From ce5e60d756aff7694a177c48ec9daa91313f261f Mon Sep 17 00:00:00 2001 From: Mykhailo Bobrovskyi Date: Tue, 3 Dec 2024 14:54:01 +0200 Subject: [PATCH] Initial kjob. --- .gitignore | 27 + .golangci.yaml | 71 + .shellcheckrc | 7 + LICENSE | 2 +- Makefile | 265 + README.md | 85 + apis/v1alpha1/application_profile_types.go | 172 + apis/v1alpha1/doc.go | 20 + apis/v1alpha1/groupversion_info.go | 39 + apis/v1alpha1/job_template_types.go | 63 + apis/v1alpha1/ray_cluster_template_types.go | 63 + apis/v1alpha1/ray_job_template_types.go | 63 + apis/v1alpha1/volumebundle_types.go | 68 + apis/v1alpha1/zz_generated.deepcopy.go | 450 + client-go/clientset/versioned/clientset.go | 119 + .../versioned/fake/clientset_generated.go | 88 + client-go/clientset/versioned/fake/doc.go | 19 + .../clientset/versioned/fake/register.go | 55 + client-go/clientset/versioned/scheme/doc.go | 19 + .../clientset/versioned/scheme/register.go | 55 + .../typed/apis/v1alpha1/apis_client.go | 126 + .../typed/apis/v1alpha1/applicationprofile.go | 66 + .../versioned/typed/apis/v1alpha1/doc.go | 19 + .../versioned/typed/apis/v1alpha1/fake/doc.go | 19 + .../apis/v1alpha1/fake/fake_apis_client.go | 55 + .../v1alpha1/fake/fake_applicationprofile.go | 133 + .../apis/v1alpha1/fake/fake_jobtemplate.go | 133 + .../v1alpha1/fake/fake_rayclustertemplate.go | 133 + .../apis/v1alpha1/fake/fake_rayjobtemplate.go | 133 + .../apis/v1alpha1/fake/fake_volumebundle.go | 133 + .../apis/v1alpha1/generated_expansion.go | 28 + .../typed/apis/v1alpha1/jobtemplate.go | 66 + .../typed/apis/v1alpha1/rayclustertemplate.go | 66 + .../typed/apis/v1alpha1/rayjobtemplate.go | 66 + .../typed/apis/v1alpha1/volumebundle.go | 66 + cmd/kjobctl-docs/generators/doc.go | 376 + cmd/kjobctl-docs/main.go | 51 + cmd/kjobctl-docs/templates/main.md | 50 + cmd/kjobctl-docs/templates/options.html | 24 + cmd/kjobctl/main.go | 29 + .../kjobctl.x-k8s.io_applicationprofiles.yaml | 210 + .../bases/kjobctl.x-k8s.io_jobtemplates.yaml | 8406 +++++ .../kjobctl.x-k8s.io_rayclustertemplates.yaml | 17400 +++++++++++ .../kjobctl.x-k8s.io_rayjobtemplates.yaml | 25625 ++++++++++++++++ .../bases/kjobctl.x-k8s.io_volumebundles.yaml | 2007 ++ config/crd/kustomization.yaml | 15 + config/default/kustomization.yaml | 18 + config/samples/README.md | 3 + config/samples/interactive-sample.yaml | 22 + config/samples/job-sample.yaml | 31 + config/samples/ray-cluster-sample.yaml | 120 + config/samples/ray-job-sample.yaml | 159 + config/samples/slurm/script.sh | 6 + config/samples/slurm/slurm-sample.yaml | 64 + docs/_index.md | 24 + docs/commands/kjobctl.md | 207 + docs/commands/kjobctl_create/_index.md | 261 + .../kjobctl_create_interactive.md | 369 + .../kjobctl_create/kjobctl_create_job.md | 372 + .../kjobctl_create_raycluster.md | 364 + .../kjobctl_create/kjobctl_create_rayjob.md | 383 + .../kjobctl_create/kjobctl_create_slurm.md | 354 + docs/commands/kjobctl_delete/_index.md | 237 + .../kjobctl_delete_interactive.md | 281 + .../kjobctl_delete/kjobctl_delete_job.md | 281 + .../kjobctl_delete_raycluster.md | 281 + .../kjobctl_delete/kjobctl_delete_rayjob.md | 281 + .../kjobctl_delete/kjobctl_delete_slurm.md | 281 + docs/commands/kjobctl_describe/_index.md | 263 + docs/commands/kjobctl_list/_index.md | 228 + .../kjobctl_list/kjobctl_list_interactive.md | 311 + .../commands/kjobctl_list/kjobctl_list_job.md | 311 + .../kjobctl_list/kjobctl_list_raycluster.md | 311 + .../kjobctl_list/kjobctl_list_rayjob.md | 311 + .../kjobctl_list/kjobctl_list_slurm.md | 311 + docs/commands/kjobctl_printcrds/_index.md | 230 + docs/installation.md | 59 + docs/run_slurm.md | 246 + go.mod | 102 + go.sum | 328 + hack/boilerplate.go.txt | 15 + hack/e2e-test.sh | 99 + hack/multiplatform-build.sh | 52 + hack/shellcheck/Dockerfile | 1 + hack/shellcheck/verify.sh | 59 + hack/tools/go.mod | 76 + hack/tools/go.sum | 289 + hack/tools/gzip/gzip.go | 37 + hack/tools/pinversion.go | 34 + hack/update-codegen.sh | 44 + pkg/builder/builder.go | 697 + pkg/builder/builder_test.go | 472 + pkg/builder/interactive_builder.go | 65 + pkg/builder/interactive_builder_test.go | 218 + pkg/builder/job_builder.go | 69 + pkg/builder/job_builder_test.go | 222 + pkg/builder/ray_cluster_builder.go | 61 + pkg/builder/ray_cluster_builder_test.go | 267 + pkg/builder/ray_job_builder.go | 77 + pkg/builder/ray_job_builder_test.go | 300 + pkg/builder/slurm_builder.go | 775 + pkg/builder/slurm_builder_test.go | 413 + .../templates/slurm_entrypoint_script.sh.tmpl | 20 + .../slurm_init_entrypoint_script.sh.tmpl | 77 + pkg/cmd/cmd.go | 87 + pkg/cmd/completion/completion.go | 276 + pkg/cmd/completion/completion_test.go | 216 + pkg/cmd/create/create.go | 861 + pkg/cmd/create/create_test.go | 2340 ++ pkg/cmd/create/helpers.go | 68 + pkg/cmd/delete/delete.go | 49 + pkg/cmd/delete/delete_interactive.go | 176 + pkg/cmd/delete/delete_interactive_test.go | 190 + pkg/cmd/delete/delete_job.go | 182 + pkg/cmd/delete/delete_job_test.go | 205 + pkg/cmd/delete/delete_ray_cluster.go | 176 + pkg/cmd/delete/delete_ray_cluster_test.go | 190 + pkg/cmd/delete/delete_ray_job.go | 176 + pkg/cmd/delete/delete_ray_job_test.go | 190 + pkg/cmd/delete/delete_slurm.go | 184 + pkg/cmd/delete/delete_slurm_test.go | 205 + pkg/cmd/delete/helpers.go | 52 + pkg/cmd/describe/describe.go | 352 + pkg/cmd/describe/describe_printer.go | 335 + pkg/cmd/describe/describe_test.go | 830 + pkg/cmd/describe/helpers.go | 868 + pkg/cmd/list/helpers.go | 47 + pkg/cmd/list/list.go | 50 + pkg/cmd/list/list_interactive.go | 220 + pkg/cmd/list/list_interactive_printer.go | 105 + pkg/cmd/list/list_interactive_test.go | 306 + pkg/cmd/list/list_job.go | 223 + pkg/cmd/list/list_job_printer.go | 117 + pkg/cmd/list/list_job_test.go | 390 + pkg/cmd/list/list_ray_cluster.go | 220 + pkg/cmd/list/list_ray_cluster_printer.go | 114 + pkg/cmd/list/list_ray_cluster_test.go | 381 + pkg/cmd/list/list_ray_job.go | 220 + pkg/cmd/list/list_ray_job_printer.go | 120 + pkg/cmd/list/list_ray_job_test.go | 387 + pkg/cmd/list/list_slurm.go | 223 + pkg/cmd/list/list_slurm_test.go | 389 + pkg/cmd/list/list_test.go | 206 + pkg/cmd/printcrds/embed/manifest.gz | Bin 0 -> 432291 bytes pkg/cmd/printcrds/printcrds.go | 59 + pkg/cmd/printcrds/printcrds_test.go | 51 + pkg/cmd/testing/fake.go | 148 + pkg/cmd/util/client_getter.go | 129 + pkg/cmd/util/dry_run.go | 85 + pkg/cmd/util/helpers.go | 55 + pkg/constants/constants.go | 35 + pkg/parser/array_indexes.go | 170 + pkg/parser/array_indexes_test.go | 115 + pkg/parser/slurm.go | 185 + pkg/parser/slurm_test.go | 141 + pkg/parser/time_limit.go | 148 + pkg/parser/time_limit_test.go | 121 + .../wrappers/application_profile_wrappers.go | 59 + pkg/testing/wrappers/configmap_wrappers.go | 83 + pkg/testing/wrappers/container_wrappers.go | 82 + pkg/testing/wrappers/job_template_wrappers.go | 167 + pkg/testing/wrappers/job_wrappers.go | 197 + pkg/testing/wrappers/pod_template_wrappers.go | 153 + pkg/testing/wrappers/pod_wrappers.go | 121 + .../wrappers/ray_cluster_spec_wrapper.go | 192 + .../wrappers/ray_cluster_template_wrappers.go | 75 + pkg/testing/wrappers/ray_cluster_wrappers.go | 178 + .../wrappers/ray_job_template_wrappers.go | 81 + pkg/testing/wrappers/ray_job_wrappers.go | 181 + pkg/testing/wrappers/service_wrappers.go | 92 + .../wrappers/supported_mode_wrappers.go | 39 + .../wrappers/volume_bundle_wrappers.go | 71 + .../wrappers/worker_group_spec_wrapper.go | 94 + test/e2e/e2e_test.go | 53 + test/e2e/slurm_test.go | 369 + test/e2e/suite_test.go | 54 + test/framework/framework.go | 80 + test/integration/kjobctl/create_test.go | 172 + test/integration/kjobctl/list_test.go | 346 + test/integration/kjobctl/suite_test.go | 61 + test/integration/kjobctl/util.go | 33 + test/util/constants.go | 27 + test/util/util.go | 171 + 183 files changed, 85632 insertions(+), 1 deletion(-) create mode 100644 .gitignore create mode 100644 .golangci.yaml create mode 100644 .shellcheckrc create mode 100644 Makefile create mode 100644 apis/v1alpha1/application_profile_types.go create mode 100644 apis/v1alpha1/doc.go create mode 100644 apis/v1alpha1/groupversion_info.go create mode 100644 apis/v1alpha1/job_template_types.go create mode 100644 apis/v1alpha1/ray_cluster_template_types.go create mode 100644 apis/v1alpha1/ray_job_template_types.go create mode 100644 apis/v1alpha1/volumebundle_types.go create mode 100644 apis/v1alpha1/zz_generated.deepcopy.go create mode 100644 client-go/clientset/versioned/clientset.go create mode 100644 client-go/clientset/versioned/fake/clientset_generated.go create mode 100644 client-go/clientset/versioned/fake/doc.go create mode 100644 client-go/clientset/versioned/fake/register.go create mode 100644 client-go/clientset/versioned/scheme/doc.go create mode 100644 client-go/clientset/versioned/scheme/register.go create mode 100644 client-go/clientset/versioned/typed/apis/v1alpha1/apis_client.go create mode 100644 client-go/clientset/versioned/typed/apis/v1alpha1/applicationprofile.go create mode 100644 client-go/clientset/versioned/typed/apis/v1alpha1/doc.go create mode 100644 client-go/clientset/versioned/typed/apis/v1alpha1/fake/doc.go create mode 100644 client-go/clientset/versioned/typed/apis/v1alpha1/fake/fake_apis_client.go create mode 100644 client-go/clientset/versioned/typed/apis/v1alpha1/fake/fake_applicationprofile.go create mode 100644 client-go/clientset/versioned/typed/apis/v1alpha1/fake/fake_jobtemplate.go create mode 100644 client-go/clientset/versioned/typed/apis/v1alpha1/fake/fake_rayclustertemplate.go create mode 100644 client-go/clientset/versioned/typed/apis/v1alpha1/fake/fake_rayjobtemplate.go create mode 100644 client-go/clientset/versioned/typed/apis/v1alpha1/fake/fake_volumebundle.go create mode 100644 client-go/clientset/versioned/typed/apis/v1alpha1/generated_expansion.go create mode 100644 client-go/clientset/versioned/typed/apis/v1alpha1/jobtemplate.go create mode 100644 client-go/clientset/versioned/typed/apis/v1alpha1/rayclustertemplate.go create mode 100644 client-go/clientset/versioned/typed/apis/v1alpha1/rayjobtemplate.go create mode 100644 client-go/clientset/versioned/typed/apis/v1alpha1/volumebundle.go create mode 100644 cmd/kjobctl-docs/generators/doc.go create mode 100644 cmd/kjobctl-docs/main.go create mode 100644 cmd/kjobctl-docs/templates/main.md create mode 100644 cmd/kjobctl-docs/templates/options.html create mode 100644 cmd/kjobctl/main.go create mode 100644 config/crd/bases/kjobctl.x-k8s.io_applicationprofiles.yaml create mode 100644 config/crd/bases/kjobctl.x-k8s.io_jobtemplates.yaml create mode 100644 config/crd/bases/kjobctl.x-k8s.io_rayclustertemplates.yaml create mode 100644 config/crd/bases/kjobctl.x-k8s.io_rayjobtemplates.yaml create mode 100644 config/crd/bases/kjobctl.x-k8s.io_volumebundles.yaml create mode 100644 config/crd/kustomization.yaml create mode 100644 config/default/kustomization.yaml create mode 100644 config/samples/README.md create mode 100644 config/samples/interactive-sample.yaml create mode 100644 config/samples/job-sample.yaml create mode 100644 config/samples/ray-cluster-sample.yaml create mode 100644 config/samples/ray-job-sample.yaml create mode 100644 config/samples/slurm/script.sh create mode 100644 config/samples/slurm/slurm-sample.yaml create mode 100644 docs/_index.md create mode 100644 docs/commands/kjobctl.md create mode 100644 docs/commands/kjobctl_create/_index.md create mode 100644 docs/commands/kjobctl_create/kjobctl_create_interactive.md create mode 100644 docs/commands/kjobctl_create/kjobctl_create_job.md create mode 100644 docs/commands/kjobctl_create/kjobctl_create_raycluster.md create mode 100644 docs/commands/kjobctl_create/kjobctl_create_rayjob.md create mode 100644 docs/commands/kjobctl_create/kjobctl_create_slurm.md create mode 100644 docs/commands/kjobctl_delete/_index.md create mode 100644 docs/commands/kjobctl_delete/kjobctl_delete_interactive.md create mode 100644 docs/commands/kjobctl_delete/kjobctl_delete_job.md create mode 100644 docs/commands/kjobctl_delete/kjobctl_delete_raycluster.md create mode 100644 docs/commands/kjobctl_delete/kjobctl_delete_rayjob.md create mode 100644 docs/commands/kjobctl_delete/kjobctl_delete_slurm.md create mode 100644 docs/commands/kjobctl_describe/_index.md create mode 100644 docs/commands/kjobctl_list/_index.md create mode 100644 docs/commands/kjobctl_list/kjobctl_list_interactive.md create mode 100644 docs/commands/kjobctl_list/kjobctl_list_job.md create mode 100644 docs/commands/kjobctl_list/kjobctl_list_raycluster.md create mode 100644 docs/commands/kjobctl_list/kjobctl_list_rayjob.md create mode 100644 docs/commands/kjobctl_list/kjobctl_list_slurm.md create mode 100644 docs/commands/kjobctl_printcrds/_index.md create mode 100644 docs/installation.md create mode 100644 docs/run_slurm.md create mode 100644 go.mod create mode 100644 go.sum create mode 100644 hack/boilerplate.go.txt create mode 100755 hack/e2e-test.sh create mode 100755 hack/multiplatform-build.sh create mode 100644 hack/shellcheck/Dockerfile create mode 100755 hack/shellcheck/verify.sh create mode 100644 hack/tools/go.mod create mode 100644 hack/tools/go.sum create mode 100644 hack/tools/gzip/gzip.go create mode 100644 hack/tools/pinversion.go create mode 100755 hack/update-codegen.sh create mode 100644 pkg/builder/builder.go create mode 100644 pkg/builder/builder_test.go create mode 100644 pkg/builder/interactive_builder.go create mode 100644 pkg/builder/interactive_builder_test.go create mode 100644 pkg/builder/job_builder.go create mode 100644 pkg/builder/job_builder_test.go create mode 100644 pkg/builder/ray_cluster_builder.go create mode 100644 pkg/builder/ray_cluster_builder_test.go create mode 100644 pkg/builder/ray_job_builder.go create mode 100644 pkg/builder/ray_job_builder_test.go create mode 100644 pkg/builder/slurm_builder.go create mode 100644 pkg/builder/slurm_builder_test.go create mode 100644 pkg/builder/templates/slurm_entrypoint_script.sh.tmpl create mode 100644 pkg/builder/templates/slurm_init_entrypoint_script.sh.tmpl create mode 100644 pkg/cmd/cmd.go create mode 100644 pkg/cmd/completion/completion.go create mode 100644 pkg/cmd/completion/completion_test.go create mode 100644 pkg/cmd/create/create.go create mode 100644 pkg/cmd/create/create_test.go create mode 100644 pkg/cmd/create/helpers.go create mode 100644 pkg/cmd/delete/delete.go create mode 100644 pkg/cmd/delete/delete_interactive.go create mode 100644 pkg/cmd/delete/delete_interactive_test.go create mode 100644 pkg/cmd/delete/delete_job.go create mode 100644 pkg/cmd/delete/delete_job_test.go create mode 100644 pkg/cmd/delete/delete_ray_cluster.go create mode 100644 pkg/cmd/delete/delete_ray_cluster_test.go create mode 100644 pkg/cmd/delete/delete_ray_job.go create mode 100644 pkg/cmd/delete/delete_ray_job_test.go create mode 100644 pkg/cmd/delete/delete_slurm.go create mode 100644 pkg/cmd/delete/delete_slurm_test.go create mode 100644 pkg/cmd/delete/helpers.go create mode 100644 pkg/cmd/describe/describe.go create mode 100644 pkg/cmd/describe/describe_printer.go create mode 100644 pkg/cmd/describe/describe_test.go create mode 100644 pkg/cmd/describe/helpers.go create mode 100644 pkg/cmd/list/helpers.go create mode 100644 pkg/cmd/list/list.go create mode 100644 pkg/cmd/list/list_interactive.go create mode 100644 pkg/cmd/list/list_interactive_printer.go create mode 100644 pkg/cmd/list/list_interactive_test.go create mode 100644 pkg/cmd/list/list_job.go create mode 100644 pkg/cmd/list/list_job_printer.go create mode 100644 pkg/cmd/list/list_job_test.go create mode 100644 pkg/cmd/list/list_ray_cluster.go create mode 100644 pkg/cmd/list/list_ray_cluster_printer.go create mode 100644 pkg/cmd/list/list_ray_cluster_test.go create mode 100644 pkg/cmd/list/list_ray_job.go create mode 100644 pkg/cmd/list/list_ray_job_printer.go create mode 100644 pkg/cmd/list/list_ray_job_test.go create mode 100644 pkg/cmd/list/list_slurm.go create mode 100644 pkg/cmd/list/list_slurm_test.go create mode 100644 pkg/cmd/list/list_test.go create mode 100644 pkg/cmd/printcrds/embed/manifest.gz create mode 100644 pkg/cmd/printcrds/printcrds.go create mode 100644 pkg/cmd/printcrds/printcrds_test.go create mode 100644 pkg/cmd/testing/fake.go create mode 100644 pkg/cmd/util/client_getter.go create mode 100644 pkg/cmd/util/dry_run.go create mode 100644 pkg/cmd/util/helpers.go create mode 100644 pkg/constants/constants.go create mode 100644 pkg/parser/array_indexes.go create mode 100644 pkg/parser/array_indexes_test.go create mode 100644 pkg/parser/slurm.go create mode 100644 pkg/parser/slurm_test.go create mode 100644 pkg/parser/time_limit.go create mode 100644 pkg/parser/time_limit_test.go create mode 100644 pkg/testing/wrappers/application_profile_wrappers.go create mode 100644 pkg/testing/wrappers/configmap_wrappers.go create mode 100644 pkg/testing/wrappers/container_wrappers.go create mode 100644 pkg/testing/wrappers/job_template_wrappers.go create mode 100644 pkg/testing/wrappers/job_wrappers.go create mode 100644 pkg/testing/wrappers/pod_template_wrappers.go create mode 100644 pkg/testing/wrappers/pod_wrappers.go create mode 100644 pkg/testing/wrappers/ray_cluster_spec_wrapper.go create mode 100644 pkg/testing/wrappers/ray_cluster_template_wrappers.go create mode 100644 pkg/testing/wrappers/ray_cluster_wrappers.go create mode 100644 pkg/testing/wrappers/ray_job_template_wrappers.go create mode 100644 pkg/testing/wrappers/ray_job_wrappers.go create mode 100644 pkg/testing/wrappers/service_wrappers.go create mode 100644 pkg/testing/wrappers/supported_mode_wrappers.go create mode 100644 pkg/testing/wrappers/volume_bundle_wrappers.go create mode 100644 pkg/testing/wrappers/worker_group_spec_wrapper.go create mode 100644 test/e2e/e2e_test.go create mode 100644 test/e2e/slurm_test.go create mode 100644 test/e2e/suite_test.go create mode 100644 test/framework/framework.go create mode 100644 test/integration/kjobctl/create_test.go create mode 100644 test/integration/kjobctl/list_test.go create mode 100644 test/integration/kjobctl/suite_test.go create mode 100644 test/integration/kjobctl/util.go create mode 100644 test/util/constants.go create mode 100644 test/util/util.go diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..ada68ff --- /dev/null +++ b/.gitignore @@ -0,0 +1,27 @@ +# Binaries for programs and plugins +*.exe +*.exe~ +*.dll +*.so +*.dylib +bin/* +Dockerfile.cross + +# Test binary, built with `go test -c` +*.test + +# Output of the go coverage tool, specifically when used with LiteIDE +*.out + +# Go workspace file +go.work + +# Kubernetes Generated files - skip generated files, except for vendored files +!vendor/**/zz_generated.* + +# editor and IDE paraphernalia +.idea +.vscode +*.swp +*.swo +*~ diff --git a/.golangci.yaml b/.golangci.yaml new file mode 100644 index 0000000..5909125 --- /dev/null +++ b/.golangci.yaml @@ -0,0 +1,71 @@ +# golangci-lint configuration file +# see: https://golangci-lint.run/usage/configuration/ + +# Settings of specific linters +linters-settings: + gocritic: + enabled-checks: + - dupImport + disabled-checks: + - appendAssign + - exitAfterDefer + govet: + enable: + - nilness + gci: + sections: + - standard # Standard section: captures all standard packages. + - default # Default section: contains all imports that could not be matched to another section type. + - prefix(sigs.k8s.io/kueue/cmd/experimental/kjobctl) # Custom section: groups all imports with the specified Prefix. + - blank # Blank section: contains all blank imports. This section is not present unless explicitly enabled. + - dot # Dot section: contains all dot imports. + skip-generated: true # Skip generated files. + perfsprint: + int-conversion: false + errorf: true + sprintf1: false + strconcat: false + revive: + enable-all-rules: false + rules: + - name: empty-lines + - name: var-naming + - name: redundant-import-alias + +# Settings for enabling and disabling linters +linters: + enable: + - copyloopvar + - dupword + - durationcheck + - gci + - ginkgolinter + - gocritic + - govet + - loggercheck + - misspell + - perfsprint + - revive + - unconvert + +# Settings related to issues +issues: + # Which dirs to exclude: issues from them won't be reported + exclude-dirs: + - bin + # Excluding configuration per-path, per-linter, per-text and per-source + exclude-rules: + - linters: + - staticcheck + # TODO(#768): Drop when incrementing the API version. + text: "SA1019: constants.QueueAnnotation is deprecated" + - linters: + - staticcheck + text: "SA1019: j.RayCluster.Status.State is deprecated: the State field is replaced by the Conditions field." + - linters: + - staticcheck + text: "SA1019: rayCluster.Status.State is deprecated: the State field is replaced by the Conditions field." + # Show all issues from a linter + max-issues-per-linter: 0 + # Show all issues with the same text + max-same-issues: 0 diff --git a/.shellcheckrc b/.shellcheckrc new file mode 100644 index 0000000..01ec289 --- /dev/null +++ b/.shellcheckrc @@ -0,0 +1,7 @@ +# shellcheck configuration file +# see: https://github.com/koalaman/shellcheck/wiki + +# Allow following sourced files that are not specified in the command, +# we need this because we specify one file at a time in order to trivially +# detect which files are failing +external-sources=true diff --git a/LICENSE b/LICENSE index 8dada3e..15b8c14 100644 --- a/LICENSE +++ b/LICENSE @@ -186,7 +186,7 @@ same "printed page" as the copyright notice for easier identification within third-party archives. - Copyright {yyyy} {name of copyright owner} + Copyright 2024 The Kubernetes Authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..e15788a --- /dev/null +++ b/Makefile @@ -0,0 +1,265 @@ +# ENVTEST_K8S_VERSION refers to the version of kubebuilder assets to be downloaded by envtest binary. +ENVTEST_K8S_VERSION = 1.31.0 + +ifeq ($(shell uname),Darwin) + GOFLAGS ?= -ldflags=-linkmode=internal +endif + +# Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set) +ifeq (,$(shell go env GOBIN)) +GOBIN=$(shell go env GOPATH)/bin +else +GOBIN=$(shell go env GOBIN) +endif + +CGO_ENABLED ?= 0 + +GO_CMD ?= go +GO_FMT ?= gofmt + +PROJECT_DIR := $(shell dirname $(abspath $(lastword $(MAKEFILE_LIST)))) +ARTIFACTS ?= $(PROJECT_DIR)/bin +TOOLS_DIR ?= $(PROJECT_DIR)/hack/tools +CLI_PLATFORMS ?= linux/amd64,linux/arm64,darwin/amd64,darwin/arm64 +EXTERNAL_CRDS_DIR ?= $(PROJECT_DIR)/dep-crds + +MANIFEST_DIR := $(PROJECT_DIR)/config/crd +MANIFEST_SOURCES := $(shell find $(MANIFEST_DIR) -name "*.yaml") +EMBEDDED_MANIFEST_DIR := $(PROJECT_DIR)/pkg/cmd/printcrds/embed +EMBEDDED_MANIFEST := $(EMBEDDED_MANIFEST_DIR)/manifest.gz + +# Number of processes to use during integration tests to run specs within a +# suite in parallel. Suites still run sequentially. User may set this value to 1 +# to run without parallelism. +INTEGRATION_NPROCS ?= 4 +# Folder where the integration tests are located. +INTEGRATION_TARGET ?= $(PROJECT_DIR)/test/integration/... + +# Setting SHELL to bash allows bash commands to be executed by recipes. +# Options are set to exit when a recipe line exits non-zero or a piped command fails. +SHELL = /usr/bin/env bash -o pipefail +.SHELLFLAGS = -ec + +# For local testing, we should allow user to use different kind cluster name +# Default will delete default kind cluster +KIND_CLUSTER_NAME ?= kind +E2E_KIND_VERSION ?= kindest/node:v1.31.0 +K8S_VERSION = $(E2E_KIND_VERSION:kindest/node:v%=%) + +##@ General + +# The help target prints out all targets with their descriptions organized +# beneath their categories. The categories are represented by '##@' and the +# target descriptions by '##'. The awk command is responsible for reading the +# entire set of makefiles included in this invocation, looking for lines of the +# file as xyz: ## something, and then pretty-format the target and help. Then, +# if there's a line with ##@ something, that gets pretty-printed as a category. +# More info on the usage of ANSI control characters for terminal formatting: +# https://en.wikipedia.org/wiki/ANSI_escape_code#SGR_parameters +# More info on the awk command: +# http://linuxcommand.org/lc3_adv_awk.php + +.PHONY: help +help: ## Display this help. + @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf " \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) + +##@ Release +.PHONY: artifacts +artifacts: kustomize ## Generate release artifacts. + if [ -d artifacts ]; then rm -rf artifacts; fi + mkdir -p artifacts + $(KUSTOMIZE) build config/default -o artifacts/manifests.yaml + CGO_ENABLED=$(CGO_ENABLED) GO_CMD="$(GO_CMD)" LD_FLAGS="$(LD_FLAGS)" BUILD_DIR="artifacts" BUILD_NAME=kubectl-kjob PLATFORMS="$(CLI_PLATFORMS)" ./hack/multiplatform-build.sh ./cmd/kjobctl/main.go + +##@ Development + +.PHONY: manifests +manifests: controller-gen ## Generate ClusterRole and CustomResourceDefinition objects. + # We need to allow dangerous types because on RayJobSpec still uses float32 type. + $(CONTROLLER_GEN) crd:allowDangerousTypes=true paths="./..." output:crd:artifacts:config=config/crd/bases + +.PHONY: generate +generate: gomod-download controller-gen ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations. + $(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./..." + TOOLS_DIR=${TOOLS_DIR} ./hack/update-codegen.sh $(GO_CMD) + +.PHONY: verify +verify: gomod-verify ci-lint shell-lint fmt-verify manifests generate generate-kjobctl-docs embeded-manifest + git --no-pager diff --exit-code config/crd apis client-go docs pkg/cmd/printcrds/embed + +.PHONY: gomod-verify +gomod-verify: + $(GO_CMD) mod tidy + git --no-pager diff --exit-code go.mod go.sum + +.PHONY: ci-lint +ci-lint: golangci-lint + $(GOLANGCI_LINT) run --timeout 15m0s + +.PHONY: shell-lint +shell-lint: ## Run shell linting. + $(PROJECT_DIR)/hack/shellcheck/verify.sh + +.PHONY: fmt-verify +fmt-verify: + @out=`$(GO_FMT) -w -l -d $$(find . -name '*.go')`; \ + if [ -n "$$out" ]; then \ + echo "$$out"; \ + exit 1; \ + fi + +$(EMBEDDED_MANIFEST): kustomize $(MANIFEST_SOURCES) + mkdir -p $(EMBEDDED_MANIFEST_DIR) + # This is embedded in the kjobctl binary. + $(KUSTOMIZE) build $(MANIFEST_DIR) | $(GO_CMD) run $(PROJECT_DIR)/hack/tools/gzip/gzip.go > $(EMBEDDED_MANIFEST) + +embeded-manifest: $(EMBEDDED_MANIFEST) + +.PHONY: gomod-download +gomod-download: + $(GO_CMD) mod download + +.PHONY: vet +vet: ## Run go vet against code. + $(GO_CMD) vet ./... + +.PHONY: test +test: verify vet test-unit test-integration test-e2e ## Run all tests. + +.PHONY: test-unit +test-unit: gomod-download gotestsum embeded-manifest ## Run unit tests. + $(GOTESTSUM) --junitfile $(ARTIFACTS)/junit-unit.xml -- $(GOFLAGS) $(GO_TEST_FLAGS) $(shell $(GO_CMD) list ./... | grep -v '/test/') -coverpkg=./... -coverprofile $(ARTIFACTS)/cover.out + +.PHONY: test-integration +test-integration: gomod-download envtest ginkgo embeded-manifest ray-operator-crd ## Run integration tests. + KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) -p path)" \ + KUEUE_BIN=$(PROJECT_DIR)/bin \ + ENVTEST_K8S_VERSION=$(ENVTEST_K8S_VERSION) \ + $(GINKGO) $(GINKGO_ARGS) -procs=$(INTEGRATION_NPROCS) --race --junit-report=junit-integration.xml --output-dir=$(ARTIFACTS) -v $(INTEGRATION_TARGET) + +CREATE_KIND_CLUSTER ?= true +.PHONY: test-e2e +test-e2e: kind kubectl-kjob + @echo Running e2e for k8s ${K8S_VERSION} + E2E_KIND_VERSION=$(E2E_KIND_VERSION) KIND_CLUSTER_NAME=$(KIND_CLUSTER_NAME) CREATE_KIND_CLUSTER=$(CREATE_KIND_CLUSTER) ARTIFACTS="$(ARTIFACTS)/$@" GINKGO_ARGS="$(GINKGO_ARGS)" ./hack/e2e-test.sh + +.PHONY: lint +lint: golangci-lint ## Run golangci-lint linter + $(GOLANGCI_LINT) run + +.PHONY: lint-fix +lint-fix: golangci-lint ## Run golangci-lint linter and perform fixes + $(GOLANGCI_LINT) run --fix + +##@ Deployment + +ifndef ignore-not-found + ignore-not-found = false +endif + +.PHONY: install +install: manifests kustomize ## Install CRDs into the K8s cluster specified in ~/.kube/config. + $(KUSTOMIZE) build config/crd | $(KUBECTL) apply --server-side -f - + +.PHONY: uninstall +uninstall: manifests kustomize ## Uninstall CRDs from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion. + $(KUSTOMIZE) build config/crd | $(KUBECTL) delete --ignore-not-found=$(ignore-not-found) -f - + + +.PHONY: kubectl-kjob +kubectl-kjob: embeded-manifest ## Build kubectl-kjob binary file to bin directory. + CGO_ENABLED=$(CGO_ENABLED) $(GO_BUILD_ENV) $(GO_CMD) build -ldflags="$(LD_FLAGS)" -o bin/kubectl-kjob cmd/kjobctl/main.go + +.PHONY: kjobctl-docs +kjobctl-docs: ## Build kjobctl-docs binary file to bin directory. + $(GO_BUILD_ENV) $(GO_CMD) build -ldflags="$(LD_FLAGS)" -o $(PROJECT_DIR)/bin/kjobctl-docs ./cmd/kjobctl-docs/main.go + +.PHONY: generate-kjobctl-docs +generate-kjobctl-docs: kjobctl-docs ## Generate kjobctl docs. + rm -Rf $(PROJECT_DIR)/docs/commands/* + $(PROJECT_DIR)/bin/kjobctl-docs \ + $(PROJECT_DIR)/cmd/kjobctl-docs/templates \ + $(PROJECT_DIR)/docs/commands + +##@ Dependencies + +## Location to install dependencies to +LOCALBIN ?= $(shell pwd)/bin +$(LOCALBIN): + mkdir -p $(LOCALBIN) + +## Tool Versions + +# Use go.mod go version as source. +KUSTOMIZE_VERSION ?= $(shell cd $(TOOLS_DIR) && $(GO_CMD) list -m -f '{{.Version}}' sigs.k8s.io/kustomize/kustomize/v5) +CONTROLLER_GEN_VERSION ?= $(shell cd $(TOOLS_DIR) && $(GO_CMD) list -m -f '{{.Version}}' sigs.k8s.io/controller-tools) +ENVTEST_VERSION ?= $(shell cd $(TOOLS_DIR) && $(GO_CMD) list -m -f '{{.Version}}' sigs.k8s.io/controller-runtime/tools/setup-envtest) +GOLANGCI_LINT_VERSION ?= $(shell cd $(TOOLS_DIR) && $(GO_CMD) list -m -f '{{.Version}}' github.com/golangci/golangci-lint) +GOTESTSUM_VERSION ?= $(shell cd $(TOOLS_DIR) && $(GO_CMD) list -m -f '{{.Version}}' gotest.tools/gotestsum) +GINKGO_VERSION ?= $(shell cd $(TOOLS_DIR) && $(GO_CMD) list -m -f '{{.Version}}' github.com/onsi/ginkgo/v2) +KIND_VERSION ?= $(shell cd $(TOOLS_DIR); $(GO_CMD) list -m -f '{{.Version}}' sigs.k8s.io/kind) + +## Tool Binaries +KUBECTL ?= kubectl +KUSTOMIZE ?= $(LOCALBIN)/kustomize-$(KUSTOMIZE_VERSION) +CONTROLLER_GEN ?= $(LOCALBIN)/controller-gen-$(CONTROLLER_GEN_VERSION) +ENVTEST ?= $(LOCALBIN)/setup-envtest-$(ENVTEST_VERSION) +GOLANGCI_LINT ?= $(LOCALBIN)/golangci-lint-$(GOLANGCI_LINT_VERSION) +GOTESTSUM ?= $(LOCALBIN)/gotestsum-$(GOTESTSUM_VERSION) +GINKGO ?= $(LOCALBIN)/ginkgo-$(GINKGO_VERSION) + +.PHONY: kustomize +kustomize: $(KUSTOMIZE) ## Download kustomize locally if necessary. +$(KUSTOMIZE): $(LOCALBIN) + $(call go-install-tool,$(KUSTOMIZE),sigs.k8s.io/kustomize/kustomize/v5,$(KUSTOMIZE_VERSION)) + +.PHONY: controller-gen +controller-gen: $(CONTROLLER_GEN) ## Download controller-gen locally if necessary. +$(CONTROLLER_GEN): $(LOCALBIN) + $(call go-install-tool,$(CONTROLLER_GEN),sigs.k8s.io/controller-tools/cmd/controller-gen,$(CONTROLLER_GEN_VERSION)) + +.PHONY: envtest +envtest: $(ENVTEST) ## Download setup-envtest locally if necessary. +$(ENVTEST): $(LOCALBIN) + $(call go-install-tool,$(ENVTEST),sigs.k8s.io/controller-runtime/tools/setup-envtest,$(ENVTEST_VERSION)) + +.PHONY: golangci-lint +golangci-lint: $(GOLANGCI_LINT) ## Download golangci-lint locally if necessary. +$(GOLANGCI_LINT): $(LOCALBIN) + $(call go-install-tool,$(GOLANGCI_LINT),github.com/golangci/golangci-lint/cmd/golangci-lint,$(GOLANGCI_LINT_VERSION)) + +.PHONY: gotestsum +gotestsum: ## Download gotestsum locally if necessary. + $(call go-install-tool,$(GOTESTSUM),gotest.tools/gotestsum,$(GOTESTSUM_VERSION)) + +.PHONY: ginkgo +ginkgo: ## Download ginkgo locally if necessary. + $(call go-install-tool,$(GINKGO),github.com/onsi/ginkgo/v2/ginkgo,$(GINKGO_VERSION)) + +KIND = $(PROJECT_DIR)/bin/kind +.PHONY: kind +kind: ## Download kind locally if necessary. + @GOBIN=$(PROJECT_DIR)/bin GO111MODULE=on $(GO_CMD) install sigs.k8s.io/kind@$(KIND_VERSION) + +##@ External CRDs + +RAY_ROOT = $(shell $(GO_CMD) list -m -mod=readonly -f "{{.Dir}}" github.com/ray-project/kuberay/ray-operator) +.PHONY: ray-operator-crd +ray-operator-crd: ## Copy the CRDs from the ray-operator to the dep-crds directory. + mkdir -p $(EXTERNAL_CRDS_DIR)/ray-operator/ + cp -f $(RAY_ROOT)/config/crd/bases/* $(EXTERNAL_CRDS_DIR)/ray-operator/ + +# go-install-tool will 'go install' any package with custom target and name of binary, if it doesn't exist +# $1 - target path with name of binary (ideally with version) +# $2 - package url which can be installed +# $3 - specific version of package +define go-install-tool +@[ -f $(1) ] || { \ +set -e; \ +package=$(2)@$(3) ;\ +echo "Downloading $${package}" ;\ +GOBIN=$(LOCALBIN) go install $${package} ;\ +mv "$$(echo "$(1)" | sed "s/-$(3)$$//")" $(1) ;\ +} ;\ +ln -sf $(1) "$$(echo "$(1)" | sed "s/-$(3)$$//")" +endef diff --git a/README.md b/README.md index c9d9061..6dfbfd5 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,91 @@ KJob: Tool for CLI-loving ML researchers +## Description +The `kubectl-kjob` plugin, `kjobctl`, allows you to list, describe and create jobs. + +Read the [overview](docs/_index.md) to learn more. + +## Getting Started + +### Prerequisites +- go version v1.23+ +- kubectl version v1.27+. +- Access to a Kubernetes v1.27+ cluster. + +### To Install + +**Install the CRDs into the cluster:** + +```sh +make install +``` + +**Install `kubectl kjob` plugin:** + +```sh +make kubectl-kjob +sudo cp ./bin/kubectl-kjob /usr/local/bin/kubectl-kjob +``` + +**Additionally, you can create an alias `kjobctl` to allow shorter syntax:** + +```sh +echo 'alias kjobctl="kubectl kjob"' >> ~/.bashrc +# Or if you are using ZSH +echo 'alias kjobctl="kubectl kjob"' >> ~/.zshrc +``` + +**Autocompletion:** + +```bash +echo '[[ $commands[kubectl-kjob] ]] && source <(kubectl-kjob completion bash)' >> ~/.bashrc +# Or if you are using ZSH +echo '[[ $commands[kubectl-kjob] ]] && source <(kubectl-kjob completion zsh)' >> ~/.zshrc + +cat <kubectl_complete-kjob +#!/usr/bin/env sh + +# Call the __complete command passing it all arguments +kubectl kjob __complete "\$@" +EOF + +chmod u+x kubectl_complete-kjob +sudo mv kubectl_complete-kjob /usr/local/bin/kubectl_complete-kjob +``` + +### To Uninstall + +**Delete the APIs(CRDs) from the cluster:** + +```sh +make uninstall +``` + +**Delete `kubectl kjob` plugin:** + +```sh +sudo rm /usr/local/bin/kubectl-kjob +``` + +**NOTE:** Run `make help` for more information on all potential `make` targets + +## License + +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + ## Community, discussion, contribution, and support Learn how to engage with the Kubernetes community on the [community page](http://kubernetes.io/community/). diff --git a/apis/v1alpha1/application_profile_types.go b/apis/v1alpha1/application_profile_types.go new file mode 100644 index 0000000..d8397e9 --- /dev/null +++ b/apis/v1alpha1/application_profile_types.go @@ -0,0 +1,172 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// VolumeBundleReference is the name of the VolumeBundle. +// +// +kubebuilder:validation:MaxLength=253 +// +kubebuilder:validation:Pattern="^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$" +type VolumeBundleReference string + +type ApplicationProfileMode string + +const ( + InteractiveMode ApplicationProfileMode = "Interactive" + JobMode ApplicationProfileMode = "Job" + RayJobMode ApplicationProfileMode = "RayJob" + RayClusterMode ApplicationProfileMode = "RayCluster" + SlurmMode ApplicationProfileMode = "Slurm" +) + +// +kubebuilder:validation:Enum=cmd;parallelism;completions;replicas;min-replicas;max-replicas;request;localqueue;raycluster;array;cpus-per-task;error;gpus-per-task;input;job-name;mem-per-cpu;mem-per-gpu;mem-per-task;nodes;ntasks;output;partition +type Flag string + +const ( + CmdFlag Flag = "cmd" + ParallelismFlag Flag = "parallelism" + CompletionsFlag Flag = "completions" + ReplicasFlag Flag = "replicas" + MinReplicasFlag Flag = "min-replicas" + MaxReplicasFlag Flag = "max-replicas" + RequestFlag Flag = "request" + LocalQueueFlag Flag = "localqueue" + RayClusterFlag Flag = "raycluster" + ArrayFlag Flag = "array" + CpusPerTaskFlag Flag = "cpus-per-task" + ErrorFlag Flag = "error" + GpusPerTaskFlag Flag = "gpus-per-task" + InputFlag Flag = "input" + JobNameFlag Flag = "job-name" + MemPerNodeFlag Flag = "mem" + MemPerCPUFlag Flag = "mem-per-cpu" + MemPerGPUFlag Flag = "mem-per-gpu" + MemPerTaskFlag Flag = "mem-per-task" + NodesFlag Flag = "nodes" + NTasksFlag Flag = "ntasks" + OutputFlag Flag = "output" + PartitionFlag Flag = "partition" + PriorityFlag Flag = "priority" + TimeFlag Flag = "time" +) + +// TemplateReference is the name of the template. +// +// +kubebuilder:validation:MaxLength=253 +// +kubebuilder:validation:Pattern="^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$" +type TemplateReference string + +// +kubebuilder:validation:XValidation:rule="!has(self.requiredFlags) || !('replicas' in self.requiredFlags) || self.name in ['RayJob', 'RayCluster']", message="replicas flag can be used only on RayJob and RayCluster modes" +// +kubebuilder:validation:XValidation:rule="!has(self.requiredFlags) || !('min-replicas' in self.requiredFlags) || self.name in ['RayJob', 'RayCluster']", message="min-replicas flag can be used only on RayJob and RayCluster modes" +// +kubebuilder:validation:XValidation:rule="!has(self.requiredFlags) || !('max-replicas' in self.requiredFlags) || self.name in ['RayJob', 'RayCluster']", message="max-replicas flag can be used only on RayJob and RayCluster modes" +// +kubebuilder:validation:XValidation:rule="!has(self.requiredFlags) || !('request' in self.requiredFlags) || self.name in ['Job', 'Interactive', 'RayJob']", message="request flag can be used only on Job and Interactive modes" +// +kubebuilder:validation:XValidation:rule="!has(self.requiredFlags) || !('cmd' in self.requiredFlags) || self.name in ['Job', 'Interactive', 'RayJob']", message="cmd flag can be used only on Job, Interactive and RayJob modes" +// +kubebuilder:validation:XValidation:rule="!has(self.requiredFlags) || !('raycluster' in self.requiredFlags) || self.name == 'RayJob'", message="raycluster flag can be used only on RayJob mode" +// +kubebuilder:validation:XValidation:rule="!has(self.requiredFlags) || !('raycluster' in self.requiredFlags) || !('localqueue' in self.requiredFlags || 'replicas' in self.requiredFlags || 'min-replicas' in self.requiredFlags || 'max-replicas' in self.requiredFlags)", message="if raycluster flag are set none of localqueue, replicas, min-replicas and max-replicas can be" +// +kubebuilder:validation:XValidation:rule="!has(self.requiredFlags) || !('array' in self.requiredFlags) || self.name == 'Slurm'", message="array flag can be used only on Slurm mode" +// +kubebuilder:validation:XValidation:rule="!has(self.requiredFlags) || !('cpus-per-task' in self.requiredFlags) || self.name == 'Slurm'", message="cpus-per-task flag can be used only on Slurm mode" +// +kubebuilder:validation:XValidation:rule="!has(self.requiredFlags) || !('error' in self.requiredFlags) || self.name == 'Slurm'", message="error flag can be used only on Slurm mode" +// +kubebuilder:validation:XValidation:rule="!has(self.requiredFlags) || !('gpus-per-task' in self.requiredFlags) || self.name == 'Slurm'", message="gpus-per-task flag can be used only on Slurm mode" +// +kubebuilder:validation:XValidation:rule="!has(self.requiredFlags) || !('input' in self.requiredFlags) || self.name == 'Slurm'", message="input flag can be used only on Slurm mode" +// +kubebuilder:validation:XValidation:rule="!has(self.requiredFlags) || !('job-name' in self.requiredFlags) || self.name == 'Slurm'", message="job-name flag can be used only on Slurm mode" +// +kubebuilder:validation:XValidation:rule="!has(self.requiredFlags) || !('mem' in self.requiredFlags) || self.name == 'Slurm'", message="mem flag can be used only on Slurm mode" +// +kubebuilder:validation:XValidation:rule="!has(self.requiredFlags) || !('mem-per-cpu' in self.requiredFlags) || self.name == 'Slurm'", message="mem-per-cpu flag can be used only on Slurm mode" +// +kubebuilder:validation:XValidation:rule="!has(self.requiredFlags) || !('mem-per-gpu' in self.requiredFlags) || self.name == 'Slurm'", message="mem-per-gpu flag can be used only on Slurm mode" +// +kubebuilder:validation:XValidation:rule="!has(self.requiredFlags) || !('mem-per-task' in self.requiredFlags) || self.name == 'Slurm'", message="mem-per-task flag can be used only on Slurm mode" +// +kubebuilder:validation:XValidation:rule="!has(self.requiredFlags) || !('nodes' in self.requiredFlags) || self.name == 'Slurm'", message="nodes flag can be used only on Slurm mode" +// +kubebuilder:validation:XValidation:rule="!has(self.requiredFlags) || !('ntasks' in self.requiredFlags) || self.name == 'Slurm'", message="ntasks flag can be used only on Slurm mode" +// +kubebuilder:validation:XValidation:rule="!has(self.requiredFlags) || !('output' in self.requiredFlags) || self.name == 'Slurm'", message="output flag can be used only on Slurm mode" +// +kubebuilder:validation:XValidation:rule="!has(self.requiredFlags) || !('partition' in self.requiredFlags) || self.name == 'Slurm'", message="partition flag can be used only on Slurm mode" +// +kubebuilder:validation:XValidation:rule="!has(self.requiredFlags) || self.name != 'Slurm' || !('parallelism' in self.requiredFlags)", message="parallelism flag can't be used on Slurm mode" +// +kubebuilder:validation:XValidation:rule="!has(self.requiredFlags) || self.name != 'Slurm' || !('completions' in self.requiredFlags)", message="completions flag can't be used on Slurm mode" +type SupportedMode struct { + // name determines which template will be used and which object will eventually be created. + // Possible values are Interactive, Job, RayJob, RayCluster and Slurm. + // + // +kubebuilder:validation:Required + // +kubebuilder:validation:Enum=Interactive;Job;RayJob;RayCluster;Slurm + Name ApplicationProfileMode `json:"name"` + + // template is the name of the template. + // Template type depends on ApplicationProfileMode: + // - on Interactive mode it must be v1/PodTemplate + // - on Job mode it must be kjobctl.x-k8s.io/v1alpha1/JobTemplate + // - on RayJob mode it must be kjobctl.x-k8s.io/v1alpha1/RayJobTemplate + // - on RayCluster mode it must be kjobctl.x-k8s.io/v1alpha1/RayClusterTemplate + // - on Slurm mode it must be kjobctl.x-k8s.io/v1alpha1/JobTemplate + // + // +kubebuilder:validation:Required + Template TemplateReference `json:"template"` + + // requiredFlags point which cli flags are required to be passed in order to fill the gaps in the templates. + // Possible values are cmd, parallelism, completions, replicas, min-replicas, max-replicas, request, localqueue, and raycluster. + // replicas, min-replicas, and max-replicas flags used only for RayJob and RayCluster mode. + // The raycluster flag used only for the RayJob mode. + // The request flag used only for Interactive and Job modes. + // The cmd flag used only for Interactive, Job, and RayJob. + // The time and priority flags can be used in all modes. + // If the raycluster flag are set, none of localqueue, replicas, min-replicas, or max-replicas can be set. + // For the Slurm mode, the possible values are: array, cpus-per-task, error, gpus-per-task, input, job-name, mem, mem-per-cpu, + // mem-per-gpu, mem-per-task, nodes, ntasks, output, partition, localqueue. + // + // cmd and requests values are going to be added only to the first primary container. + // + // +optional + // +listType=set + // +kubebuilder:validation:MaxItems=14 + RequiredFlags []Flag `json:"requiredFlags,omitempty"` +} + +// ApplicationProfileSpec defines the desired state of ApplicationProfile +type ApplicationProfileSpec struct { + // +listType=map + // +listMapKey=name + // +kubebuilder:validation:Required + SupportedModes []SupportedMode `json:"supportedModes"` + + // +optional + // +listType=set + VolumeBundles []VolumeBundleReference `json:"volumeBundles,omitempty"` +} + +// +genclient +// +kubebuilder:object:root=true + +// ApplicationProfile is the Schema for the applicationprofiles API +type ApplicationProfile struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec ApplicationProfileSpec `json:"spec,omitempty"` +} + +// +kubebuilder:object:root=true + +// ApplicationProfileList contains a list of ApplicationProfile +type ApplicationProfileList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + + Items []ApplicationProfile `json:"items"` +} + +func init() { + SchemeBuilder.Register(&ApplicationProfile{}, &ApplicationProfileList{}) +} diff --git a/apis/v1alpha1/doc.go b/apis/v1alpha1/doc.go new file mode 100644 index 0000000..75ca753 --- /dev/null +++ b/apis/v1alpha1/doc.go @@ -0,0 +1,20 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// +kubebuilder:object:generate=true +// +groupName=kjobctl.x-k8s.io + +package v1alpha1 diff --git a/apis/v1alpha1/groupversion_info.go b/apis/v1alpha1/groupversion_info.go new file mode 100644 index 0000000..2b59c4f --- /dev/null +++ b/apis/v1alpha1/groupversion_info.go @@ -0,0 +1,39 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package v1alpha1 contains API Schema definitions for the kjobctl.x-k8s.io v1alpha1 API group +// +kubebuilder:object:generate=true +// +groupName=kjobctl.x-k8s.io +package v1alpha1 + +import ( + "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/controller-runtime/pkg/scheme" +) + +var ( + // GroupVersion is group version used to register these objects + GroupVersion = schema.GroupVersion{Group: "kjobctl.x-k8s.io", Version: "v1alpha1"} + + // SchemeGroupVersion is alias to GroupVersion for client-go libraries. + SchemeGroupVersion = GroupVersion + + // SchemeBuilder is used to add go types to the GroupVersionKind scheme + SchemeBuilder = &scheme.Builder{GroupVersion: GroupVersion} + + // AddToScheme adds the types in this group-version to the given scheme. + AddToScheme = SchemeBuilder.AddToScheme +) diff --git a/apis/v1alpha1/job_template_types.go b/apis/v1alpha1/job_template_types.go new file mode 100644 index 0000000..0340cd1 --- /dev/null +++ b/apis/v1alpha1/job_template_types.go @@ -0,0 +1,63 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + batchv1 "k8s.io/api/batch/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// JobTemplateSpec describes the data a job should have when created from a template +type JobTemplateSpec struct { + // Standard object's metadata. + // + // +optional + metav1.ObjectMeta `json:"metadata,omitempty"` + + // Specification of the desired behavior of the job. + // + // +kubebuilder:validation:Required + Spec batchv1.JobSpec `json:"spec"` +} + +// +genclient +// +kubebuilder:object:root=true + +// JobTemplate is the Schema for the jobtemplate API +type JobTemplate struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + // Template defines the jobs that will be created from this pod template. + // + // +kubebuilder:validation:Required + Template JobTemplateSpec `json:"template,omitempty"` +} + +// +kubebuilder:object:root=true + +// JobTemplateList contains a list of JobTemplate +type JobTemplateList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + + Items []JobTemplate `json:"items"` +} + +func init() { + SchemeBuilder.Register(&JobTemplate{}, &JobTemplateList{}) +} diff --git a/apis/v1alpha1/ray_cluster_template_types.go b/apis/v1alpha1/ray_cluster_template_types.go new file mode 100644 index 0000000..c21c47c --- /dev/null +++ b/apis/v1alpha1/ray_cluster_template_types.go @@ -0,0 +1,63 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// RayClusterTemplateSpec describes the data a raycluster should have when created from a template +type RayClusterTemplateSpec struct { + // Standard object's metadata. + // + // +optional + metav1.ObjectMeta `json:"metadata,omitempty"` + + // Specification of the desired behavior of the raycluster. + // + // +kubebuilder:validation:Required + Spec rayv1.RayClusterSpec `json:"spec"` +} + +// +genclient +// +kubebuilder:object:root=true + +// RayClusterTemplate is the Schema for the rayclustertemplate API +type RayClusterTemplate struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + // Template defines rayclusters that will be created from this raycluster template. + // + // +kubebuilder:validation:Required + Template RayClusterTemplateSpec `json:"template,omitempty"` +} + +// +kubebuilder:object:root=true + +// RayClusterTemplateList contains a list of RayClusterTemplate +type RayClusterTemplateList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + + Items []RayClusterTemplate `json:"items"` +} + +func init() { + SchemeBuilder.Register(&RayClusterTemplate{}, &RayClusterTemplateList{}) +} diff --git a/apis/v1alpha1/ray_job_template_types.go b/apis/v1alpha1/ray_job_template_types.go new file mode 100644 index 0000000..f31f0d7 --- /dev/null +++ b/apis/v1alpha1/ray_job_template_types.go @@ -0,0 +1,63 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// RayJobTemplateSpec describes the data a rayjob should have when created from a template +type RayJobTemplateSpec struct { + // Standard object's metadata. + // + // +optional + metav1.ObjectMeta `json:"metadata,omitempty"` + + // Specification of the desired behavior of the rayjob. + // + // +kubebuilder:validation:Required + Spec rayv1.RayJobSpec `json:"spec"` +} + +// +genclient +// +kubebuilder:object:root=true + +// RayJobTemplate is the Schema for the rayjobtemplate API +type RayJobTemplate struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + // Template defines rayjobs that will be created from this rayjob template. + // + // +kubebuilder:validation:Required + Template RayJobTemplateSpec `json:"template,omitempty"` +} + +// +kubebuilder:object:root=true + +// RayJobTemplateList contains a list of RayJobTemplate +type RayJobTemplateList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + + Items []RayJobTemplate `json:"items"` +} + +func init() { + SchemeBuilder.Register(&RayJobTemplate{}, &RayJobTemplateList{}) +} diff --git a/apis/v1alpha1/volumebundle_types.go b/apis/v1alpha1/volumebundle_types.go new file mode 100644 index 0000000..87c2a5f --- /dev/null +++ b/apis/v1alpha1/volumebundle_types.go @@ -0,0 +1,68 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// VolumeBundleSpec defines the desired state of VolumeBundle +type VolumeBundleSpec struct { + // volumes is a set of volumes that will be added to all pods of the job. + // +listType=map + // +listMapKey=name + // +kubebuilder:validation:MinItems=1 + Volumes []corev1.Volume `json:"volumes"` + + // containerVolumeMounts is a list of locations in each container of a pod where the volumes will be mounted. + // +listType=map + // +listMapKey=mountPath + // +kubebuilder:validation:MinItems=1 + ContainerVolumeMounts []corev1.VolumeMount `json:"containerVolumeMounts"` + + // envVars are environment variables that refer to absolute paths in the container filesystem. + // These key/value pairs will be available in containers as environment variables. + // +optional + // +listType=map + // +listMapKey=name + EnvVars []corev1.EnvVar `json:"envVars,omitempty"` +} + +// +genclient +// +kubebuilder:object:root=true + +// VolumeBundle is the Schema for the volumebundles API +type VolumeBundle struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec VolumeBundleSpec `json:"spec,omitempty"` +} + +// +kubebuilder:object:root=true + +// VolumeBundleList contains a list of VolumeBundle +type VolumeBundleList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []VolumeBundle `json:"items"` +} + +func init() { + SchemeBuilder.Register(&VolumeBundle{}, &VolumeBundleList{}) +} diff --git a/apis/v1alpha1/zz_generated.deepcopy.go b/apis/v1alpha1/zz_generated.deepcopy.go new file mode 100644 index 0000000..148dc45 --- /dev/null +++ b/apis/v1alpha1/zz_generated.deepcopy.go @@ -0,0 +1,450 @@ +//go:build !ignore_autogenerated + +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by controller-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + "k8s.io/api/core/v1" + runtime "k8s.io/apimachinery/pkg/runtime" +) + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ApplicationProfile) DeepCopyInto(out *ApplicationProfile) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ApplicationProfile. +func (in *ApplicationProfile) DeepCopy() *ApplicationProfile { + if in == nil { + return nil + } + out := new(ApplicationProfile) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *ApplicationProfile) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ApplicationProfileList) DeepCopyInto(out *ApplicationProfileList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]ApplicationProfile, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ApplicationProfileList. +func (in *ApplicationProfileList) DeepCopy() *ApplicationProfileList { + if in == nil { + return nil + } + out := new(ApplicationProfileList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *ApplicationProfileList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ApplicationProfileSpec) DeepCopyInto(out *ApplicationProfileSpec) { + *out = *in + if in.SupportedModes != nil { + in, out := &in.SupportedModes, &out.SupportedModes + *out = make([]SupportedMode, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.VolumeBundles != nil { + in, out := &in.VolumeBundles, &out.VolumeBundles + *out = make([]VolumeBundleReference, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ApplicationProfileSpec. +func (in *ApplicationProfileSpec) DeepCopy() *ApplicationProfileSpec { + if in == nil { + return nil + } + out := new(ApplicationProfileSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *JobTemplate) DeepCopyInto(out *JobTemplate) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Template.DeepCopyInto(&out.Template) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new JobTemplate. +func (in *JobTemplate) DeepCopy() *JobTemplate { + if in == nil { + return nil + } + out := new(JobTemplate) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *JobTemplate) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *JobTemplateList) DeepCopyInto(out *JobTemplateList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]JobTemplate, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new JobTemplateList. +func (in *JobTemplateList) DeepCopy() *JobTemplateList { + if in == nil { + return nil + } + out := new(JobTemplateList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *JobTemplateList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *JobTemplateSpec) DeepCopyInto(out *JobTemplateSpec) { + *out = *in + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new JobTemplateSpec. +func (in *JobTemplateSpec) DeepCopy() *JobTemplateSpec { + if in == nil { + return nil + } + out := new(JobTemplateSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RayClusterTemplate) DeepCopyInto(out *RayClusterTemplate) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Template.DeepCopyInto(&out.Template) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RayClusterTemplate. +func (in *RayClusterTemplate) DeepCopy() *RayClusterTemplate { + if in == nil { + return nil + } + out := new(RayClusterTemplate) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *RayClusterTemplate) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RayClusterTemplateList) DeepCopyInto(out *RayClusterTemplateList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]RayClusterTemplate, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RayClusterTemplateList. +func (in *RayClusterTemplateList) DeepCopy() *RayClusterTemplateList { + if in == nil { + return nil + } + out := new(RayClusterTemplateList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *RayClusterTemplateList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RayClusterTemplateSpec) DeepCopyInto(out *RayClusterTemplateSpec) { + *out = *in + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RayClusterTemplateSpec. +func (in *RayClusterTemplateSpec) DeepCopy() *RayClusterTemplateSpec { + if in == nil { + return nil + } + out := new(RayClusterTemplateSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RayJobTemplate) DeepCopyInto(out *RayJobTemplate) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Template.DeepCopyInto(&out.Template) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RayJobTemplate. +func (in *RayJobTemplate) DeepCopy() *RayJobTemplate { + if in == nil { + return nil + } + out := new(RayJobTemplate) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *RayJobTemplate) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RayJobTemplateList) DeepCopyInto(out *RayJobTemplateList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]RayJobTemplate, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RayJobTemplateList. +func (in *RayJobTemplateList) DeepCopy() *RayJobTemplateList { + if in == nil { + return nil + } + out := new(RayJobTemplateList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *RayJobTemplateList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RayJobTemplateSpec) DeepCopyInto(out *RayJobTemplateSpec) { + *out = *in + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RayJobTemplateSpec. +func (in *RayJobTemplateSpec) DeepCopy() *RayJobTemplateSpec { + if in == nil { + return nil + } + out := new(RayJobTemplateSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SupportedMode) DeepCopyInto(out *SupportedMode) { + *out = *in + if in.RequiredFlags != nil { + in, out := &in.RequiredFlags, &out.RequiredFlags + *out = make([]Flag, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SupportedMode. +func (in *SupportedMode) DeepCopy() *SupportedMode { + if in == nil { + return nil + } + out := new(SupportedMode) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *VolumeBundle) DeepCopyInto(out *VolumeBundle) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VolumeBundle. +func (in *VolumeBundle) DeepCopy() *VolumeBundle { + if in == nil { + return nil + } + out := new(VolumeBundle) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *VolumeBundle) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *VolumeBundleList) DeepCopyInto(out *VolumeBundleList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]VolumeBundle, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VolumeBundleList. +func (in *VolumeBundleList) DeepCopy() *VolumeBundleList { + if in == nil { + return nil + } + out := new(VolumeBundleList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *VolumeBundleList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *VolumeBundleSpec) DeepCopyInto(out *VolumeBundleSpec) { + *out = *in + if in.Volumes != nil { + in, out := &in.Volumes, &out.Volumes + *out = make([]v1.Volume, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.ContainerVolumeMounts != nil { + in, out := &in.ContainerVolumeMounts, &out.ContainerVolumeMounts + *out = make([]v1.VolumeMount, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.EnvVars != nil { + in, out := &in.EnvVars, &out.EnvVars + *out = make([]v1.EnvVar, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VolumeBundleSpec. +func (in *VolumeBundleSpec) DeepCopy() *VolumeBundleSpec { + if in == nil { + return nil + } + out := new(VolumeBundleSpec) + in.DeepCopyInto(out) + return out +} diff --git a/client-go/clientset/versioned/clientset.go b/client-go/clientset/versioned/clientset.go new file mode 100644 index 0000000..ba83e32 --- /dev/null +++ b/client-go/clientset/versioned/clientset.go @@ -0,0 +1,119 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by client-gen. DO NOT EDIT. + +package versioned + +import ( + "fmt" + "net/http" + + discovery "k8s.io/client-go/discovery" + rest "k8s.io/client-go/rest" + flowcontrol "k8s.io/client-go/util/flowcontrol" + kjobctlv1alpha1 "sigs.k8s.io/kueue/cmd/experimental/kjobctl/client-go/clientset/versioned/typed/apis/v1alpha1" +) + +type Interface interface { + Discovery() discovery.DiscoveryInterface + KjobctlV1alpha1() kjobctlv1alpha1.KjobctlV1alpha1Interface +} + +// Clientset contains the clients for groups. +type Clientset struct { + *discovery.DiscoveryClient + kjobctlV1alpha1 *kjobctlv1alpha1.KjobctlV1alpha1Client +} + +// KjobctlV1alpha1 retrieves the KjobctlV1alpha1Client +func (c *Clientset) KjobctlV1alpha1() kjobctlv1alpha1.KjobctlV1alpha1Interface { + return c.kjobctlV1alpha1 +} + +// Discovery retrieves the DiscoveryClient +func (c *Clientset) Discovery() discovery.DiscoveryInterface { + if c == nil { + return nil + } + return c.DiscoveryClient +} + +// NewForConfig creates a new Clientset for the given config. +// If config's RateLimiter is not set and QPS and Burst are acceptable, +// NewForConfig will generate a rate-limiter in configShallowCopy. +// NewForConfig is equivalent to NewForConfigAndClient(c, httpClient), +// where httpClient was generated with rest.HTTPClientFor(c). +func NewForConfig(c *rest.Config) (*Clientset, error) { + configShallowCopy := *c + + if configShallowCopy.UserAgent == "" { + configShallowCopy.UserAgent = rest.DefaultKubernetesUserAgent() + } + + // share the transport between all clients + httpClient, err := rest.HTTPClientFor(&configShallowCopy) + if err != nil { + return nil, err + } + + return NewForConfigAndClient(&configShallowCopy, httpClient) +} + +// NewForConfigAndClient creates a new Clientset for the given config and http client. +// Note the http client provided takes precedence over the configured transport values. +// If config's RateLimiter is not set and QPS and Burst are acceptable, +// NewForConfigAndClient will generate a rate-limiter in configShallowCopy. +func NewForConfigAndClient(c *rest.Config, httpClient *http.Client) (*Clientset, error) { + configShallowCopy := *c + if configShallowCopy.RateLimiter == nil && configShallowCopy.QPS > 0 { + if configShallowCopy.Burst <= 0 { + return nil, fmt.Errorf("burst is required to be greater than 0 when RateLimiter is not set and QPS is set to greater than 0") + } + configShallowCopy.RateLimiter = flowcontrol.NewTokenBucketRateLimiter(configShallowCopy.QPS, configShallowCopy.Burst) + } + + var cs Clientset + var err error + cs.kjobctlV1alpha1, err = kjobctlv1alpha1.NewForConfigAndClient(&configShallowCopy, httpClient) + if err != nil { + return nil, err + } + + cs.DiscoveryClient, err = discovery.NewDiscoveryClientForConfigAndClient(&configShallowCopy, httpClient) + if err != nil { + return nil, err + } + return &cs, nil +} + +// NewForConfigOrDie creates a new Clientset for the given config and +// panics if there is an error in the config. +func NewForConfigOrDie(c *rest.Config) *Clientset { + cs, err := NewForConfig(c) + if err != nil { + panic(err) + } + return cs +} + +// New creates a new Clientset for the given RESTClient. +func New(c rest.Interface) *Clientset { + var cs Clientset + cs.kjobctlV1alpha1 = kjobctlv1alpha1.New(c) + + cs.DiscoveryClient = discovery.NewDiscoveryClient(c) + return &cs +} diff --git a/client-go/clientset/versioned/fake/clientset_generated.go b/client-go/clientset/versioned/fake/clientset_generated.go new file mode 100644 index 0000000..0f9beb6 --- /dev/null +++ b/client-go/clientset/versioned/fake/clientset_generated.go @@ -0,0 +1,88 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by client-gen. DO NOT EDIT. + +package fake + +import ( + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/watch" + "k8s.io/client-go/discovery" + fakediscovery "k8s.io/client-go/discovery/fake" + "k8s.io/client-go/testing" + clientset "sigs.k8s.io/kueue/cmd/experimental/kjobctl/client-go/clientset/versioned" + kjobctlv1alpha1 "sigs.k8s.io/kueue/cmd/experimental/kjobctl/client-go/clientset/versioned/typed/apis/v1alpha1" + fakekjobctlv1alpha1 "sigs.k8s.io/kueue/cmd/experimental/kjobctl/client-go/clientset/versioned/typed/apis/v1alpha1/fake" +) + +// NewSimpleClientset returns a clientset that will respond with the provided objects. +// It's backed by a very simple object tracker that processes creates, updates and deletions as-is, +// without applying any field management, validations and/or defaults. It shouldn't be considered a replacement +// for a real clientset and is mostly useful in simple unit tests. +// +// DEPRECATED: NewClientset replaces this with support for field management, which significantly improves +// server side apply testing. NewClientset is only available when apply configurations are generated (e.g. +// via --with-applyconfig). +func NewSimpleClientset(objects ...runtime.Object) *Clientset { + o := testing.NewObjectTracker(scheme, codecs.UniversalDecoder()) + for _, obj := range objects { + if err := o.Add(obj); err != nil { + panic(err) + } + } + + cs := &Clientset{tracker: o} + cs.discovery = &fakediscovery.FakeDiscovery{Fake: &cs.Fake} + cs.AddReactor("*", "*", testing.ObjectReaction(o)) + cs.AddWatchReactor("*", func(action testing.Action) (handled bool, ret watch.Interface, err error) { + gvr := action.GetResource() + ns := action.GetNamespace() + watch, err := o.Watch(gvr, ns) + if err != nil { + return false, nil, err + } + return true, watch, nil + }) + + return cs +} + +// Clientset implements clientset.Interface. Meant to be embedded into a +// struct to get a default implementation. This makes faking out just the method +// you want to test easier. +type Clientset struct { + testing.Fake + discovery *fakediscovery.FakeDiscovery + tracker testing.ObjectTracker +} + +func (c *Clientset) Discovery() discovery.DiscoveryInterface { + return c.discovery +} + +func (c *Clientset) Tracker() testing.ObjectTracker { + return c.tracker +} + +var ( + _ clientset.Interface = &Clientset{} + _ testing.FakeClient = &Clientset{} +) + +// KjobctlV1alpha1 retrieves the KjobctlV1alpha1Client +func (c *Clientset) KjobctlV1alpha1() kjobctlv1alpha1.KjobctlV1alpha1Interface { + return &fakekjobctlv1alpha1.FakeKjobctlV1alpha1{Fake: &c.Fake} +} diff --git a/client-go/clientset/versioned/fake/doc.go b/client-go/clientset/versioned/fake/doc.go new file mode 100644 index 0000000..634bd02 --- /dev/null +++ b/client-go/clientset/versioned/fake/doc.go @@ -0,0 +1,19 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by client-gen. DO NOT EDIT. + +// This package has the automatically generated fake clientset. +package fake diff --git a/client-go/clientset/versioned/fake/register.go b/client-go/clientset/versioned/fake/register.go new file mode 100644 index 0000000..a4e627a --- /dev/null +++ b/client-go/clientset/versioned/fake/register.go @@ -0,0 +1,55 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by client-gen. DO NOT EDIT. + +package fake + +import ( + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + runtime "k8s.io/apimachinery/pkg/runtime" + schema "k8s.io/apimachinery/pkg/runtime/schema" + serializer "k8s.io/apimachinery/pkg/runtime/serializer" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + kjobctlv1alpha1 "sigs.k8s.io/kueue/cmd/experimental/kjobctl/apis/v1alpha1" +) + +var scheme = runtime.NewScheme() +var codecs = serializer.NewCodecFactory(scheme) + +var localSchemeBuilder = runtime.SchemeBuilder{ + kjobctlv1alpha1.AddToScheme, +} + +// AddToScheme adds all types of this clientset into the given scheme. This allows composition +// of clientsets, like in: +// +// import ( +// "k8s.io/client-go/kubernetes" +// clientsetscheme "k8s.io/client-go/kubernetes/scheme" +// aggregatorclientsetscheme "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset/scheme" +// ) +// +// kclientset, _ := kubernetes.NewForConfig(c) +// _ = aggregatorclientsetscheme.AddToScheme(clientsetscheme.Scheme) +// +// After this, RawExtensions in Kubernetes types will serialize kube-aggregator types +// correctly. +var AddToScheme = localSchemeBuilder.AddToScheme + +func init() { + v1.AddToGroupVersion(scheme, schema.GroupVersion{Version: "v1"}) + utilruntime.Must(AddToScheme(scheme)) +} diff --git a/client-go/clientset/versioned/scheme/doc.go b/client-go/clientset/versioned/scheme/doc.go new file mode 100644 index 0000000..40e42c2 --- /dev/null +++ b/client-go/clientset/versioned/scheme/doc.go @@ -0,0 +1,19 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by client-gen. DO NOT EDIT. + +// This package contains the scheme of the automatically generated clientset. +package scheme diff --git a/client-go/clientset/versioned/scheme/register.go b/client-go/clientset/versioned/scheme/register.go new file mode 100644 index 0000000..c335f95 --- /dev/null +++ b/client-go/clientset/versioned/scheme/register.go @@ -0,0 +1,55 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by client-gen. DO NOT EDIT. + +package scheme + +import ( + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + runtime "k8s.io/apimachinery/pkg/runtime" + schema "k8s.io/apimachinery/pkg/runtime/schema" + serializer "k8s.io/apimachinery/pkg/runtime/serializer" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + kjobctlv1alpha1 "sigs.k8s.io/kueue/cmd/experimental/kjobctl/apis/v1alpha1" +) + +var Scheme = runtime.NewScheme() +var Codecs = serializer.NewCodecFactory(Scheme) +var ParameterCodec = runtime.NewParameterCodec(Scheme) +var localSchemeBuilder = runtime.SchemeBuilder{ + kjobctlv1alpha1.AddToScheme, +} + +// AddToScheme adds all types of this clientset into the given scheme. This allows composition +// of clientsets, like in: +// +// import ( +// "k8s.io/client-go/kubernetes" +// clientsetscheme "k8s.io/client-go/kubernetes/scheme" +// aggregatorclientsetscheme "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset/scheme" +// ) +// +// kclientset, _ := kubernetes.NewForConfig(c) +// _ = aggregatorclientsetscheme.AddToScheme(clientsetscheme.Scheme) +// +// After this, RawExtensions in Kubernetes types will serialize kube-aggregator types +// correctly. +var AddToScheme = localSchemeBuilder.AddToScheme + +func init() { + v1.AddToGroupVersion(Scheme, schema.GroupVersion{Version: "v1"}) + utilruntime.Must(AddToScheme(Scheme)) +} diff --git a/client-go/clientset/versioned/typed/apis/v1alpha1/apis_client.go b/client-go/clientset/versioned/typed/apis/v1alpha1/apis_client.go new file mode 100644 index 0000000..9204ced --- /dev/null +++ b/client-go/clientset/versioned/typed/apis/v1alpha1/apis_client.go @@ -0,0 +1,126 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by client-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + "net/http" + + rest "k8s.io/client-go/rest" + v1alpha1 "sigs.k8s.io/kueue/cmd/experimental/kjobctl/apis/v1alpha1" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/client-go/clientset/versioned/scheme" +) + +type KjobctlV1alpha1Interface interface { + RESTClient() rest.Interface + ApplicationProfilesGetter + JobTemplatesGetter + RayClusterTemplatesGetter + RayJobTemplatesGetter + VolumeBundlesGetter +} + +// KjobctlV1alpha1Client is used to interact with features provided by the kjobctl.x-k8s.io group. +type KjobctlV1alpha1Client struct { + restClient rest.Interface +} + +func (c *KjobctlV1alpha1Client) ApplicationProfiles(namespace string) ApplicationProfileInterface { + return newApplicationProfiles(c, namespace) +} + +func (c *KjobctlV1alpha1Client) JobTemplates(namespace string) JobTemplateInterface { + return newJobTemplates(c, namespace) +} + +func (c *KjobctlV1alpha1Client) RayClusterTemplates(namespace string) RayClusterTemplateInterface { + return newRayClusterTemplates(c, namespace) +} + +func (c *KjobctlV1alpha1Client) RayJobTemplates(namespace string) RayJobTemplateInterface { + return newRayJobTemplates(c, namespace) +} + +func (c *KjobctlV1alpha1Client) VolumeBundles(namespace string) VolumeBundleInterface { + return newVolumeBundles(c, namespace) +} + +// NewForConfig creates a new KjobctlV1alpha1Client for the given config. +// NewForConfig is equivalent to NewForConfigAndClient(c, httpClient), +// where httpClient was generated with rest.HTTPClientFor(c). +func NewForConfig(c *rest.Config) (*KjobctlV1alpha1Client, error) { + config := *c + if err := setConfigDefaults(&config); err != nil { + return nil, err + } + httpClient, err := rest.HTTPClientFor(&config) + if err != nil { + return nil, err + } + return NewForConfigAndClient(&config, httpClient) +} + +// NewForConfigAndClient creates a new KjobctlV1alpha1Client for the given config and http client. +// Note the http client provided takes precedence over the configured transport values. +func NewForConfigAndClient(c *rest.Config, h *http.Client) (*KjobctlV1alpha1Client, error) { + config := *c + if err := setConfigDefaults(&config); err != nil { + return nil, err + } + client, err := rest.RESTClientForConfigAndClient(&config, h) + if err != nil { + return nil, err + } + return &KjobctlV1alpha1Client{client}, nil +} + +// NewForConfigOrDie creates a new KjobctlV1alpha1Client for the given config and +// panics if there is an error in the config. +func NewForConfigOrDie(c *rest.Config) *KjobctlV1alpha1Client { + client, err := NewForConfig(c) + if err != nil { + panic(err) + } + return client +} + +// New creates a new KjobctlV1alpha1Client for the given RESTClient. +func New(c rest.Interface) *KjobctlV1alpha1Client { + return &KjobctlV1alpha1Client{c} +} + +func setConfigDefaults(config *rest.Config) error { + gv := v1alpha1.SchemeGroupVersion + config.GroupVersion = &gv + config.APIPath = "/apis" + config.NegotiatedSerializer = scheme.Codecs.WithoutConversion() + + if config.UserAgent == "" { + config.UserAgent = rest.DefaultKubernetesUserAgent() + } + + return nil +} + +// RESTClient returns a RESTClient that is used to communicate +// with API server by this client implementation. +func (c *KjobctlV1alpha1Client) RESTClient() rest.Interface { + if c == nil { + return nil + } + return c.restClient +} diff --git a/client-go/clientset/versioned/typed/apis/v1alpha1/applicationprofile.go b/client-go/clientset/versioned/typed/apis/v1alpha1/applicationprofile.go new file mode 100644 index 0000000..e57894b --- /dev/null +++ b/client-go/clientset/versioned/typed/apis/v1alpha1/applicationprofile.go @@ -0,0 +1,66 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by client-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + "context" + + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + types "k8s.io/apimachinery/pkg/types" + watch "k8s.io/apimachinery/pkg/watch" + gentype "k8s.io/client-go/gentype" + v1alpha1 "sigs.k8s.io/kueue/cmd/experimental/kjobctl/apis/v1alpha1" + scheme "sigs.k8s.io/kueue/cmd/experimental/kjobctl/client-go/clientset/versioned/scheme" +) + +// ApplicationProfilesGetter has a method to return a ApplicationProfileInterface. +// A group's client should implement this interface. +type ApplicationProfilesGetter interface { + ApplicationProfiles(namespace string) ApplicationProfileInterface +} + +// ApplicationProfileInterface has methods to work with ApplicationProfile resources. +type ApplicationProfileInterface interface { + Create(ctx context.Context, applicationProfile *v1alpha1.ApplicationProfile, opts v1.CreateOptions) (*v1alpha1.ApplicationProfile, error) + Update(ctx context.Context, applicationProfile *v1alpha1.ApplicationProfile, opts v1.UpdateOptions) (*v1alpha1.ApplicationProfile, error) + Delete(ctx context.Context, name string, opts v1.DeleteOptions) error + DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error + Get(ctx context.Context, name string, opts v1.GetOptions) (*v1alpha1.ApplicationProfile, error) + List(ctx context.Context, opts v1.ListOptions) (*v1alpha1.ApplicationProfileList, error) + Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) + Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *v1alpha1.ApplicationProfile, err error) + ApplicationProfileExpansion +} + +// applicationProfiles implements ApplicationProfileInterface +type applicationProfiles struct { + *gentype.ClientWithList[*v1alpha1.ApplicationProfile, *v1alpha1.ApplicationProfileList] +} + +// newApplicationProfiles returns a ApplicationProfiles +func newApplicationProfiles(c *KjobctlV1alpha1Client, namespace string) *applicationProfiles { + return &applicationProfiles{ + gentype.NewClientWithList[*v1alpha1.ApplicationProfile, *v1alpha1.ApplicationProfileList]( + "applicationprofiles", + c.RESTClient(), + scheme.ParameterCodec, + namespace, + func() *v1alpha1.ApplicationProfile { return &v1alpha1.ApplicationProfile{} }, + func() *v1alpha1.ApplicationProfileList { return &v1alpha1.ApplicationProfileList{} }), + } +} diff --git a/client-go/clientset/versioned/typed/apis/v1alpha1/doc.go b/client-go/clientset/versioned/typed/apis/v1alpha1/doc.go new file mode 100644 index 0000000..28991e2 --- /dev/null +++ b/client-go/clientset/versioned/typed/apis/v1alpha1/doc.go @@ -0,0 +1,19 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by client-gen. DO NOT EDIT. + +// This package has the automatically generated typed clients. +package v1alpha1 diff --git a/client-go/clientset/versioned/typed/apis/v1alpha1/fake/doc.go b/client-go/clientset/versioned/typed/apis/v1alpha1/fake/doc.go new file mode 100644 index 0000000..fbfccbb --- /dev/null +++ b/client-go/clientset/versioned/typed/apis/v1alpha1/fake/doc.go @@ -0,0 +1,19 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by client-gen. DO NOT EDIT. + +// Package fake has the automatically generated clients. +package fake diff --git a/client-go/clientset/versioned/typed/apis/v1alpha1/fake/fake_apis_client.go b/client-go/clientset/versioned/typed/apis/v1alpha1/fake/fake_apis_client.go new file mode 100644 index 0000000..d5bd37c --- /dev/null +++ b/client-go/clientset/versioned/typed/apis/v1alpha1/fake/fake_apis_client.go @@ -0,0 +1,55 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by client-gen. DO NOT EDIT. + +package fake + +import ( + rest "k8s.io/client-go/rest" + testing "k8s.io/client-go/testing" + v1alpha1 "sigs.k8s.io/kueue/cmd/experimental/kjobctl/client-go/clientset/versioned/typed/apis/v1alpha1" +) + +type FakeKjobctlV1alpha1 struct { + *testing.Fake +} + +func (c *FakeKjobctlV1alpha1) ApplicationProfiles(namespace string) v1alpha1.ApplicationProfileInterface { + return &FakeApplicationProfiles{c, namespace} +} + +func (c *FakeKjobctlV1alpha1) JobTemplates(namespace string) v1alpha1.JobTemplateInterface { + return &FakeJobTemplates{c, namespace} +} + +func (c *FakeKjobctlV1alpha1) RayClusterTemplates(namespace string) v1alpha1.RayClusterTemplateInterface { + return &FakeRayClusterTemplates{c, namespace} +} + +func (c *FakeKjobctlV1alpha1) RayJobTemplates(namespace string) v1alpha1.RayJobTemplateInterface { + return &FakeRayJobTemplates{c, namespace} +} + +func (c *FakeKjobctlV1alpha1) VolumeBundles(namespace string) v1alpha1.VolumeBundleInterface { + return &FakeVolumeBundles{c, namespace} +} + +// RESTClient returns a RESTClient that is used to communicate +// with API server by this client implementation. +func (c *FakeKjobctlV1alpha1) RESTClient() rest.Interface { + var ret *rest.RESTClient + return ret +} diff --git a/client-go/clientset/versioned/typed/apis/v1alpha1/fake/fake_applicationprofile.go b/client-go/clientset/versioned/typed/apis/v1alpha1/fake/fake_applicationprofile.go new file mode 100644 index 0000000..80963e1 --- /dev/null +++ b/client-go/clientset/versioned/typed/apis/v1alpha1/fake/fake_applicationprofile.go @@ -0,0 +1,133 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by client-gen. DO NOT EDIT. + +package fake + +import ( + "context" + + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + labels "k8s.io/apimachinery/pkg/labels" + types "k8s.io/apimachinery/pkg/types" + watch "k8s.io/apimachinery/pkg/watch" + testing "k8s.io/client-go/testing" + v1alpha1 "sigs.k8s.io/kueue/cmd/experimental/kjobctl/apis/v1alpha1" +) + +// FakeApplicationProfiles implements ApplicationProfileInterface +type FakeApplicationProfiles struct { + Fake *FakeKjobctlV1alpha1 + ns string +} + +var applicationprofilesResource = v1alpha1.SchemeGroupVersion.WithResource("applicationprofiles") + +var applicationprofilesKind = v1alpha1.SchemeGroupVersion.WithKind("ApplicationProfile") + +// Get takes name of the applicationProfile, and returns the corresponding applicationProfile object, and an error if there is any. +func (c *FakeApplicationProfiles) Get(ctx context.Context, name string, options v1.GetOptions) (result *v1alpha1.ApplicationProfile, err error) { + emptyResult := &v1alpha1.ApplicationProfile{} + obj, err := c.Fake. + Invokes(testing.NewGetActionWithOptions(applicationprofilesResource, c.ns, name, options), emptyResult) + + if obj == nil { + return emptyResult, err + } + return obj.(*v1alpha1.ApplicationProfile), err +} + +// List takes label and field selectors, and returns the list of ApplicationProfiles that match those selectors. +func (c *FakeApplicationProfiles) List(ctx context.Context, opts v1.ListOptions) (result *v1alpha1.ApplicationProfileList, err error) { + emptyResult := &v1alpha1.ApplicationProfileList{} + obj, err := c.Fake. + Invokes(testing.NewListActionWithOptions(applicationprofilesResource, applicationprofilesKind, c.ns, opts), emptyResult) + + if obj == nil { + return emptyResult, err + } + + label, _, _ := testing.ExtractFromListOptions(opts) + if label == nil { + label = labels.Everything() + } + list := &v1alpha1.ApplicationProfileList{ListMeta: obj.(*v1alpha1.ApplicationProfileList).ListMeta} + for _, item := range obj.(*v1alpha1.ApplicationProfileList).Items { + if label.Matches(labels.Set(item.Labels)) { + list.Items = append(list.Items, item) + } + } + return list, err +} + +// Watch returns a watch.Interface that watches the requested applicationProfiles. +func (c *FakeApplicationProfiles) Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) { + return c.Fake. + InvokesWatch(testing.NewWatchActionWithOptions(applicationprofilesResource, c.ns, opts)) + +} + +// Create takes the representation of a applicationProfile and creates it. Returns the server's representation of the applicationProfile, and an error, if there is any. +func (c *FakeApplicationProfiles) Create(ctx context.Context, applicationProfile *v1alpha1.ApplicationProfile, opts v1.CreateOptions) (result *v1alpha1.ApplicationProfile, err error) { + emptyResult := &v1alpha1.ApplicationProfile{} + obj, err := c.Fake. + Invokes(testing.NewCreateActionWithOptions(applicationprofilesResource, c.ns, applicationProfile, opts), emptyResult) + + if obj == nil { + return emptyResult, err + } + return obj.(*v1alpha1.ApplicationProfile), err +} + +// Update takes the representation of a applicationProfile and updates it. Returns the server's representation of the applicationProfile, and an error, if there is any. +func (c *FakeApplicationProfiles) Update(ctx context.Context, applicationProfile *v1alpha1.ApplicationProfile, opts v1.UpdateOptions) (result *v1alpha1.ApplicationProfile, err error) { + emptyResult := &v1alpha1.ApplicationProfile{} + obj, err := c.Fake. + Invokes(testing.NewUpdateActionWithOptions(applicationprofilesResource, c.ns, applicationProfile, opts), emptyResult) + + if obj == nil { + return emptyResult, err + } + return obj.(*v1alpha1.ApplicationProfile), err +} + +// Delete takes name of the applicationProfile and deletes it. Returns an error if one occurs. +func (c *FakeApplicationProfiles) Delete(ctx context.Context, name string, opts v1.DeleteOptions) error { + _, err := c.Fake. + Invokes(testing.NewDeleteActionWithOptions(applicationprofilesResource, c.ns, name, opts), &v1alpha1.ApplicationProfile{}) + + return err +} + +// DeleteCollection deletes a collection of objects. +func (c *FakeApplicationProfiles) DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error { + action := testing.NewDeleteCollectionActionWithOptions(applicationprofilesResource, c.ns, opts, listOpts) + + _, err := c.Fake.Invokes(action, &v1alpha1.ApplicationProfileList{}) + return err +} + +// Patch applies the patch and returns the patched applicationProfile. +func (c *FakeApplicationProfiles) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *v1alpha1.ApplicationProfile, err error) { + emptyResult := &v1alpha1.ApplicationProfile{} + obj, err := c.Fake. + Invokes(testing.NewPatchSubresourceActionWithOptions(applicationprofilesResource, c.ns, name, pt, data, opts, subresources...), emptyResult) + + if obj == nil { + return emptyResult, err + } + return obj.(*v1alpha1.ApplicationProfile), err +} diff --git a/client-go/clientset/versioned/typed/apis/v1alpha1/fake/fake_jobtemplate.go b/client-go/clientset/versioned/typed/apis/v1alpha1/fake/fake_jobtemplate.go new file mode 100644 index 0000000..456d176 --- /dev/null +++ b/client-go/clientset/versioned/typed/apis/v1alpha1/fake/fake_jobtemplate.go @@ -0,0 +1,133 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by client-gen. DO NOT EDIT. + +package fake + +import ( + "context" + + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + labels "k8s.io/apimachinery/pkg/labels" + types "k8s.io/apimachinery/pkg/types" + watch "k8s.io/apimachinery/pkg/watch" + testing "k8s.io/client-go/testing" + v1alpha1 "sigs.k8s.io/kueue/cmd/experimental/kjobctl/apis/v1alpha1" +) + +// FakeJobTemplates implements JobTemplateInterface +type FakeJobTemplates struct { + Fake *FakeKjobctlV1alpha1 + ns string +} + +var jobtemplatesResource = v1alpha1.SchemeGroupVersion.WithResource("jobtemplates") + +var jobtemplatesKind = v1alpha1.SchemeGroupVersion.WithKind("JobTemplate") + +// Get takes name of the jobTemplate, and returns the corresponding jobTemplate object, and an error if there is any. +func (c *FakeJobTemplates) Get(ctx context.Context, name string, options v1.GetOptions) (result *v1alpha1.JobTemplate, err error) { + emptyResult := &v1alpha1.JobTemplate{} + obj, err := c.Fake. + Invokes(testing.NewGetActionWithOptions(jobtemplatesResource, c.ns, name, options), emptyResult) + + if obj == nil { + return emptyResult, err + } + return obj.(*v1alpha1.JobTemplate), err +} + +// List takes label and field selectors, and returns the list of JobTemplates that match those selectors. +func (c *FakeJobTemplates) List(ctx context.Context, opts v1.ListOptions) (result *v1alpha1.JobTemplateList, err error) { + emptyResult := &v1alpha1.JobTemplateList{} + obj, err := c.Fake. + Invokes(testing.NewListActionWithOptions(jobtemplatesResource, jobtemplatesKind, c.ns, opts), emptyResult) + + if obj == nil { + return emptyResult, err + } + + label, _, _ := testing.ExtractFromListOptions(opts) + if label == nil { + label = labels.Everything() + } + list := &v1alpha1.JobTemplateList{ListMeta: obj.(*v1alpha1.JobTemplateList).ListMeta} + for _, item := range obj.(*v1alpha1.JobTemplateList).Items { + if label.Matches(labels.Set(item.Labels)) { + list.Items = append(list.Items, item) + } + } + return list, err +} + +// Watch returns a watch.Interface that watches the requested jobTemplates. +func (c *FakeJobTemplates) Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) { + return c.Fake. + InvokesWatch(testing.NewWatchActionWithOptions(jobtemplatesResource, c.ns, opts)) + +} + +// Create takes the representation of a jobTemplate and creates it. Returns the server's representation of the jobTemplate, and an error, if there is any. +func (c *FakeJobTemplates) Create(ctx context.Context, jobTemplate *v1alpha1.JobTemplate, opts v1.CreateOptions) (result *v1alpha1.JobTemplate, err error) { + emptyResult := &v1alpha1.JobTemplate{} + obj, err := c.Fake. + Invokes(testing.NewCreateActionWithOptions(jobtemplatesResource, c.ns, jobTemplate, opts), emptyResult) + + if obj == nil { + return emptyResult, err + } + return obj.(*v1alpha1.JobTemplate), err +} + +// Update takes the representation of a jobTemplate and updates it. Returns the server's representation of the jobTemplate, and an error, if there is any. +func (c *FakeJobTemplates) Update(ctx context.Context, jobTemplate *v1alpha1.JobTemplate, opts v1.UpdateOptions) (result *v1alpha1.JobTemplate, err error) { + emptyResult := &v1alpha1.JobTemplate{} + obj, err := c.Fake. + Invokes(testing.NewUpdateActionWithOptions(jobtemplatesResource, c.ns, jobTemplate, opts), emptyResult) + + if obj == nil { + return emptyResult, err + } + return obj.(*v1alpha1.JobTemplate), err +} + +// Delete takes name of the jobTemplate and deletes it. Returns an error if one occurs. +func (c *FakeJobTemplates) Delete(ctx context.Context, name string, opts v1.DeleteOptions) error { + _, err := c.Fake. + Invokes(testing.NewDeleteActionWithOptions(jobtemplatesResource, c.ns, name, opts), &v1alpha1.JobTemplate{}) + + return err +} + +// DeleteCollection deletes a collection of objects. +func (c *FakeJobTemplates) DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error { + action := testing.NewDeleteCollectionActionWithOptions(jobtemplatesResource, c.ns, opts, listOpts) + + _, err := c.Fake.Invokes(action, &v1alpha1.JobTemplateList{}) + return err +} + +// Patch applies the patch and returns the patched jobTemplate. +func (c *FakeJobTemplates) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *v1alpha1.JobTemplate, err error) { + emptyResult := &v1alpha1.JobTemplate{} + obj, err := c.Fake. + Invokes(testing.NewPatchSubresourceActionWithOptions(jobtemplatesResource, c.ns, name, pt, data, opts, subresources...), emptyResult) + + if obj == nil { + return emptyResult, err + } + return obj.(*v1alpha1.JobTemplate), err +} diff --git a/client-go/clientset/versioned/typed/apis/v1alpha1/fake/fake_rayclustertemplate.go b/client-go/clientset/versioned/typed/apis/v1alpha1/fake/fake_rayclustertemplate.go new file mode 100644 index 0000000..4e0cb86 --- /dev/null +++ b/client-go/clientset/versioned/typed/apis/v1alpha1/fake/fake_rayclustertemplate.go @@ -0,0 +1,133 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by client-gen. DO NOT EDIT. + +package fake + +import ( + "context" + + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + labels "k8s.io/apimachinery/pkg/labels" + types "k8s.io/apimachinery/pkg/types" + watch "k8s.io/apimachinery/pkg/watch" + testing "k8s.io/client-go/testing" + v1alpha1 "sigs.k8s.io/kueue/cmd/experimental/kjobctl/apis/v1alpha1" +) + +// FakeRayClusterTemplates implements RayClusterTemplateInterface +type FakeRayClusterTemplates struct { + Fake *FakeKjobctlV1alpha1 + ns string +} + +var rayclustertemplatesResource = v1alpha1.SchemeGroupVersion.WithResource("rayclustertemplates") + +var rayclustertemplatesKind = v1alpha1.SchemeGroupVersion.WithKind("RayClusterTemplate") + +// Get takes name of the rayClusterTemplate, and returns the corresponding rayClusterTemplate object, and an error if there is any. +func (c *FakeRayClusterTemplates) Get(ctx context.Context, name string, options v1.GetOptions) (result *v1alpha1.RayClusterTemplate, err error) { + emptyResult := &v1alpha1.RayClusterTemplate{} + obj, err := c.Fake. + Invokes(testing.NewGetActionWithOptions(rayclustertemplatesResource, c.ns, name, options), emptyResult) + + if obj == nil { + return emptyResult, err + } + return obj.(*v1alpha1.RayClusterTemplate), err +} + +// List takes label and field selectors, and returns the list of RayClusterTemplates that match those selectors. +func (c *FakeRayClusterTemplates) List(ctx context.Context, opts v1.ListOptions) (result *v1alpha1.RayClusterTemplateList, err error) { + emptyResult := &v1alpha1.RayClusterTemplateList{} + obj, err := c.Fake. + Invokes(testing.NewListActionWithOptions(rayclustertemplatesResource, rayclustertemplatesKind, c.ns, opts), emptyResult) + + if obj == nil { + return emptyResult, err + } + + label, _, _ := testing.ExtractFromListOptions(opts) + if label == nil { + label = labels.Everything() + } + list := &v1alpha1.RayClusterTemplateList{ListMeta: obj.(*v1alpha1.RayClusterTemplateList).ListMeta} + for _, item := range obj.(*v1alpha1.RayClusterTemplateList).Items { + if label.Matches(labels.Set(item.Labels)) { + list.Items = append(list.Items, item) + } + } + return list, err +} + +// Watch returns a watch.Interface that watches the requested rayClusterTemplates. +func (c *FakeRayClusterTemplates) Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) { + return c.Fake. + InvokesWatch(testing.NewWatchActionWithOptions(rayclustertemplatesResource, c.ns, opts)) + +} + +// Create takes the representation of a rayClusterTemplate and creates it. Returns the server's representation of the rayClusterTemplate, and an error, if there is any. +func (c *FakeRayClusterTemplates) Create(ctx context.Context, rayClusterTemplate *v1alpha1.RayClusterTemplate, opts v1.CreateOptions) (result *v1alpha1.RayClusterTemplate, err error) { + emptyResult := &v1alpha1.RayClusterTemplate{} + obj, err := c.Fake. + Invokes(testing.NewCreateActionWithOptions(rayclustertemplatesResource, c.ns, rayClusterTemplate, opts), emptyResult) + + if obj == nil { + return emptyResult, err + } + return obj.(*v1alpha1.RayClusterTemplate), err +} + +// Update takes the representation of a rayClusterTemplate and updates it. Returns the server's representation of the rayClusterTemplate, and an error, if there is any. +func (c *FakeRayClusterTemplates) Update(ctx context.Context, rayClusterTemplate *v1alpha1.RayClusterTemplate, opts v1.UpdateOptions) (result *v1alpha1.RayClusterTemplate, err error) { + emptyResult := &v1alpha1.RayClusterTemplate{} + obj, err := c.Fake. + Invokes(testing.NewUpdateActionWithOptions(rayclustertemplatesResource, c.ns, rayClusterTemplate, opts), emptyResult) + + if obj == nil { + return emptyResult, err + } + return obj.(*v1alpha1.RayClusterTemplate), err +} + +// Delete takes name of the rayClusterTemplate and deletes it. Returns an error if one occurs. +func (c *FakeRayClusterTemplates) Delete(ctx context.Context, name string, opts v1.DeleteOptions) error { + _, err := c.Fake. + Invokes(testing.NewDeleteActionWithOptions(rayclustertemplatesResource, c.ns, name, opts), &v1alpha1.RayClusterTemplate{}) + + return err +} + +// DeleteCollection deletes a collection of objects. +func (c *FakeRayClusterTemplates) DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error { + action := testing.NewDeleteCollectionActionWithOptions(rayclustertemplatesResource, c.ns, opts, listOpts) + + _, err := c.Fake.Invokes(action, &v1alpha1.RayClusterTemplateList{}) + return err +} + +// Patch applies the patch and returns the patched rayClusterTemplate. +func (c *FakeRayClusterTemplates) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *v1alpha1.RayClusterTemplate, err error) { + emptyResult := &v1alpha1.RayClusterTemplate{} + obj, err := c.Fake. + Invokes(testing.NewPatchSubresourceActionWithOptions(rayclustertemplatesResource, c.ns, name, pt, data, opts, subresources...), emptyResult) + + if obj == nil { + return emptyResult, err + } + return obj.(*v1alpha1.RayClusterTemplate), err +} diff --git a/client-go/clientset/versioned/typed/apis/v1alpha1/fake/fake_rayjobtemplate.go b/client-go/clientset/versioned/typed/apis/v1alpha1/fake/fake_rayjobtemplate.go new file mode 100644 index 0000000..2c9ea71 --- /dev/null +++ b/client-go/clientset/versioned/typed/apis/v1alpha1/fake/fake_rayjobtemplate.go @@ -0,0 +1,133 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by client-gen. DO NOT EDIT. + +package fake + +import ( + "context" + + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + labels "k8s.io/apimachinery/pkg/labels" + types "k8s.io/apimachinery/pkg/types" + watch "k8s.io/apimachinery/pkg/watch" + testing "k8s.io/client-go/testing" + v1alpha1 "sigs.k8s.io/kueue/cmd/experimental/kjobctl/apis/v1alpha1" +) + +// FakeRayJobTemplates implements RayJobTemplateInterface +type FakeRayJobTemplates struct { + Fake *FakeKjobctlV1alpha1 + ns string +} + +var rayjobtemplatesResource = v1alpha1.SchemeGroupVersion.WithResource("rayjobtemplates") + +var rayjobtemplatesKind = v1alpha1.SchemeGroupVersion.WithKind("RayJobTemplate") + +// Get takes name of the rayJobTemplate, and returns the corresponding rayJobTemplate object, and an error if there is any. +func (c *FakeRayJobTemplates) Get(ctx context.Context, name string, options v1.GetOptions) (result *v1alpha1.RayJobTemplate, err error) { + emptyResult := &v1alpha1.RayJobTemplate{} + obj, err := c.Fake. + Invokes(testing.NewGetActionWithOptions(rayjobtemplatesResource, c.ns, name, options), emptyResult) + + if obj == nil { + return emptyResult, err + } + return obj.(*v1alpha1.RayJobTemplate), err +} + +// List takes label and field selectors, and returns the list of RayJobTemplates that match those selectors. +func (c *FakeRayJobTemplates) List(ctx context.Context, opts v1.ListOptions) (result *v1alpha1.RayJobTemplateList, err error) { + emptyResult := &v1alpha1.RayJobTemplateList{} + obj, err := c.Fake. + Invokes(testing.NewListActionWithOptions(rayjobtemplatesResource, rayjobtemplatesKind, c.ns, opts), emptyResult) + + if obj == nil { + return emptyResult, err + } + + label, _, _ := testing.ExtractFromListOptions(opts) + if label == nil { + label = labels.Everything() + } + list := &v1alpha1.RayJobTemplateList{ListMeta: obj.(*v1alpha1.RayJobTemplateList).ListMeta} + for _, item := range obj.(*v1alpha1.RayJobTemplateList).Items { + if label.Matches(labels.Set(item.Labels)) { + list.Items = append(list.Items, item) + } + } + return list, err +} + +// Watch returns a watch.Interface that watches the requested rayJobTemplates. +func (c *FakeRayJobTemplates) Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) { + return c.Fake. + InvokesWatch(testing.NewWatchActionWithOptions(rayjobtemplatesResource, c.ns, opts)) + +} + +// Create takes the representation of a rayJobTemplate and creates it. Returns the server's representation of the rayJobTemplate, and an error, if there is any. +func (c *FakeRayJobTemplates) Create(ctx context.Context, rayJobTemplate *v1alpha1.RayJobTemplate, opts v1.CreateOptions) (result *v1alpha1.RayJobTemplate, err error) { + emptyResult := &v1alpha1.RayJobTemplate{} + obj, err := c.Fake. + Invokes(testing.NewCreateActionWithOptions(rayjobtemplatesResource, c.ns, rayJobTemplate, opts), emptyResult) + + if obj == nil { + return emptyResult, err + } + return obj.(*v1alpha1.RayJobTemplate), err +} + +// Update takes the representation of a rayJobTemplate and updates it. Returns the server's representation of the rayJobTemplate, and an error, if there is any. +func (c *FakeRayJobTemplates) Update(ctx context.Context, rayJobTemplate *v1alpha1.RayJobTemplate, opts v1.UpdateOptions) (result *v1alpha1.RayJobTemplate, err error) { + emptyResult := &v1alpha1.RayJobTemplate{} + obj, err := c.Fake. + Invokes(testing.NewUpdateActionWithOptions(rayjobtemplatesResource, c.ns, rayJobTemplate, opts), emptyResult) + + if obj == nil { + return emptyResult, err + } + return obj.(*v1alpha1.RayJobTemplate), err +} + +// Delete takes name of the rayJobTemplate and deletes it. Returns an error if one occurs. +func (c *FakeRayJobTemplates) Delete(ctx context.Context, name string, opts v1.DeleteOptions) error { + _, err := c.Fake. + Invokes(testing.NewDeleteActionWithOptions(rayjobtemplatesResource, c.ns, name, opts), &v1alpha1.RayJobTemplate{}) + + return err +} + +// DeleteCollection deletes a collection of objects. +func (c *FakeRayJobTemplates) DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error { + action := testing.NewDeleteCollectionActionWithOptions(rayjobtemplatesResource, c.ns, opts, listOpts) + + _, err := c.Fake.Invokes(action, &v1alpha1.RayJobTemplateList{}) + return err +} + +// Patch applies the patch and returns the patched rayJobTemplate. +func (c *FakeRayJobTemplates) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *v1alpha1.RayJobTemplate, err error) { + emptyResult := &v1alpha1.RayJobTemplate{} + obj, err := c.Fake. + Invokes(testing.NewPatchSubresourceActionWithOptions(rayjobtemplatesResource, c.ns, name, pt, data, opts, subresources...), emptyResult) + + if obj == nil { + return emptyResult, err + } + return obj.(*v1alpha1.RayJobTemplate), err +} diff --git a/client-go/clientset/versioned/typed/apis/v1alpha1/fake/fake_volumebundle.go b/client-go/clientset/versioned/typed/apis/v1alpha1/fake/fake_volumebundle.go new file mode 100644 index 0000000..7626eb0 --- /dev/null +++ b/client-go/clientset/versioned/typed/apis/v1alpha1/fake/fake_volumebundle.go @@ -0,0 +1,133 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by client-gen. DO NOT EDIT. + +package fake + +import ( + "context" + + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + labels "k8s.io/apimachinery/pkg/labels" + types "k8s.io/apimachinery/pkg/types" + watch "k8s.io/apimachinery/pkg/watch" + testing "k8s.io/client-go/testing" + v1alpha1 "sigs.k8s.io/kueue/cmd/experimental/kjobctl/apis/v1alpha1" +) + +// FakeVolumeBundles implements VolumeBundleInterface +type FakeVolumeBundles struct { + Fake *FakeKjobctlV1alpha1 + ns string +} + +var volumebundlesResource = v1alpha1.SchemeGroupVersion.WithResource("volumebundles") + +var volumebundlesKind = v1alpha1.SchemeGroupVersion.WithKind("VolumeBundle") + +// Get takes name of the volumeBundle, and returns the corresponding volumeBundle object, and an error if there is any. +func (c *FakeVolumeBundles) Get(ctx context.Context, name string, options v1.GetOptions) (result *v1alpha1.VolumeBundle, err error) { + emptyResult := &v1alpha1.VolumeBundle{} + obj, err := c.Fake. + Invokes(testing.NewGetActionWithOptions(volumebundlesResource, c.ns, name, options), emptyResult) + + if obj == nil { + return emptyResult, err + } + return obj.(*v1alpha1.VolumeBundle), err +} + +// List takes label and field selectors, and returns the list of VolumeBundles that match those selectors. +func (c *FakeVolumeBundles) List(ctx context.Context, opts v1.ListOptions) (result *v1alpha1.VolumeBundleList, err error) { + emptyResult := &v1alpha1.VolumeBundleList{} + obj, err := c.Fake. + Invokes(testing.NewListActionWithOptions(volumebundlesResource, volumebundlesKind, c.ns, opts), emptyResult) + + if obj == nil { + return emptyResult, err + } + + label, _, _ := testing.ExtractFromListOptions(opts) + if label == nil { + label = labels.Everything() + } + list := &v1alpha1.VolumeBundleList{ListMeta: obj.(*v1alpha1.VolumeBundleList).ListMeta} + for _, item := range obj.(*v1alpha1.VolumeBundleList).Items { + if label.Matches(labels.Set(item.Labels)) { + list.Items = append(list.Items, item) + } + } + return list, err +} + +// Watch returns a watch.Interface that watches the requested volumeBundles. +func (c *FakeVolumeBundles) Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) { + return c.Fake. + InvokesWatch(testing.NewWatchActionWithOptions(volumebundlesResource, c.ns, opts)) + +} + +// Create takes the representation of a volumeBundle and creates it. Returns the server's representation of the volumeBundle, and an error, if there is any. +func (c *FakeVolumeBundles) Create(ctx context.Context, volumeBundle *v1alpha1.VolumeBundle, opts v1.CreateOptions) (result *v1alpha1.VolumeBundle, err error) { + emptyResult := &v1alpha1.VolumeBundle{} + obj, err := c.Fake. + Invokes(testing.NewCreateActionWithOptions(volumebundlesResource, c.ns, volumeBundle, opts), emptyResult) + + if obj == nil { + return emptyResult, err + } + return obj.(*v1alpha1.VolumeBundle), err +} + +// Update takes the representation of a volumeBundle and updates it. Returns the server's representation of the volumeBundle, and an error, if there is any. +func (c *FakeVolumeBundles) Update(ctx context.Context, volumeBundle *v1alpha1.VolumeBundle, opts v1.UpdateOptions) (result *v1alpha1.VolumeBundle, err error) { + emptyResult := &v1alpha1.VolumeBundle{} + obj, err := c.Fake. + Invokes(testing.NewUpdateActionWithOptions(volumebundlesResource, c.ns, volumeBundle, opts), emptyResult) + + if obj == nil { + return emptyResult, err + } + return obj.(*v1alpha1.VolumeBundle), err +} + +// Delete takes name of the volumeBundle and deletes it. Returns an error if one occurs. +func (c *FakeVolumeBundles) Delete(ctx context.Context, name string, opts v1.DeleteOptions) error { + _, err := c.Fake. + Invokes(testing.NewDeleteActionWithOptions(volumebundlesResource, c.ns, name, opts), &v1alpha1.VolumeBundle{}) + + return err +} + +// DeleteCollection deletes a collection of objects. +func (c *FakeVolumeBundles) DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error { + action := testing.NewDeleteCollectionActionWithOptions(volumebundlesResource, c.ns, opts, listOpts) + + _, err := c.Fake.Invokes(action, &v1alpha1.VolumeBundleList{}) + return err +} + +// Patch applies the patch and returns the patched volumeBundle. +func (c *FakeVolumeBundles) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *v1alpha1.VolumeBundle, err error) { + emptyResult := &v1alpha1.VolumeBundle{} + obj, err := c.Fake. + Invokes(testing.NewPatchSubresourceActionWithOptions(volumebundlesResource, c.ns, name, pt, data, opts, subresources...), emptyResult) + + if obj == nil { + return emptyResult, err + } + return obj.(*v1alpha1.VolumeBundle), err +} diff --git a/client-go/clientset/versioned/typed/apis/v1alpha1/generated_expansion.go b/client-go/clientset/versioned/typed/apis/v1alpha1/generated_expansion.go new file mode 100644 index 0000000..5b06194 --- /dev/null +++ b/client-go/clientset/versioned/typed/apis/v1alpha1/generated_expansion.go @@ -0,0 +1,28 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by client-gen. DO NOT EDIT. + +package v1alpha1 + +type ApplicationProfileExpansion interface{} + +type JobTemplateExpansion interface{} + +type RayClusterTemplateExpansion interface{} + +type RayJobTemplateExpansion interface{} + +type VolumeBundleExpansion interface{} diff --git a/client-go/clientset/versioned/typed/apis/v1alpha1/jobtemplate.go b/client-go/clientset/versioned/typed/apis/v1alpha1/jobtemplate.go new file mode 100644 index 0000000..3d94830 --- /dev/null +++ b/client-go/clientset/versioned/typed/apis/v1alpha1/jobtemplate.go @@ -0,0 +1,66 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by client-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + "context" + + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + types "k8s.io/apimachinery/pkg/types" + watch "k8s.io/apimachinery/pkg/watch" + gentype "k8s.io/client-go/gentype" + v1alpha1 "sigs.k8s.io/kueue/cmd/experimental/kjobctl/apis/v1alpha1" + scheme "sigs.k8s.io/kueue/cmd/experimental/kjobctl/client-go/clientset/versioned/scheme" +) + +// JobTemplatesGetter has a method to return a JobTemplateInterface. +// A group's client should implement this interface. +type JobTemplatesGetter interface { + JobTemplates(namespace string) JobTemplateInterface +} + +// JobTemplateInterface has methods to work with JobTemplate resources. +type JobTemplateInterface interface { + Create(ctx context.Context, jobTemplate *v1alpha1.JobTemplate, opts v1.CreateOptions) (*v1alpha1.JobTemplate, error) + Update(ctx context.Context, jobTemplate *v1alpha1.JobTemplate, opts v1.UpdateOptions) (*v1alpha1.JobTemplate, error) + Delete(ctx context.Context, name string, opts v1.DeleteOptions) error + DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error + Get(ctx context.Context, name string, opts v1.GetOptions) (*v1alpha1.JobTemplate, error) + List(ctx context.Context, opts v1.ListOptions) (*v1alpha1.JobTemplateList, error) + Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) + Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *v1alpha1.JobTemplate, err error) + JobTemplateExpansion +} + +// jobTemplates implements JobTemplateInterface +type jobTemplates struct { + *gentype.ClientWithList[*v1alpha1.JobTemplate, *v1alpha1.JobTemplateList] +} + +// newJobTemplates returns a JobTemplates +func newJobTemplates(c *KjobctlV1alpha1Client, namespace string) *jobTemplates { + return &jobTemplates{ + gentype.NewClientWithList[*v1alpha1.JobTemplate, *v1alpha1.JobTemplateList]( + "jobtemplates", + c.RESTClient(), + scheme.ParameterCodec, + namespace, + func() *v1alpha1.JobTemplate { return &v1alpha1.JobTemplate{} }, + func() *v1alpha1.JobTemplateList { return &v1alpha1.JobTemplateList{} }), + } +} diff --git a/client-go/clientset/versioned/typed/apis/v1alpha1/rayclustertemplate.go b/client-go/clientset/versioned/typed/apis/v1alpha1/rayclustertemplate.go new file mode 100644 index 0000000..02ca456 --- /dev/null +++ b/client-go/clientset/versioned/typed/apis/v1alpha1/rayclustertemplate.go @@ -0,0 +1,66 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by client-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + "context" + + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + types "k8s.io/apimachinery/pkg/types" + watch "k8s.io/apimachinery/pkg/watch" + gentype "k8s.io/client-go/gentype" + v1alpha1 "sigs.k8s.io/kueue/cmd/experimental/kjobctl/apis/v1alpha1" + scheme "sigs.k8s.io/kueue/cmd/experimental/kjobctl/client-go/clientset/versioned/scheme" +) + +// RayClusterTemplatesGetter has a method to return a RayClusterTemplateInterface. +// A group's client should implement this interface. +type RayClusterTemplatesGetter interface { + RayClusterTemplates(namespace string) RayClusterTemplateInterface +} + +// RayClusterTemplateInterface has methods to work with RayClusterTemplate resources. +type RayClusterTemplateInterface interface { + Create(ctx context.Context, rayClusterTemplate *v1alpha1.RayClusterTemplate, opts v1.CreateOptions) (*v1alpha1.RayClusterTemplate, error) + Update(ctx context.Context, rayClusterTemplate *v1alpha1.RayClusterTemplate, opts v1.UpdateOptions) (*v1alpha1.RayClusterTemplate, error) + Delete(ctx context.Context, name string, opts v1.DeleteOptions) error + DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error + Get(ctx context.Context, name string, opts v1.GetOptions) (*v1alpha1.RayClusterTemplate, error) + List(ctx context.Context, opts v1.ListOptions) (*v1alpha1.RayClusterTemplateList, error) + Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) + Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *v1alpha1.RayClusterTemplate, err error) + RayClusterTemplateExpansion +} + +// rayClusterTemplates implements RayClusterTemplateInterface +type rayClusterTemplates struct { + *gentype.ClientWithList[*v1alpha1.RayClusterTemplate, *v1alpha1.RayClusterTemplateList] +} + +// newRayClusterTemplates returns a RayClusterTemplates +func newRayClusterTemplates(c *KjobctlV1alpha1Client, namespace string) *rayClusterTemplates { + return &rayClusterTemplates{ + gentype.NewClientWithList[*v1alpha1.RayClusterTemplate, *v1alpha1.RayClusterTemplateList]( + "rayclustertemplates", + c.RESTClient(), + scheme.ParameterCodec, + namespace, + func() *v1alpha1.RayClusterTemplate { return &v1alpha1.RayClusterTemplate{} }, + func() *v1alpha1.RayClusterTemplateList { return &v1alpha1.RayClusterTemplateList{} }), + } +} diff --git a/client-go/clientset/versioned/typed/apis/v1alpha1/rayjobtemplate.go b/client-go/clientset/versioned/typed/apis/v1alpha1/rayjobtemplate.go new file mode 100644 index 0000000..3c61696 --- /dev/null +++ b/client-go/clientset/versioned/typed/apis/v1alpha1/rayjobtemplate.go @@ -0,0 +1,66 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by client-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + "context" + + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + types "k8s.io/apimachinery/pkg/types" + watch "k8s.io/apimachinery/pkg/watch" + gentype "k8s.io/client-go/gentype" + v1alpha1 "sigs.k8s.io/kueue/cmd/experimental/kjobctl/apis/v1alpha1" + scheme "sigs.k8s.io/kueue/cmd/experimental/kjobctl/client-go/clientset/versioned/scheme" +) + +// RayJobTemplatesGetter has a method to return a RayJobTemplateInterface. +// A group's client should implement this interface. +type RayJobTemplatesGetter interface { + RayJobTemplates(namespace string) RayJobTemplateInterface +} + +// RayJobTemplateInterface has methods to work with RayJobTemplate resources. +type RayJobTemplateInterface interface { + Create(ctx context.Context, rayJobTemplate *v1alpha1.RayJobTemplate, opts v1.CreateOptions) (*v1alpha1.RayJobTemplate, error) + Update(ctx context.Context, rayJobTemplate *v1alpha1.RayJobTemplate, opts v1.UpdateOptions) (*v1alpha1.RayJobTemplate, error) + Delete(ctx context.Context, name string, opts v1.DeleteOptions) error + DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error + Get(ctx context.Context, name string, opts v1.GetOptions) (*v1alpha1.RayJobTemplate, error) + List(ctx context.Context, opts v1.ListOptions) (*v1alpha1.RayJobTemplateList, error) + Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) + Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *v1alpha1.RayJobTemplate, err error) + RayJobTemplateExpansion +} + +// rayJobTemplates implements RayJobTemplateInterface +type rayJobTemplates struct { + *gentype.ClientWithList[*v1alpha1.RayJobTemplate, *v1alpha1.RayJobTemplateList] +} + +// newRayJobTemplates returns a RayJobTemplates +func newRayJobTemplates(c *KjobctlV1alpha1Client, namespace string) *rayJobTemplates { + return &rayJobTemplates{ + gentype.NewClientWithList[*v1alpha1.RayJobTemplate, *v1alpha1.RayJobTemplateList]( + "rayjobtemplates", + c.RESTClient(), + scheme.ParameterCodec, + namespace, + func() *v1alpha1.RayJobTemplate { return &v1alpha1.RayJobTemplate{} }, + func() *v1alpha1.RayJobTemplateList { return &v1alpha1.RayJobTemplateList{} }), + } +} diff --git a/client-go/clientset/versioned/typed/apis/v1alpha1/volumebundle.go b/client-go/clientset/versioned/typed/apis/v1alpha1/volumebundle.go new file mode 100644 index 0000000..afa0590 --- /dev/null +++ b/client-go/clientset/versioned/typed/apis/v1alpha1/volumebundle.go @@ -0,0 +1,66 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by client-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + "context" + + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + types "k8s.io/apimachinery/pkg/types" + watch "k8s.io/apimachinery/pkg/watch" + gentype "k8s.io/client-go/gentype" + v1alpha1 "sigs.k8s.io/kueue/cmd/experimental/kjobctl/apis/v1alpha1" + scheme "sigs.k8s.io/kueue/cmd/experimental/kjobctl/client-go/clientset/versioned/scheme" +) + +// VolumeBundlesGetter has a method to return a VolumeBundleInterface. +// A group's client should implement this interface. +type VolumeBundlesGetter interface { + VolumeBundles(namespace string) VolumeBundleInterface +} + +// VolumeBundleInterface has methods to work with VolumeBundle resources. +type VolumeBundleInterface interface { + Create(ctx context.Context, volumeBundle *v1alpha1.VolumeBundle, opts v1.CreateOptions) (*v1alpha1.VolumeBundle, error) + Update(ctx context.Context, volumeBundle *v1alpha1.VolumeBundle, opts v1.UpdateOptions) (*v1alpha1.VolumeBundle, error) + Delete(ctx context.Context, name string, opts v1.DeleteOptions) error + DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error + Get(ctx context.Context, name string, opts v1.GetOptions) (*v1alpha1.VolumeBundle, error) + List(ctx context.Context, opts v1.ListOptions) (*v1alpha1.VolumeBundleList, error) + Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) + Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *v1alpha1.VolumeBundle, err error) + VolumeBundleExpansion +} + +// volumeBundles implements VolumeBundleInterface +type volumeBundles struct { + *gentype.ClientWithList[*v1alpha1.VolumeBundle, *v1alpha1.VolumeBundleList] +} + +// newVolumeBundles returns a VolumeBundles +func newVolumeBundles(c *KjobctlV1alpha1Client, namespace string) *volumeBundles { + return &volumeBundles{ + gentype.NewClientWithList[*v1alpha1.VolumeBundle, *v1alpha1.VolumeBundleList]( + "volumebundles", + c.RESTClient(), + scheme.ParameterCodec, + namespace, + func() *v1alpha1.VolumeBundle { return &v1alpha1.VolumeBundle{} }, + func() *v1alpha1.VolumeBundleList { return &v1alpha1.VolumeBundleList{} }), + } +} diff --git a/cmd/kjobctl-docs/generators/doc.go b/cmd/kjobctl-docs/generators/doc.go new file mode 100644 index 0000000..e5007da --- /dev/null +++ b/cmd/kjobctl-docs/generators/doc.go @@ -0,0 +1,376 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package generators + +import ( + "fmt" + "html" + "html/template" + "io" + "os" + "os/user" + "path/filepath" + "sort" + "strings" + + "github.com/spf13/cobra" + "github.com/spf13/pflag" +) + +type Reference struct { + NoList bool + Title string + Description string + ShowUseLine bool + UseLine string + ShowExample bool + Example string + Options []Option + InheritedOptions []Option + Links []Link +} + +type Option struct { + Name string + Shorthand string + Usage string + ValueType string + DefVal string + NoOptDefVal string +} + +type Link struct { + Name string + Description string + Path string +} + +type byName []*cobra.Command + +func (s byName) Len() int { return len(s) } +func (s byName) Swap(i, j int) { s[i], s[j] = s[j], s[i] } +func (s byName) Less(i, j int) bool { return s[i].Name() < s[j].Name() } + +func GenMarkdownTree(cmd *cobra.Command, templatesDir, outputDir string) error { + identity := func(s string) string { return s } + emptyStr := func(s string) string { return "" } + return GenMarkdownTreeCustom(cmd, templatesDir, outputDir, "", emptyStr, identity) +} + +func GenMarkdownTreeCustom(cmd *cobra.Command, templatesDir, outputDir string, subdir string, filePrepender, linkHandler func(string) string) error { + for _, c := range cmd.Commands() { + if !c.IsAvailableCommand() || c.IsAdditionalHelpTopicCommand() { + continue + } + // CommandPath example: 'kjobctl create localqueue' + parts := strings.Split(c.CommandPath(), " ") + childSubdir := "" + if len(parts) > 1 { + childSubdir = parts[0] + "_" + parts[1] + fullPath := filepath.Join(outputDir, childSubdir) + if _, err := os.Stat(fullPath); os.IsNotExist(err) { + if err := os.Mkdir(fullPath, 0770); err != nil { + return err + } + } + } + if err := GenMarkdownTreeCustom(c, templatesDir, outputDir, childSubdir, filePrepender, linkHandler); err != nil { + return err + } + } + + fullname := strings.ReplaceAll(cmd.CommandPath(), " ", "_") + ".md" + indexFile := false + if len(subdir) > 0 { + parts := strings.Split(cmd.CommandPath(), " ") + if len(parts) == 2 { + indexFile = true + fullname = "_index.md" + } + fullname = filepath.Join(subdir, fullname) + } + filename := filepath.Join(outputDir, fullname) + + f, err := os.Create(filename) + if err != nil { + return err + } + defer f.Close() + + if _, err := io.WriteString(f, filePrepender(filename)); err != nil { + return err + } + if err := GenReference(cmd, templatesDir, f, linkHandler, indexFile); err != nil { + return err + } + return nil +} + +func GenReference(cmd *cobra.Command, templatesDir string, w io.Writer, linkHandler func(string) string, indexFile bool) error { + ref := getReference(cmd, linkHandler, indexFile) + + tmpl, err := template.New("main.md"). + Funcs(template.FuncMap{ + "html": htmlFuncMap, + "usage": usageFuncMap, + }). + ParseGlob(fmt.Sprintf("%s/*", templatesDir)) + if err != nil { + return err + } + if err := tmpl.Execute(w, ref); err != nil { + return err + } + + return nil +} + +func htmlFuncMap(s string) template.HTML { + return template.HTML(s) +} + +func usageFuncMap(usage string) template.HTML { + usage = html.EscapeString(usage) + usage = strings.TrimSuffix(usage, "\\n") + usage = strings.ReplaceAll(usage, "\\n", "
") + return htmlFuncMap(usage) +} + +func getReference(cmd *cobra.Command, linkHandler func(string) string, indexFile bool) *Reference { + cmd.InitDefaultHelpCmd() + cmd.InitDefaultHelpFlag() + + ref := &Reference{ + NoList: indexFile, + Title: strings.TrimSpace(cmd.CommandPath()), + Description: cmd.Long, + Example: cmd.Example, + Options: getOptions(cmd.NonInheritedFlags()), + InheritedOptions: getOptions(cmd.InheritedFlags()), + Links: getLinks(cmd, linkHandler, indexFile), + } + + if len(ref.Description) == 0 { + ref.Description = cmd.Short + } + + if cmd.Runnable() { + ref.UseLine = cmd.UseLine() + } + + return ref +} + +func getOptions(f *pflag.FlagSet) []Option { + var options []Option + + f.VisitAll(func(flag *pflag.Flag) { + if len(flag.Deprecated) > 0 || flag.Hidden { + return + } + + option := Option{ + Name: flag.Name, + Usage: flag.Usage, + ValueType: getValueType(flag), + DefVal: getDefaultVal(flag), + NoOptDefVal: getNoOptDefVal(flag), + } + + if len(flag.ShorthandDeprecated) == 0 { + option.Shorthand = flag.Shorthand + } + + options = append(options, option) + }) + + return options +} + +func getValueType(flag *pflag.Flag) string { + name := flag.Value.Type() + switch name { + case "bool": + return "" + case "float64", "float32": + return "float" + case "int64", "int32", "int8": + return "int" + case "uint64", "uint32", "uint8": + return "uint" + case "stringSlice", "stringArray": + return "strings" + case "float32Slice", "float64Slice": + return "floats" + case "intSlice", "int32Slice", "int64Slice": + return "ints" + case "uintSlice": + return "uints" + case "boolSlice": + return "bools" + case "durationSlice": + return "durations" + case "stringToString": + return "" + case "stringToInt": + return "" + default: + return name + } +} + +func getNoOptDefVal(flag *pflag.Flag) string { + if len(flag.NoOptDefVal) == 0 { + return "" + } + + switch flag.Value.Type() { + case "string": + return fmt.Sprintf("[=\"%s\"]", flag.NoOptDefVal) + case "bool": + if flag.NoOptDefVal != "true" { + return fmt.Sprintf("[=%s]", flag.NoOptDefVal) + } else { + return "" + } + default: + return fmt.Sprintf("[=%s]", flag.NoOptDefVal) + } +} + +func getDefaultVal(flag *pflag.Flag) string { + var defaultVal string + + if !defaultIsZeroValue(flag) { + defaultVal = flag.DefValue + switch flag.Value.Type() { + case "string": + // There are cases where the string is very-very long, split + // it to mutiple lines manually + if len(defaultVal) > 40 { + defaultVal = strings.ReplaceAll(defaultVal, ",", ",
") + } + // clean up kjobctl cache-dir flag value + if flag.Name == "cache-dir" { + myUser, err := user.Current() + if err == nil { + noprefix := strings.TrimPrefix(defaultVal, myUser.HomeDir) + defaultVal = fmt.Sprintf("$HOME%s", noprefix) + } + } + defaultVal = fmt.Sprintf("\"%s\"", defaultVal) + case "stringSlice": + // For string slices, the default value should not contain '[' ]r ']' + defaultVal = strings.TrimPrefix(defaultVal, "[") + defaultVal = strings.TrimSuffix(defaultVal, "]") + defaultVal = strings.ReplaceAll(defaultVal, " ", "") + defaultVal = fmt.Sprintf("\"%s\"", defaultVal) + } + } + + return defaultVal +} + +func getLinks(cmd *cobra.Command, linkHandler func(string) string, indexFile bool) []Link { + var links []Link + + if cmd.HasParent() { + parent := cmd.Parent() + + parts := strings.Split(parent.CommandPath(), " ") + commandPath := strings.Join(parts, "_") + + var path string + if indexFile { + path = "../" + commandPath + ".md" + } else { + path = "_index.md" + } + + link := Link{ + Name: commandPath, + Path: linkHandler(path), + Description: parent.Short, + } + + cmd.VisitParents(func(c *cobra.Command) { + if c.DisableAutoGenTag { + cmd.DisableAutoGenTag = c.DisableAutoGenTag + } + }) + + links = append(links, link) + } + + name := cmd.CommandPath() + children := cmd.Commands() + sort.Sort(byName(children)) + + for _, child := range children { + if !child.IsAvailableCommand() || child.IsAdditionalHelpTopicCommand() { + continue + } + cname := name + " " + child.Name() + path := strings.ReplaceAll(cname, " ", "_") + if indexFile { + path += ".md" + } else { + path += "/_index.md" + } + + link := Link{ + Name: cname, + Path: linkHandler(path), + Description: child.Short, + } + + links = append(links, link) + } + + return links +} + +func defaultIsZeroValue(f *pflag.Flag) bool { + switch f.Value.Type() { + case "bool": + return f.DefValue == "false" + case "duration": + return f.DefValue == "0" || f.DefValue == "0s" + case "int", "int8", "int32", "int64", "uint", "uint8", "uint16", "uint32", "count", "float32", "float64": + return f.DefValue == "0" + case "string": + return f.DefValue == "" + case "ip", "ipMask", "ipNet": + return f.DefValue == "" + case "intSlice", "stringSlice", "stringArray": + return f.DefValue == "[]" + case "namedCertKey": + return f.DefValue == "[]" + default: + switch f.Value.String() { + case "false": + return true + case "": + return true + case "": + return true + case "0": + return true + } + return false + } +} diff --git a/cmd/kjobctl-docs/main.go b/cmd/kjobctl-docs/main.go new file mode 100644 index 0000000..218c57a --- /dev/null +++ b/cmd/kjobctl-docs/main.go @@ -0,0 +1,51 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + goflag "flag" + "log" + "os" + + "github.com/spf13/pflag" + cliflag "k8s.io/component-base/cli/flag" + + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/cmd/kjobctl-docs/generators" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/cmd" +) + +func main() { + var ( + templatesDir string + outputDir string + ) + + if len(os.Args) == 3 { + templatesDir = os.Args[1] + outputDir = os.Args[2] + } else { + log.Fatalf("usage: %s ", os.Args[0]) + } + + cmd := cmd.NewDefaultKjobctlCmd() + pflag.CommandLine.SetNormalizeFunc(cliflag.WordSepNormalizeFunc) + pflag.CommandLine.AddGoFlagSet(goflag.CommandLine) + + if err := generators.GenMarkdownTree(cmd, templatesDir, outputDir); err != nil { + panic(err) + } +} diff --git a/cmd/kjobctl-docs/templates/main.md b/cmd/kjobctl-docs/templates/main.md new file mode 100644 index 0000000..f66042c --- /dev/null +++ b/cmd/kjobctl-docs/templates/main.md @@ -0,0 +1,50 @@ +{{ html ``}} + +# {{.Title}} + + +## Synopsis + + +{{.Description}} +{{- if .UseLine}} + +``` +{{.UseLine}} +``` +{{- end}} +{{- if .Example}} + + +## Examples + +``` +{{html .Example}} +``` +{{- end}} +{{- if .Options}} + + +## Options + +{{template "options" .Options}} +{{- end}} +{{- if .InheritedOptions}} + + +## Options inherited from parent commands + +{{- template "options" .InheritedOptions}} +{{- end}} +{{- if .Links}} + + +## See Also +{{range .Links}} +* [{{.Name}}]({{.Path}}){{"\t"}} - {{.Description}} +{{- end}} +{{- end}} + diff --git a/cmd/kjobctl-docs/templates/options.html b/cmd/kjobctl-docs/templates/options.html new file mode 100644 index 0000000..55280da --- /dev/null +++ b/cmd/kjobctl-docs/templates/options.html @@ -0,0 +1,24 @@ +{{define "options"}} + + + + + + + {{- range .}} + + + + + + + + {{- end}} + +
{{if .Shorthand}}-{{.Shorthand}}, {{end}}--{{.Name}} + {{- if .ValueType}} {{.ValueType}}{{end}} + {{- if .NoOptDefVal}} {{.NoOptDefVal}}{{end}} + {{- if .DefVal}}     Default: {{.DefVal}}{{end}}
+

{{usage .Usage}}

+
+{{end}} \ No newline at end of file diff --git a/cmd/kjobctl/main.go b/cmd/kjobctl/main.go new file mode 100644 index 0000000..6697caf --- /dev/null +++ b/cmd/kjobctl/main.go @@ -0,0 +1,29 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "os" + + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/cmd" +) + +func main() { + if err := cmd.NewDefaultKjobctlCmd().Execute(); err != nil { + os.Exit(1) + } +} diff --git a/config/crd/bases/kjobctl.x-k8s.io_applicationprofiles.yaml b/config/crd/bases/kjobctl.x-k8s.io_applicationprofiles.yaml new file mode 100644 index 0000000..474db35 --- /dev/null +++ b/config/crd/bases/kjobctl.x-k8s.io_applicationprofiles.yaml @@ -0,0 +1,210 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.16.5 + name: applicationprofiles.kjobctl.x-k8s.io +spec: + group: kjobctl.x-k8s.io + names: + kind: ApplicationProfile + listKind: ApplicationProfileList + plural: applicationprofiles + singular: applicationprofile + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: ApplicationProfile is the Schema for the applicationprofiles + API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: ApplicationProfileSpec defines the desired state of ApplicationProfile + properties: + supportedModes: + items: + properties: + name: + description: |- + name determines which template will be used and which object will eventually be created. + Possible values are Interactive, Job, RayJob, RayCluster and Slurm. + enum: + - Interactive + - Job + - RayJob + - RayCluster + - Slurm + type: string + requiredFlags: + description: |- + requiredFlags point which cli flags are required to be passed in order to fill the gaps in the templates. + Possible values are cmd, parallelism, completions, replicas, min-replicas, max-replicas, request, localqueue, and raycluster. + replicas, min-replicas, and max-replicas flags used only for RayJob and RayCluster mode. + The raycluster flag used only for the RayJob mode. + The request flag used only for Interactive and Job modes. + The cmd flag used only for Interactive, Job, and RayJob. + The time and priority flags can be used in all modes. + If the raycluster flag are set, none of localqueue, replicas, min-replicas, or max-replicas can be set. + For the Slurm mode, the possible values are: array, cpus-per-task, error, gpus-per-task, input, job-name, mem, mem-per-cpu, + mem-per-gpu, mem-per-task, nodes, ntasks, output, partition, localqueue. + + cmd and requests values are going to be added only to the first primary container. + items: + enum: + - cmd + - parallelism + - completions + - replicas + - min-replicas + - max-replicas + - request + - localqueue + - raycluster + - array + - cpus-per-task + - error + - gpus-per-task + - input + - job-name + - mem-per-cpu + - mem-per-gpu + - mem-per-task + - nodes + - ntasks + - output + - partition + type: string + maxItems: 14 + type: array + x-kubernetes-list-type: set + template: + description: |- + template is the name of the template. + Template type depends on ApplicationProfileMode: + - on Interactive mode it must be v1/PodTemplate + - on Job mode it must be kjobctl.x-k8s.io/v1alpha1/JobTemplate + - on RayJob mode it must be kjobctl.x-k8s.io/v1alpha1/RayJobTemplate + - on RayCluster mode it must be kjobctl.x-k8s.io/v1alpha1/RayClusterTemplate + - on Slurm mode it must be kjobctl.x-k8s.io/v1alpha1/JobTemplate + maxLength: 253 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + required: + - name + - template + type: object + x-kubernetes-validations: + - message: replicas flag can be used only on RayJob and RayCluster + modes + rule: '!has(self.requiredFlags) || !(''replicas'' in self.requiredFlags) + || self.name in [''RayJob'', ''RayCluster'']' + - message: min-replicas flag can be used only on RayJob and RayCluster + modes + rule: '!has(self.requiredFlags) || !(''min-replicas'' in self.requiredFlags) + || self.name in [''RayJob'', ''RayCluster'']' + - message: max-replicas flag can be used only on RayJob and RayCluster + modes + rule: '!has(self.requiredFlags) || !(''max-replicas'' in self.requiredFlags) + || self.name in [''RayJob'', ''RayCluster'']' + - message: request flag can be used only on Job and Interactive + modes + rule: '!has(self.requiredFlags) || !(''request'' in self.requiredFlags) + || self.name in [''Job'', ''Interactive'', ''RayJob'']' + - message: cmd flag can be used only on Job, Interactive and RayJob + modes + rule: '!has(self.requiredFlags) || !(''cmd'' in self.requiredFlags) + || self.name in [''Job'', ''Interactive'', ''RayJob'']' + - message: raycluster flag can be used only on RayJob mode + rule: '!has(self.requiredFlags) || !(''raycluster'' in self.requiredFlags) + || self.name == ''RayJob''' + - message: if raycluster flag are set none of localqueue, replicas, + min-replicas and max-replicas can be + rule: '!has(self.requiredFlags) || !(''raycluster'' in self.requiredFlags) + || !(''localqueue'' in self.requiredFlags || ''replicas'' in + self.requiredFlags || ''min-replicas'' in self.requiredFlags + || ''max-replicas'' in self.requiredFlags)' + - message: array flag can be used only on Slurm mode + rule: '!has(self.requiredFlags) || !(''array'' in self.requiredFlags) + || self.name == ''Slurm''' + - message: cpus-per-task flag can be used only on Slurm mode + rule: '!has(self.requiredFlags) || !(''cpus-per-task'' in self.requiredFlags) + || self.name == ''Slurm''' + - message: error flag can be used only on Slurm mode + rule: '!has(self.requiredFlags) || !(''error'' in self.requiredFlags) + || self.name == ''Slurm''' + - message: gpus-per-task flag can be used only on Slurm mode + rule: '!has(self.requiredFlags) || !(''gpus-per-task'' in self.requiredFlags) + || self.name == ''Slurm''' + - message: input flag can be used only on Slurm mode + rule: '!has(self.requiredFlags) || !(''input'' in self.requiredFlags) + || self.name == ''Slurm''' + - message: job-name flag can be used only on Slurm mode + rule: '!has(self.requiredFlags) || !(''job-name'' in self.requiredFlags) + || self.name == ''Slurm''' + - message: mem flag can be used only on Slurm mode + rule: '!has(self.requiredFlags) || !(''mem'' in self.requiredFlags) + || self.name == ''Slurm''' + - message: mem-per-cpu flag can be used only on Slurm mode + rule: '!has(self.requiredFlags) || !(''mem-per-cpu'' in self.requiredFlags) + || self.name == ''Slurm''' + - message: mem-per-gpu flag can be used only on Slurm mode + rule: '!has(self.requiredFlags) || !(''mem-per-gpu'' in self.requiredFlags) + || self.name == ''Slurm''' + - message: mem-per-task flag can be used only on Slurm mode + rule: '!has(self.requiredFlags) || !(''mem-per-task'' in self.requiredFlags) + || self.name == ''Slurm''' + - message: nodes flag can be used only on Slurm mode + rule: '!has(self.requiredFlags) || !(''nodes'' in self.requiredFlags) + || self.name == ''Slurm''' + - message: ntasks flag can be used only on Slurm mode + rule: '!has(self.requiredFlags) || !(''ntasks'' in self.requiredFlags) + || self.name == ''Slurm''' + - message: output flag can be used only on Slurm mode + rule: '!has(self.requiredFlags) || !(''output'' in self.requiredFlags) + || self.name == ''Slurm''' + - message: partition flag can be used only on Slurm mode + rule: '!has(self.requiredFlags) || !(''partition'' in self.requiredFlags) + || self.name == ''Slurm''' + - message: parallelism flag can't be used on Slurm mode + rule: '!has(self.requiredFlags) || self.name != ''Slurm'' || !(''parallelism'' + in self.requiredFlags)' + - message: completions flag can't be used on Slurm mode + rule: '!has(self.requiredFlags) || self.name != ''Slurm'' || !(''completions'' + in self.requiredFlags)' + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + volumeBundles: + items: + description: VolumeBundleReference is the name of the VolumeBundle. + maxLength: 253 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + type: array + x-kubernetes-list-type: set + required: + - supportedModes + type: object + type: object + served: true + storage: true diff --git a/config/crd/bases/kjobctl.x-k8s.io_jobtemplates.yaml b/config/crd/bases/kjobctl.x-k8s.io_jobtemplates.yaml new file mode 100644 index 0000000..e171d88 --- /dev/null +++ b/config/crd/bases/kjobctl.x-k8s.io_jobtemplates.yaml @@ -0,0 +1,8406 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.16.5 + name: jobtemplates.kjobctl.x-k8s.io +spec: + group: kjobctl.x-k8s.io + names: + kind: JobTemplate + listKind: JobTemplateList + plural: jobtemplates + singular: jobtemplate + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: JobTemplate is the Schema for the jobtemplate API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + template: + description: Template defines the jobs that will be created from this + pod template. + properties: + metadata: + description: Standard object's metadata. + type: object + spec: + description: Specification of the desired behavior of the job. + properties: + activeDeadlineSeconds: + description: |- + Specifies the duration in seconds relative to the startTime that the job + may be continuously active before the system tries to terminate it; value + must be positive integer. If a Job is suspended (at creation or through an + update), this timer will effectively be stopped and reset when the Job is + resumed again. + format: int64 + type: integer + backoffLimit: + description: |- + Specifies the number of retries before marking this job failed. + Defaults to 6 + format: int32 + type: integer + backoffLimitPerIndex: + description: |- + Specifies the limit for the number of retries within an + index before marking this index as failed. When enabled the number of + failures per index is kept in the pod's + batch.kubernetes.io/job-index-failure-count annotation. It can only + be set when Job's completionMode=Indexed, and the Pod's restart + policy is Never. The field is immutable. + This field is beta-level. It can be used when the `JobBackoffLimitPerIndex` + feature gate is enabled (enabled by default). + format: int32 + type: integer + completionMode: + description: |- + completionMode specifies how Pod completions are tracked. It can be + `NonIndexed` (default) or `Indexed`. + + `NonIndexed` means that the Job is considered complete when there have + been .spec.completions successfully completed Pods. Each Pod completion is + homologous to each other. + + `Indexed` means that the Pods of a + Job get an associated completion index from 0 to (.spec.completions - 1), + available in the annotation batch.kubernetes.io/job-completion-index. + The Job is considered complete when there is one successfully completed Pod + for each index. + When value is `Indexed`, .spec.completions must be specified and + `.spec.parallelism` must be less than or equal to 10^5. + In addition, The Pod name takes the form + `$(job-name)-$(index)-$(random-string)`, + the Pod hostname takes the form `$(job-name)-$(index)`. + + More completion modes can be added in the future. + If the Job controller observes a mode that it doesn't recognize, which + is possible during upgrades due to version skew, the controller + skips updates for the Job. + type: string + completions: + description: |- + Specifies the desired number of successfully finished pods the + job should be run with. Setting to null means that the success of any + pod signals the success of all pods, and allows parallelism to have any positive + value. Setting to 1 means that parallelism is limited to 1 and the success of that + pod signals the success of the job. + More info: https://kubernetes.io/docs/concepts/workloads/controllers/jobs-run-to-completion/ + format: int32 + type: integer + managedBy: + description: |- + ManagedBy field indicates the controller that manages a Job. The k8s Job + controller reconciles jobs which don't have this field at all or the field + value is the reserved string `kubernetes.io/job-controller`, but skips + reconciling Jobs with a custom value for this field. + The value must be a valid domain-prefixed path (e.g. acme.io/foo) - + all characters before the first "/" must be a valid subdomain as defined + by RFC 1123. All characters trailing the first "/" must be valid HTTP Path + characters as defined by RFC 3986. The value cannot exceed 63 characters. + This field is immutable. + + This field is alpha-level. The job controller accepts setting the field + when the feature gate JobManagedBy is enabled (disabled by default). + type: string + manualSelector: + description: |- + manualSelector controls generation of pod labels and pod selectors. + Leave `manualSelector` unset unless you are certain what you are doing. + When false or unset, the system pick labels unique to this job + and appends those labels to the pod template. When true, + the user is responsible for picking unique labels and specifying + the selector. Failure to pick a unique label may cause this + and other jobs to not function correctly. However, You may see + `manualSelector=true` in jobs that were created with the old `extensions/v1beta1` + API. + More info: https://kubernetes.io/docs/concepts/workloads/controllers/jobs-run-to-completion/#specifying-your-own-pod-selector + type: boolean + maxFailedIndexes: + description: |- + Specifies the maximal number of failed indexes before marking the Job as + failed, when backoffLimitPerIndex is set. Once the number of failed + indexes exceeds this number the entire Job is marked as Failed and its + execution is terminated. When left as null the job continues execution of + all of its indexes and is marked with the `Complete` Job condition. + It can only be specified when backoffLimitPerIndex is set. + It can be null or up to completions. It is required and must be + less than or equal to 10^4 when is completions greater than 10^5. + This field is beta-level. It can be used when the `JobBackoffLimitPerIndex` + feature gate is enabled (enabled by default). + format: int32 + type: integer + parallelism: + description: |- + Specifies the maximum desired number of pods the job should + run at any given time. The actual number of pods running in steady state will + be less than this number when ((.spec.completions - .status.successful) < .spec.parallelism), + i.e. when the work left to do is less than max parallelism. + More info: https://kubernetes.io/docs/concepts/workloads/controllers/jobs-run-to-completion/ + format: int32 + type: integer + podFailurePolicy: + description: |- + Specifies the policy of handling failed pods. In particular, it allows to + specify the set of actions and conditions which need to be + satisfied to take the associated action. + If empty, the default behaviour applies - the counter of failed pods, + represented by the jobs's .status.failed field, is incremented and it is + checked against the backoffLimit. This field cannot be used in combination + with restartPolicy=OnFailure. + properties: + rules: + description: |- + A list of pod failure policy rules. The rules are evaluated in order. + Once a rule matches a Pod failure, the remaining of the rules are ignored. + When no rule matches the Pod failure, the default handling applies - the + counter of pod failures is incremented and it is checked against + the backoffLimit. At most 20 elements are allowed. + items: + description: |- + PodFailurePolicyRule describes how a pod failure is handled when the requirements are met. + One of onExitCodes and onPodConditions, but not both, can be used in each rule. + properties: + action: + description: |- + Specifies the action taken on a pod failure when the requirements are satisfied. + Possible values are: + + - FailJob: indicates that the pod's job is marked as Failed and all + running pods are terminated. + - FailIndex: indicates that the pod's index is marked as Failed and will + not be restarted. + This value is beta-level. It can be used when the + `JobBackoffLimitPerIndex` feature gate is enabled (enabled by default). + - Ignore: indicates that the counter towards the .backoffLimit is not + incremented and a replacement pod is created. + - Count: indicates that the pod is handled in the default way - the + counter towards the .backoffLimit is incremented. + Additional values are considered to be added in the future. Clients should + react to an unknown action by skipping the rule. + type: string + onExitCodes: + description: Represents the requirement on the container + exit codes. + properties: + containerName: + description: |- + Restricts the check for exit codes to the container with the + specified name. When null, the rule applies to all containers. + When specified, it should match one the container or initContainer + names in the pod template. + type: string + operator: + description: |- + Represents the relationship between the container exit code(s) and the + specified values. Containers completed with success (exit code 0) are + excluded from the requirement check. Possible values are: + + - In: the requirement is satisfied if at least one container exit code + (might be multiple if there are multiple containers not restricted + by the 'containerName' field) is in the set of specified values. + - NotIn: the requirement is satisfied if at least one container exit code + (might be multiple if there are multiple containers not restricted + by the 'containerName' field) is not in the set of specified values. + Additional values are considered to be added in the future. Clients should + react to an unknown operator by assuming the requirement is not satisfied. + type: string + values: + description: |- + Specifies the set of values. Each returned container exit code (might be + multiple in case of multiple containers) is checked against this set of + values with respect to the operator. The list of values must be ordered + and must not contain duplicates. Value '0' cannot be used for the In operator. + At least one element is required. At most 255 elements are allowed. + items: + format: int32 + type: integer + type: array + x-kubernetes-list-type: set + required: + - operator + - values + type: object + onPodConditions: + description: |- + Represents the requirement on the pod conditions. The requirement is represented + as a list of pod condition patterns. The requirement is satisfied if at + least one pattern matches an actual pod condition. At most 20 elements are allowed. + items: + description: |- + PodFailurePolicyOnPodConditionsPattern describes a pattern for matching + an actual pod condition type. + properties: + status: + description: |- + Specifies the required Pod condition status. To match a pod condition + it is required that the specified status equals the pod condition status. + Defaults to True. + type: string + type: + description: |- + Specifies the required Pod condition type. To match a pod condition + it is required that specified type equals the pod condition type. + type: string + required: + - status + - type + type: object + type: array + x-kubernetes-list-type: atomic + required: + - action + type: object + type: array + x-kubernetes-list-type: atomic + required: + - rules + type: object + podReplacementPolicy: + description: |- + podReplacementPolicy specifies when to create replacement Pods. + Possible values are: + - TerminatingOrFailed means that we recreate pods + when they are terminating (has a metadata.deletionTimestamp) or failed. + - Failed means to wait until a previously created Pod is fully terminated (has phase + Failed or Succeeded) before creating a replacement Pod. + + When using podFailurePolicy, Failed is the the only allowed value. + TerminatingOrFailed and Failed are allowed values when podFailurePolicy is not in use. + This is an beta field. To use this, enable the JobPodReplacementPolicy feature toggle. + This is on by default. + type: string + selector: + description: |- + A label query over pods that should match the pod count. + Normally, the system sets this field for you. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#label-selectors + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + successPolicy: + description: |- + successPolicy specifies the policy when the Job can be declared as succeeded. + If empty, the default behavior applies - the Job is declared as succeeded + only when the number of succeeded pods equals to the completions. + When the field is specified, it must be immutable and works only for the Indexed Jobs. + Once the Job meets the SuccessPolicy, the lingering pods are terminated. + + This field is beta-level. To use this field, you must enable the + `JobSuccessPolicy` feature gate (enabled by default). + properties: + rules: + description: |- + rules represents the list of alternative rules for the declaring the Jobs + as successful before `.status.succeeded >= .spec.completions`. Once any of the rules are met, + the "SucceededCriteriaMet" condition is added, and the lingering pods are removed. + The terminal state for such a Job has the "Complete" condition. + Additionally, these rules are evaluated in order; Once the Job meets one of the rules, + other rules are ignored. At most 20 elements are allowed. + items: + description: |- + SuccessPolicyRule describes rule for declaring a Job as succeeded. + Each rule must have at least one of the "succeededIndexes" or "succeededCount" specified. + properties: + succeededCount: + description: |- + succeededCount specifies the minimal required size of the actual set of the succeeded indexes + for the Job. When succeededCount is used along with succeededIndexes, the check is + constrained only to the set of indexes specified by succeededIndexes. + For example, given that succeededIndexes is "1-4", succeededCount is "3", + and completed indexes are "1", "3", and "5", the Job isn't declared as succeeded + because only "1" and "3" indexes are considered in that rules. + When this field is null, this doesn't default to any value and + is never evaluated at any time. + When specified it needs to be a positive integer. + format: int32 + type: integer + succeededIndexes: + description: |- + succeededIndexes specifies the set of indexes + which need to be contained in the actual set of the succeeded indexes for the Job. + The list of indexes must be within 0 to ".spec.completions-1" and + must not contain duplicates. At least one element is required. + The indexes are represented as intervals separated by commas. + The intervals can be a decimal integer or a pair of decimal integers separated by a hyphen. + The number are listed in represented by the first and last element of the series, + separated by a hyphen. + For example, if the completed indexes are 1, 3, 4, 5 and 7, they are + represented as "1,3-5,7". + When this field is null, this field doesn't default to any value + and is never evaluated at any time. + type: string + type: object + type: array + x-kubernetes-list-type: atomic + required: + - rules + type: object + suspend: + description: |- + suspend specifies whether the Job controller should create Pods or not. If + a Job is created with suspend set to true, no Pods are created by the Job + controller. If a Job is suspended after creation (i.e. the flag goes from + false to true), the Job controller will delete all active Pods associated + with this Job. Users must design their workload to gracefully handle this. + Suspending a Job will reset the StartTime field of the Job, effectively + resetting the ActiveDeadlineSeconds timer too. Defaults to false. + type: boolean + template: + description: |- + Describes the pod that will be created when executing a job. + The only allowed template.spec.restartPolicy values are "Never" or "OnFailure". + More info: https://kubernetes.io/docs/concepts/workloads/controllers/jobs-run-to-completion/ + properties: + metadata: + description: |- + Standard object's metadata. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata + type: object + spec: + description: |- + Specification of the desired behavior of the pod. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status + properties: + activeDeadlineSeconds: + description: |- + Optional duration in seconds the pod may be active on the node relative to + StartTime before the system will actively try to mark it failed and kill associated containers. + Value must be a positive integer. + format: int64 + type: integer + affinity: + description: If specified, the pod's scheduling constraints + properties: + nodeAffinity: + description: Describes node affinity scheduling rules + for the pod. + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node matches the corresponding matchExpressions; the + node(s) with the highest sum are the most preferred. + items: + description: |- + An empty preferred scheduling term matches all objects with implicit weight 0 + (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op). + properties: + preference: + description: A node selector term, associated + with the corresponding weight. + properties: + matchExpressions: + description: A list of node selector + requirements by node's labels. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that + the selector applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + description: A list of node selector + requirements by node's fields. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that + the selector applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + weight: + description: Weight associated with matching + the corresponding nodeSelectorTerm, in + the range 1-100. + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to an update), the system + may or may not try to eventually evict the pod from its node. + properties: + nodeSelectorTerms: + description: Required. A list of node selector + terms. The terms are ORed. + items: + description: |- + A null or empty node selector term matches no objects. The requirements of + them are ANDed. + The TopologySelectorTerm type implements a subset of the NodeSelectorTerm. + properties: + matchExpressions: + description: A list of node selector + requirements by node's labels. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that + the selector applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + description: A list of node selector + requirements by node's fields. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that + the selector applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + type: array + x-kubernetes-list-type: atomic + required: + - nodeSelectorTerms + type: object + x-kubernetes-map-type: atomic + type: object + podAffinity: + description: Describes pod affinity scheduling rules + (e.g. co-locate this pod in the same node, zone, + etc. as some other pod(s)). + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the + node(s) with the highest sum are the most preferred. + items: + description: The weights of all of the matched + WeightedPodAffinityTerm fields are added per-node + to find the most preferred node(s) + properties: + podAffinityTerm: + description: Required. A pod affinity term, + associated with the corresponding weight. + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions is + a list of label selector requirements. + The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is + a list of label selector requirements. + The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + weight: + description: |- + weight associated with matching the corresponding podAffinityTerm, + in the range 1-100. + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to a pod label update), the + system may or may not try to eventually evict the pod from its node. + When there are multiple elements, the lists of nodes corresponding to each + podAffinityTerm are intersected, i.e. all terms must be satisfied. + items: + description: |- + Defines a set of pods (namely those matching the labelSelector + relative to the given namespace(s)) that this pod should be + co-located (affinity) or not co-located (anti-affinity) with, + where co-located is defined as running on a node whose value of + the label with key matches that of any node on which + a pod of the set of pods is running + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The + requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The + requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + podAntiAffinity: + description: Describes pod anti-affinity scheduling + rules (e.g. avoid putting this pod in the same node, + zone, etc. as some other pod(s)). + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the anti-affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling anti-affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the + node(s) with the highest sum are the most preferred. + items: + description: The weights of all of the matched + WeightedPodAffinityTerm fields are added per-node + to find the most preferred node(s) + properties: + podAffinityTerm: + description: Required. A pod affinity term, + associated with the corresponding weight. + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions is + a list of label selector requirements. + The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is + a list of label selector requirements. + The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + weight: + description: |- + weight associated with matching the corresponding podAffinityTerm, + in the range 1-100. + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the anti-affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the anti-affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to a pod label update), the + system may or may not try to eventually evict the pod from its node. + When there are multiple elements, the lists of nodes corresponding to each + podAffinityTerm are intersected, i.e. all terms must be satisfied. + items: + description: |- + Defines a set of pods (namely those matching the labelSelector + relative to the given namespace(s)) that this pod should be + co-located (affinity) or not co-located (anti-affinity) with, + where co-located is defined as running on a node whose value of + the label with key matches that of any node on which + a pod of the set of pods is running + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The + requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The + requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + type: object + automountServiceAccountToken: + description: AutomountServiceAccountToken indicates whether + a service account token should be automatically mounted. + type: boolean + containers: + description: |- + List of containers belonging to the pod. + Containers cannot currently be added or removed. + There must be at least one container in a Pod. + Cannot be updated. + items: + description: A single application container that you + want to run within a pod. + properties: + args: + description: |- + Arguments to the entrypoint. + The container image's CMD is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell + items: + type: string + type: array + x-kubernetes-list-type: atomic + command: + description: |- + Entrypoint array. Not executed within a shell. + The container image's ENTRYPOINT is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell + items: + type: string + type: array + x-kubernetes-list-type: atomic + env: + description: |- + List of environment variables to set in the container. + Cannot be updated. + items: + description: EnvVar represents an environment + variable present in a Container. + properties: + name: + description: Name of the environment variable. + Must be a C_IDENTIFIER. + type: string + value: + description: |- + Variable references $(VAR_NAME) are expanded + using the previously defined environment variables in the container and + any service environment variables. If a variable cannot be resolved, + the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. + "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless of whether the variable + exists or not. + Defaults to "". + type: string + valueFrom: + description: Source for the environment variable's + value. Cannot be used if value is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the ConfigMap + or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + description: |- + Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, + spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. + properties: + apiVersion: + description: Version of the schema + the FieldPath is written in terms + of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to + select in the specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. + properties: + containerName: + description: 'Container name: required + for volumes, optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output + format of the exposed resources, + defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to + select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + description: Selects a key of a secret + in the pod's namespace + properties: + key: + description: The key of the secret + to select from. Must be a valid + secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret + or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + envFrom: + description: |- + List of sources to populate environment variables in the container. + The keys defined within a source must be a C_IDENTIFIER. All invalid keys + will be reported as an event when the container is starting. When a key exists in multiple + sources, the value associated with the last source will take precedence. + Values defined by an Env with a duplicate key will take precedence. + Cannot be updated. + items: + description: EnvFromSource represents the source + of a set of ConfigMaps + properties: + configMapRef: + description: The ConfigMap to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the ConfigMap + must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + description: An optional identifier to prepend + to each key in the ConfigMap. Must be a + C_IDENTIFIER. + type: string + secretRef: + description: The Secret to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret + must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array + x-kubernetes-list-type: atomic + image: + description: |- + Container image name. + More info: https://kubernetes.io/docs/concepts/containers/images + This field is optional to allow higher level config management to default or override + container images in workload controllers like Deployments and StatefulSets. + type: string + imagePullPolicy: + description: |- + Image pull policy. + One of Always, Never, IfNotPresent. + Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/containers/images#updating-images + type: string + lifecycle: + description: |- + Actions that the management system should take in response to container lifecycle events. + Cannot be updated. + properties: + postStart: + description: |- + PostStart is called immediately after a container is created. If the handler fails, + the container is terminated and restarted according to its restart policy. + Other management of the container blocks until the hook completes. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies the action to + take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies the http + request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in + the request. HTTP allows repeated + headers. + items: + description: HTTPHeader describes + a custom header to be used in HTTP + probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field + value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP + server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents the duration + that the container should sleep before + being terminated. + properties: + seconds: + description: Seconds is the number of + seconds to sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for the backward compatibility. There are no validation of this field and + lifecycle hooks will fail in runtime when tcp handler is specified. + properties: + host: + description: 'Optional: Host name to + connect to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + preStop: + description: |- + PreStop is called immediately before a container is terminated due to an + API request or management event such as liveness/startup probe failure, + preemption, resource contention, etc. The handler is not called if the + container crashes or exits. The Pod's termination grace period countdown begins before the + PreStop hook is executed. Regardless of the outcome of the handler, the + container will eventually terminate within the Pod's termination grace + period (unless delayed by finalizers). Other management of the container blocks until the hook completes + or until the termination grace period is reached. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies the action to + take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies the http + request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in + the request. HTTP allows repeated + headers. + items: + description: HTTPHeader describes + a custom header to be used in HTTP + probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field + value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP + server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents the duration + that the container should sleep before + being terminated. + properties: + seconds: + description: Seconds is the number of + seconds to sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for the backward compatibility. There are no validation of this field and + lifecycle hooks will fail in runtime when tcp handler is specified. + properties: + host: + description: 'Optional: Host name to + connect to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + type: object + livenessProbe: + description: |- + Periodic probe of container liveness. + Container will be restarted if the probe fails. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies the action to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving + a GRPC port. + properties: + port: + description: Port number of the gRPC service. + Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http request + to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the + request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom + header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP + server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action involving + a TCP port. + properties: + host: + description: 'Optional: Host name to connect + to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + name: + description: |- + Name of the container specified as a DNS_LABEL. + Each container in a pod must have a unique name (DNS_LABEL). + Cannot be updated. + type: string + ports: + description: |- + List of ports to expose from the container. Not specifying a port here + DOES NOT prevent that port from being exposed. Any port which is + listening on the default "0.0.0.0" address inside a container will be + accessible from the network. + Modifying this array with strategic merge patch may corrupt the data. + For more information See https://github.com/kubernetes/kubernetes/issues/108255. + Cannot be updated. + items: + description: ContainerPort represents a network + port in a single container. + properties: + containerPort: + description: |- + Number of port to expose on the pod's IP address. + This must be a valid port number, 0 < x < 65536. + format: int32 + type: integer + hostIP: + description: What host IP to bind the external + port to. + type: string + hostPort: + description: |- + Number of port to expose on the host. + If specified, this must be a valid port number, 0 < x < 65536. + If HostNetwork is specified, this must match ContainerPort. + Most containers do not need this. + format: int32 + type: integer + name: + description: |- + If specified, this must be an IANA_SVC_NAME and unique within the pod. Each + named port in a pod must have a unique name. Name for the port that can be + referred to by services. + type: string + protocol: + default: TCP + description: |- + Protocol for port. Must be UDP, TCP, or SCTP. + Defaults to "TCP". + type: string + required: + - containerPort + type: object + type: array + x-kubernetes-list-map-keys: + - containerPort + - protocol + x-kubernetes-list-type: map + readinessProbe: + description: |- + Periodic probe of container service readiness. + Container will be removed from service endpoints if the probe fails. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies the action to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving + a GRPC port. + properties: + port: + description: Port number of the gRPC service. + Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http request + to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the + request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom + header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP + server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action involving + a TCP port. + properties: + host: + description: 'Optional: Host name to connect + to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + resizePolicy: + description: Resources resize policy for the container. + items: + description: ContainerResizePolicy represents + resource resize policy for the container. + properties: + resourceName: + description: |- + Name of the resource to which this resource resize policy applies. + Supported values: cpu, memory. + type: string + restartPolicy: + description: |- + Restart policy to apply when specified resource is resized. + If not specified, it defaults to NotRequired. + type: string + required: + - resourceName + - restartPolicy + type: object + type: array + x-kubernetes-list-type: atomic + resources: + description: |- + Compute Resources required by this container. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one + entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + restartPolicy: + description: |- + RestartPolicy defines the restart behavior of individual containers in a pod. + This field may only be set for init containers, and the only allowed value is "Always". + For non-init containers or when this field is not specified, + the restart behavior is defined by the Pod's restart policy and the container type. + Setting the RestartPolicy as "Always" for the init container will have the following effect: + this init container will be continually restarted on + exit until all regular containers have terminated. Once all regular + containers have completed, all init containers with restartPolicy "Always" + will be shut down. This lifecycle differs from normal init containers and + is often referred to as a "sidecar" container. Although this init + container still starts in the init container sequence, it does not wait + for the container to complete before proceeding to the next init + container. Instead, the next init container starts immediately after this + init container is started, or after any startupProbe has successfully + completed. + type: string + securityContext: + description: |- + SecurityContext defines the security options the container should be run with. + If set, the fields of SecurityContext override the equivalent fields of PodSecurityContext. + More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ + properties: + allowPrivilegeEscalation: + description: |- + AllowPrivilegeEscalation controls whether a process can gain more + privileges than its parent process. This bool directly controls if + the no_new_privs flag will be set on the container process. + AllowPrivilegeEscalation is true always when the container is: + 1) run as Privileged + 2) has CAP_SYS_ADMIN + Note that this field cannot be set when spec.os.name is windows. + type: boolean + appArmorProfile: + description: |- + appArmorProfile is the AppArmor options to use by this container. If set, this profile + overrides the pod's appArmorProfile. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile loaded on the node that should be used. + The profile must be preconfigured on the node to work. + Must match the loaded name of the profile. + Must be set if and only if type is "Localhost". + type: string + type: + description: |- + type indicates which kind of AppArmor profile will be applied. + Valid options are: + Localhost - a profile pre-loaded on the node. + RuntimeDefault - the container runtime's default profile. + Unconfined - no AppArmor enforcement. + type: string + required: + - type + type: object + capabilities: + description: |- + The capabilities to add/drop when running containers. + Defaults to the default set of capabilities granted by the container runtime. + Note that this field cannot be set when spec.os.name is windows. + properties: + add: + description: Added capabilities + items: + description: Capability represent POSIX + capabilities type + type: string + type: array + x-kubernetes-list-type: atomic + drop: + description: Removed capabilities + items: + description: Capability represent POSIX + capabilities type + type: string + type: array + x-kubernetes-list-type: atomic + type: object + privileged: + description: |- + Run container in privileged mode. + Processes in privileged containers are essentially equivalent to root on the host. + Defaults to false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + procMount: + description: |- + procMount denotes the type of proc mount to use for the containers. + The default value is Default which uses the container runtime defaults for + readonly paths and masked paths. + This requires the ProcMountType feature flag to be enabled. + Note that this field cannot be set when spec.os.name is windows. + type: string + readOnlyRootFilesystem: + description: |- + Whether this container has a read-only root filesystem. + Default is false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + runAsGroup: + description: |- + The GID to run the entrypoint of the container process. + Uses runtime default if unset. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + runAsNonRoot: + description: |- + Indicates that the container must run as a non-root user. + If true, the Kubelet will validate the image at runtime to ensure that it + does not run as UID 0 (root) and fail to start the container if it does. + If unset or false, no such validation will be performed. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: |- + The UID to run the entrypoint of the container process. + Defaults to user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + seLinuxOptions: + description: |- + The SELinux context to be applied to the container. + If unspecified, the container runtime will allocate a random SELinux context for each + container. May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + properties: + level: + description: Level is SELinux level label + that applies to the container. + type: string + role: + description: Role is a SELinux role label + that applies to the container. + type: string + type: + description: Type is a SELinux type label + that applies to the container. + type: string + user: + description: User is a SELinux user label + that applies to the container. + type: string + type: object + seccompProfile: + description: |- + The seccomp options to use by this container. If seccomp options are + provided at both the pod & container level, the container options + override the pod options. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile defined in a file on the node should be used. + The profile must be preconfigured on the node to work. + Must be a descending path, relative to the kubelet's configured seccomp profile location. + Must be set if type is "Localhost". Must NOT be set for any other type. + type: string + type: + description: |- + type indicates which kind of seccomp profile will be applied. + Valid options are: + + Localhost - a profile defined in a file on the node should be used. + RuntimeDefault - the container runtime default profile should be used. + Unconfined - no profile should be applied. + type: string + required: + - type + type: object + windowsOptions: + description: |- + The Windows specific settings applied to all containers. + If unspecified, the options from the PodSecurityContext will be used. + If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is linux. + properties: + gmsaCredentialSpec: + description: |- + GMSACredentialSpec is where the GMSA admission webhook + (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the + GMSA credential spec named by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName is the + name of the GMSA credential spec to use. + type: string + hostProcess: + description: |- + HostProcess determines if a container should be run as a 'Host Process' container. + All of a Pod's containers must have the same effective HostProcess value + (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). + In addition, if HostProcess is true then HostNetwork must also be set to true. + type: boolean + runAsUserName: + description: |- + The UserName in Windows to run the entrypoint of the container process. + Defaults to the user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: string + type: object + type: object + startupProbe: + description: |- + StartupProbe indicates that the Pod has successfully initialized. + If specified, no other probes are executed until this completes successfully. + If this probe fails, the Pod will be restarted, just as if the livenessProbe failed. + This can be used to provide different probe parameters at the beginning of a Pod's lifecycle, + when it might take a long time to load data or warm a cache, than during steady-state operation. + This cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies the action to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving + a GRPC port. + properties: + port: + description: Port number of the gRPC service. + Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http request + to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the + request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom + header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP + server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action involving + a TCP port. + properties: + host: + description: 'Optional: Host name to connect + to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + stdin: + description: |- + Whether this container should allocate a buffer for stdin in the container runtime. If this + is not set, reads from stdin in the container will always result in EOF. + Default is false. + type: boolean + stdinOnce: + description: |- + Whether the container runtime should close the stdin channel after it has been opened by + a single attach. When stdin is true the stdin stream will remain open across multiple attach + sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the + first client attaches to stdin, and then remains open and accepts data until the client disconnects, + at which time stdin is closed and remains closed until the container is restarted. If this + flag is false, a container processes that reads from stdin will never receive an EOF. + Default is false + type: boolean + terminationMessagePath: + description: |- + Optional: Path at which the file to which the container's termination message + will be written is mounted into the container's filesystem. + Message written is intended to be brief final status, such as an assertion failure message. + Will be truncated by the node if greater than 4096 bytes. The total message length across + all containers will be limited to 12kb. + Defaults to /dev/termination-log. + Cannot be updated. + type: string + terminationMessagePolicy: + description: |- + Indicate how the termination message should be populated. File will use the contents of + terminationMessagePath to populate the container status message on both success and failure. + FallbackToLogsOnError will use the last chunk of container log output if the termination + message file is empty and the container exited with an error. + The log output is limited to 2048 bytes or 80 lines, whichever is smaller. + Defaults to File. + Cannot be updated. + type: string + tty: + description: |- + Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. + Default is false. + type: boolean + volumeDevices: + description: volumeDevices is the list of block + devices to be used by the container. + items: + description: volumeDevice describes a mapping + of a raw block device within a container. + properties: + devicePath: + description: devicePath is the path inside + of the container that the device will be + mapped to. + type: string + name: + description: name must match the name of a + persistentVolumeClaim in the pod + type: string + required: + - devicePath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - devicePath + x-kubernetes-list-type: map + volumeMounts: + description: |- + Pod volumes to mount into the container's filesystem. + Cannot be updated. + items: + description: VolumeMount describes a mounting + of a Volume within a container. + properties: + mountPath: + description: |- + Path within the container at which the volume should be mounted. Must + not contain ':'. + type: string + mountPropagation: + description: |- + mountPropagation determines how mounts are propagated from the host + to container and the other way around. + When not set, MountPropagationNone is used. + This field is beta in 1.10. + When RecursiveReadOnly is set to IfPossible or to Enabled, MountPropagation must be None or unspecified + (which defaults to None). + type: string + name: + description: This must match the Name of a + Volume. + type: string + readOnly: + description: |- + Mounted read-only if true, read-write otherwise (false or unspecified). + Defaults to false. + type: boolean + recursiveReadOnly: + description: |- + RecursiveReadOnly specifies whether read-only mounts should be handled + recursively. + + If ReadOnly is false, this field has no meaning and must be unspecified. + + If ReadOnly is true, and this field is set to Disabled, the mount is not made + recursively read-only. If this field is set to IfPossible, the mount is made + recursively read-only, if it is supported by the container runtime. If this + field is set to Enabled, the mount is made recursively read-only if it is + supported by the container runtime, otherwise the pod will not be started and + an error will be generated to indicate the reason. + + If this field is set to IfPossible or Enabled, MountPropagation must be set to + None (or be unspecified, which defaults to None). + + If this field is not specified, it is treated as an equivalent of Disabled. + type: string + subPath: + description: |- + Path within the volume from which the container's volume should be mounted. + Defaults to "" (volume's root). + type: string + subPathExpr: + description: |- + Expanded path within the volume from which the container's volume should be mounted. + Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. + Defaults to "" (volume's root). + SubPathExpr and SubPath are mutually exclusive. + type: string + required: + - mountPath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - mountPath + x-kubernetes-list-type: map + workingDir: + description: |- + Container's working directory. + If not specified, the container runtime's default will be used, which + might be configured in the container image. + Cannot be updated. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + dnsConfig: + description: |- + Specifies the DNS parameters of a pod. + Parameters specified here will be merged to the generated DNS + configuration based on DNSPolicy. + properties: + nameservers: + description: |- + A list of DNS name server IP addresses. + This will be appended to the base nameservers generated from DNSPolicy. + Duplicated nameservers will be removed. + items: + type: string + type: array + x-kubernetes-list-type: atomic + options: + description: |- + A list of DNS resolver options. + This will be merged with the base options generated from DNSPolicy. + Duplicated entries will be removed. Resolution options given in Options + will override those that appear in the base DNSPolicy. + items: + description: PodDNSConfigOption defines DNS resolver + options of a pod. + properties: + name: + description: Required. + type: string + value: + type: string + type: object + type: array + x-kubernetes-list-type: atomic + searches: + description: |- + A list of DNS search domains for host-name lookup. + This will be appended to the base search paths generated from DNSPolicy. + Duplicated search paths will be removed. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + dnsPolicy: + description: |- + Set DNS policy for the pod. + Defaults to "ClusterFirst". + Valid values are 'ClusterFirstWithHostNet', 'ClusterFirst', 'Default' or 'None'. + DNS parameters given in DNSConfig will be merged with the policy selected with DNSPolicy. + To have DNS options set along with hostNetwork, you have to specify DNS policy + explicitly to 'ClusterFirstWithHostNet'. + type: string + enableServiceLinks: + description: |- + EnableServiceLinks indicates whether information about services should be injected into pod's + environment variables, matching the syntax of Docker links. + Optional: Defaults to true. + type: boolean + ephemeralContainers: + description: |- + List of ephemeral containers run in this pod. Ephemeral containers may be run in an existing + pod to perform user-initiated actions such as debugging. This list cannot be specified when + creating a pod, and it cannot be modified by updating the pod spec. In order to add an + ephemeral container to an existing pod, use the pod's ephemeralcontainers subresource. + items: + description: |- + An EphemeralContainer is a temporary container that you may add to an existing Pod for + user-initiated activities such as debugging. Ephemeral containers have no resource or + scheduling guarantees, and they will not be restarted when they exit or when a Pod is + removed or restarted. The kubelet may evict a Pod if an ephemeral container causes the + Pod to exceed its resource allocation. + + To add an ephemeral container, use the ephemeralcontainers subresource of an existing + Pod. Ephemeral containers may not be removed or restarted. + properties: + args: + description: |- + Arguments to the entrypoint. + The image's CMD is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell + items: + type: string + type: array + x-kubernetes-list-type: atomic + command: + description: |- + Entrypoint array. Not executed within a shell. + The image's ENTRYPOINT is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell + items: + type: string + type: array + x-kubernetes-list-type: atomic + env: + description: |- + List of environment variables to set in the container. + Cannot be updated. + items: + description: EnvVar represents an environment + variable present in a Container. + properties: + name: + description: Name of the environment variable. + Must be a C_IDENTIFIER. + type: string + value: + description: |- + Variable references $(VAR_NAME) are expanded + using the previously defined environment variables in the container and + any service environment variables. If a variable cannot be resolved, + the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. + "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless of whether the variable + exists or not. + Defaults to "". + type: string + valueFrom: + description: Source for the environment variable's + value. Cannot be used if value is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the ConfigMap + or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + description: |- + Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, + spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. + properties: + apiVersion: + description: Version of the schema + the FieldPath is written in terms + of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to + select in the specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. + properties: + containerName: + description: 'Container name: required + for volumes, optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output + format of the exposed resources, + defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to + select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + description: Selects a key of a secret + in the pod's namespace + properties: + key: + description: The key of the secret + to select from. Must be a valid + secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret + or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + envFrom: + description: |- + List of sources to populate environment variables in the container. + The keys defined within a source must be a C_IDENTIFIER. All invalid keys + will be reported as an event when the container is starting. When a key exists in multiple + sources, the value associated with the last source will take precedence. + Values defined by an Env with a duplicate key will take precedence. + Cannot be updated. + items: + description: EnvFromSource represents the source + of a set of ConfigMaps + properties: + configMapRef: + description: The ConfigMap to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the ConfigMap + must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + description: An optional identifier to prepend + to each key in the ConfigMap. Must be a + C_IDENTIFIER. + type: string + secretRef: + description: The Secret to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret + must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array + x-kubernetes-list-type: atomic + image: + description: |- + Container image name. + More info: https://kubernetes.io/docs/concepts/containers/images + type: string + imagePullPolicy: + description: |- + Image pull policy. + One of Always, Never, IfNotPresent. + Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/containers/images#updating-images + type: string + lifecycle: + description: Lifecycle is not allowed for ephemeral + containers. + properties: + postStart: + description: |- + PostStart is called immediately after a container is created. If the handler fails, + the container is terminated and restarted according to its restart policy. + Other management of the container blocks until the hook completes. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies the action to + take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies the http + request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in + the request. HTTP allows repeated + headers. + items: + description: HTTPHeader describes + a custom header to be used in HTTP + probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field + value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP + server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents the duration + that the container should sleep before + being terminated. + properties: + seconds: + description: Seconds is the number of + seconds to sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for the backward compatibility. There are no validation of this field and + lifecycle hooks will fail in runtime when tcp handler is specified. + properties: + host: + description: 'Optional: Host name to + connect to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + preStop: + description: |- + PreStop is called immediately before a container is terminated due to an + API request or management event such as liveness/startup probe failure, + preemption, resource contention, etc. The handler is not called if the + container crashes or exits. The Pod's termination grace period countdown begins before the + PreStop hook is executed. Regardless of the outcome of the handler, the + container will eventually terminate within the Pod's termination grace + period (unless delayed by finalizers). Other management of the container blocks until the hook completes + or until the termination grace period is reached. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies the action to + take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies the http + request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in + the request. HTTP allows repeated + headers. + items: + description: HTTPHeader describes + a custom header to be used in HTTP + probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field + value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP + server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents the duration + that the container should sleep before + being terminated. + properties: + seconds: + description: Seconds is the number of + seconds to sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for the backward compatibility. There are no validation of this field and + lifecycle hooks will fail in runtime when tcp handler is specified. + properties: + host: + description: 'Optional: Host name to + connect to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + type: object + livenessProbe: + description: Probes are not allowed for ephemeral + containers. + properties: + exec: + description: Exec specifies the action to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving + a GRPC port. + properties: + port: + description: Port number of the gRPC service. + Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http request + to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the + request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom + header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP + server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action involving + a TCP port. + properties: + host: + description: 'Optional: Host name to connect + to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + name: + description: |- + Name of the ephemeral container specified as a DNS_LABEL. + This name must be unique among all containers, init containers and ephemeral containers. + type: string + ports: + description: Ports are not allowed for ephemeral + containers. + items: + description: ContainerPort represents a network + port in a single container. + properties: + containerPort: + description: |- + Number of port to expose on the pod's IP address. + This must be a valid port number, 0 < x < 65536. + format: int32 + type: integer + hostIP: + description: What host IP to bind the external + port to. + type: string + hostPort: + description: |- + Number of port to expose on the host. + If specified, this must be a valid port number, 0 < x < 65536. + If HostNetwork is specified, this must match ContainerPort. + Most containers do not need this. + format: int32 + type: integer + name: + description: |- + If specified, this must be an IANA_SVC_NAME and unique within the pod. Each + named port in a pod must have a unique name. Name for the port that can be + referred to by services. + type: string + protocol: + default: TCP + description: |- + Protocol for port. Must be UDP, TCP, or SCTP. + Defaults to "TCP". + type: string + required: + - containerPort + type: object + type: array + x-kubernetes-list-map-keys: + - containerPort + - protocol + x-kubernetes-list-type: map + readinessProbe: + description: Probes are not allowed for ephemeral + containers. + properties: + exec: + description: Exec specifies the action to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving + a GRPC port. + properties: + port: + description: Port number of the gRPC service. + Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http request + to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the + request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom + header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP + server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action involving + a TCP port. + properties: + host: + description: 'Optional: Host name to connect + to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + resizePolicy: + description: Resources resize policy for the container. + items: + description: ContainerResizePolicy represents + resource resize policy for the container. + properties: + resourceName: + description: |- + Name of the resource to which this resource resize policy applies. + Supported values: cpu, memory. + type: string + restartPolicy: + description: |- + Restart policy to apply when specified resource is resized. + If not specified, it defaults to NotRequired. + type: string + required: + - resourceName + - restartPolicy + type: object + type: array + x-kubernetes-list-type: atomic + resources: + description: |- + Resources are not allowed for ephemeral containers. Ephemeral containers use spare resources + already allocated to the pod. + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one + entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + restartPolicy: + description: |- + Restart policy for the container to manage the restart behavior of each + container within a pod. + This may only be set for init containers. You cannot set this field on + ephemeral containers. + type: string + securityContext: + description: |- + Optional: SecurityContext defines the security options the ephemeral container should be run with. + If set, the fields of SecurityContext override the equivalent fields of PodSecurityContext. + properties: + allowPrivilegeEscalation: + description: |- + AllowPrivilegeEscalation controls whether a process can gain more + privileges than its parent process. This bool directly controls if + the no_new_privs flag will be set on the container process. + AllowPrivilegeEscalation is true always when the container is: + 1) run as Privileged + 2) has CAP_SYS_ADMIN + Note that this field cannot be set when spec.os.name is windows. + type: boolean + appArmorProfile: + description: |- + appArmorProfile is the AppArmor options to use by this container. If set, this profile + overrides the pod's appArmorProfile. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile loaded on the node that should be used. + The profile must be preconfigured on the node to work. + Must match the loaded name of the profile. + Must be set if and only if type is "Localhost". + type: string + type: + description: |- + type indicates which kind of AppArmor profile will be applied. + Valid options are: + Localhost - a profile pre-loaded on the node. + RuntimeDefault - the container runtime's default profile. + Unconfined - no AppArmor enforcement. + type: string + required: + - type + type: object + capabilities: + description: |- + The capabilities to add/drop when running containers. + Defaults to the default set of capabilities granted by the container runtime. + Note that this field cannot be set when spec.os.name is windows. + properties: + add: + description: Added capabilities + items: + description: Capability represent POSIX + capabilities type + type: string + type: array + x-kubernetes-list-type: atomic + drop: + description: Removed capabilities + items: + description: Capability represent POSIX + capabilities type + type: string + type: array + x-kubernetes-list-type: atomic + type: object + privileged: + description: |- + Run container in privileged mode. + Processes in privileged containers are essentially equivalent to root on the host. + Defaults to false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + procMount: + description: |- + procMount denotes the type of proc mount to use for the containers. + The default value is Default which uses the container runtime defaults for + readonly paths and masked paths. + This requires the ProcMountType feature flag to be enabled. + Note that this field cannot be set when spec.os.name is windows. + type: string + readOnlyRootFilesystem: + description: |- + Whether this container has a read-only root filesystem. + Default is false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + runAsGroup: + description: |- + The GID to run the entrypoint of the container process. + Uses runtime default if unset. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + runAsNonRoot: + description: |- + Indicates that the container must run as a non-root user. + If true, the Kubelet will validate the image at runtime to ensure that it + does not run as UID 0 (root) and fail to start the container if it does. + If unset or false, no such validation will be performed. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: |- + The UID to run the entrypoint of the container process. + Defaults to user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + seLinuxOptions: + description: |- + The SELinux context to be applied to the container. + If unspecified, the container runtime will allocate a random SELinux context for each + container. May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + properties: + level: + description: Level is SELinux level label + that applies to the container. + type: string + role: + description: Role is a SELinux role label + that applies to the container. + type: string + type: + description: Type is a SELinux type label + that applies to the container. + type: string + user: + description: User is a SELinux user label + that applies to the container. + type: string + type: object + seccompProfile: + description: |- + The seccomp options to use by this container. If seccomp options are + provided at both the pod & container level, the container options + override the pod options. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile defined in a file on the node should be used. + The profile must be preconfigured on the node to work. + Must be a descending path, relative to the kubelet's configured seccomp profile location. + Must be set if type is "Localhost". Must NOT be set for any other type. + type: string + type: + description: |- + type indicates which kind of seccomp profile will be applied. + Valid options are: + + Localhost - a profile defined in a file on the node should be used. + RuntimeDefault - the container runtime default profile should be used. + Unconfined - no profile should be applied. + type: string + required: + - type + type: object + windowsOptions: + description: |- + The Windows specific settings applied to all containers. + If unspecified, the options from the PodSecurityContext will be used. + If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is linux. + properties: + gmsaCredentialSpec: + description: |- + GMSACredentialSpec is where the GMSA admission webhook + (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the + GMSA credential spec named by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName is the + name of the GMSA credential spec to use. + type: string + hostProcess: + description: |- + HostProcess determines if a container should be run as a 'Host Process' container. + All of a Pod's containers must have the same effective HostProcess value + (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). + In addition, if HostProcess is true then HostNetwork must also be set to true. + type: boolean + runAsUserName: + description: |- + The UserName in Windows to run the entrypoint of the container process. + Defaults to the user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: string + type: object + type: object + startupProbe: + description: Probes are not allowed for ephemeral + containers. + properties: + exec: + description: Exec specifies the action to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving + a GRPC port. + properties: + port: + description: Port number of the gRPC service. + Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http request + to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the + request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom + header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP + server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action involving + a TCP port. + properties: + host: + description: 'Optional: Host name to connect + to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + stdin: + description: |- + Whether this container should allocate a buffer for stdin in the container runtime. If this + is not set, reads from stdin in the container will always result in EOF. + Default is false. + type: boolean + stdinOnce: + description: |- + Whether the container runtime should close the stdin channel after it has been opened by + a single attach. When stdin is true the stdin stream will remain open across multiple attach + sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the + first client attaches to stdin, and then remains open and accepts data until the client disconnects, + at which time stdin is closed and remains closed until the container is restarted. If this + flag is false, a container processes that reads from stdin will never receive an EOF. + Default is false + type: boolean + targetContainerName: + description: |- + If set, the name of the container from PodSpec that this ephemeral container targets. + The ephemeral container will be run in the namespaces (IPC, PID, etc) of this container. + If not set then the ephemeral container uses the namespaces configured in the Pod spec. + + The container runtime must implement support for this feature. If the runtime does not + support namespace targeting then the result of setting this field is undefined. + type: string + terminationMessagePath: + description: |- + Optional: Path at which the file to which the container's termination message + will be written is mounted into the container's filesystem. + Message written is intended to be brief final status, such as an assertion failure message. + Will be truncated by the node if greater than 4096 bytes. The total message length across + all containers will be limited to 12kb. + Defaults to /dev/termination-log. + Cannot be updated. + type: string + terminationMessagePolicy: + description: |- + Indicate how the termination message should be populated. File will use the contents of + terminationMessagePath to populate the container status message on both success and failure. + FallbackToLogsOnError will use the last chunk of container log output if the termination + message file is empty and the container exited with an error. + The log output is limited to 2048 bytes or 80 lines, whichever is smaller. + Defaults to File. + Cannot be updated. + type: string + tty: + description: |- + Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. + Default is false. + type: boolean + volumeDevices: + description: volumeDevices is the list of block + devices to be used by the container. + items: + description: volumeDevice describes a mapping + of a raw block device within a container. + properties: + devicePath: + description: devicePath is the path inside + of the container that the device will be + mapped to. + type: string + name: + description: name must match the name of a + persistentVolumeClaim in the pod + type: string + required: + - devicePath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - devicePath + x-kubernetes-list-type: map + volumeMounts: + description: |- + Pod volumes to mount into the container's filesystem. Subpath mounts are not allowed for ephemeral containers. + Cannot be updated. + items: + description: VolumeMount describes a mounting + of a Volume within a container. + properties: + mountPath: + description: |- + Path within the container at which the volume should be mounted. Must + not contain ':'. + type: string + mountPropagation: + description: |- + mountPropagation determines how mounts are propagated from the host + to container and the other way around. + When not set, MountPropagationNone is used. + This field is beta in 1.10. + When RecursiveReadOnly is set to IfPossible or to Enabled, MountPropagation must be None or unspecified + (which defaults to None). + type: string + name: + description: This must match the Name of a + Volume. + type: string + readOnly: + description: |- + Mounted read-only if true, read-write otherwise (false or unspecified). + Defaults to false. + type: boolean + recursiveReadOnly: + description: |- + RecursiveReadOnly specifies whether read-only mounts should be handled + recursively. + + If ReadOnly is false, this field has no meaning and must be unspecified. + + If ReadOnly is true, and this field is set to Disabled, the mount is not made + recursively read-only. If this field is set to IfPossible, the mount is made + recursively read-only, if it is supported by the container runtime. If this + field is set to Enabled, the mount is made recursively read-only if it is + supported by the container runtime, otherwise the pod will not be started and + an error will be generated to indicate the reason. + + If this field is set to IfPossible or Enabled, MountPropagation must be set to + None (or be unspecified, which defaults to None). + + If this field is not specified, it is treated as an equivalent of Disabled. + type: string + subPath: + description: |- + Path within the volume from which the container's volume should be mounted. + Defaults to "" (volume's root). + type: string + subPathExpr: + description: |- + Expanded path within the volume from which the container's volume should be mounted. + Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. + Defaults to "" (volume's root). + SubPathExpr and SubPath are mutually exclusive. + type: string + required: + - mountPath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - mountPath + x-kubernetes-list-type: map + workingDir: + description: |- + Container's working directory. + If not specified, the container runtime's default will be used, which + might be configured in the container image. + Cannot be updated. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + hostAliases: + description: |- + HostAliases is an optional list of hosts and IPs that will be injected into the pod's hosts + file if specified. + items: + description: |- + HostAlias holds the mapping between IP and hostnames that will be injected as an entry in the + pod's hosts file. + properties: + hostnames: + description: Hostnames for the above IP address. + items: + type: string + type: array + x-kubernetes-list-type: atomic + ip: + description: IP address of the host file entry. + type: string + required: + - ip + type: object + type: array + x-kubernetes-list-map-keys: + - ip + x-kubernetes-list-type: map + hostIPC: + description: |- + Use the host's ipc namespace. + Optional: Default to false. + type: boolean + hostNetwork: + description: |- + Host networking requested for this pod. Use the host's network namespace. + If this option is set, the ports that will be used must be specified. + Default to false. + type: boolean + hostPID: + description: |- + Use the host's pid namespace. + Optional: Default to false. + type: boolean + hostUsers: + description: |- + Use the host's user namespace. + Optional: Default to true. + If set to true or not present, the pod will be run in the host user namespace, useful + for when the pod needs a feature only available to the host user namespace, such as + loading a kernel module with CAP_SYS_MODULE. + When set to false, a new userns is created for the pod. Setting false is useful for + mitigating container breakout vulnerabilities even allowing users to run their + containers as root without actually having root privileges on the host. + This field is alpha-level and is only honored by servers that enable the UserNamespacesSupport feature. + type: boolean + hostname: + description: |- + Specifies the hostname of the Pod + If not specified, the pod's hostname will be set to a system-defined value. + type: string + imagePullSecrets: + description: |- + ImagePullSecrets is an optional list of references to secrets in the same namespace to use for pulling any of the images used by this PodSpec. + If specified, these secrets will be passed to individual puller implementations for them to use. + More info: https://kubernetes.io/docs/concepts/containers/images#specifying-imagepullsecrets-on-a-pod + items: + description: |- + LocalObjectReference contains enough information to let you locate the + referenced object inside the same namespace. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + initContainers: + description: |- + List of initialization containers belonging to the pod. + Init containers are executed in order prior to containers being started. If any + init container fails, the pod is considered to have failed and is handled according + to its restartPolicy. The name for an init container or normal container must be + unique among all containers. + Init containers may not have Lifecycle actions, Readiness probes, Liveness probes, or Startup probes. + The resourceRequirements of an init container are taken into account during scheduling + by finding the highest request/limit for each resource type, and then using the max of + of that value or the sum of the normal containers. Limits are applied to init containers + in a similar fashion. + Init containers cannot currently be added or removed. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/init-containers/ + items: + description: A single application container that you + want to run within a pod. + properties: + args: + description: |- + Arguments to the entrypoint. + The container image's CMD is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell + items: + type: string + type: array + x-kubernetes-list-type: atomic + command: + description: |- + Entrypoint array. Not executed within a shell. + The container image's ENTRYPOINT is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell + items: + type: string + type: array + x-kubernetes-list-type: atomic + env: + description: |- + List of environment variables to set in the container. + Cannot be updated. + items: + description: EnvVar represents an environment + variable present in a Container. + properties: + name: + description: Name of the environment variable. + Must be a C_IDENTIFIER. + type: string + value: + description: |- + Variable references $(VAR_NAME) are expanded + using the previously defined environment variables in the container and + any service environment variables. If a variable cannot be resolved, + the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. + "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless of whether the variable + exists or not. + Defaults to "". + type: string + valueFrom: + description: Source for the environment variable's + value. Cannot be used if value is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the ConfigMap + or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + description: |- + Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, + spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. + properties: + apiVersion: + description: Version of the schema + the FieldPath is written in terms + of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to + select in the specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. + properties: + containerName: + description: 'Container name: required + for volumes, optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output + format of the exposed resources, + defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to + select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + description: Selects a key of a secret + in the pod's namespace + properties: + key: + description: The key of the secret + to select from. Must be a valid + secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret + or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + envFrom: + description: |- + List of sources to populate environment variables in the container. + The keys defined within a source must be a C_IDENTIFIER. All invalid keys + will be reported as an event when the container is starting. When a key exists in multiple + sources, the value associated with the last source will take precedence. + Values defined by an Env with a duplicate key will take precedence. + Cannot be updated. + items: + description: EnvFromSource represents the source + of a set of ConfigMaps + properties: + configMapRef: + description: The ConfigMap to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the ConfigMap + must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + description: An optional identifier to prepend + to each key in the ConfigMap. Must be a + C_IDENTIFIER. + type: string + secretRef: + description: The Secret to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret + must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array + x-kubernetes-list-type: atomic + image: + description: |- + Container image name. + More info: https://kubernetes.io/docs/concepts/containers/images + This field is optional to allow higher level config management to default or override + container images in workload controllers like Deployments and StatefulSets. + type: string + imagePullPolicy: + description: |- + Image pull policy. + One of Always, Never, IfNotPresent. + Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/containers/images#updating-images + type: string + lifecycle: + description: |- + Actions that the management system should take in response to container lifecycle events. + Cannot be updated. + properties: + postStart: + description: |- + PostStart is called immediately after a container is created. If the handler fails, + the container is terminated and restarted according to its restart policy. + Other management of the container blocks until the hook completes. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies the action to + take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies the http + request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in + the request. HTTP allows repeated + headers. + items: + description: HTTPHeader describes + a custom header to be used in HTTP + probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field + value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP + server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents the duration + that the container should sleep before + being terminated. + properties: + seconds: + description: Seconds is the number of + seconds to sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for the backward compatibility. There are no validation of this field and + lifecycle hooks will fail in runtime when tcp handler is specified. + properties: + host: + description: 'Optional: Host name to + connect to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + preStop: + description: |- + PreStop is called immediately before a container is terminated due to an + API request or management event such as liveness/startup probe failure, + preemption, resource contention, etc. The handler is not called if the + container crashes or exits. The Pod's termination grace period countdown begins before the + PreStop hook is executed. Regardless of the outcome of the handler, the + container will eventually terminate within the Pod's termination grace + period (unless delayed by finalizers). Other management of the container blocks until the hook completes + or until the termination grace period is reached. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies the action to + take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies the http + request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in + the request. HTTP allows repeated + headers. + items: + description: HTTPHeader describes + a custom header to be used in HTTP + probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field + value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP + server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents the duration + that the container should sleep before + being terminated. + properties: + seconds: + description: Seconds is the number of + seconds to sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for the backward compatibility. There are no validation of this field and + lifecycle hooks will fail in runtime when tcp handler is specified. + properties: + host: + description: 'Optional: Host name to + connect to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + type: object + livenessProbe: + description: |- + Periodic probe of container liveness. + Container will be restarted if the probe fails. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies the action to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving + a GRPC port. + properties: + port: + description: Port number of the gRPC service. + Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http request + to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the + request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom + header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP + server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action involving + a TCP port. + properties: + host: + description: 'Optional: Host name to connect + to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + name: + description: |- + Name of the container specified as a DNS_LABEL. + Each container in a pod must have a unique name (DNS_LABEL). + Cannot be updated. + type: string + ports: + description: |- + List of ports to expose from the container. Not specifying a port here + DOES NOT prevent that port from being exposed. Any port which is + listening on the default "0.0.0.0" address inside a container will be + accessible from the network. + Modifying this array with strategic merge patch may corrupt the data. + For more information See https://github.com/kubernetes/kubernetes/issues/108255. + Cannot be updated. + items: + description: ContainerPort represents a network + port in a single container. + properties: + containerPort: + description: |- + Number of port to expose on the pod's IP address. + This must be a valid port number, 0 < x < 65536. + format: int32 + type: integer + hostIP: + description: What host IP to bind the external + port to. + type: string + hostPort: + description: |- + Number of port to expose on the host. + If specified, this must be a valid port number, 0 < x < 65536. + If HostNetwork is specified, this must match ContainerPort. + Most containers do not need this. + format: int32 + type: integer + name: + description: |- + If specified, this must be an IANA_SVC_NAME and unique within the pod. Each + named port in a pod must have a unique name. Name for the port that can be + referred to by services. + type: string + protocol: + default: TCP + description: |- + Protocol for port. Must be UDP, TCP, or SCTP. + Defaults to "TCP". + type: string + required: + - containerPort + type: object + type: array + x-kubernetes-list-map-keys: + - containerPort + - protocol + x-kubernetes-list-type: map + readinessProbe: + description: |- + Periodic probe of container service readiness. + Container will be removed from service endpoints if the probe fails. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies the action to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving + a GRPC port. + properties: + port: + description: Port number of the gRPC service. + Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http request + to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the + request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom + header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP + server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action involving + a TCP port. + properties: + host: + description: 'Optional: Host name to connect + to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + resizePolicy: + description: Resources resize policy for the container. + items: + description: ContainerResizePolicy represents + resource resize policy for the container. + properties: + resourceName: + description: |- + Name of the resource to which this resource resize policy applies. + Supported values: cpu, memory. + type: string + restartPolicy: + description: |- + Restart policy to apply when specified resource is resized. + If not specified, it defaults to NotRequired. + type: string + required: + - resourceName + - restartPolicy + type: object + type: array + x-kubernetes-list-type: atomic + resources: + description: |- + Compute Resources required by this container. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one + entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + restartPolicy: + description: |- + RestartPolicy defines the restart behavior of individual containers in a pod. + This field may only be set for init containers, and the only allowed value is "Always". + For non-init containers or when this field is not specified, + the restart behavior is defined by the Pod's restart policy and the container type. + Setting the RestartPolicy as "Always" for the init container will have the following effect: + this init container will be continually restarted on + exit until all regular containers have terminated. Once all regular + containers have completed, all init containers with restartPolicy "Always" + will be shut down. This lifecycle differs from normal init containers and + is often referred to as a "sidecar" container. Although this init + container still starts in the init container sequence, it does not wait + for the container to complete before proceeding to the next init + container. Instead, the next init container starts immediately after this + init container is started, or after any startupProbe has successfully + completed. + type: string + securityContext: + description: |- + SecurityContext defines the security options the container should be run with. + If set, the fields of SecurityContext override the equivalent fields of PodSecurityContext. + More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ + properties: + allowPrivilegeEscalation: + description: |- + AllowPrivilegeEscalation controls whether a process can gain more + privileges than its parent process. This bool directly controls if + the no_new_privs flag will be set on the container process. + AllowPrivilegeEscalation is true always when the container is: + 1) run as Privileged + 2) has CAP_SYS_ADMIN + Note that this field cannot be set when spec.os.name is windows. + type: boolean + appArmorProfile: + description: |- + appArmorProfile is the AppArmor options to use by this container. If set, this profile + overrides the pod's appArmorProfile. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile loaded on the node that should be used. + The profile must be preconfigured on the node to work. + Must match the loaded name of the profile. + Must be set if and only if type is "Localhost". + type: string + type: + description: |- + type indicates which kind of AppArmor profile will be applied. + Valid options are: + Localhost - a profile pre-loaded on the node. + RuntimeDefault - the container runtime's default profile. + Unconfined - no AppArmor enforcement. + type: string + required: + - type + type: object + capabilities: + description: |- + The capabilities to add/drop when running containers. + Defaults to the default set of capabilities granted by the container runtime. + Note that this field cannot be set when spec.os.name is windows. + properties: + add: + description: Added capabilities + items: + description: Capability represent POSIX + capabilities type + type: string + type: array + x-kubernetes-list-type: atomic + drop: + description: Removed capabilities + items: + description: Capability represent POSIX + capabilities type + type: string + type: array + x-kubernetes-list-type: atomic + type: object + privileged: + description: |- + Run container in privileged mode. + Processes in privileged containers are essentially equivalent to root on the host. + Defaults to false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + procMount: + description: |- + procMount denotes the type of proc mount to use for the containers. + The default value is Default which uses the container runtime defaults for + readonly paths and masked paths. + This requires the ProcMountType feature flag to be enabled. + Note that this field cannot be set when spec.os.name is windows. + type: string + readOnlyRootFilesystem: + description: |- + Whether this container has a read-only root filesystem. + Default is false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + runAsGroup: + description: |- + The GID to run the entrypoint of the container process. + Uses runtime default if unset. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + runAsNonRoot: + description: |- + Indicates that the container must run as a non-root user. + If true, the Kubelet will validate the image at runtime to ensure that it + does not run as UID 0 (root) and fail to start the container if it does. + If unset or false, no such validation will be performed. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: |- + The UID to run the entrypoint of the container process. + Defaults to user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + seLinuxOptions: + description: |- + The SELinux context to be applied to the container. + If unspecified, the container runtime will allocate a random SELinux context for each + container. May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + properties: + level: + description: Level is SELinux level label + that applies to the container. + type: string + role: + description: Role is a SELinux role label + that applies to the container. + type: string + type: + description: Type is a SELinux type label + that applies to the container. + type: string + user: + description: User is a SELinux user label + that applies to the container. + type: string + type: object + seccompProfile: + description: |- + The seccomp options to use by this container. If seccomp options are + provided at both the pod & container level, the container options + override the pod options. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile defined in a file on the node should be used. + The profile must be preconfigured on the node to work. + Must be a descending path, relative to the kubelet's configured seccomp profile location. + Must be set if type is "Localhost". Must NOT be set for any other type. + type: string + type: + description: |- + type indicates which kind of seccomp profile will be applied. + Valid options are: + + Localhost - a profile defined in a file on the node should be used. + RuntimeDefault - the container runtime default profile should be used. + Unconfined - no profile should be applied. + type: string + required: + - type + type: object + windowsOptions: + description: |- + The Windows specific settings applied to all containers. + If unspecified, the options from the PodSecurityContext will be used. + If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is linux. + properties: + gmsaCredentialSpec: + description: |- + GMSACredentialSpec is where the GMSA admission webhook + (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the + GMSA credential spec named by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName is the + name of the GMSA credential spec to use. + type: string + hostProcess: + description: |- + HostProcess determines if a container should be run as a 'Host Process' container. + All of a Pod's containers must have the same effective HostProcess value + (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). + In addition, if HostProcess is true then HostNetwork must also be set to true. + type: boolean + runAsUserName: + description: |- + The UserName in Windows to run the entrypoint of the container process. + Defaults to the user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: string + type: object + type: object + startupProbe: + description: |- + StartupProbe indicates that the Pod has successfully initialized. + If specified, no other probes are executed until this completes successfully. + If this probe fails, the Pod will be restarted, just as if the livenessProbe failed. + This can be used to provide different probe parameters at the beginning of a Pod's lifecycle, + when it might take a long time to load data or warm a cache, than during steady-state operation. + This cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies the action to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving + a GRPC port. + properties: + port: + description: Port number of the gRPC service. + Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http request + to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the + request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom + header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP + server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action involving + a TCP port. + properties: + host: + description: 'Optional: Host name to connect + to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + stdin: + description: |- + Whether this container should allocate a buffer for stdin in the container runtime. If this + is not set, reads from stdin in the container will always result in EOF. + Default is false. + type: boolean + stdinOnce: + description: |- + Whether the container runtime should close the stdin channel after it has been opened by + a single attach. When stdin is true the stdin stream will remain open across multiple attach + sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the + first client attaches to stdin, and then remains open and accepts data until the client disconnects, + at which time stdin is closed and remains closed until the container is restarted. If this + flag is false, a container processes that reads from stdin will never receive an EOF. + Default is false + type: boolean + terminationMessagePath: + description: |- + Optional: Path at which the file to which the container's termination message + will be written is mounted into the container's filesystem. + Message written is intended to be brief final status, such as an assertion failure message. + Will be truncated by the node if greater than 4096 bytes. The total message length across + all containers will be limited to 12kb. + Defaults to /dev/termination-log. + Cannot be updated. + type: string + terminationMessagePolicy: + description: |- + Indicate how the termination message should be populated. File will use the contents of + terminationMessagePath to populate the container status message on both success and failure. + FallbackToLogsOnError will use the last chunk of container log output if the termination + message file is empty and the container exited with an error. + The log output is limited to 2048 bytes or 80 lines, whichever is smaller. + Defaults to File. + Cannot be updated. + type: string + tty: + description: |- + Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. + Default is false. + type: boolean + volumeDevices: + description: volumeDevices is the list of block + devices to be used by the container. + items: + description: volumeDevice describes a mapping + of a raw block device within a container. + properties: + devicePath: + description: devicePath is the path inside + of the container that the device will be + mapped to. + type: string + name: + description: name must match the name of a + persistentVolumeClaim in the pod + type: string + required: + - devicePath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - devicePath + x-kubernetes-list-type: map + volumeMounts: + description: |- + Pod volumes to mount into the container's filesystem. + Cannot be updated. + items: + description: VolumeMount describes a mounting + of a Volume within a container. + properties: + mountPath: + description: |- + Path within the container at which the volume should be mounted. Must + not contain ':'. + type: string + mountPropagation: + description: |- + mountPropagation determines how mounts are propagated from the host + to container and the other way around. + When not set, MountPropagationNone is used. + This field is beta in 1.10. + When RecursiveReadOnly is set to IfPossible or to Enabled, MountPropagation must be None or unspecified + (which defaults to None). + type: string + name: + description: This must match the Name of a + Volume. + type: string + readOnly: + description: |- + Mounted read-only if true, read-write otherwise (false or unspecified). + Defaults to false. + type: boolean + recursiveReadOnly: + description: |- + RecursiveReadOnly specifies whether read-only mounts should be handled + recursively. + + If ReadOnly is false, this field has no meaning and must be unspecified. + + If ReadOnly is true, and this field is set to Disabled, the mount is not made + recursively read-only. If this field is set to IfPossible, the mount is made + recursively read-only, if it is supported by the container runtime. If this + field is set to Enabled, the mount is made recursively read-only if it is + supported by the container runtime, otherwise the pod will not be started and + an error will be generated to indicate the reason. + + If this field is set to IfPossible or Enabled, MountPropagation must be set to + None (or be unspecified, which defaults to None). + + If this field is not specified, it is treated as an equivalent of Disabled. + type: string + subPath: + description: |- + Path within the volume from which the container's volume should be mounted. + Defaults to "" (volume's root). + type: string + subPathExpr: + description: |- + Expanded path within the volume from which the container's volume should be mounted. + Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. + Defaults to "" (volume's root). + SubPathExpr and SubPath are mutually exclusive. + type: string + required: + - mountPath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - mountPath + x-kubernetes-list-type: map + workingDir: + description: |- + Container's working directory. + If not specified, the container runtime's default will be used, which + might be configured in the container image. + Cannot be updated. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + nodeName: + description: |- + NodeName indicates in which node this pod is scheduled. + If empty, this pod is a candidate for scheduling by the scheduler defined in schedulerName. + Once this field is set, the kubelet for this node becomes responsible for the lifecycle of this pod. + This field should not be used to express a desire for the pod to be scheduled on a specific node. + https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodename + type: string + nodeSelector: + additionalProperties: + type: string + description: |- + NodeSelector is a selector which must be true for the pod to fit on a node. + Selector which must match a node's labels for the pod to be scheduled on that node. + More info: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ + type: object + x-kubernetes-map-type: atomic + os: + description: |- + Specifies the OS of the containers in the pod. + Some pod and container fields are restricted if this is set. + + If the OS field is set to linux, the following fields must be unset: + -securityContext.windowsOptions + + If the OS field is set to windows, following fields must be unset: + - spec.hostPID + - spec.hostIPC + - spec.hostUsers + - spec.securityContext.appArmorProfile + - spec.securityContext.seLinuxOptions + - spec.securityContext.seccompProfile + - spec.securityContext.fsGroup + - spec.securityContext.fsGroupChangePolicy + - spec.securityContext.sysctls + - spec.shareProcessNamespace + - spec.securityContext.runAsUser + - spec.securityContext.runAsGroup + - spec.securityContext.supplementalGroups + - spec.securityContext.supplementalGroupsPolicy + - spec.containers[*].securityContext.appArmorProfile + - spec.containers[*].securityContext.seLinuxOptions + - spec.containers[*].securityContext.seccompProfile + - spec.containers[*].securityContext.capabilities + - spec.containers[*].securityContext.readOnlyRootFilesystem + - spec.containers[*].securityContext.privileged + - spec.containers[*].securityContext.allowPrivilegeEscalation + - spec.containers[*].securityContext.procMount + - spec.containers[*].securityContext.runAsUser + - spec.containers[*].securityContext.runAsGroup + properties: + name: + description: |- + Name is the name of the operating system. The currently supported values are linux and windows. + Additional value may be defined in future and can be one of: + https://github.com/opencontainers/runtime-spec/blob/master/config.md#platform-specific-configuration + Clients should expect to handle additional values and treat unrecognized values in this field as os: null + type: string + required: + - name + type: object + overhead: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Overhead represents the resource overhead associated with running a pod for a given RuntimeClass. + This field will be autopopulated at admission time by the RuntimeClass admission controller. If + the RuntimeClass admission controller is enabled, overhead must not be set in Pod create requests. + The RuntimeClass admission controller will reject Pod create requests which have the overhead already + set. If RuntimeClass is configured and selected in the PodSpec, Overhead will be set to the value + defined in the corresponding RuntimeClass, otherwise it will remain unset and treated as zero. + More info: https://git.k8s.io/enhancements/keps/sig-node/688-pod-overhead/README.md + type: object + preemptionPolicy: + description: |- + PreemptionPolicy is the Policy for preempting pods with lower priority. + One of Never, PreemptLowerPriority. + Defaults to PreemptLowerPriority if unset. + type: string + priority: + description: |- + The priority value. Various system components use this field to find the + priority of the pod. When Priority Admission Controller is enabled, it + prevents users from setting this field. The admission controller populates + this field from PriorityClassName. + The higher the value, the higher the priority. + format: int32 + type: integer + priorityClassName: + description: |- + If specified, indicates the pod's priority. "system-node-critical" and + "system-cluster-critical" are two special keywords which indicate the + highest priorities with the former being the highest priority. Any other + name must be defined by creating a PriorityClass object with that name. + If not specified, the pod priority will be default or zero if there is no + default. + type: string + readinessGates: + description: |- + If specified, all readiness gates will be evaluated for pod readiness. + A pod is ready when all its containers are ready AND + all conditions specified in the readiness gates have status equal to "True" + More info: https://git.k8s.io/enhancements/keps/sig-network/580-pod-readiness-gates + items: + description: PodReadinessGate contains the reference + to a pod condition + properties: + conditionType: + description: ConditionType refers to a condition + in the pod's condition list with matching type. + type: string + required: + - conditionType + type: object + type: array + x-kubernetes-list-type: atomic + resourceClaims: + description: |- + ResourceClaims defines which ResourceClaims must be allocated + and reserved before the Pod is allowed to start. The resources + will be made available to those containers which consume them + by name. + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + This field is immutable. + items: + description: |- + PodResourceClaim references exactly one ResourceClaim, either directly + or by naming a ResourceClaimTemplate which is then turned into a ResourceClaim + for the pod. + + It adds a name to it that uniquely identifies the ResourceClaim inside the Pod. + Containers that need access to the ResourceClaim reference it with this name. + properties: + name: + description: |- + Name uniquely identifies this resource claim inside the pod. + This must be a DNS_LABEL. + type: string + resourceClaimName: + description: |- + ResourceClaimName is the name of a ResourceClaim object in the same + namespace as this pod. + + Exactly one of ResourceClaimName and ResourceClaimTemplateName must + be set. + type: string + resourceClaimTemplateName: + description: |- + ResourceClaimTemplateName is the name of a ResourceClaimTemplate + object in the same namespace as this pod. + + The template will be used to create a new ResourceClaim, which will + be bound to this pod. When this pod is deleted, the ResourceClaim + will also be deleted. The pod name and resource name, along with a + generated component, will be used to form a unique name for the + ResourceClaim, which will be recorded in pod.status.resourceClaimStatuses. + + This field is immutable and no changes will be made to the + corresponding ResourceClaim by the control plane after creating the + ResourceClaim. + + Exactly one of ResourceClaimName and ResourceClaimTemplateName must + be set. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + restartPolicy: + description: |- + Restart policy for all containers within the pod. + One of Always, OnFailure, Never. In some contexts, only a subset of those values may be permitted. + Default to Always. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#restart-policy + type: string + runtimeClassName: + description: |- + RuntimeClassName refers to a RuntimeClass object in the node.k8s.io group, which should be used + to run this pod. If no RuntimeClass resource matches the named class, the pod will not be run. + If unset or empty, the "legacy" RuntimeClass will be used, which is an implicit class with an + empty definition that uses the default runtime handler. + More info: https://git.k8s.io/enhancements/keps/sig-node/585-runtime-class + type: string + schedulerName: + description: |- + If specified, the pod will be dispatched by specified scheduler. + If not specified, the pod will be dispatched by default scheduler. + type: string + schedulingGates: + description: |- + SchedulingGates is an opaque list of values that if specified will block scheduling the pod. + If schedulingGates is not empty, the pod will stay in the SchedulingGated state and the + scheduler will not attempt to schedule the pod. + + SchedulingGates can only be set at pod creation time, and be removed only afterwards. + items: + description: PodSchedulingGate is associated to a Pod + to guard its scheduling. + properties: + name: + description: |- + Name of the scheduling gate. + Each scheduling gate must have a unique name field. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + securityContext: + description: |- + SecurityContext holds pod-level security attributes and common container settings. + Optional: Defaults to empty. See type description for default values of each field. + properties: + appArmorProfile: + description: |- + appArmorProfile is the AppArmor options to use by the containers in this pod. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile loaded on the node that should be used. + The profile must be preconfigured on the node to work. + Must match the loaded name of the profile. + Must be set if and only if type is "Localhost". + type: string + type: + description: |- + type indicates which kind of AppArmor profile will be applied. + Valid options are: + Localhost - a profile pre-loaded on the node. + RuntimeDefault - the container runtime's default profile. + Unconfined - no AppArmor enforcement. + type: string + required: + - type + type: object + fsGroup: + description: |- + A special supplemental group that applies to all containers in a pod. + Some volume types allow the Kubelet to change the ownership of that volume + to be owned by the pod: + + 1. The owning GID will be the FSGroup + 2. The setgid bit is set (new files created in the volume will be owned by FSGroup) + 3. The permission bits are OR'd with rw-rw---- + + If unset, the Kubelet will not modify the ownership and permissions of any volume. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + fsGroupChangePolicy: + description: |- + fsGroupChangePolicy defines behavior of changing ownership and permission of the volume + before being exposed inside Pod. This field will only apply to + volume types which support fsGroup based ownership(and permissions). + It will have no effect on ephemeral volume types such as: secret, configmaps + and emptydir. + Valid values are "OnRootMismatch" and "Always". If not specified, "Always" is used. + Note that this field cannot be set when spec.os.name is windows. + type: string + runAsGroup: + description: |- + The GID to run the entrypoint of the container process. + Uses runtime default if unset. + May also be set in SecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence + for that container. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + runAsNonRoot: + description: |- + Indicates that the container must run as a non-root user. + If true, the Kubelet will validate the image at runtime to ensure that it + does not run as UID 0 (root) and fail to start the container if it does. + If unset or false, no such validation will be performed. + May also be set in SecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: |- + The UID to run the entrypoint of the container process. + Defaults to user specified in image metadata if unspecified. + May also be set in SecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence + for that container. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + seLinuxOptions: + description: |- + The SELinux context to be applied to all containers. + If unspecified, the container runtime will allocate a random SELinux context for each + container. May also be set in SecurityContext. If set in + both SecurityContext and PodSecurityContext, the value specified in SecurityContext + takes precedence for that container. + Note that this field cannot be set when spec.os.name is windows. + properties: + level: + description: Level is SELinux level label that + applies to the container. + type: string + role: + description: Role is a SELinux role label that + applies to the container. + type: string + type: + description: Type is a SELinux type label that + applies to the container. + type: string + user: + description: User is a SELinux user label that + applies to the container. + type: string + type: object + seccompProfile: + description: |- + The seccomp options to use by the containers in this pod. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile defined in a file on the node should be used. + The profile must be preconfigured on the node to work. + Must be a descending path, relative to the kubelet's configured seccomp profile location. + Must be set if type is "Localhost". Must NOT be set for any other type. + type: string + type: + description: |- + type indicates which kind of seccomp profile will be applied. + Valid options are: + + Localhost - a profile defined in a file on the node should be used. + RuntimeDefault - the container runtime default profile should be used. + Unconfined - no profile should be applied. + type: string + required: + - type + type: object + supplementalGroups: + description: |- + A list of groups applied to the first process run in each container, in + addition to the container's primary GID and fsGroup (if specified). If + the SupplementalGroupsPolicy feature is enabled, the + supplementalGroupsPolicy field determines whether these are in addition + to or instead of any group memberships defined in the container image. + If unspecified, no additional groups are added, though group memberships + defined in the container image may still be used, depending on the + supplementalGroupsPolicy field. + Note that this field cannot be set when spec.os.name is windows. + items: + format: int64 + type: integer + type: array + x-kubernetes-list-type: atomic + supplementalGroupsPolicy: + description: |- + Defines how supplemental groups of the first container processes are calculated. + Valid values are "Merge" and "Strict". If not specified, "Merge" is used. + (Alpha) Using the field requires the SupplementalGroupsPolicy feature gate to be enabled + and the container runtime must implement support for this feature. + Note that this field cannot be set when spec.os.name is windows. + type: string + sysctls: + description: |- + Sysctls hold a list of namespaced sysctls used for the pod. Pods with unsupported + sysctls (by the container runtime) might fail to launch. + Note that this field cannot be set when spec.os.name is windows. + items: + description: Sysctl defines a kernel parameter to + be set + properties: + name: + description: Name of a property to set + type: string + value: + description: Value of a property to set + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + windowsOptions: + description: |- + The Windows specific settings applied to all containers. + If unspecified, the options within a container's SecurityContext will be used. + If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is linux. + properties: + gmsaCredentialSpec: + description: |- + GMSACredentialSpec is where the GMSA admission webhook + (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the + GMSA credential spec named by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName is the name + of the GMSA credential spec to use. + type: string + hostProcess: + description: |- + HostProcess determines if a container should be run as a 'Host Process' container. + All of a Pod's containers must have the same effective HostProcess value + (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). + In addition, if HostProcess is true then HostNetwork must also be set to true. + type: boolean + runAsUserName: + description: |- + The UserName in Windows to run the entrypoint of the container process. + Defaults to the user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: string + type: object + type: object + serviceAccount: + description: |- + DeprecatedServiceAccount is a deprecated alias for ServiceAccountName. + Deprecated: Use serviceAccountName instead. + type: string + serviceAccountName: + description: |- + ServiceAccountName is the name of the ServiceAccount to use to run this pod. + More info: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ + type: string + setHostnameAsFQDN: + description: |- + If true the pod's hostname will be configured as the pod's FQDN, rather than the leaf name (the default). + In Linux containers, this means setting the FQDN in the hostname field of the kernel (the nodename field of struct utsname). + In Windows containers, this means setting the registry value of hostname for the registry key HKEY_LOCAL_MACHINE\\SYSTEM\\CurrentControlSet\\Services\\Tcpip\\Parameters to FQDN. + If a pod does not have FQDN, this has no effect. + Default to false. + type: boolean + shareProcessNamespace: + description: |- + Share a single process namespace between all of the containers in a pod. + When this is set containers will be able to view and signal processes from other containers + in the same pod, and the first process in each container will not be assigned PID 1. + HostPID and ShareProcessNamespace cannot both be set. + Optional: Default to false. + type: boolean + subdomain: + description: |- + If specified, the fully qualified Pod hostname will be "...svc.". + If not specified, the pod will not have a domainname at all. + type: string + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully. May be decreased in delete request. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + If this value is nil, the default grace period will be used instead. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + Defaults to 30 seconds. + format: int64 + type: integer + tolerations: + description: If specified, the pod's tolerations. + items: + description: |- + The pod this Toleration is attached to tolerates any taint that matches + the triple using the matching operator . + properties: + effect: + description: |- + Effect indicates the taint effect to match. Empty means match all taint effects. + When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: |- + Key is the taint key that the toleration applies to. Empty means match all taint keys. + If the key is empty, operator must be Exists; this combination means to match all values and all keys. + type: string + operator: + description: |- + Operator represents a key's relationship to the value. + Valid operators are Exists and Equal. Defaults to Equal. + Exists is equivalent to wildcard for value, so that a pod can + tolerate all taints of a particular category. + type: string + tolerationSeconds: + description: |- + TolerationSeconds represents the period of time the toleration (which must be + of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, + it is not set, which means tolerate the taint forever (do not evict). Zero and + negative values will be treated as 0 (evict immediately) by the system. + format: int64 + type: integer + value: + description: |- + Value is the taint value the toleration matches to. + If the operator is Exists, the value should be empty, otherwise just a regular string. + type: string + type: object + type: array + x-kubernetes-list-type: atomic + topologySpreadConstraints: + description: |- + TopologySpreadConstraints describes how a group of pods ought to spread across topology + domains. Scheduler will schedule pods in a way which abides by the constraints. + All topologySpreadConstraints are ANDed. + items: + description: TopologySpreadConstraint specifies how + to spread matching pods among the given topology. + properties: + labelSelector: + description: |- + LabelSelector is used to find matching pods. + Pods that match this label selector are counted to determine the number of pods + in their corresponding topology domain. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that + the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select the pods over which + spreading will be calculated. The keys are used to lookup values from the + incoming pod labels, those key-value labels are ANDed with labelSelector + to select the group of existing pods over which spreading will be calculated + for the incoming pod. The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + MatchLabelKeys cannot be set when LabelSelector isn't set. + Keys that don't exist in the incoming pod labels will + be ignored. A null or empty list means only match against labelSelector. + + This is a beta field and requires the MatchLabelKeysInPodTopologySpread feature gate to be enabled (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + maxSkew: + description: |- + MaxSkew describes the degree to which pods may be unevenly distributed. + When `whenUnsatisfiable=DoNotSchedule`, it is the maximum permitted difference + between the number of matching pods in the target topology and the global minimum. + The global minimum is the minimum number of matching pods in an eligible domain + or zero if the number of eligible domains is less than MinDomains. + For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same + labelSelector spread as 2/2/1: + In this case, the global minimum is 1. + | zone1 | zone2 | zone3 | + | P P | P P | P | + - if MaxSkew is 1, incoming pod can only be scheduled to zone3 to become 2/2/2; + scheduling it onto zone1(zone2) would make the ActualSkew(3-1) on zone1(zone2) + violate MaxSkew(1). + - if MaxSkew is 2, incoming pod can be scheduled onto any zone. + When `whenUnsatisfiable=ScheduleAnyway`, it is used to give higher precedence + to topologies that satisfy it. + It's a required field. Default value is 1 and 0 is not allowed. + format: int32 + type: integer + minDomains: + description: |- + MinDomains indicates a minimum number of eligible domains. + When the number of eligible domains with matching topology keys is less than minDomains, + Pod Topology Spread treats "global minimum" as 0, and then the calculation of Skew is performed. + And when the number of eligible domains with matching topology keys equals or greater than minDomains, + this value has no effect on scheduling. + As a result, when the number of eligible domains is less than minDomains, + scheduler won't schedule more than maxSkew Pods to those domains. + If value is nil, the constraint behaves as if MinDomains is equal to 1. + Valid values are integers greater than 0. + When value is not nil, WhenUnsatisfiable must be DoNotSchedule. + + For example, in a 3-zone cluster, MaxSkew is set to 2, MinDomains is set to 5 and pods with the same + labelSelector spread as 2/2/2: + | zone1 | zone2 | zone3 | + | P P | P P | P P | + The number of domains is less than 5(MinDomains), so "global minimum" is treated as 0. + In this situation, new pod with the same labelSelector cannot be scheduled, + because computed skew will be 3(3 - 0) if new Pod is scheduled to any of the three zones, + it will violate MaxSkew. + format: int32 + type: integer + nodeAffinityPolicy: + description: |- + NodeAffinityPolicy indicates how we will treat Pod's nodeAffinity/nodeSelector + when calculating pod topology spread skew. Options are: + - Honor: only nodes matching nodeAffinity/nodeSelector are included in the calculations. + - Ignore: nodeAffinity/nodeSelector are ignored. All nodes are included in the calculations. + + If this value is nil, the behavior is equivalent to the Honor policy. + This is a beta-level feature default enabled by the NodeInclusionPolicyInPodTopologySpread feature flag. + type: string + nodeTaintsPolicy: + description: |- + NodeTaintsPolicy indicates how we will treat node taints when calculating + pod topology spread skew. Options are: + - Honor: nodes without taints, along with tainted nodes for which the incoming pod + has a toleration, are included. + - Ignore: node taints are ignored. All nodes are included. + + If this value is nil, the behavior is equivalent to the Ignore policy. + This is a beta-level feature default enabled by the NodeInclusionPolicyInPodTopologySpread feature flag. + type: string + topologyKey: + description: |- + TopologyKey is the key of node labels. Nodes that have a label with this key + and identical values are considered to be in the same topology. + We consider each as a "bucket", and try to put balanced number + of pods into each bucket. + We define a domain as a particular instance of a topology. + Also, we define an eligible domain as a domain whose nodes meet the requirements of + nodeAffinityPolicy and nodeTaintsPolicy. + e.g. If TopologyKey is "kubernetes.io/hostname", each Node is a domain of that topology. + And, if TopologyKey is "topology.kubernetes.io/zone", each zone is a domain of that topology. + It's a required field. + type: string + whenUnsatisfiable: + description: |- + WhenUnsatisfiable indicates how to deal with a pod if it doesn't satisfy + the spread constraint. + - DoNotSchedule (default) tells the scheduler not to schedule it. + - ScheduleAnyway tells the scheduler to schedule the pod in any location, + but giving higher precedence to topologies that would help reduce the + skew. + A constraint is considered "Unsatisfiable" for an incoming pod + if and only if every possible node assignment for that pod would violate + "MaxSkew" on some topology. + For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same + labelSelector spread as 3/1/1: + | zone1 | zone2 | zone3 | + | P P P | P | P | + If WhenUnsatisfiable is set to DoNotSchedule, incoming pod can only be scheduled + to zone2(zone3) to become 3/2/1(3/1/2) as ActualSkew(2-1) on zone2(zone3) satisfies + MaxSkew(1). In other words, the cluster can still be imbalanced, but scheduler + won't make it *more* imbalanced. + It's a required field. + type: string + required: + - maxSkew + - topologyKey + - whenUnsatisfiable + type: object + type: array + x-kubernetes-list-map-keys: + - topologyKey + - whenUnsatisfiable + x-kubernetes-list-type: map + volumes: + description: |- + List of volumes that can be mounted by containers belonging to the pod. + More info: https://kubernetes.io/docs/concepts/storage/volumes + items: + description: Volume represents a named volume in a pod + that may be accessed by any container in the pod. + properties: + awsElasticBlockStore: + description: |- + awsElasticBlockStore represents an AWS Disk resource that is attached to a + kubelet's host machine and then exposed to the pod. + More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore + properties: + fsType: + description: |- + fsType is the filesystem type of the volume that you want to mount. + Tip: Ensure that the filesystem type is supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore + type: string + partition: + description: |- + partition is the partition in the volume that you want to mount. + If omitted, the default is to mount by volume name. + Examples: For volume /dev/sda1, you specify the partition as "1". + Similarly, the volume partition for /dev/sda is "0" (or you can leave the property empty). + format: int32 + type: integer + readOnly: + description: |- + readOnly value true will force the readOnly setting in VolumeMounts. + More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore + type: boolean + volumeID: + description: |- + volumeID is unique ID of the persistent disk resource in AWS (Amazon EBS volume). + More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore + type: string + required: + - volumeID + type: object + azureDisk: + description: azureDisk represents an Azure Data + Disk mount on the host and bind mount to the pod. + properties: + cachingMode: + description: 'cachingMode is the Host Caching + mode: None, Read Only, Read Write.' + type: string + diskName: + description: diskName is the Name of the data + disk in the blob storage + type: string + diskURI: + description: diskURI is the URI of data disk + in the blob storage + type: string + fsType: + default: ext4 + description: |- + fsType is Filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + kind: + description: 'kind expected values are Shared: + multiple blob disks per storage account Dedicated: + single blob disk per storage account Managed: + azure managed data disk (only in managed availability + set). defaults to shared' + type: string + readOnly: + default: false + description: |- + readOnly Defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + required: + - diskName + - diskURI + type: object + azureFile: + description: azureFile represents an Azure File + Service mount on the host and bind mount to the + pod. + properties: + readOnly: + description: |- + readOnly defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + secretName: + description: secretName is the name of secret + that contains Azure Storage Account Name and + Key + type: string + shareName: + description: shareName is the azure share Name + type: string + required: + - secretName + - shareName + type: object + cephfs: + description: cephFS represents a Ceph FS mount on + the host that shares a pod's lifetime + properties: + monitors: + description: |- + monitors is Required: Monitors is a collection of Ceph monitors + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + items: + type: string + type: array + x-kubernetes-list-type: atomic + path: + description: 'path is Optional: Used as the + mounted root, rather than the full Ceph tree, + default is /' + type: string + readOnly: + description: |- + readOnly is Optional: Defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + type: boolean + secretFile: + description: |- + secretFile is Optional: SecretFile is the path to key ring for User, default is /etc/ceph/user.secret + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + type: string + secretRef: + description: |- + secretRef is Optional: SecretRef is reference to the authentication secret for User, default is empty. + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + user: + description: |- + user is optional: User is the rados user name, default is admin + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + type: string + required: + - monitors + type: object + cinder: + description: |- + cinder represents a cinder volume attached and mounted on kubelets host machine. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md + properties: + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md + type: string + readOnly: + description: |- + readOnly defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md + type: boolean + secretRef: + description: |- + secretRef is optional: points to a secret object containing parameters used to connect + to OpenStack. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + volumeID: + description: |- + volumeID used to identify the volume in cinder. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md + type: string + required: + - volumeID + type: object + configMap: + description: configMap represents a configMap that + should populate this volume + properties: + defaultMode: + description: |- + defaultMode is optional: mode bits used to set permissions on created files by default. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + Defaults to 0644. + Directories within the path are not affected by this setting. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + items: + description: |- + items if unspecified, each key-value pair in the Data field of the referenced + ConfigMap will be projected into the volume as a file whose name is the + key and content is the value. If specified, the listed keys will be + projected into the specified paths, and unlisted keys will not be + present. If a key is specified which is not present in the ConfigMap, + the volume setup will error unless it is marked optional. Paths must be + relative and may not contain the '..' path or start with '..'. + items: + description: Maps a string key to a path within + a volume. + properties: + key: + description: key is the key to project. + type: string + mode: + description: |- + mode is Optional: mode bits used to set permissions on this file. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: |- + path is the relative path of the file to map the key to. + May not be an absolute path. + May not contain the path element '..'. + May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: optional specify whether the ConfigMap + or its keys must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + csi: + description: csi (Container Storage Interface) represents + ephemeral storage that is handled by certain external + CSI drivers (Beta feature). + properties: + driver: + description: |- + driver is the name of the CSI driver that handles this volume. + Consult with your admin for the correct name as registered in the cluster. + type: string + fsType: + description: |- + fsType to mount. Ex. "ext4", "xfs", "ntfs". + If not provided, the empty value is passed to the associated CSI driver + which will determine the default filesystem to apply. + type: string + nodePublishSecretRef: + description: |- + nodePublishSecretRef is a reference to the secret object containing + sensitive information to pass to the CSI driver to complete the CSI + NodePublishVolume and NodeUnpublishVolume calls. + This field is optional, and may be empty if no secret is required. If the + secret object contains more than one secret, all secret references are passed. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + readOnly: + description: |- + readOnly specifies a read-only configuration for the volume. + Defaults to false (read/write). + type: boolean + volumeAttributes: + additionalProperties: + type: string + description: |- + volumeAttributes stores driver-specific properties that are passed to the CSI + driver. Consult your driver's documentation for supported values. + type: object + required: + - driver + type: object + downwardAPI: + description: downwardAPI represents downward API + about the pod that should populate this volume + properties: + defaultMode: + description: |- + Optional: mode bits to use on created files by default. Must be a + Optional: mode bits used to set permissions on created files by default. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + Defaults to 0644. + Directories within the path are not affected by this setting. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + items: + description: Items is a list of downward API + volume file + items: + description: DownwardAPIVolumeFile represents + information to create the file containing + the pod field + properties: + fieldRef: + description: 'Required: Selects a field + of the pod: only annotations, labels, + name, namespace and uid are supported.' + properties: + apiVersion: + description: Version of the schema + the FieldPath is written in terms + of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to + select in the specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + mode: + description: |- + Optional: mode bits used to set permissions on this file, must be an octal value + between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: 'Required: Path is the relative + path name of the file to be created. + Must not be absolute or contain the + ''..'' path. Must be utf-8 encoded. + The first item of the relative path + must not start with ''..''' + type: string + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, requests.cpu and requests.memory) are currently supported. + properties: + containerName: + description: 'Container name: required + for volumes, optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output + format of the exposed resources, + defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to + select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + required: + - path + type: object + type: array + x-kubernetes-list-type: atomic + type: object + emptyDir: + description: |- + emptyDir represents a temporary directory that shares a pod's lifetime. + More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir + properties: + medium: + description: |- + medium represents what type of storage medium should back this directory. + The default is "" which means to use the node's default medium. + Must be an empty string (default) or Memory. + More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir + type: string + sizeLimit: + anyOf: + - type: integer + - type: string + description: |- + sizeLimit is the total amount of local storage required for this EmptyDir volume. + The size limit is also applicable for memory medium. + The maximum usage on memory medium EmptyDir would be the minimum value between + the SizeLimit specified here and the sum of memory limits of all containers in a pod. + The default is nil which means that the limit is undefined. + More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + ephemeral: + description: |- + ephemeral represents a volume that is handled by a cluster storage driver. + The volume's lifecycle is tied to the pod that defines it - it will be created before the pod starts, + and deleted when the pod is removed. + + Use this if: + a) the volume is only needed while the pod runs, + b) features of normal volumes like restoring from snapshot or capacity + tracking are needed, + c) the storage driver is specified through a storage class, and + d) the storage driver supports dynamic volume provisioning through + a PersistentVolumeClaim (see EphemeralVolumeSource for more + information on the connection between this volume type + and PersistentVolumeClaim). + + Use PersistentVolumeClaim or one of the vendor-specific + APIs for volumes that persist for longer than the lifecycle + of an individual pod. + + Use CSI for light-weight local ephemeral volumes if the CSI driver is meant to + be used that way - see the documentation of the driver for + more information. + + A pod can use both types of ephemeral volumes and + persistent volumes at the same time. + properties: + volumeClaimTemplate: + description: |- + Will be used to create a stand-alone PVC to provision the volume. + The pod in which this EphemeralVolumeSource is embedded will be the + owner of the PVC, i.e. the PVC will be deleted together with the + pod. The name of the PVC will be `-` where + `` is the name from the `PodSpec.Volumes` array + entry. Pod validation will reject the pod if the concatenated name + is not valid for a PVC (for example, too long). + + An existing PVC with that name that is not owned by the pod + will *not* be used for the pod to avoid using an unrelated + volume by mistake. Starting the pod is then blocked until + the unrelated PVC is removed. If such a pre-created PVC is + meant to be used by the pod, the PVC has to updated with an + owner reference to the pod once the pod exists. Normally + this should not be necessary, but it may be useful when + manually reconstructing a broken cluster. + + This field is read-only and no changes will be made by Kubernetes + to the PVC after it has been created. + + Required, must not be nil. + properties: + metadata: + description: |- + May contain labels and annotations that will be copied into the PVC + when creating it. No other fields are allowed and will be rejected during + validation. + type: object + spec: + description: |- + The specification for the PersistentVolumeClaim. The entire content is + copied unchanged into the PVC that gets created from this + template. The same fields as in a PersistentVolumeClaim + are also valid here. + properties: + accessModes: + description: |- + accessModes contains the desired access modes the volume should have. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1 + items: + type: string + type: array + x-kubernetes-list-type: atomic + dataSource: + description: |- + dataSource field can be used to specify either: + * An existing VolumeSnapshot object (snapshot.storage.k8s.io/VolumeSnapshot) + * An existing PVC (PersistentVolumeClaim) + If the provisioner or an external controller can support the specified data source, + it will create a new volume based on the contents of the specified data source. + When the AnyVolumeDataSource feature gate is enabled, dataSource contents will be copied to dataSourceRef, + and dataSourceRef contents will be copied to dataSource when dataSourceRef.namespace is not specified. + If the namespace is specified, then dataSourceRef will not be copied to dataSource. + properties: + apiGroup: + description: |- + APIGroup is the group for the resource being referenced. + If APIGroup is not specified, the specified Kind must be in the core API group. + For any other third-party types, APIGroup is required. + type: string + kind: + description: Kind is the type of + resource being referenced + type: string + name: + description: Name is the name of + resource being referenced + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + dataSourceRef: + description: |- + dataSourceRef specifies the object from which to populate the volume with data, if a non-empty + volume is desired. This may be any object from a non-empty API group (non + core object) or a PersistentVolumeClaim object. + When this field is specified, volume binding will only succeed if the type of + the specified object matches some installed volume populator or dynamic + provisioner. + This field will replace the functionality of the dataSource field and as such + if both fields are non-empty, they must have the same value. For backwards + compatibility, when namespace isn't specified in dataSourceRef, + both fields (dataSource and dataSourceRef) will be set to the same + value automatically if one of them is empty and the other is non-empty. + When namespace is specified in dataSourceRef, + dataSource isn't set to the same value and must be empty. + There are three important differences between dataSource and dataSourceRef: + * While dataSource only allows two specific types of objects, dataSourceRef + allows any non-core object, as well as PersistentVolumeClaim objects. + * While dataSource ignores disallowed values (dropping them), dataSourceRef + preserves all values, and generates an error if a disallowed value is + specified. + * While dataSource only allows local objects, dataSourceRef allows objects + in any namespaces. + (Beta) Using this field requires the AnyVolumeDataSource feature gate to be enabled. + (Alpha) Using the namespace field of dataSourceRef requires the CrossNamespaceVolumeDataSource feature gate to be enabled. + properties: + apiGroup: + description: |- + APIGroup is the group for the resource being referenced. + If APIGroup is not specified, the specified Kind must be in the core API group. + For any other third-party types, APIGroup is required. + type: string + kind: + description: Kind is the type of + resource being referenced + type: string + name: + description: Name is the name of + resource being referenced + type: string + namespace: + description: |- + Namespace is the namespace of resource being referenced + Note that when a namespace is specified, a gateway.networking.k8s.io/ReferenceGrant object is required in the referent namespace to allow that namespace's owner to accept the reference. See the ReferenceGrant documentation for details. + (Alpha) This field requires the CrossNamespaceVolumeDataSource feature gate to be enabled. + type: string + required: + - kind + - name + type: object + resources: + description: |- + resources represents the minimum resources the volume should have. + If RecoverVolumeExpansionFailure feature is enabled users are allowed to specify resource requirements + that are lower than previous value but must still be higher than capacity recorded in the + status field of the claim. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + selector: + description: selector is a label query + over volumes to consider for binding. + properties: + matchExpressions: + description: matchExpressions is + a list of label selector requirements. + The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + storageClassName: + description: |- + storageClassName is the name of the StorageClass required by the claim. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1 + type: string + volumeAttributesClassName: + description: |- + volumeAttributesClassName may be used to set the VolumeAttributesClass used by this claim. + If specified, the CSI driver will create or update the volume with the attributes defined + in the corresponding VolumeAttributesClass. This has a different purpose than storageClassName, + it can be changed after the claim is created. An empty string value means that no VolumeAttributesClass + will be applied to the claim but it's not allowed to reset this field to empty string once it is set. + If unspecified and the PersistentVolumeClaim is unbound, the default VolumeAttributesClass + will be set by the persistentvolume controller if it exists. + If the resource referred to by volumeAttributesClass does not exist, this PersistentVolumeClaim will be + set to a Pending state, as reflected by the modifyVolumeStatus field, until such as a resource + exists. + More info: https://kubernetes.io/docs/concepts/storage/volume-attributes-classes/ + (Beta) Using this field requires the VolumeAttributesClass feature gate to be enabled (off by default). + type: string + volumeMode: + description: |- + volumeMode defines what type of volume is required by the claim. + Value of Filesystem is implied when not included in claim spec. + type: string + volumeName: + description: volumeName is the binding + reference to the PersistentVolume + backing this claim. + type: string + type: object + required: + - spec + type: object + type: object + fc: + description: fc represents a Fibre Channel resource + that is attached to a kubelet's host machine and + then exposed to the pod. + properties: + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + lun: + description: 'lun is Optional: FC target lun + number' + format: int32 + type: integer + readOnly: + description: |- + readOnly is Optional: Defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + targetWWNs: + description: 'targetWWNs is Optional: FC target + worldwide names (WWNs)' + items: + type: string + type: array + x-kubernetes-list-type: atomic + wwids: + description: |- + wwids Optional: FC volume world wide identifiers (wwids) + Either wwids or combination of targetWWNs and lun must be set, but not both simultaneously. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + flexVolume: + description: |- + flexVolume represents a generic volume resource that is + provisioned/attached using an exec based plugin. + properties: + driver: + description: driver is the name of the driver + to use for this volume. + type: string + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". The default filesystem depends on FlexVolume script. + type: string + options: + additionalProperties: + type: string + description: 'options is Optional: this field + holds extra command options if any.' + type: object + readOnly: + description: |- + readOnly is Optional: defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + secretRef: + description: |- + secretRef is Optional: secretRef is reference to the secret object containing + sensitive information to pass to the plugin scripts. This may be + empty if no secret object is specified. If the secret object + contains more than one secret, all secrets are passed to the plugin + scripts. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + required: + - driver + type: object + flocker: + description: flocker represents a Flocker volume + attached to a kubelet's host machine. This depends + on the Flocker control service being running + properties: + datasetName: + description: |- + datasetName is Name of the dataset stored as metadata -> name on the dataset for Flocker + should be considered as deprecated + type: string + datasetUUID: + description: datasetUUID is the UUID of the + dataset. This is unique identifier of a Flocker + dataset + type: string + type: object + gcePersistentDisk: + description: |- + gcePersistentDisk represents a GCE Disk resource that is attached to a + kubelet's host machine and then exposed to the pod. + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + properties: + fsType: + description: |- + fsType is filesystem type of the volume that you want to mount. + Tip: Ensure that the filesystem type is supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + type: string + partition: + description: |- + partition is the partition in the volume that you want to mount. + If omitted, the default is to mount by volume name. + Examples: For volume /dev/sda1, you specify the partition as "1". + Similarly, the volume partition for /dev/sda is "0" (or you can leave the property empty). + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + format: int32 + type: integer + pdName: + description: |- + pdName is unique name of the PD resource in GCE. Used to identify the disk in GCE. + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + type: string + readOnly: + description: |- + readOnly here will force the ReadOnly setting in VolumeMounts. + Defaults to false. + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + type: boolean + required: + - pdName + type: object + gitRepo: + description: |- + gitRepo represents a git repository at a particular revision. + DEPRECATED: GitRepo is deprecated. To provision a container with a git repo, mount an + EmptyDir into an InitContainer that clones the repo using git, then mount the EmptyDir + into the Pod's container. + properties: + directory: + description: |- + directory is the target directory name. + Must not contain or start with '..'. If '.' is supplied, the volume directory will be the + git repository. Otherwise, if specified, the volume will contain the git repository in + the subdirectory with the given name. + type: string + repository: + description: repository is the URL + type: string + revision: + description: revision is the commit hash for + the specified revision. + type: string + required: + - repository + type: object + glusterfs: + description: |- + glusterfs represents a Glusterfs mount on the host that shares a pod's lifetime. + More info: https://examples.k8s.io/volumes/glusterfs/README.md + properties: + endpoints: + description: |- + endpoints is the endpoint name that details Glusterfs topology. + More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod + type: string + path: + description: |- + path is the Glusterfs volume path. + More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod + type: string + readOnly: + description: |- + readOnly here will force the Glusterfs volume to be mounted with read-only permissions. + Defaults to false. + More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod + type: boolean + required: + - endpoints + - path + type: object + hostPath: + description: |- + hostPath represents a pre-existing file or directory on the host + machine that is directly exposed to the container. This is generally + used for system agents or other privileged things that are allowed + to see the host machine. Most containers will NOT need this. + More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath + properties: + path: + description: |- + path of the directory on the host. + If the path is a symlink, it will follow the link to the real path. + More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath + type: string + type: + description: |- + type for HostPath Volume + Defaults to "" + More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath + type: string + required: + - path + type: object + image: + description: |- + image represents an OCI object (a container image or artifact) pulled and mounted on the kubelet's host machine. + The volume is resolved at pod startup depending on which PullPolicy value is provided: + + - Always: the kubelet always attempts to pull the reference. Container creation will fail If the pull fails. + - Never: the kubelet never pulls the reference and only uses a local image or artifact. Container creation will fail if the reference isn't present. + - IfNotPresent: the kubelet pulls if the reference isn't already present on disk. Container creation will fail if the reference isn't present and the pull fails. + + The volume gets re-resolved if the pod gets deleted and recreated, which means that new remote content will become available on pod recreation. + A failure to resolve or pull the image during pod startup will block containers from starting and may add significant latency. Failures will be retried using normal volume backoff and will be reported on the pod reason and message. + The types of objects that may be mounted by this volume are defined by the container runtime implementation on a host machine and at minimum must include all valid types supported by the container image field. + The OCI object gets mounted in a single directory (spec.containers[*].volumeMounts.mountPath) by merging the manifest layers in the same way as for container images. + The volume will be mounted read-only (ro) and non-executable files (noexec). + Sub path mounts for containers are not supported (spec.containers[*].volumeMounts.subpath). + The field spec.securityContext.fsGroupChangePolicy has no effect on this volume type. + properties: + pullPolicy: + description: |- + Policy for pulling OCI objects. Possible values are: + Always: the kubelet always attempts to pull the reference. Container creation will fail If the pull fails. + Never: the kubelet never pulls the reference and only uses a local image or artifact. Container creation will fail if the reference isn't present. + IfNotPresent: the kubelet pulls if the reference isn't already present on disk. Container creation will fail if the reference isn't present and the pull fails. + Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. + type: string + reference: + description: |- + Required: Image or artifact reference to be used. + Behaves in the same way as pod.spec.containers[*].image. + Pull secrets will be assembled in the same way as for the container image by looking up node credentials, SA image pull secrets, and pod spec image pull secrets. + More info: https://kubernetes.io/docs/concepts/containers/images + This field is optional to allow higher level config management to default or override + container images in workload controllers like Deployments and StatefulSets. + type: string + type: object + iscsi: + description: |- + iscsi represents an ISCSI Disk resource that is attached to a + kubelet's host machine and then exposed to the pod. + More info: https://examples.k8s.io/volumes/iscsi/README.md + properties: + chapAuthDiscovery: + description: chapAuthDiscovery defines whether + support iSCSI Discovery CHAP authentication + type: boolean + chapAuthSession: + description: chapAuthSession defines whether + support iSCSI Session CHAP authentication + type: boolean + fsType: + description: |- + fsType is the filesystem type of the volume that you want to mount. + Tip: Ensure that the filesystem type is supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://kubernetes.io/docs/concepts/storage/volumes#iscsi + type: string + initiatorName: + description: |- + initiatorName is the custom iSCSI Initiator Name. + If initiatorName is specified with iscsiInterface simultaneously, new iSCSI interface + : will be created for the connection. + type: string + iqn: + description: iqn is the target iSCSI Qualified + Name. + type: string + iscsiInterface: + default: default + description: |- + iscsiInterface is the interface Name that uses an iSCSI transport. + Defaults to 'default' (tcp). + type: string + lun: + description: lun represents iSCSI Target Lun + number. + format: int32 + type: integer + portals: + description: |- + portals is the iSCSI Target Portal List. The portal is either an IP or ip_addr:port if the port + is other than default (typically TCP ports 860 and 3260). + items: + type: string + type: array + x-kubernetes-list-type: atomic + readOnly: + description: |- + readOnly here will force the ReadOnly setting in VolumeMounts. + Defaults to false. + type: boolean + secretRef: + description: secretRef is the CHAP Secret for + iSCSI target and initiator authentication + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + targetPortal: + description: |- + targetPortal is iSCSI Target Portal. The Portal is either an IP or ip_addr:port if the port + is other than default (typically TCP ports 860 and 3260). + type: string + required: + - iqn + - lun + - targetPortal + type: object + name: + description: |- + name of the volume. + Must be a DNS_LABEL and unique within the pod. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + nfs: + description: |- + nfs represents an NFS mount on the host that shares a pod's lifetime + More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs + properties: + path: + description: |- + path that is exported by the NFS server. + More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs + type: string + readOnly: + description: |- + readOnly here will force the NFS export to be mounted with read-only permissions. + Defaults to false. + More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs + type: boolean + server: + description: |- + server is the hostname or IP address of the NFS server. + More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs + type: string + required: + - path + - server + type: object + persistentVolumeClaim: + description: |- + persistentVolumeClaimVolumeSource represents a reference to a + PersistentVolumeClaim in the same namespace. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims + properties: + claimName: + description: |- + claimName is the name of a PersistentVolumeClaim in the same namespace as the pod using this volume. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims + type: string + readOnly: + description: |- + readOnly Will force the ReadOnly setting in VolumeMounts. + Default false. + type: boolean + required: + - claimName + type: object + photonPersistentDisk: + description: photonPersistentDisk represents a PhotonController + persistent disk attached and mounted on kubelets + host machine + properties: + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + pdID: + description: pdID is the ID that identifies + Photon Controller persistent disk + type: string + required: + - pdID + type: object + portworxVolume: + description: portworxVolume represents a portworx + volume attached and mounted on kubelets host machine + properties: + fsType: + description: |- + fSType represents the filesystem type to mount + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs". Implicitly inferred to be "ext4" if unspecified. + type: string + readOnly: + description: |- + readOnly defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + volumeID: + description: volumeID uniquely identifies a + Portworx volume + type: string + required: + - volumeID + type: object + projected: + description: projected items for all in one resources + secrets, configmaps, and downward API + properties: + defaultMode: + description: |- + defaultMode are the mode bits used to set permissions on created files by default. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + Directories within the path are not affected by this setting. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + sources: + description: |- + sources is the list of volume projections. Each entry in this list + handles one source. + items: + description: |- + Projection that may be projected along with other supported volume types. + Exactly one of these fields must be set. + properties: + clusterTrustBundle: + description: |- + ClusterTrustBundle allows a pod to access the `.spec.trustBundle` field + of ClusterTrustBundle objects in an auto-updating file. + + Alpha, gated by the ClusterTrustBundleProjection feature gate. + + ClusterTrustBundle objects can either be selected by name, or by the + combination of signer name and a label selector. + + Kubelet performs aggressive normalization of the PEM contents written + into the pod filesystem. Esoteric PEM features such as inter-block + comments and block headers are stripped. Certificates are deduplicated. + The ordering of certificates within the file is arbitrary, and Kubelet + may change the order over time. + properties: + labelSelector: + description: |- + Select all ClusterTrustBundles that match this label selector. Only has + effect if signerName is set. Mutually-exclusive with name. If unset, + interpreted as "match nothing". If set but empty, interpreted as "match + everything". + properties: + matchExpressions: + description: matchExpressions + is a list of label selector + requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the + label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + name: + description: |- + Select a single ClusterTrustBundle by object name. Mutually-exclusive + with signerName and labelSelector. + type: string + optional: + description: |- + If true, don't block pod startup if the referenced ClusterTrustBundle(s) + aren't available. If using name, then the named ClusterTrustBundle is + allowed not to exist. If using signerName, then the combination of + signerName and labelSelector is allowed to match zero + ClusterTrustBundles. + type: boolean + path: + description: Relative path from the + volume root to write the bundle. + type: string + signerName: + description: |- + Select all ClusterTrustBundles that match this signer name. + Mutually-exclusive with name. The contents of all selected + ClusterTrustBundles will be unified and deduplicated. + type: string + required: + - path + type: object + configMap: + description: configMap information about + the configMap data to project + properties: + items: + description: |- + items if unspecified, each key-value pair in the Data field of the referenced + ConfigMap will be projected into the volume as a file whose name is the + key and content is the value. If specified, the listed keys will be + projected into the specified paths, and unlisted keys will not be + present. If a key is specified which is not present in the ConfigMap, + the volume setup will error unless it is marked optional. Paths must be + relative and may not contain the '..' path or start with '..'. + items: + description: Maps a string key to + a path within a volume. + properties: + key: + description: key is the key + to project. + type: string + mode: + description: |- + mode is Optional: mode bits used to set permissions on this file. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: |- + path is the relative path of the file to map the key to. + May not be an absolute path. + May not contain the path element '..'. + May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: optional specify whether + the ConfigMap or its keys must be + defined + type: boolean + type: object + x-kubernetes-map-type: atomic + downwardAPI: + description: downwardAPI information about + the downwardAPI data to project + properties: + items: + description: Items is a list of DownwardAPIVolume + file + items: + description: DownwardAPIVolumeFile + represents information to create + the file containing the pod field + properties: + fieldRef: + description: 'Required: Selects + a field of the pod: only annotations, + labels, name, namespace and + uid are supported.' + properties: + apiVersion: + description: Version of + the schema the FieldPath + is written in terms of, + defaults to "v1". + type: string + fieldPath: + description: Path of the + field to select in the + specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + mode: + description: |- + Optional: mode bits used to set permissions on this file, must be an octal value + between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: 'Required: Path + is the relative path name + of the file to be created. + Must not be absolute or contain + the ''..'' path. Must be utf-8 + encoded. The first item of + the relative path must not + start with ''..''' + type: string + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, requests.cpu and requests.memory) are currently supported. + properties: + containerName: + description: 'Container + name: required for volumes, + optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the + output format of the exposed + resources, defaults to + "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: + resource to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + required: + - path + type: object + type: array + x-kubernetes-list-type: atomic + type: object + secret: + description: secret information about + the secret data to project + properties: + items: + description: |- + items if unspecified, each key-value pair in the Data field of the referenced + Secret will be projected into the volume as a file whose name is the + key and content is the value. If specified, the listed keys will be + projected into the specified paths, and unlisted keys will not be + present. If a key is specified which is not present in the Secret, + the volume setup will error unless it is marked optional. Paths must be + relative and may not contain the '..' path or start with '..'. + items: + description: Maps a string key to + a path within a volume. + properties: + key: + description: key is the key + to project. + type: string + mode: + description: |- + mode is Optional: mode bits used to set permissions on this file. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: |- + path is the relative path of the file to map the key to. + May not be an absolute path. + May not contain the path element '..'. + May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: optional field specify + whether the Secret or its key must + be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + serviceAccountToken: + description: serviceAccountToken is information + about the serviceAccountToken data to + project + properties: + audience: + description: |- + audience is the intended audience of the token. A recipient of a token + must identify itself with an identifier specified in the audience of the + token, and otherwise should reject the token. The audience defaults to the + identifier of the apiserver. + type: string + expirationSeconds: + description: |- + expirationSeconds is the requested duration of validity of the service + account token. As the token approaches expiration, the kubelet volume + plugin will proactively rotate the service account token. The kubelet will + start trying to rotate the token if the token is older than 80 percent of + its time to live or if the token is older than 24 hours.Defaults to 1 hour + and must be at least 10 minutes. + format: int64 + type: integer + path: + description: |- + path is the path relative to the mount point of the file to project the + token into. + type: string + required: + - path + type: object + type: object + type: array + x-kubernetes-list-type: atomic + type: object + quobyte: + description: quobyte represents a Quobyte mount + on the host that shares a pod's lifetime + properties: + group: + description: |- + group to map volume access to + Default is no group + type: string + readOnly: + description: |- + readOnly here will force the Quobyte volume to be mounted with read-only permissions. + Defaults to false. + type: boolean + registry: + description: |- + registry represents a single or multiple Quobyte Registry services + specified as a string as host:port pair (multiple entries are separated with commas) + which acts as the central registry for volumes + type: string + tenant: + description: |- + tenant owning the given Quobyte volume in the Backend + Used with dynamically provisioned Quobyte volumes, value is set by the plugin + type: string + user: + description: |- + user to map volume access to + Defaults to serivceaccount user + type: string + volume: + description: volume is a string that references + an already created Quobyte volume by name. + type: string + required: + - registry + - volume + type: object + rbd: + description: |- + rbd represents a Rados Block Device mount on the host that shares a pod's lifetime. + More info: https://examples.k8s.io/volumes/rbd/README.md + properties: + fsType: + description: |- + fsType is the filesystem type of the volume that you want to mount. + Tip: Ensure that the filesystem type is supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://kubernetes.io/docs/concepts/storage/volumes#rbd + type: string + image: + description: |- + image is the rados image name. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: string + keyring: + default: /etc/ceph/keyring + description: |- + keyring is the path to key ring for RBDUser. + Default is /etc/ceph/keyring. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: string + monitors: + description: |- + monitors is a collection of Ceph monitors. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + items: + type: string + type: array + x-kubernetes-list-type: atomic + pool: + default: rbd + description: |- + pool is the rados pool name. + Default is rbd. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: string + readOnly: + description: |- + readOnly here will force the ReadOnly setting in VolumeMounts. + Defaults to false. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: boolean + secretRef: + description: |- + secretRef is name of the authentication secret for RBDUser. If provided + overrides keyring. + Default is nil. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + user: + default: admin + description: |- + user is the rados user name. + Default is admin. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: string + required: + - image + - monitors + type: object + scaleIO: + description: scaleIO represents a ScaleIO persistent + volume attached and mounted on Kubernetes nodes. + properties: + fsType: + default: xfs + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". + Default is "xfs". + type: string + gateway: + description: gateway is the host address of + the ScaleIO API Gateway. + type: string + protectionDomain: + description: protectionDomain is the name of + the ScaleIO Protection Domain for the configured + storage. + type: string + readOnly: + description: |- + readOnly Defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + secretRef: + description: |- + secretRef references to the secret for ScaleIO user and other + sensitive information. If this is not provided, Login operation will fail. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + sslEnabled: + description: sslEnabled Flag enable/disable + SSL communication with Gateway, default false + type: boolean + storageMode: + default: ThinProvisioned + description: |- + storageMode indicates whether the storage for a volume should be ThickProvisioned or ThinProvisioned. + Default is ThinProvisioned. + type: string + storagePool: + description: storagePool is the ScaleIO Storage + Pool associated with the protection domain. + type: string + system: + description: system is the name of the storage + system as configured in ScaleIO. + type: string + volumeName: + description: |- + volumeName is the name of a volume already created in the ScaleIO system + that is associated with this volume source. + type: string + required: + - gateway + - secretRef + - system + type: object + secret: + description: |- + secret represents a secret that should populate this volume. + More info: https://kubernetes.io/docs/concepts/storage/volumes#secret + properties: + defaultMode: + description: |- + defaultMode is Optional: mode bits used to set permissions on created files by default. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values + for mode bits. Defaults to 0644. + Directories within the path are not affected by this setting. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + items: + description: |- + items If unspecified, each key-value pair in the Data field of the referenced + Secret will be projected into the volume as a file whose name is the + key and content is the value. If specified, the listed keys will be + projected into the specified paths, and unlisted keys will not be + present. If a key is specified which is not present in the Secret, + the volume setup will error unless it is marked optional. Paths must be + relative and may not contain the '..' path or start with '..'. + items: + description: Maps a string key to a path within + a volume. + properties: + key: + description: key is the key to project. + type: string + mode: + description: |- + mode is Optional: mode bits used to set permissions on this file. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: |- + path is the relative path of the file to map the key to. + May not be an absolute path. + May not contain the path element '..'. + May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + optional: + description: optional field specify whether + the Secret or its keys must be defined + type: boolean + secretName: + description: |- + secretName is the name of the secret in the pod's namespace to use. + More info: https://kubernetes.io/docs/concepts/storage/volumes#secret + type: string + type: object + storageos: + description: storageOS represents a StorageOS volume + attached and mounted on Kubernetes nodes. + properties: + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + readOnly: + description: |- + readOnly defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + secretRef: + description: |- + secretRef specifies the secret to use for obtaining the StorageOS API + credentials. If not specified, default values will be attempted. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + volumeName: + description: |- + volumeName is the human-readable name of the StorageOS volume. Volume + names are only unique within a namespace. + type: string + volumeNamespace: + description: |- + volumeNamespace specifies the scope of the volume within StorageOS. If no + namespace is specified then the Pod's namespace will be used. This allows the + Kubernetes name scoping to be mirrored within StorageOS for tighter integration. + Set VolumeName to any name to override the default behaviour. + Set to "default" if you are not using namespaces within StorageOS. + Namespaces that do not pre-exist within StorageOS will be created. + type: string + type: object + vsphereVolume: + description: vsphereVolume represents a vSphere + volume attached and mounted on kubelets host machine + properties: + fsType: + description: |- + fsType is filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + storagePolicyID: + description: storagePolicyID is the storage + Policy Based Management (SPBM) profile ID + associated with the StoragePolicyName. + type: string + storagePolicyName: + description: storagePolicyName is the storage + Policy Based Management (SPBM) profile name. + type: string + volumePath: + description: volumePath is the path that identifies + vSphere volume vmdk + type: string + required: + - volumePath + type: object + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + required: + - containers + type: object + type: object + ttlSecondsAfterFinished: + description: |- + ttlSecondsAfterFinished limits the lifetime of a Job that has finished + execution (either Complete or Failed). If this field is set, + ttlSecondsAfterFinished after the Job finishes, it is eligible to be + automatically deleted. When the Job is being deleted, its lifecycle + guarantees (e.g. finalizers) will be honored. If this field is unset, + the Job won't be automatically deleted. If this field is set to zero, + the Job becomes eligible to be deleted immediately after it finishes. + format: int32 + type: integer + required: + - template + type: object + required: + - spec + type: object + required: + - template + type: object + served: true + storage: true diff --git a/config/crd/bases/kjobctl.x-k8s.io_rayclustertemplates.yaml b/config/crd/bases/kjobctl.x-k8s.io_rayclustertemplates.yaml new file mode 100644 index 0000000..657bcf5 --- /dev/null +++ b/config/crd/bases/kjobctl.x-k8s.io_rayclustertemplates.yaml @@ -0,0 +1,17400 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.16.5 + name: rayclustertemplates.kjobctl.x-k8s.io +spec: + group: kjobctl.x-k8s.io + names: + kind: RayClusterTemplate + listKind: RayClusterTemplateList + plural: rayclustertemplates + singular: rayclustertemplate + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: RayClusterTemplate is the Schema for the rayclustertemplate API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + template: + description: Template defines rayclusters that will be created from this + raycluster template. + properties: + metadata: + description: Standard object's metadata. + type: object + spec: + description: Specification of the desired behavior of the raycluster. + properties: + autoscalerOptions: + description: AutoscalerOptions specifies optional configuration + for the Ray autoscaler. + properties: + env: + description: Optional list of environment variables to set + in the autoscaler container. + items: + description: EnvVar represents an environment variable present + in a Container. + properties: + name: + description: Name of the environment variable. Must + be a C_IDENTIFIER. + type: string + value: + description: |- + Variable references $(VAR_NAME) are expanded + using the previously defined environment variables in the container and + any service environment variables. If a variable cannot be resolved, + the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. + "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless of whether the variable + exists or not. + Defaults to "". + type: string + valueFrom: + description: Source for the environment variable's value. + Cannot be used if value is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the ConfigMap or + its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + description: |- + Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, + spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. + properties: + apiVersion: + description: Version of the schema the FieldPath + is written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select in + the specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. + properties: + containerName: + description: 'Container name: required for volumes, + optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format of + the exposed resources, defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + description: Selects a key of a secret in the pod's + namespace + properties: + key: + description: The key of the secret to select + from. Must be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret or its + key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + envFrom: + description: Optional list of sources to populate environment + variables in the autoscaler container. + items: + description: EnvFromSource represents the source of a set + of ConfigMaps + properties: + configMapRef: + description: The ConfigMap to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the ConfigMap must + be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + description: An optional identifier to prepend to each + key in the ConfigMap. Must be a C_IDENTIFIER. + type: string + secretRef: + description: The Secret to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret must be + defined + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array + idleTimeoutSeconds: + description: |- + IdleTimeoutSeconds is the number of seconds to wait before scaling down a worker pod which is not using Ray resources. + Defaults to 60 (one minute). It is not read by the KubeRay operator but by the Ray autoscaler. + format: int32 + type: integer + image: + description: Image optionally overrides the autoscaler's container + image. This override is for provided for autoscaler testing + and development. + type: string + imagePullPolicy: + description: ImagePullPolicy optionally overrides the autoscaler + container's image pull policy. This override is for provided + for autoscaler testing and development. + type: string + resources: + description: |- + Resources specifies optional resource request and limit overrides for the autoscaler container. + Default values: 500m CPU request and limit. 512Mi memory request and limit. + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + securityContext: + description: |- + SecurityContext defines the security options the container should be run with. + If set, the fields of SecurityContext override the equivalent fields of PodSecurityContext. + More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ + properties: + allowPrivilegeEscalation: + description: |- + AllowPrivilegeEscalation controls whether a process can gain more + privileges than its parent process. This bool directly controls if + the no_new_privs flag will be set on the container process. + AllowPrivilegeEscalation is true always when the container is: + 1) run as Privileged + 2) has CAP_SYS_ADMIN + Note that this field cannot be set when spec.os.name is windows. + type: boolean + appArmorProfile: + description: |- + appArmorProfile is the AppArmor options to use by this container. If set, this profile + overrides the pod's appArmorProfile. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile loaded on the node that should be used. + The profile must be preconfigured on the node to work. + Must match the loaded name of the profile. + Must be set if and only if type is "Localhost". + type: string + type: + description: |- + type indicates which kind of AppArmor profile will be applied. + Valid options are: + Localhost - a profile pre-loaded on the node. + RuntimeDefault - the container runtime's default profile. + Unconfined - no AppArmor enforcement. + type: string + required: + - type + type: object + capabilities: + description: |- + The capabilities to add/drop when running containers. + Defaults to the default set of capabilities granted by the container runtime. + Note that this field cannot be set when spec.os.name is windows. + properties: + add: + description: Added capabilities + items: + description: Capability represent POSIX capabilities + type + type: string + type: array + x-kubernetes-list-type: atomic + drop: + description: Removed capabilities + items: + description: Capability represent POSIX capabilities + type + type: string + type: array + x-kubernetes-list-type: atomic + type: object + privileged: + description: |- + Run container in privileged mode. + Processes in privileged containers are essentially equivalent to root on the host. + Defaults to false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + procMount: + description: |- + procMount denotes the type of proc mount to use for the containers. + The default value is Default which uses the container runtime defaults for + readonly paths and masked paths. + This requires the ProcMountType feature flag to be enabled. + Note that this field cannot be set when spec.os.name is windows. + type: string + readOnlyRootFilesystem: + description: |- + Whether this container has a read-only root filesystem. + Default is false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + runAsGroup: + description: |- + The GID to run the entrypoint of the container process. + Uses runtime default if unset. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + runAsNonRoot: + description: |- + Indicates that the container must run as a non-root user. + If true, the Kubelet will validate the image at runtime to ensure that it + does not run as UID 0 (root) and fail to start the container if it does. + If unset or false, no such validation will be performed. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: |- + The UID to run the entrypoint of the container process. + Defaults to user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + seLinuxOptions: + description: |- + The SELinux context to be applied to the container. + If unspecified, the container runtime will allocate a random SELinux context for each + container. May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + properties: + level: + description: Level is SELinux level label that applies + to the container. + type: string + role: + description: Role is a SELinux role label that applies + to the container. + type: string + type: + description: Type is a SELinux type label that applies + to the container. + type: string + user: + description: User is a SELinux user label that applies + to the container. + type: string + type: object + seccompProfile: + description: |- + The seccomp options to use by this container. If seccomp options are + provided at both the pod & container level, the container options + override the pod options. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile defined in a file on the node should be used. + The profile must be preconfigured on the node to work. + Must be a descending path, relative to the kubelet's configured seccomp profile location. + Must be set if type is "Localhost". Must NOT be set for any other type. + type: string + type: + description: |- + type indicates which kind of seccomp profile will be applied. + Valid options are: + + Localhost - a profile defined in a file on the node should be used. + RuntimeDefault - the container runtime default profile should be used. + Unconfined - no profile should be applied. + type: string + required: + - type + type: object + windowsOptions: + description: |- + The Windows specific settings applied to all containers. + If unspecified, the options from the PodSecurityContext will be used. + If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is linux. + properties: + gmsaCredentialSpec: + description: |- + GMSACredentialSpec is where the GMSA admission webhook + (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the + GMSA credential spec named by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName is the name of + the GMSA credential spec to use. + type: string + hostProcess: + description: |- + HostProcess determines if a container should be run as a 'Host Process' container. + All of a Pod's containers must have the same effective HostProcess value + (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). + In addition, if HostProcess is true then HostNetwork must also be set to true. + type: boolean + runAsUserName: + description: |- + The UserName in Windows to run the entrypoint of the container process. + Defaults to the user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: string + type: object + type: object + upscalingMode: + description: |- + UpscalingMode is "Conservative", "Default", or "Aggressive." + Conservative: Upscaling is rate-limited; the number of pending worker pods is at most the size of the Ray cluster. + Default: Upscaling is not rate-limited. + Aggressive: An alias for Default; upscaling is not rate-limited. + It is not read by the KubeRay operator but by the Ray autoscaler. + enum: + - Default + - Aggressive + - Conservative + type: string + volumeMounts: + description: Optional list of volumeMounts. This is needed + for enabling TLS for the autoscaler container. + items: + description: VolumeMount describes a mounting of a Volume + within a container. + properties: + mountPath: + description: |- + Path within the container at which the volume should be mounted. Must + not contain ':'. + type: string + mountPropagation: + description: |- + mountPropagation determines how mounts are propagated from the host + to container and the other way around. + When not set, MountPropagationNone is used. + This field is beta in 1.10. + When RecursiveReadOnly is set to IfPossible or to Enabled, MountPropagation must be None or unspecified + (which defaults to None). + type: string + name: + description: This must match the Name of a Volume. + type: string + readOnly: + description: |- + Mounted read-only if true, read-write otherwise (false or unspecified). + Defaults to false. + type: boolean + recursiveReadOnly: + description: |- + RecursiveReadOnly specifies whether read-only mounts should be handled + recursively. + + If ReadOnly is false, this field has no meaning and must be unspecified. + + If ReadOnly is true, and this field is set to Disabled, the mount is not made + recursively read-only. If this field is set to IfPossible, the mount is made + recursively read-only, if it is supported by the container runtime. If this + field is set to Enabled, the mount is made recursively read-only if it is + supported by the container runtime, otherwise the pod will not be started and + an error will be generated to indicate the reason. + + If this field is set to IfPossible or Enabled, MountPropagation must be set to + None (or be unspecified, which defaults to None). + + If this field is not specified, it is treated as an equivalent of Disabled. + type: string + subPath: + description: |- + Path within the volume from which the container's volume should be mounted. + Defaults to "" (volume's root). + type: string + subPathExpr: + description: |- + Expanded path within the volume from which the container's volume should be mounted. + Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. + Defaults to "" (volume's root). + SubPathExpr and SubPath are mutually exclusive. + type: string + required: + - mountPath + - name + type: object + type: array + type: object + enableInTreeAutoscaling: + description: EnableInTreeAutoscaling indicates whether operator + should create in tree autoscaling configs + type: boolean + headGroupSpec: + description: |- + INSERT ADDITIONAL SPEC FIELDS - desired state of cluster + Important: Run "make" to regenerate code after modifying this file + HeadGroupSpecs are the spec for the head pod + properties: + enableIngress: + description: EnableIngress indicates whether operator should + create ingress object for head service or not. + type: boolean + headService: + description: HeadService is the Kubernetes service of the + head pod. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + description: |- + Standard object's metadata. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata + type: object + spec: + description: |- + Spec defines the behavior of a service. + https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status + properties: + allocateLoadBalancerNodePorts: + description: |- + allocateLoadBalancerNodePorts defines if NodePorts will be automatically + allocated for services with type LoadBalancer. Default is "true". It + may be set to "false" if the cluster load-balancer does not rely on + NodePorts. If the caller requests specific NodePorts (by specifying a + value), those requests will be respected, regardless of this field. + This field may only be set for services with type LoadBalancer and will + be cleared if the type is changed to any other type. + type: boolean + clusterIP: + description: |- + clusterIP is the IP address of the service and is usually assigned + randomly. If an address is specified manually, is in-range (as per + system configuration), and is not in use, it will be allocated to the + service; otherwise creation of the service will fail. This field may not + be changed through updates unless the type field is also being changed + to ExternalName (which requires this field to be blank) or the type + field is being changed from ExternalName (in which case this field may + optionally be specified, as describe above). Valid values are "None", + empty string (""), or a valid IP address. Setting this to "None" makes a + "headless service" (no virtual IP), which is useful when direct endpoint + connections are preferred and proxying is not required. Only applies to + types ClusterIP, NodePort, and LoadBalancer. If this field is specified + when creating a Service of type ExternalName, creation will fail. This + field will be wiped when updating a Service to type ExternalName. + More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies + type: string + clusterIPs: + description: |- + ClusterIPs is a list of IP addresses assigned to this service, and are + usually assigned randomly. If an address is specified manually, is + in-range (as per system configuration), and is not in use, it will be + allocated to the service; otherwise creation of the service will fail. + This field may not be changed through updates unless the type field is + also being changed to ExternalName (which requires this field to be + empty) or the type field is being changed from ExternalName (in which + case this field may optionally be specified, as describe above). Valid + values are "None", empty string (""), or a valid IP address. Setting + this to "None" makes a "headless service" (no virtual IP), which is + useful when direct endpoint connections are preferred and proxying is + not required. Only applies to types ClusterIP, NodePort, and + LoadBalancer. If this field is specified when creating a Service of type + ExternalName, creation will fail. This field will be wiped when updating + a Service to type ExternalName. If this field is not specified, it will + be initialized from the clusterIP field. If this field is specified, + clients must ensure that clusterIPs[0] and clusterIP have the same + value. + + This field may hold a maximum of two entries (dual-stack IPs, in either order). + These IPs must correspond to the values of the ipFamilies field. Both + clusterIPs and ipFamilies are governed by the ipFamilyPolicy field. + More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies + items: + type: string + type: array + x-kubernetes-list-type: atomic + externalIPs: + description: |- + externalIPs is a list of IP addresses for which nodes in the cluster + will also accept traffic for this service. These IPs are not managed by + Kubernetes. The user is responsible for ensuring that traffic arrives + at a node with this IP. A common example is external load-balancers + that are not part of the Kubernetes system. + items: + type: string + type: array + x-kubernetes-list-type: atomic + externalName: + description: |- + externalName is the external reference that discovery mechanisms will + return as an alias for this service (e.g. a DNS CNAME record). No + proxying will be involved. Must be a lowercase RFC-1123 hostname + (https://tools.ietf.org/html/rfc1123) and requires `type` to be "ExternalName". + type: string + externalTrafficPolicy: + description: |- + externalTrafficPolicy describes how nodes distribute service traffic they + receive on one of the Service's "externally-facing" addresses (NodePorts, + ExternalIPs, and LoadBalancer IPs). If set to "Local", the proxy will configure + the service in a way that assumes that external load balancers will take care + of balancing the service traffic between nodes, and so each node will deliver + traffic only to the node-local endpoints of the service, without masquerading + the client source IP. (Traffic mistakenly sent to a node with no endpoints will + be dropped.) The default value, "Cluster", uses the standard behavior of + routing to all endpoints evenly (possibly modified by topology and other + features). Note that traffic sent to an External IP or LoadBalancer IP from + within the cluster will always get "Cluster" semantics, but clients sending to + a NodePort from within the cluster may need to take traffic policy into account + when picking a node. + type: string + healthCheckNodePort: + description: |- + healthCheckNodePort specifies the healthcheck nodePort for the service. + This only applies when type is set to LoadBalancer and + externalTrafficPolicy is set to Local. If a value is specified, is + in-range, and is not in use, it will be used. If not specified, a value + will be automatically allocated. External systems (e.g. load-balancers) + can use this port to determine if a given node holds endpoints for this + service or not. If this field is specified when creating a Service + which does not need it, creation will fail. This field will be wiped + when updating a Service to no longer need it (e.g. changing type). + This field cannot be updated once set. + format: int32 + type: integer + internalTrafficPolicy: + description: |- + InternalTrafficPolicy describes how nodes distribute service traffic they + receive on the ClusterIP. If set to "Local", the proxy will assume that pods + only want to talk to endpoints of the service on the same node as the pod, + dropping the traffic if there are no local endpoints. The default value, + "Cluster", uses the standard behavior of routing to all endpoints evenly + (possibly modified by topology and other features). + type: string + ipFamilies: + description: |- + IPFamilies is a list of IP families (e.g. IPv4, IPv6) assigned to this + service. This field is usually assigned automatically based on cluster + configuration and the ipFamilyPolicy field. If this field is specified + manually, the requested family is available in the cluster, + and ipFamilyPolicy allows it, it will be used; otherwise creation of + the service will fail. This field is conditionally mutable: it allows + for adding or removing a secondary IP family, but it does not allow + changing the primary IP family of the Service. Valid values are "IPv4" + and "IPv6". This field only applies to Services of types ClusterIP, + NodePort, and LoadBalancer, and does apply to "headless" services. + This field will be wiped when updating a Service to type ExternalName. + + This field may hold a maximum of two entries (dual-stack families, in + either order). These families must correspond to the values of the + clusterIPs field, if specified. Both clusterIPs and ipFamilies are + governed by the ipFamilyPolicy field. + items: + description: |- + IPFamily represents the IP Family (IPv4 or IPv6). This type is used + to express the family of an IP expressed by a type (e.g. service.spec.ipFamilies). + type: string + type: array + x-kubernetes-list-type: atomic + ipFamilyPolicy: + description: |- + IPFamilyPolicy represents the dual-stack-ness requested or required by + this Service. If there is no value provided, then this field will be set + to SingleStack. Services can be "SingleStack" (a single IP family), + "PreferDualStack" (two IP families on dual-stack configured clusters or + a single IP family on single-stack clusters), or "RequireDualStack" + (two IP families on dual-stack configured clusters, otherwise fail). The + ipFamilies and clusterIPs fields depend on the value of this field. This + field will be wiped when updating a service to type ExternalName. + type: string + loadBalancerClass: + description: |- + loadBalancerClass is the class of the load balancer implementation this Service belongs to. + If specified, the value of this field must be a label-style identifier, with an optional prefix, + e.g. "internal-vip" or "example.com/internal-vip". Unprefixed names are reserved for end-users. + This field can only be set when the Service type is 'LoadBalancer'. If not set, the default load + balancer implementation is used, today this is typically done through the cloud provider integration, + but should apply for any default implementation. If set, it is assumed that a load balancer + implementation is watching for Services with a matching class. Any default load balancer + implementation (e.g. cloud providers) should ignore Services that set this field. + This field can only be set when creating or updating a Service to type 'LoadBalancer'. + Once set, it can not be changed. This field will be wiped when a service is updated to a non 'LoadBalancer' type. + type: string + loadBalancerIP: + description: |- + Only applies to Service Type: LoadBalancer. + This feature depends on whether the underlying cloud-provider supports specifying + the loadBalancerIP when a load balancer is created. + This field will be ignored if the cloud-provider does not support the feature. + Deprecated: This field was under-specified and its meaning varies across implementations. + Using it is non-portable and it may not support dual-stack. + Users are encouraged to use implementation-specific annotations when available. + type: string + loadBalancerSourceRanges: + description: |- + If specified and supported by the platform, this will restrict traffic through the cloud-provider + load-balancer will be restricted to the specified client IPs. This field will be ignored if the + cloud-provider does not support the feature." + More info: https://kubernetes.io/docs/tasks/access-application-cluster/create-external-load-balancer/ + items: + type: string + type: array + x-kubernetes-list-type: atomic + ports: + description: |- + The list of ports that are exposed by this service. + More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies + items: + description: ServicePort contains information on + service's port. + properties: + appProtocol: + description: |- + The application protocol for this port. + This is used as a hint for implementations to offer richer behavior for protocols that they understand. + This field follows standard Kubernetes label syntax. + Valid values are either: + + * Un-prefixed protocol names - reserved for IANA standard service names (as per + RFC-6335 and https://www.iana.org/assignments/service-names). + + * Kubernetes-defined prefixed names: + * 'kubernetes.io/h2c' - HTTP/2 prior knowledge over cleartext as described in https://www.rfc-editor.org/rfc/rfc9113.html#name-starting-http-2-with-prior- + * 'kubernetes.io/ws' - WebSocket over cleartext as described in https://www.rfc-editor.org/rfc/rfc6455 + * 'kubernetes.io/wss' - WebSocket over TLS as described in https://www.rfc-editor.org/rfc/rfc6455 + + * Other protocols should use implementation-defined prefixed names such as + mycompany.com/my-custom-protocol. + type: string + name: + description: |- + The name of this port within the service. This must be a DNS_LABEL. + All ports within a ServiceSpec must have unique names. When considering + the endpoints for a Service, this must match the 'name' field in the + EndpointPort. + Optional if only one ServicePort is defined on this service. + type: string + nodePort: + description: |- + The port on each node on which this service is exposed when type is + NodePort or LoadBalancer. Usually assigned by the system. If a value is + specified, in-range, and not in use it will be used, otherwise the + operation will fail. If not specified, a port will be allocated if this + Service requires one. If this field is specified when creating a + Service which does not need it, creation will fail. This field will be + wiped when updating a Service to no longer need it (e.g. changing type + from NodePort to ClusterIP). + More info: https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport + format: int32 + type: integer + port: + description: The port that will be exposed by + this service. + format: int32 + type: integer + protocol: + default: TCP + description: |- + The IP protocol for this port. Supports "TCP", "UDP", and "SCTP". + Default is TCP. + type: string + targetPort: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the pods targeted by the service. + Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. + If this is a string, it will be looked up as a named port in the + target Pod's container ports. If this is not specified, the value + of the 'port' field is used (an identity map). + This field is ignored for services with clusterIP=None, and should be + omitted or set equal to the 'port' field. + More info: https://kubernetes.io/docs/concepts/services-networking/service/#defining-a-service + x-kubernetes-int-or-string: true + required: + - port + type: object + type: array + x-kubernetes-list-map-keys: + - port + - protocol + x-kubernetes-list-type: map + publishNotReadyAddresses: + description: |- + publishNotReadyAddresses indicates that any agent which deals with endpoints for this + Service should disregard any indications of ready/not-ready. + The primary use case for setting this field is for a StatefulSet's Headless Service to + propagate SRV DNS records for its Pods for the purpose of peer discovery. + The Kubernetes controllers that generate Endpoints and EndpointSlice resources for + Services interpret this to mean that all endpoints are considered "ready" even if the + Pods themselves are not. Agents which consume only Kubernetes generated endpoints + through the Endpoints or EndpointSlice resources can safely assume this behavior. + type: boolean + selector: + additionalProperties: + type: string + description: |- + Route service traffic to pods with label keys and values matching this + selector. If empty or not present, the service is assumed to have an + external process managing its endpoints, which Kubernetes will not + modify. Only applies to types ClusterIP, NodePort, and LoadBalancer. + Ignored if type is ExternalName. + More info: https://kubernetes.io/docs/concepts/services-networking/service/ + type: object + x-kubernetes-map-type: atomic + sessionAffinity: + description: |- + Supports "ClientIP" and "None". Used to maintain session affinity. + Enable client IP based session affinity. + Must be ClientIP or None. + Defaults to None. + More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies + type: string + sessionAffinityConfig: + description: sessionAffinityConfig contains the configurations + of session affinity. + properties: + clientIP: + description: clientIP contains the configurations + of Client IP based session affinity. + properties: + timeoutSeconds: + description: |- + timeoutSeconds specifies the seconds of ClientIP type session sticky time. + The value must be >0 && <=86400(for 1 day) if ServiceAffinity == "ClientIP". + Default value is 10800(for 3 hours). + format: int32 + type: integer + type: object + type: object + trafficDistribution: + description: |- + TrafficDistribution offers a way to express preferences for how traffic is + distributed to Service endpoints. Implementations can use this field as a + hint, but are not required to guarantee strict adherence. If the field is + not set, the implementation will apply its default routing strategy. If set + to "PreferClose", implementations should prioritize endpoints that are + topologically close (e.g., same zone). + This is an alpha field and requires enabling ServiceTrafficDistribution feature. + type: string + type: + description: |- + type determines how the Service is exposed. Defaults to ClusterIP. Valid + options are ExternalName, ClusterIP, NodePort, and LoadBalancer. + "ClusterIP" allocates a cluster-internal IP address for load-balancing + to endpoints. Endpoints are determined by the selector or if that is not + specified, by manual construction of an Endpoints object or + EndpointSlice objects. If clusterIP is "None", no virtual IP is + allocated and the endpoints are published as a set of endpoints rather + than a virtual IP. + "NodePort" builds on ClusterIP and allocates a port on every node which + routes to the same endpoints as the clusterIP. + "LoadBalancer" builds on NodePort and creates an external load-balancer + (if supported in the current cloud) which routes to the same endpoints + as the clusterIP. + "ExternalName" aliases this service to the specified externalName. + Several other fields do not apply to ExternalName services. + More info: https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types + type: string + type: object + status: + description: |- + Most recently observed status of the service. + Populated by the system. + Read-only. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status + properties: + conditions: + description: Current service state + items: + description: Condition contains details for one + aspect of the current state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of + True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase + or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + loadBalancer: + description: |- + LoadBalancer contains the current status of the load-balancer, + if one is present. + properties: + ingress: + description: |- + Ingress is a list containing ingress points for the load-balancer. + Traffic intended for the service should be sent to these ingress points. + items: + description: |- + LoadBalancerIngress represents the status of a load-balancer ingress point: + traffic intended for the service should be sent to an ingress point. + properties: + hostname: + description: |- + Hostname is set for load-balancer ingress points that are DNS based + (typically AWS load-balancers) + type: string + ip: + description: |- + IP is set for load-balancer ingress points that are IP based + (typically GCE or OpenStack load-balancers) + type: string + ipMode: + description: |- + IPMode specifies how the load-balancer IP behaves, and may only be specified when the ip field is specified. + Setting this to "VIP" indicates that traffic is delivered to the node with + the destination set to the load-balancer's IP and port. + Setting this to "Proxy" indicates that traffic is delivered to the node or pod with + the destination set to the node's IP and node port or the pod's IP and port. + Service implementations may use this information to adjust traffic routing. + type: string + ports: + description: |- + Ports is a list of records of service ports + If used, every port defined in the service should have an entry in it + items: + properties: + error: + description: |- + Error is to record the problem with the service port + The format of the error shall comply with the following rules: + - built-in error values shall be specified in this file and those shall use + CamelCase names + - cloud provider specific error values must have names that comply with the + format foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + port: + description: Port is the port number + of the service port of which status + is recorded here + format: int32 + type: integer + protocol: + description: |- + Protocol is the protocol of the service port of which status is recorded here + The supported values are: "TCP", "UDP", "SCTP" + type: string + required: + - error + - port + - protocol + type: object + type: array + x-kubernetes-list-type: atomic + type: object + type: array + x-kubernetes-list-type: atomic + type: object + type: object + type: object + rayStartParams: + additionalProperties: + type: string + description: 'RayStartParams are the params of the start command: + node-manager-port, object-store-memory, ...' + type: object + serviceType: + description: ServiceType is Kubernetes service type of the + head service. it will be used by the workers to connect + to the head pod + type: string + template: + description: Template is the exact pod template used in K8s + depoyments, statefulsets, etc. + properties: + metadata: + description: |- + Standard object's metadata. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata + type: object + spec: + description: |- + Specification of the desired behavior of the pod. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status + properties: + activeDeadlineSeconds: + description: |- + Optional duration in seconds the pod may be active on the node relative to + StartTime before the system will actively try to mark it failed and kill associated containers. + Value must be a positive integer. + format: int64 + type: integer + affinity: + description: If specified, the pod's scheduling constraints + properties: + nodeAffinity: + description: Describes node affinity scheduling + rules for the pod. + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node matches the corresponding matchExpressions; the + node(s) with the highest sum are the most preferred. + items: + description: |- + An empty preferred scheduling term matches all objects with implicit weight 0 + (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op). + properties: + preference: + description: A node selector term, associated + with the corresponding weight. + properties: + matchExpressions: + description: A list of node selector + requirements by node's labels. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key + that the selector applies + to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + description: A list of node selector + requirements by node's fields. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key + that the selector applies + to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + weight: + description: Weight associated with + matching the corresponding nodeSelectorTerm, + in the range 1-100. + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to an update), the system + may or may not try to eventually evict the pod from its node. + properties: + nodeSelectorTerms: + description: Required. A list of node + selector terms. The terms are ORed. + items: + description: |- + A null or empty node selector term matches no objects. The requirements of + them are ANDed. + The TopologySelectorTerm type implements a subset of the NodeSelectorTerm. + properties: + matchExpressions: + description: A list of node selector + requirements by node's labels. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key + that the selector applies + to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + description: A list of node selector + requirements by node's fields. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key + that the selector applies + to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + type: array + x-kubernetes-list-type: atomic + required: + - nodeSelectorTerms + type: object + x-kubernetes-map-type: atomic + type: object + podAffinity: + description: Describes pod affinity scheduling + rules (e.g. co-locate this pod in the same node, + zone, etc. as some other pod(s)). + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the + node(s) with the highest sum are the most preferred. + items: + description: The weights of all of the matched + WeightedPodAffinityTerm fields are added + per-node to find the most preferred node(s) + properties: + podAffinityTerm: + description: Required. A pod affinity + term, associated with the corresponding + weight. + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions + is a list of label selector + requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the + label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions + is a list of label selector + requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the + label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + weight: + description: |- + weight associated with matching the corresponding podAffinityTerm, + in the range 1-100. + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to a pod label update), the + system may or may not try to eventually evict the pod from its node. + When there are multiple elements, the lists of nodes corresponding to each + podAffinityTerm are intersected, i.e. all terms must be satisfied. + items: + description: |- + Defines a set of pods (namely those matching the labelSelector + relative to the given namespace(s)) that this pod should be + co-located (affinity) or not co-located (anti-affinity) with, + where co-located is defined as running on a node whose value of + the label with key matches that of any node on which + a pod of the set of pods is running + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions is + a list of label selector requirements. + The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is + a list of label selector requirements. + The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + podAntiAffinity: + description: Describes pod anti-affinity scheduling + rules (e.g. avoid putting this pod in the same + node, zone, etc. as some other pod(s)). + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the anti-affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling anti-affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the + node(s) with the highest sum are the most preferred. + items: + description: The weights of all of the matched + WeightedPodAffinityTerm fields are added + per-node to find the most preferred node(s) + properties: + podAffinityTerm: + description: Required. A pod affinity + term, associated with the corresponding + weight. + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions + is a list of label selector + requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the + label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions + is a list of label selector + requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the + label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + weight: + description: |- + weight associated with matching the corresponding podAffinityTerm, + in the range 1-100. + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the anti-affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the anti-affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to a pod label update), the + system may or may not try to eventually evict the pod from its node. + When there are multiple elements, the lists of nodes corresponding to each + podAffinityTerm are intersected, i.e. all terms must be satisfied. + items: + description: |- + Defines a set of pods (namely those matching the labelSelector + relative to the given namespace(s)) that this pod should be + co-located (affinity) or not co-located (anti-affinity) with, + where co-located is defined as running on a node whose value of + the label with key matches that of any node on which + a pod of the set of pods is running + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions is + a list of label selector requirements. + The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is + a list of label selector requirements. + The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + type: object + automountServiceAccountToken: + description: AutomountServiceAccountToken indicates + whether a service account token should be automatically + mounted. + type: boolean + containers: + description: |- + List of containers belonging to the pod. + Containers cannot currently be added or removed. + There must be at least one container in a Pod. + Cannot be updated. + items: + description: A single application container that + you want to run within a pod. + properties: + args: + description: |- + Arguments to the entrypoint. + The container image's CMD is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell + items: + type: string + type: array + x-kubernetes-list-type: atomic + command: + description: |- + Entrypoint array. Not executed within a shell. + The container image's ENTRYPOINT is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell + items: + type: string + type: array + x-kubernetes-list-type: atomic + env: + description: |- + List of environment variables to set in the container. + Cannot be updated. + items: + description: EnvVar represents an environment + variable present in a Container. + properties: + name: + description: Name of the environment variable. + Must be a C_IDENTIFIER. + type: string + value: + description: |- + Variable references $(VAR_NAME) are expanded + using the previously defined environment variables in the container and + any service environment variables. If a variable cannot be resolved, + the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. + "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless of whether the variable + exists or not. + Defaults to "". + type: string + valueFrom: + description: Source for the environment + variable's value. Cannot be used if + value is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the + ConfigMap or its key must be + defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + description: |- + Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, + spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. + properties: + apiVersion: + description: Version of the schema + the FieldPath is written in + terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field + to select in the specified API + version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. + properties: + containerName: + description: 'Container name: + required for volumes, optional + for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output + format of the exposed resources, + defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource + to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + description: Selects a key of a secret + in the pod's namespace + properties: + key: + description: The key of the secret + to select from. Must be a valid + secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the + Secret or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + envFrom: + description: |- + List of sources to populate environment variables in the container. + The keys defined within a source must be a C_IDENTIFIER. All invalid keys + will be reported as an event when the container is starting. When a key exists in multiple + sources, the value associated with the last source will take precedence. + Values defined by an Env with a duplicate key will take precedence. + Cannot be updated. + items: + description: EnvFromSource represents the + source of a set of ConfigMaps + properties: + configMapRef: + description: The ConfigMap to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the ConfigMap + must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + description: An optional identifier to + prepend to each key in the ConfigMap. + Must be a C_IDENTIFIER. + type: string + secretRef: + description: The Secret to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret + must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array + x-kubernetes-list-type: atomic + image: + description: |- + Container image name. + More info: https://kubernetes.io/docs/concepts/containers/images + This field is optional to allow higher level config management to default or override + container images in workload controllers like Deployments and StatefulSets. + type: string + imagePullPolicy: + description: |- + Image pull policy. + One of Always, Never, IfNotPresent. + Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/containers/images#updating-images + type: string + lifecycle: + description: |- + Actions that the management system should take in response to container lifecycle events. + Cannot be updated. + properties: + postStart: + description: |- + PostStart is called immediately after a container is created. If the handler fails, + the container is terminated and restarted according to its restart policy. + Other management of the container blocks until the hook completes. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies the action + to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies the http + request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set + in the request. HTTP allows repeated + headers. + items: + description: HTTPHeader describes + a custom header to be used in + HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field + value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the + HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents the duration + that the container should sleep before + being terminated. + properties: + seconds: + description: Seconds is the number + of seconds to sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for the backward compatibility. There are no validation of this field and + lifecycle hooks will fail in runtime when tcp handler is specified. + properties: + host: + description: 'Optional: Host name + to connect to, defaults to the + pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + preStop: + description: |- + PreStop is called immediately before a container is terminated due to an + API request or management event such as liveness/startup probe failure, + preemption, resource contention, etc. The handler is not called if the + container crashes or exits. The Pod's termination grace period countdown begins before the + PreStop hook is executed. Regardless of the outcome of the handler, the + container will eventually terminate within the Pod's termination grace + period (unless delayed by finalizers). Other management of the container blocks until the hook completes + or until the termination grace period is reached. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies the action + to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies the http + request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set + in the request. HTTP allows repeated + headers. + items: + description: HTTPHeader describes + a custom header to be used in + HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field + value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the + HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents the duration + that the container should sleep before + being terminated. + properties: + seconds: + description: Seconds is the number + of seconds to sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for the backward compatibility. There are no validation of this field and + lifecycle hooks will fail in runtime when tcp handler is specified. + properties: + host: + description: 'Optional: Host name + to connect to, defaults to the + pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + type: object + livenessProbe: + description: |- + Periodic probe of container liveness. + Container will be restarted if the probe fails. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies the action to + take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving + a GRPC port. + properties: + port: + description: Port number of the gRPC + service. Number must be in the range + 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http + request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in + the request. HTTP allows repeated + headers. + items: + description: HTTPHeader describes + a custom header to be used in HTTP + probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field + value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP + server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action + involving a TCP port. + properties: + host: + description: 'Optional: Host name to + connect to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + name: + description: |- + Name of the container specified as a DNS_LABEL. + Each container in a pod must have a unique name (DNS_LABEL). + Cannot be updated. + type: string + ports: + description: |- + List of ports to expose from the container. Not specifying a port here + DOES NOT prevent that port from being exposed. Any port which is + listening on the default "0.0.0.0" address inside a container will be + accessible from the network. + Modifying this array with strategic merge patch may corrupt the data. + For more information See https://github.com/kubernetes/kubernetes/issues/108255. + Cannot be updated. + items: + description: ContainerPort represents a network + port in a single container. + properties: + containerPort: + description: |- + Number of port to expose on the pod's IP address. + This must be a valid port number, 0 < x < 65536. + format: int32 + type: integer + hostIP: + description: What host IP to bind the + external port to. + type: string + hostPort: + description: |- + Number of port to expose on the host. + If specified, this must be a valid port number, 0 < x < 65536. + If HostNetwork is specified, this must match ContainerPort. + Most containers do not need this. + format: int32 + type: integer + name: + description: |- + If specified, this must be an IANA_SVC_NAME and unique within the pod. Each + named port in a pod must have a unique name. Name for the port that can be + referred to by services. + type: string + protocol: + default: TCP + description: |- + Protocol for port. Must be UDP, TCP, or SCTP. + Defaults to "TCP". + type: string + required: + - containerPort + type: object + type: array + x-kubernetes-list-map-keys: + - containerPort + - protocol + x-kubernetes-list-type: map + readinessProbe: + description: |- + Periodic probe of container service readiness. + Container will be removed from service endpoints if the probe fails. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies the action to + take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving + a GRPC port. + properties: + port: + description: Port number of the gRPC + service. Number must be in the range + 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http + request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in + the request. HTTP allows repeated + headers. + items: + description: HTTPHeader describes + a custom header to be used in HTTP + probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field + value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP + server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action + involving a TCP port. + properties: + host: + description: 'Optional: Host name to + connect to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + resizePolicy: + description: Resources resize policy for the + container. + items: + description: ContainerResizePolicy represents + resource resize policy for the container. + properties: + resourceName: + description: |- + Name of the resource to which this resource resize policy applies. + Supported values: cpu, memory. + type: string + restartPolicy: + description: |- + Restart policy to apply when specified resource is resized. + If not specified, it defaults to NotRequired. + type: string + required: + - resourceName + - restartPolicy + type: object + type: array + x-kubernetes-list-type: atomic + resources: + description: |- + Compute Resources required by this container. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references + one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + restartPolicy: + description: |- + RestartPolicy defines the restart behavior of individual containers in a pod. + This field may only be set for init containers, and the only allowed value is "Always". + For non-init containers or when this field is not specified, + the restart behavior is defined by the Pod's restart policy and the container type. + Setting the RestartPolicy as "Always" for the init container will have the following effect: + this init container will be continually restarted on + exit until all regular containers have terminated. Once all regular + containers have completed, all init containers with restartPolicy "Always" + will be shut down. This lifecycle differs from normal init containers and + is often referred to as a "sidecar" container. Although this init + container still starts in the init container sequence, it does not wait + for the container to complete before proceeding to the next init + container. Instead, the next init container starts immediately after this + init container is started, or after any startupProbe has successfully + completed. + type: string + securityContext: + description: |- + SecurityContext defines the security options the container should be run with. + If set, the fields of SecurityContext override the equivalent fields of PodSecurityContext. + More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ + properties: + allowPrivilegeEscalation: + description: |- + AllowPrivilegeEscalation controls whether a process can gain more + privileges than its parent process. This bool directly controls if + the no_new_privs flag will be set on the container process. + AllowPrivilegeEscalation is true always when the container is: + 1) run as Privileged + 2) has CAP_SYS_ADMIN + Note that this field cannot be set when spec.os.name is windows. + type: boolean + appArmorProfile: + description: |- + appArmorProfile is the AppArmor options to use by this container. If set, this profile + overrides the pod's appArmorProfile. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile loaded on the node that should be used. + The profile must be preconfigured on the node to work. + Must match the loaded name of the profile. + Must be set if and only if type is "Localhost". + type: string + type: + description: |- + type indicates which kind of AppArmor profile will be applied. + Valid options are: + Localhost - a profile pre-loaded on the node. + RuntimeDefault - the container runtime's default profile. + Unconfined - no AppArmor enforcement. + type: string + required: + - type + type: object + capabilities: + description: |- + The capabilities to add/drop when running containers. + Defaults to the default set of capabilities granted by the container runtime. + Note that this field cannot be set when spec.os.name is windows. + properties: + add: + description: Added capabilities + items: + description: Capability represent + POSIX capabilities type + type: string + type: array + x-kubernetes-list-type: atomic + drop: + description: Removed capabilities + items: + description: Capability represent + POSIX capabilities type + type: string + type: array + x-kubernetes-list-type: atomic + type: object + privileged: + description: |- + Run container in privileged mode. + Processes in privileged containers are essentially equivalent to root on the host. + Defaults to false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + procMount: + description: |- + procMount denotes the type of proc mount to use for the containers. + The default value is Default which uses the container runtime defaults for + readonly paths and masked paths. + This requires the ProcMountType feature flag to be enabled. + Note that this field cannot be set when spec.os.name is windows. + type: string + readOnlyRootFilesystem: + description: |- + Whether this container has a read-only root filesystem. + Default is false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + runAsGroup: + description: |- + The GID to run the entrypoint of the container process. + Uses runtime default if unset. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + runAsNonRoot: + description: |- + Indicates that the container must run as a non-root user. + If true, the Kubelet will validate the image at runtime to ensure that it + does not run as UID 0 (root) and fail to start the container if it does. + If unset or false, no such validation will be performed. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: |- + The UID to run the entrypoint of the container process. + Defaults to user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + seLinuxOptions: + description: |- + The SELinux context to be applied to the container. + If unspecified, the container runtime will allocate a random SELinux context for each + container. May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + properties: + level: + description: Level is SELinux level + label that applies to the container. + type: string + role: + description: Role is a SELinux role + label that applies to the container. + type: string + type: + description: Type is a SELinux type + label that applies to the container. + type: string + user: + description: User is a SELinux user + label that applies to the container. + type: string + type: object + seccompProfile: + description: |- + The seccomp options to use by this container. If seccomp options are + provided at both the pod & container level, the container options + override the pod options. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile defined in a file on the node should be used. + The profile must be preconfigured on the node to work. + Must be a descending path, relative to the kubelet's configured seccomp profile location. + Must be set if type is "Localhost". Must NOT be set for any other type. + type: string + type: + description: |- + type indicates which kind of seccomp profile will be applied. + Valid options are: + + Localhost - a profile defined in a file on the node should be used. + RuntimeDefault - the container runtime default profile should be used. + Unconfined - no profile should be applied. + type: string + required: + - type + type: object + windowsOptions: + description: |- + The Windows specific settings applied to all containers. + If unspecified, the options from the PodSecurityContext will be used. + If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is linux. + properties: + gmsaCredentialSpec: + description: |- + GMSACredentialSpec is where the GMSA admission webhook + (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the + GMSA credential spec named by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName + is the name of the GMSA credential + spec to use. + type: string + hostProcess: + description: |- + HostProcess determines if a container should be run as a 'Host Process' container. + All of a Pod's containers must have the same effective HostProcess value + (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). + In addition, if HostProcess is true then HostNetwork must also be set to true. + type: boolean + runAsUserName: + description: |- + The UserName in Windows to run the entrypoint of the container process. + Defaults to the user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: string + type: object + type: object + startupProbe: + description: |- + StartupProbe indicates that the Pod has successfully initialized. + If specified, no other probes are executed until this completes successfully. + If this probe fails, the Pod will be restarted, just as if the livenessProbe failed. + This can be used to provide different probe parameters at the beginning of a Pod's lifecycle, + when it might take a long time to load data or warm a cache, than during steady-state operation. + This cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies the action to + take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving + a GRPC port. + properties: + port: + description: Port number of the gRPC + service. Number must be in the range + 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http + request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in + the request. HTTP allows repeated + headers. + items: + description: HTTPHeader describes + a custom header to be used in HTTP + probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field + value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP + server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action + involving a TCP port. + properties: + host: + description: 'Optional: Host name to + connect to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + stdin: + description: |- + Whether this container should allocate a buffer for stdin in the container runtime. If this + is not set, reads from stdin in the container will always result in EOF. + Default is false. + type: boolean + stdinOnce: + description: |- + Whether the container runtime should close the stdin channel after it has been opened by + a single attach. When stdin is true the stdin stream will remain open across multiple attach + sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the + first client attaches to stdin, and then remains open and accepts data until the client disconnects, + at which time stdin is closed and remains closed until the container is restarted. If this + flag is false, a container processes that reads from stdin will never receive an EOF. + Default is false + type: boolean + terminationMessagePath: + description: |- + Optional: Path at which the file to which the container's termination message + will be written is mounted into the container's filesystem. + Message written is intended to be brief final status, such as an assertion failure message. + Will be truncated by the node if greater than 4096 bytes. The total message length across + all containers will be limited to 12kb. + Defaults to /dev/termination-log. + Cannot be updated. + type: string + terminationMessagePolicy: + description: |- + Indicate how the termination message should be populated. File will use the contents of + terminationMessagePath to populate the container status message on both success and failure. + FallbackToLogsOnError will use the last chunk of container log output if the termination + message file is empty and the container exited with an error. + The log output is limited to 2048 bytes or 80 lines, whichever is smaller. + Defaults to File. + Cannot be updated. + type: string + tty: + description: |- + Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. + Default is false. + type: boolean + volumeDevices: + description: volumeDevices is the list of block + devices to be used by the container. + items: + description: volumeDevice describes a mapping + of a raw block device within a container. + properties: + devicePath: + description: devicePath is the path inside + of the container that the device will + be mapped to. + type: string + name: + description: name must match the name + of a persistentVolumeClaim in the pod + type: string + required: + - devicePath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - devicePath + x-kubernetes-list-type: map + volumeMounts: + description: |- + Pod volumes to mount into the container's filesystem. + Cannot be updated. + items: + description: VolumeMount describes a mounting + of a Volume within a container. + properties: + mountPath: + description: |- + Path within the container at which the volume should be mounted. Must + not contain ':'. + type: string + mountPropagation: + description: |- + mountPropagation determines how mounts are propagated from the host + to container and the other way around. + When not set, MountPropagationNone is used. + This field is beta in 1.10. + When RecursiveReadOnly is set to IfPossible or to Enabled, MountPropagation must be None or unspecified + (which defaults to None). + type: string + name: + description: This must match the Name + of a Volume. + type: string + readOnly: + description: |- + Mounted read-only if true, read-write otherwise (false or unspecified). + Defaults to false. + type: boolean + recursiveReadOnly: + description: |- + RecursiveReadOnly specifies whether read-only mounts should be handled + recursively. + + If ReadOnly is false, this field has no meaning and must be unspecified. + + If ReadOnly is true, and this field is set to Disabled, the mount is not made + recursively read-only. If this field is set to IfPossible, the mount is made + recursively read-only, if it is supported by the container runtime. If this + field is set to Enabled, the mount is made recursively read-only if it is + supported by the container runtime, otherwise the pod will not be started and + an error will be generated to indicate the reason. + + If this field is set to IfPossible or Enabled, MountPropagation must be set to + None (or be unspecified, which defaults to None). + + If this field is not specified, it is treated as an equivalent of Disabled. + type: string + subPath: + description: |- + Path within the volume from which the container's volume should be mounted. + Defaults to "" (volume's root). + type: string + subPathExpr: + description: |- + Expanded path within the volume from which the container's volume should be mounted. + Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. + Defaults to "" (volume's root). + SubPathExpr and SubPath are mutually exclusive. + type: string + required: + - mountPath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - mountPath + x-kubernetes-list-type: map + workingDir: + description: |- + Container's working directory. + If not specified, the container runtime's default will be used, which + might be configured in the container image. + Cannot be updated. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + dnsConfig: + description: |- + Specifies the DNS parameters of a pod. + Parameters specified here will be merged to the generated DNS + configuration based on DNSPolicy. + properties: + nameservers: + description: |- + A list of DNS name server IP addresses. + This will be appended to the base nameservers generated from DNSPolicy. + Duplicated nameservers will be removed. + items: + type: string + type: array + x-kubernetes-list-type: atomic + options: + description: |- + A list of DNS resolver options. + This will be merged with the base options generated from DNSPolicy. + Duplicated entries will be removed. Resolution options given in Options + will override those that appear in the base DNSPolicy. + items: + description: PodDNSConfigOption defines DNS + resolver options of a pod. + properties: + name: + description: Required. + type: string + value: + type: string + type: object + type: array + x-kubernetes-list-type: atomic + searches: + description: |- + A list of DNS search domains for host-name lookup. + This will be appended to the base search paths generated from DNSPolicy. + Duplicated search paths will be removed. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + dnsPolicy: + description: |- + Set DNS policy for the pod. + Defaults to "ClusterFirst". + Valid values are 'ClusterFirstWithHostNet', 'ClusterFirst', 'Default' or 'None'. + DNS parameters given in DNSConfig will be merged with the policy selected with DNSPolicy. + To have DNS options set along with hostNetwork, you have to specify DNS policy + explicitly to 'ClusterFirstWithHostNet'. + type: string + enableServiceLinks: + description: |- + EnableServiceLinks indicates whether information about services should be injected into pod's + environment variables, matching the syntax of Docker links. + Optional: Defaults to true. + type: boolean + ephemeralContainers: + description: |- + List of ephemeral containers run in this pod. Ephemeral containers may be run in an existing + pod to perform user-initiated actions such as debugging. This list cannot be specified when + creating a pod, and it cannot be modified by updating the pod spec. In order to add an + ephemeral container to an existing pod, use the pod's ephemeralcontainers subresource. + items: + description: |- + An EphemeralContainer is a temporary container that you may add to an existing Pod for + user-initiated activities such as debugging. Ephemeral containers have no resource or + scheduling guarantees, and they will not be restarted when they exit or when a Pod is + removed or restarted. The kubelet may evict a Pod if an ephemeral container causes the + Pod to exceed its resource allocation. + + To add an ephemeral container, use the ephemeralcontainers subresource of an existing + Pod. Ephemeral containers may not be removed or restarted. + properties: + args: + description: |- + Arguments to the entrypoint. + The image's CMD is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell + items: + type: string + type: array + x-kubernetes-list-type: atomic + command: + description: |- + Entrypoint array. Not executed within a shell. + The image's ENTRYPOINT is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell + items: + type: string + type: array + x-kubernetes-list-type: atomic + env: + description: |- + List of environment variables to set in the container. + Cannot be updated. + items: + description: EnvVar represents an environment + variable present in a Container. + properties: + name: + description: Name of the environment variable. + Must be a C_IDENTIFIER. + type: string + value: + description: |- + Variable references $(VAR_NAME) are expanded + using the previously defined environment variables in the container and + any service environment variables. If a variable cannot be resolved, + the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. + "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless of whether the variable + exists or not. + Defaults to "". + type: string + valueFrom: + description: Source for the environment + variable's value. Cannot be used if + value is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the + ConfigMap or its key must be + defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + description: |- + Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, + spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. + properties: + apiVersion: + description: Version of the schema + the FieldPath is written in + terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field + to select in the specified API + version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. + properties: + containerName: + description: 'Container name: + required for volumes, optional + for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output + format of the exposed resources, + defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource + to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + description: Selects a key of a secret + in the pod's namespace + properties: + key: + description: The key of the secret + to select from. Must be a valid + secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the + Secret or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + envFrom: + description: |- + List of sources to populate environment variables in the container. + The keys defined within a source must be a C_IDENTIFIER. All invalid keys + will be reported as an event when the container is starting. When a key exists in multiple + sources, the value associated with the last source will take precedence. + Values defined by an Env with a duplicate key will take precedence. + Cannot be updated. + items: + description: EnvFromSource represents the + source of a set of ConfigMaps + properties: + configMapRef: + description: The ConfigMap to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the ConfigMap + must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + description: An optional identifier to + prepend to each key in the ConfigMap. + Must be a C_IDENTIFIER. + type: string + secretRef: + description: The Secret to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret + must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array + x-kubernetes-list-type: atomic + image: + description: |- + Container image name. + More info: https://kubernetes.io/docs/concepts/containers/images + type: string + imagePullPolicy: + description: |- + Image pull policy. + One of Always, Never, IfNotPresent. + Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/containers/images#updating-images + type: string + lifecycle: + description: Lifecycle is not allowed for ephemeral + containers. + properties: + postStart: + description: |- + PostStart is called immediately after a container is created. If the handler fails, + the container is terminated and restarted according to its restart policy. + Other management of the container blocks until the hook completes. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies the action + to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies the http + request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set + in the request. HTTP allows repeated + headers. + items: + description: HTTPHeader describes + a custom header to be used in + HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field + value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the + HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents the duration + that the container should sleep before + being terminated. + properties: + seconds: + description: Seconds is the number + of seconds to sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for the backward compatibility. There are no validation of this field and + lifecycle hooks will fail in runtime when tcp handler is specified. + properties: + host: + description: 'Optional: Host name + to connect to, defaults to the + pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + preStop: + description: |- + PreStop is called immediately before a container is terminated due to an + API request or management event such as liveness/startup probe failure, + preemption, resource contention, etc. The handler is not called if the + container crashes or exits. The Pod's termination grace period countdown begins before the + PreStop hook is executed. Regardless of the outcome of the handler, the + container will eventually terminate within the Pod's termination grace + period (unless delayed by finalizers). Other management of the container blocks until the hook completes + or until the termination grace period is reached. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies the action + to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies the http + request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set + in the request. HTTP allows repeated + headers. + items: + description: HTTPHeader describes + a custom header to be used in + HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field + value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the + HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents the duration + that the container should sleep before + being terminated. + properties: + seconds: + description: Seconds is the number + of seconds to sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for the backward compatibility. There are no validation of this field and + lifecycle hooks will fail in runtime when tcp handler is specified. + properties: + host: + description: 'Optional: Host name + to connect to, defaults to the + pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + type: object + livenessProbe: + description: Probes are not allowed for ephemeral + containers. + properties: + exec: + description: Exec specifies the action to + take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving + a GRPC port. + properties: + port: + description: Port number of the gRPC + service. Number must be in the range + 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http + request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in + the request. HTTP allows repeated + headers. + items: + description: HTTPHeader describes + a custom header to be used in HTTP + probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field + value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP + server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action + involving a TCP port. + properties: + host: + description: 'Optional: Host name to + connect to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + name: + description: |- + Name of the ephemeral container specified as a DNS_LABEL. + This name must be unique among all containers, init containers and ephemeral containers. + type: string + ports: + description: Ports are not allowed for ephemeral + containers. + items: + description: ContainerPort represents a network + port in a single container. + properties: + containerPort: + description: |- + Number of port to expose on the pod's IP address. + This must be a valid port number, 0 < x < 65536. + format: int32 + type: integer + hostIP: + description: What host IP to bind the + external port to. + type: string + hostPort: + description: |- + Number of port to expose on the host. + If specified, this must be a valid port number, 0 < x < 65536. + If HostNetwork is specified, this must match ContainerPort. + Most containers do not need this. + format: int32 + type: integer + name: + description: |- + If specified, this must be an IANA_SVC_NAME and unique within the pod. Each + named port in a pod must have a unique name. Name for the port that can be + referred to by services. + type: string + protocol: + default: TCP + description: |- + Protocol for port. Must be UDP, TCP, or SCTP. + Defaults to "TCP". + type: string + required: + - containerPort + type: object + type: array + x-kubernetes-list-map-keys: + - containerPort + - protocol + x-kubernetes-list-type: map + readinessProbe: + description: Probes are not allowed for ephemeral + containers. + properties: + exec: + description: Exec specifies the action to + take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving + a GRPC port. + properties: + port: + description: Port number of the gRPC + service. Number must be in the range + 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http + request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in + the request. HTTP allows repeated + headers. + items: + description: HTTPHeader describes + a custom header to be used in HTTP + probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field + value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP + server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action + involving a TCP port. + properties: + host: + description: 'Optional: Host name to + connect to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + resizePolicy: + description: Resources resize policy for the + container. + items: + description: ContainerResizePolicy represents + resource resize policy for the container. + properties: + resourceName: + description: |- + Name of the resource to which this resource resize policy applies. + Supported values: cpu, memory. + type: string + restartPolicy: + description: |- + Restart policy to apply when specified resource is resized. + If not specified, it defaults to NotRequired. + type: string + required: + - resourceName + - restartPolicy + type: object + type: array + x-kubernetes-list-type: atomic + resources: + description: |- + Resources are not allowed for ephemeral containers. Ephemeral containers use spare resources + already allocated to the pod. + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references + one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + restartPolicy: + description: |- + Restart policy for the container to manage the restart behavior of each + container within a pod. + This may only be set for init containers. You cannot set this field on + ephemeral containers. + type: string + securityContext: + description: |- + Optional: SecurityContext defines the security options the ephemeral container should be run with. + If set, the fields of SecurityContext override the equivalent fields of PodSecurityContext. + properties: + allowPrivilegeEscalation: + description: |- + AllowPrivilegeEscalation controls whether a process can gain more + privileges than its parent process. This bool directly controls if + the no_new_privs flag will be set on the container process. + AllowPrivilegeEscalation is true always when the container is: + 1) run as Privileged + 2) has CAP_SYS_ADMIN + Note that this field cannot be set when spec.os.name is windows. + type: boolean + appArmorProfile: + description: |- + appArmorProfile is the AppArmor options to use by this container. If set, this profile + overrides the pod's appArmorProfile. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile loaded on the node that should be used. + The profile must be preconfigured on the node to work. + Must match the loaded name of the profile. + Must be set if and only if type is "Localhost". + type: string + type: + description: |- + type indicates which kind of AppArmor profile will be applied. + Valid options are: + Localhost - a profile pre-loaded on the node. + RuntimeDefault - the container runtime's default profile. + Unconfined - no AppArmor enforcement. + type: string + required: + - type + type: object + capabilities: + description: |- + The capabilities to add/drop when running containers. + Defaults to the default set of capabilities granted by the container runtime. + Note that this field cannot be set when spec.os.name is windows. + properties: + add: + description: Added capabilities + items: + description: Capability represent + POSIX capabilities type + type: string + type: array + x-kubernetes-list-type: atomic + drop: + description: Removed capabilities + items: + description: Capability represent + POSIX capabilities type + type: string + type: array + x-kubernetes-list-type: atomic + type: object + privileged: + description: |- + Run container in privileged mode. + Processes in privileged containers are essentially equivalent to root on the host. + Defaults to false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + procMount: + description: |- + procMount denotes the type of proc mount to use for the containers. + The default value is Default which uses the container runtime defaults for + readonly paths and masked paths. + This requires the ProcMountType feature flag to be enabled. + Note that this field cannot be set when spec.os.name is windows. + type: string + readOnlyRootFilesystem: + description: |- + Whether this container has a read-only root filesystem. + Default is false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + runAsGroup: + description: |- + The GID to run the entrypoint of the container process. + Uses runtime default if unset. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + runAsNonRoot: + description: |- + Indicates that the container must run as a non-root user. + If true, the Kubelet will validate the image at runtime to ensure that it + does not run as UID 0 (root) and fail to start the container if it does. + If unset or false, no such validation will be performed. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: |- + The UID to run the entrypoint of the container process. + Defaults to user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + seLinuxOptions: + description: |- + The SELinux context to be applied to the container. + If unspecified, the container runtime will allocate a random SELinux context for each + container. May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + properties: + level: + description: Level is SELinux level + label that applies to the container. + type: string + role: + description: Role is a SELinux role + label that applies to the container. + type: string + type: + description: Type is a SELinux type + label that applies to the container. + type: string + user: + description: User is a SELinux user + label that applies to the container. + type: string + type: object + seccompProfile: + description: |- + The seccomp options to use by this container. If seccomp options are + provided at both the pod & container level, the container options + override the pod options. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile defined in a file on the node should be used. + The profile must be preconfigured on the node to work. + Must be a descending path, relative to the kubelet's configured seccomp profile location. + Must be set if type is "Localhost". Must NOT be set for any other type. + type: string + type: + description: |- + type indicates which kind of seccomp profile will be applied. + Valid options are: + + Localhost - a profile defined in a file on the node should be used. + RuntimeDefault - the container runtime default profile should be used. + Unconfined - no profile should be applied. + type: string + required: + - type + type: object + windowsOptions: + description: |- + The Windows specific settings applied to all containers. + If unspecified, the options from the PodSecurityContext will be used. + If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is linux. + properties: + gmsaCredentialSpec: + description: |- + GMSACredentialSpec is where the GMSA admission webhook + (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the + GMSA credential spec named by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName + is the name of the GMSA credential + spec to use. + type: string + hostProcess: + description: |- + HostProcess determines if a container should be run as a 'Host Process' container. + All of a Pod's containers must have the same effective HostProcess value + (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). + In addition, if HostProcess is true then HostNetwork must also be set to true. + type: boolean + runAsUserName: + description: |- + The UserName in Windows to run the entrypoint of the container process. + Defaults to the user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: string + type: object + type: object + startupProbe: + description: Probes are not allowed for ephemeral + containers. + properties: + exec: + description: Exec specifies the action to + take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving + a GRPC port. + properties: + port: + description: Port number of the gRPC + service. Number must be in the range + 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http + request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in + the request. HTTP allows repeated + headers. + items: + description: HTTPHeader describes + a custom header to be used in HTTP + probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field + value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP + server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action + involving a TCP port. + properties: + host: + description: 'Optional: Host name to + connect to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + stdin: + description: |- + Whether this container should allocate a buffer for stdin in the container runtime. If this + is not set, reads from stdin in the container will always result in EOF. + Default is false. + type: boolean + stdinOnce: + description: |- + Whether the container runtime should close the stdin channel after it has been opened by + a single attach. When stdin is true the stdin stream will remain open across multiple attach + sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the + first client attaches to stdin, and then remains open and accepts data until the client disconnects, + at which time stdin is closed and remains closed until the container is restarted. If this + flag is false, a container processes that reads from stdin will never receive an EOF. + Default is false + type: boolean + targetContainerName: + description: |- + If set, the name of the container from PodSpec that this ephemeral container targets. + The ephemeral container will be run in the namespaces (IPC, PID, etc) of this container. + If not set then the ephemeral container uses the namespaces configured in the Pod spec. + + The container runtime must implement support for this feature. If the runtime does not + support namespace targeting then the result of setting this field is undefined. + type: string + terminationMessagePath: + description: |- + Optional: Path at which the file to which the container's termination message + will be written is mounted into the container's filesystem. + Message written is intended to be brief final status, such as an assertion failure message. + Will be truncated by the node if greater than 4096 bytes. The total message length across + all containers will be limited to 12kb. + Defaults to /dev/termination-log. + Cannot be updated. + type: string + terminationMessagePolicy: + description: |- + Indicate how the termination message should be populated. File will use the contents of + terminationMessagePath to populate the container status message on both success and failure. + FallbackToLogsOnError will use the last chunk of container log output if the termination + message file is empty and the container exited with an error. + The log output is limited to 2048 bytes or 80 lines, whichever is smaller. + Defaults to File. + Cannot be updated. + type: string + tty: + description: |- + Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. + Default is false. + type: boolean + volumeDevices: + description: volumeDevices is the list of block + devices to be used by the container. + items: + description: volumeDevice describes a mapping + of a raw block device within a container. + properties: + devicePath: + description: devicePath is the path inside + of the container that the device will + be mapped to. + type: string + name: + description: name must match the name + of a persistentVolumeClaim in the pod + type: string + required: + - devicePath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - devicePath + x-kubernetes-list-type: map + volumeMounts: + description: |- + Pod volumes to mount into the container's filesystem. Subpath mounts are not allowed for ephemeral containers. + Cannot be updated. + items: + description: VolumeMount describes a mounting + of a Volume within a container. + properties: + mountPath: + description: |- + Path within the container at which the volume should be mounted. Must + not contain ':'. + type: string + mountPropagation: + description: |- + mountPropagation determines how mounts are propagated from the host + to container and the other way around. + When not set, MountPropagationNone is used. + This field is beta in 1.10. + When RecursiveReadOnly is set to IfPossible or to Enabled, MountPropagation must be None or unspecified + (which defaults to None). + type: string + name: + description: This must match the Name + of a Volume. + type: string + readOnly: + description: |- + Mounted read-only if true, read-write otherwise (false or unspecified). + Defaults to false. + type: boolean + recursiveReadOnly: + description: |- + RecursiveReadOnly specifies whether read-only mounts should be handled + recursively. + + If ReadOnly is false, this field has no meaning and must be unspecified. + + If ReadOnly is true, and this field is set to Disabled, the mount is not made + recursively read-only. If this field is set to IfPossible, the mount is made + recursively read-only, if it is supported by the container runtime. If this + field is set to Enabled, the mount is made recursively read-only if it is + supported by the container runtime, otherwise the pod will not be started and + an error will be generated to indicate the reason. + + If this field is set to IfPossible or Enabled, MountPropagation must be set to + None (or be unspecified, which defaults to None). + + If this field is not specified, it is treated as an equivalent of Disabled. + type: string + subPath: + description: |- + Path within the volume from which the container's volume should be mounted. + Defaults to "" (volume's root). + type: string + subPathExpr: + description: |- + Expanded path within the volume from which the container's volume should be mounted. + Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. + Defaults to "" (volume's root). + SubPathExpr and SubPath are mutually exclusive. + type: string + required: + - mountPath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - mountPath + x-kubernetes-list-type: map + workingDir: + description: |- + Container's working directory. + If not specified, the container runtime's default will be used, which + might be configured in the container image. + Cannot be updated. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + hostAliases: + description: |- + HostAliases is an optional list of hosts and IPs that will be injected into the pod's hosts + file if specified. + items: + description: |- + HostAlias holds the mapping between IP and hostnames that will be injected as an entry in the + pod's hosts file. + properties: + hostnames: + description: Hostnames for the above IP address. + items: + type: string + type: array + x-kubernetes-list-type: atomic + ip: + description: IP address of the host file entry. + type: string + required: + - ip + type: object + type: array + x-kubernetes-list-map-keys: + - ip + x-kubernetes-list-type: map + hostIPC: + description: |- + Use the host's ipc namespace. + Optional: Default to false. + type: boolean + hostNetwork: + description: |- + Host networking requested for this pod. Use the host's network namespace. + If this option is set, the ports that will be used must be specified. + Default to false. + type: boolean + hostPID: + description: |- + Use the host's pid namespace. + Optional: Default to false. + type: boolean + hostUsers: + description: |- + Use the host's user namespace. + Optional: Default to true. + If set to true or not present, the pod will be run in the host user namespace, useful + for when the pod needs a feature only available to the host user namespace, such as + loading a kernel module with CAP_SYS_MODULE. + When set to false, a new userns is created for the pod. Setting false is useful for + mitigating container breakout vulnerabilities even allowing users to run their + containers as root without actually having root privileges on the host. + This field is alpha-level and is only honored by servers that enable the UserNamespacesSupport feature. + type: boolean + hostname: + description: |- + Specifies the hostname of the Pod + If not specified, the pod's hostname will be set to a system-defined value. + type: string + imagePullSecrets: + description: |- + ImagePullSecrets is an optional list of references to secrets in the same namespace to use for pulling any of the images used by this PodSpec. + If specified, these secrets will be passed to individual puller implementations for them to use. + More info: https://kubernetes.io/docs/concepts/containers/images#specifying-imagepullsecrets-on-a-pod + items: + description: |- + LocalObjectReference contains enough information to let you locate the + referenced object inside the same namespace. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + initContainers: + description: |- + List of initialization containers belonging to the pod. + Init containers are executed in order prior to containers being started. If any + init container fails, the pod is considered to have failed and is handled according + to its restartPolicy. The name for an init container or normal container must be + unique among all containers. + Init containers may not have Lifecycle actions, Readiness probes, Liveness probes, or Startup probes. + The resourceRequirements of an init container are taken into account during scheduling + by finding the highest request/limit for each resource type, and then using the max of + of that value or the sum of the normal containers. Limits are applied to init containers + in a similar fashion. + Init containers cannot currently be added or removed. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/init-containers/ + items: + description: A single application container that + you want to run within a pod. + properties: + args: + description: |- + Arguments to the entrypoint. + The container image's CMD is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell + items: + type: string + type: array + x-kubernetes-list-type: atomic + command: + description: |- + Entrypoint array. Not executed within a shell. + The container image's ENTRYPOINT is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell + items: + type: string + type: array + x-kubernetes-list-type: atomic + env: + description: |- + List of environment variables to set in the container. + Cannot be updated. + items: + description: EnvVar represents an environment + variable present in a Container. + properties: + name: + description: Name of the environment variable. + Must be a C_IDENTIFIER. + type: string + value: + description: |- + Variable references $(VAR_NAME) are expanded + using the previously defined environment variables in the container and + any service environment variables. If a variable cannot be resolved, + the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. + "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless of whether the variable + exists or not. + Defaults to "". + type: string + valueFrom: + description: Source for the environment + variable's value. Cannot be used if + value is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the + ConfigMap or its key must be + defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + description: |- + Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, + spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. + properties: + apiVersion: + description: Version of the schema + the FieldPath is written in + terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field + to select in the specified API + version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. + properties: + containerName: + description: 'Container name: + required for volumes, optional + for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output + format of the exposed resources, + defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource + to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + description: Selects a key of a secret + in the pod's namespace + properties: + key: + description: The key of the secret + to select from. Must be a valid + secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the + Secret or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + envFrom: + description: |- + List of sources to populate environment variables in the container. + The keys defined within a source must be a C_IDENTIFIER. All invalid keys + will be reported as an event when the container is starting. When a key exists in multiple + sources, the value associated with the last source will take precedence. + Values defined by an Env with a duplicate key will take precedence. + Cannot be updated. + items: + description: EnvFromSource represents the + source of a set of ConfigMaps + properties: + configMapRef: + description: The ConfigMap to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the ConfigMap + must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + description: An optional identifier to + prepend to each key in the ConfigMap. + Must be a C_IDENTIFIER. + type: string + secretRef: + description: The Secret to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret + must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array + x-kubernetes-list-type: atomic + image: + description: |- + Container image name. + More info: https://kubernetes.io/docs/concepts/containers/images + This field is optional to allow higher level config management to default or override + container images in workload controllers like Deployments and StatefulSets. + type: string + imagePullPolicy: + description: |- + Image pull policy. + One of Always, Never, IfNotPresent. + Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/containers/images#updating-images + type: string + lifecycle: + description: |- + Actions that the management system should take in response to container lifecycle events. + Cannot be updated. + properties: + postStart: + description: |- + PostStart is called immediately after a container is created. If the handler fails, + the container is terminated and restarted according to its restart policy. + Other management of the container blocks until the hook completes. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies the action + to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies the http + request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set + in the request. HTTP allows repeated + headers. + items: + description: HTTPHeader describes + a custom header to be used in + HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field + value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the + HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents the duration + that the container should sleep before + being terminated. + properties: + seconds: + description: Seconds is the number + of seconds to sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for the backward compatibility. There are no validation of this field and + lifecycle hooks will fail in runtime when tcp handler is specified. + properties: + host: + description: 'Optional: Host name + to connect to, defaults to the + pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + preStop: + description: |- + PreStop is called immediately before a container is terminated due to an + API request or management event such as liveness/startup probe failure, + preemption, resource contention, etc. The handler is not called if the + container crashes or exits. The Pod's termination grace period countdown begins before the + PreStop hook is executed. Regardless of the outcome of the handler, the + container will eventually terminate within the Pod's termination grace + period (unless delayed by finalizers). Other management of the container blocks until the hook completes + or until the termination grace period is reached. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies the action + to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies the http + request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set + in the request. HTTP allows repeated + headers. + items: + description: HTTPHeader describes + a custom header to be used in + HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field + value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the + HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents the duration + that the container should sleep before + being terminated. + properties: + seconds: + description: Seconds is the number + of seconds to sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for the backward compatibility. There are no validation of this field and + lifecycle hooks will fail in runtime when tcp handler is specified. + properties: + host: + description: 'Optional: Host name + to connect to, defaults to the + pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + type: object + livenessProbe: + description: |- + Periodic probe of container liveness. + Container will be restarted if the probe fails. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies the action to + take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving + a GRPC port. + properties: + port: + description: Port number of the gRPC + service. Number must be in the range + 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http + request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in + the request. HTTP allows repeated + headers. + items: + description: HTTPHeader describes + a custom header to be used in HTTP + probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field + value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP + server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action + involving a TCP port. + properties: + host: + description: 'Optional: Host name to + connect to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + name: + description: |- + Name of the container specified as a DNS_LABEL. + Each container in a pod must have a unique name (DNS_LABEL). + Cannot be updated. + type: string + ports: + description: |- + List of ports to expose from the container. Not specifying a port here + DOES NOT prevent that port from being exposed. Any port which is + listening on the default "0.0.0.0" address inside a container will be + accessible from the network. + Modifying this array with strategic merge patch may corrupt the data. + For more information See https://github.com/kubernetes/kubernetes/issues/108255. + Cannot be updated. + items: + description: ContainerPort represents a network + port in a single container. + properties: + containerPort: + description: |- + Number of port to expose on the pod's IP address. + This must be a valid port number, 0 < x < 65536. + format: int32 + type: integer + hostIP: + description: What host IP to bind the + external port to. + type: string + hostPort: + description: |- + Number of port to expose on the host. + If specified, this must be a valid port number, 0 < x < 65536. + If HostNetwork is specified, this must match ContainerPort. + Most containers do not need this. + format: int32 + type: integer + name: + description: |- + If specified, this must be an IANA_SVC_NAME and unique within the pod. Each + named port in a pod must have a unique name. Name for the port that can be + referred to by services. + type: string + protocol: + default: TCP + description: |- + Protocol for port. Must be UDP, TCP, or SCTP. + Defaults to "TCP". + type: string + required: + - containerPort + type: object + type: array + x-kubernetes-list-map-keys: + - containerPort + - protocol + x-kubernetes-list-type: map + readinessProbe: + description: |- + Periodic probe of container service readiness. + Container will be removed from service endpoints if the probe fails. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies the action to + take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving + a GRPC port. + properties: + port: + description: Port number of the gRPC + service. Number must be in the range + 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http + request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in + the request. HTTP allows repeated + headers. + items: + description: HTTPHeader describes + a custom header to be used in HTTP + probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field + value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP + server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action + involving a TCP port. + properties: + host: + description: 'Optional: Host name to + connect to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + resizePolicy: + description: Resources resize policy for the + container. + items: + description: ContainerResizePolicy represents + resource resize policy for the container. + properties: + resourceName: + description: |- + Name of the resource to which this resource resize policy applies. + Supported values: cpu, memory. + type: string + restartPolicy: + description: |- + Restart policy to apply when specified resource is resized. + If not specified, it defaults to NotRequired. + type: string + required: + - resourceName + - restartPolicy + type: object + type: array + x-kubernetes-list-type: atomic + resources: + description: |- + Compute Resources required by this container. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references + one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + restartPolicy: + description: |- + RestartPolicy defines the restart behavior of individual containers in a pod. + This field may only be set for init containers, and the only allowed value is "Always". + For non-init containers or when this field is not specified, + the restart behavior is defined by the Pod's restart policy and the container type. + Setting the RestartPolicy as "Always" for the init container will have the following effect: + this init container will be continually restarted on + exit until all regular containers have terminated. Once all regular + containers have completed, all init containers with restartPolicy "Always" + will be shut down. This lifecycle differs from normal init containers and + is often referred to as a "sidecar" container. Although this init + container still starts in the init container sequence, it does not wait + for the container to complete before proceeding to the next init + container. Instead, the next init container starts immediately after this + init container is started, or after any startupProbe has successfully + completed. + type: string + securityContext: + description: |- + SecurityContext defines the security options the container should be run with. + If set, the fields of SecurityContext override the equivalent fields of PodSecurityContext. + More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ + properties: + allowPrivilegeEscalation: + description: |- + AllowPrivilegeEscalation controls whether a process can gain more + privileges than its parent process. This bool directly controls if + the no_new_privs flag will be set on the container process. + AllowPrivilegeEscalation is true always when the container is: + 1) run as Privileged + 2) has CAP_SYS_ADMIN + Note that this field cannot be set when spec.os.name is windows. + type: boolean + appArmorProfile: + description: |- + appArmorProfile is the AppArmor options to use by this container. If set, this profile + overrides the pod's appArmorProfile. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile loaded on the node that should be used. + The profile must be preconfigured on the node to work. + Must match the loaded name of the profile. + Must be set if and only if type is "Localhost". + type: string + type: + description: |- + type indicates which kind of AppArmor profile will be applied. + Valid options are: + Localhost - a profile pre-loaded on the node. + RuntimeDefault - the container runtime's default profile. + Unconfined - no AppArmor enforcement. + type: string + required: + - type + type: object + capabilities: + description: |- + The capabilities to add/drop when running containers. + Defaults to the default set of capabilities granted by the container runtime. + Note that this field cannot be set when spec.os.name is windows. + properties: + add: + description: Added capabilities + items: + description: Capability represent + POSIX capabilities type + type: string + type: array + x-kubernetes-list-type: atomic + drop: + description: Removed capabilities + items: + description: Capability represent + POSIX capabilities type + type: string + type: array + x-kubernetes-list-type: atomic + type: object + privileged: + description: |- + Run container in privileged mode. + Processes in privileged containers are essentially equivalent to root on the host. + Defaults to false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + procMount: + description: |- + procMount denotes the type of proc mount to use for the containers. + The default value is Default which uses the container runtime defaults for + readonly paths and masked paths. + This requires the ProcMountType feature flag to be enabled. + Note that this field cannot be set when spec.os.name is windows. + type: string + readOnlyRootFilesystem: + description: |- + Whether this container has a read-only root filesystem. + Default is false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + runAsGroup: + description: |- + The GID to run the entrypoint of the container process. + Uses runtime default if unset. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + runAsNonRoot: + description: |- + Indicates that the container must run as a non-root user. + If true, the Kubelet will validate the image at runtime to ensure that it + does not run as UID 0 (root) and fail to start the container if it does. + If unset or false, no such validation will be performed. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: |- + The UID to run the entrypoint of the container process. + Defaults to user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + seLinuxOptions: + description: |- + The SELinux context to be applied to the container. + If unspecified, the container runtime will allocate a random SELinux context for each + container. May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + properties: + level: + description: Level is SELinux level + label that applies to the container. + type: string + role: + description: Role is a SELinux role + label that applies to the container. + type: string + type: + description: Type is a SELinux type + label that applies to the container. + type: string + user: + description: User is a SELinux user + label that applies to the container. + type: string + type: object + seccompProfile: + description: |- + The seccomp options to use by this container. If seccomp options are + provided at both the pod & container level, the container options + override the pod options. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile defined in a file on the node should be used. + The profile must be preconfigured on the node to work. + Must be a descending path, relative to the kubelet's configured seccomp profile location. + Must be set if type is "Localhost". Must NOT be set for any other type. + type: string + type: + description: |- + type indicates which kind of seccomp profile will be applied. + Valid options are: + + Localhost - a profile defined in a file on the node should be used. + RuntimeDefault - the container runtime default profile should be used. + Unconfined - no profile should be applied. + type: string + required: + - type + type: object + windowsOptions: + description: |- + The Windows specific settings applied to all containers. + If unspecified, the options from the PodSecurityContext will be used. + If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is linux. + properties: + gmsaCredentialSpec: + description: |- + GMSACredentialSpec is where the GMSA admission webhook + (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the + GMSA credential spec named by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName + is the name of the GMSA credential + spec to use. + type: string + hostProcess: + description: |- + HostProcess determines if a container should be run as a 'Host Process' container. + All of a Pod's containers must have the same effective HostProcess value + (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). + In addition, if HostProcess is true then HostNetwork must also be set to true. + type: boolean + runAsUserName: + description: |- + The UserName in Windows to run the entrypoint of the container process. + Defaults to the user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: string + type: object + type: object + startupProbe: + description: |- + StartupProbe indicates that the Pod has successfully initialized. + If specified, no other probes are executed until this completes successfully. + If this probe fails, the Pod will be restarted, just as if the livenessProbe failed. + This can be used to provide different probe parameters at the beginning of a Pod's lifecycle, + when it might take a long time to load data or warm a cache, than during steady-state operation. + This cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies the action to + take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving + a GRPC port. + properties: + port: + description: Port number of the gRPC + service. Number must be in the range + 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http + request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in + the request. HTTP allows repeated + headers. + items: + description: HTTPHeader describes + a custom header to be used in HTTP + probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field + value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP + server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action + involving a TCP port. + properties: + host: + description: 'Optional: Host name to + connect to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + stdin: + description: |- + Whether this container should allocate a buffer for stdin in the container runtime. If this + is not set, reads from stdin in the container will always result in EOF. + Default is false. + type: boolean + stdinOnce: + description: |- + Whether the container runtime should close the stdin channel after it has been opened by + a single attach. When stdin is true the stdin stream will remain open across multiple attach + sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the + first client attaches to stdin, and then remains open and accepts data until the client disconnects, + at which time stdin is closed and remains closed until the container is restarted. If this + flag is false, a container processes that reads from stdin will never receive an EOF. + Default is false + type: boolean + terminationMessagePath: + description: |- + Optional: Path at which the file to which the container's termination message + will be written is mounted into the container's filesystem. + Message written is intended to be brief final status, such as an assertion failure message. + Will be truncated by the node if greater than 4096 bytes. The total message length across + all containers will be limited to 12kb. + Defaults to /dev/termination-log. + Cannot be updated. + type: string + terminationMessagePolicy: + description: |- + Indicate how the termination message should be populated. File will use the contents of + terminationMessagePath to populate the container status message on both success and failure. + FallbackToLogsOnError will use the last chunk of container log output if the termination + message file is empty and the container exited with an error. + The log output is limited to 2048 bytes or 80 lines, whichever is smaller. + Defaults to File. + Cannot be updated. + type: string + tty: + description: |- + Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. + Default is false. + type: boolean + volumeDevices: + description: volumeDevices is the list of block + devices to be used by the container. + items: + description: volumeDevice describes a mapping + of a raw block device within a container. + properties: + devicePath: + description: devicePath is the path inside + of the container that the device will + be mapped to. + type: string + name: + description: name must match the name + of a persistentVolumeClaim in the pod + type: string + required: + - devicePath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - devicePath + x-kubernetes-list-type: map + volumeMounts: + description: |- + Pod volumes to mount into the container's filesystem. + Cannot be updated. + items: + description: VolumeMount describes a mounting + of a Volume within a container. + properties: + mountPath: + description: |- + Path within the container at which the volume should be mounted. Must + not contain ':'. + type: string + mountPropagation: + description: |- + mountPropagation determines how mounts are propagated from the host + to container and the other way around. + When not set, MountPropagationNone is used. + This field is beta in 1.10. + When RecursiveReadOnly is set to IfPossible or to Enabled, MountPropagation must be None or unspecified + (which defaults to None). + type: string + name: + description: This must match the Name + of a Volume. + type: string + readOnly: + description: |- + Mounted read-only if true, read-write otherwise (false or unspecified). + Defaults to false. + type: boolean + recursiveReadOnly: + description: |- + RecursiveReadOnly specifies whether read-only mounts should be handled + recursively. + + If ReadOnly is false, this field has no meaning and must be unspecified. + + If ReadOnly is true, and this field is set to Disabled, the mount is not made + recursively read-only. If this field is set to IfPossible, the mount is made + recursively read-only, if it is supported by the container runtime. If this + field is set to Enabled, the mount is made recursively read-only if it is + supported by the container runtime, otherwise the pod will not be started and + an error will be generated to indicate the reason. + + If this field is set to IfPossible or Enabled, MountPropagation must be set to + None (or be unspecified, which defaults to None). + + If this field is not specified, it is treated as an equivalent of Disabled. + type: string + subPath: + description: |- + Path within the volume from which the container's volume should be mounted. + Defaults to "" (volume's root). + type: string + subPathExpr: + description: |- + Expanded path within the volume from which the container's volume should be mounted. + Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. + Defaults to "" (volume's root). + SubPathExpr and SubPath are mutually exclusive. + type: string + required: + - mountPath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - mountPath + x-kubernetes-list-type: map + workingDir: + description: |- + Container's working directory. + If not specified, the container runtime's default will be used, which + might be configured in the container image. + Cannot be updated. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + nodeName: + description: |- + NodeName indicates in which node this pod is scheduled. + If empty, this pod is a candidate for scheduling by the scheduler defined in schedulerName. + Once this field is set, the kubelet for this node becomes responsible for the lifecycle of this pod. + This field should not be used to express a desire for the pod to be scheduled on a specific node. + https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodename + type: string + nodeSelector: + additionalProperties: + type: string + description: |- + NodeSelector is a selector which must be true for the pod to fit on a node. + Selector which must match a node's labels for the pod to be scheduled on that node. + More info: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ + type: object + x-kubernetes-map-type: atomic + os: + description: |- + Specifies the OS of the containers in the pod. + Some pod and container fields are restricted if this is set. + + If the OS field is set to linux, the following fields must be unset: + -securityContext.windowsOptions + + If the OS field is set to windows, following fields must be unset: + - spec.hostPID + - spec.hostIPC + - spec.hostUsers + - spec.securityContext.appArmorProfile + - spec.securityContext.seLinuxOptions + - spec.securityContext.seccompProfile + - spec.securityContext.fsGroup + - spec.securityContext.fsGroupChangePolicy + - spec.securityContext.sysctls + - spec.shareProcessNamespace + - spec.securityContext.runAsUser + - spec.securityContext.runAsGroup + - spec.securityContext.supplementalGroups + - spec.securityContext.supplementalGroupsPolicy + - spec.containers[*].securityContext.appArmorProfile + - spec.containers[*].securityContext.seLinuxOptions + - spec.containers[*].securityContext.seccompProfile + - spec.containers[*].securityContext.capabilities + - spec.containers[*].securityContext.readOnlyRootFilesystem + - spec.containers[*].securityContext.privileged + - spec.containers[*].securityContext.allowPrivilegeEscalation + - spec.containers[*].securityContext.procMount + - spec.containers[*].securityContext.runAsUser + - spec.containers[*].securityContext.runAsGroup + properties: + name: + description: |- + Name is the name of the operating system. The currently supported values are linux and windows. + Additional value may be defined in future and can be one of: + https://github.com/opencontainers/runtime-spec/blob/master/config.md#platform-specific-configuration + Clients should expect to handle additional values and treat unrecognized values in this field as os: null + type: string + required: + - name + type: object + overhead: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Overhead represents the resource overhead associated with running a pod for a given RuntimeClass. + This field will be autopopulated at admission time by the RuntimeClass admission controller. If + the RuntimeClass admission controller is enabled, overhead must not be set in Pod create requests. + The RuntimeClass admission controller will reject Pod create requests which have the overhead already + set. If RuntimeClass is configured and selected in the PodSpec, Overhead will be set to the value + defined in the corresponding RuntimeClass, otherwise it will remain unset and treated as zero. + More info: https://git.k8s.io/enhancements/keps/sig-node/688-pod-overhead/README.md + type: object + preemptionPolicy: + description: |- + PreemptionPolicy is the Policy for preempting pods with lower priority. + One of Never, PreemptLowerPriority. + Defaults to PreemptLowerPriority if unset. + type: string + priority: + description: |- + The priority value. Various system components use this field to find the + priority of the pod. When Priority Admission Controller is enabled, it + prevents users from setting this field. The admission controller populates + this field from PriorityClassName. + The higher the value, the higher the priority. + format: int32 + type: integer + priorityClassName: + description: |- + If specified, indicates the pod's priority. "system-node-critical" and + "system-cluster-critical" are two special keywords which indicate the + highest priorities with the former being the highest priority. Any other + name must be defined by creating a PriorityClass object with that name. + If not specified, the pod priority will be default or zero if there is no + default. + type: string + readinessGates: + description: |- + If specified, all readiness gates will be evaluated for pod readiness. + A pod is ready when all its containers are ready AND + all conditions specified in the readiness gates have status equal to "True" + More info: https://git.k8s.io/enhancements/keps/sig-network/580-pod-readiness-gates + items: + description: PodReadinessGate contains the reference + to a pod condition + properties: + conditionType: + description: ConditionType refers to a condition + in the pod's condition list with matching + type. + type: string + required: + - conditionType + type: object + type: array + x-kubernetes-list-type: atomic + resourceClaims: + description: |- + ResourceClaims defines which ResourceClaims must be allocated + and reserved before the Pod is allowed to start. The resources + will be made available to those containers which consume them + by name. + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + This field is immutable. + items: + description: |- + PodResourceClaim references exactly one ResourceClaim, either directly + or by naming a ResourceClaimTemplate which is then turned into a ResourceClaim + for the pod. + + It adds a name to it that uniquely identifies the ResourceClaim inside the Pod. + Containers that need access to the ResourceClaim reference it with this name. + properties: + name: + description: |- + Name uniquely identifies this resource claim inside the pod. + This must be a DNS_LABEL. + type: string + resourceClaimName: + description: |- + ResourceClaimName is the name of a ResourceClaim object in the same + namespace as this pod. + + Exactly one of ResourceClaimName and ResourceClaimTemplateName must + be set. + type: string + resourceClaimTemplateName: + description: |- + ResourceClaimTemplateName is the name of a ResourceClaimTemplate + object in the same namespace as this pod. + + The template will be used to create a new ResourceClaim, which will + be bound to this pod. When this pod is deleted, the ResourceClaim + will also be deleted. The pod name and resource name, along with a + generated component, will be used to form a unique name for the + ResourceClaim, which will be recorded in pod.status.resourceClaimStatuses. + + This field is immutable and no changes will be made to the + corresponding ResourceClaim by the control plane after creating the + ResourceClaim. + + Exactly one of ResourceClaimName and ResourceClaimTemplateName must + be set. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + restartPolicy: + description: |- + Restart policy for all containers within the pod. + One of Always, OnFailure, Never. In some contexts, only a subset of those values may be permitted. + Default to Always. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#restart-policy + type: string + runtimeClassName: + description: |- + RuntimeClassName refers to a RuntimeClass object in the node.k8s.io group, which should be used + to run this pod. If no RuntimeClass resource matches the named class, the pod will not be run. + If unset or empty, the "legacy" RuntimeClass will be used, which is an implicit class with an + empty definition that uses the default runtime handler. + More info: https://git.k8s.io/enhancements/keps/sig-node/585-runtime-class + type: string + schedulerName: + description: |- + If specified, the pod will be dispatched by specified scheduler. + If not specified, the pod will be dispatched by default scheduler. + type: string + schedulingGates: + description: |- + SchedulingGates is an opaque list of values that if specified will block scheduling the pod. + If schedulingGates is not empty, the pod will stay in the SchedulingGated state and the + scheduler will not attempt to schedule the pod. + + SchedulingGates can only be set at pod creation time, and be removed only afterwards. + items: + description: PodSchedulingGate is associated to + a Pod to guard its scheduling. + properties: + name: + description: |- + Name of the scheduling gate. + Each scheduling gate must have a unique name field. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + securityContext: + description: |- + SecurityContext holds pod-level security attributes and common container settings. + Optional: Defaults to empty. See type description for default values of each field. + properties: + appArmorProfile: + description: |- + appArmorProfile is the AppArmor options to use by the containers in this pod. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile loaded on the node that should be used. + The profile must be preconfigured on the node to work. + Must match the loaded name of the profile. + Must be set if and only if type is "Localhost". + type: string + type: + description: |- + type indicates which kind of AppArmor profile will be applied. + Valid options are: + Localhost - a profile pre-loaded on the node. + RuntimeDefault - the container runtime's default profile. + Unconfined - no AppArmor enforcement. + type: string + required: + - type + type: object + fsGroup: + description: |- + A special supplemental group that applies to all containers in a pod. + Some volume types allow the Kubelet to change the ownership of that volume + to be owned by the pod: + + 1. The owning GID will be the FSGroup + 2. The setgid bit is set (new files created in the volume will be owned by FSGroup) + 3. The permission bits are OR'd with rw-rw---- + + If unset, the Kubelet will not modify the ownership and permissions of any volume. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + fsGroupChangePolicy: + description: |- + fsGroupChangePolicy defines behavior of changing ownership and permission of the volume + before being exposed inside Pod. This field will only apply to + volume types which support fsGroup based ownership(and permissions). + It will have no effect on ephemeral volume types such as: secret, configmaps + and emptydir. + Valid values are "OnRootMismatch" and "Always". If not specified, "Always" is used. + Note that this field cannot be set when spec.os.name is windows. + type: string + runAsGroup: + description: |- + The GID to run the entrypoint of the container process. + Uses runtime default if unset. + May also be set in SecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence + for that container. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + runAsNonRoot: + description: |- + Indicates that the container must run as a non-root user. + If true, the Kubelet will validate the image at runtime to ensure that it + does not run as UID 0 (root) and fail to start the container if it does. + If unset or false, no such validation will be performed. + May also be set in SecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: |- + The UID to run the entrypoint of the container process. + Defaults to user specified in image metadata if unspecified. + May also be set in SecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence + for that container. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + seLinuxOptions: + description: |- + The SELinux context to be applied to all containers. + If unspecified, the container runtime will allocate a random SELinux context for each + container. May also be set in SecurityContext. If set in + both SecurityContext and PodSecurityContext, the value specified in SecurityContext + takes precedence for that container. + Note that this field cannot be set when spec.os.name is windows. + properties: + level: + description: Level is SELinux level label + that applies to the container. + type: string + role: + description: Role is a SELinux role label + that applies to the container. + type: string + type: + description: Type is a SELinux type label + that applies to the container. + type: string + user: + description: User is a SELinux user label + that applies to the container. + type: string + type: object + seccompProfile: + description: |- + The seccomp options to use by the containers in this pod. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile defined in a file on the node should be used. + The profile must be preconfigured on the node to work. + Must be a descending path, relative to the kubelet's configured seccomp profile location. + Must be set if type is "Localhost". Must NOT be set for any other type. + type: string + type: + description: |- + type indicates which kind of seccomp profile will be applied. + Valid options are: + + Localhost - a profile defined in a file on the node should be used. + RuntimeDefault - the container runtime default profile should be used. + Unconfined - no profile should be applied. + type: string + required: + - type + type: object + supplementalGroups: + description: |- + A list of groups applied to the first process run in each container, in + addition to the container's primary GID and fsGroup (if specified). If + the SupplementalGroupsPolicy feature is enabled, the + supplementalGroupsPolicy field determines whether these are in addition + to or instead of any group memberships defined in the container image. + If unspecified, no additional groups are added, though group memberships + defined in the container image may still be used, depending on the + supplementalGroupsPolicy field. + Note that this field cannot be set when spec.os.name is windows. + items: + format: int64 + type: integer + type: array + x-kubernetes-list-type: atomic + supplementalGroupsPolicy: + description: |- + Defines how supplemental groups of the first container processes are calculated. + Valid values are "Merge" and "Strict". If not specified, "Merge" is used. + (Alpha) Using the field requires the SupplementalGroupsPolicy feature gate to be enabled + and the container runtime must implement support for this feature. + Note that this field cannot be set when spec.os.name is windows. + type: string + sysctls: + description: |- + Sysctls hold a list of namespaced sysctls used for the pod. Pods with unsupported + sysctls (by the container runtime) might fail to launch. + Note that this field cannot be set when spec.os.name is windows. + items: + description: Sysctl defines a kernel parameter + to be set + properties: + name: + description: Name of a property to set + type: string + value: + description: Value of a property to set + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + windowsOptions: + description: |- + The Windows specific settings applied to all containers. + If unspecified, the options within a container's SecurityContext will be used. + If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is linux. + properties: + gmsaCredentialSpec: + description: |- + GMSACredentialSpec is where the GMSA admission webhook + (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the + GMSA credential spec named by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName is the + name of the GMSA credential spec to use. + type: string + hostProcess: + description: |- + HostProcess determines if a container should be run as a 'Host Process' container. + All of a Pod's containers must have the same effective HostProcess value + (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). + In addition, if HostProcess is true then HostNetwork must also be set to true. + type: boolean + runAsUserName: + description: |- + The UserName in Windows to run the entrypoint of the container process. + Defaults to the user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: string + type: object + type: object + serviceAccount: + description: |- + DeprecatedServiceAccount is a deprecated alias for ServiceAccountName. + Deprecated: Use serviceAccountName instead. + type: string + serviceAccountName: + description: |- + ServiceAccountName is the name of the ServiceAccount to use to run this pod. + More info: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ + type: string + setHostnameAsFQDN: + description: |- + If true the pod's hostname will be configured as the pod's FQDN, rather than the leaf name (the default). + In Linux containers, this means setting the FQDN in the hostname field of the kernel (the nodename field of struct utsname). + In Windows containers, this means setting the registry value of hostname for the registry key HKEY_LOCAL_MACHINE\\SYSTEM\\CurrentControlSet\\Services\\Tcpip\\Parameters to FQDN. + If a pod does not have FQDN, this has no effect. + Default to false. + type: boolean + shareProcessNamespace: + description: |- + Share a single process namespace between all of the containers in a pod. + When this is set containers will be able to view and signal processes from other containers + in the same pod, and the first process in each container will not be assigned PID 1. + HostPID and ShareProcessNamespace cannot both be set. + Optional: Default to false. + type: boolean + subdomain: + description: |- + If specified, the fully qualified Pod hostname will be "...svc.". + If not specified, the pod will not have a domainname at all. + type: string + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully. May be decreased in delete request. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + If this value is nil, the default grace period will be used instead. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + Defaults to 30 seconds. + format: int64 + type: integer + tolerations: + description: If specified, the pod's tolerations. + items: + description: |- + The pod this Toleration is attached to tolerates any taint that matches + the triple using the matching operator . + properties: + effect: + description: |- + Effect indicates the taint effect to match. Empty means match all taint effects. + When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: |- + Key is the taint key that the toleration applies to. Empty means match all taint keys. + If the key is empty, operator must be Exists; this combination means to match all values and all keys. + type: string + operator: + description: |- + Operator represents a key's relationship to the value. + Valid operators are Exists and Equal. Defaults to Equal. + Exists is equivalent to wildcard for value, so that a pod can + tolerate all taints of a particular category. + type: string + tolerationSeconds: + description: |- + TolerationSeconds represents the period of time the toleration (which must be + of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, + it is not set, which means tolerate the taint forever (do not evict). Zero and + negative values will be treated as 0 (evict immediately) by the system. + format: int64 + type: integer + value: + description: |- + Value is the taint value the toleration matches to. + If the operator is Exists, the value should be empty, otherwise just a regular string. + type: string + type: object + type: array + x-kubernetes-list-type: atomic + topologySpreadConstraints: + description: |- + TopologySpreadConstraints describes how a group of pods ought to spread across topology + domains. Scheduler will schedule pods in a way which abides by the constraints. + All topologySpreadConstraints are ANDed. + items: + description: TopologySpreadConstraint specifies + how to spread matching pods among the given topology. + properties: + labelSelector: + description: |- + LabelSelector is used to find matching pods. + Pods that match this label selector are counted to determine the number of pods + in their corresponding topology domain. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key + that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select the pods over which + spreading will be calculated. The keys are used to lookup values from the + incoming pod labels, those key-value labels are ANDed with labelSelector + to select the group of existing pods over which spreading will be calculated + for the incoming pod. The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + MatchLabelKeys cannot be set when LabelSelector isn't set. + Keys that don't exist in the incoming pod labels will + be ignored. A null or empty list means only match against labelSelector. + + This is a beta field and requires the MatchLabelKeysInPodTopologySpread feature gate to be enabled (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + maxSkew: + description: |- + MaxSkew describes the degree to which pods may be unevenly distributed. + When `whenUnsatisfiable=DoNotSchedule`, it is the maximum permitted difference + between the number of matching pods in the target topology and the global minimum. + The global minimum is the minimum number of matching pods in an eligible domain + or zero if the number of eligible domains is less than MinDomains. + For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same + labelSelector spread as 2/2/1: + In this case, the global minimum is 1. + | zone1 | zone2 | zone3 | + | P P | P P | P | + - if MaxSkew is 1, incoming pod can only be scheduled to zone3 to become 2/2/2; + scheduling it onto zone1(zone2) would make the ActualSkew(3-1) on zone1(zone2) + violate MaxSkew(1). + - if MaxSkew is 2, incoming pod can be scheduled onto any zone. + When `whenUnsatisfiable=ScheduleAnyway`, it is used to give higher precedence + to topologies that satisfy it. + It's a required field. Default value is 1 and 0 is not allowed. + format: int32 + type: integer + minDomains: + description: |- + MinDomains indicates a minimum number of eligible domains. + When the number of eligible domains with matching topology keys is less than minDomains, + Pod Topology Spread treats "global minimum" as 0, and then the calculation of Skew is performed. + And when the number of eligible domains with matching topology keys equals or greater than minDomains, + this value has no effect on scheduling. + As a result, when the number of eligible domains is less than minDomains, + scheduler won't schedule more than maxSkew Pods to those domains. + If value is nil, the constraint behaves as if MinDomains is equal to 1. + Valid values are integers greater than 0. + When value is not nil, WhenUnsatisfiable must be DoNotSchedule. + + For example, in a 3-zone cluster, MaxSkew is set to 2, MinDomains is set to 5 and pods with the same + labelSelector spread as 2/2/2: + | zone1 | zone2 | zone3 | + | P P | P P | P P | + The number of domains is less than 5(MinDomains), so "global minimum" is treated as 0. + In this situation, new pod with the same labelSelector cannot be scheduled, + because computed skew will be 3(3 - 0) if new Pod is scheduled to any of the three zones, + it will violate MaxSkew. + format: int32 + type: integer + nodeAffinityPolicy: + description: |- + NodeAffinityPolicy indicates how we will treat Pod's nodeAffinity/nodeSelector + when calculating pod topology spread skew. Options are: + - Honor: only nodes matching nodeAffinity/nodeSelector are included in the calculations. + - Ignore: nodeAffinity/nodeSelector are ignored. All nodes are included in the calculations. + + If this value is nil, the behavior is equivalent to the Honor policy. + This is a beta-level feature default enabled by the NodeInclusionPolicyInPodTopologySpread feature flag. + type: string + nodeTaintsPolicy: + description: |- + NodeTaintsPolicy indicates how we will treat node taints when calculating + pod topology spread skew. Options are: + - Honor: nodes without taints, along with tainted nodes for which the incoming pod + has a toleration, are included. + - Ignore: node taints are ignored. All nodes are included. + + If this value is nil, the behavior is equivalent to the Ignore policy. + This is a beta-level feature default enabled by the NodeInclusionPolicyInPodTopologySpread feature flag. + type: string + topologyKey: + description: |- + TopologyKey is the key of node labels. Nodes that have a label with this key + and identical values are considered to be in the same topology. + We consider each as a "bucket", and try to put balanced number + of pods into each bucket. + We define a domain as a particular instance of a topology. + Also, we define an eligible domain as a domain whose nodes meet the requirements of + nodeAffinityPolicy and nodeTaintsPolicy. + e.g. If TopologyKey is "kubernetes.io/hostname", each Node is a domain of that topology. + And, if TopologyKey is "topology.kubernetes.io/zone", each zone is a domain of that topology. + It's a required field. + type: string + whenUnsatisfiable: + description: |- + WhenUnsatisfiable indicates how to deal with a pod if it doesn't satisfy + the spread constraint. + - DoNotSchedule (default) tells the scheduler not to schedule it. + - ScheduleAnyway tells the scheduler to schedule the pod in any location, + but giving higher precedence to topologies that would help reduce the + skew. + A constraint is considered "Unsatisfiable" for an incoming pod + if and only if every possible node assignment for that pod would violate + "MaxSkew" on some topology. + For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same + labelSelector spread as 3/1/1: + | zone1 | zone2 | zone3 | + | P P P | P | P | + If WhenUnsatisfiable is set to DoNotSchedule, incoming pod can only be scheduled + to zone2(zone3) to become 3/2/1(3/1/2) as ActualSkew(2-1) on zone2(zone3) satisfies + MaxSkew(1). In other words, the cluster can still be imbalanced, but scheduler + won't make it *more* imbalanced. + It's a required field. + type: string + required: + - maxSkew + - topologyKey + - whenUnsatisfiable + type: object + type: array + x-kubernetes-list-map-keys: + - topologyKey + - whenUnsatisfiable + x-kubernetes-list-type: map + volumes: + description: |- + List of volumes that can be mounted by containers belonging to the pod. + More info: https://kubernetes.io/docs/concepts/storage/volumes + items: + description: Volume represents a named volume in + a pod that may be accessed by any container in + the pod. + properties: + awsElasticBlockStore: + description: |- + awsElasticBlockStore represents an AWS Disk resource that is attached to a + kubelet's host machine and then exposed to the pod. + More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore + properties: + fsType: + description: |- + fsType is the filesystem type of the volume that you want to mount. + Tip: Ensure that the filesystem type is supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore + type: string + partition: + description: |- + partition is the partition in the volume that you want to mount. + If omitted, the default is to mount by volume name. + Examples: For volume /dev/sda1, you specify the partition as "1". + Similarly, the volume partition for /dev/sda is "0" (or you can leave the property empty). + format: int32 + type: integer + readOnly: + description: |- + readOnly value true will force the readOnly setting in VolumeMounts. + More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore + type: boolean + volumeID: + description: |- + volumeID is unique ID of the persistent disk resource in AWS (Amazon EBS volume). + More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore + type: string + required: + - volumeID + type: object + azureDisk: + description: azureDisk represents an Azure Data + Disk mount on the host and bind mount to the + pod. + properties: + cachingMode: + description: 'cachingMode is the Host Caching + mode: None, Read Only, Read Write.' + type: string + diskName: + description: diskName is the Name of the + data disk in the blob storage + type: string + diskURI: + description: diskURI is the URI of data + disk in the blob storage + type: string + fsType: + default: ext4 + description: |- + fsType is Filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + kind: + description: 'kind expected values are Shared: + multiple blob disks per storage account Dedicated: + single blob disk per storage account Managed: + azure managed data disk (only in managed + availability set). defaults to shared' + type: string + readOnly: + default: false + description: |- + readOnly Defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + required: + - diskName + - diskURI + type: object + azureFile: + description: azureFile represents an Azure File + Service mount on the host and bind mount to + the pod. + properties: + readOnly: + description: |- + readOnly defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + secretName: + description: secretName is the name of + secret that contains Azure Storage Account + Name and Key + type: string + shareName: + description: shareName is the azure share + Name + type: string + required: + - secretName + - shareName + type: object + cephfs: + description: cephFS represents a Ceph FS mount + on the host that shares a pod's lifetime + properties: + monitors: + description: |- + monitors is Required: Monitors is a collection of Ceph monitors + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + items: + type: string + type: array + x-kubernetes-list-type: atomic + path: + description: 'path is Optional: Used as + the mounted root, rather than the full + Ceph tree, default is /' + type: string + readOnly: + description: |- + readOnly is Optional: Defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + type: boolean + secretFile: + description: |- + secretFile is Optional: SecretFile is the path to key ring for User, default is /etc/ceph/user.secret + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + type: string + secretRef: + description: |- + secretRef is Optional: SecretRef is reference to the authentication secret for User, default is empty. + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + user: + description: |- + user is optional: User is the rados user name, default is admin + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + type: string + required: + - monitors + type: object + cinder: + description: |- + cinder represents a cinder volume attached and mounted on kubelets host machine. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md + properties: + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md + type: string + readOnly: + description: |- + readOnly defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md + type: boolean + secretRef: + description: |- + secretRef is optional: points to a secret object containing parameters used to connect + to OpenStack. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + volumeID: + description: |- + volumeID used to identify the volume in cinder. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md + type: string + required: + - volumeID + type: object + configMap: + description: configMap represents a configMap + that should populate this volume + properties: + defaultMode: + description: |- + defaultMode is optional: mode bits used to set permissions on created files by default. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + Defaults to 0644. + Directories within the path are not affected by this setting. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + items: + description: |- + items if unspecified, each key-value pair in the Data field of the referenced + ConfigMap will be projected into the volume as a file whose name is the + key and content is the value. If specified, the listed keys will be + projected into the specified paths, and unlisted keys will not be + present. If a key is specified which is not present in the ConfigMap, + the volume setup will error unless it is marked optional. Paths must be + relative and may not contain the '..' path or start with '..'. + items: + description: Maps a string key to a path + within a volume. + properties: + key: + description: key is the key to project. + type: string + mode: + description: |- + mode is Optional: mode bits used to set permissions on this file. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: |- + path is the relative path of the file to map the key to. + May not be an absolute path. + May not contain the path element '..'. + May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: optional specify whether the + ConfigMap or its keys must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + csi: + description: csi (Container Storage Interface) + represents ephemeral storage that is handled + by certain external CSI drivers (Beta feature). + properties: + driver: + description: |- + driver is the name of the CSI driver that handles this volume. + Consult with your admin for the correct name as registered in the cluster. + type: string + fsType: + description: |- + fsType to mount. Ex. "ext4", "xfs", "ntfs". + If not provided, the empty value is passed to the associated CSI driver + which will determine the default filesystem to apply. + type: string + nodePublishSecretRef: + description: |- + nodePublishSecretRef is a reference to the secret object containing + sensitive information to pass to the CSI driver to complete the CSI + NodePublishVolume and NodeUnpublishVolume calls. + This field is optional, and may be empty if no secret is required. If the + secret object contains more than one secret, all secret references are passed. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + readOnly: + description: |- + readOnly specifies a read-only configuration for the volume. + Defaults to false (read/write). + type: boolean + volumeAttributes: + additionalProperties: + type: string + description: |- + volumeAttributes stores driver-specific properties that are passed to the CSI + driver. Consult your driver's documentation for supported values. + type: object + required: + - driver + type: object + downwardAPI: + description: downwardAPI represents downward + API about the pod that should populate this + volume + properties: + defaultMode: + description: |- + Optional: mode bits to use on created files by default. Must be a + Optional: mode bits used to set permissions on created files by default. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + Defaults to 0644. + Directories within the path are not affected by this setting. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + items: + description: Items is a list of downward + API volume file + items: + description: DownwardAPIVolumeFile represents + information to create the file containing + the pod field + properties: + fieldRef: + description: 'Required: Selects a + field of the pod: only annotations, + labels, name, namespace and uid + are supported.' + properties: + apiVersion: + description: Version of the schema + the FieldPath is written in + terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field + to select in the specified API + version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + mode: + description: |- + Optional: mode bits used to set permissions on this file, must be an octal value + between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: 'Required: Path is the + relative path name of the file to + be created. Must not be absolute + or contain the ''..'' path. Must + be utf-8 encoded. The first item + of the relative path must not start + with ''..''' + type: string + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, requests.cpu and requests.memory) are currently supported. + properties: + containerName: + description: 'Container name: + required for volumes, optional + for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output + format of the exposed resources, + defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource + to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + required: + - path + type: object + type: array + x-kubernetes-list-type: atomic + type: object + emptyDir: + description: |- + emptyDir represents a temporary directory that shares a pod's lifetime. + More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir + properties: + medium: + description: |- + medium represents what type of storage medium should back this directory. + The default is "" which means to use the node's default medium. + Must be an empty string (default) or Memory. + More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir + type: string + sizeLimit: + anyOf: + - type: integer + - type: string + description: |- + sizeLimit is the total amount of local storage required for this EmptyDir volume. + The size limit is also applicable for memory medium. + The maximum usage on memory medium EmptyDir would be the minimum value between + the SizeLimit specified here and the sum of memory limits of all containers in a pod. + The default is nil which means that the limit is undefined. + More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + ephemeral: + description: |- + ephemeral represents a volume that is handled by a cluster storage driver. + The volume's lifecycle is tied to the pod that defines it - it will be created before the pod starts, + and deleted when the pod is removed. + + Use this if: + a) the volume is only needed while the pod runs, + b) features of normal volumes like restoring from snapshot or capacity + tracking are needed, + c) the storage driver is specified through a storage class, and + d) the storage driver supports dynamic volume provisioning through + a PersistentVolumeClaim (see EphemeralVolumeSource for more + information on the connection between this volume type + and PersistentVolumeClaim). + + Use PersistentVolumeClaim or one of the vendor-specific + APIs for volumes that persist for longer than the lifecycle + of an individual pod. + + Use CSI for light-weight local ephemeral volumes if the CSI driver is meant to + be used that way - see the documentation of the driver for + more information. + + A pod can use both types of ephemeral volumes and + persistent volumes at the same time. + properties: + volumeClaimTemplate: + description: |- + Will be used to create a stand-alone PVC to provision the volume. + The pod in which this EphemeralVolumeSource is embedded will be the + owner of the PVC, i.e. the PVC will be deleted together with the + pod. The name of the PVC will be `-` where + `` is the name from the `PodSpec.Volumes` array + entry. Pod validation will reject the pod if the concatenated name + is not valid for a PVC (for example, too long). + + An existing PVC with that name that is not owned by the pod + will *not* be used for the pod to avoid using an unrelated + volume by mistake. Starting the pod is then blocked until + the unrelated PVC is removed. If such a pre-created PVC is + meant to be used by the pod, the PVC has to updated with an + owner reference to the pod once the pod exists. Normally + this should not be necessary, but it may be useful when + manually reconstructing a broken cluster. + + This field is read-only and no changes will be made by Kubernetes + to the PVC after it has been created. + + Required, must not be nil. + properties: + metadata: + description: |- + May contain labels and annotations that will be copied into the PVC + when creating it. No other fields are allowed and will be rejected during + validation. + type: object + spec: + description: |- + The specification for the PersistentVolumeClaim. The entire content is + copied unchanged into the PVC that gets created from this + template. The same fields as in a PersistentVolumeClaim + are also valid here. + properties: + accessModes: + description: |- + accessModes contains the desired access modes the volume should have. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1 + items: + type: string + type: array + x-kubernetes-list-type: atomic + dataSource: + description: |- + dataSource field can be used to specify either: + * An existing VolumeSnapshot object (snapshot.storage.k8s.io/VolumeSnapshot) + * An existing PVC (PersistentVolumeClaim) + If the provisioner or an external controller can support the specified data source, + it will create a new volume based on the contents of the specified data source. + When the AnyVolumeDataSource feature gate is enabled, dataSource contents will be copied to dataSourceRef, + and dataSourceRef contents will be copied to dataSource when dataSourceRef.namespace is not specified. + If the namespace is specified, then dataSourceRef will not be copied to dataSource. + properties: + apiGroup: + description: |- + APIGroup is the group for the resource being referenced. + If APIGroup is not specified, the specified Kind must be in the core API group. + For any other third-party types, APIGroup is required. + type: string + kind: + description: Kind is the type + of resource being referenced + type: string + name: + description: Name is the name + of resource being referenced + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + dataSourceRef: + description: |- + dataSourceRef specifies the object from which to populate the volume with data, if a non-empty + volume is desired. This may be any object from a non-empty API group (non + core object) or a PersistentVolumeClaim object. + When this field is specified, volume binding will only succeed if the type of + the specified object matches some installed volume populator or dynamic + provisioner. + This field will replace the functionality of the dataSource field and as such + if both fields are non-empty, they must have the same value. For backwards + compatibility, when namespace isn't specified in dataSourceRef, + both fields (dataSource and dataSourceRef) will be set to the same + value automatically if one of them is empty and the other is non-empty. + When namespace is specified in dataSourceRef, + dataSource isn't set to the same value and must be empty. + There are three important differences between dataSource and dataSourceRef: + * While dataSource only allows two specific types of objects, dataSourceRef + allows any non-core object, as well as PersistentVolumeClaim objects. + * While dataSource ignores disallowed values (dropping them), dataSourceRef + preserves all values, and generates an error if a disallowed value is + specified. + * While dataSource only allows local objects, dataSourceRef allows objects + in any namespaces. + (Beta) Using this field requires the AnyVolumeDataSource feature gate to be enabled. + (Alpha) Using the namespace field of dataSourceRef requires the CrossNamespaceVolumeDataSource feature gate to be enabled. + properties: + apiGroup: + description: |- + APIGroup is the group for the resource being referenced. + If APIGroup is not specified, the specified Kind must be in the core API group. + For any other third-party types, APIGroup is required. + type: string + kind: + description: Kind is the type + of resource being referenced + type: string + name: + description: Name is the name + of resource being referenced + type: string + namespace: + description: |- + Namespace is the namespace of resource being referenced + Note that when a namespace is specified, a gateway.networking.k8s.io/ReferenceGrant object is required in the referent namespace to allow that namespace's owner to accept the reference. See the ReferenceGrant documentation for details. + (Alpha) This field requires the CrossNamespaceVolumeDataSource feature gate to be enabled. + type: string + required: + - kind + - name + type: object + resources: + description: |- + resources represents the minimum resources the volume should have. + If RecoverVolumeExpansionFailure feature is enabled users are allowed to specify resource requirements + that are lower than previous value but must still be higher than capacity recorded in the + status field of the claim. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + selector: + description: selector is a label + query over volumes to consider + for binding. + properties: + matchExpressions: + description: matchExpressions + is a list of label selector + requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the + label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + storageClassName: + description: |- + storageClassName is the name of the StorageClass required by the claim. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1 + type: string + volumeAttributesClassName: + description: |- + volumeAttributesClassName may be used to set the VolumeAttributesClass used by this claim. + If specified, the CSI driver will create or update the volume with the attributes defined + in the corresponding VolumeAttributesClass. This has a different purpose than storageClassName, + it can be changed after the claim is created. An empty string value means that no VolumeAttributesClass + will be applied to the claim but it's not allowed to reset this field to empty string once it is set. + If unspecified and the PersistentVolumeClaim is unbound, the default VolumeAttributesClass + will be set by the persistentvolume controller if it exists. + If the resource referred to by volumeAttributesClass does not exist, this PersistentVolumeClaim will be + set to a Pending state, as reflected by the modifyVolumeStatus field, until such as a resource + exists. + More info: https://kubernetes.io/docs/concepts/storage/volume-attributes-classes/ + (Beta) Using this field requires the VolumeAttributesClass feature gate to be enabled (off by default). + type: string + volumeMode: + description: |- + volumeMode defines what type of volume is required by the claim. + Value of Filesystem is implied when not included in claim spec. + type: string + volumeName: + description: volumeName is the binding + reference to the PersistentVolume + backing this claim. + type: string + type: object + required: + - spec + type: object + type: object + fc: + description: fc represents a Fibre Channel resource + that is attached to a kubelet's host machine + and then exposed to the pod. + properties: + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + lun: + description: 'lun is Optional: FC target + lun number' + format: int32 + type: integer + readOnly: + description: |- + readOnly is Optional: Defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + targetWWNs: + description: 'targetWWNs is Optional: FC + target worldwide names (WWNs)' + items: + type: string + type: array + x-kubernetes-list-type: atomic + wwids: + description: |- + wwids Optional: FC volume world wide identifiers (wwids) + Either wwids or combination of targetWWNs and lun must be set, but not both simultaneously. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + flexVolume: + description: |- + flexVolume represents a generic volume resource that is + provisioned/attached using an exec based plugin. + properties: + driver: + description: driver is the name of the driver + to use for this volume. + type: string + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". The default filesystem depends on FlexVolume script. + type: string + options: + additionalProperties: + type: string + description: 'options is Optional: this + field holds extra command options if any.' + type: object + readOnly: + description: |- + readOnly is Optional: defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + secretRef: + description: |- + secretRef is Optional: secretRef is reference to the secret object containing + sensitive information to pass to the plugin scripts. This may be + empty if no secret object is specified. If the secret object + contains more than one secret, all secrets are passed to the plugin + scripts. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + required: + - driver + type: object + flocker: + description: flocker represents a Flocker volume + attached to a kubelet's host machine. This + depends on the Flocker control service being + running + properties: + datasetName: + description: |- + datasetName is Name of the dataset stored as metadata -> name on the dataset for Flocker + should be considered as deprecated + type: string + datasetUUID: + description: datasetUUID is the UUID of + the dataset. This is unique identifier + of a Flocker dataset + type: string + type: object + gcePersistentDisk: + description: |- + gcePersistentDisk represents a GCE Disk resource that is attached to a + kubelet's host machine and then exposed to the pod. + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + properties: + fsType: + description: |- + fsType is filesystem type of the volume that you want to mount. + Tip: Ensure that the filesystem type is supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + type: string + partition: + description: |- + partition is the partition in the volume that you want to mount. + If omitted, the default is to mount by volume name. + Examples: For volume /dev/sda1, you specify the partition as "1". + Similarly, the volume partition for /dev/sda is "0" (or you can leave the property empty). + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + format: int32 + type: integer + pdName: + description: |- + pdName is unique name of the PD resource in GCE. Used to identify the disk in GCE. + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + type: string + readOnly: + description: |- + readOnly here will force the ReadOnly setting in VolumeMounts. + Defaults to false. + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + type: boolean + required: + - pdName + type: object + gitRepo: + description: |- + gitRepo represents a git repository at a particular revision. + DEPRECATED: GitRepo is deprecated. To provision a container with a git repo, mount an + EmptyDir into an InitContainer that clones the repo using git, then mount the EmptyDir + into the Pod's container. + properties: + directory: + description: |- + directory is the target directory name. + Must not contain or start with '..'. If '.' is supplied, the volume directory will be the + git repository. Otherwise, if specified, the volume will contain the git repository in + the subdirectory with the given name. + type: string + repository: + description: repository is the URL + type: string + revision: + description: revision is the commit hash + for the specified revision. + type: string + required: + - repository + type: object + glusterfs: + description: |- + glusterfs represents a Glusterfs mount on the host that shares a pod's lifetime. + More info: https://examples.k8s.io/volumes/glusterfs/README.md + properties: + endpoints: + description: |- + endpoints is the endpoint name that details Glusterfs topology. + More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod + type: string + path: + description: |- + path is the Glusterfs volume path. + More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod + type: string + readOnly: + description: |- + readOnly here will force the Glusterfs volume to be mounted with read-only permissions. + Defaults to false. + More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod + type: boolean + required: + - endpoints + - path + type: object + hostPath: + description: |- + hostPath represents a pre-existing file or directory on the host + machine that is directly exposed to the container. This is generally + used for system agents or other privileged things that are allowed + to see the host machine. Most containers will NOT need this. + More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath + properties: + path: + description: |- + path of the directory on the host. + If the path is a symlink, it will follow the link to the real path. + More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath + type: string + type: + description: |- + type for HostPath Volume + Defaults to "" + More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath + type: string + required: + - path + type: object + image: + description: |- + image represents an OCI object (a container image or artifact) pulled and mounted on the kubelet's host machine. + The volume is resolved at pod startup depending on which PullPolicy value is provided: + + - Always: the kubelet always attempts to pull the reference. Container creation will fail If the pull fails. + - Never: the kubelet never pulls the reference and only uses a local image or artifact. Container creation will fail if the reference isn't present. + - IfNotPresent: the kubelet pulls if the reference isn't already present on disk. Container creation will fail if the reference isn't present and the pull fails. + + The volume gets re-resolved if the pod gets deleted and recreated, which means that new remote content will become available on pod recreation. + A failure to resolve or pull the image during pod startup will block containers from starting and may add significant latency. Failures will be retried using normal volume backoff and will be reported on the pod reason and message. + The types of objects that may be mounted by this volume are defined by the container runtime implementation on a host machine and at minimum must include all valid types supported by the container image field. + The OCI object gets mounted in a single directory (spec.containers[*].volumeMounts.mountPath) by merging the manifest layers in the same way as for container images. + The volume will be mounted read-only (ro) and non-executable files (noexec). + Sub path mounts for containers are not supported (spec.containers[*].volumeMounts.subpath). + The field spec.securityContext.fsGroupChangePolicy has no effect on this volume type. + properties: + pullPolicy: + description: |- + Policy for pulling OCI objects. Possible values are: + Always: the kubelet always attempts to pull the reference. Container creation will fail If the pull fails. + Never: the kubelet never pulls the reference and only uses a local image or artifact. Container creation will fail if the reference isn't present. + IfNotPresent: the kubelet pulls if the reference isn't already present on disk. Container creation will fail if the reference isn't present and the pull fails. + Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. + type: string + reference: + description: |- + Required: Image or artifact reference to be used. + Behaves in the same way as pod.spec.containers[*].image. + Pull secrets will be assembled in the same way as for the container image by looking up node credentials, SA image pull secrets, and pod spec image pull secrets. + More info: https://kubernetes.io/docs/concepts/containers/images + This field is optional to allow higher level config management to default or override + container images in workload controllers like Deployments and StatefulSets. + type: string + type: object + iscsi: + description: |- + iscsi represents an ISCSI Disk resource that is attached to a + kubelet's host machine and then exposed to the pod. + More info: https://examples.k8s.io/volumes/iscsi/README.md + properties: + chapAuthDiscovery: + description: chapAuthDiscovery defines whether + support iSCSI Discovery CHAP authentication + type: boolean + chapAuthSession: + description: chapAuthSession defines whether + support iSCSI Session CHAP authentication + type: boolean + fsType: + description: |- + fsType is the filesystem type of the volume that you want to mount. + Tip: Ensure that the filesystem type is supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://kubernetes.io/docs/concepts/storage/volumes#iscsi + type: string + initiatorName: + description: |- + initiatorName is the custom iSCSI Initiator Name. + If initiatorName is specified with iscsiInterface simultaneously, new iSCSI interface + : will be created for the connection. + type: string + iqn: + description: iqn is the target iSCSI Qualified + Name. + type: string + iscsiInterface: + default: default + description: |- + iscsiInterface is the interface Name that uses an iSCSI transport. + Defaults to 'default' (tcp). + type: string + lun: + description: lun represents iSCSI Target + Lun number. + format: int32 + type: integer + portals: + description: |- + portals is the iSCSI Target Portal List. The portal is either an IP or ip_addr:port if the port + is other than default (typically TCP ports 860 and 3260). + items: + type: string + type: array + x-kubernetes-list-type: atomic + readOnly: + description: |- + readOnly here will force the ReadOnly setting in VolumeMounts. + Defaults to false. + type: boolean + secretRef: + description: secretRef is the CHAP Secret + for iSCSI target and initiator authentication + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + targetPortal: + description: |- + targetPortal is iSCSI Target Portal. The Portal is either an IP or ip_addr:port if the port + is other than default (typically TCP ports 860 and 3260). + type: string + required: + - iqn + - lun + - targetPortal + type: object + name: + description: |- + name of the volume. + Must be a DNS_LABEL and unique within the pod. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + nfs: + description: |- + nfs represents an NFS mount on the host that shares a pod's lifetime + More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs + properties: + path: + description: |- + path that is exported by the NFS server. + More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs + type: string + readOnly: + description: |- + readOnly here will force the NFS export to be mounted with read-only permissions. + Defaults to false. + More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs + type: boolean + server: + description: |- + server is the hostname or IP address of the NFS server. + More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs + type: string + required: + - path + - server + type: object + persistentVolumeClaim: + description: |- + persistentVolumeClaimVolumeSource represents a reference to a + PersistentVolumeClaim in the same namespace. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims + properties: + claimName: + description: |- + claimName is the name of a PersistentVolumeClaim in the same namespace as the pod using this volume. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims + type: string + readOnly: + description: |- + readOnly Will force the ReadOnly setting in VolumeMounts. + Default false. + type: boolean + required: + - claimName + type: object + photonPersistentDisk: + description: photonPersistentDisk represents + a PhotonController persistent disk attached + and mounted on kubelets host machine + properties: + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + pdID: + description: pdID is the ID that identifies + Photon Controller persistent disk + type: string + required: + - pdID + type: object + portworxVolume: + description: portworxVolume represents a portworx + volume attached and mounted on kubelets host + machine + properties: + fsType: + description: |- + fSType represents the filesystem type to mount + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs". Implicitly inferred to be "ext4" if unspecified. + type: string + readOnly: + description: |- + readOnly defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + volumeID: + description: volumeID uniquely identifies + a Portworx volume + type: string + required: + - volumeID + type: object + projected: + description: projected items for all in one + resources secrets, configmaps, and downward + API + properties: + defaultMode: + description: |- + defaultMode are the mode bits used to set permissions on created files by default. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + Directories within the path are not affected by this setting. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + sources: + description: |- + sources is the list of volume projections. Each entry in this list + handles one source. + items: + description: |- + Projection that may be projected along with other supported volume types. + Exactly one of these fields must be set. + properties: + clusterTrustBundle: + description: |- + ClusterTrustBundle allows a pod to access the `.spec.trustBundle` field + of ClusterTrustBundle objects in an auto-updating file. + + Alpha, gated by the ClusterTrustBundleProjection feature gate. + + ClusterTrustBundle objects can either be selected by name, or by the + combination of signer name and a label selector. + + Kubelet performs aggressive normalization of the PEM contents written + into the pod filesystem. Esoteric PEM features such as inter-block + comments and block headers are stripped. Certificates are deduplicated. + The ordering of certificates within the file is arbitrary, and Kubelet + may change the order over time. + properties: + labelSelector: + description: |- + Select all ClusterTrustBundles that match this label selector. Only has + effect if signerName is set. Mutually-exclusive with name. If unset, + interpreted as "match nothing". If set but empty, interpreted as "match + everything". + properties: + matchExpressions: + description: matchExpressions + is a list of label selector + requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is + the label key that + the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + name: + description: |- + Select a single ClusterTrustBundle by object name. Mutually-exclusive + with signerName and labelSelector. + type: string + optional: + description: |- + If true, don't block pod startup if the referenced ClusterTrustBundle(s) + aren't available. If using name, then the named ClusterTrustBundle is + allowed not to exist. If using signerName, then the combination of + signerName and labelSelector is allowed to match zero + ClusterTrustBundles. + type: boolean + path: + description: Relative path from + the volume root to write the + bundle. + type: string + signerName: + description: |- + Select all ClusterTrustBundles that match this signer name. + Mutually-exclusive with name. The contents of all selected + ClusterTrustBundles will be unified and deduplicated. + type: string + required: + - path + type: object + configMap: + description: configMap information + about the configMap data to project + properties: + items: + description: |- + items if unspecified, each key-value pair in the Data field of the referenced + ConfigMap will be projected into the volume as a file whose name is the + key and content is the value. If specified, the listed keys will be + projected into the specified paths, and unlisted keys will not be + present. If a key is specified which is not present in the ConfigMap, + the volume setup will error unless it is marked optional. Paths must be + relative and may not contain the '..' path or start with '..'. + items: + description: Maps a string key + to a path within a volume. + properties: + key: + description: key is the + key to project. + type: string + mode: + description: |- + mode is Optional: mode bits used to set permissions on this file. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: |- + path is the relative path of the file to map the key to. + May not be an absolute path. + May not contain the path element '..'. + May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: optional specify + whether the ConfigMap or its + keys must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + downwardAPI: + description: downwardAPI information + about the downwardAPI data to project + properties: + items: + description: Items is a list of + DownwardAPIVolume file + items: + description: DownwardAPIVolumeFile + represents information to + create the file containing + the pod field + properties: + fieldRef: + description: 'Required: + Selects a field of the + pod: only annotations, + labels, name, namespace + and uid are supported.' + properties: + apiVersion: + description: Version + of the schema the + FieldPath is written + in terms of, defaults + to "v1". + type: string + fieldPath: + description: Path of + the field to select + in the specified API + version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + mode: + description: |- + Optional: mode bits used to set permissions on this file, must be an octal value + between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: 'Required: + Path is the relative + path name of the file + to be created. Must not + be absolute or contain + the ''..'' path. Must + be utf-8 encoded. The + first item of the relative + path must not start with + ''..''' + type: string + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, requests.cpu and requests.memory) are currently supported. + properties: + containerName: + description: 'Container + name: required for + volumes, optional + for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies + the output format + of the exposed resources, + defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: + resource to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + required: + - path + type: object + type: array + x-kubernetes-list-type: atomic + type: object + secret: + description: secret information about + the secret data to project + properties: + items: + description: |- + items if unspecified, each key-value pair in the Data field of the referenced + Secret will be projected into the volume as a file whose name is the + key and content is the value. If specified, the listed keys will be + projected into the specified paths, and unlisted keys will not be + present. If a key is specified which is not present in the Secret, + the volume setup will error unless it is marked optional. Paths must be + relative and may not contain the '..' path or start with '..'. + items: + description: Maps a string key + to a path within a volume. + properties: + key: + description: key is the + key to project. + type: string + mode: + description: |- + mode is Optional: mode bits used to set permissions on this file. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: |- + path is the relative path of the file to map the key to. + May not be an absolute path. + May not contain the path element '..'. + May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: optional field specify + whether the Secret or its key + must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + serviceAccountToken: + description: serviceAccountToken is + information about the serviceAccountToken + data to project + properties: + audience: + description: |- + audience is the intended audience of the token. A recipient of a token + must identify itself with an identifier specified in the audience of the + token, and otherwise should reject the token. The audience defaults to the + identifier of the apiserver. + type: string + expirationSeconds: + description: |- + expirationSeconds is the requested duration of validity of the service + account token. As the token approaches expiration, the kubelet volume + plugin will proactively rotate the service account token. The kubelet will + start trying to rotate the token if the token is older than 80 percent of + its time to live or if the token is older than 24 hours.Defaults to 1 hour + and must be at least 10 minutes. + format: int64 + type: integer + path: + description: |- + path is the path relative to the mount point of the file to project the + token into. + type: string + required: + - path + type: object + type: object + type: array + x-kubernetes-list-type: atomic + type: object + quobyte: + description: quobyte represents a Quobyte mount + on the host that shares a pod's lifetime + properties: + group: + description: |- + group to map volume access to + Default is no group + type: string + readOnly: + description: |- + readOnly here will force the Quobyte volume to be mounted with read-only permissions. + Defaults to false. + type: boolean + registry: + description: |- + registry represents a single or multiple Quobyte Registry services + specified as a string as host:port pair (multiple entries are separated with commas) + which acts as the central registry for volumes + type: string + tenant: + description: |- + tenant owning the given Quobyte volume in the Backend + Used with dynamically provisioned Quobyte volumes, value is set by the plugin + type: string + user: + description: |- + user to map volume access to + Defaults to serivceaccount user + type: string + volume: + description: volume is a string that references + an already created Quobyte volume by name. + type: string + required: + - registry + - volume + type: object + rbd: + description: |- + rbd represents a Rados Block Device mount on the host that shares a pod's lifetime. + More info: https://examples.k8s.io/volumes/rbd/README.md + properties: + fsType: + description: |- + fsType is the filesystem type of the volume that you want to mount. + Tip: Ensure that the filesystem type is supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://kubernetes.io/docs/concepts/storage/volumes#rbd + type: string + image: + description: |- + image is the rados image name. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: string + keyring: + default: /etc/ceph/keyring + description: |- + keyring is the path to key ring for RBDUser. + Default is /etc/ceph/keyring. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: string + monitors: + description: |- + monitors is a collection of Ceph monitors. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + items: + type: string + type: array + x-kubernetes-list-type: atomic + pool: + default: rbd + description: |- + pool is the rados pool name. + Default is rbd. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: string + readOnly: + description: |- + readOnly here will force the ReadOnly setting in VolumeMounts. + Defaults to false. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: boolean + secretRef: + description: |- + secretRef is name of the authentication secret for RBDUser. If provided + overrides keyring. + Default is nil. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + user: + default: admin + description: |- + user is the rados user name. + Default is admin. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: string + required: + - image + - monitors + type: object + scaleIO: + description: scaleIO represents a ScaleIO persistent + volume attached and mounted on Kubernetes + nodes. + properties: + fsType: + default: xfs + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". + Default is "xfs". + type: string + gateway: + description: gateway is the host address + of the ScaleIO API Gateway. + type: string + protectionDomain: + description: protectionDomain is the name + of the ScaleIO Protection Domain for the + configured storage. + type: string + readOnly: + description: |- + readOnly Defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + secretRef: + description: |- + secretRef references to the secret for ScaleIO user and other + sensitive information. If this is not provided, Login operation will fail. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + sslEnabled: + description: sslEnabled Flag enable/disable + SSL communication with Gateway, default + false + type: boolean + storageMode: + default: ThinProvisioned + description: |- + storageMode indicates whether the storage for a volume should be ThickProvisioned or ThinProvisioned. + Default is ThinProvisioned. + type: string + storagePool: + description: storagePool is the ScaleIO + Storage Pool associated with the protection + domain. + type: string + system: + description: system is the name of the storage + system as configured in ScaleIO. + type: string + volumeName: + description: |- + volumeName is the name of a volume already created in the ScaleIO system + that is associated with this volume source. + type: string + required: + - gateway + - secretRef + - system + type: object + secret: + description: |- + secret represents a secret that should populate this volume. + More info: https://kubernetes.io/docs/concepts/storage/volumes#secret + properties: + defaultMode: + description: |- + defaultMode is Optional: mode bits used to set permissions on created files by default. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values + for mode bits. Defaults to 0644. + Directories within the path are not affected by this setting. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + items: + description: |- + items If unspecified, each key-value pair in the Data field of the referenced + Secret will be projected into the volume as a file whose name is the + key and content is the value. If specified, the listed keys will be + projected into the specified paths, and unlisted keys will not be + present. If a key is specified which is not present in the Secret, + the volume setup will error unless it is marked optional. Paths must be + relative and may not contain the '..' path or start with '..'. + items: + description: Maps a string key to a path + within a volume. + properties: + key: + description: key is the key to project. + type: string + mode: + description: |- + mode is Optional: mode bits used to set permissions on this file. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: |- + path is the relative path of the file to map the key to. + May not be an absolute path. + May not contain the path element '..'. + May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + optional: + description: optional field specify whether + the Secret or its keys must be defined + type: boolean + secretName: + description: |- + secretName is the name of the secret in the pod's namespace to use. + More info: https://kubernetes.io/docs/concepts/storage/volumes#secret + type: string + type: object + storageos: + description: storageOS represents a StorageOS + volume attached and mounted on Kubernetes + nodes. + properties: + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + readOnly: + description: |- + readOnly defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + secretRef: + description: |- + secretRef specifies the secret to use for obtaining the StorageOS API + credentials. If not specified, default values will be attempted. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + volumeName: + description: |- + volumeName is the human-readable name of the StorageOS volume. Volume + names are only unique within a namespace. + type: string + volumeNamespace: + description: |- + volumeNamespace specifies the scope of the volume within StorageOS. If no + namespace is specified then the Pod's namespace will be used. This allows the + Kubernetes name scoping to be mirrored within StorageOS for tighter integration. + Set VolumeName to any name to override the default behaviour. + Set to "default" if you are not using namespaces within StorageOS. + Namespaces that do not pre-exist within StorageOS will be created. + type: string + type: object + vsphereVolume: + description: vsphereVolume represents a vSphere + volume attached and mounted on kubelets host + machine + properties: + fsType: + description: |- + fsType is filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + storagePolicyID: + description: storagePolicyID is the storage + Policy Based Management (SPBM) profile + ID associated with the StoragePolicyName. + type: string + storagePolicyName: + description: storagePolicyName is the storage + Policy Based Management (SPBM) profile + name. + type: string + volumePath: + description: volumePath is the path that + identifies vSphere volume vmdk + type: string + required: + - volumePath + type: object + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + required: + - containers + type: object + type: object + required: + - rayStartParams + - template + type: object + headServiceAnnotations: + additionalProperties: + type: string + type: object + rayVersion: + description: RayVersion is used to determine the command for the + Kubernetes Job managed by RayJob + type: string + suspend: + description: |- + Suspend indicates whether a RayCluster should be suspended. + A suspended RayCluster will have head pods and worker pods deleted. + type: boolean + workerGroupSpecs: + description: WorkerGroupSpecs are the specs for the worker pods + items: + description: WorkerGroupSpec are the specs for the worker pods + properties: + groupName: + description: we can have multiple worker groups, we distinguish + them by name + type: string + maxReplicas: + default: 2147483647 + description: MaxReplicas denotes the maximum number of desired + Pods for this worker group, and the default value is maxInt32. + format: int32 + type: integer + minReplicas: + default: 0 + description: MinReplicas denotes the minimum number of desired + Pods for this worker group. + format: int32 + type: integer + numOfHosts: + default: 1 + description: NumOfHosts denotes the number of hosts to create + per replica. The default value is 1. + format: int32 + type: integer + rayStartParams: + additionalProperties: + type: string + description: 'RayStartParams are the params of the start + command: address, object-store-memory, ...' + type: object + replicas: + default: 0 + description: Replicas is the number of desired Pods for + this worker group. See https://github.com/ray-project/kuberay/pull/1443 + for more details about the reason for making this field + optional. + format: int32 + type: integer + scaleStrategy: + description: ScaleStrategy defines which pods to remove + properties: + workersToDelete: + description: WorkersToDelete workers to be deleted + items: + type: string + type: array + type: object + template: + description: Template is a pod template for the worker + properties: + metadata: + description: |- + Standard object's metadata. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata + type: object + spec: + description: |- + Specification of the desired behavior of the pod. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status + properties: + activeDeadlineSeconds: + description: |- + Optional duration in seconds the pod may be active on the node relative to + StartTime before the system will actively try to mark it failed and kill associated containers. + Value must be a positive integer. + format: int64 + type: integer + affinity: + description: If specified, the pod's scheduling + constraints + properties: + nodeAffinity: + description: Describes node affinity scheduling + rules for the pod. + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node matches the corresponding matchExpressions; the + node(s) with the highest sum are the most preferred. + items: + description: |- + An empty preferred scheduling term matches all objects with implicit weight 0 + (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op). + properties: + preference: + description: A node selector term, + associated with the corresponding + weight. + properties: + matchExpressions: + description: A list of node selector + requirements by node's labels. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key + that the selector applies + to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + description: A list of node selector + requirements by node's fields. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key + that the selector applies + to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + weight: + description: Weight associated with + matching the corresponding nodeSelectorTerm, + in the range 1-100. + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to an update), the system + may or may not try to eventually evict the pod from its node. + properties: + nodeSelectorTerms: + description: Required. A list of node + selector terms. The terms are ORed. + items: + description: |- + A null or empty node selector term matches no objects. The requirements of + them are ANDed. + The TopologySelectorTerm type implements a subset of the NodeSelectorTerm. + properties: + matchExpressions: + description: A list of node selector + requirements by node's labels. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key + that the selector applies + to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + description: A list of node selector + requirements by node's fields. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key + that the selector applies + to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + type: array + x-kubernetes-list-type: atomic + required: + - nodeSelectorTerms + type: object + x-kubernetes-map-type: atomic + type: object + podAffinity: + description: Describes pod affinity scheduling + rules (e.g. co-locate this pod in the same + node, zone, etc. as some other pod(s)). + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the + node(s) with the highest sum are the most preferred. + items: + description: The weights of all of the + matched WeightedPodAffinityTerm fields + are added per-node to find the most + preferred node(s) + properties: + podAffinityTerm: + description: Required. A pod affinity + term, associated with the corresponding + weight. + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions + is a list of label selector + requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is + the label key that + the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions + is a list of label selector + requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is + the label key that + the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + weight: + description: |- + weight associated with matching the corresponding podAffinityTerm, + in the range 1-100. + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to a pod label update), the + system may or may not try to eventually evict the pod from its node. + When there are multiple elements, the lists of nodes corresponding to each + podAffinityTerm are intersected, i.e. all terms must be satisfied. + items: + description: |- + Defines a set of pods (namely those matching the labelSelector + relative to the given namespace(s)) that this pod should be + co-located (affinity) or not co-located (anti-affinity) with, + where co-located is defined as running on a node whose value of + the label with key matches that of any node on which + a pod of the set of pods is running + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions + is a list of label selector + requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the + label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions + is a list of label selector + requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the + label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + podAntiAffinity: + description: Describes pod anti-affinity scheduling + rules (e.g. avoid putting this pod in the + same node, zone, etc. as some other pod(s)). + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the anti-affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling anti-affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the + node(s) with the highest sum are the most preferred. + items: + description: The weights of all of the + matched WeightedPodAffinityTerm fields + are added per-node to find the most + preferred node(s) + properties: + podAffinityTerm: + description: Required. A pod affinity + term, associated with the corresponding + weight. + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions + is a list of label selector + requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is + the label key that + the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions + is a list of label selector + requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is + the label key that + the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + weight: + description: |- + weight associated with matching the corresponding podAffinityTerm, + in the range 1-100. + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the anti-affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the anti-affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to a pod label update), the + system may or may not try to eventually evict the pod from its node. + When there are multiple elements, the lists of nodes corresponding to each + podAffinityTerm are intersected, i.e. all terms must be satisfied. + items: + description: |- + Defines a set of pods (namely those matching the labelSelector + relative to the given namespace(s)) that this pod should be + co-located (affinity) or not co-located (anti-affinity) with, + where co-located is defined as running on a node whose value of + the label with key matches that of any node on which + a pod of the set of pods is running + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions + is a list of label selector + requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the + label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions + is a list of label selector + requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the + label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + type: object + automountServiceAccountToken: + description: AutomountServiceAccountToken indicates + whether a service account token should be automatically + mounted. + type: boolean + containers: + description: |- + List of containers belonging to the pod. + Containers cannot currently be added or removed. + There must be at least one container in a Pod. + Cannot be updated. + items: + description: A single application container that + you want to run within a pod. + properties: + args: + description: |- + Arguments to the entrypoint. + The container image's CMD is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell + items: + type: string + type: array + x-kubernetes-list-type: atomic + command: + description: |- + Entrypoint array. Not executed within a shell. + The container image's ENTRYPOINT is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell + items: + type: string + type: array + x-kubernetes-list-type: atomic + env: + description: |- + List of environment variables to set in the container. + Cannot be updated. + items: + description: EnvVar represents an environment + variable present in a Container. + properties: + name: + description: Name of the environment + variable. Must be a C_IDENTIFIER. + type: string + value: + description: |- + Variable references $(VAR_NAME) are expanded + using the previously defined environment variables in the container and + any service environment variables. If a variable cannot be resolved, + the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. + "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless of whether the variable + exists or not. + Defaults to "". + type: string + valueFrom: + description: Source for the environment + variable's value. Cannot be used if + value is not empty. + properties: + configMapKeyRef: + description: Selects a key of a + ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether + the ConfigMap or its key must + be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + description: |- + Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, + spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. + properties: + apiVersion: + description: Version of the + schema the FieldPath is written + in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field + to select in the specified + API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. + properties: + containerName: + description: 'Container name: + required for volumes, optional + for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output + format of the exposed resources, + defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource + to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + description: Selects a key of a + secret in the pod's namespace + properties: + key: + description: The key of the + secret to select from. Must + be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether + the Secret or its key must + be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + envFrom: + description: |- + List of sources to populate environment variables in the container. + The keys defined within a source must be a C_IDENTIFIER. All invalid keys + will be reported as an event when the container is starting. When a key exists in multiple + sources, the value associated with the last source will take precedence. + Values defined by an Env with a duplicate key will take precedence. + Cannot be updated. + items: + description: EnvFromSource represents the + source of a set of ConfigMaps + properties: + configMapRef: + description: The ConfigMap to select + from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the + ConfigMap must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + description: An optional identifier + to prepend to each key in the ConfigMap. + Must be a C_IDENTIFIER. + type: string + secretRef: + description: The Secret to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the + Secret must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array + x-kubernetes-list-type: atomic + image: + description: |- + Container image name. + More info: https://kubernetes.io/docs/concepts/containers/images + This field is optional to allow higher level config management to default or override + container images in workload controllers like Deployments and StatefulSets. + type: string + imagePullPolicy: + description: |- + Image pull policy. + One of Always, Never, IfNotPresent. + Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/containers/images#updating-images + type: string + lifecycle: + description: |- + Actions that the management system should take in response to container lifecycle events. + Cannot be updated. + properties: + postStart: + description: |- + PostStart is called immediately after a container is created. If the handler fails, + the container is terminated and restarted according to its restart policy. + Other management of the container blocks until the hook completes. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies the action + to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies the + http request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to + set in the request. HTTP allows + repeated headers. + items: + description: HTTPHeader describes + a custom header to be used + in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header + field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on + the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents the + duration that the container should + sleep before being terminated. + properties: + seconds: + description: Seconds is the number + of seconds to sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for the backward compatibility. There are no validation of this field and + lifecycle hooks will fail in runtime when tcp handler is specified. + properties: + host: + description: 'Optional: Host name + to connect to, defaults to the + pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + preStop: + description: |- + PreStop is called immediately before a container is terminated due to an + API request or management event such as liveness/startup probe failure, + preemption, resource contention, etc. The handler is not called if the + container crashes or exits. The Pod's termination grace period countdown begins before the + PreStop hook is executed. Regardless of the outcome of the handler, the + container will eventually terminate within the Pod's termination grace + period (unless delayed by finalizers). Other management of the container blocks until the hook completes + or until the termination grace period is reached. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies the action + to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies the + http request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to + set in the request. HTTP allows + repeated headers. + items: + description: HTTPHeader describes + a custom header to be used + in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header + field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on + the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents the + duration that the container should + sleep before being terminated. + properties: + seconds: + description: Seconds is the number + of seconds to sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for the backward compatibility. There are no validation of this field and + lifecycle hooks will fail in runtime when tcp handler is specified. + properties: + host: + description: 'Optional: Host name + to connect to, defaults to the + pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + type: object + livenessProbe: + description: |- + Periodic probe of container liveness. + Container will be restarted if the probe fails. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies the action + to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action + involving a GRPC port. + properties: + port: + description: Port number of the gRPC + service. Number must be in the range + 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http + request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set + in the request. HTTP allows repeated + headers. + items: + description: HTTPHeader describes + a custom header to be used in + HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field + value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the + HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action + involving a TCP port. + properties: + host: + description: 'Optional: Host name + to connect to, defaults to the pod + IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + name: + description: |- + Name of the container specified as a DNS_LABEL. + Each container in a pod must have a unique name (DNS_LABEL). + Cannot be updated. + type: string + ports: + description: |- + List of ports to expose from the container. Not specifying a port here + DOES NOT prevent that port from being exposed. Any port which is + listening on the default "0.0.0.0" address inside a container will be + accessible from the network. + Modifying this array with strategic merge patch may corrupt the data. + For more information See https://github.com/kubernetes/kubernetes/issues/108255. + Cannot be updated. + items: + description: ContainerPort represents a + network port in a single container. + properties: + containerPort: + description: |- + Number of port to expose on the pod's IP address. + This must be a valid port number, 0 < x < 65536. + format: int32 + type: integer + hostIP: + description: What host IP to bind the + external port to. + type: string + hostPort: + description: |- + Number of port to expose on the host. + If specified, this must be a valid port number, 0 < x < 65536. + If HostNetwork is specified, this must match ContainerPort. + Most containers do not need this. + format: int32 + type: integer + name: + description: |- + If specified, this must be an IANA_SVC_NAME and unique within the pod. Each + named port in a pod must have a unique name. Name for the port that can be + referred to by services. + type: string + protocol: + default: TCP + description: |- + Protocol for port. Must be UDP, TCP, or SCTP. + Defaults to "TCP". + type: string + required: + - containerPort + type: object + type: array + x-kubernetes-list-map-keys: + - containerPort + - protocol + x-kubernetes-list-type: map + readinessProbe: + description: |- + Periodic probe of container service readiness. + Container will be removed from service endpoints if the probe fails. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies the action + to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action + involving a GRPC port. + properties: + port: + description: Port number of the gRPC + service. Number must be in the range + 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http + request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set + in the request. HTTP allows repeated + headers. + items: + description: HTTPHeader describes + a custom header to be used in + HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field + value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the + HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action + involving a TCP port. + properties: + host: + description: 'Optional: Host name + to connect to, defaults to the pod + IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + resizePolicy: + description: Resources resize policy for the + container. + items: + description: ContainerResizePolicy represents + resource resize policy for the container. + properties: + resourceName: + description: |- + Name of the resource to which this resource resize policy applies. + Supported values: cpu, memory. + type: string + restartPolicy: + description: |- + Restart policy to apply when specified resource is resized. + If not specified, it defaults to NotRequired. + type: string + required: + - resourceName + - restartPolicy + type: object + type: array + x-kubernetes-list-type: atomic + resources: + description: |- + Compute Resources required by this container. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references + one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + restartPolicy: + description: |- + RestartPolicy defines the restart behavior of individual containers in a pod. + This field may only be set for init containers, and the only allowed value is "Always". + For non-init containers or when this field is not specified, + the restart behavior is defined by the Pod's restart policy and the container type. + Setting the RestartPolicy as "Always" for the init container will have the following effect: + this init container will be continually restarted on + exit until all regular containers have terminated. Once all regular + containers have completed, all init containers with restartPolicy "Always" + will be shut down. This lifecycle differs from normal init containers and + is often referred to as a "sidecar" container. Although this init + container still starts in the init container sequence, it does not wait + for the container to complete before proceeding to the next init + container. Instead, the next init container starts immediately after this + init container is started, or after any startupProbe has successfully + completed. + type: string + securityContext: + description: |- + SecurityContext defines the security options the container should be run with. + If set, the fields of SecurityContext override the equivalent fields of PodSecurityContext. + More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ + properties: + allowPrivilegeEscalation: + description: |- + AllowPrivilegeEscalation controls whether a process can gain more + privileges than its parent process. This bool directly controls if + the no_new_privs flag will be set on the container process. + AllowPrivilegeEscalation is true always when the container is: + 1) run as Privileged + 2) has CAP_SYS_ADMIN + Note that this field cannot be set when spec.os.name is windows. + type: boolean + appArmorProfile: + description: |- + appArmorProfile is the AppArmor options to use by this container. If set, this profile + overrides the pod's appArmorProfile. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile loaded on the node that should be used. + The profile must be preconfigured on the node to work. + Must match the loaded name of the profile. + Must be set if and only if type is "Localhost". + type: string + type: + description: |- + type indicates which kind of AppArmor profile will be applied. + Valid options are: + Localhost - a profile pre-loaded on the node. + RuntimeDefault - the container runtime's default profile. + Unconfined - no AppArmor enforcement. + type: string + required: + - type + type: object + capabilities: + description: |- + The capabilities to add/drop when running containers. + Defaults to the default set of capabilities granted by the container runtime. + Note that this field cannot be set when spec.os.name is windows. + properties: + add: + description: Added capabilities + items: + description: Capability represent + POSIX capabilities type + type: string + type: array + x-kubernetes-list-type: atomic + drop: + description: Removed capabilities + items: + description: Capability represent + POSIX capabilities type + type: string + type: array + x-kubernetes-list-type: atomic + type: object + privileged: + description: |- + Run container in privileged mode. + Processes in privileged containers are essentially equivalent to root on the host. + Defaults to false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + procMount: + description: |- + procMount denotes the type of proc mount to use for the containers. + The default value is Default which uses the container runtime defaults for + readonly paths and masked paths. + This requires the ProcMountType feature flag to be enabled. + Note that this field cannot be set when spec.os.name is windows. + type: string + readOnlyRootFilesystem: + description: |- + Whether this container has a read-only root filesystem. + Default is false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + runAsGroup: + description: |- + The GID to run the entrypoint of the container process. + Uses runtime default if unset. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + runAsNonRoot: + description: |- + Indicates that the container must run as a non-root user. + If true, the Kubelet will validate the image at runtime to ensure that it + does not run as UID 0 (root) and fail to start the container if it does. + If unset or false, no such validation will be performed. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: |- + The UID to run the entrypoint of the container process. + Defaults to user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + seLinuxOptions: + description: |- + The SELinux context to be applied to the container. + If unspecified, the container runtime will allocate a random SELinux context for each + container. May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + properties: + level: + description: Level is SELinux level + label that applies to the container. + type: string + role: + description: Role is a SELinux role + label that applies to the container. + type: string + type: + description: Type is a SELinux type + label that applies to the container. + type: string + user: + description: User is a SELinux user + label that applies to the container. + type: string + type: object + seccompProfile: + description: |- + The seccomp options to use by this container. If seccomp options are + provided at both the pod & container level, the container options + override the pod options. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile defined in a file on the node should be used. + The profile must be preconfigured on the node to work. + Must be a descending path, relative to the kubelet's configured seccomp profile location. + Must be set if type is "Localhost". Must NOT be set for any other type. + type: string + type: + description: |- + type indicates which kind of seccomp profile will be applied. + Valid options are: + + Localhost - a profile defined in a file on the node should be used. + RuntimeDefault - the container runtime default profile should be used. + Unconfined - no profile should be applied. + type: string + required: + - type + type: object + windowsOptions: + description: |- + The Windows specific settings applied to all containers. + If unspecified, the options from the PodSecurityContext will be used. + If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is linux. + properties: + gmsaCredentialSpec: + description: |- + GMSACredentialSpec is where the GMSA admission webhook + (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the + GMSA credential spec named by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName + is the name of the GMSA credential + spec to use. + type: string + hostProcess: + description: |- + HostProcess determines if a container should be run as a 'Host Process' container. + All of a Pod's containers must have the same effective HostProcess value + (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). + In addition, if HostProcess is true then HostNetwork must also be set to true. + type: boolean + runAsUserName: + description: |- + The UserName in Windows to run the entrypoint of the container process. + Defaults to the user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: string + type: object + type: object + startupProbe: + description: |- + StartupProbe indicates that the Pod has successfully initialized. + If specified, no other probes are executed until this completes successfully. + If this probe fails, the Pod will be restarted, just as if the livenessProbe failed. + This can be used to provide different probe parameters at the beginning of a Pod's lifecycle, + when it might take a long time to load data or warm a cache, than during steady-state operation. + This cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies the action + to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action + involving a GRPC port. + properties: + port: + description: Port number of the gRPC + service. Number must be in the range + 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http + request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set + in the request. HTTP allows repeated + headers. + items: + description: HTTPHeader describes + a custom header to be used in + HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field + value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the + HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action + involving a TCP port. + properties: + host: + description: 'Optional: Host name + to connect to, defaults to the pod + IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + stdin: + description: |- + Whether this container should allocate a buffer for stdin in the container runtime. If this + is not set, reads from stdin in the container will always result in EOF. + Default is false. + type: boolean + stdinOnce: + description: |- + Whether the container runtime should close the stdin channel after it has been opened by + a single attach. When stdin is true the stdin stream will remain open across multiple attach + sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the + first client attaches to stdin, and then remains open and accepts data until the client disconnects, + at which time stdin is closed and remains closed until the container is restarted. If this + flag is false, a container processes that reads from stdin will never receive an EOF. + Default is false + type: boolean + terminationMessagePath: + description: |- + Optional: Path at which the file to which the container's termination message + will be written is mounted into the container's filesystem. + Message written is intended to be brief final status, such as an assertion failure message. + Will be truncated by the node if greater than 4096 bytes. The total message length across + all containers will be limited to 12kb. + Defaults to /dev/termination-log. + Cannot be updated. + type: string + terminationMessagePolicy: + description: |- + Indicate how the termination message should be populated. File will use the contents of + terminationMessagePath to populate the container status message on both success and failure. + FallbackToLogsOnError will use the last chunk of container log output if the termination + message file is empty and the container exited with an error. + The log output is limited to 2048 bytes or 80 lines, whichever is smaller. + Defaults to File. + Cannot be updated. + type: string + tty: + description: |- + Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. + Default is false. + type: boolean + volumeDevices: + description: volumeDevices is the list of + block devices to be used by the container. + items: + description: volumeDevice describes a mapping + of a raw block device within a container. + properties: + devicePath: + description: devicePath is the path + inside of the container that the device + will be mapped to. + type: string + name: + description: name must match the name + of a persistentVolumeClaim in the + pod + type: string + required: + - devicePath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - devicePath + x-kubernetes-list-type: map + volumeMounts: + description: |- + Pod volumes to mount into the container's filesystem. + Cannot be updated. + items: + description: VolumeMount describes a mounting + of a Volume within a container. + properties: + mountPath: + description: |- + Path within the container at which the volume should be mounted. Must + not contain ':'. + type: string + mountPropagation: + description: |- + mountPropagation determines how mounts are propagated from the host + to container and the other way around. + When not set, MountPropagationNone is used. + This field is beta in 1.10. + When RecursiveReadOnly is set to IfPossible or to Enabled, MountPropagation must be None or unspecified + (which defaults to None). + type: string + name: + description: This must match the Name + of a Volume. + type: string + readOnly: + description: |- + Mounted read-only if true, read-write otherwise (false or unspecified). + Defaults to false. + type: boolean + recursiveReadOnly: + description: |- + RecursiveReadOnly specifies whether read-only mounts should be handled + recursively. + + If ReadOnly is false, this field has no meaning and must be unspecified. + + If ReadOnly is true, and this field is set to Disabled, the mount is not made + recursively read-only. If this field is set to IfPossible, the mount is made + recursively read-only, if it is supported by the container runtime. If this + field is set to Enabled, the mount is made recursively read-only if it is + supported by the container runtime, otherwise the pod will not be started and + an error will be generated to indicate the reason. + + If this field is set to IfPossible or Enabled, MountPropagation must be set to + None (or be unspecified, which defaults to None). + + If this field is not specified, it is treated as an equivalent of Disabled. + type: string + subPath: + description: |- + Path within the volume from which the container's volume should be mounted. + Defaults to "" (volume's root). + type: string + subPathExpr: + description: |- + Expanded path within the volume from which the container's volume should be mounted. + Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. + Defaults to "" (volume's root). + SubPathExpr and SubPath are mutually exclusive. + type: string + required: + - mountPath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - mountPath + x-kubernetes-list-type: map + workingDir: + description: |- + Container's working directory. + If not specified, the container runtime's default will be used, which + might be configured in the container image. + Cannot be updated. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + dnsConfig: + description: |- + Specifies the DNS parameters of a pod. + Parameters specified here will be merged to the generated DNS + configuration based on DNSPolicy. + properties: + nameservers: + description: |- + A list of DNS name server IP addresses. + This will be appended to the base nameservers generated from DNSPolicy. + Duplicated nameservers will be removed. + items: + type: string + type: array + x-kubernetes-list-type: atomic + options: + description: |- + A list of DNS resolver options. + This will be merged with the base options generated from DNSPolicy. + Duplicated entries will be removed. Resolution options given in Options + will override those that appear in the base DNSPolicy. + items: + description: PodDNSConfigOption defines DNS + resolver options of a pod. + properties: + name: + description: Required. + type: string + value: + type: string + type: object + type: array + x-kubernetes-list-type: atomic + searches: + description: |- + A list of DNS search domains for host-name lookup. + This will be appended to the base search paths generated from DNSPolicy. + Duplicated search paths will be removed. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + dnsPolicy: + description: |- + Set DNS policy for the pod. + Defaults to "ClusterFirst". + Valid values are 'ClusterFirstWithHostNet', 'ClusterFirst', 'Default' or 'None'. + DNS parameters given in DNSConfig will be merged with the policy selected with DNSPolicy. + To have DNS options set along with hostNetwork, you have to specify DNS policy + explicitly to 'ClusterFirstWithHostNet'. + type: string + enableServiceLinks: + description: |- + EnableServiceLinks indicates whether information about services should be injected into pod's + environment variables, matching the syntax of Docker links. + Optional: Defaults to true. + type: boolean + ephemeralContainers: + description: |- + List of ephemeral containers run in this pod. Ephemeral containers may be run in an existing + pod to perform user-initiated actions such as debugging. This list cannot be specified when + creating a pod, and it cannot be modified by updating the pod spec. In order to add an + ephemeral container to an existing pod, use the pod's ephemeralcontainers subresource. + items: + description: |- + An EphemeralContainer is a temporary container that you may add to an existing Pod for + user-initiated activities such as debugging. Ephemeral containers have no resource or + scheduling guarantees, and they will not be restarted when they exit or when a Pod is + removed or restarted. The kubelet may evict a Pod if an ephemeral container causes the + Pod to exceed its resource allocation. + + To add an ephemeral container, use the ephemeralcontainers subresource of an existing + Pod. Ephemeral containers may not be removed or restarted. + properties: + args: + description: |- + Arguments to the entrypoint. + The image's CMD is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell + items: + type: string + type: array + x-kubernetes-list-type: atomic + command: + description: |- + Entrypoint array. Not executed within a shell. + The image's ENTRYPOINT is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell + items: + type: string + type: array + x-kubernetes-list-type: atomic + env: + description: |- + List of environment variables to set in the container. + Cannot be updated. + items: + description: EnvVar represents an environment + variable present in a Container. + properties: + name: + description: Name of the environment + variable. Must be a C_IDENTIFIER. + type: string + value: + description: |- + Variable references $(VAR_NAME) are expanded + using the previously defined environment variables in the container and + any service environment variables. If a variable cannot be resolved, + the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. + "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless of whether the variable + exists or not. + Defaults to "". + type: string + valueFrom: + description: Source for the environment + variable's value. Cannot be used if + value is not empty. + properties: + configMapKeyRef: + description: Selects a key of a + ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether + the ConfigMap or its key must + be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + description: |- + Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, + spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. + properties: + apiVersion: + description: Version of the + schema the FieldPath is written + in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field + to select in the specified + API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. + properties: + containerName: + description: 'Container name: + required for volumes, optional + for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output + format of the exposed resources, + defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource + to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + description: Selects a key of a + secret in the pod's namespace + properties: + key: + description: The key of the + secret to select from. Must + be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether + the Secret or its key must + be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + envFrom: + description: |- + List of sources to populate environment variables in the container. + The keys defined within a source must be a C_IDENTIFIER. All invalid keys + will be reported as an event when the container is starting. When a key exists in multiple + sources, the value associated with the last source will take precedence. + Values defined by an Env with a duplicate key will take precedence. + Cannot be updated. + items: + description: EnvFromSource represents the + source of a set of ConfigMaps + properties: + configMapRef: + description: The ConfigMap to select + from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the + ConfigMap must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + description: An optional identifier + to prepend to each key in the ConfigMap. + Must be a C_IDENTIFIER. + type: string + secretRef: + description: The Secret to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the + Secret must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array + x-kubernetes-list-type: atomic + image: + description: |- + Container image name. + More info: https://kubernetes.io/docs/concepts/containers/images + type: string + imagePullPolicy: + description: |- + Image pull policy. + One of Always, Never, IfNotPresent. + Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/containers/images#updating-images + type: string + lifecycle: + description: Lifecycle is not allowed for + ephemeral containers. + properties: + postStart: + description: |- + PostStart is called immediately after a container is created. If the handler fails, + the container is terminated and restarted according to its restart policy. + Other management of the container blocks until the hook completes. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies the action + to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies the + http request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to + set in the request. HTTP allows + repeated headers. + items: + description: HTTPHeader describes + a custom header to be used + in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header + field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on + the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents the + duration that the container should + sleep before being terminated. + properties: + seconds: + description: Seconds is the number + of seconds to sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for the backward compatibility. There are no validation of this field and + lifecycle hooks will fail in runtime when tcp handler is specified. + properties: + host: + description: 'Optional: Host name + to connect to, defaults to the + pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + preStop: + description: |- + PreStop is called immediately before a container is terminated due to an + API request or management event such as liveness/startup probe failure, + preemption, resource contention, etc. The handler is not called if the + container crashes or exits. The Pod's termination grace period countdown begins before the + PreStop hook is executed. Regardless of the outcome of the handler, the + container will eventually terminate within the Pod's termination grace + period (unless delayed by finalizers). Other management of the container blocks until the hook completes + or until the termination grace period is reached. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies the action + to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies the + http request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to + set in the request. HTTP allows + repeated headers. + items: + description: HTTPHeader describes + a custom header to be used + in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header + field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on + the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents the + duration that the container should + sleep before being terminated. + properties: + seconds: + description: Seconds is the number + of seconds to sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for the backward compatibility. There are no validation of this field and + lifecycle hooks will fail in runtime when tcp handler is specified. + properties: + host: + description: 'Optional: Host name + to connect to, defaults to the + pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + type: object + livenessProbe: + description: Probes are not allowed for ephemeral + containers. + properties: + exec: + description: Exec specifies the action + to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action + involving a GRPC port. + properties: + port: + description: Port number of the gRPC + service. Number must be in the range + 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http + request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set + in the request. HTTP allows repeated + headers. + items: + description: HTTPHeader describes + a custom header to be used in + HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field + value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the + HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action + involving a TCP port. + properties: + host: + description: 'Optional: Host name + to connect to, defaults to the pod + IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + name: + description: |- + Name of the ephemeral container specified as a DNS_LABEL. + This name must be unique among all containers, init containers and ephemeral containers. + type: string + ports: + description: Ports are not allowed for ephemeral + containers. + items: + description: ContainerPort represents a + network port in a single container. + properties: + containerPort: + description: |- + Number of port to expose on the pod's IP address. + This must be a valid port number, 0 < x < 65536. + format: int32 + type: integer + hostIP: + description: What host IP to bind the + external port to. + type: string + hostPort: + description: |- + Number of port to expose on the host. + If specified, this must be a valid port number, 0 < x < 65536. + If HostNetwork is specified, this must match ContainerPort. + Most containers do not need this. + format: int32 + type: integer + name: + description: |- + If specified, this must be an IANA_SVC_NAME and unique within the pod. Each + named port in a pod must have a unique name. Name for the port that can be + referred to by services. + type: string + protocol: + default: TCP + description: |- + Protocol for port. Must be UDP, TCP, or SCTP. + Defaults to "TCP". + type: string + required: + - containerPort + type: object + type: array + x-kubernetes-list-map-keys: + - containerPort + - protocol + x-kubernetes-list-type: map + readinessProbe: + description: Probes are not allowed for ephemeral + containers. + properties: + exec: + description: Exec specifies the action + to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action + involving a GRPC port. + properties: + port: + description: Port number of the gRPC + service. Number must be in the range + 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http + request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set + in the request. HTTP allows repeated + headers. + items: + description: HTTPHeader describes + a custom header to be used in + HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field + value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the + HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action + involving a TCP port. + properties: + host: + description: 'Optional: Host name + to connect to, defaults to the pod + IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + resizePolicy: + description: Resources resize policy for the + container. + items: + description: ContainerResizePolicy represents + resource resize policy for the container. + properties: + resourceName: + description: |- + Name of the resource to which this resource resize policy applies. + Supported values: cpu, memory. + type: string + restartPolicy: + description: |- + Restart policy to apply when specified resource is resized. + If not specified, it defaults to NotRequired. + type: string + required: + - resourceName + - restartPolicy + type: object + type: array + x-kubernetes-list-type: atomic + resources: + description: |- + Resources are not allowed for ephemeral containers. Ephemeral containers use spare resources + already allocated to the pod. + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references + one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + restartPolicy: + description: |- + Restart policy for the container to manage the restart behavior of each + container within a pod. + This may only be set for init containers. You cannot set this field on + ephemeral containers. + type: string + securityContext: + description: |- + Optional: SecurityContext defines the security options the ephemeral container should be run with. + If set, the fields of SecurityContext override the equivalent fields of PodSecurityContext. + properties: + allowPrivilegeEscalation: + description: |- + AllowPrivilegeEscalation controls whether a process can gain more + privileges than its parent process. This bool directly controls if + the no_new_privs flag will be set on the container process. + AllowPrivilegeEscalation is true always when the container is: + 1) run as Privileged + 2) has CAP_SYS_ADMIN + Note that this field cannot be set when spec.os.name is windows. + type: boolean + appArmorProfile: + description: |- + appArmorProfile is the AppArmor options to use by this container. If set, this profile + overrides the pod's appArmorProfile. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile loaded on the node that should be used. + The profile must be preconfigured on the node to work. + Must match the loaded name of the profile. + Must be set if and only if type is "Localhost". + type: string + type: + description: |- + type indicates which kind of AppArmor profile will be applied. + Valid options are: + Localhost - a profile pre-loaded on the node. + RuntimeDefault - the container runtime's default profile. + Unconfined - no AppArmor enforcement. + type: string + required: + - type + type: object + capabilities: + description: |- + The capabilities to add/drop when running containers. + Defaults to the default set of capabilities granted by the container runtime. + Note that this field cannot be set when spec.os.name is windows. + properties: + add: + description: Added capabilities + items: + description: Capability represent + POSIX capabilities type + type: string + type: array + x-kubernetes-list-type: atomic + drop: + description: Removed capabilities + items: + description: Capability represent + POSIX capabilities type + type: string + type: array + x-kubernetes-list-type: atomic + type: object + privileged: + description: |- + Run container in privileged mode. + Processes in privileged containers are essentially equivalent to root on the host. + Defaults to false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + procMount: + description: |- + procMount denotes the type of proc mount to use for the containers. + The default value is Default which uses the container runtime defaults for + readonly paths and masked paths. + This requires the ProcMountType feature flag to be enabled. + Note that this field cannot be set when spec.os.name is windows. + type: string + readOnlyRootFilesystem: + description: |- + Whether this container has a read-only root filesystem. + Default is false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + runAsGroup: + description: |- + The GID to run the entrypoint of the container process. + Uses runtime default if unset. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + runAsNonRoot: + description: |- + Indicates that the container must run as a non-root user. + If true, the Kubelet will validate the image at runtime to ensure that it + does not run as UID 0 (root) and fail to start the container if it does. + If unset or false, no such validation will be performed. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: |- + The UID to run the entrypoint of the container process. + Defaults to user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + seLinuxOptions: + description: |- + The SELinux context to be applied to the container. + If unspecified, the container runtime will allocate a random SELinux context for each + container. May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + properties: + level: + description: Level is SELinux level + label that applies to the container. + type: string + role: + description: Role is a SELinux role + label that applies to the container. + type: string + type: + description: Type is a SELinux type + label that applies to the container. + type: string + user: + description: User is a SELinux user + label that applies to the container. + type: string + type: object + seccompProfile: + description: |- + The seccomp options to use by this container. If seccomp options are + provided at both the pod & container level, the container options + override the pod options. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile defined in a file on the node should be used. + The profile must be preconfigured on the node to work. + Must be a descending path, relative to the kubelet's configured seccomp profile location. + Must be set if type is "Localhost". Must NOT be set for any other type. + type: string + type: + description: |- + type indicates which kind of seccomp profile will be applied. + Valid options are: + + Localhost - a profile defined in a file on the node should be used. + RuntimeDefault - the container runtime default profile should be used. + Unconfined - no profile should be applied. + type: string + required: + - type + type: object + windowsOptions: + description: |- + The Windows specific settings applied to all containers. + If unspecified, the options from the PodSecurityContext will be used. + If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is linux. + properties: + gmsaCredentialSpec: + description: |- + GMSACredentialSpec is where the GMSA admission webhook + (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the + GMSA credential spec named by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName + is the name of the GMSA credential + spec to use. + type: string + hostProcess: + description: |- + HostProcess determines if a container should be run as a 'Host Process' container. + All of a Pod's containers must have the same effective HostProcess value + (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). + In addition, if HostProcess is true then HostNetwork must also be set to true. + type: boolean + runAsUserName: + description: |- + The UserName in Windows to run the entrypoint of the container process. + Defaults to the user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: string + type: object + type: object + startupProbe: + description: Probes are not allowed for ephemeral + containers. + properties: + exec: + description: Exec specifies the action + to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action + involving a GRPC port. + properties: + port: + description: Port number of the gRPC + service. Number must be in the range + 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http + request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set + in the request. HTTP allows repeated + headers. + items: + description: HTTPHeader describes + a custom header to be used in + HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field + value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the + HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action + involving a TCP port. + properties: + host: + description: 'Optional: Host name + to connect to, defaults to the pod + IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + stdin: + description: |- + Whether this container should allocate a buffer for stdin in the container runtime. If this + is not set, reads from stdin in the container will always result in EOF. + Default is false. + type: boolean + stdinOnce: + description: |- + Whether the container runtime should close the stdin channel after it has been opened by + a single attach. When stdin is true the stdin stream will remain open across multiple attach + sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the + first client attaches to stdin, and then remains open and accepts data until the client disconnects, + at which time stdin is closed and remains closed until the container is restarted. If this + flag is false, a container processes that reads from stdin will never receive an EOF. + Default is false + type: boolean + targetContainerName: + description: |- + If set, the name of the container from PodSpec that this ephemeral container targets. + The ephemeral container will be run in the namespaces (IPC, PID, etc) of this container. + If not set then the ephemeral container uses the namespaces configured in the Pod spec. + + The container runtime must implement support for this feature. If the runtime does not + support namespace targeting then the result of setting this field is undefined. + type: string + terminationMessagePath: + description: |- + Optional: Path at which the file to which the container's termination message + will be written is mounted into the container's filesystem. + Message written is intended to be brief final status, such as an assertion failure message. + Will be truncated by the node if greater than 4096 bytes. The total message length across + all containers will be limited to 12kb. + Defaults to /dev/termination-log. + Cannot be updated. + type: string + terminationMessagePolicy: + description: |- + Indicate how the termination message should be populated. File will use the contents of + terminationMessagePath to populate the container status message on both success and failure. + FallbackToLogsOnError will use the last chunk of container log output if the termination + message file is empty and the container exited with an error. + The log output is limited to 2048 bytes or 80 lines, whichever is smaller. + Defaults to File. + Cannot be updated. + type: string + tty: + description: |- + Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. + Default is false. + type: boolean + volumeDevices: + description: volumeDevices is the list of + block devices to be used by the container. + items: + description: volumeDevice describes a mapping + of a raw block device within a container. + properties: + devicePath: + description: devicePath is the path + inside of the container that the device + will be mapped to. + type: string + name: + description: name must match the name + of a persistentVolumeClaim in the + pod + type: string + required: + - devicePath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - devicePath + x-kubernetes-list-type: map + volumeMounts: + description: |- + Pod volumes to mount into the container's filesystem. Subpath mounts are not allowed for ephemeral containers. + Cannot be updated. + items: + description: VolumeMount describes a mounting + of a Volume within a container. + properties: + mountPath: + description: |- + Path within the container at which the volume should be mounted. Must + not contain ':'. + type: string + mountPropagation: + description: |- + mountPropagation determines how mounts are propagated from the host + to container and the other way around. + When not set, MountPropagationNone is used. + This field is beta in 1.10. + When RecursiveReadOnly is set to IfPossible or to Enabled, MountPropagation must be None or unspecified + (which defaults to None). + type: string + name: + description: This must match the Name + of a Volume. + type: string + readOnly: + description: |- + Mounted read-only if true, read-write otherwise (false or unspecified). + Defaults to false. + type: boolean + recursiveReadOnly: + description: |- + RecursiveReadOnly specifies whether read-only mounts should be handled + recursively. + + If ReadOnly is false, this field has no meaning and must be unspecified. + + If ReadOnly is true, and this field is set to Disabled, the mount is not made + recursively read-only. If this field is set to IfPossible, the mount is made + recursively read-only, if it is supported by the container runtime. If this + field is set to Enabled, the mount is made recursively read-only if it is + supported by the container runtime, otherwise the pod will not be started and + an error will be generated to indicate the reason. + + If this field is set to IfPossible or Enabled, MountPropagation must be set to + None (or be unspecified, which defaults to None). + + If this field is not specified, it is treated as an equivalent of Disabled. + type: string + subPath: + description: |- + Path within the volume from which the container's volume should be mounted. + Defaults to "" (volume's root). + type: string + subPathExpr: + description: |- + Expanded path within the volume from which the container's volume should be mounted. + Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. + Defaults to "" (volume's root). + SubPathExpr and SubPath are mutually exclusive. + type: string + required: + - mountPath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - mountPath + x-kubernetes-list-type: map + workingDir: + description: |- + Container's working directory. + If not specified, the container runtime's default will be used, which + might be configured in the container image. + Cannot be updated. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + hostAliases: + description: |- + HostAliases is an optional list of hosts and IPs that will be injected into the pod's hosts + file if specified. + items: + description: |- + HostAlias holds the mapping between IP and hostnames that will be injected as an entry in the + pod's hosts file. + properties: + hostnames: + description: Hostnames for the above IP address. + items: + type: string + type: array + x-kubernetes-list-type: atomic + ip: + description: IP address of the host file entry. + type: string + required: + - ip + type: object + type: array + x-kubernetes-list-map-keys: + - ip + x-kubernetes-list-type: map + hostIPC: + description: |- + Use the host's ipc namespace. + Optional: Default to false. + type: boolean + hostNetwork: + description: |- + Host networking requested for this pod. Use the host's network namespace. + If this option is set, the ports that will be used must be specified. + Default to false. + type: boolean + hostPID: + description: |- + Use the host's pid namespace. + Optional: Default to false. + type: boolean + hostUsers: + description: |- + Use the host's user namespace. + Optional: Default to true. + If set to true or not present, the pod will be run in the host user namespace, useful + for when the pod needs a feature only available to the host user namespace, such as + loading a kernel module with CAP_SYS_MODULE. + When set to false, a new userns is created for the pod. Setting false is useful for + mitigating container breakout vulnerabilities even allowing users to run their + containers as root without actually having root privileges on the host. + This field is alpha-level and is only honored by servers that enable the UserNamespacesSupport feature. + type: boolean + hostname: + description: |- + Specifies the hostname of the Pod + If not specified, the pod's hostname will be set to a system-defined value. + type: string + imagePullSecrets: + description: |- + ImagePullSecrets is an optional list of references to secrets in the same namespace to use for pulling any of the images used by this PodSpec. + If specified, these secrets will be passed to individual puller implementations for them to use. + More info: https://kubernetes.io/docs/concepts/containers/images#specifying-imagepullsecrets-on-a-pod + items: + description: |- + LocalObjectReference contains enough information to let you locate the + referenced object inside the same namespace. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + initContainers: + description: |- + List of initialization containers belonging to the pod. + Init containers are executed in order prior to containers being started. If any + init container fails, the pod is considered to have failed and is handled according + to its restartPolicy. The name for an init container or normal container must be + unique among all containers. + Init containers may not have Lifecycle actions, Readiness probes, Liveness probes, or Startup probes. + The resourceRequirements of an init container are taken into account during scheduling + by finding the highest request/limit for each resource type, and then using the max of + of that value or the sum of the normal containers. Limits are applied to init containers + in a similar fashion. + Init containers cannot currently be added or removed. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/init-containers/ + items: + description: A single application container that + you want to run within a pod. + properties: + args: + description: |- + Arguments to the entrypoint. + The container image's CMD is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell + items: + type: string + type: array + x-kubernetes-list-type: atomic + command: + description: |- + Entrypoint array. Not executed within a shell. + The container image's ENTRYPOINT is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell + items: + type: string + type: array + x-kubernetes-list-type: atomic + env: + description: |- + List of environment variables to set in the container. + Cannot be updated. + items: + description: EnvVar represents an environment + variable present in a Container. + properties: + name: + description: Name of the environment + variable. Must be a C_IDENTIFIER. + type: string + value: + description: |- + Variable references $(VAR_NAME) are expanded + using the previously defined environment variables in the container and + any service environment variables. If a variable cannot be resolved, + the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. + "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless of whether the variable + exists or not. + Defaults to "". + type: string + valueFrom: + description: Source for the environment + variable's value. Cannot be used if + value is not empty. + properties: + configMapKeyRef: + description: Selects a key of a + ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether + the ConfigMap or its key must + be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + description: |- + Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, + spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. + properties: + apiVersion: + description: Version of the + schema the FieldPath is written + in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field + to select in the specified + API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. + properties: + containerName: + description: 'Container name: + required for volumes, optional + for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output + format of the exposed resources, + defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource + to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + description: Selects a key of a + secret in the pod's namespace + properties: + key: + description: The key of the + secret to select from. Must + be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether + the Secret or its key must + be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + envFrom: + description: |- + List of sources to populate environment variables in the container. + The keys defined within a source must be a C_IDENTIFIER. All invalid keys + will be reported as an event when the container is starting. When a key exists in multiple + sources, the value associated with the last source will take precedence. + Values defined by an Env with a duplicate key will take precedence. + Cannot be updated. + items: + description: EnvFromSource represents the + source of a set of ConfigMaps + properties: + configMapRef: + description: The ConfigMap to select + from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the + ConfigMap must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + description: An optional identifier + to prepend to each key in the ConfigMap. + Must be a C_IDENTIFIER. + type: string + secretRef: + description: The Secret to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the + Secret must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array + x-kubernetes-list-type: atomic + image: + description: |- + Container image name. + More info: https://kubernetes.io/docs/concepts/containers/images + This field is optional to allow higher level config management to default or override + container images in workload controllers like Deployments and StatefulSets. + type: string + imagePullPolicy: + description: |- + Image pull policy. + One of Always, Never, IfNotPresent. + Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/containers/images#updating-images + type: string + lifecycle: + description: |- + Actions that the management system should take in response to container lifecycle events. + Cannot be updated. + properties: + postStart: + description: |- + PostStart is called immediately after a container is created. If the handler fails, + the container is terminated and restarted according to its restart policy. + Other management of the container blocks until the hook completes. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies the action + to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies the + http request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to + set in the request. HTTP allows + repeated headers. + items: + description: HTTPHeader describes + a custom header to be used + in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header + field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on + the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents the + duration that the container should + sleep before being terminated. + properties: + seconds: + description: Seconds is the number + of seconds to sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for the backward compatibility. There are no validation of this field and + lifecycle hooks will fail in runtime when tcp handler is specified. + properties: + host: + description: 'Optional: Host name + to connect to, defaults to the + pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + preStop: + description: |- + PreStop is called immediately before a container is terminated due to an + API request or management event such as liveness/startup probe failure, + preemption, resource contention, etc. The handler is not called if the + container crashes or exits. The Pod's termination grace period countdown begins before the + PreStop hook is executed. Regardless of the outcome of the handler, the + container will eventually terminate within the Pod's termination grace + period (unless delayed by finalizers). Other management of the container blocks until the hook completes + or until the termination grace period is reached. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies the action + to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies the + http request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to + set in the request. HTTP allows + repeated headers. + items: + description: HTTPHeader describes + a custom header to be used + in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header + field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on + the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents the + duration that the container should + sleep before being terminated. + properties: + seconds: + description: Seconds is the number + of seconds to sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for the backward compatibility. There are no validation of this field and + lifecycle hooks will fail in runtime when tcp handler is specified. + properties: + host: + description: 'Optional: Host name + to connect to, defaults to the + pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + type: object + livenessProbe: + description: |- + Periodic probe of container liveness. + Container will be restarted if the probe fails. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies the action + to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action + involving a GRPC port. + properties: + port: + description: Port number of the gRPC + service. Number must be in the range + 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http + request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set + in the request. HTTP allows repeated + headers. + items: + description: HTTPHeader describes + a custom header to be used in + HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field + value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the + HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action + involving a TCP port. + properties: + host: + description: 'Optional: Host name + to connect to, defaults to the pod + IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + name: + description: |- + Name of the container specified as a DNS_LABEL. + Each container in a pod must have a unique name (DNS_LABEL). + Cannot be updated. + type: string + ports: + description: |- + List of ports to expose from the container. Not specifying a port here + DOES NOT prevent that port from being exposed. Any port which is + listening on the default "0.0.0.0" address inside a container will be + accessible from the network. + Modifying this array with strategic merge patch may corrupt the data. + For more information See https://github.com/kubernetes/kubernetes/issues/108255. + Cannot be updated. + items: + description: ContainerPort represents a + network port in a single container. + properties: + containerPort: + description: |- + Number of port to expose on the pod's IP address. + This must be a valid port number, 0 < x < 65536. + format: int32 + type: integer + hostIP: + description: What host IP to bind the + external port to. + type: string + hostPort: + description: |- + Number of port to expose on the host. + If specified, this must be a valid port number, 0 < x < 65536. + If HostNetwork is specified, this must match ContainerPort. + Most containers do not need this. + format: int32 + type: integer + name: + description: |- + If specified, this must be an IANA_SVC_NAME and unique within the pod. Each + named port in a pod must have a unique name. Name for the port that can be + referred to by services. + type: string + protocol: + default: TCP + description: |- + Protocol for port. Must be UDP, TCP, or SCTP. + Defaults to "TCP". + type: string + required: + - containerPort + type: object + type: array + x-kubernetes-list-map-keys: + - containerPort + - protocol + x-kubernetes-list-type: map + readinessProbe: + description: |- + Periodic probe of container service readiness. + Container will be removed from service endpoints if the probe fails. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies the action + to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action + involving a GRPC port. + properties: + port: + description: Port number of the gRPC + service. Number must be in the range + 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http + request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set + in the request. HTTP allows repeated + headers. + items: + description: HTTPHeader describes + a custom header to be used in + HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field + value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the + HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action + involving a TCP port. + properties: + host: + description: 'Optional: Host name + to connect to, defaults to the pod + IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + resizePolicy: + description: Resources resize policy for the + container. + items: + description: ContainerResizePolicy represents + resource resize policy for the container. + properties: + resourceName: + description: |- + Name of the resource to which this resource resize policy applies. + Supported values: cpu, memory. + type: string + restartPolicy: + description: |- + Restart policy to apply when specified resource is resized. + If not specified, it defaults to NotRequired. + type: string + required: + - resourceName + - restartPolicy + type: object + type: array + x-kubernetes-list-type: atomic + resources: + description: |- + Compute Resources required by this container. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references + one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + restartPolicy: + description: |- + RestartPolicy defines the restart behavior of individual containers in a pod. + This field may only be set for init containers, and the only allowed value is "Always". + For non-init containers or when this field is not specified, + the restart behavior is defined by the Pod's restart policy and the container type. + Setting the RestartPolicy as "Always" for the init container will have the following effect: + this init container will be continually restarted on + exit until all regular containers have terminated. Once all regular + containers have completed, all init containers with restartPolicy "Always" + will be shut down. This lifecycle differs from normal init containers and + is often referred to as a "sidecar" container. Although this init + container still starts in the init container sequence, it does not wait + for the container to complete before proceeding to the next init + container. Instead, the next init container starts immediately after this + init container is started, or after any startupProbe has successfully + completed. + type: string + securityContext: + description: |- + SecurityContext defines the security options the container should be run with. + If set, the fields of SecurityContext override the equivalent fields of PodSecurityContext. + More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ + properties: + allowPrivilegeEscalation: + description: |- + AllowPrivilegeEscalation controls whether a process can gain more + privileges than its parent process. This bool directly controls if + the no_new_privs flag will be set on the container process. + AllowPrivilegeEscalation is true always when the container is: + 1) run as Privileged + 2) has CAP_SYS_ADMIN + Note that this field cannot be set when spec.os.name is windows. + type: boolean + appArmorProfile: + description: |- + appArmorProfile is the AppArmor options to use by this container. If set, this profile + overrides the pod's appArmorProfile. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile loaded on the node that should be used. + The profile must be preconfigured on the node to work. + Must match the loaded name of the profile. + Must be set if and only if type is "Localhost". + type: string + type: + description: |- + type indicates which kind of AppArmor profile will be applied. + Valid options are: + Localhost - a profile pre-loaded on the node. + RuntimeDefault - the container runtime's default profile. + Unconfined - no AppArmor enforcement. + type: string + required: + - type + type: object + capabilities: + description: |- + The capabilities to add/drop when running containers. + Defaults to the default set of capabilities granted by the container runtime. + Note that this field cannot be set when spec.os.name is windows. + properties: + add: + description: Added capabilities + items: + description: Capability represent + POSIX capabilities type + type: string + type: array + x-kubernetes-list-type: atomic + drop: + description: Removed capabilities + items: + description: Capability represent + POSIX capabilities type + type: string + type: array + x-kubernetes-list-type: atomic + type: object + privileged: + description: |- + Run container in privileged mode. + Processes in privileged containers are essentially equivalent to root on the host. + Defaults to false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + procMount: + description: |- + procMount denotes the type of proc mount to use for the containers. + The default value is Default which uses the container runtime defaults for + readonly paths and masked paths. + This requires the ProcMountType feature flag to be enabled. + Note that this field cannot be set when spec.os.name is windows. + type: string + readOnlyRootFilesystem: + description: |- + Whether this container has a read-only root filesystem. + Default is false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + runAsGroup: + description: |- + The GID to run the entrypoint of the container process. + Uses runtime default if unset. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + runAsNonRoot: + description: |- + Indicates that the container must run as a non-root user. + If true, the Kubelet will validate the image at runtime to ensure that it + does not run as UID 0 (root) and fail to start the container if it does. + If unset or false, no such validation will be performed. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: |- + The UID to run the entrypoint of the container process. + Defaults to user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + seLinuxOptions: + description: |- + The SELinux context to be applied to the container. + If unspecified, the container runtime will allocate a random SELinux context for each + container. May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + properties: + level: + description: Level is SELinux level + label that applies to the container. + type: string + role: + description: Role is a SELinux role + label that applies to the container. + type: string + type: + description: Type is a SELinux type + label that applies to the container. + type: string + user: + description: User is a SELinux user + label that applies to the container. + type: string + type: object + seccompProfile: + description: |- + The seccomp options to use by this container. If seccomp options are + provided at both the pod & container level, the container options + override the pod options. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile defined in a file on the node should be used. + The profile must be preconfigured on the node to work. + Must be a descending path, relative to the kubelet's configured seccomp profile location. + Must be set if type is "Localhost". Must NOT be set for any other type. + type: string + type: + description: |- + type indicates which kind of seccomp profile will be applied. + Valid options are: + + Localhost - a profile defined in a file on the node should be used. + RuntimeDefault - the container runtime default profile should be used. + Unconfined - no profile should be applied. + type: string + required: + - type + type: object + windowsOptions: + description: |- + The Windows specific settings applied to all containers. + If unspecified, the options from the PodSecurityContext will be used. + If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is linux. + properties: + gmsaCredentialSpec: + description: |- + GMSACredentialSpec is where the GMSA admission webhook + (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the + GMSA credential spec named by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName + is the name of the GMSA credential + spec to use. + type: string + hostProcess: + description: |- + HostProcess determines if a container should be run as a 'Host Process' container. + All of a Pod's containers must have the same effective HostProcess value + (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). + In addition, if HostProcess is true then HostNetwork must also be set to true. + type: boolean + runAsUserName: + description: |- + The UserName in Windows to run the entrypoint of the container process. + Defaults to the user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: string + type: object + type: object + startupProbe: + description: |- + StartupProbe indicates that the Pod has successfully initialized. + If specified, no other probes are executed until this completes successfully. + If this probe fails, the Pod will be restarted, just as if the livenessProbe failed. + This can be used to provide different probe parameters at the beginning of a Pod's lifecycle, + when it might take a long time to load data or warm a cache, than during steady-state operation. + This cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies the action + to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action + involving a GRPC port. + properties: + port: + description: Port number of the gRPC + service. Number must be in the range + 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http + request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set + in the request. HTTP allows repeated + headers. + items: + description: HTTPHeader describes + a custom header to be used in + HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field + value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the + HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action + involving a TCP port. + properties: + host: + description: 'Optional: Host name + to connect to, defaults to the pod + IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + stdin: + description: |- + Whether this container should allocate a buffer for stdin in the container runtime. If this + is not set, reads from stdin in the container will always result in EOF. + Default is false. + type: boolean + stdinOnce: + description: |- + Whether the container runtime should close the stdin channel after it has been opened by + a single attach. When stdin is true the stdin stream will remain open across multiple attach + sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the + first client attaches to stdin, and then remains open and accepts data until the client disconnects, + at which time stdin is closed and remains closed until the container is restarted. If this + flag is false, a container processes that reads from stdin will never receive an EOF. + Default is false + type: boolean + terminationMessagePath: + description: |- + Optional: Path at which the file to which the container's termination message + will be written is mounted into the container's filesystem. + Message written is intended to be brief final status, such as an assertion failure message. + Will be truncated by the node if greater than 4096 bytes. The total message length across + all containers will be limited to 12kb. + Defaults to /dev/termination-log. + Cannot be updated. + type: string + terminationMessagePolicy: + description: |- + Indicate how the termination message should be populated. File will use the contents of + terminationMessagePath to populate the container status message on both success and failure. + FallbackToLogsOnError will use the last chunk of container log output if the termination + message file is empty and the container exited with an error. + The log output is limited to 2048 bytes or 80 lines, whichever is smaller. + Defaults to File. + Cannot be updated. + type: string + tty: + description: |- + Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. + Default is false. + type: boolean + volumeDevices: + description: volumeDevices is the list of + block devices to be used by the container. + items: + description: volumeDevice describes a mapping + of a raw block device within a container. + properties: + devicePath: + description: devicePath is the path + inside of the container that the device + will be mapped to. + type: string + name: + description: name must match the name + of a persistentVolumeClaim in the + pod + type: string + required: + - devicePath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - devicePath + x-kubernetes-list-type: map + volumeMounts: + description: |- + Pod volumes to mount into the container's filesystem. + Cannot be updated. + items: + description: VolumeMount describes a mounting + of a Volume within a container. + properties: + mountPath: + description: |- + Path within the container at which the volume should be mounted. Must + not contain ':'. + type: string + mountPropagation: + description: |- + mountPropagation determines how mounts are propagated from the host + to container and the other way around. + When not set, MountPropagationNone is used. + This field is beta in 1.10. + When RecursiveReadOnly is set to IfPossible or to Enabled, MountPropagation must be None or unspecified + (which defaults to None). + type: string + name: + description: This must match the Name + of a Volume. + type: string + readOnly: + description: |- + Mounted read-only if true, read-write otherwise (false or unspecified). + Defaults to false. + type: boolean + recursiveReadOnly: + description: |- + RecursiveReadOnly specifies whether read-only mounts should be handled + recursively. + + If ReadOnly is false, this field has no meaning and must be unspecified. + + If ReadOnly is true, and this field is set to Disabled, the mount is not made + recursively read-only. If this field is set to IfPossible, the mount is made + recursively read-only, if it is supported by the container runtime. If this + field is set to Enabled, the mount is made recursively read-only if it is + supported by the container runtime, otherwise the pod will not be started and + an error will be generated to indicate the reason. + + If this field is set to IfPossible or Enabled, MountPropagation must be set to + None (or be unspecified, which defaults to None). + + If this field is not specified, it is treated as an equivalent of Disabled. + type: string + subPath: + description: |- + Path within the volume from which the container's volume should be mounted. + Defaults to "" (volume's root). + type: string + subPathExpr: + description: |- + Expanded path within the volume from which the container's volume should be mounted. + Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. + Defaults to "" (volume's root). + SubPathExpr and SubPath are mutually exclusive. + type: string + required: + - mountPath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - mountPath + x-kubernetes-list-type: map + workingDir: + description: |- + Container's working directory. + If not specified, the container runtime's default will be used, which + might be configured in the container image. + Cannot be updated. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + nodeName: + description: |- + NodeName indicates in which node this pod is scheduled. + If empty, this pod is a candidate for scheduling by the scheduler defined in schedulerName. + Once this field is set, the kubelet for this node becomes responsible for the lifecycle of this pod. + This field should not be used to express a desire for the pod to be scheduled on a specific node. + https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodename + type: string + nodeSelector: + additionalProperties: + type: string + description: |- + NodeSelector is a selector which must be true for the pod to fit on a node. + Selector which must match a node's labels for the pod to be scheduled on that node. + More info: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ + type: object + x-kubernetes-map-type: atomic + os: + description: |- + Specifies the OS of the containers in the pod. + Some pod and container fields are restricted if this is set. + + If the OS field is set to linux, the following fields must be unset: + -securityContext.windowsOptions + + If the OS field is set to windows, following fields must be unset: + - spec.hostPID + - spec.hostIPC + - spec.hostUsers + - spec.securityContext.appArmorProfile + - spec.securityContext.seLinuxOptions + - spec.securityContext.seccompProfile + - spec.securityContext.fsGroup + - spec.securityContext.fsGroupChangePolicy + - spec.securityContext.sysctls + - spec.shareProcessNamespace + - spec.securityContext.runAsUser + - spec.securityContext.runAsGroup + - spec.securityContext.supplementalGroups + - spec.securityContext.supplementalGroupsPolicy + - spec.containers[*].securityContext.appArmorProfile + - spec.containers[*].securityContext.seLinuxOptions + - spec.containers[*].securityContext.seccompProfile + - spec.containers[*].securityContext.capabilities + - spec.containers[*].securityContext.readOnlyRootFilesystem + - spec.containers[*].securityContext.privileged + - spec.containers[*].securityContext.allowPrivilegeEscalation + - spec.containers[*].securityContext.procMount + - spec.containers[*].securityContext.runAsUser + - spec.containers[*].securityContext.runAsGroup + properties: + name: + description: |- + Name is the name of the operating system. The currently supported values are linux and windows. + Additional value may be defined in future and can be one of: + https://github.com/opencontainers/runtime-spec/blob/master/config.md#platform-specific-configuration + Clients should expect to handle additional values and treat unrecognized values in this field as os: null + type: string + required: + - name + type: object + overhead: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Overhead represents the resource overhead associated with running a pod for a given RuntimeClass. + This field will be autopopulated at admission time by the RuntimeClass admission controller. If + the RuntimeClass admission controller is enabled, overhead must not be set in Pod create requests. + The RuntimeClass admission controller will reject Pod create requests which have the overhead already + set. If RuntimeClass is configured and selected in the PodSpec, Overhead will be set to the value + defined in the corresponding RuntimeClass, otherwise it will remain unset and treated as zero. + More info: https://git.k8s.io/enhancements/keps/sig-node/688-pod-overhead/README.md + type: object + preemptionPolicy: + description: |- + PreemptionPolicy is the Policy for preempting pods with lower priority. + One of Never, PreemptLowerPriority. + Defaults to PreemptLowerPriority if unset. + type: string + priority: + description: |- + The priority value. Various system components use this field to find the + priority of the pod. When Priority Admission Controller is enabled, it + prevents users from setting this field. The admission controller populates + this field from PriorityClassName. + The higher the value, the higher the priority. + format: int32 + type: integer + priorityClassName: + description: |- + If specified, indicates the pod's priority. "system-node-critical" and + "system-cluster-critical" are two special keywords which indicate the + highest priorities with the former being the highest priority. Any other + name must be defined by creating a PriorityClass object with that name. + If not specified, the pod priority will be default or zero if there is no + default. + type: string + readinessGates: + description: |- + If specified, all readiness gates will be evaluated for pod readiness. + A pod is ready when all its containers are ready AND + all conditions specified in the readiness gates have status equal to "True" + More info: https://git.k8s.io/enhancements/keps/sig-network/580-pod-readiness-gates + items: + description: PodReadinessGate contains the reference + to a pod condition + properties: + conditionType: + description: ConditionType refers to a condition + in the pod's condition list with matching + type. + type: string + required: + - conditionType + type: object + type: array + x-kubernetes-list-type: atomic + resourceClaims: + description: |- + ResourceClaims defines which ResourceClaims must be allocated + and reserved before the Pod is allowed to start. The resources + will be made available to those containers which consume them + by name. + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + This field is immutable. + items: + description: |- + PodResourceClaim references exactly one ResourceClaim, either directly + or by naming a ResourceClaimTemplate which is then turned into a ResourceClaim + for the pod. + + It adds a name to it that uniquely identifies the ResourceClaim inside the Pod. + Containers that need access to the ResourceClaim reference it with this name. + properties: + name: + description: |- + Name uniquely identifies this resource claim inside the pod. + This must be a DNS_LABEL. + type: string + resourceClaimName: + description: |- + ResourceClaimName is the name of a ResourceClaim object in the same + namespace as this pod. + + Exactly one of ResourceClaimName and ResourceClaimTemplateName must + be set. + type: string + resourceClaimTemplateName: + description: |- + ResourceClaimTemplateName is the name of a ResourceClaimTemplate + object in the same namespace as this pod. + + The template will be used to create a new ResourceClaim, which will + be bound to this pod. When this pod is deleted, the ResourceClaim + will also be deleted. The pod name and resource name, along with a + generated component, will be used to form a unique name for the + ResourceClaim, which will be recorded in pod.status.resourceClaimStatuses. + + This field is immutable and no changes will be made to the + corresponding ResourceClaim by the control plane after creating the + ResourceClaim. + + Exactly one of ResourceClaimName and ResourceClaimTemplateName must + be set. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + restartPolicy: + description: |- + Restart policy for all containers within the pod. + One of Always, OnFailure, Never. In some contexts, only a subset of those values may be permitted. + Default to Always. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#restart-policy + type: string + runtimeClassName: + description: |- + RuntimeClassName refers to a RuntimeClass object in the node.k8s.io group, which should be used + to run this pod. If no RuntimeClass resource matches the named class, the pod will not be run. + If unset or empty, the "legacy" RuntimeClass will be used, which is an implicit class with an + empty definition that uses the default runtime handler. + More info: https://git.k8s.io/enhancements/keps/sig-node/585-runtime-class + type: string + schedulerName: + description: |- + If specified, the pod will be dispatched by specified scheduler. + If not specified, the pod will be dispatched by default scheduler. + type: string + schedulingGates: + description: |- + SchedulingGates is an opaque list of values that if specified will block scheduling the pod. + If schedulingGates is not empty, the pod will stay in the SchedulingGated state and the + scheduler will not attempt to schedule the pod. + + SchedulingGates can only be set at pod creation time, and be removed only afterwards. + items: + description: PodSchedulingGate is associated to + a Pod to guard its scheduling. + properties: + name: + description: |- + Name of the scheduling gate. + Each scheduling gate must have a unique name field. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + securityContext: + description: |- + SecurityContext holds pod-level security attributes and common container settings. + Optional: Defaults to empty. See type description for default values of each field. + properties: + appArmorProfile: + description: |- + appArmorProfile is the AppArmor options to use by the containers in this pod. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile loaded on the node that should be used. + The profile must be preconfigured on the node to work. + Must match the loaded name of the profile. + Must be set if and only if type is "Localhost". + type: string + type: + description: |- + type indicates which kind of AppArmor profile will be applied. + Valid options are: + Localhost - a profile pre-loaded on the node. + RuntimeDefault - the container runtime's default profile. + Unconfined - no AppArmor enforcement. + type: string + required: + - type + type: object + fsGroup: + description: |- + A special supplemental group that applies to all containers in a pod. + Some volume types allow the Kubelet to change the ownership of that volume + to be owned by the pod: + + 1. The owning GID will be the FSGroup + 2. The setgid bit is set (new files created in the volume will be owned by FSGroup) + 3. The permission bits are OR'd with rw-rw---- + + If unset, the Kubelet will not modify the ownership and permissions of any volume. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + fsGroupChangePolicy: + description: |- + fsGroupChangePolicy defines behavior of changing ownership and permission of the volume + before being exposed inside Pod. This field will only apply to + volume types which support fsGroup based ownership(and permissions). + It will have no effect on ephemeral volume types such as: secret, configmaps + and emptydir. + Valid values are "OnRootMismatch" and "Always". If not specified, "Always" is used. + Note that this field cannot be set when spec.os.name is windows. + type: string + runAsGroup: + description: |- + The GID to run the entrypoint of the container process. + Uses runtime default if unset. + May also be set in SecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence + for that container. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + runAsNonRoot: + description: |- + Indicates that the container must run as a non-root user. + If true, the Kubelet will validate the image at runtime to ensure that it + does not run as UID 0 (root) and fail to start the container if it does. + If unset or false, no such validation will be performed. + May also be set in SecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: |- + The UID to run the entrypoint of the container process. + Defaults to user specified in image metadata if unspecified. + May also be set in SecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence + for that container. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + seLinuxOptions: + description: |- + The SELinux context to be applied to all containers. + If unspecified, the container runtime will allocate a random SELinux context for each + container. May also be set in SecurityContext. If set in + both SecurityContext and PodSecurityContext, the value specified in SecurityContext + takes precedence for that container. + Note that this field cannot be set when spec.os.name is windows. + properties: + level: + description: Level is SELinux level label + that applies to the container. + type: string + role: + description: Role is a SELinux role label + that applies to the container. + type: string + type: + description: Type is a SELinux type label + that applies to the container. + type: string + user: + description: User is a SELinux user label + that applies to the container. + type: string + type: object + seccompProfile: + description: |- + The seccomp options to use by the containers in this pod. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile defined in a file on the node should be used. + The profile must be preconfigured on the node to work. + Must be a descending path, relative to the kubelet's configured seccomp profile location. + Must be set if type is "Localhost". Must NOT be set for any other type. + type: string + type: + description: |- + type indicates which kind of seccomp profile will be applied. + Valid options are: + + Localhost - a profile defined in a file on the node should be used. + RuntimeDefault - the container runtime default profile should be used. + Unconfined - no profile should be applied. + type: string + required: + - type + type: object + supplementalGroups: + description: |- + A list of groups applied to the first process run in each container, in + addition to the container's primary GID and fsGroup (if specified). If + the SupplementalGroupsPolicy feature is enabled, the + supplementalGroupsPolicy field determines whether these are in addition + to or instead of any group memberships defined in the container image. + If unspecified, no additional groups are added, though group memberships + defined in the container image may still be used, depending on the + supplementalGroupsPolicy field. + Note that this field cannot be set when spec.os.name is windows. + items: + format: int64 + type: integer + type: array + x-kubernetes-list-type: atomic + supplementalGroupsPolicy: + description: |- + Defines how supplemental groups of the first container processes are calculated. + Valid values are "Merge" and "Strict". If not specified, "Merge" is used. + (Alpha) Using the field requires the SupplementalGroupsPolicy feature gate to be enabled + and the container runtime must implement support for this feature. + Note that this field cannot be set when spec.os.name is windows. + type: string + sysctls: + description: |- + Sysctls hold a list of namespaced sysctls used for the pod. Pods with unsupported + sysctls (by the container runtime) might fail to launch. + Note that this field cannot be set when spec.os.name is windows. + items: + description: Sysctl defines a kernel parameter + to be set + properties: + name: + description: Name of a property to set + type: string + value: + description: Value of a property to set + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + windowsOptions: + description: |- + The Windows specific settings applied to all containers. + If unspecified, the options within a container's SecurityContext will be used. + If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is linux. + properties: + gmsaCredentialSpec: + description: |- + GMSACredentialSpec is where the GMSA admission webhook + (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the + GMSA credential spec named by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName is the + name of the GMSA credential spec to use. + type: string + hostProcess: + description: |- + HostProcess determines if a container should be run as a 'Host Process' container. + All of a Pod's containers must have the same effective HostProcess value + (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). + In addition, if HostProcess is true then HostNetwork must also be set to true. + type: boolean + runAsUserName: + description: |- + The UserName in Windows to run the entrypoint of the container process. + Defaults to the user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: string + type: object + type: object + serviceAccount: + description: |- + DeprecatedServiceAccount is a deprecated alias for ServiceAccountName. + Deprecated: Use serviceAccountName instead. + type: string + serviceAccountName: + description: |- + ServiceAccountName is the name of the ServiceAccount to use to run this pod. + More info: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ + type: string + setHostnameAsFQDN: + description: |- + If true the pod's hostname will be configured as the pod's FQDN, rather than the leaf name (the default). + In Linux containers, this means setting the FQDN in the hostname field of the kernel (the nodename field of struct utsname). + In Windows containers, this means setting the registry value of hostname for the registry key HKEY_LOCAL_MACHINE\\SYSTEM\\CurrentControlSet\\Services\\Tcpip\\Parameters to FQDN. + If a pod does not have FQDN, this has no effect. + Default to false. + type: boolean + shareProcessNamespace: + description: |- + Share a single process namespace between all of the containers in a pod. + When this is set containers will be able to view and signal processes from other containers + in the same pod, and the first process in each container will not be assigned PID 1. + HostPID and ShareProcessNamespace cannot both be set. + Optional: Default to false. + type: boolean + subdomain: + description: |- + If specified, the fully qualified Pod hostname will be "...svc.". + If not specified, the pod will not have a domainname at all. + type: string + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully. May be decreased in delete request. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + If this value is nil, the default grace period will be used instead. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + Defaults to 30 seconds. + format: int64 + type: integer + tolerations: + description: If specified, the pod's tolerations. + items: + description: |- + The pod this Toleration is attached to tolerates any taint that matches + the triple using the matching operator . + properties: + effect: + description: |- + Effect indicates the taint effect to match. Empty means match all taint effects. + When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: |- + Key is the taint key that the toleration applies to. Empty means match all taint keys. + If the key is empty, operator must be Exists; this combination means to match all values and all keys. + type: string + operator: + description: |- + Operator represents a key's relationship to the value. + Valid operators are Exists and Equal. Defaults to Equal. + Exists is equivalent to wildcard for value, so that a pod can + tolerate all taints of a particular category. + type: string + tolerationSeconds: + description: |- + TolerationSeconds represents the period of time the toleration (which must be + of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, + it is not set, which means tolerate the taint forever (do not evict). Zero and + negative values will be treated as 0 (evict immediately) by the system. + format: int64 + type: integer + value: + description: |- + Value is the taint value the toleration matches to. + If the operator is Exists, the value should be empty, otherwise just a regular string. + type: string + type: object + type: array + x-kubernetes-list-type: atomic + topologySpreadConstraints: + description: |- + TopologySpreadConstraints describes how a group of pods ought to spread across topology + domains. Scheduler will schedule pods in a way which abides by the constraints. + All topologySpreadConstraints are ANDed. + items: + description: TopologySpreadConstraint specifies + how to spread matching pods among the given + topology. + properties: + labelSelector: + description: |- + LabelSelector is used to find matching pods. + Pods that match this label selector are counted to determine the number of pods + in their corresponding topology domain. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The + requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key + that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select the pods over which + spreading will be calculated. The keys are used to lookup values from the + incoming pod labels, those key-value labels are ANDed with labelSelector + to select the group of existing pods over which spreading will be calculated + for the incoming pod. The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + MatchLabelKeys cannot be set when LabelSelector isn't set. + Keys that don't exist in the incoming pod labels will + be ignored. A null or empty list means only match against labelSelector. + + This is a beta field and requires the MatchLabelKeysInPodTopologySpread feature gate to be enabled (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + maxSkew: + description: |- + MaxSkew describes the degree to which pods may be unevenly distributed. + When `whenUnsatisfiable=DoNotSchedule`, it is the maximum permitted difference + between the number of matching pods in the target topology and the global minimum. + The global minimum is the minimum number of matching pods in an eligible domain + or zero if the number of eligible domains is less than MinDomains. + For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same + labelSelector spread as 2/2/1: + In this case, the global minimum is 1. + | zone1 | zone2 | zone3 | + | P P | P P | P | + - if MaxSkew is 1, incoming pod can only be scheduled to zone3 to become 2/2/2; + scheduling it onto zone1(zone2) would make the ActualSkew(3-1) on zone1(zone2) + violate MaxSkew(1). + - if MaxSkew is 2, incoming pod can be scheduled onto any zone. + When `whenUnsatisfiable=ScheduleAnyway`, it is used to give higher precedence + to topologies that satisfy it. + It's a required field. Default value is 1 and 0 is not allowed. + format: int32 + type: integer + minDomains: + description: |- + MinDomains indicates a minimum number of eligible domains. + When the number of eligible domains with matching topology keys is less than minDomains, + Pod Topology Spread treats "global minimum" as 0, and then the calculation of Skew is performed. + And when the number of eligible domains with matching topology keys equals or greater than minDomains, + this value has no effect on scheduling. + As a result, when the number of eligible domains is less than minDomains, + scheduler won't schedule more than maxSkew Pods to those domains. + If value is nil, the constraint behaves as if MinDomains is equal to 1. + Valid values are integers greater than 0. + When value is not nil, WhenUnsatisfiable must be DoNotSchedule. + + For example, in a 3-zone cluster, MaxSkew is set to 2, MinDomains is set to 5 and pods with the same + labelSelector spread as 2/2/2: + | zone1 | zone2 | zone3 | + | P P | P P | P P | + The number of domains is less than 5(MinDomains), so "global minimum" is treated as 0. + In this situation, new pod with the same labelSelector cannot be scheduled, + because computed skew will be 3(3 - 0) if new Pod is scheduled to any of the three zones, + it will violate MaxSkew. + format: int32 + type: integer + nodeAffinityPolicy: + description: |- + NodeAffinityPolicy indicates how we will treat Pod's nodeAffinity/nodeSelector + when calculating pod topology spread skew. Options are: + - Honor: only nodes matching nodeAffinity/nodeSelector are included in the calculations. + - Ignore: nodeAffinity/nodeSelector are ignored. All nodes are included in the calculations. + + If this value is nil, the behavior is equivalent to the Honor policy. + This is a beta-level feature default enabled by the NodeInclusionPolicyInPodTopologySpread feature flag. + type: string + nodeTaintsPolicy: + description: |- + NodeTaintsPolicy indicates how we will treat node taints when calculating + pod topology spread skew. Options are: + - Honor: nodes without taints, along with tainted nodes for which the incoming pod + has a toleration, are included. + - Ignore: node taints are ignored. All nodes are included. + + If this value is nil, the behavior is equivalent to the Ignore policy. + This is a beta-level feature default enabled by the NodeInclusionPolicyInPodTopologySpread feature flag. + type: string + topologyKey: + description: |- + TopologyKey is the key of node labels. Nodes that have a label with this key + and identical values are considered to be in the same topology. + We consider each as a "bucket", and try to put balanced number + of pods into each bucket. + We define a domain as a particular instance of a topology. + Also, we define an eligible domain as a domain whose nodes meet the requirements of + nodeAffinityPolicy and nodeTaintsPolicy. + e.g. If TopologyKey is "kubernetes.io/hostname", each Node is a domain of that topology. + And, if TopologyKey is "topology.kubernetes.io/zone", each zone is a domain of that topology. + It's a required field. + type: string + whenUnsatisfiable: + description: |- + WhenUnsatisfiable indicates how to deal with a pod if it doesn't satisfy + the spread constraint. + - DoNotSchedule (default) tells the scheduler not to schedule it. + - ScheduleAnyway tells the scheduler to schedule the pod in any location, + but giving higher precedence to topologies that would help reduce the + skew. + A constraint is considered "Unsatisfiable" for an incoming pod + if and only if every possible node assignment for that pod would violate + "MaxSkew" on some topology. + For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same + labelSelector spread as 3/1/1: + | zone1 | zone2 | zone3 | + | P P P | P | P | + If WhenUnsatisfiable is set to DoNotSchedule, incoming pod can only be scheduled + to zone2(zone3) to become 3/2/1(3/1/2) as ActualSkew(2-1) on zone2(zone3) satisfies + MaxSkew(1). In other words, the cluster can still be imbalanced, but scheduler + won't make it *more* imbalanced. + It's a required field. + type: string + required: + - maxSkew + - topologyKey + - whenUnsatisfiable + type: object + type: array + x-kubernetes-list-map-keys: + - topologyKey + - whenUnsatisfiable + x-kubernetes-list-type: map + volumes: + description: |- + List of volumes that can be mounted by containers belonging to the pod. + More info: https://kubernetes.io/docs/concepts/storage/volumes + items: + description: Volume represents a named volume + in a pod that may be accessed by any container + in the pod. + properties: + awsElasticBlockStore: + description: |- + awsElasticBlockStore represents an AWS Disk resource that is attached to a + kubelet's host machine and then exposed to the pod. + More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore + properties: + fsType: + description: |- + fsType is the filesystem type of the volume that you want to mount. + Tip: Ensure that the filesystem type is supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore + type: string + partition: + description: |- + partition is the partition in the volume that you want to mount. + If omitted, the default is to mount by volume name. + Examples: For volume /dev/sda1, you specify the partition as "1". + Similarly, the volume partition for /dev/sda is "0" (or you can leave the property empty). + format: int32 + type: integer + readOnly: + description: |- + readOnly value true will force the readOnly setting in VolumeMounts. + More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore + type: boolean + volumeID: + description: |- + volumeID is unique ID of the persistent disk resource in AWS (Amazon EBS volume). + More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore + type: string + required: + - volumeID + type: object + azureDisk: + description: azureDisk represents an Azure + Data Disk mount on the host and bind mount + to the pod. + properties: + cachingMode: + description: 'cachingMode is the Host + Caching mode: None, Read Only, Read + Write.' + type: string + diskName: + description: diskName is the Name of the + data disk in the blob storage + type: string + diskURI: + description: diskURI is the URI of data + disk in the blob storage + type: string + fsType: + default: ext4 + description: |- + fsType is Filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + kind: + description: 'kind expected values are + Shared: multiple blob disks per storage + account Dedicated: single blob disk + per storage account Managed: azure + managed data disk (only in managed availability + set). defaults to shared' + type: string + readOnly: + default: false + description: |- + readOnly Defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + required: + - diskName + - diskURI + type: object + azureFile: + description: azureFile represents an Azure + File Service mount on the host and bind + mount to the pod. + properties: + readOnly: + description: |- + readOnly defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + secretName: + description: secretName is the name of + secret that contains Azure Storage Account + Name and Key + type: string + shareName: + description: shareName is the azure share + Name + type: string + required: + - secretName + - shareName + type: object + cephfs: + description: cephFS represents a Ceph FS mount + on the host that shares a pod's lifetime + properties: + monitors: + description: |- + monitors is Required: Monitors is a collection of Ceph monitors + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + items: + type: string + type: array + x-kubernetes-list-type: atomic + path: + description: 'path is Optional: Used as + the mounted root, rather than the full + Ceph tree, default is /' + type: string + readOnly: + description: |- + readOnly is Optional: Defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + type: boolean + secretFile: + description: |- + secretFile is Optional: SecretFile is the path to key ring for User, default is /etc/ceph/user.secret + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + type: string + secretRef: + description: |- + secretRef is Optional: SecretRef is reference to the authentication secret for User, default is empty. + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + user: + description: |- + user is optional: User is the rados user name, default is admin + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + type: string + required: + - monitors + type: object + cinder: + description: |- + cinder represents a cinder volume attached and mounted on kubelets host machine. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md + properties: + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md + type: string + readOnly: + description: |- + readOnly defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md + type: boolean + secretRef: + description: |- + secretRef is optional: points to a secret object containing parameters used to connect + to OpenStack. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + volumeID: + description: |- + volumeID used to identify the volume in cinder. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md + type: string + required: + - volumeID + type: object + configMap: + description: configMap represents a configMap + that should populate this volume + properties: + defaultMode: + description: |- + defaultMode is optional: mode bits used to set permissions on created files by default. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + Defaults to 0644. + Directories within the path are not affected by this setting. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + items: + description: |- + items if unspecified, each key-value pair in the Data field of the referenced + ConfigMap will be projected into the volume as a file whose name is the + key and content is the value. If specified, the listed keys will be + projected into the specified paths, and unlisted keys will not be + present. If a key is specified which is not present in the ConfigMap, + the volume setup will error unless it is marked optional. Paths must be + relative and may not contain the '..' path or start with '..'. + items: + description: Maps a string key to a + path within a volume. + properties: + key: + description: key is the key to project. + type: string + mode: + description: |- + mode is Optional: mode bits used to set permissions on this file. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: |- + path is the relative path of the file to map the key to. + May not be an absolute path. + May not contain the path element '..'. + May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: optional specify whether + the ConfigMap or its keys must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + csi: + description: csi (Container Storage Interface) + represents ephemeral storage that is handled + by certain external CSI drivers (Beta feature). + properties: + driver: + description: |- + driver is the name of the CSI driver that handles this volume. + Consult with your admin for the correct name as registered in the cluster. + type: string + fsType: + description: |- + fsType to mount. Ex. "ext4", "xfs", "ntfs". + If not provided, the empty value is passed to the associated CSI driver + which will determine the default filesystem to apply. + type: string + nodePublishSecretRef: + description: |- + nodePublishSecretRef is a reference to the secret object containing + sensitive information to pass to the CSI driver to complete the CSI + NodePublishVolume and NodeUnpublishVolume calls. + This field is optional, and may be empty if no secret is required. If the + secret object contains more than one secret, all secret references are passed. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + readOnly: + description: |- + readOnly specifies a read-only configuration for the volume. + Defaults to false (read/write). + type: boolean + volumeAttributes: + additionalProperties: + type: string + description: |- + volumeAttributes stores driver-specific properties that are passed to the CSI + driver. Consult your driver's documentation for supported values. + type: object + required: + - driver + type: object + downwardAPI: + description: downwardAPI represents downward + API about the pod that should populate this + volume + properties: + defaultMode: + description: |- + Optional: mode bits to use on created files by default. Must be a + Optional: mode bits used to set permissions on created files by default. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + Defaults to 0644. + Directories within the path are not affected by this setting. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + items: + description: Items is a list of downward + API volume file + items: + description: DownwardAPIVolumeFile represents + information to create the file containing + the pod field + properties: + fieldRef: + description: 'Required: Selects + a field of the pod: only annotations, + labels, name, namespace and uid + are supported.' + properties: + apiVersion: + description: Version of the + schema the FieldPath is written + in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field + to select in the specified + API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + mode: + description: |- + Optional: mode bits used to set permissions on this file, must be an octal value + between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: 'Required: Path is the + relative path name of the file + to be created. Must not be absolute + or contain the ''..'' path. Must + be utf-8 encoded. The first item + of the relative path must not + start with ''..''' + type: string + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, requests.cpu and requests.memory) are currently supported. + properties: + containerName: + description: 'Container name: + required for volumes, optional + for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output + format of the exposed resources, + defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource + to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + required: + - path + type: object + type: array + x-kubernetes-list-type: atomic + type: object + emptyDir: + description: |- + emptyDir represents a temporary directory that shares a pod's lifetime. + More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir + properties: + medium: + description: |- + medium represents what type of storage medium should back this directory. + The default is "" which means to use the node's default medium. + Must be an empty string (default) or Memory. + More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir + type: string + sizeLimit: + anyOf: + - type: integer + - type: string + description: |- + sizeLimit is the total amount of local storage required for this EmptyDir volume. + The size limit is also applicable for memory medium. + The maximum usage on memory medium EmptyDir would be the minimum value between + the SizeLimit specified here and the sum of memory limits of all containers in a pod. + The default is nil which means that the limit is undefined. + More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + ephemeral: + description: |- + ephemeral represents a volume that is handled by a cluster storage driver. + The volume's lifecycle is tied to the pod that defines it - it will be created before the pod starts, + and deleted when the pod is removed. + + Use this if: + a) the volume is only needed while the pod runs, + b) features of normal volumes like restoring from snapshot or capacity + tracking are needed, + c) the storage driver is specified through a storage class, and + d) the storage driver supports dynamic volume provisioning through + a PersistentVolumeClaim (see EphemeralVolumeSource for more + information on the connection between this volume type + and PersistentVolumeClaim). + + Use PersistentVolumeClaim or one of the vendor-specific + APIs for volumes that persist for longer than the lifecycle + of an individual pod. + + Use CSI for light-weight local ephemeral volumes if the CSI driver is meant to + be used that way - see the documentation of the driver for + more information. + + A pod can use both types of ephemeral volumes and + persistent volumes at the same time. + properties: + volumeClaimTemplate: + description: |- + Will be used to create a stand-alone PVC to provision the volume. + The pod in which this EphemeralVolumeSource is embedded will be the + owner of the PVC, i.e. the PVC will be deleted together with the + pod. The name of the PVC will be `-` where + `` is the name from the `PodSpec.Volumes` array + entry. Pod validation will reject the pod if the concatenated name + is not valid for a PVC (for example, too long). + + An existing PVC with that name that is not owned by the pod + will *not* be used for the pod to avoid using an unrelated + volume by mistake. Starting the pod is then blocked until + the unrelated PVC is removed. If such a pre-created PVC is + meant to be used by the pod, the PVC has to updated with an + owner reference to the pod once the pod exists. Normally + this should not be necessary, but it may be useful when + manually reconstructing a broken cluster. + + This field is read-only and no changes will be made by Kubernetes + to the PVC after it has been created. + + Required, must not be nil. + properties: + metadata: + description: |- + May contain labels and annotations that will be copied into the PVC + when creating it. No other fields are allowed and will be rejected during + validation. + type: object + spec: + description: |- + The specification for the PersistentVolumeClaim. The entire content is + copied unchanged into the PVC that gets created from this + template. The same fields as in a PersistentVolumeClaim + are also valid here. + properties: + accessModes: + description: |- + accessModes contains the desired access modes the volume should have. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1 + items: + type: string + type: array + x-kubernetes-list-type: atomic + dataSource: + description: |- + dataSource field can be used to specify either: + * An existing VolumeSnapshot object (snapshot.storage.k8s.io/VolumeSnapshot) + * An existing PVC (PersistentVolumeClaim) + If the provisioner or an external controller can support the specified data source, + it will create a new volume based on the contents of the specified data source. + When the AnyVolumeDataSource feature gate is enabled, dataSource contents will be copied to dataSourceRef, + and dataSourceRef contents will be copied to dataSource when dataSourceRef.namespace is not specified. + If the namespace is specified, then dataSourceRef will not be copied to dataSource. + properties: + apiGroup: + description: |- + APIGroup is the group for the resource being referenced. + If APIGroup is not specified, the specified Kind must be in the core API group. + For any other third-party types, APIGroup is required. + type: string + kind: + description: Kind is the type + of resource being referenced + type: string + name: + description: Name is the name + of resource being referenced + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + dataSourceRef: + description: |- + dataSourceRef specifies the object from which to populate the volume with data, if a non-empty + volume is desired. This may be any object from a non-empty API group (non + core object) or a PersistentVolumeClaim object. + When this field is specified, volume binding will only succeed if the type of + the specified object matches some installed volume populator or dynamic + provisioner. + This field will replace the functionality of the dataSource field and as such + if both fields are non-empty, they must have the same value. For backwards + compatibility, when namespace isn't specified in dataSourceRef, + both fields (dataSource and dataSourceRef) will be set to the same + value automatically if one of them is empty and the other is non-empty. + When namespace is specified in dataSourceRef, + dataSource isn't set to the same value and must be empty. + There are three important differences between dataSource and dataSourceRef: + * While dataSource only allows two specific types of objects, dataSourceRef + allows any non-core object, as well as PersistentVolumeClaim objects. + * While dataSource ignores disallowed values (dropping them), dataSourceRef + preserves all values, and generates an error if a disallowed value is + specified. + * While dataSource only allows local objects, dataSourceRef allows objects + in any namespaces. + (Beta) Using this field requires the AnyVolumeDataSource feature gate to be enabled. + (Alpha) Using the namespace field of dataSourceRef requires the CrossNamespaceVolumeDataSource feature gate to be enabled. + properties: + apiGroup: + description: |- + APIGroup is the group for the resource being referenced. + If APIGroup is not specified, the specified Kind must be in the core API group. + For any other third-party types, APIGroup is required. + type: string + kind: + description: Kind is the type + of resource being referenced + type: string + name: + description: Name is the name + of resource being referenced + type: string + namespace: + description: |- + Namespace is the namespace of resource being referenced + Note that when a namespace is specified, a gateway.networking.k8s.io/ReferenceGrant object is required in the referent namespace to allow that namespace's owner to accept the reference. See the ReferenceGrant documentation for details. + (Alpha) This field requires the CrossNamespaceVolumeDataSource feature gate to be enabled. + type: string + required: + - kind + - name + type: object + resources: + description: |- + resources represents the minimum resources the volume should have. + If RecoverVolumeExpansionFailure feature is enabled users are allowed to specify resource requirements + that are lower than previous value but must still be higher than capacity recorded in the + status field of the claim. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + selector: + description: selector is a label + query over volumes to consider + for binding. + properties: + matchExpressions: + description: matchExpressions + is a list of label selector + requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is + the label key that + the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + storageClassName: + description: |- + storageClassName is the name of the StorageClass required by the claim. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1 + type: string + volumeAttributesClassName: + description: |- + volumeAttributesClassName may be used to set the VolumeAttributesClass used by this claim. + If specified, the CSI driver will create or update the volume with the attributes defined + in the corresponding VolumeAttributesClass. This has a different purpose than storageClassName, + it can be changed after the claim is created. An empty string value means that no VolumeAttributesClass + will be applied to the claim but it's not allowed to reset this field to empty string once it is set. + If unspecified and the PersistentVolumeClaim is unbound, the default VolumeAttributesClass + will be set by the persistentvolume controller if it exists. + If the resource referred to by volumeAttributesClass does not exist, this PersistentVolumeClaim will be + set to a Pending state, as reflected by the modifyVolumeStatus field, until such as a resource + exists. + More info: https://kubernetes.io/docs/concepts/storage/volume-attributes-classes/ + (Beta) Using this field requires the VolumeAttributesClass feature gate to be enabled (off by default). + type: string + volumeMode: + description: |- + volumeMode defines what type of volume is required by the claim. + Value of Filesystem is implied when not included in claim spec. + type: string + volumeName: + description: volumeName is the + binding reference to the PersistentVolume + backing this claim. + type: string + type: object + required: + - spec + type: object + type: object + fc: + description: fc represents a Fibre Channel + resource that is attached to a kubelet's + host machine and then exposed to the pod. + properties: + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + lun: + description: 'lun is Optional: FC target + lun number' + format: int32 + type: integer + readOnly: + description: |- + readOnly is Optional: Defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + targetWWNs: + description: 'targetWWNs is Optional: + FC target worldwide names (WWNs)' + items: + type: string + type: array + x-kubernetes-list-type: atomic + wwids: + description: |- + wwids Optional: FC volume world wide identifiers (wwids) + Either wwids or combination of targetWWNs and lun must be set, but not both simultaneously. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + flexVolume: + description: |- + flexVolume represents a generic volume resource that is + provisioned/attached using an exec based plugin. + properties: + driver: + description: driver is the name of the + driver to use for this volume. + type: string + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". The default filesystem depends on FlexVolume script. + type: string + options: + additionalProperties: + type: string + description: 'options is Optional: this + field holds extra command options if + any.' + type: object + readOnly: + description: |- + readOnly is Optional: defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + secretRef: + description: |- + secretRef is Optional: secretRef is reference to the secret object containing + sensitive information to pass to the plugin scripts. This may be + empty if no secret object is specified. If the secret object + contains more than one secret, all secrets are passed to the plugin + scripts. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + required: + - driver + type: object + flocker: + description: flocker represents a Flocker + volume attached to a kubelet's host machine. + This depends on the Flocker control service + being running + properties: + datasetName: + description: |- + datasetName is Name of the dataset stored as metadata -> name on the dataset for Flocker + should be considered as deprecated + type: string + datasetUUID: + description: datasetUUID is the UUID of + the dataset. This is unique identifier + of a Flocker dataset + type: string + type: object + gcePersistentDisk: + description: |- + gcePersistentDisk represents a GCE Disk resource that is attached to a + kubelet's host machine and then exposed to the pod. + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + properties: + fsType: + description: |- + fsType is filesystem type of the volume that you want to mount. + Tip: Ensure that the filesystem type is supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + type: string + partition: + description: |- + partition is the partition in the volume that you want to mount. + If omitted, the default is to mount by volume name. + Examples: For volume /dev/sda1, you specify the partition as "1". + Similarly, the volume partition for /dev/sda is "0" (or you can leave the property empty). + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + format: int32 + type: integer + pdName: + description: |- + pdName is unique name of the PD resource in GCE. Used to identify the disk in GCE. + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + type: string + readOnly: + description: |- + readOnly here will force the ReadOnly setting in VolumeMounts. + Defaults to false. + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + type: boolean + required: + - pdName + type: object + gitRepo: + description: |- + gitRepo represents a git repository at a particular revision. + DEPRECATED: GitRepo is deprecated. To provision a container with a git repo, mount an + EmptyDir into an InitContainer that clones the repo using git, then mount the EmptyDir + into the Pod's container. + properties: + directory: + description: |- + directory is the target directory name. + Must not contain or start with '..'. If '.' is supplied, the volume directory will be the + git repository. Otherwise, if specified, the volume will contain the git repository in + the subdirectory with the given name. + type: string + repository: + description: repository is the URL + type: string + revision: + description: revision is the commit hash + for the specified revision. + type: string + required: + - repository + type: object + glusterfs: + description: |- + glusterfs represents a Glusterfs mount on the host that shares a pod's lifetime. + More info: https://examples.k8s.io/volumes/glusterfs/README.md + properties: + endpoints: + description: |- + endpoints is the endpoint name that details Glusterfs topology. + More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod + type: string + path: + description: |- + path is the Glusterfs volume path. + More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod + type: string + readOnly: + description: |- + readOnly here will force the Glusterfs volume to be mounted with read-only permissions. + Defaults to false. + More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod + type: boolean + required: + - endpoints + - path + type: object + hostPath: + description: |- + hostPath represents a pre-existing file or directory on the host + machine that is directly exposed to the container. This is generally + used for system agents or other privileged things that are allowed + to see the host machine. Most containers will NOT need this. + More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath + properties: + path: + description: |- + path of the directory on the host. + If the path is a symlink, it will follow the link to the real path. + More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath + type: string + type: + description: |- + type for HostPath Volume + Defaults to "" + More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath + type: string + required: + - path + type: object + image: + description: |- + image represents an OCI object (a container image or artifact) pulled and mounted on the kubelet's host machine. + The volume is resolved at pod startup depending on which PullPolicy value is provided: + + - Always: the kubelet always attempts to pull the reference. Container creation will fail If the pull fails. + - Never: the kubelet never pulls the reference and only uses a local image or artifact. Container creation will fail if the reference isn't present. + - IfNotPresent: the kubelet pulls if the reference isn't already present on disk. Container creation will fail if the reference isn't present and the pull fails. + + The volume gets re-resolved if the pod gets deleted and recreated, which means that new remote content will become available on pod recreation. + A failure to resolve or pull the image during pod startup will block containers from starting and may add significant latency. Failures will be retried using normal volume backoff and will be reported on the pod reason and message. + The types of objects that may be mounted by this volume are defined by the container runtime implementation on a host machine and at minimum must include all valid types supported by the container image field. + The OCI object gets mounted in a single directory (spec.containers[*].volumeMounts.mountPath) by merging the manifest layers in the same way as for container images. + The volume will be mounted read-only (ro) and non-executable files (noexec). + Sub path mounts for containers are not supported (spec.containers[*].volumeMounts.subpath). + The field spec.securityContext.fsGroupChangePolicy has no effect on this volume type. + properties: + pullPolicy: + description: |- + Policy for pulling OCI objects. Possible values are: + Always: the kubelet always attempts to pull the reference. Container creation will fail If the pull fails. + Never: the kubelet never pulls the reference and only uses a local image or artifact. Container creation will fail if the reference isn't present. + IfNotPresent: the kubelet pulls if the reference isn't already present on disk. Container creation will fail if the reference isn't present and the pull fails. + Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. + type: string + reference: + description: |- + Required: Image or artifact reference to be used. + Behaves in the same way as pod.spec.containers[*].image. + Pull secrets will be assembled in the same way as for the container image by looking up node credentials, SA image pull secrets, and pod spec image pull secrets. + More info: https://kubernetes.io/docs/concepts/containers/images + This field is optional to allow higher level config management to default or override + container images in workload controllers like Deployments and StatefulSets. + type: string + type: object + iscsi: + description: |- + iscsi represents an ISCSI Disk resource that is attached to a + kubelet's host machine and then exposed to the pod. + More info: https://examples.k8s.io/volumes/iscsi/README.md + properties: + chapAuthDiscovery: + description: chapAuthDiscovery defines + whether support iSCSI Discovery CHAP + authentication + type: boolean + chapAuthSession: + description: chapAuthSession defines whether + support iSCSI Session CHAP authentication + type: boolean + fsType: + description: |- + fsType is the filesystem type of the volume that you want to mount. + Tip: Ensure that the filesystem type is supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://kubernetes.io/docs/concepts/storage/volumes#iscsi + type: string + initiatorName: + description: |- + initiatorName is the custom iSCSI Initiator Name. + If initiatorName is specified with iscsiInterface simultaneously, new iSCSI interface + : will be created for the connection. + type: string + iqn: + description: iqn is the target iSCSI Qualified + Name. + type: string + iscsiInterface: + default: default + description: |- + iscsiInterface is the interface Name that uses an iSCSI transport. + Defaults to 'default' (tcp). + type: string + lun: + description: lun represents iSCSI Target + Lun number. + format: int32 + type: integer + portals: + description: |- + portals is the iSCSI Target Portal List. The portal is either an IP or ip_addr:port if the port + is other than default (typically TCP ports 860 and 3260). + items: + type: string + type: array + x-kubernetes-list-type: atomic + readOnly: + description: |- + readOnly here will force the ReadOnly setting in VolumeMounts. + Defaults to false. + type: boolean + secretRef: + description: secretRef is the CHAP Secret + for iSCSI target and initiator authentication + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + targetPortal: + description: |- + targetPortal is iSCSI Target Portal. The Portal is either an IP or ip_addr:port if the port + is other than default (typically TCP ports 860 and 3260). + type: string + required: + - iqn + - lun + - targetPortal + type: object + name: + description: |- + name of the volume. + Must be a DNS_LABEL and unique within the pod. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + nfs: + description: |- + nfs represents an NFS mount on the host that shares a pod's lifetime + More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs + properties: + path: + description: |- + path that is exported by the NFS server. + More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs + type: string + readOnly: + description: |- + readOnly here will force the NFS export to be mounted with read-only permissions. + Defaults to false. + More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs + type: boolean + server: + description: |- + server is the hostname or IP address of the NFS server. + More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs + type: string + required: + - path + - server + type: object + persistentVolumeClaim: + description: |- + persistentVolumeClaimVolumeSource represents a reference to a + PersistentVolumeClaim in the same namespace. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims + properties: + claimName: + description: |- + claimName is the name of a PersistentVolumeClaim in the same namespace as the pod using this volume. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims + type: string + readOnly: + description: |- + readOnly Will force the ReadOnly setting in VolumeMounts. + Default false. + type: boolean + required: + - claimName + type: object + photonPersistentDisk: + description: photonPersistentDisk represents + a PhotonController persistent disk attached + and mounted on kubelets host machine + properties: + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + pdID: + description: pdID is the ID that identifies + Photon Controller persistent disk + type: string + required: + - pdID + type: object + portworxVolume: + description: portworxVolume represents a portworx + volume attached and mounted on kubelets + host machine + properties: + fsType: + description: |- + fSType represents the filesystem type to mount + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs". Implicitly inferred to be "ext4" if unspecified. + type: string + readOnly: + description: |- + readOnly defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + volumeID: + description: volumeID uniquely identifies + a Portworx volume + type: string + required: + - volumeID + type: object + projected: + description: projected items for all in one + resources secrets, configmaps, and downward + API + properties: + defaultMode: + description: |- + defaultMode are the mode bits used to set permissions on created files by default. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + Directories within the path are not affected by this setting. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + sources: + description: |- + sources is the list of volume projections. Each entry in this list + handles one source. + items: + description: |- + Projection that may be projected along with other supported volume types. + Exactly one of these fields must be set. + properties: + clusterTrustBundle: + description: |- + ClusterTrustBundle allows a pod to access the `.spec.trustBundle` field + of ClusterTrustBundle objects in an auto-updating file. + + Alpha, gated by the ClusterTrustBundleProjection feature gate. + + ClusterTrustBundle objects can either be selected by name, or by the + combination of signer name and a label selector. + + Kubelet performs aggressive normalization of the PEM contents written + into the pod filesystem. Esoteric PEM features such as inter-block + comments and block headers are stripped. Certificates are deduplicated. + The ordering of certificates within the file is arbitrary, and Kubelet + may change the order over time. + properties: + labelSelector: + description: |- + Select all ClusterTrustBundles that match this label selector. Only has + effect if signerName is set. Mutually-exclusive with name. If unset, + interpreted as "match nothing". If set but empty, interpreted as "match + everything". + properties: + matchExpressions: + description: matchExpressions + is a list of label selector + requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is + the label key that + the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + name: + description: |- + Select a single ClusterTrustBundle by object name. Mutually-exclusive + with signerName and labelSelector. + type: string + optional: + description: |- + If true, don't block pod startup if the referenced ClusterTrustBundle(s) + aren't available. If using name, then the named ClusterTrustBundle is + allowed not to exist. If using signerName, then the combination of + signerName and labelSelector is allowed to match zero + ClusterTrustBundles. + type: boolean + path: + description: Relative path from + the volume root to write the + bundle. + type: string + signerName: + description: |- + Select all ClusterTrustBundles that match this signer name. + Mutually-exclusive with name. The contents of all selected + ClusterTrustBundles will be unified and deduplicated. + type: string + required: + - path + type: object + configMap: + description: configMap information + about the configMap data to project + properties: + items: + description: |- + items if unspecified, each key-value pair in the Data field of the referenced + ConfigMap will be projected into the volume as a file whose name is the + key and content is the value. If specified, the listed keys will be + projected into the specified paths, and unlisted keys will not be + present. If a key is specified which is not present in the ConfigMap, + the volume setup will error unless it is marked optional. Paths must be + relative and may not contain the '..' path or start with '..'. + items: + description: Maps a string + key to a path within a volume. + properties: + key: + description: key is the + key to project. + type: string + mode: + description: |- + mode is Optional: mode bits used to set permissions on this file. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: |- + path is the relative path of the file to map the key to. + May not be an absolute path. + May not contain the path element '..'. + May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: optional specify + whether the ConfigMap or its + keys must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + downwardAPI: + description: downwardAPI information + about the downwardAPI data to + project + properties: + items: + description: Items is a list + of DownwardAPIVolume file + items: + description: DownwardAPIVolumeFile + represents information to + create the file containing + the pod field + properties: + fieldRef: + description: 'Required: + Selects a field of the + pod: only annotations, + labels, name, namespace + and uid are supported.' + properties: + apiVersion: + description: Version + of the schema the + FieldPath is written + in terms of, defaults + to "v1". + type: string + fieldPath: + description: Path + of the field to + select in the specified + API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + mode: + description: |- + Optional: mode bits used to set permissions on this file, must be an octal value + between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: 'Required: + Path is the relative + path name of the file + to be created. Must + not be absolute or contain + the ''..'' path. Must + be utf-8 encoded. The + first item of the relative + path must not start + with ''..''' + type: string + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, requests.cpu and requests.memory) are currently supported. + properties: + containerName: + description: 'Container + name: required for + volumes, optional + for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies + the output format + of the exposed resources, + defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: + resource to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + required: + - path + type: object + type: array + x-kubernetes-list-type: atomic + type: object + secret: + description: secret information + about the secret data to project + properties: + items: + description: |- + items if unspecified, each key-value pair in the Data field of the referenced + Secret will be projected into the volume as a file whose name is the + key and content is the value. If specified, the listed keys will be + projected into the specified paths, and unlisted keys will not be + present. If a key is specified which is not present in the Secret, + the volume setup will error unless it is marked optional. Paths must be + relative and may not contain the '..' path or start with '..'. + items: + description: Maps a string + key to a path within a volume. + properties: + key: + description: key is the + key to project. + type: string + mode: + description: |- + mode is Optional: mode bits used to set permissions on this file. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: |- + path is the relative path of the file to map the key to. + May not be an absolute path. + May not contain the path element '..'. + May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: optional field + specify whether the Secret + or its key must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + serviceAccountToken: + description: serviceAccountToken + is information about the serviceAccountToken + data to project + properties: + audience: + description: |- + audience is the intended audience of the token. A recipient of a token + must identify itself with an identifier specified in the audience of the + token, and otherwise should reject the token. The audience defaults to the + identifier of the apiserver. + type: string + expirationSeconds: + description: |- + expirationSeconds is the requested duration of validity of the service + account token. As the token approaches expiration, the kubelet volume + plugin will proactively rotate the service account token. The kubelet will + start trying to rotate the token if the token is older than 80 percent of + its time to live or if the token is older than 24 hours.Defaults to 1 hour + and must be at least 10 minutes. + format: int64 + type: integer + path: + description: |- + path is the path relative to the mount point of the file to project the + token into. + type: string + required: + - path + type: object + type: object + type: array + x-kubernetes-list-type: atomic + type: object + quobyte: + description: quobyte represents a Quobyte + mount on the host that shares a pod's lifetime + properties: + group: + description: |- + group to map volume access to + Default is no group + type: string + readOnly: + description: |- + readOnly here will force the Quobyte volume to be mounted with read-only permissions. + Defaults to false. + type: boolean + registry: + description: |- + registry represents a single or multiple Quobyte Registry services + specified as a string as host:port pair (multiple entries are separated with commas) + which acts as the central registry for volumes + type: string + tenant: + description: |- + tenant owning the given Quobyte volume in the Backend + Used with dynamically provisioned Quobyte volumes, value is set by the plugin + type: string + user: + description: |- + user to map volume access to + Defaults to serivceaccount user + type: string + volume: + description: volume is a string that references + an already created Quobyte volume by + name. + type: string + required: + - registry + - volume + type: object + rbd: + description: |- + rbd represents a Rados Block Device mount on the host that shares a pod's lifetime. + More info: https://examples.k8s.io/volumes/rbd/README.md + properties: + fsType: + description: |- + fsType is the filesystem type of the volume that you want to mount. + Tip: Ensure that the filesystem type is supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://kubernetes.io/docs/concepts/storage/volumes#rbd + type: string + image: + description: |- + image is the rados image name. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: string + keyring: + default: /etc/ceph/keyring + description: |- + keyring is the path to key ring for RBDUser. + Default is /etc/ceph/keyring. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: string + monitors: + description: |- + monitors is a collection of Ceph monitors. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + items: + type: string + type: array + x-kubernetes-list-type: atomic + pool: + default: rbd + description: |- + pool is the rados pool name. + Default is rbd. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: string + readOnly: + description: |- + readOnly here will force the ReadOnly setting in VolumeMounts. + Defaults to false. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: boolean + secretRef: + description: |- + secretRef is name of the authentication secret for RBDUser. If provided + overrides keyring. + Default is nil. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + user: + default: admin + description: |- + user is the rados user name. + Default is admin. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: string + required: + - image + - monitors + type: object + scaleIO: + description: scaleIO represents a ScaleIO + persistent volume attached and mounted on + Kubernetes nodes. + properties: + fsType: + default: xfs + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". + Default is "xfs". + type: string + gateway: + description: gateway is the host address + of the ScaleIO API Gateway. + type: string + protectionDomain: + description: protectionDomain is the name + of the ScaleIO Protection Domain for + the configured storage. + type: string + readOnly: + description: |- + readOnly Defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + secretRef: + description: |- + secretRef references to the secret for ScaleIO user and other + sensitive information. If this is not provided, Login operation will fail. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + sslEnabled: + description: sslEnabled Flag enable/disable + SSL communication with Gateway, default + false + type: boolean + storageMode: + default: ThinProvisioned + description: |- + storageMode indicates whether the storage for a volume should be ThickProvisioned or ThinProvisioned. + Default is ThinProvisioned. + type: string + storagePool: + description: storagePool is the ScaleIO + Storage Pool associated with the protection + domain. + type: string + system: + description: system is the name of the + storage system as configured in ScaleIO. + type: string + volumeName: + description: |- + volumeName is the name of a volume already created in the ScaleIO system + that is associated with this volume source. + type: string + required: + - gateway + - secretRef + - system + type: object + secret: + description: |- + secret represents a secret that should populate this volume. + More info: https://kubernetes.io/docs/concepts/storage/volumes#secret + properties: + defaultMode: + description: |- + defaultMode is Optional: mode bits used to set permissions on created files by default. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values + for mode bits. Defaults to 0644. + Directories within the path are not affected by this setting. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + items: + description: |- + items If unspecified, each key-value pair in the Data field of the referenced + Secret will be projected into the volume as a file whose name is the + key and content is the value. If specified, the listed keys will be + projected into the specified paths, and unlisted keys will not be + present. If a key is specified which is not present in the Secret, + the volume setup will error unless it is marked optional. Paths must be + relative and may not contain the '..' path or start with '..'. + items: + description: Maps a string key to a + path within a volume. + properties: + key: + description: key is the key to project. + type: string + mode: + description: |- + mode is Optional: mode bits used to set permissions on this file. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: |- + path is the relative path of the file to map the key to. + May not be an absolute path. + May not contain the path element '..'. + May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + optional: + description: optional field specify whether + the Secret or its keys must be defined + type: boolean + secretName: + description: |- + secretName is the name of the secret in the pod's namespace to use. + More info: https://kubernetes.io/docs/concepts/storage/volumes#secret + type: string + type: object + storageos: + description: storageOS represents a StorageOS + volume attached and mounted on Kubernetes + nodes. + properties: + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + readOnly: + description: |- + readOnly defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + secretRef: + description: |- + secretRef specifies the secret to use for obtaining the StorageOS API + credentials. If not specified, default values will be attempted. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + volumeName: + description: |- + volumeName is the human-readable name of the StorageOS volume. Volume + names are only unique within a namespace. + type: string + volumeNamespace: + description: |- + volumeNamespace specifies the scope of the volume within StorageOS. If no + namespace is specified then the Pod's namespace will be used. This allows the + Kubernetes name scoping to be mirrored within StorageOS for tighter integration. + Set VolumeName to any name to override the default behaviour. + Set to "default" if you are not using namespaces within StorageOS. + Namespaces that do not pre-exist within StorageOS will be created. + type: string + type: object + vsphereVolume: + description: vsphereVolume represents a vSphere + volume attached and mounted on kubelets + host machine + properties: + fsType: + description: |- + fsType is filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + storagePolicyID: + description: storagePolicyID is the storage + Policy Based Management (SPBM) profile + ID associated with the StoragePolicyName. + type: string + storagePolicyName: + description: storagePolicyName is the + storage Policy Based Management (SPBM) + profile name. + type: string + volumePath: + description: volumePath is the path that + identifies vSphere volume vmdk + type: string + required: + - volumePath + type: object + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + required: + - containers + type: object + type: object + required: + - groupName + - maxReplicas + - minReplicas + - rayStartParams + - template + type: object + type: array + required: + - headGroupSpec + type: object + required: + - spec + type: object + required: + - template + type: object + served: true + storage: true diff --git a/config/crd/bases/kjobctl.x-k8s.io_rayjobtemplates.yaml b/config/crd/bases/kjobctl.x-k8s.io_rayjobtemplates.yaml new file mode 100644 index 0000000..8a0405c --- /dev/null +++ b/config/crd/bases/kjobctl.x-k8s.io_rayjobtemplates.yaml @@ -0,0 +1,25625 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.16.5 + name: rayjobtemplates.kjobctl.x-k8s.io +spec: + group: kjobctl.x-k8s.io + names: + kind: RayJobTemplate + listKind: RayJobTemplateList + plural: rayjobtemplates + singular: rayjobtemplate + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: RayJobTemplate is the Schema for the rayjobtemplate API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + template: + description: Template defines rayjobs that will be created from this rayjob + template. + properties: + metadata: + description: Standard object's metadata. + type: object + spec: + description: Specification of the desired behavior of the rayjob. + properties: + activeDeadlineSeconds: + description: |- + ActiveDeadlineSeconds is the duration in seconds that the RayJob may be active before + KubeRay actively tries to terminate the RayJob; value must be positive integer. + format: int32 + type: integer + backoffLimit: + default: 0 + description: |- + Specifies the number of retries before marking this job failed. + Each retry creates a new RayCluster. + format: int32 + type: integer + clusterSelector: + additionalProperties: + type: string + description: clusterSelector is used to select running rayclusters + by labels + type: object + entrypoint: + description: |- + INSERT ADDITIONAL SPEC FIELDS - desired state of cluster + Important: Run "make" to regenerate code after modifying this file + type: string + entrypointNumCpus: + description: EntrypointNumCpus specifies the number of cpus to + reserve for the entrypoint command. + type: number + entrypointNumGpus: + description: EntrypointNumGpus specifies the number of gpus to + reserve for the entrypoint command. + type: number + entrypointResources: + description: |- + EntrypointResources specifies the custom resources and quantities to reserve for the + entrypoint command. + type: string + jobId: + description: If jobId is not set, a new jobId will be auto-generated. + type: string + metadata: + additionalProperties: + type: string + description: Metadata is data to store along with this job. + type: object + rayClusterSpec: + description: RayClusterSpec is the cluster template to run the + job + properties: + autoscalerOptions: + description: AutoscalerOptions specifies optional configuration + for the Ray autoscaler. + properties: + env: + description: Optional list of environment variables to + set in the autoscaler container. + items: + description: EnvVar represents an environment variable + present in a Container. + properties: + name: + description: Name of the environment variable. Must + be a C_IDENTIFIER. + type: string + value: + description: |- + Variable references $(VAR_NAME) are expanded + using the previously defined environment variables in the container and + any service environment variables. If a variable cannot be resolved, + the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. + "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless of whether the variable + exists or not. + Defaults to "". + type: string + valueFrom: + description: Source for the environment variable's + value. Cannot be used if value is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the ConfigMap + or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + description: |- + Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, + spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. + properties: + apiVersion: + description: Version of the schema the FieldPath + is written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select + in the specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. + properties: + containerName: + description: 'Container name: required for + volumes, optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format + of the exposed resources, defaults to + "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + description: Selects a key of a secret in the + pod's namespace + properties: + key: + description: The key of the secret to select + from. Must be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret + or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + envFrom: + description: Optional list of sources to populate environment + variables in the autoscaler container. + items: + description: EnvFromSource represents the source of + a set of ConfigMaps + properties: + configMapRef: + description: The ConfigMap to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the ConfigMap must + be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + description: An optional identifier to prepend to + each key in the ConfigMap. Must be a C_IDENTIFIER. + type: string + secretRef: + description: The Secret to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret must + be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array + idleTimeoutSeconds: + description: |- + IdleTimeoutSeconds is the number of seconds to wait before scaling down a worker pod which is not using Ray resources. + Defaults to 60 (one minute). It is not read by the KubeRay operator but by the Ray autoscaler. + format: int32 + type: integer + image: + description: Image optionally overrides the autoscaler's + container image. This override is for provided for autoscaler + testing and development. + type: string + imagePullPolicy: + description: ImagePullPolicy optionally overrides the + autoscaler container's image pull policy. This override + is for provided for autoscaler testing and development. + type: string + resources: + description: |- + Resources specifies optional resource request and limit overrides for the autoscaler container. + Default values: 500m CPU request and limit. 512Mi memory request and limit. + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry + in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + securityContext: + description: |- + SecurityContext defines the security options the container should be run with. + If set, the fields of SecurityContext override the equivalent fields of PodSecurityContext. + More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ + properties: + allowPrivilegeEscalation: + description: |- + AllowPrivilegeEscalation controls whether a process can gain more + privileges than its parent process. This bool directly controls if + the no_new_privs flag will be set on the container process. + AllowPrivilegeEscalation is true always when the container is: + 1) run as Privileged + 2) has CAP_SYS_ADMIN + Note that this field cannot be set when spec.os.name is windows. + type: boolean + appArmorProfile: + description: |- + appArmorProfile is the AppArmor options to use by this container. If set, this profile + overrides the pod's appArmorProfile. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile loaded on the node that should be used. + The profile must be preconfigured on the node to work. + Must match the loaded name of the profile. + Must be set if and only if type is "Localhost". + type: string + type: + description: |- + type indicates which kind of AppArmor profile will be applied. + Valid options are: + Localhost - a profile pre-loaded on the node. + RuntimeDefault - the container runtime's default profile. + Unconfined - no AppArmor enforcement. + type: string + required: + - type + type: object + capabilities: + description: |- + The capabilities to add/drop when running containers. + Defaults to the default set of capabilities granted by the container runtime. + Note that this field cannot be set when spec.os.name is windows. + properties: + add: + description: Added capabilities + items: + description: Capability represent POSIX capabilities + type + type: string + type: array + x-kubernetes-list-type: atomic + drop: + description: Removed capabilities + items: + description: Capability represent POSIX capabilities + type + type: string + type: array + x-kubernetes-list-type: atomic + type: object + privileged: + description: |- + Run container in privileged mode. + Processes in privileged containers are essentially equivalent to root on the host. + Defaults to false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + procMount: + description: |- + procMount denotes the type of proc mount to use for the containers. + The default value is Default which uses the container runtime defaults for + readonly paths and masked paths. + This requires the ProcMountType feature flag to be enabled. + Note that this field cannot be set when spec.os.name is windows. + type: string + readOnlyRootFilesystem: + description: |- + Whether this container has a read-only root filesystem. + Default is false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + runAsGroup: + description: |- + The GID to run the entrypoint of the container process. + Uses runtime default if unset. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + runAsNonRoot: + description: |- + Indicates that the container must run as a non-root user. + If true, the Kubelet will validate the image at runtime to ensure that it + does not run as UID 0 (root) and fail to start the container if it does. + If unset or false, no such validation will be performed. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: |- + The UID to run the entrypoint of the container process. + Defaults to user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + seLinuxOptions: + description: |- + The SELinux context to be applied to the container. + If unspecified, the container runtime will allocate a random SELinux context for each + container. May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + properties: + level: + description: Level is SELinux level label that + applies to the container. + type: string + role: + description: Role is a SELinux role label that + applies to the container. + type: string + type: + description: Type is a SELinux type label that + applies to the container. + type: string + user: + description: User is a SELinux user label that + applies to the container. + type: string + type: object + seccompProfile: + description: |- + The seccomp options to use by this container. If seccomp options are + provided at both the pod & container level, the container options + override the pod options. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile defined in a file on the node should be used. + The profile must be preconfigured on the node to work. + Must be a descending path, relative to the kubelet's configured seccomp profile location. + Must be set if type is "Localhost". Must NOT be set for any other type. + type: string + type: + description: |- + type indicates which kind of seccomp profile will be applied. + Valid options are: + + Localhost - a profile defined in a file on the node should be used. + RuntimeDefault - the container runtime default profile should be used. + Unconfined - no profile should be applied. + type: string + required: + - type + type: object + windowsOptions: + description: |- + The Windows specific settings applied to all containers. + If unspecified, the options from the PodSecurityContext will be used. + If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is linux. + properties: + gmsaCredentialSpec: + description: |- + GMSACredentialSpec is where the GMSA admission webhook + (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the + GMSA credential spec named by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName is the name + of the GMSA credential spec to use. + type: string + hostProcess: + description: |- + HostProcess determines if a container should be run as a 'Host Process' container. + All of a Pod's containers must have the same effective HostProcess value + (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). + In addition, if HostProcess is true then HostNetwork must also be set to true. + type: boolean + runAsUserName: + description: |- + The UserName in Windows to run the entrypoint of the container process. + Defaults to the user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: string + type: object + type: object + upscalingMode: + description: |- + UpscalingMode is "Conservative", "Default", or "Aggressive." + Conservative: Upscaling is rate-limited; the number of pending worker pods is at most the size of the Ray cluster. + Default: Upscaling is not rate-limited. + Aggressive: An alias for Default; upscaling is not rate-limited. + It is not read by the KubeRay operator but by the Ray autoscaler. + enum: + - Default + - Aggressive + - Conservative + type: string + volumeMounts: + description: Optional list of volumeMounts. This is needed + for enabling TLS for the autoscaler container. + items: + description: VolumeMount describes a mounting of a Volume + within a container. + properties: + mountPath: + description: |- + Path within the container at which the volume should be mounted. Must + not contain ':'. + type: string + mountPropagation: + description: |- + mountPropagation determines how mounts are propagated from the host + to container and the other way around. + When not set, MountPropagationNone is used. + This field is beta in 1.10. + When RecursiveReadOnly is set to IfPossible or to Enabled, MountPropagation must be None or unspecified + (which defaults to None). + type: string + name: + description: This must match the Name of a Volume. + type: string + readOnly: + description: |- + Mounted read-only if true, read-write otherwise (false or unspecified). + Defaults to false. + type: boolean + recursiveReadOnly: + description: |- + RecursiveReadOnly specifies whether read-only mounts should be handled + recursively. + + If ReadOnly is false, this field has no meaning and must be unspecified. + + If ReadOnly is true, and this field is set to Disabled, the mount is not made + recursively read-only. If this field is set to IfPossible, the mount is made + recursively read-only, if it is supported by the container runtime. If this + field is set to Enabled, the mount is made recursively read-only if it is + supported by the container runtime, otherwise the pod will not be started and + an error will be generated to indicate the reason. + + If this field is set to IfPossible or Enabled, MountPropagation must be set to + None (or be unspecified, which defaults to None). + + If this field is not specified, it is treated as an equivalent of Disabled. + type: string + subPath: + description: |- + Path within the volume from which the container's volume should be mounted. + Defaults to "" (volume's root). + type: string + subPathExpr: + description: |- + Expanded path within the volume from which the container's volume should be mounted. + Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. + Defaults to "" (volume's root). + SubPathExpr and SubPath are mutually exclusive. + type: string + required: + - mountPath + - name + type: object + type: array + type: object + enableInTreeAutoscaling: + description: EnableInTreeAutoscaling indicates whether operator + should create in tree autoscaling configs + type: boolean + headGroupSpec: + description: |- + INSERT ADDITIONAL SPEC FIELDS - desired state of cluster + Important: Run "make" to regenerate code after modifying this file + HeadGroupSpecs are the spec for the head pod + properties: + enableIngress: + description: EnableIngress indicates whether operator + should create ingress object for head service or not. + type: boolean + headService: + description: HeadService is the Kubernetes service of + the head pod. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + description: |- + Standard object's metadata. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata + type: object + spec: + description: |- + Spec defines the behavior of a service. + https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status + properties: + allocateLoadBalancerNodePorts: + description: |- + allocateLoadBalancerNodePorts defines if NodePorts will be automatically + allocated for services with type LoadBalancer. Default is "true". It + may be set to "false" if the cluster load-balancer does not rely on + NodePorts. If the caller requests specific NodePorts (by specifying a + value), those requests will be respected, regardless of this field. + This field may only be set for services with type LoadBalancer and will + be cleared if the type is changed to any other type. + type: boolean + clusterIP: + description: |- + clusterIP is the IP address of the service and is usually assigned + randomly. If an address is specified manually, is in-range (as per + system configuration), and is not in use, it will be allocated to the + service; otherwise creation of the service will fail. This field may not + be changed through updates unless the type field is also being changed + to ExternalName (which requires this field to be blank) or the type + field is being changed from ExternalName (in which case this field may + optionally be specified, as describe above). Valid values are "None", + empty string (""), or a valid IP address. Setting this to "None" makes a + "headless service" (no virtual IP), which is useful when direct endpoint + connections are preferred and proxying is not required. Only applies to + types ClusterIP, NodePort, and LoadBalancer. If this field is specified + when creating a Service of type ExternalName, creation will fail. This + field will be wiped when updating a Service to type ExternalName. + More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies + type: string + clusterIPs: + description: |- + ClusterIPs is a list of IP addresses assigned to this service, and are + usually assigned randomly. If an address is specified manually, is + in-range (as per system configuration), and is not in use, it will be + allocated to the service; otherwise creation of the service will fail. + This field may not be changed through updates unless the type field is + also being changed to ExternalName (which requires this field to be + empty) or the type field is being changed from ExternalName (in which + case this field may optionally be specified, as describe above). Valid + values are "None", empty string (""), or a valid IP address. Setting + this to "None" makes a "headless service" (no virtual IP), which is + useful when direct endpoint connections are preferred and proxying is + not required. Only applies to types ClusterIP, NodePort, and + LoadBalancer. If this field is specified when creating a Service of type + ExternalName, creation will fail. This field will be wiped when updating + a Service to type ExternalName. If this field is not specified, it will + be initialized from the clusterIP field. If this field is specified, + clients must ensure that clusterIPs[0] and clusterIP have the same + value. + + This field may hold a maximum of two entries (dual-stack IPs, in either order). + These IPs must correspond to the values of the ipFamilies field. Both + clusterIPs and ipFamilies are governed by the ipFamilyPolicy field. + More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies + items: + type: string + type: array + x-kubernetes-list-type: atomic + externalIPs: + description: |- + externalIPs is a list of IP addresses for which nodes in the cluster + will also accept traffic for this service. These IPs are not managed by + Kubernetes. The user is responsible for ensuring that traffic arrives + at a node with this IP. A common example is external load-balancers + that are not part of the Kubernetes system. + items: + type: string + type: array + x-kubernetes-list-type: atomic + externalName: + description: |- + externalName is the external reference that discovery mechanisms will + return as an alias for this service (e.g. a DNS CNAME record). No + proxying will be involved. Must be a lowercase RFC-1123 hostname + (https://tools.ietf.org/html/rfc1123) and requires `type` to be "ExternalName". + type: string + externalTrafficPolicy: + description: |- + externalTrafficPolicy describes how nodes distribute service traffic they + receive on one of the Service's "externally-facing" addresses (NodePorts, + ExternalIPs, and LoadBalancer IPs). If set to "Local", the proxy will configure + the service in a way that assumes that external load balancers will take care + of balancing the service traffic between nodes, and so each node will deliver + traffic only to the node-local endpoints of the service, without masquerading + the client source IP. (Traffic mistakenly sent to a node with no endpoints will + be dropped.) The default value, "Cluster", uses the standard behavior of + routing to all endpoints evenly (possibly modified by topology and other + features). Note that traffic sent to an External IP or LoadBalancer IP from + within the cluster will always get "Cluster" semantics, but clients sending to + a NodePort from within the cluster may need to take traffic policy into account + when picking a node. + type: string + healthCheckNodePort: + description: |- + healthCheckNodePort specifies the healthcheck nodePort for the service. + This only applies when type is set to LoadBalancer and + externalTrafficPolicy is set to Local. If a value is specified, is + in-range, and is not in use, it will be used. If not specified, a value + will be automatically allocated. External systems (e.g. load-balancers) + can use this port to determine if a given node holds endpoints for this + service or not. If this field is specified when creating a Service + which does not need it, creation will fail. This field will be wiped + when updating a Service to no longer need it (e.g. changing type). + This field cannot be updated once set. + format: int32 + type: integer + internalTrafficPolicy: + description: |- + InternalTrafficPolicy describes how nodes distribute service traffic they + receive on the ClusterIP. If set to "Local", the proxy will assume that pods + only want to talk to endpoints of the service on the same node as the pod, + dropping the traffic if there are no local endpoints. The default value, + "Cluster", uses the standard behavior of routing to all endpoints evenly + (possibly modified by topology and other features). + type: string + ipFamilies: + description: |- + IPFamilies is a list of IP families (e.g. IPv4, IPv6) assigned to this + service. This field is usually assigned automatically based on cluster + configuration and the ipFamilyPolicy field. If this field is specified + manually, the requested family is available in the cluster, + and ipFamilyPolicy allows it, it will be used; otherwise creation of + the service will fail. This field is conditionally mutable: it allows + for adding or removing a secondary IP family, but it does not allow + changing the primary IP family of the Service. Valid values are "IPv4" + and "IPv6". This field only applies to Services of types ClusterIP, + NodePort, and LoadBalancer, and does apply to "headless" services. + This field will be wiped when updating a Service to type ExternalName. + + This field may hold a maximum of two entries (dual-stack families, in + either order). These families must correspond to the values of the + clusterIPs field, if specified. Both clusterIPs and ipFamilies are + governed by the ipFamilyPolicy field. + items: + description: |- + IPFamily represents the IP Family (IPv4 or IPv6). This type is used + to express the family of an IP expressed by a type (e.g. service.spec.ipFamilies). + type: string + type: array + x-kubernetes-list-type: atomic + ipFamilyPolicy: + description: |- + IPFamilyPolicy represents the dual-stack-ness requested or required by + this Service. If there is no value provided, then this field will be set + to SingleStack. Services can be "SingleStack" (a single IP family), + "PreferDualStack" (two IP families on dual-stack configured clusters or + a single IP family on single-stack clusters), or "RequireDualStack" + (two IP families on dual-stack configured clusters, otherwise fail). The + ipFamilies and clusterIPs fields depend on the value of this field. This + field will be wiped when updating a service to type ExternalName. + type: string + loadBalancerClass: + description: |- + loadBalancerClass is the class of the load balancer implementation this Service belongs to. + If specified, the value of this field must be a label-style identifier, with an optional prefix, + e.g. "internal-vip" or "example.com/internal-vip". Unprefixed names are reserved for end-users. + This field can only be set when the Service type is 'LoadBalancer'. If not set, the default load + balancer implementation is used, today this is typically done through the cloud provider integration, + but should apply for any default implementation. If set, it is assumed that a load balancer + implementation is watching for Services with a matching class. Any default load balancer + implementation (e.g. cloud providers) should ignore Services that set this field. + This field can only be set when creating or updating a Service to type 'LoadBalancer'. + Once set, it can not be changed. This field will be wiped when a service is updated to a non 'LoadBalancer' type. + type: string + loadBalancerIP: + description: |- + Only applies to Service Type: LoadBalancer. + This feature depends on whether the underlying cloud-provider supports specifying + the loadBalancerIP when a load balancer is created. + This field will be ignored if the cloud-provider does not support the feature. + Deprecated: This field was under-specified and its meaning varies across implementations. + Using it is non-portable and it may not support dual-stack. + Users are encouraged to use implementation-specific annotations when available. + type: string + loadBalancerSourceRanges: + description: |- + If specified and supported by the platform, this will restrict traffic through the cloud-provider + load-balancer will be restricted to the specified client IPs. This field will be ignored if the + cloud-provider does not support the feature." + More info: https://kubernetes.io/docs/tasks/access-application-cluster/create-external-load-balancer/ + items: + type: string + type: array + x-kubernetes-list-type: atomic + ports: + description: |- + The list of ports that are exposed by this service. + More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies + items: + description: ServicePort contains information + on service's port. + properties: + appProtocol: + description: |- + The application protocol for this port. + This is used as a hint for implementations to offer richer behavior for protocols that they understand. + This field follows standard Kubernetes label syntax. + Valid values are either: + + * Un-prefixed protocol names - reserved for IANA standard service names (as per + RFC-6335 and https://www.iana.org/assignments/service-names). + + * Kubernetes-defined prefixed names: + * 'kubernetes.io/h2c' - HTTP/2 prior knowledge over cleartext as described in https://www.rfc-editor.org/rfc/rfc9113.html#name-starting-http-2-with-prior- + * 'kubernetes.io/ws' - WebSocket over cleartext as described in https://www.rfc-editor.org/rfc/rfc6455 + * 'kubernetes.io/wss' - WebSocket over TLS as described in https://www.rfc-editor.org/rfc/rfc6455 + + * Other protocols should use implementation-defined prefixed names such as + mycompany.com/my-custom-protocol. + type: string + name: + description: |- + The name of this port within the service. This must be a DNS_LABEL. + All ports within a ServiceSpec must have unique names. When considering + the endpoints for a Service, this must match the 'name' field in the + EndpointPort. + Optional if only one ServicePort is defined on this service. + type: string + nodePort: + description: |- + The port on each node on which this service is exposed when type is + NodePort or LoadBalancer. Usually assigned by the system. If a value is + specified, in-range, and not in use it will be used, otherwise the + operation will fail. If not specified, a port will be allocated if this + Service requires one. If this field is specified when creating a + Service which does not need it, creation will fail. This field will be + wiped when updating a Service to no longer need it (e.g. changing type + from NodePort to ClusterIP). + More info: https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport + format: int32 + type: integer + port: + description: The port that will be exposed + by this service. + format: int32 + type: integer + protocol: + default: TCP + description: |- + The IP protocol for this port. Supports "TCP", "UDP", and "SCTP". + Default is TCP. + type: string + targetPort: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the pods targeted by the service. + Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. + If this is a string, it will be looked up as a named port in the + target Pod's container ports. If this is not specified, the value + of the 'port' field is used (an identity map). + This field is ignored for services with clusterIP=None, and should be + omitted or set equal to the 'port' field. + More info: https://kubernetes.io/docs/concepts/services-networking/service/#defining-a-service + x-kubernetes-int-or-string: true + required: + - port + type: object + type: array + x-kubernetes-list-map-keys: + - port + - protocol + x-kubernetes-list-type: map + publishNotReadyAddresses: + description: |- + publishNotReadyAddresses indicates that any agent which deals with endpoints for this + Service should disregard any indications of ready/not-ready. + The primary use case for setting this field is for a StatefulSet's Headless Service to + propagate SRV DNS records for its Pods for the purpose of peer discovery. + The Kubernetes controllers that generate Endpoints and EndpointSlice resources for + Services interpret this to mean that all endpoints are considered "ready" even if the + Pods themselves are not. Agents which consume only Kubernetes generated endpoints + through the Endpoints or EndpointSlice resources can safely assume this behavior. + type: boolean + selector: + additionalProperties: + type: string + description: |- + Route service traffic to pods with label keys and values matching this + selector. If empty or not present, the service is assumed to have an + external process managing its endpoints, which Kubernetes will not + modify. Only applies to types ClusterIP, NodePort, and LoadBalancer. + Ignored if type is ExternalName. + More info: https://kubernetes.io/docs/concepts/services-networking/service/ + type: object + x-kubernetes-map-type: atomic + sessionAffinity: + description: |- + Supports "ClientIP" and "None". Used to maintain session affinity. + Enable client IP based session affinity. + Must be ClientIP or None. + Defaults to None. + More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies + type: string + sessionAffinityConfig: + description: sessionAffinityConfig contains the + configurations of session affinity. + properties: + clientIP: + description: clientIP contains the configurations + of Client IP based session affinity. + properties: + timeoutSeconds: + description: |- + timeoutSeconds specifies the seconds of ClientIP type session sticky time. + The value must be >0 && <=86400(for 1 day) if ServiceAffinity == "ClientIP". + Default value is 10800(for 3 hours). + format: int32 + type: integer + type: object + type: object + trafficDistribution: + description: |- + TrafficDistribution offers a way to express preferences for how traffic is + distributed to Service endpoints. Implementations can use this field as a + hint, but are not required to guarantee strict adherence. If the field is + not set, the implementation will apply its default routing strategy. If set + to "PreferClose", implementations should prioritize endpoints that are + topologically close (e.g., same zone). + This is an alpha field and requires enabling ServiceTrafficDistribution feature. + type: string + type: + description: |- + type determines how the Service is exposed. Defaults to ClusterIP. Valid + options are ExternalName, ClusterIP, NodePort, and LoadBalancer. + "ClusterIP" allocates a cluster-internal IP address for load-balancing + to endpoints. Endpoints are determined by the selector or if that is not + specified, by manual construction of an Endpoints object or + EndpointSlice objects. If clusterIP is "None", no virtual IP is + allocated and the endpoints are published as a set of endpoints rather + than a virtual IP. + "NodePort" builds on ClusterIP and allocates a port on every node which + routes to the same endpoints as the clusterIP. + "LoadBalancer" builds on NodePort and creates an external load-balancer + (if supported in the current cloud) which routes to the same endpoints + as the clusterIP. + "ExternalName" aliases this service to the specified externalName. + Several other fields do not apply to ExternalName services. + More info: https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types + type: string + type: object + status: + description: |- + Most recently observed status of the service. + Populated by the system. + Read-only. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status + properties: + conditions: + description: Current service state + items: + description: Condition contains details for + one aspect of the current state of this API + Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one + of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase + or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + loadBalancer: + description: |- + LoadBalancer contains the current status of the load-balancer, + if one is present. + properties: + ingress: + description: |- + Ingress is a list containing ingress points for the load-balancer. + Traffic intended for the service should be sent to these ingress points. + items: + description: |- + LoadBalancerIngress represents the status of a load-balancer ingress point: + traffic intended for the service should be sent to an ingress point. + properties: + hostname: + description: |- + Hostname is set for load-balancer ingress points that are DNS based + (typically AWS load-balancers) + type: string + ip: + description: |- + IP is set for load-balancer ingress points that are IP based + (typically GCE or OpenStack load-balancers) + type: string + ipMode: + description: |- + IPMode specifies how the load-balancer IP behaves, and may only be specified when the ip field is specified. + Setting this to "VIP" indicates that traffic is delivered to the node with + the destination set to the load-balancer's IP and port. + Setting this to "Proxy" indicates that traffic is delivered to the node or pod with + the destination set to the node's IP and node port or the pod's IP and port. + Service implementations may use this information to adjust traffic routing. + type: string + ports: + description: |- + Ports is a list of records of service ports + If used, every port defined in the service should have an entry in it + items: + properties: + error: + description: |- + Error is to record the problem with the service port + The format of the error shall comply with the following rules: + - built-in error values shall be specified in this file and those shall use + CamelCase names + - cloud provider specific error values must have names that comply with the + format foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + port: + description: Port is the port + number of the service port of + which status is recorded here + format: int32 + type: integer + protocol: + description: |- + Protocol is the protocol of the service port of which status is recorded here + The supported values are: "TCP", "UDP", "SCTP" + type: string + required: + - error + - port + - protocol + type: object + type: array + x-kubernetes-list-type: atomic + type: object + type: array + x-kubernetes-list-type: atomic + type: object + type: object + type: object + rayStartParams: + additionalProperties: + type: string + description: 'RayStartParams are the params of the start + command: node-manager-port, object-store-memory, ...' + type: object + serviceType: + description: ServiceType is Kubernetes service type of + the head service. it will be used by the workers to + connect to the head pod + type: string + template: + description: Template is the exact pod template used in + K8s depoyments, statefulsets, etc. + properties: + metadata: + description: |- + Standard object's metadata. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata + type: object + spec: + description: |- + Specification of the desired behavior of the pod. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status + properties: + activeDeadlineSeconds: + description: |- + Optional duration in seconds the pod may be active on the node relative to + StartTime before the system will actively try to mark it failed and kill associated containers. + Value must be a positive integer. + format: int64 + type: integer + affinity: + description: If specified, the pod's scheduling + constraints + properties: + nodeAffinity: + description: Describes node affinity scheduling + rules for the pod. + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node matches the corresponding matchExpressions; the + node(s) with the highest sum are the most preferred. + items: + description: |- + An empty preferred scheduling term matches all objects with implicit weight 0 + (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op). + properties: + preference: + description: A node selector term, + associated with the corresponding + weight. + properties: + matchExpressions: + description: A list of node + selector requirements by node's + labels. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label + key that the selector + applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + description: A list of node + selector requirements by node's + fields. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label + key that the selector + applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + weight: + description: Weight associated with + matching the corresponding nodeSelectorTerm, + in the range 1-100. + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to an update), the system + may or may not try to eventually evict the pod from its node. + properties: + nodeSelectorTerms: + description: Required. A list of node + selector terms. The terms are ORed. + items: + description: |- + A null or empty node selector term matches no objects. The requirements of + them are ANDed. + The TopologySelectorTerm type implements a subset of the NodeSelectorTerm. + properties: + matchExpressions: + description: A list of node + selector requirements by node's + labels. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label + key that the selector + applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + description: A list of node + selector requirements by node's + fields. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label + key that the selector + applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + type: array + x-kubernetes-list-type: atomic + required: + - nodeSelectorTerms + type: object + x-kubernetes-map-type: atomic + type: object + podAffinity: + description: Describes pod affinity scheduling + rules (e.g. co-locate this pod in the same + node, zone, etc. as some other pod(s)). + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the + node(s) with the highest sum are the most preferred. + items: + description: The weights of all of the + matched WeightedPodAffinityTerm fields + are added per-node to find the most + preferred node(s) + properties: + podAffinityTerm: + description: Required. A pod affinity + term, associated with the corresponding + weight. + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions + is a list of label selector + requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is + the label key that + the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions + is a list of label selector + requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is + the label key that + the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + weight: + description: |- + weight associated with matching the corresponding podAffinityTerm, + in the range 1-100. + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to a pod label update), the + system may or may not try to eventually evict the pod from its node. + When there are multiple elements, the lists of nodes corresponding to each + podAffinityTerm are intersected, i.e. all terms must be satisfied. + items: + description: |- + Defines a set of pods (namely those matching the labelSelector + relative to the given namespace(s)) that this pod should be + co-located (affinity) or not co-located (anti-affinity) with, + where co-located is defined as running on a node whose value of + the label with key matches that of any node on which + a pod of the set of pods is running + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions + is a list of label selector + requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the + label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions + is a list of label selector + requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the + label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + podAntiAffinity: + description: Describes pod anti-affinity scheduling + rules (e.g. avoid putting this pod in the + same node, zone, etc. as some other pod(s)). + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the anti-affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling anti-affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the + node(s) with the highest sum are the most preferred. + items: + description: The weights of all of the + matched WeightedPodAffinityTerm fields + are added per-node to find the most + preferred node(s) + properties: + podAffinityTerm: + description: Required. A pod affinity + term, associated with the corresponding + weight. + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions + is a list of label selector + requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is + the label key that + the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions + is a list of label selector + requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is + the label key that + the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + weight: + description: |- + weight associated with matching the corresponding podAffinityTerm, + in the range 1-100. + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the anti-affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the anti-affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to a pod label update), the + system may or may not try to eventually evict the pod from its node. + When there are multiple elements, the lists of nodes corresponding to each + podAffinityTerm are intersected, i.e. all terms must be satisfied. + items: + description: |- + Defines a set of pods (namely those matching the labelSelector + relative to the given namespace(s)) that this pod should be + co-located (affinity) or not co-located (anti-affinity) with, + where co-located is defined as running on a node whose value of + the label with key matches that of any node on which + a pod of the set of pods is running + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions + is a list of label selector + requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the + label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions + is a list of label selector + requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the + label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + type: object + automountServiceAccountToken: + description: AutomountServiceAccountToken indicates + whether a service account token should be automatically + mounted. + type: boolean + containers: + description: |- + List of containers belonging to the pod. + Containers cannot currently be added or removed. + There must be at least one container in a Pod. + Cannot be updated. + items: + description: A single application container + that you want to run within a pod. + properties: + args: + description: |- + Arguments to the entrypoint. + The container image's CMD is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell + items: + type: string + type: array + x-kubernetes-list-type: atomic + command: + description: |- + Entrypoint array. Not executed within a shell. + The container image's ENTRYPOINT is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell + items: + type: string + type: array + x-kubernetes-list-type: atomic + env: + description: |- + List of environment variables to set in the container. + Cannot be updated. + items: + description: EnvVar represents an environment + variable present in a Container. + properties: + name: + description: Name of the environment + variable. Must be a C_IDENTIFIER. + type: string + value: + description: |- + Variable references $(VAR_NAME) are expanded + using the previously defined environment variables in the container and + any service environment variables. If a variable cannot be resolved, + the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. + "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless of whether the variable + exists or not. + Defaults to "". + type: string + valueFrom: + description: Source for the environment + variable's value. Cannot be used + if value is not empty. + properties: + configMapKeyRef: + description: Selects a key of + a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether + the ConfigMap or its key + must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + description: |- + Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, + spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. + properties: + apiVersion: + description: Version of the + schema the FieldPath is + written in terms of, defaults + to "v1". + type: string + fieldPath: + description: Path of the field + to select in the specified + API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. + properties: + containerName: + description: 'Container name: + required for volumes, optional + for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the + output format of the exposed + resources, defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource + to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + description: Selects a key of + a secret in the pod's namespace + properties: + key: + description: The key of the + secret to select from. Must + be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether + the Secret or its key must + be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + envFrom: + description: |- + List of sources to populate environment variables in the container. + The keys defined within a source must be a C_IDENTIFIER. All invalid keys + will be reported as an event when the container is starting. When a key exists in multiple + sources, the value associated with the last source will take precedence. + Values defined by an Env with a duplicate key will take precedence. + Cannot be updated. + items: + description: EnvFromSource represents + the source of a set of ConfigMaps + properties: + configMapRef: + description: The ConfigMap to select + from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the + ConfigMap must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + description: An optional identifier + to prepend to each key in the ConfigMap. + Must be a C_IDENTIFIER. + type: string + secretRef: + description: The Secret to select + from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the + Secret must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array + x-kubernetes-list-type: atomic + image: + description: |- + Container image name. + More info: https://kubernetes.io/docs/concepts/containers/images + This field is optional to allow higher level config management to default or override + container images in workload controllers like Deployments and StatefulSets. + type: string + imagePullPolicy: + description: |- + Image pull policy. + One of Always, Never, IfNotPresent. + Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/containers/images#updating-images + type: string + lifecycle: + description: |- + Actions that the management system should take in response to container lifecycle events. + Cannot be updated. + properties: + postStart: + description: |- + PostStart is called immediately after a container is created. If the handler fails, + the container is terminated and restarted according to its restart policy. + Other management of the container blocks until the hook completes. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies the + action to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies the + http request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers + to set in the request. HTTP + allows repeated headers. + items: + description: HTTPHeader describes + a custom header to be used + in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header + field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access + on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents the + duration that the container should + sleep before being terminated. + properties: + seconds: + description: Seconds is the + number of seconds to sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for the backward compatibility. There are no validation of this field and + lifecycle hooks will fail in runtime when tcp handler is specified. + properties: + host: + description: 'Optional: Host + name to connect to, defaults + to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + preStop: + description: |- + PreStop is called immediately before a container is terminated due to an + API request or management event such as liveness/startup probe failure, + preemption, resource contention, etc. The handler is not called if the + container crashes or exits. The Pod's termination grace period countdown begins before the + PreStop hook is executed. Regardless of the outcome of the handler, the + container will eventually terminate within the Pod's termination grace + period (unless delayed by finalizers). Other management of the container blocks until the hook completes + or until the termination grace period is reached. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies the + action to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies the + http request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers + to set in the request. HTTP + allows repeated headers. + items: + description: HTTPHeader describes + a custom header to be used + in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header + field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access + on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents the + duration that the container should + sleep before being terminated. + properties: + seconds: + description: Seconds is the + number of seconds to sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for the backward compatibility. There are no validation of this field and + lifecycle hooks will fail in runtime when tcp handler is specified. + properties: + host: + description: 'Optional: Host + name to connect to, defaults + to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + type: object + livenessProbe: + description: |- + Periodic probe of container liveness. + Container will be restarted if the probe fails. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies the action + to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action + involving a GRPC port. + properties: + port: + description: Port number of the + gRPC service. Number must be in + the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http + request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set + in the request. HTTP allows repeated + headers. + items: + description: HTTPHeader describes + a custom header to be used in + HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field + value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the + HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an + action involving a TCP port. + properties: + host: + description: 'Optional: Host name + to connect to, defaults to the + pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + name: + description: |- + Name of the container specified as a DNS_LABEL. + Each container in a pod must have a unique name (DNS_LABEL). + Cannot be updated. + type: string + ports: + description: |- + List of ports to expose from the container. Not specifying a port here + DOES NOT prevent that port from being exposed. Any port which is + listening on the default "0.0.0.0" address inside a container will be + accessible from the network. + Modifying this array with strategic merge patch may corrupt the data. + For more information See https://github.com/kubernetes/kubernetes/issues/108255. + Cannot be updated. + items: + description: ContainerPort represents + a network port in a single container. + properties: + containerPort: + description: |- + Number of port to expose on the pod's IP address. + This must be a valid port number, 0 < x < 65536. + format: int32 + type: integer + hostIP: + description: What host IP to bind + the external port to. + type: string + hostPort: + description: |- + Number of port to expose on the host. + If specified, this must be a valid port number, 0 < x < 65536. + If HostNetwork is specified, this must match ContainerPort. + Most containers do not need this. + format: int32 + type: integer + name: + description: |- + If specified, this must be an IANA_SVC_NAME and unique within the pod. Each + named port in a pod must have a unique name. Name for the port that can be + referred to by services. + type: string + protocol: + default: TCP + description: |- + Protocol for port. Must be UDP, TCP, or SCTP. + Defaults to "TCP". + type: string + required: + - containerPort + type: object + type: array + x-kubernetes-list-map-keys: + - containerPort + - protocol + x-kubernetes-list-type: map + readinessProbe: + description: |- + Periodic probe of container service readiness. + Container will be removed from service endpoints if the probe fails. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies the action + to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action + involving a GRPC port. + properties: + port: + description: Port number of the + gRPC service. Number must be in + the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http + request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set + in the request. HTTP allows repeated + headers. + items: + description: HTTPHeader describes + a custom header to be used in + HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field + value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the + HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an + action involving a TCP port. + properties: + host: + description: 'Optional: Host name + to connect to, defaults to the + pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + resizePolicy: + description: Resources resize policy for + the container. + items: + description: ContainerResizePolicy represents + resource resize policy for the container. + properties: + resourceName: + description: |- + Name of the resource to which this resource resize policy applies. + Supported values: cpu, memory. + type: string + restartPolicy: + description: |- + Restart policy to apply when specified resource is resized. + If not specified, it defaults to NotRequired. + type: string + required: + - resourceName + - restartPolicy + type: object + type: array + x-kubernetes-list-type: atomic + resources: + description: |- + Compute Resources required by this container. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references + one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + restartPolicy: + description: |- + RestartPolicy defines the restart behavior of individual containers in a pod. + This field may only be set for init containers, and the only allowed value is "Always". + For non-init containers or when this field is not specified, + the restart behavior is defined by the Pod's restart policy and the container type. + Setting the RestartPolicy as "Always" for the init container will have the following effect: + this init container will be continually restarted on + exit until all regular containers have terminated. Once all regular + containers have completed, all init containers with restartPolicy "Always" + will be shut down. This lifecycle differs from normal init containers and + is often referred to as a "sidecar" container. Although this init + container still starts in the init container sequence, it does not wait + for the container to complete before proceeding to the next init + container. Instead, the next init container starts immediately after this + init container is started, or after any startupProbe has successfully + completed. + type: string + securityContext: + description: |- + SecurityContext defines the security options the container should be run with. + If set, the fields of SecurityContext override the equivalent fields of PodSecurityContext. + More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ + properties: + allowPrivilegeEscalation: + description: |- + AllowPrivilegeEscalation controls whether a process can gain more + privileges than its parent process. This bool directly controls if + the no_new_privs flag will be set on the container process. + AllowPrivilegeEscalation is true always when the container is: + 1) run as Privileged + 2) has CAP_SYS_ADMIN + Note that this field cannot be set when spec.os.name is windows. + type: boolean + appArmorProfile: + description: |- + appArmorProfile is the AppArmor options to use by this container. If set, this profile + overrides the pod's appArmorProfile. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile loaded on the node that should be used. + The profile must be preconfigured on the node to work. + Must match the loaded name of the profile. + Must be set if and only if type is "Localhost". + type: string + type: + description: |- + type indicates which kind of AppArmor profile will be applied. + Valid options are: + Localhost - a profile pre-loaded on the node. + RuntimeDefault - the container runtime's default profile. + Unconfined - no AppArmor enforcement. + type: string + required: + - type + type: object + capabilities: + description: |- + The capabilities to add/drop when running containers. + Defaults to the default set of capabilities granted by the container runtime. + Note that this field cannot be set when spec.os.name is windows. + properties: + add: + description: Added capabilities + items: + description: Capability represent + POSIX capabilities type + type: string + type: array + x-kubernetes-list-type: atomic + drop: + description: Removed capabilities + items: + description: Capability represent + POSIX capabilities type + type: string + type: array + x-kubernetes-list-type: atomic + type: object + privileged: + description: |- + Run container in privileged mode. + Processes in privileged containers are essentially equivalent to root on the host. + Defaults to false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + procMount: + description: |- + procMount denotes the type of proc mount to use for the containers. + The default value is Default which uses the container runtime defaults for + readonly paths and masked paths. + This requires the ProcMountType feature flag to be enabled. + Note that this field cannot be set when spec.os.name is windows. + type: string + readOnlyRootFilesystem: + description: |- + Whether this container has a read-only root filesystem. + Default is false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + runAsGroup: + description: |- + The GID to run the entrypoint of the container process. + Uses runtime default if unset. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + runAsNonRoot: + description: |- + Indicates that the container must run as a non-root user. + If true, the Kubelet will validate the image at runtime to ensure that it + does not run as UID 0 (root) and fail to start the container if it does. + If unset or false, no such validation will be performed. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: |- + The UID to run the entrypoint of the container process. + Defaults to user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + seLinuxOptions: + description: |- + The SELinux context to be applied to the container. + If unspecified, the container runtime will allocate a random SELinux context for each + container. May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + properties: + level: + description: Level is SELinux level + label that applies to the container. + type: string + role: + description: Role is a SELinux role + label that applies to the container. + type: string + type: + description: Type is a SELinux type + label that applies to the container. + type: string + user: + description: User is a SELinux user + label that applies to the container. + type: string + type: object + seccompProfile: + description: |- + The seccomp options to use by this container. If seccomp options are + provided at both the pod & container level, the container options + override the pod options. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile defined in a file on the node should be used. + The profile must be preconfigured on the node to work. + Must be a descending path, relative to the kubelet's configured seccomp profile location. + Must be set if type is "Localhost". Must NOT be set for any other type. + type: string + type: + description: |- + type indicates which kind of seccomp profile will be applied. + Valid options are: + + Localhost - a profile defined in a file on the node should be used. + RuntimeDefault - the container runtime default profile should be used. + Unconfined - no profile should be applied. + type: string + required: + - type + type: object + windowsOptions: + description: |- + The Windows specific settings applied to all containers. + If unspecified, the options from the PodSecurityContext will be used. + If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is linux. + properties: + gmsaCredentialSpec: + description: |- + GMSACredentialSpec is where the GMSA admission webhook + (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the + GMSA credential spec named by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName + is the name of the GMSA credential + spec to use. + type: string + hostProcess: + description: |- + HostProcess determines if a container should be run as a 'Host Process' container. + All of a Pod's containers must have the same effective HostProcess value + (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). + In addition, if HostProcess is true then HostNetwork must also be set to true. + type: boolean + runAsUserName: + description: |- + The UserName in Windows to run the entrypoint of the container process. + Defaults to the user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: string + type: object + type: object + startupProbe: + description: |- + StartupProbe indicates that the Pod has successfully initialized. + If specified, no other probes are executed until this completes successfully. + If this probe fails, the Pod will be restarted, just as if the livenessProbe failed. + This can be used to provide different probe parameters at the beginning of a Pod's lifecycle, + when it might take a long time to load data or warm a cache, than during steady-state operation. + This cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies the action + to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action + involving a GRPC port. + properties: + port: + description: Port number of the + gRPC service. Number must be in + the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http + request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set + in the request. HTTP allows repeated + headers. + items: + description: HTTPHeader describes + a custom header to be used in + HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field + value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the + HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an + action involving a TCP port. + properties: + host: + description: 'Optional: Host name + to connect to, defaults to the + pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + stdin: + description: |- + Whether this container should allocate a buffer for stdin in the container runtime. If this + is not set, reads from stdin in the container will always result in EOF. + Default is false. + type: boolean + stdinOnce: + description: |- + Whether the container runtime should close the stdin channel after it has been opened by + a single attach. When stdin is true the stdin stream will remain open across multiple attach + sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the + first client attaches to stdin, and then remains open and accepts data until the client disconnects, + at which time stdin is closed and remains closed until the container is restarted. If this + flag is false, a container processes that reads from stdin will never receive an EOF. + Default is false + type: boolean + terminationMessagePath: + description: |- + Optional: Path at which the file to which the container's termination message + will be written is mounted into the container's filesystem. + Message written is intended to be brief final status, such as an assertion failure message. + Will be truncated by the node if greater than 4096 bytes. The total message length across + all containers will be limited to 12kb. + Defaults to /dev/termination-log. + Cannot be updated. + type: string + terminationMessagePolicy: + description: |- + Indicate how the termination message should be populated. File will use the contents of + terminationMessagePath to populate the container status message on both success and failure. + FallbackToLogsOnError will use the last chunk of container log output if the termination + message file is empty and the container exited with an error. + The log output is limited to 2048 bytes or 80 lines, whichever is smaller. + Defaults to File. + Cannot be updated. + type: string + tty: + description: |- + Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. + Default is false. + type: boolean + volumeDevices: + description: volumeDevices is the list of + block devices to be used by the container. + items: + description: volumeDevice describes a + mapping of a raw block device within + a container. + properties: + devicePath: + description: devicePath is the path + inside of the container that the + device will be mapped to. + type: string + name: + description: name must match the name + of a persistentVolumeClaim in the + pod + type: string + required: + - devicePath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - devicePath + x-kubernetes-list-type: map + volumeMounts: + description: |- + Pod volumes to mount into the container's filesystem. + Cannot be updated. + items: + description: VolumeMount describes a mounting + of a Volume within a container. + properties: + mountPath: + description: |- + Path within the container at which the volume should be mounted. Must + not contain ':'. + type: string + mountPropagation: + description: |- + mountPropagation determines how mounts are propagated from the host + to container and the other way around. + When not set, MountPropagationNone is used. + This field is beta in 1.10. + When RecursiveReadOnly is set to IfPossible or to Enabled, MountPropagation must be None or unspecified + (which defaults to None). + type: string + name: + description: This must match the Name + of a Volume. + type: string + readOnly: + description: |- + Mounted read-only if true, read-write otherwise (false or unspecified). + Defaults to false. + type: boolean + recursiveReadOnly: + description: |- + RecursiveReadOnly specifies whether read-only mounts should be handled + recursively. + + If ReadOnly is false, this field has no meaning and must be unspecified. + + If ReadOnly is true, and this field is set to Disabled, the mount is not made + recursively read-only. If this field is set to IfPossible, the mount is made + recursively read-only, if it is supported by the container runtime. If this + field is set to Enabled, the mount is made recursively read-only if it is + supported by the container runtime, otherwise the pod will not be started and + an error will be generated to indicate the reason. + + If this field is set to IfPossible or Enabled, MountPropagation must be set to + None (or be unspecified, which defaults to None). + + If this field is not specified, it is treated as an equivalent of Disabled. + type: string + subPath: + description: |- + Path within the volume from which the container's volume should be mounted. + Defaults to "" (volume's root). + type: string + subPathExpr: + description: |- + Expanded path within the volume from which the container's volume should be mounted. + Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. + Defaults to "" (volume's root). + SubPathExpr and SubPath are mutually exclusive. + type: string + required: + - mountPath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - mountPath + x-kubernetes-list-type: map + workingDir: + description: |- + Container's working directory. + If not specified, the container runtime's default will be used, which + might be configured in the container image. + Cannot be updated. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + dnsConfig: + description: |- + Specifies the DNS parameters of a pod. + Parameters specified here will be merged to the generated DNS + configuration based on DNSPolicy. + properties: + nameservers: + description: |- + A list of DNS name server IP addresses. + This will be appended to the base nameservers generated from DNSPolicy. + Duplicated nameservers will be removed. + items: + type: string + type: array + x-kubernetes-list-type: atomic + options: + description: |- + A list of DNS resolver options. + This will be merged with the base options generated from DNSPolicy. + Duplicated entries will be removed. Resolution options given in Options + will override those that appear in the base DNSPolicy. + items: + description: PodDNSConfigOption defines + DNS resolver options of a pod. + properties: + name: + description: Required. + type: string + value: + type: string + type: object + type: array + x-kubernetes-list-type: atomic + searches: + description: |- + A list of DNS search domains for host-name lookup. + This will be appended to the base search paths generated from DNSPolicy. + Duplicated search paths will be removed. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + dnsPolicy: + description: |- + Set DNS policy for the pod. + Defaults to "ClusterFirst". + Valid values are 'ClusterFirstWithHostNet', 'ClusterFirst', 'Default' or 'None'. + DNS parameters given in DNSConfig will be merged with the policy selected with DNSPolicy. + To have DNS options set along with hostNetwork, you have to specify DNS policy + explicitly to 'ClusterFirstWithHostNet'. + type: string + enableServiceLinks: + description: |- + EnableServiceLinks indicates whether information about services should be injected into pod's + environment variables, matching the syntax of Docker links. + Optional: Defaults to true. + type: boolean + ephemeralContainers: + description: |- + List of ephemeral containers run in this pod. Ephemeral containers may be run in an existing + pod to perform user-initiated actions such as debugging. This list cannot be specified when + creating a pod, and it cannot be modified by updating the pod spec. In order to add an + ephemeral container to an existing pod, use the pod's ephemeralcontainers subresource. + items: + description: |- + An EphemeralContainer is a temporary container that you may add to an existing Pod for + user-initiated activities such as debugging. Ephemeral containers have no resource or + scheduling guarantees, and they will not be restarted when they exit or when a Pod is + removed or restarted. The kubelet may evict a Pod if an ephemeral container causes the + Pod to exceed its resource allocation. + + To add an ephemeral container, use the ephemeralcontainers subresource of an existing + Pod. Ephemeral containers may not be removed or restarted. + properties: + args: + description: |- + Arguments to the entrypoint. + The image's CMD is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell + items: + type: string + type: array + x-kubernetes-list-type: atomic + command: + description: |- + Entrypoint array. Not executed within a shell. + The image's ENTRYPOINT is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell + items: + type: string + type: array + x-kubernetes-list-type: atomic + env: + description: |- + List of environment variables to set in the container. + Cannot be updated. + items: + description: EnvVar represents an environment + variable present in a Container. + properties: + name: + description: Name of the environment + variable. Must be a C_IDENTIFIER. + type: string + value: + description: |- + Variable references $(VAR_NAME) are expanded + using the previously defined environment variables in the container and + any service environment variables. If a variable cannot be resolved, + the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. + "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless of whether the variable + exists or not. + Defaults to "". + type: string + valueFrom: + description: Source for the environment + variable's value. Cannot be used + if value is not empty. + properties: + configMapKeyRef: + description: Selects a key of + a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether + the ConfigMap or its key + must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + description: |- + Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, + spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. + properties: + apiVersion: + description: Version of the + schema the FieldPath is + written in terms of, defaults + to "v1". + type: string + fieldPath: + description: Path of the field + to select in the specified + API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. + properties: + containerName: + description: 'Container name: + required for volumes, optional + for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the + output format of the exposed + resources, defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource + to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + description: Selects a key of + a secret in the pod's namespace + properties: + key: + description: The key of the + secret to select from. Must + be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether + the Secret or its key must + be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + envFrom: + description: |- + List of sources to populate environment variables in the container. + The keys defined within a source must be a C_IDENTIFIER. All invalid keys + will be reported as an event when the container is starting. When a key exists in multiple + sources, the value associated with the last source will take precedence. + Values defined by an Env with a duplicate key will take precedence. + Cannot be updated. + items: + description: EnvFromSource represents + the source of a set of ConfigMaps + properties: + configMapRef: + description: The ConfigMap to select + from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the + ConfigMap must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + description: An optional identifier + to prepend to each key in the ConfigMap. + Must be a C_IDENTIFIER. + type: string + secretRef: + description: The Secret to select + from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the + Secret must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array + x-kubernetes-list-type: atomic + image: + description: |- + Container image name. + More info: https://kubernetes.io/docs/concepts/containers/images + type: string + imagePullPolicy: + description: |- + Image pull policy. + One of Always, Never, IfNotPresent. + Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/containers/images#updating-images + type: string + lifecycle: + description: Lifecycle is not allowed for + ephemeral containers. + properties: + postStart: + description: |- + PostStart is called immediately after a container is created. If the handler fails, + the container is terminated and restarted according to its restart policy. + Other management of the container blocks until the hook completes. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies the + action to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies the + http request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers + to set in the request. HTTP + allows repeated headers. + items: + description: HTTPHeader describes + a custom header to be used + in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header + field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access + on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents the + duration that the container should + sleep before being terminated. + properties: + seconds: + description: Seconds is the + number of seconds to sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for the backward compatibility. There are no validation of this field and + lifecycle hooks will fail in runtime when tcp handler is specified. + properties: + host: + description: 'Optional: Host + name to connect to, defaults + to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + preStop: + description: |- + PreStop is called immediately before a container is terminated due to an + API request or management event such as liveness/startup probe failure, + preemption, resource contention, etc. The handler is not called if the + container crashes or exits. The Pod's termination grace period countdown begins before the + PreStop hook is executed. Regardless of the outcome of the handler, the + container will eventually terminate within the Pod's termination grace + period (unless delayed by finalizers). Other management of the container blocks until the hook completes + or until the termination grace period is reached. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies the + action to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies the + http request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers + to set in the request. HTTP + allows repeated headers. + items: + description: HTTPHeader describes + a custom header to be used + in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header + field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access + on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents the + duration that the container should + sleep before being terminated. + properties: + seconds: + description: Seconds is the + number of seconds to sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for the backward compatibility. There are no validation of this field and + lifecycle hooks will fail in runtime when tcp handler is specified. + properties: + host: + description: 'Optional: Host + name to connect to, defaults + to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + type: object + livenessProbe: + description: Probes are not allowed for + ephemeral containers. + properties: + exec: + description: Exec specifies the action + to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action + involving a GRPC port. + properties: + port: + description: Port number of the + gRPC service. Number must be in + the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http + request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set + in the request. HTTP allows repeated + headers. + items: + description: HTTPHeader describes + a custom header to be used in + HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field + value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the + HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an + action involving a TCP port. + properties: + host: + description: 'Optional: Host name + to connect to, defaults to the + pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + name: + description: |- + Name of the ephemeral container specified as a DNS_LABEL. + This name must be unique among all containers, init containers and ephemeral containers. + type: string + ports: + description: Ports are not allowed for ephemeral + containers. + items: + description: ContainerPort represents + a network port in a single container. + properties: + containerPort: + description: |- + Number of port to expose on the pod's IP address. + This must be a valid port number, 0 < x < 65536. + format: int32 + type: integer + hostIP: + description: What host IP to bind + the external port to. + type: string + hostPort: + description: |- + Number of port to expose on the host. + If specified, this must be a valid port number, 0 < x < 65536. + If HostNetwork is specified, this must match ContainerPort. + Most containers do not need this. + format: int32 + type: integer + name: + description: |- + If specified, this must be an IANA_SVC_NAME and unique within the pod. Each + named port in a pod must have a unique name. Name for the port that can be + referred to by services. + type: string + protocol: + default: TCP + description: |- + Protocol for port. Must be UDP, TCP, or SCTP. + Defaults to "TCP". + type: string + required: + - containerPort + type: object + type: array + x-kubernetes-list-map-keys: + - containerPort + - protocol + x-kubernetes-list-type: map + readinessProbe: + description: Probes are not allowed for + ephemeral containers. + properties: + exec: + description: Exec specifies the action + to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action + involving a GRPC port. + properties: + port: + description: Port number of the + gRPC service. Number must be in + the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http + request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set + in the request. HTTP allows repeated + headers. + items: + description: HTTPHeader describes + a custom header to be used in + HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field + value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the + HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an + action involving a TCP port. + properties: + host: + description: 'Optional: Host name + to connect to, defaults to the + pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + resizePolicy: + description: Resources resize policy for + the container. + items: + description: ContainerResizePolicy represents + resource resize policy for the container. + properties: + resourceName: + description: |- + Name of the resource to which this resource resize policy applies. + Supported values: cpu, memory. + type: string + restartPolicy: + description: |- + Restart policy to apply when specified resource is resized. + If not specified, it defaults to NotRequired. + type: string + required: + - resourceName + - restartPolicy + type: object + type: array + x-kubernetes-list-type: atomic + resources: + description: |- + Resources are not allowed for ephemeral containers. Ephemeral containers use spare resources + already allocated to the pod. + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references + one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + restartPolicy: + description: |- + Restart policy for the container to manage the restart behavior of each + container within a pod. + This may only be set for init containers. You cannot set this field on + ephemeral containers. + type: string + securityContext: + description: |- + Optional: SecurityContext defines the security options the ephemeral container should be run with. + If set, the fields of SecurityContext override the equivalent fields of PodSecurityContext. + properties: + allowPrivilegeEscalation: + description: |- + AllowPrivilegeEscalation controls whether a process can gain more + privileges than its parent process. This bool directly controls if + the no_new_privs flag will be set on the container process. + AllowPrivilegeEscalation is true always when the container is: + 1) run as Privileged + 2) has CAP_SYS_ADMIN + Note that this field cannot be set when spec.os.name is windows. + type: boolean + appArmorProfile: + description: |- + appArmorProfile is the AppArmor options to use by this container. If set, this profile + overrides the pod's appArmorProfile. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile loaded on the node that should be used. + The profile must be preconfigured on the node to work. + Must match the loaded name of the profile. + Must be set if and only if type is "Localhost". + type: string + type: + description: |- + type indicates which kind of AppArmor profile will be applied. + Valid options are: + Localhost - a profile pre-loaded on the node. + RuntimeDefault - the container runtime's default profile. + Unconfined - no AppArmor enforcement. + type: string + required: + - type + type: object + capabilities: + description: |- + The capabilities to add/drop when running containers. + Defaults to the default set of capabilities granted by the container runtime. + Note that this field cannot be set when spec.os.name is windows. + properties: + add: + description: Added capabilities + items: + description: Capability represent + POSIX capabilities type + type: string + type: array + x-kubernetes-list-type: atomic + drop: + description: Removed capabilities + items: + description: Capability represent + POSIX capabilities type + type: string + type: array + x-kubernetes-list-type: atomic + type: object + privileged: + description: |- + Run container in privileged mode. + Processes in privileged containers are essentially equivalent to root on the host. + Defaults to false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + procMount: + description: |- + procMount denotes the type of proc mount to use for the containers. + The default value is Default which uses the container runtime defaults for + readonly paths and masked paths. + This requires the ProcMountType feature flag to be enabled. + Note that this field cannot be set when spec.os.name is windows. + type: string + readOnlyRootFilesystem: + description: |- + Whether this container has a read-only root filesystem. + Default is false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + runAsGroup: + description: |- + The GID to run the entrypoint of the container process. + Uses runtime default if unset. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + runAsNonRoot: + description: |- + Indicates that the container must run as a non-root user. + If true, the Kubelet will validate the image at runtime to ensure that it + does not run as UID 0 (root) and fail to start the container if it does. + If unset or false, no such validation will be performed. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: |- + The UID to run the entrypoint of the container process. + Defaults to user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + seLinuxOptions: + description: |- + The SELinux context to be applied to the container. + If unspecified, the container runtime will allocate a random SELinux context for each + container. May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + properties: + level: + description: Level is SELinux level + label that applies to the container. + type: string + role: + description: Role is a SELinux role + label that applies to the container. + type: string + type: + description: Type is a SELinux type + label that applies to the container. + type: string + user: + description: User is a SELinux user + label that applies to the container. + type: string + type: object + seccompProfile: + description: |- + The seccomp options to use by this container. If seccomp options are + provided at both the pod & container level, the container options + override the pod options. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile defined in a file on the node should be used. + The profile must be preconfigured on the node to work. + Must be a descending path, relative to the kubelet's configured seccomp profile location. + Must be set if type is "Localhost". Must NOT be set for any other type. + type: string + type: + description: |- + type indicates which kind of seccomp profile will be applied. + Valid options are: + + Localhost - a profile defined in a file on the node should be used. + RuntimeDefault - the container runtime default profile should be used. + Unconfined - no profile should be applied. + type: string + required: + - type + type: object + windowsOptions: + description: |- + The Windows specific settings applied to all containers. + If unspecified, the options from the PodSecurityContext will be used. + If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is linux. + properties: + gmsaCredentialSpec: + description: |- + GMSACredentialSpec is where the GMSA admission webhook + (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the + GMSA credential spec named by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName + is the name of the GMSA credential + spec to use. + type: string + hostProcess: + description: |- + HostProcess determines if a container should be run as a 'Host Process' container. + All of a Pod's containers must have the same effective HostProcess value + (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). + In addition, if HostProcess is true then HostNetwork must also be set to true. + type: boolean + runAsUserName: + description: |- + The UserName in Windows to run the entrypoint of the container process. + Defaults to the user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: string + type: object + type: object + startupProbe: + description: Probes are not allowed for + ephemeral containers. + properties: + exec: + description: Exec specifies the action + to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action + involving a GRPC port. + properties: + port: + description: Port number of the + gRPC service. Number must be in + the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http + request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set + in the request. HTTP allows repeated + headers. + items: + description: HTTPHeader describes + a custom header to be used in + HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field + value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the + HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an + action involving a TCP port. + properties: + host: + description: 'Optional: Host name + to connect to, defaults to the + pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + stdin: + description: |- + Whether this container should allocate a buffer for stdin in the container runtime. If this + is not set, reads from stdin in the container will always result in EOF. + Default is false. + type: boolean + stdinOnce: + description: |- + Whether the container runtime should close the stdin channel after it has been opened by + a single attach. When stdin is true the stdin stream will remain open across multiple attach + sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the + first client attaches to stdin, and then remains open and accepts data until the client disconnects, + at which time stdin is closed and remains closed until the container is restarted. If this + flag is false, a container processes that reads from stdin will never receive an EOF. + Default is false + type: boolean + targetContainerName: + description: |- + If set, the name of the container from PodSpec that this ephemeral container targets. + The ephemeral container will be run in the namespaces (IPC, PID, etc) of this container. + If not set then the ephemeral container uses the namespaces configured in the Pod spec. + + The container runtime must implement support for this feature. If the runtime does not + support namespace targeting then the result of setting this field is undefined. + type: string + terminationMessagePath: + description: |- + Optional: Path at which the file to which the container's termination message + will be written is mounted into the container's filesystem. + Message written is intended to be brief final status, such as an assertion failure message. + Will be truncated by the node if greater than 4096 bytes. The total message length across + all containers will be limited to 12kb. + Defaults to /dev/termination-log. + Cannot be updated. + type: string + terminationMessagePolicy: + description: |- + Indicate how the termination message should be populated. File will use the contents of + terminationMessagePath to populate the container status message on both success and failure. + FallbackToLogsOnError will use the last chunk of container log output if the termination + message file is empty and the container exited with an error. + The log output is limited to 2048 bytes or 80 lines, whichever is smaller. + Defaults to File. + Cannot be updated. + type: string + tty: + description: |- + Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. + Default is false. + type: boolean + volumeDevices: + description: volumeDevices is the list of + block devices to be used by the container. + items: + description: volumeDevice describes a + mapping of a raw block device within + a container. + properties: + devicePath: + description: devicePath is the path + inside of the container that the + device will be mapped to. + type: string + name: + description: name must match the name + of a persistentVolumeClaim in the + pod + type: string + required: + - devicePath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - devicePath + x-kubernetes-list-type: map + volumeMounts: + description: |- + Pod volumes to mount into the container's filesystem. Subpath mounts are not allowed for ephemeral containers. + Cannot be updated. + items: + description: VolumeMount describes a mounting + of a Volume within a container. + properties: + mountPath: + description: |- + Path within the container at which the volume should be mounted. Must + not contain ':'. + type: string + mountPropagation: + description: |- + mountPropagation determines how mounts are propagated from the host + to container and the other way around. + When not set, MountPropagationNone is used. + This field is beta in 1.10. + When RecursiveReadOnly is set to IfPossible or to Enabled, MountPropagation must be None or unspecified + (which defaults to None). + type: string + name: + description: This must match the Name + of a Volume. + type: string + readOnly: + description: |- + Mounted read-only if true, read-write otherwise (false or unspecified). + Defaults to false. + type: boolean + recursiveReadOnly: + description: |- + RecursiveReadOnly specifies whether read-only mounts should be handled + recursively. + + If ReadOnly is false, this field has no meaning and must be unspecified. + + If ReadOnly is true, and this field is set to Disabled, the mount is not made + recursively read-only. If this field is set to IfPossible, the mount is made + recursively read-only, if it is supported by the container runtime. If this + field is set to Enabled, the mount is made recursively read-only if it is + supported by the container runtime, otherwise the pod will not be started and + an error will be generated to indicate the reason. + + If this field is set to IfPossible or Enabled, MountPropagation must be set to + None (or be unspecified, which defaults to None). + + If this field is not specified, it is treated as an equivalent of Disabled. + type: string + subPath: + description: |- + Path within the volume from which the container's volume should be mounted. + Defaults to "" (volume's root). + type: string + subPathExpr: + description: |- + Expanded path within the volume from which the container's volume should be mounted. + Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. + Defaults to "" (volume's root). + SubPathExpr and SubPath are mutually exclusive. + type: string + required: + - mountPath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - mountPath + x-kubernetes-list-type: map + workingDir: + description: |- + Container's working directory. + If not specified, the container runtime's default will be used, which + might be configured in the container image. + Cannot be updated. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + hostAliases: + description: |- + HostAliases is an optional list of hosts and IPs that will be injected into the pod's hosts + file if specified. + items: + description: |- + HostAlias holds the mapping between IP and hostnames that will be injected as an entry in the + pod's hosts file. + properties: + hostnames: + description: Hostnames for the above IP + address. + items: + type: string + type: array + x-kubernetes-list-type: atomic + ip: + description: IP address of the host file + entry. + type: string + required: + - ip + type: object + type: array + x-kubernetes-list-map-keys: + - ip + x-kubernetes-list-type: map + hostIPC: + description: |- + Use the host's ipc namespace. + Optional: Default to false. + type: boolean + hostNetwork: + description: |- + Host networking requested for this pod. Use the host's network namespace. + If this option is set, the ports that will be used must be specified. + Default to false. + type: boolean + hostPID: + description: |- + Use the host's pid namespace. + Optional: Default to false. + type: boolean + hostUsers: + description: |- + Use the host's user namespace. + Optional: Default to true. + If set to true or not present, the pod will be run in the host user namespace, useful + for when the pod needs a feature only available to the host user namespace, such as + loading a kernel module with CAP_SYS_MODULE. + When set to false, a new userns is created for the pod. Setting false is useful for + mitigating container breakout vulnerabilities even allowing users to run their + containers as root without actually having root privileges on the host. + This field is alpha-level and is only honored by servers that enable the UserNamespacesSupport feature. + type: boolean + hostname: + description: |- + Specifies the hostname of the Pod + If not specified, the pod's hostname will be set to a system-defined value. + type: string + imagePullSecrets: + description: |- + ImagePullSecrets is an optional list of references to secrets in the same namespace to use for pulling any of the images used by this PodSpec. + If specified, these secrets will be passed to individual puller implementations for them to use. + More info: https://kubernetes.io/docs/concepts/containers/images#specifying-imagepullsecrets-on-a-pod + items: + description: |- + LocalObjectReference contains enough information to let you locate the + referenced object inside the same namespace. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + initContainers: + description: |- + List of initialization containers belonging to the pod. + Init containers are executed in order prior to containers being started. If any + init container fails, the pod is considered to have failed and is handled according + to its restartPolicy. The name for an init container or normal container must be + unique among all containers. + Init containers may not have Lifecycle actions, Readiness probes, Liveness probes, or Startup probes. + The resourceRequirements of an init container are taken into account during scheduling + by finding the highest request/limit for each resource type, and then using the max of + of that value or the sum of the normal containers. Limits are applied to init containers + in a similar fashion. + Init containers cannot currently be added or removed. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/init-containers/ + items: + description: A single application container + that you want to run within a pod. + properties: + args: + description: |- + Arguments to the entrypoint. + The container image's CMD is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell + items: + type: string + type: array + x-kubernetes-list-type: atomic + command: + description: |- + Entrypoint array. Not executed within a shell. + The container image's ENTRYPOINT is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell + items: + type: string + type: array + x-kubernetes-list-type: atomic + env: + description: |- + List of environment variables to set in the container. + Cannot be updated. + items: + description: EnvVar represents an environment + variable present in a Container. + properties: + name: + description: Name of the environment + variable. Must be a C_IDENTIFIER. + type: string + value: + description: |- + Variable references $(VAR_NAME) are expanded + using the previously defined environment variables in the container and + any service environment variables. If a variable cannot be resolved, + the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. + "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless of whether the variable + exists or not. + Defaults to "". + type: string + valueFrom: + description: Source for the environment + variable's value. Cannot be used + if value is not empty. + properties: + configMapKeyRef: + description: Selects a key of + a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether + the ConfigMap or its key + must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + description: |- + Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, + spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. + properties: + apiVersion: + description: Version of the + schema the FieldPath is + written in terms of, defaults + to "v1". + type: string + fieldPath: + description: Path of the field + to select in the specified + API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. + properties: + containerName: + description: 'Container name: + required for volumes, optional + for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the + output format of the exposed + resources, defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource + to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + description: Selects a key of + a secret in the pod's namespace + properties: + key: + description: The key of the + secret to select from. Must + be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether + the Secret or its key must + be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + envFrom: + description: |- + List of sources to populate environment variables in the container. + The keys defined within a source must be a C_IDENTIFIER. All invalid keys + will be reported as an event when the container is starting. When a key exists in multiple + sources, the value associated with the last source will take precedence. + Values defined by an Env with a duplicate key will take precedence. + Cannot be updated. + items: + description: EnvFromSource represents + the source of a set of ConfigMaps + properties: + configMapRef: + description: The ConfigMap to select + from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the + ConfigMap must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + description: An optional identifier + to prepend to each key in the ConfigMap. + Must be a C_IDENTIFIER. + type: string + secretRef: + description: The Secret to select + from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the + Secret must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array + x-kubernetes-list-type: atomic + image: + description: |- + Container image name. + More info: https://kubernetes.io/docs/concepts/containers/images + This field is optional to allow higher level config management to default or override + container images in workload controllers like Deployments and StatefulSets. + type: string + imagePullPolicy: + description: |- + Image pull policy. + One of Always, Never, IfNotPresent. + Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/containers/images#updating-images + type: string + lifecycle: + description: |- + Actions that the management system should take in response to container lifecycle events. + Cannot be updated. + properties: + postStart: + description: |- + PostStart is called immediately after a container is created. If the handler fails, + the container is terminated and restarted according to its restart policy. + Other management of the container blocks until the hook completes. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies the + action to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies the + http request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers + to set in the request. HTTP + allows repeated headers. + items: + description: HTTPHeader describes + a custom header to be used + in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header + field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access + on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents the + duration that the container should + sleep before being terminated. + properties: + seconds: + description: Seconds is the + number of seconds to sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for the backward compatibility. There are no validation of this field and + lifecycle hooks will fail in runtime when tcp handler is specified. + properties: + host: + description: 'Optional: Host + name to connect to, defaults + to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + preStop: + description: |- + PreStop is called immediately before a container is terminated due to an + API request or management event such as liveness/startup probe failure, + preemption, resource contention, etc. The handler is not called if the + container crashes or exits. The Pod's termination grace period countdown begins before the + PreStop hook is executed. Regardless of the outcome of the handler, the + container will eventually terminate within the Pod's termination grace + period (unless delayed by finalizers). Other management of the container blocks until the hook completes + or until the termination grace period is reached. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies the + action to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies the + http request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers + to set in the request. HTTP + allows repeated headers. + items: + description: HTTPHeader describes + a custom header to be used + in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header + field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access + on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents the + duration that the container should + sleep before being terminated. + properties: + seconds: + description: Seconds is the + number of seconds to sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for the backward compatibility. There are no validation of this field and + lifecycle hooks will fail in runtime when tcp handler is specified. + properties: + host: + description: 'Optional: Host + name to connect to, defaults + to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + type: object + livenessProbe: + description: |- + Periodic probe of container liveness. + Container will be restarted if the probe fails. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies the action + to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action + involving a GRPC port. + properties: + port: + description: Port number of the + gRPC service. Number must be in + the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http + request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set + in the request. HTTP allows repeated + headers. + items: + description: HTTPHeader describes + a custom header to be used in + HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field + value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the + HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an + action involving a TCP port. + properties: + host: + description: 'Optional: Host name + to connect to, defaults to the + pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + name: + description: |- + Name of the container specified as a DNS_LABEL. + Each container in a pod must have a unique name (DNS_LABEL). + Cannot be updated. + type: string + ports: + description: |- + List of ports to expose from the container. Not specifying a port here + DOES NOT prevent that port from being exposed. Any port which is + listening on the default "0.0.0.0" address inside a container will be + accessible from the network. + Modifying this array with strategic merge patch may corrupt the data. + For more information See https://github.com/kubernetes/kubernetes/issues/108255. + Cannot be updated. + items: + description: ContainerPort represents + a network port in a single container. + properties: + containerPort: + description: |- + Number of port to expose on the pod's IP address. + This must be a valid port number, 0 < x < 65536. + format: int32 + type: integer + hostIP: + description: What host IP to bind + the external port to. + type: string + hostPort: + description: |- + Number of port to expose on the host. + If specified, this must be a valid port number, 0 < x < 65536. + If HostNetwork is specified, this must match ContainerPort. + Most containers do not need this. + format: int32 + type: integer + name: + description: |- + If specified, this must be an IANA_SVC_NAME and unique within the pod. Each + named port in a pod must have a unique name. Name for the port that can be + referred to by services. + type: string + protocol: + default: TCP + description: |- + Protocol for port. Must be UDP, TCP, or SCTP. + Defaults to "TCP". + type: string + required: + - containerPort + type: object + type: array + x-kubernetes-list-map-keys: + - containerPort + - protocol + x-kubernetes-list-type: map + readinessProbe: + description: |- + Periodic probe of container service readiness. + Container will be removed from service endpoints if the probe fails. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies the action + to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action + involving a GRPC port. + properties: + port: + description: Port number of the + gRPC service. Number must be in + the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http + request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set + in the request. HTTP allows repeated + headers. + items: + description: HTTPHeader describes + a custom header to be used in + HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field + value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the + HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an + action involving a TCP port. + properties: + host: + description: 'Optional: Host name + to connect to, defaults to the + pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + resizePolicy: + description: Resources resize policy for + the container. + items: + description: ContainerResizePolicy represents + resource resize policy for the container. + properties: + resourceName: + description: |- + Name of the resource to which this resource resize policy applies. + Supported values: cpu, memory. + type: string + restartPolicy: + description: |- + Restart policy to apply when specified resource is resized. + If not specified, it defaults to NotRequired. + type: string + required: + - resourceName + - restartPolicy + type: object + type: array + x-kubernetes-list-type: atomic + resources: + description: |- + Compute Resources required by this container. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references + one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + restartPolicy: + description: |- + RestartPolicy defines the restart behavior of individual containers in a pod. + This field may only be set for init containers, and the only allowed value is "Always". + For non-init containers or when this field is not specified, + the restart behavior is defined by the Pod's restart policy and the container type. + Setting the RestartPolicy as "Always" for the init container will have the following effect: + this init container will be continually restarted on + exit until all regular containers have terminated. Once all regular + containers have completed, all init containers with restartPolicy "Always" + will be shut down. This lifecycle differs from normal init containers and + is often referred to as a "sidecar" container. Although this init + container still starts in the init container sequence, it does not wait + for the container to complete before proceeding to the next init + container. Instead, the next init container starts immediately after this + init container is started, or after any startupProbe has successfully + completed. + type: string + securityContext: + description: |- + SecurityContext defines the security options the container should be run with. + If set, the fields of SecurityContext override the equivalent fields of PodSecurityContext. + More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ + properties: + allowPrivilegeEscalation: + description: |- + AllowPrivilegeEscalation controls whether a process can gain more + privileges than its parent process. This bool directly controls if + the no_new_privs flag will be set on the container process. + AllowPrivilegeEscalation is true always when the container is: + 1) run as Privileged + 2) has CAP_SYS_ADMIN + Note that this field cannot be set when spec.os.name is windows. + type: boolean + appArmorProfile: + description: |- + appArmorProfile is the AppArmor options to use by this container. If set, this profile + overrides the pod's appArmorProfile. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile loaded on the node that should be used. + The profile must be preconfigured on the node to work. + Must match the loaded name of the profile. + Must be set if and only if type is "Localhost". + type: string + type: + description: |- + type indicates which kind of AppArmor profile will be applied. + Valid options are: + Localhost - a profile pre-loaded on the node. + RuntimeDefault - the container runtime's default profile. + Unconfined - no AppArmor enforcement. + type: string + required: + - type + type: object + capabilities: + description: |- + The capabilities to add/drop when running containers. + Defaults to the default set of capabilities granted by the container runtime. + Note that this field cannot be set when spec.os.name is windows. + properties: + add: + description: Added capabilities + items: + description: Capability represent + POSIX capabilities type + type: string + type: array + x-kubernetes-list-type: atomic + drop: + description: Removed capabilities + items: + description: Capability represent + POSIX capabilities type + type: string + type: array + x-kubernetes-list-type: atomic + type: object + privileged: + description: |- + Run container in privileged mode. + Processes in privileged containers are essentially equivalent to root on the host. + Defaults to false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + procMount: + description: |- + procMount denotes the type of proc mount to use for the containers. + The default value is Default which uses the container runtime defaults for + readonly paths and masked paths. + This requires the ProcMountType feature flag to be enabled. + Note that this field cannot be set when spec.os.name is windows. + type: string + readOnlyRootFilesystem: + description: |- + Whether this container has a read-only root filesystem. + Default is false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + runAsGroup: + description: |- + The GID to run the entrypoint of the container process. + Uses runtime default if unset. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + runAsNonRoot: + description: |- + Indicates that the container must run as a non-root user. + If true, the Kubelet will validate the image at runtime to ensure that it + does not run as UID 0 (root) and fail to start the container if it does. + If unset or false, no such validation will be performed. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: |- + The UID to run the entrypoint of the container process. + Defaults to user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + seLinuxOptions: + description: |- + The SELinux context to be applied to the container. + If unspecified, the container runtime will allocate a random SELinux context for each + container. May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + properties: + level: + description: Level is SELinux level + label that applies to the container. + type: string + role: + description: Role is a SELinux role + label that applies to the container. + type: string + type: + description: Type is a SELinux type + label that applies to the container. + type: string + user: + description: User is a SELinux user + label that applies to the container. + type: string + type: object + seccompProfile: + description: |- + The seccomp options to use by this container. If seccomp options are + provided at both the pod & container level, the container options + override the pod options. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile defined in a file on the node should be used. + The profile must be preconfigured on the node to work. + Must be a descending path, relative to the kubelet's configured seccomp profile location. + Must be set if type is "Localhost". Must NOT be set for any other type. + type: string + type: + description: |- + type indicates which kind of seccomp profile will be applied. + Valid options are: + + Localhost - a profile defined in a file on the node should be used. + RuntimeDefault - the container runtime default profile should be used. + Unconfined - no profile should be applied. + type: string + required: + - type + type: object + windowsOptions: + description: |- + The Windows specific settings applied to all containers. + If unspecified, the options from the PodSecurityContext will be used. + If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is linux. + properties: + gmsaCredentialSpec: + description: |- + GMSACredentialSpec is where the GMSA admission webhook + (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the + GMSA credential spec named by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName + is the name of the GMSA credential + spec to use. + type: string + hostProcess: + description: |- + HostProcess determines if a container should be run as a 'Host Process' container. + All of a Pod's containers must have the same effective HostProcess value + (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). + In addition, if HostProcess is true then HostNetwork must also be set to true. + type: boolean + runAsUserName: + description: |- + The UserName in Windows to run the entrypoint of the container process. + Defaults to the user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: string + type: object + type: object + startupProbe: + description: |- + StartupProbe indicates that the Pod has successfully initialized. + If specified, no other probes are executed until this completes successfully. + If this probe fails, the Pod will be restarted, just as if the livenessProbe failed. + This can be used to provide different probe parameters at the beginning of a Pod's lifecycle, + when it might take a long time to load data or warm a cache, than during steady-state operation. + This cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies the action + to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action + involving a GRPC port. + properties: + port: + description: Port number of the + gRPC service. Number must be in + the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http + request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set + in the request. HTTP allows repeated + headers. + items: + description: HTTPHeader describes + a custom header to be used in + HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field + value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the + HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an + action involving a TCP port. + properties: + host: + description: 'Optional: Host name + to connect to, defaults to the + pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + stdin: + description: |- + Whether this container should allocate a buffer for stdin in the container runtime. If this + is not set, reads from stdin in the container will always result in EOF. + Default is false. + type: boolean + stdinOnce: + description: |- + Whether the container runtime should close the stdin channel after it has been opened by + a single attach. When stdin is true the stdin stream will remain open across multiple attach + sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the + first client attaches to stdin, and then remains open and accepts data until the client disconnects, + at which time stdin is closed and remains closed until the container is restarted. If this + flag is false, a container processes that reads from stdin will never receive an EOF. + Default is false + type: boolean + terminationMessagePath: + description: |- + Optional: Path at which the file to which the container's termination message + will be written is mounted into the container's filesystem. + Message written is intended to be brief final status, such as an assertion failure message. + Will be truncated by the node if greater than 4096 bytes. The total message length across + all containers will be limited to 12kb. + Defaults to /dev/termination-log. + Cannot be updated. + type: string + terminationMessagePolicy: + description: |- + Indicate how the termination message should be populated. File will use the contents of + terminationMessagePath to populate the container status message on both success and failure. + FallbackToLogsOnError will use the last chunk of container log output if the termination + message file is empty and the container exited with an error. + The log output is limited to 2048 bytes or 80 lines, whichever is smaller. + Defaults to File. + Cannot be updated. + type: string + tty: + description: |- + Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. + Default is false. + type: boolean + volumeDevices: + description: volumeDevices is the list of + block devices to be used by the container. + items: + description: volumeDevice describes a + mapping of a raw block device within + a container. + properties: + devicePath: + description: devicePath is the path + inside of the container that the + device will be mapped to. + type: string + name: + description: name must match the name + of a persistentVolumeClaim in the + pod + type: string + required: + - devicePath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - devicePath + x-kubernetes-list-type: map + volumeMounts: + description: |- + Pod volumes to mount into the container's filesystem. + Cannot be updated. + items: + description: VolumeMount describes a mounting + of a Volume within a container. + properties: + mountPath: + description: |- + Path within the container at which the volume should be mounted. Must + not contain ':'. + type: string + mountPropagation: + description: |- + mountPropagation determines how mounts are propagated from the host + to container and the other way around. + When not set, MountPropagationNone is used. + This field is beta in 1.10. + When RecursiveReadOnly is set to IfPossible or to Enabled, MountPropagation must be None or unspecified + (which defaults to None). + type: string + name: + description: This must match the Name + of a Volume. + type: string + readOnly: + description: |- + Mounted read-only if true, read-write otherwise (false or unspecified). + Defaults to false. + type: boolean + recursiveReadOnly: + description: |- + RecursiveReadOnly specifies whether read-only mounts should be handled + recursively. + + If ReadOnly is false, this field has no meaning and must be unspecified. + + If ReadOnly is true, and this field is set to Disabled, the mount is not made + recursively read-only. If this field is set to IfPossible, the mount is made + recursively read-only, if it is supported by the container runtime. If this + field is set to Enabled, the mount is made recursively read-only if it is + supported by the container runtime, otherwise the pod will not be started and + an error will be generated to indicate the reason. + + If this field is set to IfPossible or Enabled, MountPropagation must be set to + None (or be unspecified, which defaults to None). + + If this field is not specified, it is treated as an equivalent of Disabled. + type: string + subPath: + description: |- + Path within the volume from which the container's volume should be mounted. + Defaults to "" (volume's root). + type: string + subPathExpr: + description: |- + Expanded path within the volume from which the container's volume should be mounted. + Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. + Defaults to "" (volume's root). + SubPathExpr and SubPath are mutually exclusive. + type: string + required: + - mountPath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - mountPath + x-kubernetes-list-type: map + workingDir: + description: |- + Container's working directory. + If not specified, the container runtime's default will be used, which + might be configured in the container image. + Cannot be updated. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + nodeName: + description: |- + NodeName indicates in which node this pod is scheduled. + If empty, this pod is a candidate for scheduling by the scheduler defined in schedulerName. + Once this field is set, the kubelet for this node becomes responsible for the lifecycle of this pod. + This field should not be used to express a desire for the pod to be scheduled on a specific node. + https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodename + type: string + nodeSelector: + additionalProperties: + type: string + description: |- + NodeSelector is a selector which must be true for the pod to fit on a node. + Selector which must match a node's labels for the pod to be scheduled on that node. + More info: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ + type: object + x-kubernetes-map-type: atomic + os: + description: |- + Specifies the OS of the containers in the pod. + Some pod and container fields are restricted if this is set. + + If the OS field is set to linux, the following fields must be unset: + -securityContext.windowsOptions + + If the OS field is set to windows, following fields must be unset: + - spec.hostPID + - spec.hostIPC + - spec.hostUsers + - spec.securityContext.appArmorProfile + - spec.securityContext.seLinuxOptions + - spec.securityContext.seccompProfile + - spec.securityContext.fsGroup + - spec.securityContext.fsGroupChangePolicy + - spec.securityContext.sysctls + - spec.shareProcessNamespace + - spec.securityContext.runAsUser + - spec.securityContext.runAsGroup + - spec.securityContext.supplementalGroups + - spec.securityContext.supplementalGroupsPolicy + - spec.containers[*].securityContext.appArmorProfile + - spec.containers[*].securityContext.seLinuxOptions + - spec.containers[*].securityContext.seccompProfile + - spec.containers[*].securityContext.capabilities + - spec.containers[*].securityContext.readOnlyRootFilesystem + - spec.containers[*].securityContext.privileged + - spec.containers[*].securityContext.allowPrivilegeEscalation + - spec.containers[*].securityContext.procMount + - spec.containers[*].securityContext.runAsUser + - spec.containers[*].securityContext.runAsGroup + properties: + name: + description: |- + Name is the name of the operating system. The currently supported values are linux and windows. + Additional value may be defined in future and can be one of: + https://github.com/opencontainers/runtime-spec/blob/master/config.md#platform-specific-configuration + Clients should expect to handle additional values and treat unrecognized values in this field as os: null + type: string + required: + - name + type: object + overhead: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Overhead represents the resource overhead associated with running a pod for a given RuntimeClass. + This field will be autopopulated at admission time by the RuntimeClass admission controller. If + the RuntimeClass admission controller is enabled, overhead must not be set in Pod create requests. + The RuntimeClass admission controller will reject Pod create requests which have the overhead already + set. If RuntimeClass is configured and selected in the PodSpec, Overhead will be set to the value + defined in the corresponding RuntimeClass, otherwise it will remain unset and treated as zero. + More info: https://git.k8s.io/enhancements/keps/sig-node/688-pod-overhead/README.md + type: object + preemptionPolicy: + description: |- + PreemptionPolicy is the Policy for preempting pods with lower priority. + One of Never, PreemptLowerPriority. + Defaults to PreemptLowerPriority if unset. + type: string + priority: + description: |- + The priority value. Various system components use this field to find the + priority of the pod. When Priority Admission Controller is enabled, it + prevents users from setting this field. The admission controller populates + this field from PriorityClassName. + The higher the value, the higher the priority. + format: int32 + type: integer + priorityClassName: + description: |- + If specified, indicates the pod's priority. "system-node-critical" and + "system-cluster-critical" are two special keywords which indicate the + highest priorities with the former being the highest priority. Any other + name must be defined by creating a PriorityClass object with that name. + If not specified, the pod priority will be default or zero if there is no + default. + type: string + readinessGates: + description: |- + If specified, all readiness gates will be evaluated for pod readiness. + A pod is ready when all its containers are ready AND + all conditions specified in the readiness gates have status equal to "True" + More info: https://git.k8s.io/enhancements/keps/sig-network/580-pod-readiness-gates + items: + description: PodReadinessGate contains the reference + to a pod condition + properties: + conditionType: + description: ConditionType refers to a condition + in the pod's condition list with matching + type. + type: string + required: + - conditionType + type: object + type: array + x-kubernetes-list-type: atomic + resourceClaims: + description: |- + ResourceClaims defines which ResourceClaims must be allocated + and reserved before the Pod is allowed to start. The resources + will be made available to those containers which consume them + by name. + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + This field is immutable. + items: + description: |- + PodResourceClaim references exactly one ResourceClaim, either directly + or by naming a ResourceClaimTemplate which is then turned into a ResourceClaim + for the pod. + + It adds a name to it that uniquely identifies the ResourceClaim inside the Pod. + Containers that need access to the ResourceClaim reference it with this name. + properties: + name: + description: |- + Name uniquely identifies this resource claim inside the pod. + This must be a DNS_LABEL. + type: string + resourceClaimName: + description: |- + ResourceClaimName is the name of a ResourceClaim object in the same + namespace as this pod. + + Exactly one of ResourceClaimName and ResourceClaimTemplateName must + be set. + type: string + resourceClaimTemplateName: + description: |- + ResourceClaimTemplateName is the name of a ResourceClaimTemplate + object in the same namespace as this pod. + + The template will be used to create a new ResourceClaim, which will + be bound to this pod. When this pod is deleted, the ResourceClaim + will also be deleted. The pod name and resource name, along with a + generated component, will be used to form a unique name for the + ResourceClaim, which will be recorded in pod.status.resourceClaimStatuses. + + This field is immutable and no changes will be made to the + corresponding ResourceClaim by the control plane after creating the + ResourceClaim. + + Exactly one of ResourceClaimName and ResourceClaimTemplateName must + be set. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + restartPolicy: + description: |- + Restart policy for all containers within the pod. + One of Always, OnFailure, Never. In some contexts, only a subset of those values may be permitted. + Default to Always. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#restart-policy + type: string + runtimeClassName: + description: |- + RuntimeClassName refers to a RuntimeClass object in the node.k8s.io group, which should be used + to run this pod. If no RuntimeClass resource matches the named class, the pod will not be run. + If unset or empty, the "legacy" RuntimeClass will be used, which is an implicit class with an + empty definition that uses the default runtime handler. + More info: https://git.k8s.io/enhancements/keps/sig-node/585-runtime-class + type: string + schedulerName: + description: |- + If specified, the pod will be dispatched by specified scheduler. + If not specified, the pod will be dispatched by default scheduler. + type: string + schedulingGates: + description: |- + SchedulingGates is an opaque list of values that if specified will block scheduling the pod. + If schedulingGates is not empty, the pod will stay in the SchedulingGated state and the + scheduler will not attempt to schedule the pod. + + SchedulingGates can only be set at pod creation time, and be removed only afterwards. + items: + description: PodSchedulingGate is associated + to a Pod to guard its scheduling. + properties: + name: + description: |- + Name of the scheduling gate. + Each scheduling gate must have a unique name field. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + securityContext: + description: |- + SecurityContext holds pod-level security attributes and common container settings. + Optional: Defaults to empty. See type description for default values of each field. + properties: + appArmorProfile: + description: |- + appArmorProfile is the AppArmor options to use by the containers in this pod. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile loaded on the node that should be used. + The profile must be preconfigured on the node to work. + Must match the loaded name of the profile. + Must be set if and only if type is "Localhost". + type: string + type: + description: |- + type indicates which kind of AppArmor profile will be applied. + Valid options are: + Localhost - a profile pre-loaded on the node. + RuntimeDefault - the container runtime's default profile. + Unconfined - no AppArmor enforcement. + type: string + required: + - type + type: object + fsGroup: + description: |- + A special supplemental group that applies to all containers in a pod. + Some volume types allow the Kubelet to change the ownership of that volume + to be owned by the pod: + + 1. The owning GID will be the FSGroup + 2. The setgid bit is set (new files created in the volume will be owned by FSGroup) + 3. The permission bits are OR'd with rw-rw---- + + If unset, the Kubelet will not modify the ownership and permissions of any volume. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + fsGroupChangePolicy: + description: |- + fsGroupChangePolicy defines behavior of changing ownership and permission of the volume + before being exposed inside Pod. This field will only apply to + volume types which support fsGroup based ownership(and permissions). + It will have no effect on ephemeral volume types such as: secret, configmaps + and emptydir. + Valid values are "OnRootMismatch" and "Always". If not specified, "Always" is used. + Note that this field cannot be set when spec.os.name is windows. + type: string + runAsGroup: + description: |- + The GID to run the entrypoint of the container process. + Uses runtime default if unset. + May also be set in SecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence + for that container. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + runAsNonRoot: + description: |- + Indicates that the container must run as a non-root user. + If true, the Kubelet will validate the image at runtime to ensure that it + does not run as UID 0 (root) and fail to start the container if it does. + If unset or false, no such validation will be performed. + May also be set in SecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: |- + The UID to run the entrypoint of the container process. + Defaults to user specified in image metadata if unspecified. + May also be set in SecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence + for that container. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + seLinuxOptions: + description: |- + The SELinux context to be applied to all containers. + If unspecified, the container runtime will allocate a random SELinux context for each + container. May also be set in SecurityContext. If set in + both SecurityContext and PodSecurityContext, the value specified in SecurityContext + takes precedence for that container. + Note that this field cannot be set when spec.os.name is windows. + properties: + level: + description: Level is SELinux level label + that applies to the container. + type: string + role: + description: Role is a SELinux role label + that applies to the container. + type: string + type: + description: Type is a SELinux type label + that applies to the container. + type: string + user: + description: User is a SELinux user label + that applies to the container. + type: string + type: object + seccompProfile: + description: |- + The seccomp options to use by the containers in this pod. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile defined in a file on the node should be used. + The profile must be preconfigured on the node to work. + Must be a descending path, relative to the kubelet's configured seccomp profile location. + Must be set if type is "Localhost". Must NOT be set for any other type. + type: string + type: + description: |- + type indicates which kind of seccomp profile will be applied. + Valid options are: + + Localhost - a profile defined in a file on the node should be used. + RuntimeDefault - the container runtime default profile should be used. + Unconfined - no profile should be applied. + type: string + required: + - type + type: object + supplementalGroups: + description: |- + A list of groups applied to the first process run in each container, in + addition to the container's primary GID and fsGroup (if specified). If + the SupplementalGroupsPolicy feature is enabled, the + supplementalGroupsPolicy field determines whether these are in addition + to or instead of any group memberships defined in the container image. + If unspecified, no additional groups are added, though group memberships + defined in the container image may still be used, depending on the + supplementalGroupsPolicy field. + Note that this field cannot be set when spec.os.name is windows. + items: + format: int64 + type: integer + type: array + x-kubernetes-list-type: atomic + supplementalGroupsPolicy: + description: |- + Defines how supplemental groups of the first container processes are calculated. + Valid values are "Merge" and "Strict". If not specified, "Merge" is used. + (Alpha) Using the field requires the SupplementalGroupsPolicy feature gate to be enabled + and the container runtime must implement support for this feature. + Note that this field cannot be set when spec.os.name is windows. + type: string + sysctls: + description: |- + Sysctls hold a list of namespaced sysctls used for the pod. Pods with unsupported + sysctls (by the container runtime) might fail to launch. + Note that this field cannot be set when spec.os.name is windows. + items: + description: Sysctl defines a kernel parameter + to be set + properties: + name: + description: Name of a property to set + type: string + value: + description: Value of a property to + set + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + windowsOptions: + description: |- + The Windows specific settings applied to all containers. + If unspecified, the options within a container's SecurityContext will be used. + If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is linux. + properties: + gmsaCredentialSpec: + description: |- + GMSACredentialSpec is where the GMSA admission webhook + (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the + GMSA credential spec named by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName is + the name of the GMSA credential spec + to use. + type: string + hostProcess: + description: |- + HostProcess determines if a container should be run as a 'Host Process' container. + All of a Pod's containers must have the same effective HostProcess value + (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). + In addition, if HostProcess is true then HostNetwork must also be set to true. + type: boolean + runAsUserName: + description: |- + The UserName in Windows to run the entrypoint of the container process. + Defaults to the user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: string + type: object + type: object + serviceAccount: + description: |- + DeprecatedServiceAccount is a deprecated alias for ServiceAccountName. + Deprecated: Use serviceAccountName instead. + type: string + serviceAccountName: + description: |- + ServiceAccountName is the name of the ServiceAccount to use to run this pod. + More info: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ + type: string + setHostnameAsFQDN: + description: |- + If true the pod's hostname will be configured as the pod's FQDN, rather than the leaf name (the default). + In Linux containers, this means setting the FQDN in the hostname field of the kernel (the nodename field of struct utsname). + In Windows containers, this means setting the registry value of hostname for the registry key HKEY_LOCAL_MACHINE\\SYSTEM\\CurrentControlSet\\Services\\Tcpip\\Parameters to FQDN. + If a pod does not have FQDN, this has no effect. + Default to false. + type: boolean + shareProcessNamespace: + description: |- + Share a single process namespace between all of the containers in a pod. + When this is set containers will be able to view and signal processes from other containers + in the same pod, and the first process in each container will not be assigned PID 1. + HostPID and ShareProcessNamespace cannot both be set. + Optional: Default to false. + type: boolean + subdomain: + description: |- + If specified, the fully qualified Pod hostname will be "...svc.". + If not specified, the pod will not have a domainname at all. + type: string + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully. May be decreased in delete request. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + If this value is nil, the default grace period will be used instead. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + Defaults to 30 seconds. + format: int64 + type: integer + tolerations: + description: If specified, the pod's tolerations. + items: + description: |- + The pod this Toleration is attached to tolerates any taint that matches + the triple using the matching operator . + properties: + effect: + description: |- + Effect indicates the taint effect to match. Empty means match all taint effects. + When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: |- + Key is the taint key that the toleration applies to. Empty means match all taint keys. + If the key is empty, operator must be Exists; this combination means to match all values and all keys. + type: string + operator: + description: |- + Operator represents a key's relationship to the value. + Valid operators are Exists and Equal. Defaults to Equal. + Exists is equivalent to wildcard for value, so that a pod can + tolerate all taints of a particular category. + type: string + tolerationSeconds: + description: |- + TolerationSeconds represents the period of time the toleration (which must be + of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, + it is not set, which means tolerate the taint forever (do not evict). Zero and + negative values will be treated as 0 (evict immediately) by the system. + format: int64 + type: integer + value: + description: |- + Value is the taint value the toleration matches to. + If the operator is Exists, the value should be empty, otherwise just a regular string. + type: string + type: object + type: array + x-kubernetes-list-type: atomic + topologySpreadConstraints: + description: |- + TopologySpreadConstraints describes how a group of pods ought to spread across topology + domains. Scheduler will schedule pods in a way which abides by the constraints. + All topologySpreadConstraints are ANDed. + items: + description: TopologySpreadConstraint specifies + how to spread matching pods among the given + topology. + properties: + labelSelector: + description: |- + LabelSelector is used to find matching pods. + Pods that match this label selector are counted to determine the number of pods + in their corresponding topology domain. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The + requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select the pods over which + spreading will be calculated. The keys are used to lookup values from the + incoming pod labels, those key-value labels are ANDed with labelSelector + to select the group of existing pods over which spreading will be calculated + for the incoming pod. The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + MatchLabelKeys cannot be set when LabelSelector isn't set. + Keys that don't exist in the incoming pod labels will + be ignored. A null or empty list means only match against labelSelector. + + This is a beta field and requires the MatchLabelKeysInPodTopologySpread feature gate to be enabled (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + maxSkew: + description: |- + MaxSkew describes the degree to which pods may be unevenly distributed. + When `whenUnsatisfiable=DoNotSchedule`, it is the maximum permitted difference + between the number of matching pods in the target topology and the global minimum. + The global minimum is the minimum number of matching pods in an eligible domain + or zero if the number of eligible domains is less than MinDomains. + For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same + labelSelector spread as 2/2/1: + In this case, the global minimum is 1. + | zone1 | zone2 | zone3 | + | P P | P P | P | + - if MaxSkew is 1, incoming pod can only be scheduled to zone3 to become 2/2/2; + scheduling it onto zone1(zone2) would make the ActualSkew(3-1) on zone1(zone2) + violate MaxSkew(1). + - if MaxSkew is 2, incoming pod can be scheduled onto any zone. + When `whenUnsatisfiable=ScheduleAnyway`, it is used to give higher precedence + to topologies that satisfy it. + It's a required field. Default value is 1 and 0 is not allowed. + format: int32 + type: integer + minDomains: + description: |- + MinDomains indicates a minimum number of eligible domains. + When the number of eligible domains with matching topology keys is less than minDomains, + Pod Topology Spread treats "global minimum" as 0, and then the calculation of Skew is performed. + And when the number of eligible domains with matching topology keys equals or greater than minDomains, + this value has no effect on scheduling. + As a result, when the number of eligible domains is less than minDomains, + scheduler won't schedule more than maxSkew Pods to those domains. + If value is nil, the constraint behaves as if MinDomains is equal to 1. + Valid values are integers greater than 0. + When value is not nil, WhenUnsatisfiable must be DoNotSchedule. + + For example, in a 3-zone cluster, MaxSkew is set to 2, MinDomains is set to 5 and pods with the same + labelSelector spread as 2/2/2: + | zone1 | zone2 | zone3 | + | P P | P P | P P | + The number of domains is less than 5(MinDomains), so "global minimum" is treated as 0. + In this situation, new pod with the same labelSelector cannot be scheduled, + because computed skew will be 3(3 - 0) if new Pod is scheduled to any of the three zones, + it will violate MaxSkew. + format: int32 + type: integer + nodeAffinityPolicy: + description: |- + NodeAffinityPolicy indicates how we will treat Pod's nodeAffinity/nodeSelector + when calculating pod topology spread skew. Options are: + - Honor: only nodes matching nodeAffinity/nodeSelector are included in the calculations. + - Ignore: nodeAffinity/nodeSelector are ignored. All nodes are included in the calculations. + + If this value is nil, the behavior is equivalent to the Honor policy. + This is a beta-level feature default enabled by the NodeInclusionPolicyInPodTopologySpread feature flag. + type: string + nodeTaintsPolicy: + description: |- + NodeTaintsPolicy indicates how we will treat node taints when calculating + pod topology spread skew. Options are: + - Honor: nodes without taints, along with tainted nodes for which the incoming pod + has a toleration, are included. + - Ignore: node taints are ignored. All nodes are included. + + If this value is nil, the behavior is equivalent to the Ignore policy. + This is a beta-level feature default enabled by the NodeInclusionPolicyInPodTopologySpread feature flag. + type: string + topologyKey: + description: |- + TopologyKey is the key of node labels. Nodes that have a label with this key + and identical values are considered to be in the same topology. + We consider each as a "bucket", and try to put balanced number + of pods into each bucket. + We define a domain as a particular instance of a topology. + Also, we define an eligible domain as a domain whose nodes meet the requirements of + nodeAffinityPolicy and nodeTaintsPolicy. + e.g. If TopologyKey is "kubernetes.io/hostname", each Node is a domain of that topology. + And, if TopologyKey is "topology.kubernetes.io/zone", each zone is a domain of that topology. + It's a required field. + type: string + whenUnsatisfiable: + description: |- + WhenUnsatisfiable indicates how to deal with a pod if it doesn't satisfy + the spread constraint. + - DoNotSchedule (default) tells the scheduler not to schedule it. + - ScheduleAnyway tells the scheduler to schedule the pod in any location, + but giving higher precedence to topologies that would help reduce the + skew. + A constraint is considered "Unsatisfiable" for an incoming pod + if and only if every possible node assignment for that pod would violate + "MaxSkew" on some topology. + For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same + labelSelector spread as 3/1/1: + | zone1 | zone2 | zone3 | + | P P P | P | P | + If WhenUnsatisfiable is set to DoNotSchedule, incoming pod can only be scheduled + to zone2(zone3) to become 3/2/1(3/1/2) as ActualSkew(2-1) on zone2(zone3) satisfies + MaxSkew(1). In other words, the cluster can still be imbalanced, but scheduler + won't make it *more* imbalanced. + It's a required field. + type: string + required: + - maxSkew + - topologyKey + - whenUnsatisfiable + type: object + type: array + x-kubernetes-list-map-keys: + - topologyKey + - whenUnsatisfiable + x-kubernetes-list-type: map + volumes: + description: |- + List of volumes that can be mounted by containers belonging to the pod. + More info: https://kubernetes.io/docs/concepts/storage/volumes + items: + description: Volume represents a named volume + in a pod that may be accessed by any container + in the pod. + properties: + awsElasticBlockStore: + description: |- + awsElasticBlockStore represents an AWS Disk resource that is attached to a + kubelet's host machine and then exposed to the pod. + More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore + properties: + fsType: + description: |- + fsType is the filesystem type of the volume that you want to mount. + Tip: Ensure that the filesystem type is supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore + type: string + partition: + description: |- + partition is the partition in the volume that you want to mount. + If omitted, the default is to mount by volume name. + Examples: For volume /dev/sda1, you specify the partition as "1". + Similarly, the volume partition for /dev/sda is "0" (or you can leave the property empty). + format: int32 + type: integer + readOnly: + description: |- + readOnly value true will force the readOnly setting in VolumeMounts. + More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore + type: boolean + volumeID: + description: |- + volumeID is unique ID of the persistent disk resource in AWS (Amazon EBS volume). + More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore + type: string + required: + - volumeID + type: object + azureDisk: + description: azureDisk represents an Azure + Data Disk mount on the host and bind mount + to the pod. + properties: + cachingMode: + description: 'cachingMode is the Host + Caching mode: None, Read Only, Read + Write.' + type: string + diskName: + description: diskName is the Name of + the data disk in the blob storage + type: string + diskURI: + description: diskURI is the URI of data + disk in the blob storage + type: string + fsType: + default: ext4 + description: |- + fsType is Filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + kind: + description: 'kind expected values are + Shared: multiple blob disks per storage + account Dedicated: single blob disk + per storage account Managed: azure + managed data disk (only in managed + availability set). defaults to shared' + type: string + readOnly: + default: false + description: |- + readOnly Defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + required: + - diskName + - diskURI + type: object + azureFile: + description: azureFile represents an Azure + File Service mount on the host and bind + mount to the pod. + properties: + readOnly: + description: |- + readOnly defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + secretName: + description: secretName is the name + of secret that contains Azure Storage + Account Name and Key + type: string + shareName: + description: shareName is the azure + share Name + type: string + required: + - secretName + - shareName + type: object + cephfs: + description: cephFS represents a Ceph FS + mount on the host that shares a pod's + lifetime + properties: + monitors: + description: |- + monitors is Required: Monitors is a collection of Ceph monitors + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + items: + type: string + type: array + x-kubernetes-list-type: atomic + path: + description: 'path is Optional: Used + as the mounted root, rather than the + full Ceph tree, default is /' + type: string + readOnly: + description: |- + readOnly is Optional: Defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + type: boolean + secretFile: + description: |- + secretFile is Optional: SecretFile is the path to key ring for User, default is /etc/ceph/user.secret + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + type: string + secretRef: + description: |- + secretRef is Optional: SecretRef is reference to the authentication secret for User, default is empty. + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + user: + description: |- + user is optional: User is the rados user name, default is admin + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + type: string + required: + - monitors + type: object + cinder: + description: |- + cinder represents a cinder volume attached and mounted on kubelets host machine. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md + properties: + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md + type: string + readOnly: + description: |- + readOnly defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md + type: boolean + secretRef: + description: |- + secretRef is optional: points to a secret object containing parameters used to connect + to OpenStack. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + volumeID: + description: |- + volumeID used to identify the volume in cinder. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md + type: string + required: + - volumeID + type: object + configMap: + description: configMap represents a configMap + that should populate this volume + properties: + defaultMode: + description: |- + defaultMode is optional: mode bits used to set permissions on created files by default. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + Defaults to 0644. + Directories within the path are not affected by this setting. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + items: + description: |- + items if unspecified, each key-value pair in the Data field of the referenced + ConfigMap will be projected into the volume as a file whose name is the + key and content is the value. If specified, the listed keys will be + projected into the specified paths, and unlisted keys will not be + present. If a key is specified which is not present in the ConfigMap, + the volume setup will error unless it is marked optional. Paths must be + relative and may not contain the '..' path or start with '..'. + items: + description: Maps a string key to + a path within a volume. + properties: + key: + description: key is the key to + project. + type: string + mode: + description: |- + mode is Optional: mode bits used to set permissions on this file. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: |- + path is the relative path of the file to map the key to. + May not be an absolute path. + May not contain the path element '..'. + May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: optional specify whether + the ConfigMap or its keys must be + defined + type: boolean + type: object + x-kubernetes-map-type: atomic + csi: + description: csi (Container Storage Interface) + represents ephemeral storage that is handled + by certain external CSI drivers (Beta + feature). + properties: + driver: + description: |- + driver is the name of the CSI driver that handles this volume. + Consult with your admin for the correct name as registered in the cluster. + type: string + fsType: + description: |- + fsType to mount. Ex. "ext4", "xfs", "ntfs". + If not provided, the empty value is passed to the associated CSI driver + which will determine the default filesystem to apply. + type: string + nodePublishSecretRef: + description: |- + nodePublishSecretRef is a reference to the secret object containing + sensitive information to pass to the CSI driver to complete the CSI + NodePublishVolume and NodeUnpublishVolume calls. + This field is optional, and may be empty if no secret is required. If the + secret object contains more than one secret, all secret references are passed. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + readOnly: + description: |- + readOnly specifies a read-only configuration for the volume. + Defaults to false (read/write). + type: boolean + volumeAttributes: + additionalProperties: + type: string + description: |- + volumeAttributes stores driver-specific properties that are passed to the CSI + driver. Consult your driver's documentation for supported values. + type: object + required: + - driver + type: object + downwardAPI: + description: downwardAPI represents downward + API about the pod that should populate + this volume + properties: + defaultMode: + description: |- + Optional: mode bits to use on created files by default. Must be a + Optional: mode bits used to set permissions on created files by default. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + Defaults to 0644. + Directories within the path are not affected by this setting. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + items: + description: Items is a list of downward + API volume file + items: + description: DownwardAPIVolumeFile + represents information to create + the file containing the pod field + properties: + fieldRef: + description: 'Required: Selects + a field of the pod: only annotations, + labels, name, namespace and + uid are supported.' + properties: + apiVersion: + description: Version of the + schema the FieldPath is + written in terms of, defaults + to "v1". + type: string + fieldPath: + description: Path of the field + to select in the specified + API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + mode: + description: |- + Optional: mode bits used to set permissions on this file, must be an octal value + between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: 'Required: Path is the + relative path name of the file + to be created. Must not be absolute + or contain the ''..'' path. + Must be utf-8 encoded. The first + item of the relative path must + not start with ''..''' + type: string + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, requests.cpu and requests.memory) are currently supported. + properties: + containerName: + description: 'Container name: + required for volumes, optional + for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the + output format of the exposed + resources, defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource + to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + required: + - path + type: object + type: array + x-kubernetes-list-type: atomic + type: object + emptyDir: + description: |- + emptyDir represents a temporary directory that shares a pod's lifetime. + More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir + properties: + medium: + description: |- + medium represents what type of storage medium should back this directory. + The default is "" which means to use the node's default medium. + Must be an empty string (default) or Memory. + More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir + type: string + sizeLimit: + anyOf: + - type: integer + - type: string + description: |- + sizeLimit is the total amount of local storage required for this EmptyDir volume. + The size limit is also applicable for memory medium. + The maximum usage on memory medium EmptyDir would be the minimum value between + the SizeLimit specified here and the sum of memory limits of all containers in a pod. + The default is nil which means that the limit is undefined. + More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + ephemeral: + description: |- + ephemeral represents a volume that is handled by a cluster storage driver. + The volume's lifecycle is tied to the pod that defines it - it will be created before the pod starts, + and deleted when the pod is removed. + + Use this if: + a) the volume is only needed while the pod runs, + b) features of normal volumes like restoring from snapshot or capacity + tracking are needed, + c) the storage driver is specified through a storage class, and + d) the storage driver supports dynamic volume provisioning through + a PersistentVolumeClaim (see EphemeralVolumeSource for more + information on the connection between this volume type + and PersistentVolumeClaim). + + Use PersistentVolumeClaim or one of the vendor-specific + APIs for volumes that persist for longer than the lifecycle + of an individual pod. + + Use CSI for light-weight local ephemeral volumes if the CSI driver is meant to + be used that way - see the documentation of the driver for + more information. + + A pod can use both types of ephemeral volumes and + persistent volumes at the same time. + properties: + volumeClaimTemplate: + description: |- + Will be used to create a stand-alone PVC to provision the volume. + The pod in which this EphemeralVolumeSource is embedded will be the + owner of the PVC, i.e. the PVC will be deleted together with the + pod. The name of the PVC will be `-` where + `` is the name from the `PodSpec.Volumes` array + entry. Pod validation will reject the pod if the concatenated name + is not valid for a PVC (for example, too long). + + An existing PVC with that name that is not owned by the pod + will *not* be used for the pod to avoid using an unrelated + volume by mistake. Starting the pod is then blocked until + the unrelated PVC is removed. If such a pre-created PVC is + meant to be used by the pod, the PVC has to updated with an + owner reference to the pod once the pod exists. Normally + this should not be necessary, but it may be useful when + manually reconstructing a broken cluster. + + This field is read-only and no changes will be made by Kubernetes + to the PVC after it has been created. + + Required, must not be nil. + properties: + metadata: + description: |- + May contain labels and annotations that will be copied into the PVC + when creating it. No other fields are allowed and will be rejected during + validation. + type: object + spec: + description: |- + The specification for the PersistentVolumeClaim. The entire content is + copied unchanged into the PVC that gets created from this + template. The same fields as in a PersistentVolumeClaim + are also valid here. + properties: + accessModes: + description: |- + accessModes contains the desired access modes the volume should have. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1 + items: + type: string + type: array + x-kubernetes-list-type: atomic + dataSource: + description: |- + dataSource field can be used to specify either: + * An existing VolumeSnapshot object (snapshot.storage.k8s.io/VolumeSnapshot) + * An existing PVC (PersistentVolumeClaim) + If the provisioner or an external controller can support the specified data source, + it will create a new volume based on the contents of the specified data source. + When the AnyVolumeDataSource feature gate is enabled, dataSource contents will be copied to dataSourceRef, + and dataSourceRef contents will be copied to dataSource when dataSourceRef.namespace is not specified. + If the namespace is specified, then dataSourceRef will not be copied to dataSource. + properties: + apiGroup: + description: |- + APIGroup is the group for the resource being referenced. + If APIGroup is not specified, the specified Kind must be in the core API group. + For any other third-party types, APIGroup is required. + type: string + kind: + description: Kind is the + type of resource being + referenced + type: string + name: + description: Name is the + name of resource being + referenced + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + dataSourceRef: + description: |- + dataSourceRef specifies the object from which to populate the volume with data, if a non-empty + volume is desired. This may be any object from a non-empty API group (non + core object) or a PersistentVolumeClaim object. + When this field is specified, volume binding will only succeed if the type of + the specified object matches some installed volume populator or dynamic + provisioner. + This field will replace the functionality of the dataSource field and as such + if both fields are non-empty, they must have the same value. For backwards + compatibility, when namespace isn't specified in dataSourceRef, + both fields (dataSource and dataSourceRef) will be set to the same + value automatically if one of them is empty and the other is non-empty. + When namespace is specified in dataSourceRef, + dataSource isn't set to the same value and must be empty. + There are three important differences between dataSource and dataSourceRef: + * While dataSource only allows two specific types of objects, dataSourceRef + allows any non-core object, as well as PersistentVolumeClaim objects. + * While dataSource ignores disallowed values (dropping them), dataSourceRef + preserves all values, and generates an error if a disallowed value is + specified. + * While dataSource only allows local objects, dataSourceRef allows objects + in any namespaces. + (Beta) Using this field requires the AnyVolumeDataSource feature gate to be enabled. + (Alpha) Using the namespace field of dataSourceRef requires the CrossNamespaceVolumeDataSource feature gate to be enabled. + properties: + apiGroup: + description: |- + APIGroup is the group for the resource being referenced. + If APIGroup is not specified, the specified Kind must be in the core API group. + For any other third-party types, APIGroup is required. + type: string + kind: + description: Kind is the + type of resource being + referenced + type: string + name: + description: Name is the + name of resource being + referenced + type: string + namespace: + description: |- + Namespace is the namespace of resource being referenced + Note that when a namespace is specified, a gateway.networking.k8s.io/ReferenceGrant object is required in the referent namespace to allow that namespace's owner to accept the reference. See the ReferenceGrant documentation for details. + (Alpha) This field requires the CrossNamespaceVolumeDataSource feature gate to be enabled. + type: string + required: + - kind + - name + type: object + resources: + description: |- + resources represents the minimum resources the volume should have. + If RecoverVolumeExpansionFailure feature is enabled users are allowed to specify resource requirements + that are lower than previous value but must still be higher than capacity recorded in the + status field of the claim. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + selector: + description: selector is a label + query over volumes to consider + for binding. + properties: + matchExpressions: + description: matchExpressions + is a list of label selector + requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is + the label key that + the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + storageClassName: + description: |- + storageClassName is the name of the StorageClass required by the claim. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1 + type: string + volumeAttributesClassName: + description: |- + volumeAttributesClassName may be used to set the VolumeAttributesClass used by this claim. + If specified, the CSI driver will create or update the volume with the attributes defined + in the corresponding VolumeAttributesClass. This has a different purpose than storageClassName, + it can be changed after the claim is created. An empty string value means that no VolumeAttributesClass + will be applied to the claim but it's not allowed to reset this field to empty string once it is set. + If unspecified and the PersistentVolumeClaim is unbound, the default VolumeAttributesClass + will be set by the persistentvolume controller if it exists. + If the resource referred to by volumeAttributesClass does not exist, this PersistentVolumeClaim will be + set to a Pending state, as reflected by the modifyVolumeStatus field, until such as a resource + exists. + More info: https://kubernetes.io/docs/concepts/storage/volume-attributes-classes/ + (Beta) Using this field requires the VolumeAttributesClass feature gate to be enabled (off by default). + type: string + volumeMode: + description: |- + volumeMode defines what type of volume is required by the claim. + Value of Filesystem is implied when not included in claim spec. + type: string + volumeName: + description: volumeName is the + binding reference to the PersistentVolume + backing this claim. + type: string + type: object + required: + - spec + type: object + type: object + fc: + description: fc represents a Fibre Channel + resource that is attached to a kubelet's + host machine and then exposed to the pod. + properties: + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + lun: + description: 'lun is Optional: FC target + lun number' + format: int32 + type: integer + readOnly: + description: |- + readOnly is Optional: Defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + targetWWNs: + description: 'targetWWNs is Optional: + FC target worldwide names (WWNs)' + items: + type: string + type: array + x-kubernetes-list-type: atomic + wwids: + description: |- + wwids Optional: FC volume world wide identifiers (wwids) + Either wwids or combination of targetWWNs and lun must be set, but not both simultaneously. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + flexVolume: + description: |- + flexVolume represents a generic volume resource that is + provisioned/attached using an exec based plugin. + properties: + driver: + description: driver is the name of the + driver to use for this volume. + type: string + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". The default filesystem depends on FlexVolume script. + type: string + options: + additionalProperties: + type: string + description: 'options is Optional: this + field holds extra command options + if any.' + type: object + readOnly: + description: |- + readOnly is Optional: defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + secretRef: + description: |- + secretRef is Optional: secretRef is reference to the secret object containing + sensitive information to pass to the plugin scripts. This may be + empty if no secret object is specified. If the secret object + contains more than one secret, all secrets are passed to the plugin + scripts. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + required: + - driver + type: object + flocker: + description: flocker represents a Flocker + volume attached to a kubelet's host machine. + This depends on the Flocker control service + being running + properties: + datasetName: + description: |- + datasetName is Name of the dataset stored as metadata -> name on the dataset for Flocker + should be considered as deprecated + type: string + datasetUUID: + description: datasetUUID is the UUID + of the dataset. This is unique identifier + of a Flocker dataset + type: string + type: object + gcePersistentDisk: + description: |- + gcePersistentDisk represents a GCE Disk resource that is attached to a + kubelet's host machine and then exposed to the pod. + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + properties: + fsType: + description: |- + fsType is filesystem type of the volume that you want to mount. + Tip: Ensure that the filesystem type is supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + type: string + partition: + description: |- + partition is the partition in the volume that you want to mount. + If omitted, the default is to mount by volume name. + Examples: For volume /dev/sda1, you specify the partition as "1". + Similarly, the volume partition for /dev/sda is "0" (or you can leave the property empty). + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + format: int32 + type: integer + pdName: + description: |- + pdName is unique name of the PD resource in GCE. Used to identify the disk in GCE. + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + type: string + readOnly: + description: |- + readOnly here will force the ReadOnly setting in VolumeMounts. + Defaults to false. + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + type: boolean + required: + - pdName + type: object + gitRepo: + description: |- + gitRepo represents a git repository at a particular revision. + DEPRECATED: GitRepo is deprecated. To provision a container with a git repo, mount an + EmptyDir into an InitContainer that clones the repo using git, then mount the EmptyDir + into the Pod's container. + properties: + directory: + description: |- + directory is the target directory name. + Must not contain or start with '..'. If '.' is supplied, the volume directory will be the + git repository. Otherwise, if specified, the volume will contain the git repository in + the subdirectory with the given name. + type: string + repository: + description: repository is the URL + type: string + revision: + description: revision is the commit + hash for the specified revision. + type: string + required: + - repository + type: object + glusterfs: + description: |- + glusterfs represents a Glusterfs mount on the host that shares a pod's lifetime. + More info: https://examples.k8s.io/volumes/glusterfs/README.md + properties: + endpoints: + description: |- + endpoints is the endpoint name that details Glusterfs topology. + More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod + type: string + path: + description: |- + path is the Glusterfs volume path. + More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod + type: string + readOnly: + description: |- + readOnly here will force the Glusterfs volume to be mounted with read-only permissions. + Defaults to false. + More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod + type: boolean + required: + - endpoints + - path + type: object + hostPath: + description: |- + hostPath represents a pre-existing file or directory on the host + machine that is directly exposed to the container. This is generally + used for system agents or other privileged things that are allowed + to see the host machine. Most containers will NOT need this. + More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath + properties: + path: + description: |- + path of the directory on the host. + If the path is a symlink, it will follow the link to the real path. + More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath + type: string + type: + description: |- + type for HostPath Volume + Defaults to "" + More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath + type: string + required: + - path + type: object + image: + description: |- + image represents an OCI object (a container image or artifact) pulled and mounted on the kubelet's host machine. + The volume is resolved at pod startup depending on which PullPolicy value is provided: + + - Always: the kubelet always attempts to pull the reference. Container creation will fail If the pull fails. + - Never: the kubelet never pulls the reference and only uses a local image or artifact. Container creation will fail if the reference isn't present. + - IfNotPresent: the kubelet pulls if the reference isn't already present on disk. Container creation will fail if the reference isn't present and the pull fails. + + The volume gets re-resolved if the pod gets deleted and recreated, which means that new remote content will become available on pod recreation. + A failure to resolve or pull the image during pod startup will block containers from starting and may add significant latency. Failures will be retried using normal volume backoff and will be reported on the pod reason and message. + The types of objects that may be mounted by this volume are defined by the container runtime implementation on a host machine and at minimum must include all valid types supported by the container image field. + The OCI object gets mounted in a single directory (spec.containers[*].volumeMounts.mountPath) by merging the manifest layers in the same way as for container images. + The volume will be mounted read-only (ro) and non-executable files (noexec). + Sub path mounts for containers are not supported (spec.containers[*].volumeMounts.subpath). + The field spec.securityContext.fsGroupChangePolicy has no effect on this volume type. + properties: + pullPolicy: + description: |- + Policy for pulling OCI objects. Possible values are: + Always: the kubelet always attempts to pull the reference. Container creation will fail If the pull fails. + Never: the kubelet never pulls the reference and only uses a local image or artifact. Container creation will fail if the reference isn't present. + IfNotPresent: the kubelet pulls if the reference isn't already present on disk. Container creation will fail if the reference isn't present and the pull fails. + Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. + type: string + reference: + description: |- + Required: Image or artifact reference to be used. + Behaves in the same way as pod.spec.containers[*].image. + Pull secrets will be assembled in the same way as for the container image by looking up node credentials, SA image pull secrets, and pod spec image pull secrets. + More info: https://kubernetes.io/docs/concepts/containers/images + This field is optional to allow higher level config management to default or override + container images in workload controllers like Deployments and StatefulSets. + type: string + type: object + iscsi: + description: |- + iscsi represents an ISCSI Disk resource that is attached to a + kubelet's host machine and then exposed to the pod. + More info: https://examples.k8s.io/volumes/iscsi/README.md + properties: + chapAuthDiscovery: + description: chapAuthDiscovery defines + whether support iSCSI Discovery CHAP + authentication + type: boolean + chapAuthSession: + description: chapAuthSession defines + whether support iSCSI Session CHAP + authentication + type: boolean + fsType: + description: |- + fsType is the filesystem type of the volume that you want to mount. + Tip: Ensure that the filesystem type is supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://kubernetes.io/docs/concepts/storage/volumes#iscsi + type: string + initiatorName: + description: |- + initiatorName is the custom iSCSI Initiator Name. + If initiatorName is specified with iscsiInterface simultaneously, new iSCSI interface + : will be created for the connection. + type: string + iqn: + description: iqn is the target iSCSI + Qualified Name. + type: string + iscsiInterface: + default: default + description: |- + iscsiInterface is the interface Name that uses an iSCSI transport. + Defaults to 'default' (tcp). + type: string + lun: + description: lun represents iSCSI Target + Lun number. + format: int32 + type: integer + portals: + description: |- + portals is the iSCSI Target Portal List. The portal is either an IP or ip_addr:port if the port + is other than default (typically TCP ports 860 and 3260). + items: + type: string + type: array + x-kubernetes-list-type: atomic + readOnly: + description: |- + readOnly here will force the ReadOnly setting in VolumeMounts. + Defaults to false. + type: boolean + secretRef: + description: secretRef is the CHAP Secret + for iSCSI target and initiator authentication + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + targetPortal: + description: |- + targetPortal is iSCSI Target Portal. The Portal is either an IP or ip_addr:port if the port + is other than default (typically TCP ports 860 and 3260). + type: string + required: + - iqn + - lun + - targetPortal + type: object + name: + description: |- + name of the volume. + Must be a DNS_LABEL and unique within the pod. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + nfs: + description: |- + nfs represents an NFS mount on the host that shares a pod's lifetime + More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs + properties: + path: + description: |- + path that is exported by the NFS server. + More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs + type: string + readOnly: + description: |- + readOnly here will force the NFS export to be mounted with read-only permissions. + Defaults to false. + More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs + type: boolean + server: + description: |- + server is the hostname or IP address of the NFS server. + More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs + type: string + required: + - path + - server + type: object + persistentVolumeClaim: + description: |- + persistentVolumeClaimVolumeSource represents a reference to a + PersistentVolumeClaim in the same namespace. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims + properties: + claimName: + description: |- + claimName is the name of a PersistentVolumeClaim in the same namespace as the pod using this volume. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims + type: string + readOnly: + description: |- + readOnly Will force the ReadOnly setting in VolumeMounts. + Default false. + type: boolean + required: + - claimName + type: object + photonPersistentDisk: + description: photonPersistentDisk represents + a PhotonController persistent disk attached + and mounted on kubelets host machine + properties: + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + pdID: + description: pdID is the ID that identifies + Photon Controller persistent disk + type: string + required: + - pdID + type: object + portworxVolume: + description: portworxVolume represents a + portworx volume attached and mounted on + kubelets host machine + properties: + fsType: + description: |- + fSType represents the filesystem type to mount + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs". Implicitly inferred to be "ext4" if unspecified. + type: string + readOnly: + description: |- + readOnly defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + volumeID: + description: volumeID uniquely identifies + a Portworx volume + type: string + required: + - volumeID + type: object + projected: + description: projected items for all in + one resources secrets, configmaps, and + downward API + properties: + defaultMode: + description: |- + defaultMode are the mode bits used to set permissions on created files by default. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + Directories within the path are not affected by this setting. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + sources: + description: |- + sources is the list of volume projections. Each entry in this list + handles one source. + items: + description: |- + Projection that may be projected along with other supported volume types. + Exactly one of these fields must be set. + properties: + clusterTrustBundle: + description: |- + ClusterTrustBundle allows a pod to access the `.spec.trustBundle` field + of ClusterTrustBundle objects in an auto-updating file. + + Alpha, gated by the ClusterTrustBundleProjection feature gate. + + ClusterTrustBundle objects can either be selected by name, or by the + combination of signer name and a label selector. + + Kubelet performs aggressive normalization of the PEM contents written + into the pod filesystem. Esoteric PEM features such as inter-block + comments and block headers are stripped. Certificates are deduplicated. + The ordering of certificates within the file is arbitrary, and Kubelet + may change the order over time. + properties: + labelSelector: + description: |- + Select all ClusterTrustBundles that match this label selector. Only has + effect if signerName is set. Mutually-exclusive with name. If unset, + interpreted as "match nothing". If set but empty, interpreted as "match + everything". + properties: + matchExpressions: + description: matchExpressions + is a list of label selector + requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key + is the label key + that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + name: + description: |- + Select a single ClusterTrustBundle by object name. Mutually-exclusive + with signerName and labelSelector. + type: string + optional: + description: |- + If true, don't block pod startup if the referenced ClusterTrustBundle(s) + aren't available. If using name, then the named ClusterTrustBundle is + allowed not to exist. If using signerName, then the combination of + signerName and labelSelector is allowed to match zero + ClusterTrustBundles. + type: boolean + path: + description: Relative path + from the volume root to + write the bundle. + type: string + signerName: + description: |- + Select all ClusterTrustBundles that match this signer name. + Mutually-exclusive with name. The contents of all selected + ClusterTrustBundles will be unified and deduplicated. + type: string + required: + - path + type: object + configMap: + description: configMap information + about the configMap data to + project + properties: + items: + description: |- + items if unspecified, each key-value pair in the Data field of the referenced + ConfigMap will be projected into the volume as a file whose name is the + key and content is the value. If specified, the listed keys will be + projected into the specified paths, and unlisted keys will not be + present. If a key is specified which is not present in the ConfigMap, + the volume setup will error unless it is marked optional. Paths must be + relative and may not contain the '..' path or start with '..'. + items: + description: Maps a string + key to a path within a + volume. + properties: + key: + description: key is + the key to project. + type: string + mode: + description: |- + mode is Optional: mode bits used to set permissions on this file. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: |- + path is the relative path of the file to map the key to. + May not be an absolute path. + May not contain the path element '..'. + May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: optional specify + whether the ConfigMap or + its keys must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + downwardAPI: + description: downwardAPI information + about the downwardAPI data to + project + properties: + items: + description: Items is a list + of DownwardAPIVolume file + items: + description: DownwardAPIVolumeFile + represents information + to create the file containing + the pod field + properties: + fieldRef: + description: 'Required: + Selects a field of + the pod: only annotations, + labels, name, namespace + and uid are supported.' + properties: + apiVersion: + description: Version + of the schema + the FieldPath + is written in + terms of, defaults + to "v1". + type: string + fieldPath: + description: Path + of the field to + select in the + specified API + version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + mode: + description: |- + Optional: mode bits used to set permissions on this file, must be an octal value + between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: 'Required: + Path is the relative + path name of the file + to be created. Must + not be absolute or + contain the ''..'' + path. Must be utf-8 + encoded. The first + item of the relative + path must not start + with ''..''' + type: string + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, requests.cpu and requests.memory) are currently supported. + properties: + containerName: + description: 'Container + name: required + for volumes, optional + for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies + the output format + of the exposed + resources, defaults + to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: + resource to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + required: + - path + type: object + type: array + x-kubernetes-list-type: atomic + type: object + secret: + description: secret information + about the secret data to project + properties: + items: + description: |- + items if unspecified, each key-value pair in the Data field of the referenced + Secret will be projected into the volume as a file whose name is the + key and content is the value. If specified, the listed keys will be + projected into the specified paths, and unlisted keys will not be + present. If a key is specified which is not present in the Secret, + the volume setup will error unless it is marked optional. Paths must be + relative and may not contain the '..' path or start with '..'. + items: + description: Maps a string + key to a path within a + volume. + properties: + key: + description: key is + the key to project. + type: string + mode: + description: |- + mode is Optional: mode bits used to set permissions on this file. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: |- + path is the relative path of the file to map the key to. + May not be an absolute path. + May not contain the path element '..'. + May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: optional field + specify whether the Secret + or its key must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + serviceAccountToken: + description: serviceAccountToken + is information about the serviceAccountToken + data to project + properties: + audience: + description: |- + audience is the intended audience of the token. A recipient of a token + must identify itself with an identifier specified in the audience of the + token, and otherwise should reject the token. The audience defaults to the + identifier of the apiserver. + type: string + expirationSeconds: + description: |- + expirationSeconds is the requested duration of validity of the service + account token. As the token approaches expiration, the kubelet volume + plugin will proactively rotate the service account token. The kubelet will + start trying to rotate the token if the token is older than 80 percent of + its time to live or if the token is older than 24 hours.Defaults to 1 hour + and must be at least 10 minutes. + format: int64 + type: integer + path: + description: |- + path is the path relative to the mount point of the file to project the + token into. + type: string + required: + - path + type: object + type: object + type: array + x-kubernetes-list-type: atomic + type: object + quobyte: + description: quobyte represents a Quobyte + mount on the host that shares a pod's + lifetime + properties: + group: + description: |- + group to map volume access to + Default is no group + type: string + readOnly: + description: |- + readOnly here will force the Quobyte volume to be mounted with read-only permissions. + Defaults to false. + type: boolean + registry: + description: |- + registry represents a single or multiple Quobyte Registry services + specified as a string as host:port pair (multiple entries are separated with commas) + which acts as the central registry for volumes + type: string + tenant: + description: |- + tenant owning the given Quobyte volume in the Backend + Used with dynamically provisioned Quobyte volumes, value is set by the plugin + type: string + user: + description: |- + user to map volume access to + Defaults to serivceaccount user + type: string + volume: + description: volume is a string that + references an already created Quobyte + volume by name. + type: string + required: + - registry + - volume + type: object + rbd: + description: |- + rbd represents a Rados Block Device mount on the host that shares a pod's lifetime. + More info: https://examples.k8s.io/volumes/rbd/README.md + properties: + fsType: + description: |- + fsType is the filesystem type of the volume that you want to mount. + Tip: Ensure that the filesystem type is supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://kubernetes.io/docs/concepts/storage/volumes#rbd + type: string + image: + description: |- + image is the rados image name. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: string + keyring: + default: /etc/ceph/keyring + description: |- + keyring is the path to key ring for RBDUser. + Default is /etc/ceph/keyring. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: string + monitors: + description: |- + monitors is a collection of Ceph monitors. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + items: + type: string + type: array + x-kubernetes-list-type: atomic + pool: + default: rbd + description: |- + pool is the rados pool name. + Default is rbd. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: string + readOnly: + description: |- + readOnly here will force the ReadOnly setting in VolumeMounts. + Defaults to false. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: boolean + secretRef: + description: |- + secretRef is name of the authentication secret for RBDUser. If provided + overrides keyring. + Default is nil. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + user: + default: admin + description: |- + user is the rados user name. + Default is admin. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: string + required: + - image + - monitors + type: object + scaleIO: + description: scaleIO represents a ScaleIO + persistent volume attached and mounted + on Kubernetes nodes. + properties: + fsType: + default: xfs + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". + Default is "xfs". + type: string + gateway: + description: gateway is the host address + of the ScaleIO API Gateway. + type: string + protectionDomain: + description: protectionDomain is the + name of the ScaleIO Protection Domain + for the configured storage. + type: string + readOnly: + description: |- + readOnly Defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + secretRef: + description: |- + secretRef references to the secret for ScaleIO user and other + sensitive information. If this is not provided, Login operation will fail. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + sslEnabled: + description: sslEnabled Flag enable/disable + SSL communication with Gateway, default + false + type: boolean + storageMode: + default: ThinProvisioned + description: |- + storageMode indicates whether the storage for a volume should be ThickProvisioned or ThinProvisioned. + Default is ThinProvisioned. + type: string + storagePool: + description: storagePool is the ScaleIO + Storage Pool associated with the protection + domain. + type: string + system: + description: system is the name of the + storage system as configured in ScaleIO. + type: string + volumeName: + description: |- + volumeName is the name of a volume already created in the ScaleIO system + that is associated with this volume source. + type: string + required: + - gateway + - secretRef + - system + type: object + secret: + description: |- + secret represents a secret that should populate this volume. + More info: https://kubernetes.io/docs/concepts/storage/volumes#secret + properties: + defaultMode: + description: |- + defaultMode is Optional: mode bits used to set permissions on created files by default. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values + for mode bits. Defaults to 0644. + Directories within the path are not affected by this setting. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + items: + description: |- + items If unspecified, each key-value pair in the Data field of the referenced + Secret will be projected into the volume as a file whose name is the + key and content is the value. If specified, the listed keys will be + projected into the specified paths, and unlisted keys will not be + present. If a key is specified which is not present in the Secret, + the volume setup will error unless it is marked optional. Paths must be + relative and may not contain the '..' path or start with '..'. + items: + description: Maps a string key to + a path within a volume. + properties: + key: + description: key is the key to + project. + type: string + mode: + description: |- + mode is Optional: mode bits used to set permissions on this file. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: |- + path is the relative path of the file to map the key to. + May not be an absolute path. + May not contain the path element '..'. + May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + optional: + description: optional field specify + whether the Secret or its keys must + be defined + type: boolean + secretName: + description: |- + secretName is the name of the secret in the pod's namespace to use. + More info: https://kubernetes.io/docs/concepts/storage/volumes#secret + type: string + type: object + storageos: + description: storageOS represents a StorageOS + volume attached and mounted on Kubernetes + nodes. + properties: + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + readOnly: + description: |- + readOnly defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + secretRef: + description: |- + secretRef specifies the secret to use for obtaining the StorageOS API + credentials. If not specified, default values will be attempted. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + volumeName: + description: |- + volumeName is the human-readable name of the StorageOS volume. Volume + names are only unique within a namespace. + type: string + volumeNamespace: + description: |- + volumeNamespace specifies the scope of the volume within StorageOS. If no + namespace is specified then the Pod's namespace will be used. This allows the + Kubernetes name scoping to be mirrored within StorageOS for tighter integration. + Set VolumeName to any name to override the default behaviour. + Set to "default" if you are not using namespaces within StorageOS. + Namespaces that do not pre-exist within StorageOS will be created. + type: string + type: object + vsphereVolume: + description: vsphereVolume represents a + vSphere volume attached and mounted on + kubelets host machine + properties: + fsType: + description: |- + fsType is filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + storagePolicyID: + description: storagePolicyID is the + storage Policy Based Management (SPBM) + profile ID associated with the StoragePolicyName. + type: string + storagePolicyName: + description: storagePolicyName is the + storage Policy Based Management (SPBM) + profile name. + type: string + volumePath: + description: volumePath is the path + that identifies vSphere volume vmdk + type: string + required: + - volumePath + type: object + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + required: + - containers + type: object + type: object + required: + - rayStartParams + - template + type: object + headServiceAnnotations: + additionalProperties: + type: string + type: object + rayVersion: + description: RayVersion is used to determine the command for + the Kubernetes Job managed by RayJob + type: string + suspend: + description: |- + Suspend indicates whether a RayCluster should be suspended. + A suspended RayCluster will have head pods and worker pods deleted. + type: boolean + workerGroupSpecs: + description: WorkerGroupSpecs are the specs for the worker + pods + items: + description: WorkerGroupSpec are the specs for the worker + pods + properties: + groupName: + description: we can have multiple worker groups, we + distinguish them by name + type: string + maxReplicas: + default: 2147483647 + description: MaxReplicas denotes the maximum number + of desired Pods for this worker group, and the default + value is maxInt32. + format: int32 + type: integer + minReplicas: + default: 0 + description: MinReplicas denotes the minimum number + of desired Pods for this worker group. + format: int32 + type: integer + numOfHosts: + default: 1 + description: NumOfHosts denotes the number of hosts + to create per replica. The default value is 1. + format: int32 + type: integer + rayStartParams: + additionalProperties: + type: string + description: 'RayStartParams are the params of the start + command: address, object-store-memory, ...' + type: object + replicas: + default: 0 + description: Replicas is the number of desired Pods + for this worker group. See https://github.com/ray-project/kuberay/pull/1443 + for more details about the reason for making this + field optional. + format: int32 + type: integer + scaleStrategy: + description: ScaleStrategy defines which pods to remove + properties: + workersToDelete: + description: WorkersToDelete workers to be deleted + items: + type: string + type: array + type: object + template: + description: Template is a pod template for the worker + properties: + metadata: + description: |- + Standard object's metadata. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata + type: object + spec: + description: |- + Specification of the desired behavior of the pod. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status + properties: + activeDeadlineSeconds: + description: |- + Optional duration in seconds the pod may be active on the node relative to + StartTime before the system will actively try to mark it failed and kill associated containers. + Value must be a positive integer. + format: int64 + type: integer + affinity: + description: If specified, the pod's scheduling + constraints + properties: + nodeAffinity: + description: Describes node affinity scheduling + rules for the pod. + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node matches the corresponding matchExpressions; the + node(s) with the highest sum are the most preferred. + items: + description: |- + An empty preferred scheduling term matches all objects with implicit weight 0 + (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op). + properties: + preference: + description: A node selector term, + associated with the corresponding + weight. + properties: + matchExpressions: + description: A list of node + selector requirements by + node's labels. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label + key that the selector + applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + description: A list of node + selector requirements by + node's fields. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label + key that the selector + applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + weight: + description: Weight associated + with matching the corresponding + nodeSelectorTerm, in the range + 1-100. + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to an update), the system + may or may not try to eventually evict the pod from its node. + properties: + nodeSelectorTerms: + description: Required. A list of + node selector terms. The terms + are ORed. + items: + description: |- + A null or empty node selector term matches no objects. The requirements of + them are ANDed. + The TopologySelectorTerm type implements a subset of the NodeSelectorTerm. + properties: + matchExpressions: + description: A list of node + selector requirements by + node's labels. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label + key that the selector + applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + description: A list of node + selector requirements by + node's fields. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label + key that the selector + applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + type: array + x-kubernetes-list-type: atomic + required: + - nodeSelectorTerms + type: object + x-kubernetes-map-type: atomic + type: object + podAffinity: + description: Describes pod affinity scheduling + rules (e.g. co-locate this pod in the + same node, zone, etc. as some other pod(s)). + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the + node(s) with the highest sum are the most preferred. + items: + description: The weights of all of + the matched WeightedPodAffinityTerm + fields are added per-node to find + the most preferred node(s) + properties: + podAffinityTerm: + description: Required. A pod affinity + term, associated with the corresponding + weight. + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions + is a list of label selector + requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key + is the label key + that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions + is a list of label selector + requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key + is the label key + that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + weight: + description: |- + weight associated with matching the corresponding podAffinityTerm, + in the range 1-100. + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to a pod label update), the + system may or may not try to eventually evict the pod from its node. + When there are multiple elements, the lists of nodes corresponding to each + podAffinityTerm are intersected, i.e. all terms must be satisfied. + items: + description: |- + Defines a set of pods (namely those matching the labelSelector + relative to the given namespace(s)) that this pod should be + co-located (affinity) or not co-located (anti-affinity) with, + where co-located is defined as running on a node whose value of + the label with key matches that of any node on which + a pod of the set of pods is running + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions + is a list of label selector + requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is + the label key that + the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions + is a list of label selector + requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is + the label key that + the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + podAntiAffinity: + description: Describes pod anti-affinity + scheduling rules (e.g. avoid putting this + pod in the same node, zone, etc. as some + other pod(s)). + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the anti-affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling anti-affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the + node(s) with the highest sum are the most preferred. + items: + description: The weights of all of + the matched WeightedPodAffinityTerm + fields are added per-node to find + the most preferred node(s) + properties: + podAffinityTerm: + description: Required. A pod affinity + term, associated with the corresponding + weight. + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions + is a list of label selector + requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key + is the label key + that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions + is a list of label selector + requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key + is the label key + that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + weight: + description: |- + weight associated with matching the corresponding podAffinityTerm, + in the range 1-100. + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the anti-affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the anti-affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to a pod label update), the + system may or may not try to eventually evict the pod from its node. + When there are multiple elements, the lists of nodes corresponding to each + podAffinityTerm are intersected, i.e. all terms must be satisfied. + items: + description: |- + Defines a set of pods (namely those matching the labelSelector + relative to the given namespace(s)) that this pod should be + co-located (affinity) or not co-located (anti-affinity) with, + where co-located is defined as running on a node whose value of + the label with key matches that of any node on which + a pod of the set of pods is running + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions + is a list of label selector + requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is + the label key that + the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions + is a list of label selector + requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is + the label key that + the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + type: object + automountServiceAccountToken: + description: AutomountServiceAccountToken indicates + whether a service account token should be + automatically mounted. + type: boolean + containers: + description: |- + List of containers belonging to the pod. + Containers cannot currently be added or removed. + There must be at least one container in a Pod. + Cannot be updated. + items: + description: A single application container + that you want to run within a pod. + properties: + args: + description: |- + Arguments to the entrypoint. + The container image's CMD is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell + items: + type: string + type: array + x-kubernetes-list-type: atomic + command: + description: |- + Entrypoint array. Not executed within a shell. + The container image's ENTRYPOINT is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell + items: + type: string + type: array + x-kubernetes-list-type: atomic + env: + description: |- + List of environment variables to set in the container. + Cannot be updated. + items: + description: EnvVar represents an environment + variable present in a Container. + properties: + name: + description: Name of the environment + variable. Must be a C_IDENTIFIER. + type: string + value: + description: |- + Variable references $(VAR_NAME) are expanded + using the previously defined environment variables in the container and + any service environment variables. If a variable cannot be resolved, + the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. + "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless of whether the variable + exists or not. + Defaults to "". + type: string + valueFrom: + description: Source for the environment + variable's value. Cannot be used + if value is not empty. + properties: + configMapKeyRef: + description: Selects a key of + a ConfigMap. + properties: + key: + description: The key to + select. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether + the ConfigMap or its key + must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + description: |- + Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, + spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. + properties: + apiVersion: + description: Version of + the schema the FieldPath + is written in terms of, + defaults to "v1". + type: string + fieldPath: + description: Path of the + field to select in the + specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. + properties: + containerName: + description: 'Container + name: required for volumes, + optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the + output format of the exposed + resources, defaults to + "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: + resource to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + description: Selects a key of + a secret in the pod's namespace + properties: + key: + description: The key of + the secret to select from. Must + be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether + the Secret or its key + must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + envFrom: + description: |- + List of sources to populate environment variables in the container. + The keys defined within a source must be a C_IDENTIFIER. All invalid keys + will be reported as an event when the container is starting. When a key exists in multiple + sources, the value associated with the last source will take precedence. + Values defined by an Env with a duplicate key will take precedence. + Cannot be updated. + items: + description: EnvFromSource represents + the source of a set of ConfigMaps + properties: + configMapRef: + description: The ConfigMap to select + from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether + the ConfigMap must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + description: An optional identifier + to prepend to each key in the + ConfigMap. Must be a C_IDENTIFIER. + type: string + secretRef: + description: The Secret to select + from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether + the Secret must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array + x-kubernetes-list-type: atomic + image: + description: |- + Container image name. + More info: https://kubernetes.io/docs/concepts/containers/images + This field is optional to allow higher level config management to default or override + container images in workload controllers like Deployments and StatefulSets. + type: string + imagePullPolicy: + description: |- + Image pull policy. + One of Always, Never, IfNotPresent. + Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/containers/images#updating-images + type: string + lifecycle: + description: |- + Actions that the management system should take in response to container lifecycle events. + Cannot be updated. + properties: + postStart: + description: |- + PostStart is called immediately after a container is created. If the handler fails, + the container is terminated and restarted according to its restart policy. + Other management of the container blocks until the hook completes. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies the + action to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies + the http request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers + to set in the request. HTTP + allows repeated headers. + items: + description: HTTPHeader + describes a custom header + to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header + field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access + on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents + the duration that the container + should sleep before being terminated. + properties: + seconds: + description: Seconds is the + number of seconds to sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for the backward compatibility. There are no validation of this field and + lifecycle hooks will fail in runtime when tcp handler is specified. + properties: + host: + description: 'Optional: Host + name to connect to, defaults + to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + preStop: + description: |- + PreStop is called immediately before a container is terminated due to an + API request or management event such as liveness/startup probe failure, + preemption, resource contention, etc. The handler is not called if the + container crashes or exits. The Pod's termination grace period countdown begins before the + PreStop hook is executed. Regardless of the outcome of the handler, the + container will eventually terminate within the Pod's termination grace + period (unless delayed by finalizers). Other management of the container blocks until the hook completes + or until the termination grace period is reached. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies the + action to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies + the http request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers + to set in the request. HTTP + allows repeated headers. + items: + description: HTTPHeader + describes a custom header + to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header + field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access + on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents + the duration that the container + should sleep before being terminated. + properties: + seconds: + description: Seconds is the + number of seconds to sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for the backward compatibility. There are no validation of this field and + lifecycle hooks will fail in runtime when tcp handler is specified. + properties: + host: + description: 'Optional: Host + name to connect to, defaults + to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + type: object + livenessProbe: + description: |- + Periodic probe of container liveness. + Container will be restarted if the probe fails. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies the action + to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action + involving a GRPC port. + properties: + port: + description: Port number of the + gRPC service. Number must be + in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the + http request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to + set in the request. HTTP allows + repeated headers. + items: + description: HTTPHeader describes + a custom header to be used + in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header + field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on + the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an + action involving a TCP port. + properties: + host: + description: 'Optional: Host name + to connect to, defaults to the + pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + name: + description: |- + Name of the container specified as a DNS_LABEL. + Each container in a pod must have a unique name (DNS_LABEL). + Cannot be updated. + type: string + ports: + description: |- + List of ports to expose from the container. Not specifying a port here + DOES NOT prevent that port from being exposed. Any port which is + listening on the default "0.0.0.0" address inside a container will be + accessible from the network. + Modifying this array with strategic merge patch may corrupt the data. + For more information See https://github.com/kubernetes/kubernetes/issues/108255. + Cannot be updated. + items: + description: ContainerPort represents + a network port in a single container. + properties: + containerPort: + description: |- + Number of port to expose on the pod's IP address. + This must be a valid port number, 0 < x < 65536. + format: int32 + type: integer + hostIP: + description: What host IP to bind + the external port to. + type: string + hostPort: + description: |- + Number of port to expose on the host. + If specified, this must be a valid port number, 0 < x < 65536. + If HostNetwork is specified, this must match ContainerPort. + Most containers do not need this. + format: int32 + type: integer + name: + description: |- + If specified, this must be an IANA_SVC_NAME and unique within the pod. Each + named port in a pod must have a unique name. Name for the port that can be + referred to by services. + type: string + protocol: + default: TCP + description: |- + Protocol for port. Must be UDP, TCP, or SCTP. + Defaults to "TCP". + type: string + required: + - containerPort + type: object + type: array + x-kubernetes-list-map-keys: + - containerPort + - protocol + x-kubernetes-list-type: map + readinessProbe: + description: |- + Periodic probe of container service readiness. + Container will be removed from service endpoints if the probe fails. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies the action + to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action + involving a GRPC port. + properties: + port: + description: Port number of the + gRPC service. Number must be + in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the + http request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to + set in the request. HTTP allows + repeated headers. + items: + description: HTTPHeader describes + a custom header to be used + in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header + field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on + the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an + action involving a TCP port. + properties: + host: + description: 'Optional: Host name + to connect to, defaults to the + pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + resizePolicy: + description: Resources resize policy for + the container. + items: + description: ContainerResizePolicy represents + resource resize policy for the container. + properties: + resourceName: + description: |- + Name of the resource to which this resource resize policy applies. + Supported values: cpu, memory. + type: string + restartPolicy: + description: |- + Restart policy to apply when specified resource is resized. + If not specified, it defaults to NotRequired. + type: string + required: + - resourceName + - restartPolicy + type: object + type: array + x-kubernetes-list-type: atomic + resources: + description: |- + Compute Resources required by this container. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references + one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + restartPolicy: + description: |- + RestartPolicy defines the restart behavior of individual containers in a pod. + This field may only be set for init containers, and the only allowed value is "Always". + For non-init containers or when this field is not specified, + the restart behavior is defined by the Pod's restart policy and the container type. + Setting the RestartPolicy as "Always" for the init container will have the following effect: + this init container will be continually restarted on + exit until all regular containers have terminated. Once all regular + containers have completed, all init containers with restartPolicy "Always" + will be shut down. This lifecycle differs from normal init containers and + is often referred to as a "sidecar" container. Although this init + container still starts in the init container sequence, it does not wait + for the container to complete before proceeding to the next init + container. Instead, the next init container starts immediately after this + init container is started, or after any startupProbe has successfully + completed. + type: string + securityContext: + description: |- + SecurityContext defines the security options the container should be run with. + If set, the fields of SecurityContext override the equivalent fields of PodSecurityContext. + More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ + properties: + allowPrivilegeEscalation: + description: |- + AllowPrivilegeEscalation controls whether a process can gain more + privileges than its parent process. This bool directly controls if + the no_new_privs flag will be set on the container process. + AllowPrivilegeEscalation is true always when the container is: + 1) run as Privileged + 2) has CAP_SYS_ADMIN + Note that this field cannot be set when spec.os.name is windows. + type: boolean + appArmorProfile: + description: |- + appArmorProfile is the AppArmor options to use by this container. If set, this profile + overrides the pod's appArmorProfile. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile loaded on the node that should be used. + The profile must be preconfigured on the node to work. + Must match the loaded name of the profile. + Must be set if and only if type is "Localhost". + type: string + type: + description: |- + type indicates which kind of AppArmor profile will be applied. + Valid options are: + Localhost - a profile pre-loaded on the node. + RuntimeDefault - the container runtime's default profile. + Unconfined - no AppArmor enforcement. + type: string + required: + - type + type: object + capabilities: + description: |- + The capabilities to add/drop when running containers. + Defaults to the default set of capabilities granted by the container runtime. + Note that this field cannot be set when spec.os.name is windows. + properties: + add: + description: Added capabilities + items: + description: Capability represent + POSIX capabilities type + type: string + type: array + x-kubernetes-list-type: atomic + drop: + description: Removed capabilities + items: + description: Capability represent + POSIX capabilities type + type: string + type: array + x-kubernetes-list-type: atomic + type: object + privileged: + description: |- + Run container in privileged mode. + Processes in privileged containers are essentially equivalent to root on the host. + Defaults to false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + procMount: + description: |- + procMount denotes the type of proc mount to use for the containers. + The default value is Default which uses the container runtime defaults for + readonly paths and masked paths. + This requires the ProcMountType feature flag to be enabled. + Note that this field cannot be set when spec.os.name is windows. + type: string + readOnlyRootFilesystem: + description: |- + Whether this container has a read-only root filesystem. + Default is false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + runAsGroup: + description: |- + The GID to run the entrypoint of the container process. + Uses runtime default if unset. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + runAsNonRoot: + description: |- + Indicates that the container must run as a non-root user. + If true, the Kubelet will validate the image at runtime to ensure that it + does not run as UID 0 (root) and fail to start the container if it does. + If unset or false, no such validation will be performed. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: |- + The UID to run the entrypoint of the container process. + Defaults to user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + seLinuxOptions: + description: |- + The SELinux context to be applied to the container. + If unspecified, the container runtime will allocate a random SELinux context for each + container. May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + properties: + level: + description: Level is SELinux + level label that applies to + the container. + type: string + role: + description: Role is a SELinux + role label that applies to the + container. + type: string + type: + description: Type is a SELinux + type label that applies to the + container. + type: string + user: + description: User is a SELinux + user label that applies to the + container. + type: string + type: object + seccompProfile: + description: |- + The seccomp options to use by this container. If seccomp options are + provided at both the pod & container level, the container options + override the pod options. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile defined in a file on the node should be used. + The profile must be preconfigured on the node to work. + Must be a descending path, relative to the kubelet's configured seccomp profile location. + Must be set if type is "Localhost". Must NOT be set for any other type. + type: string + type: + description: |- + type indicates which kind of seccomp profile will be applied. + Valid options are: + + Localhost - a profile defined in a file on the node should be used. + RuntimeDefault - the container runtime default profile should be used. + Unconfined - no profile should be applied. + type: string + required: + - type + type: object + windowsOptions: + description: |- + The Windows specific settings applied to all containers. + If unspecified, the options from the PodSecurityContext will be used. + If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is linux. + properties: + gmsaCredentialSpec: + description: |- + GMSACredentialSpec is where the GMSA admission webhook + (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the + GMSA credential spec named by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName + is the name of the GMSA credential + spec to use. + type: string + hostProcess: + description: |- + HostProcess determines if a container should be run as a 'Host Process' container. + All of a Pod's containers must have the same effective HostProcess value + (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). + In addition, if HostProcess is true then HostNetwork must also be set to true. + type: boolean + runAsUserName: + description: |- + The UserName in Windows to run the entrypoint of the container process. + Defaults to the user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: string + type: object + type: object + startupProbe: + description: |- + StartupProbe indicates that the Pod has successfully initialized. + If specified, no other probes are executed until this completes successfully. + If this probe fails, the Pod will be restarted, just as if the livenessProbe failed. + This can be used to provide different probe parameters at the beginning of a Pod's lifecycle, + when it might take a long time to load data or warm a cache, than during steady-state operation. + This cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies the action + to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action + involving a GRPC port. + properties: + port: + description: Port number of the + gRPC service. Number must be + in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the + http request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to + set in the request. HTTP allows + repeated headers. + items: + description: HTTPHeader describes + a custom header to be used + in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header + field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on + the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an + action involving a TCP port. + properties: + host: + description: 'Optional: Host name + to connect to, defaults to the + pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + stdin: + description: |- + Whether this container should allocate a buffer for stdin in the container runtime. If this + is not set, reads from stdin in the container will always result in EOF. + Default is false. + type: boolean + stdinOnce: + description: |- + Whether the container runtime should close the stdin channel after it has been opened by + a single attach. When stdin is true the stdin stream will remain open across multiple attach + sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the + first client attaches to stdin, and then remains open and accepts data until the client disconnects, + at which time stdin is closed and remains closed until the container is restarted. If this + flag is false, a container processes that reads from stdin will never receive an EOF. + Default is false + type: boolean + terminationMessagePath: + description: |- + Optional: Path at which the file to which the container's termination message + will be written is mounted into the container's filesystem. + Message written is intended to be brief final status, such as an assertion failure message. + Will be truncated by the node if greater than 4096 bytes. The total message length across + all containers will be limited to 12kb. + Defaults to /dev/termination-log. + Cannot be updated. + type: string + terminationMessagePolicy: + description: |- + Indicate how the termination message should be populated. File will use the contents of + terminationMessagePath to populate the container status message on both success and failure. + FallbackToLogsOnError will use the last chunk of container log output if the termination + message file is empty and the container exited with an error. + The log output is limited to 2048 bytes or 80 lines, whichever is smaller. + Defaults to File. + Cannot be updated. + type: string + tty: + description: |- + Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. + Default is false. + type: boolean + volumeDevices: + description: volumeDevices is the list + of block devices to be used by the container. + items: + description: volumeDevice describes + a mapping of a raw block device within + a container. + properties: + devicePath: + description: devicePath is the path + inside of the container that the + device will be mapped to. + type: string + name: + description: name must match the + name of a persistentVolumeClaim + in the pod + type: string + required: + - devicePath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - devicePath + x-kubernetes-list-type: map + volumeMounts: + description: |- + Pod volumes to mount into the container's filesystem. + Cannot be updated. + items: + description: VolumeMount describes a + mounting of a Volume within a container. + properties: + mountPath: + description: |- + Path within the container at which the volume should be mounted. Must + not contain ':'. + type: string + mountPropagation: + description: |- + mountPropagation determines how mounts are propagated from the host + to container and the other way around. + When not set, MountPropagationNone is used. + This field is beta in 1.10. + When RecursiveReadOnly is set to IfPossible or to Enabled, MountPropagation must be None or unspecified + (which defaults to None). + type: string + name: + description: This must match the + Name of a Volume. + type: string + readOnly: + description: |- + Mounted read-only if true, read-write otherwise (false or unspecified). + Defaults to false. + type: boolean + recursiveReadOnly: + description: |- + RecursiveReadOnly specifies whether read-only mounts should be handled + recursively. + + If ReadOnly is false, this field has no meaning and must be unspecified. + + If ReadOnly is true, and this field is set to Disabled, the mount is not made + recursively read-only. If this field is set to IfPossible, the mount is made + recursively read-only, if it is supported by the container runtime. If this + field is set to Enabled, the mount is made recursively read-only if it is + supported by the container runtime, otherwise the pod will not be started and + an error will be generated to indicate the reason. + + If this field is set to IfPossible or Enabled, MountPropagation must be set to + None (or be unspecified, which defaults to None). + + If this field is not specified, it is treated as an equivalent of Disabled. + type: string + subPath: + description: |- + Path within the volume from which the container's volume should be mounted. + Defaults to "" (volume's root). + type: string + subPathExpr: + description: |- + Expanded path within the volume from which the container's volume should be mounted. + Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. + Defaults to "" (volume's root). + SubPathExpr and SubPath are mutually exclusive. + type: string + required: + - mountPath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - mountPath + x-kubernetes-list-type: map + workingDir: + description: |- + Container's working directory. + If not specified, the container runtime's default will be used, which + might be configured in the container image. + Cannot be updated. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + dnsConfig: + description: |- + Specifies the DNS parameters of a pod. + Parameters specified here will be merged to the generated DNS + configuration based on DNSPolicy. + properties: + nameservers: + description: |- + A list of DNS name server IP addresses. + This will be appended to the base nameservers generated from DNSPolicy. + Duplicated nameservers will be removed. + items: + type: string + type: array + x-kubernetes-list-type: atomic + options: + description: |- + A list of DNS resolver options. + This will be merged with the base options generated from DNSPolicy. + Duplicated entries will be removed. Resolution options given in Options + will override those that appear in the base DNSPolicy. + items: + description: PodDNSConfigOption defines + DNS resolver options of a pod. + properties: + name: + description: Required. + type: string + value: + type: string + type: object + type: array + x-kubernetes-list-type: atomic + searches: + description: |- + A list of DNS search domains for host-name lookup. + This will be appended to the base search paths generated from DNSPolicy. + Duplicated search paths will be removed. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + dnsPolicy: + description: |- + Set DNS policy for the pod. + Defaults to "ClusterFirst". + Valid values are 'ClusterFirstWithHostNet', 'ClusterFirst', 'Default' or 'None'. + DNS parameters given in DNSConfig will be merged with the policy selected with DNSPolicy. + To have DNS options set along with hostNetwork, you have to specify DNS policy + explicitly to 'ClusterFirstWithHostNet'. + type: string + enableServiceLinks: + description: |- + EnableServiceLinks indicates whether information about services should be injected into pod's + environment variables, matching the syntax of Docker links. + Optional: Defaults to true. + type: boolean + ephemeralContainers: + description: |- + List of ephemeral containers run in this pod. Ephemeral containers may be run in an existing + pod to perform user-initiated actions such as debugging. This list cannot be specified when + creating a pod, and it cannot be modified by updating the pod spec. In order to add an + ephemeral container to an existing pod, use the pod's ephemeralcontainers subresource. + items: + description: |- + An EphemeralContainer is a temporary container that you may add to an existing Pod for + user-initiated activities such as debugging. Ephemeral containers have no resource or + scheduling guarantees, and they will not be restarted when they exit or when a Pod is + removed or restarted. The kubelet may evict a Pod if an ephemeral container causes the + Pod to exceed its resource allocation. + + To add an ephemeral container, use the ephemeralcontainers subresource of an existing + Pod. Ephemeral containers may not be removed or restarted. + properties: + args: + description: |- + Arguments to the entrypoint. + The image's CMD is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell + items: + type: string + type: array + x-kubernetes-list-type: atomic + command: + description: |- + Entrypoint array. Not executed within a shell. + The image's ENTRYPOINT is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell + items: + type: string + type: array + x-kubernetes-list-type: atomic + env: + description: |- + List of environment variables to set in the container. + Cannot be updated. + items: + description: EnvVar represents an environment + variable present in a Container. + properties: + name: + description: Name of the environment + variable. Must be a C_IDENTIFIER. + type: string + value: + description: |- + Variable references $(VAR_NAME) are expanded + using the previously defined environment variables in the container and + any service environment variables. If a variable cannot be resolved, + the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. + "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless of whether the variable + exists or not. + Defaults to "". + type: string + valueFrom: + description: Source for the environment + variable's value. Cannot be used + if value is not empty. + properties: + configMapKeyRef: + description: Selects a key of + a ConfigMap. + properties: + key: + description: The key to + select. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether + the ConfigMap or its key + must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + description: |- + Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, + spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. + properties: + apiVersion: + description: Version of + the schema the FieldPath + is written in terms of, + defaults to "v1". + type: string + fieldPath: + description: Path of the + field to select in the + specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. + properties: + containerName: + description: 'Container + name: required for volumes, + optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the + output format of the exposed + resources, defaults to + "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: + resource to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + description: Selects a key of + a secret in the pod's namespace + properties: + key: + description: The key of + the secret to select from. Must + be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether + the Secret or its key + must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + envFrom: + description: |- + List of sources to populate environment variables in the container. + The keys defined within a source must be a C_IDENTIFIER. All invalid keys + will be reported as an event when the container is starting. When a key exists in multiple + sources, the value associated with the last source will take precedence. + Values defined by an Env with a duplicate key will take precedence. + Cannot be updated. + items: + description: EnvFromSource represents + the source of a set of ConfigMaps + properties: + configMapRef: + description: The ConfigMap to select + from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether + the ConfigMap must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + description: An optional identifier + to prepend to each key in the + ConfigMap. Must be a C_IDENTIFIER. + type: string + secretRef: + description: The Secret to select + from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether + the Secret must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array + x-kubernetes-list-type: atomic + image: + description: |- + Container image name. + More info: https://kubernetes.io/docs/concepts/containers/images + type: string + imagePullPolicy: + description: |- + Image pull policy. + One of Always, Never, IfNotPresent. + Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/containers/images#updating-images + type: string + lifecycle: + description: Lifecycle is not allowed + for ephemeral containers. + properties: + postStart: + description: |- + PostStart is called immediately after a container is created. If the handler fails, + the container is terminated and restarted according to its restart policy. + Other management of the container blocks until the hook completes. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies the + action to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies + the http request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers + to set in the request. HTTP + allows repeated headers. + items: + description: HTTPHeader + describes a custom header + to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header + field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access + on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents + the duration that the container + should sleep before being terminated. + properties: + seconds: + description: Seconds is the + number of seconds to sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for the backward compatibility. There are no validation of this field and + lifecycle hooks will fail in runtime when tcp handler is specified. + properties: + host: + description: 'Optional: Host + name to connect to, defaults + to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + preStop: + description: |- + PreStop is called immediately before a container is terminated due to an + API request or management event such as liveness/startup probe failure, + preemption, resource contention, etc. The handler is not called if the + container crashes or exits. The Pod's termination grace period countdown begins before the + PreStop hook is executed. Regardless of the outcome of the handler, the + container will eventually terminate within the Pod's termination grace + period (unless delayed by finalizers). Other management of the container blocks until the hook completes + or until the termination grace period is reached. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies the + action to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies + the http request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers + to set in the request. HTTP + allows repeated headers. + items: + description: HTTPHeader + describes a custom header + to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header + field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access + on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents + the duration that the container + should sleep before being terminated. + properties: + seconds: + description: Seconds is the + number of seconds to sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for the backward compatibility. There are no validation of this field and + lifecycle hooks will fail in runtime when tcp handler is specified. + properties: + host: + description: 'Optional: Host + name to connect to, defaults + to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + type: object + livenessProbe: + description: Probes are not allowed for + ephemeral containers. + properties: + exec: + description: Exec specifies the action + to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action + involving a GRPC port. + properties: + port: + description: Port number of the + gRPC service. Number must be + in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the + http request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to + set in the request. HTTP allows + repeated headers. + items: + description: HTTPHeader describes + a custom header to be used + in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header + field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on + the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an + action involving a TCP port. + properties: + host: + description: 'Optional: Host name + to connect to, defaults to the + pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + name: + description: |- + Name of the ephemeral container specified as a DNS_LABEL. + This name must be unique among all containers, init containers and ephemeral containers. + type: string + ports: + description: Ports are not allowed for + ephemeral containers. + items: + description: ContainerPort represents + a network port in a single container. + properties: + containerPort: + description: |- + Number of port to expose on the pod's IP address. + This must be a valid port number, 0 < x < 65536. + format: int32 + type: integer + hostIP: + description: What host IP to bind + the external port to. + type: string + hostPort: + description: |- + Number of port to expose on the host. + If specified, this must be a valid port number, 0 < x < 65536. + If HostNetwork is specified, this must match ContainerPort. + Most containers do not need this. + format: int32 + type: integer + name: + description: |- + If specified, this must be an IANA_SVC_NAME and unique within the pod. Each + named port in a pod must have a unique name. Name for the port that can be + referred to by services. + type: string + protocol: + default: TCP + description: |- + Protocol for port. Must be UDP, TCP, or SCTP. + Defaults to "TCP". + type: string + required: + - containerPort + type: object + type: array + x-kubernetes-list-map-keys: + - containerPort + - protocol + x-kubernetes-list-type: map + readinessProbe: + description: Probes are not allowed for + ephemeral containers. + properties: + exec: + description: Exec specifies the action + to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action + involving a GRPC port. + properties: + port: + description: Port number of the + gRPC service. Number must be + in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the + http request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to + set in the request. HTTP allows + repeated headers. + items: + description: HTTPHeader describes + a custom header to be used + in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header + field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on + the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an + action involving a TCP port. + properties: + host: + description: 'Optional: Host name + to connect to, defaults to the + pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + resizePolicy: + description: Resources resize policy for + the container. + items: + description: ContainerResizePolicy represents + resource resize policy for the container. + properties: + resourceName: + description: |- + Name of the resource to which this resource resize policy applies. + Supported values: cpu, memory. + type: string + restartPolicy: + description: |- + Restart policy to apply when specified resource is resized. + If not specified, it defaults to NotRequired. + type: string + required: + - resourceName + - restartPolicy + type: object + type: array + x-kubernetes-list-type: atomic + resources: + description: |- + Resources are not allowed for ephemeral containers. Ephemeral containers use spare resources + already allocated to the pod. + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references + one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + restartPolicy: + description: |- + Restart policy for the container to manage the restart behavior of each + container within a pod. + This may only be set for init containers. You cannot set this field on + ephemeral containers. + type: string + securityContext: + description: |- + Optional: SecurityContext defines the security options the ephemeral container should be run with. + If set, the fields of SecurityContext override the equivalent fields of PodSecurityContext. + properties: + allowPrivilegeEscalation: + description: |- + AllowPrivilegeEscalation controls whether a process can gain more + privileges than its parent process. This bool directly controls if + the no_new_privs flag will be set on the container process. + AllowPrivilegeEscalation is true always when the container is: + 1) run as Privileged + 2) has CAP_SYS_ADMIN + Note that this field cannot be set when spec.os.name is windows. + type: boolean + appArmorProfile: + description: |- + appArmorProfile is the AppArmor options to use by this container. If set, this profile + overrides the pod's appArmorProfile. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile loaded on the node that should be used. + The profile must be preconfigured on the node to work. + Must match the loaded name of the profile. + Must be set if and only if type is "Localhost". + type: string + type: + description: |- + type indicates which kind of AppArmor profile will be applied. + Valid options are: + Localhost - a profile pre-loaded on the node. + RuntimeDefault - the container runtime's default profile. + Unconfined - no AppArmor enforcement. + type: string + required: + - type + type: object + capabilities: + description: |- + The capabilities to add/drop when running containers. + Defaults to the default set of capabilities granted by the container runtime. + Note that this field cannot be set when spec.os.name is windows. + properties: + add: + description: Added capabilities + items: + description: Capability represent + POSIX capabilities type + type: string + type: array + x-kubernetes-list-type: atomic + drop: + description: Removed capabilities + items: + description: Capability represent + POSIX capabilities type + type: string + type: array + x-kubernetes-list-type: atomic + type: object + privileged: + description: |- + Run container in privileged mode. + Processes in privileged containers are essentially equivalent to root on the host. + Defaults to false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + procMount: + description: |- + procMount denotes the type of proc mount to use for the containers. + The default value is Default which uses the container runtime defaults for + readonly paths and masked paths. + This requires the ProcMountType feature flag to be enabled. + Note that this field cannot be set when spec.os.name is windows. + type: string + readOnlyRootFilesystem: + description: |- + Whether this container has a read-only root filesystem. + Default is false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + runAsGroup: + description: |- + The GID to run the entrypoint of the container process. + Uses runtime default if unset. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + runAsNonRoot: + description: |- + Indicates that the container must run as a non-root user. + If true, the Kubelet will validate the image at runtime to ensure that it + does not run as UID 0 (root) and fail to start the container if it does. + If unset or false, no such validation will be performed. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: |- + The UID to run the entrypoint of the container process. + Defaults to user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + seLinuxOptions: + description: |- + The SELinux context to be applied to the container. + If unspecified, the container runtime will allocate a random SELinux context for each + container. May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + properties: + level: + description: Level is SELinux + level label that applies to + the container. + type: string + role: + description: Role is a SELinux + role label that applies to the + container. + type: string + type: + description: Type is a SELinux + type label that applies to the + container. + type: string + user: + description: User is a SELinux + user label that applies to the + container. + type: string + type: object + seccompProfile: + description: |- + The seccomp options to use by this container. If seccomp options are + provided at both the pod & container level, the container options + override the pod options. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile defined in a file on the node should be used. + The profile must be preconfigured on the node to work. + Must be a descending path, relative to the kubelet's configured seccomp profile location. + Must be set if type is "Localhost". Must NOT be set for any other type. + type: string + type: + description: |- + type indicates which kind of seccomp profile will be applied. + Valid options are: + + Localhost - a profile defined in a file on the node should be used. + RuntimeDefault - the container runtime default profile should be used. + Unconfined - no profile should be applied. + type: string + required: + - type + type: object + windowsOptions: + description: |- + The Windows specific settings applied to all containers. + If unspecified, the options from the PodSecurityContext will be used. + If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is linux. + properties: + gmsaCredentialSpec: + description: |- + GMSACredentialSpec is where the GMSA admission webhook + (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the + GMSA credential spec named by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName + is the name of the GMSA credential + spec to use. + type: string + hostProcess: + description: |- + HostProcess determines if a container should be run as a 'Host Process' container. + All of a Pod's containers must have the same effective HostProcess value + (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). + In addition, if HostProcess is true then HostNetwork must also be set to true. + type: boolean + runAsUserName: + description: |- + The UserName in Windows to run the entrypoint of the container process. + Defaults to the user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: string + type: object + type: object + startupProbe: + description: Probes are not allowed for + ephemeral containers. + properties: + exec: + description: Exec specifies the action + to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action + involving a GRPC port. + properties: + port: + description: Port number of the + gRPC service. Number must be + in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the + http request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to + set in the request. HTTP allows + repeated headers. + items: + description: HTTPHeader describes + a custom header to be used + in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header + field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on + the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an + action involving a TCP port. + properties: + host: + description: 'Optional: Host name + to connect to, defaults to the + pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + stdin: + description: |- + Whether this container should allocate a buffer for stdin in the container runtime. If this + is not set, reads from stdin in the container will always result in EOF. + Default is false. + type: boolean + stdinOnce: + description: |- + Whether the container runtime should close the stdin channel after it has been opened by + a single attach. When stdin is true the stdin stream will remain open across multiple attach + sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the + first client attaches to stdin, and then remains open and accepts data until the client disconnects, + at which time stdin is closed and remains closed until the container is restarted. If this + flag is false, a container processes that reads from stdin will never receive an EOF. + Default is false + type: boolean + targetContainerName: + description: |- + If set, the name of the container from PodSpec that this ephemeral container targets. + The ephemeral container will be run in the namespaces (IPC, PID, etc) of this container. + If not set then the ephemeral container uses the namespaces configured in the Pod spec. + + The container runtime must implement support for this feature. If the runtime does not + support namespace targeting then the result of setting this field is undefined. + type: string + terminationMessagePath: + description: |- + Optional: Path at which the file to which the container's termination message + will be written is mounted into the container's filesystem. + Message written is intended to be brief final status, such as an assertion failure message. + Will be truncated by the node if greater than 4096 bytes. The total message length across + all containers will be limited to 12kb. + Defaults to /dev/termination-log. + Cannot be updated. + type: string + terminationMessagePolicy: + description: |- + Indicate how the termination message should be populated. File will use the contents of + terminationMessagePath to populate the container status message on both success and failure. + FallbackToLogsOnError will use the last chunk of container log output if the termination + message file is empty and the container exited with an error. + The log output is limited to 2048 bytes or 80 lines, whichever is smaller. + Defaults to File. + Cannot be updated. + type: string + tty: + description: |- + Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. + Default is false. + type: boolean + volumeDevices: + description: volumeDevices is the list + of block devices to be used by the container. + items: + description: volumeDevice describes + a mapping of a raw block device within + a container. + properties: + devicePath: + description: devicePath is the path + inside of the container that the + device will be mapped to. + type: string + name: + description: name must match the + name of a persistentVolumeClaim + in the pod + type: string + required: + - devicePath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - devicePath + x-kubernetes-list-type: map + volumeMounts: + description: |- + Pod volumes to mount into the container's filesystem. Subpath mounts are not allowed for ephemeral containers. + Cannot be updated. + items: + description: VolumeMount describes a + mounting of a Volume within a container. + properties: + mountPath: + description: |- + Path within the container at which the volume should be mounted. Must + not contain ':'. + type: string + mountPropagation: + description: |- + mountPropagation determines how mounts are propagated from the host + to container and the other way around. + When not set, MountPropagationNone is used. + This field is beta in 1.10. + When RecursiveReadOnly is set to IfPossible or to Enabled, MountPropagation must be None or unspecified + (which defaults to None). + type: string + name: + description: This must match the + Name of a Volume. + type: string + readOnly: + description: |- + Mounted read-only if true, read-write otherwise (false or unspecified). + Defaults to false. + type: boolean + recursiveReadOnly: + description: |- + RecursiveReadOnly specifies whether read-only mounts should be handled + recursively. + + If ReadOnly is false, this field has no meaning and must be unspecified. + + If ReadOnly is true, and this field is set to Disabled, the mount is not made + recursively read-only. If this field is set to IfPossible, the mount is made + recursively read-only, if it is supported by the container runtime. If this + field is set to Enabled, the mount is made recursively read-only if it is + supported by the container runtime, otherwise the pod will not be started and + an error will be generated to indicate the reason. + + If this field is set to IfPossible or Enabled, MountPropagation must be set to + None (or be unspecified, which defaults to None). + + If this field is not specified, it is treated as an equivalent of Disabled. + type: string + subPath: + description: |- + Path within the volume from which the container's volume should be mounted. + Defaults to "" (volume's root). + type: string + subPathExpr: + description: |- + Expanded path within the volume from which the container's volume should be mounted. + Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. + Defaults to "" (volume's root). + SubPathExpr and SubPath are mutually exclusive. + type: string + required: + - mountPath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - mountPath + x-kubernetes-list-type: map + workingDir: + description: |- + Container's working directory. + If not specified, the container runtime's default will be used, which + might be configured in the container image. + Cannot be updated. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + hostAliases: + description: |- + HostAliases is an optional list of hosts and IPs that will be injected into the pod's hosts + file if specified. + items: + description: |- + HostAlias holds the mapping between IP and hostnames that will be injected as an entry in the + pod's hosts file. + properties: + hostnames: + description: Hostnames for the above IP + address. + items: + type: string + type: array + x-kubernetes-list-type: atomic + ip: + description: IP address of the host file + entry. + type: string + required: + - ip + type: object + type: array + x-kubernetes-list-map-keys: + - ip + x-kubernetes-list-type: map + hostIPC: + description: |- + Use the host's ipc namespace. + Optional: Default to false. + type: boolean + hostNetwork: + description: |- + Host networking requested for this pod. Use the host's network namespace. + If this option is set, the ports that will be used must be specified. + Default to false. + type: boolean + hostPID: + description: |- + Use the host's pid namespace. + Optional: Default to false. + type: boolean + hostUsers: + description: |- + Use the host's user namespace. + Optional: Default to true. + If set to true or not present, the pod will be run in the host user namespace, useful + for when the pod needs a feature only available to the host user namespace, such as + loading a kernel module with CAP_SYS_MODULE. + When set to false, a new userns is created for the pod. Setting false is useful for + mitigating container breakout vulnerabilities even allowing users to run their + containers as root without actually having root privileges on the host. + This field is alpha-level and is only honored by servers that enable the UserNamespacesSupport feature. + type: boolean + hostname: + description: |- + Specifies the hostname of the Pod + If not specified, the pod's hostname will be set to a system-defined value. + type: string + imagePullSecrets: + description: |- + ImagePullSecrets is an optional list of references to secrets in the same namespace to use for pulling any of the images used by this PodSpec. + If specified, these secrets will be passed to individual puller implementations for them to use. + More info: https://kubernetes.io/docs/concepts/containers/images#specifying-imagepullsecrets-on-a-pod + items: + description: |- + LocalObjectReference contains enough information to let you locate the + referenced object inside the same namespace. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + initContainers: + description: |- + List of initialization containers belonging to the pod. + Init containers are executed in order prior to containers being started. If any + init container fails, the pod is considered to have failed and is handled according + to its restartPolicy. The name for an init container or normal container must be + unique among all containers. + Init containers may not have Lifecycle actions, Readiness probes, Liveness probes, or Startup probes. + The resourceRequirements of an init container are taken into account during scheduling + by finding the highest request/limit for each resource type, and then using the max of + of that value or the sum of the normal containers. Limits are applied to init containers + in a similar fashion. + Init containers cannot currently be added or removed. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/init-containers/ + items: + description: A single application container + that you want to run within a pod. + properties: + args: + description: |- + Arguments to the entrypoint. + The container image's CMD is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell + items: + type: string + type: array + x-kubernetes-list-type: atomic + command: + description: |- + Entrypoint array. Not executed within a shell. + The container image's ENTRYPOINT is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell + items: + type: string + type: array + x-kubernetes-list-type: atomic + env: + description: |- + List of environment variables to set in the container. + Cannot be updated. + items: + description: EnvVar represents an environment + variable present in a Container. + properties: + name: + description: Name of the environment + variable. Must be a C_IDENTIFIER. + type: string + value: + description: |- + Variable references $(VAR_NAME) are expanded + using the previously defined environment variables in the container and + any service environment variables. If a variable cannot be resolved, + the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. + "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless of whether the variable + exists or not. + Defaults to "". + type: string + valueFrom: + description: Source for the environment + variable's value. Cannot be used + if value is not empty. + properties: + configMapKeyRef: + description: Selects a key of + a ConfigMap. + properties: + key: + description: The key to + select. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether + the ConfigMap or its key + must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + description: |- + Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, + spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. + properties: + apiVersion: + description: Version of + the schema the FieldPath + is written in terms of, + defaults to "v1". + type: string + fieldPath: + description: Path of the + field to select in the + specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. + properties: + containerName: + description: 'Container + name: required for volumes, + optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the + output format of the exposed + resources, defaults to + "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: + resource to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + description: Selects a key of + a secret in the pod's namespace + properties: + key: + description: The key of + the secret to select from. Must + be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether + the Secret or its key + must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + envFrom: + description: |- + List of sources to populate environment variables in the container. + The keys defined within a source must be a C_IDENTIFIER. All invalid keys + will be reported as an event when the container is starting. When a key exists in multiple + sources, the value associated with the last source will take precedence. + Values defined by an Env with a duplicate key will take precedence. + Cannot be updated. + items: + description: EnvFromSource represents + the source of a set of ConfigMaps + properties: + configMapRef: + description: The ConfigMap to select + from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether + the ConfigMap must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + description: An optional identifier + to prepend to each key in the + ConfigMap. Must be a C_IDENTIFIER. + type: string + secretRef: + description: The Secret to select + from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether + the Secret must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array + x-kubernetes-list-type: atomic + image: + description: |- + Container image name. + More info: https://kubernetes.io/docs/concepts/containers/images + This field is optional to allow higher level config management to default or override + container images in workload controllers like Deployments and StatefulSets. + type: string + imagePullPolicy: + description: |- + Image pull policy. + One of Always, Never, IfNotPresent. + Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/containers/images#updating-images + type: string + lifecycle: + description: |- + Actions that the management system should take in response to container lifecycle events. + Cannot be updated. + properties: + postStart: + description: |- + PostStart is called immediately after a container is created. If the handler fails, + the container is terminated and restarted according to its restart policy. + Other management of the container blocks until the hook completes. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies the + action to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies + the http request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers + to set in the request. HTTP + allows repeated headers. + items: + description: HTTPHeader + describes a custom header + to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header + field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access + on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents + the duration that the container + should sleep before being terminated. + properties: + seconds: + description: Seconds is the + number of seconds to sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for the backward compatibility. There are no validation of this field and + lifecycle hooks will fail in runtime when tcp handler is specified. + properties: + host: + description: 'Optional: Host + name to connect to, defaults + to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + preStop: + description: |- + PreStop is called immediately before a container is terminated due to an + API request or management event such as liveness/startup probe failure, + preemption, resource contention, etc. The handler is not called if the + container crashes or exits. The Pod's termination grace period countdown begins before the + PreStop hook is executed. Regardless of the outcome of the handler, the + container will eventually terminate within the Pod's termination grace + period (unless delayed by finalizers). Other management of the container blocks until the hook completes + or until the termination grace period is reached. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies the + action to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies + the http request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers + to set in the request. HTTP + allows repeated headers. + items: + description: HTTPHeader + describes a custom header + to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header + field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access + on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents + the duration that the container + should sleep before being terminated. + properties: + seconds: + description: Seconds is the + number of seconds to sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for the backward compatibility. There are no validation of this field and + lifecycle hooks will fail in runtime when tcp handler is specified. + properties: + host: + description: 'Optional: Host + name to connect to, defaults + to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + type: object + livenessProbe: + description: |- + Periodic probe of container liveness. + Container will be restarted if the probe fails. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies the action + to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action + involving a GRPC port. + properties: + port: + description: Port number of the + gRPC service. Number must be + in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the + http request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to + set in the request. HTTP allows + repeated headers. + items: + description: HTTPHeader describes + a custom header to be used + in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header + field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on + the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an + action involving a TCP port. + properties: + host: + description: 'Optional: Host name + to connect to, defaults to the + pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + name: + description: |- + Name of the container specified as a DNS_LABEL. + Each container in a pod must have a unique name (DNS_LABEL). + Cannot be updated. + type: string + ports: + description: |- + List of ports to expose from the container. Not specifying a port here + DOES NOT prevent that port from being exposed. Any port which is + listening on the default "0.0.0.0" address inside a container will be + accessible from the network. + Modifying this array with strategic merge patch may corrupt the data. + For more information See https://github.com/kubernetes/kubernetes/issues/108255. + Cannot be updated. + items: + description: ContainerPort represents + a network port in a single container. + properties: + containerPort: + description: |- + Number of port to expose on the pod's IP address. + This must be a valid port number, 0 < x < 65536. + format: int32 + type: integer + hostIP: + description: What host IP to bind + the external port to. + type: string + hostPort: + description: |- + Number of port to expose on the host. + If specified, this must be a valid port number, 0 < x < 65536. + If HostNetwork is specified, this must match ContainerPort. + Most containers do not need this. + format: int32 + type: integer + name: + description: |- + If specified, this must be an IANA_SVC_NAME and unique within the pod. Each + named port in a pod must have a unique name. Name for the port that can be + referred to by services. + type: string + protocol: + default: TCP + description: |- + Protocol for port. Must be UDP, TCP, or SCTP. + Defaults to "TCP". + type: string + required: + - containerPort + type: object + type: array + x-kubernetes-list-map-keys: + - containerPort + - protocol + x-kubernetes-list-type: map + readinessProbe: + description: |- + Periodic probe of container service readiness. + Container will be removed from service endpoints if the probe fails. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies the action + to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action + involving a GRPC port. + properties: + port: + description: Port number of the + gRPC service. Number must be + in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the + http request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to + set in the request. HTTP allows + repeated headers. + items: + description: HTTPHeader describes + a custom header to be used + in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header + field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on + the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an + action involving a TCP port. + properties: + host: + description: 'Optional: Host name + to connect to, defaults to the + pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + resizePolicy: + description: Resources resize policy for + the container. + items: + description: ContainerResizePolicy represents + resource resize policy for the container. + properties: + resourceName: + description: |- + Name of the resource to which this resource resize policy applies. + Supported values: cpu, memory. + type: string + restartPolicy: + description: |- + Restart policy to apply when specified resource is resized. + If not specified, it defaults to NotRequired. + type: string + required: + - resourceName + - restartPolicy + type: object + type: array + x-kubernetes-list-type: atomic + resources: + description: |- + Compute Resources required by this container. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references + one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + restartPolicy: + description: |- + RestartPolicy defines the restart behavior of individual containers in a pod. + This field may only be set for init containers, and the only allowed value is "Always". + For non-init containers or when this field is not specified, + the restart behavior is defined by the Pod's restart policy and the container type. + Setting the RestartPolicy as "Always" for the init container will have the following effect: + this init container will be continually restarted on + exit until all regular containers have terminated. Once all regular + containers have completed, all init containers with restartPolicy "Always" + will be shut down. This lifecycle differs from normal init containers and + is often referred to as a "sidecar" container. Although this init + container still starts in the init container sequence, it does not wait + for the container to complete before proceeding to the next init + container. Instead, the next init container starts immediately after this + init container is started, or after any startupProbe has successfully + completed. + type: string + securityContext: + description: |- + SecurityContext defines the security options the container should be run with. + If set, the fields of SecurityContext override the equivalent fields of PodSecurityContext. + More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ + properties: + allowPrivilegeEscalation: + description: |- + AllowPrivilegeEscalation controls whether a process can gain more + privileges than its parent process. This bool directly controls if + the no_new_privs flag will be set on the container process. + AllowPrivilegeEscalation is true always when the container is: + 1) run as Privileged + 2) has CAP_SYS_ADMIN + Note that this field cannot be set when spec.os.name is windows. + type: boolean + appArmorProfile: + description: |- + appArmorProfile is the AppArmor options to use by this container. If set, this profile + overrides the pod's appArmorProfile. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile loaded on the node that should be used. + The profile must be preconfigured on the node to work. + Must match the loaded name of the profile. + Must be set if and only if type is "Localhost". + type: string + type: + description: |- + type indicates which kind of AppArmor profile will be applied. + Valid options are: + Localhost - a profile pre-loaded on the node. + RuntimeDefault - the container runtime's default profile. + Unconfined - no AppArmor enforcement. + type: string + required: + - type + type: object + capabilities: + description: |- + The capabilities to add/drop when running containers. + Defaults to the default set of capabilities granted by the container runtime. + Note that this field cannot be set when spec.os.name is windows. + properties: + add: + description: Added capabilities + items: + description: Capability represent + POSIX capabilities type + type: string + type: array + x-kubernetes-list-type: atomic + drop: + description: Removed capabilities + items: + description: Capability represent + POSIX capabilities type + type: string + type: array + x-kubernetes-list-type: atomic + type: object + privileged: + description: |- + Run container in privileged mode. + Processes in privileged containers are essentially equivalent to root on the host. + Defaults to false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + procMount: + description: |- + procMount denotes the type of proc mount to use for the containers. + The default value is Default which uses the container runtime defaults for + readonly paths and masked paths. + This requires the ProcMountType feature flag to be enabled. + Note that this field cannot be set when spec.os.name is windows. + type: string + readOnlyRootFilesystem: + description: |- + Whether this container has a read-only root filesystem. + Default is false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + runAsGroup: + description: |- + The GID to run the entrypoint of the container process. + Uses runtime default if unset. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + runAsNonRoot: + description: |- + Indicates that the container must run as a non-root user. + If true, the Kubelet will validate the image at runtime to ensure that it + does not run as UID 0 (root) and fail to start the container if it does. + If unset or false, no such validation will be performed. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: |- + The UID to run the entrypoint of the container process. + Defaults to user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + seLinuxOptions: + description: |- + The SELinux context to be applied to the container. + If unspecified, the container runtime will allocate a random SELinux context for each + container. May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + properties: + level: + description: Level is SELinux + level label that applies to + the container. + type: string + role: + description: Role is a SELinux + role label that applies to the + container. + type: string + type: + description: Type is a SELinux + type label that applies to the + container. + type: string + user: + description: User is a SELinux + user label that applies to the + container. + type: string + type: object + seccompProfile: + description: |- + The seccomp options to use by this container. If seccomp options are + provided at both the pod & container level, the container options + override the pod options. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile defined in a file on the node should be used. + The profile must be preconfigured on the node to work. + Must be a descending path, relative to the kubelet's configured seccomp profile location. + Must be set if type is "Localhost". Must NOT be set for any other type. + type: string + type: + description: |- + type indicates which kind of seccomp profile will be applied. + Valid options are: + + Localhost - a profile defined in a file on the node should be used. + RuntimeDefault - the container runtime default profile should be used. + Unconfined - no profile should be applied. + type: string + required: + - type + type: object + windowsOptions: + description: |- + The Windows specific settings applied to all containers. + If unspecified, the options from the PodSecurityContext will be used. + If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is linux. + properties: + gmsaCredentialSpec: + description: |- + GMSACredentialSpec is where the GMSA admission webhook + (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the + GMSA credential spec named by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName + is the name of the GMSA credential + spec to use. + type: string + hostProcess: + description: |- + HostProcess determines if a container should be run as a 'Host Process' container. + All of a Pod's containers must have the same effective HostProcess value + (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). + In addition, if HostProcess is true then HostNetwork must also be set to true. + type: boolean + runAsUserName: + description: |- + The UserName in Windows to run the entrypoint of the container process. + Defaults to the user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: string + type: object + type: object + startupProbe: + description: |- + StartupProbe indicates that the Pod has successfully initialized. + If specified, no other probes are executed until this completes successfully. + If this probe fails, the Pod will be restarted, just as if the livenessProbe failed. + This can be used to provide different probe parameters at the beginning of a Pod's lifecycle, + when it might take a long time to load data or warm a cache, than during steady-state operation. + This cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies the action + to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action + involving a GRPC port. + properties: + port: + description: Port number of the + gRPC service. Number must be + in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the + http request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to + set in the request. HTTP allows + repeated headers. + items: + description: HTTPHeader describes + a custom header to be used + in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header + field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on + the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an + action involving a TCP port. + properties: + host: + description: 'Optional: Host name + to connect to, defaults to the + pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + stdin: + description: |- + Whether this container should allocate a buffer for stdin in the container runtime. If this + is not set, reads from stdin in the container will always result in EOF. + Default is false. + type: boolean + stdinOnce: + description: |- + Whether the container runtime should close the stdin channel after it has been opened by + a single attach. When stdin is true the stdin stream will remain open across multiple attach + sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the + first client attaches to stdin, and then remains open and accepts data until the client disconnects, + at which time stdin is closed and remains closed until the container is restarted. If this + flag is false, a container processes that reads from stdin will never receive an EOF. + Default is false + type: boolean + terminationMessagePath: + description: |- + Optional: Path at which the file to which the container's termination message + will be written is mounted into the container's filesystem. + Message written is intended to be brief final status, such as an assertion failure message. + Will be truncated by the node if greater than 4096 bytes. The total message length across + all containers will be limited to 12kb. + Defaults to /dev/termination-log. + Cannot be updated. + type: string + terminationMessagePolicy: + description: |- + Indicate how the termination message should be populated. File will use the contents of + terminationMessagePath to populate the container status message on both success and failure. + FallbackToLogsOnError will use the last chunk of container log output if the termination + message file is empty and the container exited with an error. + The log output is limited to 2048 bytes or 80 lines, whichever is smaller. + Defaults to File. + Cannot be updated. + type: string + tty: + description: |- + Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. + Default is false. + type: boolean + volumeDevices: + description: volumeDevices is the list + of block devices to be used by the container. + items: + description: volumeDevice describes + a mapping of a raw block device within + a container. + properties: + devicePath: + description: devicePath is the path + inside of the container that the + device will be mapped to. + type: string + name: + description: name must match the + name of a persistentVolumeClaim + in the pod + type: string + required: + - devicePath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - devicePath + x-kubernetes-list-type: map + volumeMounts: + description: |- + Pod volumes to mount into the container's filesystem. + Cannot be updated. + items: + description: VolumeMount describes a + mounting of a Volume within a container. + properties: + mountPath: + description: |- + Path within the container at which the volume should be mounted. Must + not contain ':'. + type: string + mountPropagation: + description: |- + mountPropagation determines how mounts are propagated from the host + to container and the other way around. + When not set, MountPropagationNone is used. + This field is beta in 1.10. + When RecursiveReadOnly is set to IfPossible or to Enabled, MountPropagation must be None or unspecified + (which defaults to None). + type: string + name: + description: This must match the + Name of a Volume. + type: string + readOnly: + description: |- + Mounted read-only if true, read-write otherwise (false or unspecified). + Defaults to false. + type: boolean + recursiveReadOnly: + description: |- + RecursiveReadOnly specifies whether read-only mounts should be handled + recursively. + + If ReadOnly is false, this field has no meaning and must be unspecified. + + If ReadOnly is true, and this field is set to Disabled, the mount is not made + recursively read-only. If this field is set to IfPossible, the mount is made + recursively read-only, if it is supported by the container runtime. If this + field is set to Enabled, the mount is made recursively read-only if it is + supported by the container runtime, otherwise the pod will not be started and + an error will be generated to indicate the reason. + + If this field is set to IfPossible or Enabled, MountPropagation must be set to + None (or be unspecified, which defaults to None). + + If this field is not specified, it is treated as an equivalent of Disabled. + type: string + subPath: + description: |- + Path within the volume from which the container's volume should be mounted. + Defaults to "" (volume's root). + type: string + subPathExpr: + description: |- + Expanded path within the volume from which the container's volume should be mounted. + Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. + Defaults to "" (volume's root). + SubPathExpr and SubPath are mutually exclusive. + type: string + required: + - mountPath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - mountPath + x-kubernetes-list-type: map + workingDir: + description: |- + Container's working directory. + If not specified, the container runtime's default will be used, which + might be configured in the container image. + Cannot be updated. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + nodeName: + description: |- + NodeName indicates in which node this pod is scheduled. + If empty, this pod is a candidate for scheduling by the scheduler defined in schedulerName. + Once this field is set, the kubelet for this node becomes responsible for the lifecycle of this pod. + This field should not be used to express a desire for the pod to be scheduled on a specific node. + https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodename + type: string + nodeSelector: + additionalProperties: + type: string + description: |- + NodeSelector is a selector which must be true for the pod to fit on a node. + Selector which must match a node's labels for the pod to be scheduled on that node. + More info: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ + type: object + x-kubernetes-map-type: atomic + os: + description: |- + Specifies the OS of the containers in the pod. + Some pod and container fields are restricted if this is set. + + If the OS field is set to linux, the following fields must be unset: + -securityContext.windowsOptions + + If the OS field is set to windows, following fields must be unset: + - spec.hostPID + - spec.hostIPC + - spec.hostUsers + - spec.securityContext.appArmorProfile + - spec.securityContext.seLinuxOptions + - spec.securityContext.seccompProfile + - spec.securityContext.fsGroup + - spec.securityContext.fsGroupChangePolicy + - spec.securityContext.sysctls + - spec.shareProcessNamespace + - spec.securityContext.runAsUser + - spec.securityContext.runAsGroup + - spec.securityContext.supplementalGroups + - spec.securityContext.supplementalGroupsPolicy + - spec.containers[*].securityContext.appArmorProfile + - spec.containers[*].securityContext.seLinuxOptions + - spec.containers[*].securityContext.seccompProfile + - spec.containers[*].securityContext.capabilities + - spec.containers[*].securityContext.readOnlyRootFilesystem + - spec.containers[*].securityContext.privileged + - spec.containers[*].securityContext.allowPrivilegeEscalation + - spec.containers[*].securityContext.procMount + - spec.containers[*].securityContext.runAsUser + - spec.containers[*].securityContext.runAsGroup + properties: + name: + description: |- + Name is the name of the operating system. The currently supported values are linux and windows. + Additional value may be defined in future and can be one of: + https://github.com/opencontainers/runtime-spec/blob/master/config.md#platform-specific-configuration + Clients should expect to handle additional values and treat unrecognized values in this field as os: null + type: string + required: + - name + type: object + overhead: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Overhead represents the resource overhead associated with running a pod for a given RuntimeClass. + This field will be autopopulated at admission time by the RuntimeClass admission controller. If + the RuntimeClass admission controller is enabled, overhead must not be set in Pod create requests. + The RuntimeClass admission controller will reject Pod create requests which have the overhead already + set. If RuntimeClass is configured and selected in the PodSpec, Overhead will be set to the value + defined in the corresponding RuntimeClass, otherwise it will remain unset and treated as zero. + More info: https://git.k8s.io/enhancements/keps/sig-node/688-pod-overhead/README.md + type: object + preemptionPolicy: + description: |- + PreemptionPolicy is the Policy for preempting pods with lower priority. + One of Never, PreemptLowerPriority. + Defaults to PreemptLowerPriority if unset. + type: string + priority: + description: |- + The priority value. Various system components use this field to find the + priority of the pod. When Priority Admission Controller is enabled, it + prevents users from setting this field. The admission controller populates + this field from PriorityClassName. + The higher the value, the higher the priority. + format: int32 + type: integer + priorityClassName: + description: |- + If specified, indicates the pod's priority. "system-node-critical" and + "system-cluster-critical" are two special keywords which indicate the + highest priorities with the former being the highest priority. Any other + name must be defined by creating a PriorityClass object with that name. + If not specified, the pod priority will be default or zero if there is no + default. + type: string + readinessGates: + description: |- + If specified, all readiness gates will be evaluated for pod readiness. + A pod is ready when all its containers are ready AND + all conditions specified in the readiness gates have status equal to "True" + More info: https://git.k8s.io/enhancements/keps/sig-network/580-pod-readiness-gates + items: + description: PodReadinessGate contains the + reference to a pod condition + properties: + conditionType: + description: ConditionType refers to a + condition in the pod's condition list + with matching type. + type: string + required: + - conditionType + type: object + type: array + x-kubernetes-list-type: atomic + resourceClaims: + description: |- + ResourceClaims defines which ResourceClaims must be allocated + and reserved before the Pod is allowed to start. The resources + will be made available to those containers which consume them + by name. + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + This field is immutable. + items: + description: |- + PodResourceClaim references exactly one ResourceClaim, either directly + or by naming a ResourceClaimTemplate which is then turned into a ResourceClaim + for the pod. + + It adds a name to it that uniquely identifies the ResourceClaim inside the Pod. + Containers that need access to the ResourceClaim reference it with this name. + properties: + name: + description: |- + Name uniquely identifies this resource claim inside the pod. + This must be a DNS_LABEL. + type: string + resourceClaimName: + description: |- + ResourceClaimName is the name of a ResourceClaim object in the same + namespace as this pod. + + Exactly one of ResourceClaimName and ResourceClaimTemplateName must + be set. + type: string + resourceClaimTemplateName: + description: |- + ResourceClaimTemplateName is the name of a ResourceClaimTemplate + object in the same namespace as this pod. + + The template will be used to create a new ResourceClaim, which will + be bound to this pod. When this pod is deleted, the ResourceClaim + will also be deleted. The pod name and resource name, along with a + generated component, will be used to form a unique name for the + ResourceClaim, which will be recorded in pod.status.resourceClaimStatuses. + + This field is immutable and no changes will be made to the + corresponding ResourceClaim by the control plane after creating the + ResourceClaim. + + Exactly one of ResourceClaimName and ResourceClaimTemplateName must + be set. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + restartPolicy: + description: |- + Restart policy for all containers within the pod. + One of Always, OnFailure, Never. In some contexts, only a subset of those values may be permitted. + Default to Always. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#restart-policy + type: string + runtimeClassName: + description: |- + RuntimeClassName refers to a RuntimeClass object in the node.k8s.io group, which should be used + to run this pod. If no RuntimeClass resource matches the named class, the pod will not be run. + If unset or empty, the "legacy" RuntimeClass will be used, which is an implicit class with an + empty definition that uses the default runtime handler. + More info: https://git.k8s.io/enhancements/keps/sig-node/585-runtime-class + type: string + schedulerName: + description: |- + If specified, the pod will be dispatched by specified scheduler. + If not specified, the pod will be dispatched by default scheduler. + type: string + schedulingGates: + description: |- + SchedulingGates is an opaque list of values that if specified will block scheduling the pod. + If schedulingGates is not empty, the pod will stay in the SchedulingGated state and the + scheduler will not attempt to schedule the pod. + + SchedulingGates can only be set at pod creation time, and be removed only afterwards. + items: + description: PodSchedulingGate is associated + to a Pod to guard its scheduling. + properties: + name: + description: |- + Name of the scheduling gate. + Each scheduling gate must have a unique name field. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + securityContext: + description: |- + SecurityContext holds pod-level security attributes and common container settings. + Optional: Defaults to empty. See type description for default values of each field. + properties: + appArmorProfile: + description: |- + appArmorProfile is the AppArmor options to use by the containers in this pod. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile loaded on the node that should be used. + The profile must be preconfigured on the node to work. + Must match the loaded name of the profile. + Must be set if and only if type is "Localhost". + type: string + type: + description: |- + type indicates which kind of AppArmor profile will be applied. + Valid options are: + Localhost - a profile pre-loaded on the node. + RuntimeDefault - the container runtime's default profile. + Unconfined - no AppArmor enforcement. + type: string + required: + - type + type: object + fsGroup: + description: |- + A special supplemental group that applies to all containers in a pod. + Some volume types allow the Kubelet to change the ownership of that volume + to be owned by the pod: + + 1. The owning GID will be the FSGroup + 2. The setgid bit is set (new files created in the volume will be owned by FSGroup) + 3. The permission bits are OR'd with rw-rw---- + + If unset, the Kubelet will not modify the ownership and permissions of any volume. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + fsGroupChangePolicy: + description: |- + fsGroupChangePolicy defines behavior of changing ownership and permission of the volume + before being exposed inside Pod. This field will only apply to + volume types which support fsGroup based ownership(and permissions). + It will have no effect on ephemeral volume types such as: secret, configmaps + and emptydir. + Valid values are "OnRootMismatch" and "Always". If not specified, "Always" is used. + Note that this field cannot be set when spec.os.name is windows. + type: string + runAsGroup: + description: |- + The GID to run the entrypoint of the container process. + Uses runtime default if unset. + May also be set in SecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence + for that container. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + runAsNonRoot: + description: |- + Indicates that the container must run as a non-root user. + If true, the Kubelet will validate the image at runtime to ensure that it + does not run as UID 0 (root) and fail to start the container if it does. + If unset or false, no such validation will be performed. + May also be set in SecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: |- + The UID to run the entrypoint of the container process. + Defaults to user specified in image metadata if unspecified. + May also be set in SecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence + for that container. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + seLinuxOptions: + description: |- + The SELinux context to be applied to all containers. + If unspecified, the container runtime will allocate a random SELinux context for each + container. May also be set in SecurityContext. If set in + both SecurityContext and PodSecurityContext, the value specified in SecurityContext + takes precedence for that container. + Note that this field cannot be set when spec.os.name is windows. + properties: + level: + description: Level is SELinux level + label that applies to the container. + type: string + role: + description: Role is a SELinux role + label that applies to the container. + type: string + type: + description: Type is a SELinux type + label that applies to the container. + type: string + user: + description: User is a SELinux user + label that applies to the container. + type: string + type: object + seccompProfile: + description: |- + The seccomp options to use by the containers in this pod. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile defined in a file on the node should be used. + The profile must be preconfigured on the node to work. + Must be a descending path, relative to the kubelet's configured seccomp profile location. + Must be set if type is "Localhost". Must NOT be set for any other type. + type: string + type: + description: |- + type indicates which kind of seccomp profile will be applied. + Valid options are: + + Localhost - a profile defined in a file on the node should be used. + RuntimeDefault - the container runtime default profile should be used. + Unconfined - no profile should be applied. + type: string + required: + - type + type: object + supplementalGroups: + description: |- + A list of groups applied to the first process run in each container, in + addition to the container's primary GID and fsGroup (if specified). If + the SupplementalGroupsPolicy feature is enabled, the + supplementalGroupsPolicy field determines whether these are in addition + to or instead of any group memberships defined in the container image. + If unspecified, no additional groups are added, though group memberships + defined in the container image may still be used, depending on the + supplementalGroupsPolicy field. + Note that this field cannot be set when spec.os.name is windows. + items: + format: int64 + type: integer + type: array + x-kubernetes-list-type: atomic + supplementalGroupsPolicy: + description: |- + Defines how supplemental groups of the first container processes are calculated. + Valid values are "Merge" and "Strict". If not specified, "Merge" is used. + (Alpha) Using the field requires the SupplementalGroupsPolicy feature gate to be enabled + and the container runtime must implement support for this feature. + Note that this field cannot be set when spec.os.name is windows. + type: string + sysctls: + description: |- + Sysctls hold a list of namespaced sysctls used for the pod. Pods with unsupported + sysctls (by the container runtime) might fail to launch. + Note that this field cannot be set when spec.os.name is windows. + items: + description: Sysctl defines a kernel parameter + to be set + properties: + name: + description: Name of a property to + set + type: string + value: + description: Value of a property to + set + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + windowsOptions: + description: |- + The Windows specific settings applied to all containers. + If unspecified, the options within a container's SecurityContext will be used. + If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is linux. + properties: + gmsaCredentialSpec: + description: |- + GMSACredentialSpec is where the GMSA admission webhook + (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the + GMSA credential spec named by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName + is the name of the GMSA credential + spec to use. + type: string + hostProcess: + description: |- + HostProcess determines if a container should be run as a 'Host Process' container. + All of a Pod's containers must have the same effective HostProcess value + (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). + In addition, if HostProcess is true then HostNetwork must also be set to true. + type: boolean + runAsUserName: + description: |- + The UserName in Windows to run the entrypoint of the container process. + Defaults to the user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: string + type: object + type: object + serviceAccount: + description: |- + DeprecatedServiceAccount is a deprecated alias for ServiceAccountName. + Deprecated: Use serviceAccountName instead. + type: string + serviceAccountName: + description: |- + ServiceAccountName is the name of the ServiceAccount to use to run this pod. + More info: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ + type: string + setHostnameAsFQDN: + description: |- + If true the pod's hostname will be configured as the pod's FQDN, rather than the leaf name (the default). + In Linux containers, this means setting the FQDN in the hostname field of the kernel (the nodename field of struct utsname). + In Windows containers, this means setting the registry value of hostname for the registry key HKEY_LOCAL_MACHINE\\SYSTEM\\CurrentControlSet\\Services\\Tcpip\\Parameters to FQDN. + If a pod does not have FQDN, this has no effect. + Default to false. + type: boolean + shareProcessNamespace: + description: |- + Share a single process namespace between all of the containers in a pod. + When this is set containers will be able to view and signal processes from other containers + in the same pod, and the first process in each container will not be assigned PID 1. + HostPID and ShareProcessNamespace cannot both be set. + Optional: Default to false. + type: boolean + subdomain: + description: |- + If specified, the fully qualified Pod hostname will be "...svc.". + If not specified, the pod will not have a domainname at all. + type: string + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully. May be decreased in delete request. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + If this value is nil, the default grace period will be used instead. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + Defaults to 30 seconds. + format: int64 + type: integer + tolerations: + description: If specified, the pod's tolerations. + items: + description: |- + The pod this Toleration is attached to tolerates any taint that matches + the triple using the matching operator . + properties: + effect: + description: |- + Effect indicates the taint effect to match. Empty means match all taint effects. + When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: |- + Key is the taint key that the toleration applies to. Empty means match all taint keys. + If the key is empty, operator must be Exists; this combination means to match all values and all keys. + type: string + operator: + description: |- + Operator represents a key's relationship to the value. + Valid operators are Exists and Equal. Defaults to Equal. + Exists is equivalent to wildcard for value, so that a pod can + tolerate all taints of a particular category. + type: string + tolerationSeconds: + description: |- + TolerationSeconds represents the period of time the toleration (which must be + of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, + it is not set, which means tolerate the taint forever (do not evict). Zero and + negative values will be treated as 0 (evict immediately) by the system. + format: int64 + type: integer + value: + description: |- + Value is the taint value the toleration matches to. + If the operator is Exists, the value should be empty, otherwise just a regular string. + type: string + type: object + type: array + x-kubernetes-list-type: atomic + topologySpreadConstraints: + description: |- + TopologySpreadConstraints describes how a group of pods ought to spread across topology + domains. Scheduler will schedule pods in a way which abides by the constraints. + All topologySpreadConstraints are ANDed. + items: + description: TopologySpreadConstraint specifies + how to spread matching pods among the given + topology. + properties: + labelSelector: + description: |- + LabelSelector is used to find matching pods. + Pods that match this label selector are counted to determine the number of pods + in their corresponding topology domain. + properties: + matchExpressions: + description: matchExpressions is a + list of label selector requirements. + The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select the pods over which + spreading will be calculated. The keys are used to lookup values from the + incoming pod labels, those key-value labels are ANDed with labelSelector + to select the group of existing pods over which spreading will be calculated + for the incoming pod. The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + MatchLabelKeys cannot be set when LabelSelector isn't set. + Keys that don't exist in the incoming pod labels will + be ignored. A null or empty list means only match against labelSelector. + + This is a beta field and requires the MatchLabelKeysInPodTopologySpread feature gate to be enabled (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + maxSkew: + description: |- + MaxSkew describes the degree to which pods may be unevenly distributed. + When `whenUnsatisfiable=DoNotSchedule`, it is the maximum permitted difference + between the number of matching pods in the target topology and the global minimum. + The global minimum is the minimum number of matching pods in an eligible domain + or zero if the number of eligible domains is less than MinDomains. + For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same + labelSelector spread as 2/2/1: + In this case, the global minimum is 1. + | zone1 | zone2 | zone3 | + | P P | P P | P | + - if MaxSkew is 1, incoming pod can only be scheduled to zone3 to become 2/2/2; + scheduling it onto zone1(zone2) would make the ActualSkew(3-1) on zone1(zone2) + violate MaxSkew(1). + - if MaxSkew is 2, incoming pod can be scheduled onto any zone. + When `whenUnsatisfiable=ScheduleAnyway`, it is used to give higher precedence + to topologies that satisfy it. + It's a required field. Default value is 1 and 0 is not allowed. + format: int32 + type: integer + minDomains: + description: |- + MinDomains indicates a minimum number of eligible domains. + When the number of eligible domains with matching topology keys is less than minDomains, + Pod Topology Spread treats "global minimum" as 0, and then the calculation of Skew is performed. + And when the number of eligible domains with matching topology keys equals or greater than minDomains, + this value has no effect on scheduling. + As a result, when the number of eligible domains is less than minDomains, + scheduler won't schedule more than maxSkew Pods to those domains. + If value is nil, the constraint behaves as if MinDomains is equal to 1. + Valid values are integers greater than 0. + When value is not nil, WhenUnsatisfiable must be DoNotSchedule. + + For example, in a 3-zone cluster, MaxSkew is set to 2, MinDomains is set to 5 and pods with the same + labelSelector spread as 2/2/2: + | zone1 | zone2 | zone3 | + | P P | P P | P P | + The number of domains is less than 5(MinDomains), so "global minimum" is treated as 0. + In this situation, new pod with the same labelSelector cannot be scheduled, + because computed skew will be 3(3 - 0) if new Pod is scheduled to any of the three zones, + it will violate MaxSkew. + format: int32 + type: integer + nodeAffinityPolicy: + description: |- + NodeAffinityPolicy indicates how we will treat Pod's nodeAffinity/nodeSelector + when calculating pod topology spread skew. Options are: + - Honor: only nodes matching nodeAffinity/nodeSelector are included in the calculations. + - Ignore: nodeAffinity/nodeSelector are ignored. All nodes are included in the calculations. + + If this value is nil, the behavior is equivalent to the Honor policy. + This is a beta-level feature default enabled by the NodeInclusionPolicyInPodTopologySpread feature flag. + type: string + nodeTaintsPolicy: + description: |- + NodeTaintsPolicy indicates how we will treat node taints when calculating + pod topology spread skew. Options are: + - Honor: nodes without taints, along with tainted nodes for which the incoming pod + has a toleration, are included. + - Ignore: node taints are ignored. All nodes are included. + + If this value is nil, the behavior is equivalent to the Ignore policy. + This is a beta-level feature default enabled by the NodeInclusionPolicyInPodTopologySpread feature flag. + type: string + topologyKey: + description: |- + TopologyKey is the key of node labels. Nodes that have a label with this key + and identical values are considered to be in the same topology. + We consider each as a "bucket", and try to put balanced number + of pods into each bucket. + We define a domain as a particular instance of a topology. + Also, we define an eligible domain as a domain whose nodes meet the requirements of + nodeAffinityPolicy and nodeTaintsPolicy. + e.g. If TopologyKey is "kubernetes.io/hostname", each Node is a domain of that topology. + And, if TopologyKey is "topology.kubernetes.io/zone", each zone is a domain of that topology. + It's a required field. + type: string + whenUnsatisfiable: + description: |- + WhenUnsatisfiable indicates how to deal with a pod if it doesn't satisfy + the spread constraint. + - DoNotSchedule (default) tells the scheduler not to schedule it. + - ScheduleAnyway tells the scheduler to schedule the pod in any location, + but giving higher precedence to topologies that would help reduce the + skew. + A constraint is considered "Unsatisfiable" for an incoming pod + if and only if every possible node assignment for that pod would violate + "MaxSkew" on some topology. + For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same + labelSelector spread as 3/1/1: + | zone1 | zone2 | zone3 | + | P P P | P | P | + If WhenUnsatisfiable is set to DoNotSchedule, incoming pod can only be scheduled + to zone2(zone3) to become 3/2/1(3/1/2) as ActualSkew(2-1) on zone2(zone3) satisfies + MaxSkew(1). In other words, the cluster can still be imbalanced, but scheduler + won't make it *more* imbalanced. + It's a required field. + type: string + required: + - maxSkew + - topologyKey + - whenUnsatisfiable + type: object + type: array + x-kubernetes-list-map-keys: + - topologyKey + - whenUnsatisfiable + x-kubernetes-list-type: map + volumes: + description: |- + List of volumes that can be mounted by containers belonging to the pod. + More info: https://kubernetes.io/docs/concepts/storage/volumes + items: + description: Volume represents a named volume + in a pod that may be accessed by any container + in the pod. + properties: + awsElasticBlockStore: + description: |- + awsElasticBlockStore represents an AWS Disk resource that is attached to a + kubelet's host machine and then exposed to the pod. + More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore + properties: + fsType: + description: |- + fsType is the filesystem type of the volume that you want to mount. + Tip: Ensure that the filesystem type is supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore + type: string + partition: + description: |- + partition is the partition in the volume that you want to mount. + If omitted, the default is to mount by volume name. + Examples: For volume /dev/sda1, you specify the partition as "1". + Similarly, the volume partition for /dev/sda is "0" (or you can leave the property empty). + format: int32 + type: integer + readOnly: + description: |- + readOnly value true will force the readOnly setting in VolumeMounts. + More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore + type: boolean + volumeID: + description: |- + volumeID is unique ID of the persistent disk resource in AWS (Amazon EBS volume). + More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore + type: string + required: + - volumeID + type: object + azureDisk: + description: azureDisk represents an Azure + Data Disk mount on the host and bind + mount to the pod. + properties: + cachingMode: + description: 'cachingMode is the Host + Caching mode: None, Read Only, Read + Write.' + type: string + diskName: + description: diskName is the Name + of the data disk in the blob storage + type: string + diskURI: + description: diskURI is the URI of + data disk in the blob storage + type: string + fsType: + default: ext4 + description: |- + fsType is Filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + kind: + description: 'kind expected values + are Shared: multiple blob disks + per storage account Dedicated: + single blob disk per storage account Managed: + azure managed data disk (only in + managed availability set). defaults + to shared' + type: string + readOnly: + default: false + description: |- + readOnly Defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + required: + - diskName + - diskURI + type: object + azureFile: + description: azureFile represents an Azure + File Service mount on the host and bind + mount to the pod. + properties: + readOnly: + description: |- + readOnly defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + secretName: + description: secretName is the name + of secret that contains Azure Storage + Account Name and Key + type: string + shareName: + description: shareName is the azure + share Name + type: string + required: + - secretName + - shareName + type: object + cephfs: + description: cephFS represents a Ceph + FS mount on the host that shares a pod's + lifetime + properties: + monitors: + description: |- + monitors is Required: Monitors is a collection of Ceph monitors + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + items: + type: string + type: array + x-kubernetes-list-type: atomic + path: + description: 'path is Optional: Used + as the mounted root, rather than + the full Ceph tree, default is /' + type: string + readOnly: + description: |- + readOnly is Optional: Defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + type: boolean + secretFile: + description: |- + secretFile is Optional: SecretFile is the path to key ring for User, default is /etc/ceph/user.secret + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + type: string + secretRef: + description: |- + secretRef is Optional: SecretRef is reference to the authentication secret for User, default is empty. + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + user: + description: |- + user is optional: User is the rados user name, default is admin + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + type: string + required: + - monitors + type: object + cinder: + description: |- + cinder represents a cinder volume attached and mounted on kubelets host machine. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md + properties: + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md + type: string + readOnly: + description: |- + readOnly defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md + type: boolean + secretRef: + description: |- + secretRef is optional: points to a secret object containing parameters used to connect + to OpenStack. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + volumeID: + description: |- + volumeID used to identify the volume in cinder. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md + type: string + required: + - volumeID + type: object + configMap: + description: configMap represents a configMap + that should populate this volume + properties: + defaultMode: + description: |- + defaultMode is optional: mode bits used to set permissions on created files by default. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + Defaults to 0644. + Directories within the path are not affected by this setting. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + items: + description: |- + items if unspecified, each key-value pair in the Data field of the referenced + ConfigMap will be projected into the volume as a file whose name is the + key and content is the value. If specified, the listed keys will be + projected into the specified paths, and unlisted keys will not be + present. If a key is specified which is not present in the ConfigMap, + the volume setup will error unless it is marked optional. Paths must be + relative and may not contain the '..' path or start with '..'. + items: + description: Maps a string key to + a path within a volume. + properties: + key: + description: key is the key + to project. + type: string + mode: + description: |- + mode is Optional: mode bits used to set permissions on this file. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: |- + path is the relative path of the file to map the key to. + May not be an absolute path. + May not contain the path element '..'. + May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: optional specify whether + the ConfigMap or its keys must be + defined + type: boolean + type: object + x-kubernetes-map-type: atomic + csi: + description: csi (Container Storage Interface) + represents ephemeral storage that is + handled by certain external CSI drivers + (Beta feature). + properties: + driver: + description: |- + driver is the name of the CSI driver that handles this volume. + Consult with your admin for the correct name as registered in the cluster. + type: string + fsType: + description: |- + fsType to mount. Ex. "ext4", "xfs", "ntfs". + If not provided, the empty value is passed to the associated CSI driver + which will determine the default filesystem to apply. + type: string + nodePublishSecretRef: + description: |- + nodePublishSecretRef is a reference to the secret object containing + sensitive information to pass to the CSI driver to complete the CSI + NodePublishVolume and NodeUnpublishVolume calls. + This field is optional, and may be empty if no secret is required. If the + secret object contains more than one secret, all secret references are passed. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + readOnly: + description: |- + readOnly specifies a read-only configuration for the volume. + Defaults to false (read/write). + type: boolean + volumeAttributes: + additionalProperties: + type: string + description: |- + volumeAttributes stores driver-specific properties that are passed to the CSI + driver. Consult your driver's documentation for supported values. + type: object + required: + - driver + type: object + downwardAPI: + description: downwardAPI represents downward + API about the pod that should populate + this volume + properties: + defaultMode: + description: |- + Optional: mode bits to use on created files by default. Must be a + Optional: mode bits used to set permissions on created files by default. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + Defaults to 0644. + Directories within the path are not affected by this setting. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + items: + description: Items is a list of downward + API volume file + items: + description: DownwardAPIVolumeFile + represents information to create + the file containing the pod field + properties: + fieldRef: + description: 'Required: Selects + a field of the pod: only annotations, + labels, name, namespace and + uid are supported.' + properties: + apiVersion: + description: Version of + the schema the FieldPath + is written in terms of, + defaults to "v1". + type: string + fieldPath: + description: Path of the + field to select in the + specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + mode: + description: |- + Optional: mode bits used to set permissions on this file, must be an octal value + between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: 'Required: Path + is the relative path name + of the file to be created. + Must not be absolute or contain + the ''..'' path. Must be utf-8 + encoded. The first item of + the relative path must not + start with ''..''' + type: string + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, requests.cpu and requests.memory) are currently supported. + properties: + containerName: + description: 'Container + name: required for volumes, + optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the + output format of the exposed + resources, defaults to + "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: + resource to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + required: + - path + type: object + type: array + x-kubernetes-list-type: atomic + type: object + emptyDir: + description: |- + emptyDir represents a temporary directory that shares a pod's lifetime. + More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir + properties: + medium: + description: |- + medium represents what type of storage medium should back this directory. + The default is "" which means to use the node's default medium. + Must be an empty string (default) or Memory. + More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir + type: string + sizeLimit: + anyOf: + - type: integer + - type: string + description: |- + sizeLimit is the total amount of local storage required for this EmptyDir volume. + The size limit is also applicable for memory medium. + The maximum usage on memory medium EmptyDir would be the minimum value between + the SizeLimit specified here and the sum of memory limits of all containers in a pod. + The default is nil which means that the limit is undefined. + More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + ephemeral: + description: |- + ephemeral represents a volume that is handled by a cluster storage driver. + The volume's lifecycle is tied to the pod that defines it - it will be created before the pod starts, + and deleted when the pod is removed. + + Use this if: + a) the volume is only needed while the pod runs, + b) features of normal volumes like restoring from snapshot or capacity + tracking are needed, + c) the storage driver is specified through a storage class, and + d) the storage driver supports dynamic volume provisioning through + a PersistentVolumeClaim (see EphemeralVolumeSource for more + information on the connection between this volume type + and PersistentVolumeClaim). + + Use PersistentVolumeClaim or one of the vendor-specific + APIs for volumes that persist for longer than the lifecycle + of an individual pod. + + Use CSI for light-weight local ephemeral volumes if the CSI driver is meant to + be used that way - see the documentation of the driver for + more information. + + A pod can use both types of ephemeral volumes and + persistent volumes at the same time. + properties: + volumeClaimTemplate: + description: |- + Will be used to create a stand-alone PVC to provision the volume. + The pod in which this EphemeralVolumeSource is embedded will be the + owner of the PVC, i.e. the PVC will be deleted together with the + pod. The name of the PVC will be `-` where + `` is the name from the `PodSpec.Volumes` array + entry. Pod validation will reject the pod if the concatenated name + is not valid for a PVC (for example, too long). + + An existing PVC with that name that is not owned by the pod + will *not* be used for the pod to avoid using an unrelated + volume by mistake. Starting the pod is then blocked until + the unrelated PVC is removed. If such a pre-created PVC is + meant to be used by the pod, the PVC has to updated with an + owner reference to the pod once the pod exists. Normally + this should not be necessary, but it may be useful when + manually reconstructing a broken cluster. + + This field is read-only and no changes will be made by Kubernetes + to the PVC after it has been created. + + Required, must not be nil. + properties: + metadata: + description: |- + May contain labels and annotations that will be copied into the PVC + when creating it. No other fields are allowed and will be rejected during + validation. + type: object + spec: + description: |- + The specification for the PersistentVolumeClaim. The entire content is + copied unchanged into the PVC that gets created from this + template. The same fields as in a PersistentVolumeClaim + are also valid here. + properties: + accessModes: + description: |- + accessModes contains the desired access modes the volume should have. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1 + items: + type: string + type: array + x-kubernetes-list-type: atomic + dataSource: + description: |- + dataSource field can be used to specify either: + * An existing VolumeSnapshot object (snapshot.storage.k8s.io/VolumeSnapshot) + * An existing PVC (PersistentVolumeClaim) + If the provisioner or an external controller can support the specified data source, + it will create a new volume based on the contents of the specified data source. + When the AnyVolumeDataSource feature gate is enabled, dataSource contents will be copied to dataSourceRef, + and dataSourceRef contents will be copied to dataSource when dataSourceRef.namespace is not specified. + If the namespace is specified, then dataSourceRef will not be copied to dataSource. + properties: + apiGroup: + description: |- + APIGroup is the group for the resource being referenced. + If APIGroup is not specified, the specified Kind must be in the core API group. + For any other third-party types, APIGroup is required. + type: string + kind: + description: Kind is the + type of resource being + referenced + type: string + name: + description: Name is the + name of resource being + referenced + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + dataSourceRef: + description: |- + dataSourceRef specifies the object from which to populate the volume with data, if a non-empty + volume is desired. This may be any object from a non-empty API group (non + core object) or a PersistentVolumeClaim object. + When this field is specified, volume binding will only succeed if the type of + the specified object matches some installed volume populator or dynamic + provisioner. + This field will replace the functionality of the dataSource field and as such + if both fields are non-empty, they must have the same value. For backwards + compatibility, when namespace isn't specified in dataSourceRef, + both fields (dataSource and dataSourceRef) will be set to the same + value automatically if one of them is empty and the other is non-empty. + When namespace is specified in dataSourceRef, + dataSource isn't set to the same value and must be empty. + There are three important differences between dataSource and dataSourceRef: + * While dataSource only allows two specific types of objects, dataSourceRef + allows any non-core object, as well as PersistentVolumeClaim objects. + * While dataSource ignores disallowed values (dropping them), dataSourceRef + preserves all values, and generates an error if a disallowed value is + specified. + * While dataSource only allows local objects, dataSourceRef allows objects + in any namespaces. + (Beta) Using this field requires the AnyVolumeDataSource feature gate to be enabled. + (Alpha) Using the namespace field of dataSourceRef requires the CrossNamespaceVolumeDataSource feature gate to be enabled. + properties: + apiGroup: + description: |- + APIGroup is the group for the resource being referenced. + If APIGroup is not specified, the specified Kind must be in the core API group. + For any other third-party types, APIGroup is required. + type: string + kind: + description: Kind is the + type of resource being + referenced + type: string + name: + description: Name is the + name of resource being + referenced + type: string + namespace: + description: |- + Namespace is the namespace of resource being referenced + Note that when a namespace is specified, a gateway.networking.k8s.io/ReferenceGrant object is required in the referent namespace to allow that namespace's owner to accept the reference. See the ReferenceGrant documentation for details. + (Alpha) This field requires the CrossNamespaceVolumeDataSource feature gate to be enabled. + type: string + required: + - kind + - name + type: object + resources: + description: |- + resources represents the minimum resources the volume should have. + If RecoverVolumeExpansionFailure feature is enabled users are allowed to specify resource requirements + that are lower than previous value but must still be higher than capacity recorded in the + status field of the claim. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + selector: + description: selector is a + label query over volumes + to consider for binding. + properties: + matchExpressions: + description: matchExpressions + is a list of label selector + requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key + is the label key + that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + storageClassName: + description: |- + storageClassName is the name of the StorageClass required by the claim. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1 + type: string + volumeAttributesClassName: + description: |- + volumeAttributesClassName may be used to set the VolumeAttributesClass used by this claim. + If specified, the CSI driver will create or update the volume with the attributes defined + in the corresponding VolumeAttributesClass. This has a different purpose than storageClassName, + it can be changed after the claim is created. An empty string value means that no VolumeAttributesClass + will be applied to the claim but it's not allowed to reset this field to empty string once it is set. + If unspecified and the PersistentVolumeClaim is unbound, the default VolumeAttributesClass + will be set by the persistentvolume controller if it exists. + If the resource referred to by volumeAttributesClass does not exist, this PersistentVolumeClaim will be + set to a Pending state, as reflected by the modifyVolumeStatus field, until such as a resource + exists. + More info: https://kubernetes.io/docs/concepts/storage/volume-attributes-classes/ + (Beta) Using this field requires the VolumeAttributesClass feature gate to be enabled (off by default). + type: string + volumeMode: + description: |- + volumeMode defines what type of volume is required by the claim. + Value of Filesystem is implied when not included in claim spec. + type: string + volumeName: + description: volumeName is + the binding reference to + the PersistentVolume backing + this claim. + type: string + type: object + required: + - spec + type: object + type: object + fc: + description: fc represents a Fibre Channel + resource that is attached to a kubelet's + host machine and then exposed to the + pod. + properties: + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + lun: + description: 'lun is Optional: FC + target lun number' + format: int32 + type: integer + readOnly: + description: |- + readOnly is Optional: Defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + targetWWNs: + description: 'targetWWNs is Optional: + FC target worldwide names (WWNs)' + items: + type: string + type: array + x-kubernetes-list-type: atomic + wwids: + description: |- + wwids Optional: FC volume world wide identifiers (wwids) + Either wwids or combination of targetWWNs and lun must be set, but not both simultaneously. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + flexVolume: + description: |- + flexVolume represents a generic volume resource that is + provisioned/attached using an exec based plugin. + properties: + driver: + description: driver is the name of + the driver to use for this volume. + type: string + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". The default filesystem depends on FlexVolume script. + type: string + options: + additionalProperties: + type: string + description: 'options is Optional: + this field holds extra command options + if any.' + type: object + readOnly: + description: |- + readOnly is Optional: defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + secretRef: + description: |- + secretRef is Optional: secretRef is reference to the secret object containing + sensitive information to pass to the plugin scripts. This may be + empty if no secret object is specified. If the secret object + contains more than one secret, all secrets are passed to the plugin + scripts. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + required: + - driver + type: object + flocker: + description: flocker represents a Flocker + volume attached to a kubelet's host + machine. This depends on the Flocker + control service being running + properties: + datasetName: + description: |- + datasetName is Name of the dataset stored as metadata -> name on the dataset for Flocker + should be considered as deprecated + type: string + datasetUUID: + description: datasetUUID is the UUID + of the dataset. This is unique identifier + of a Flocker dataset + type: string + type: object + gcePersistentDisk: + description: |- + gcePersistentDisk represents a GCE Disk resource that is attached to a + kubelet's host machine and then exposed to the pod. + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + properties: + fsType: + description: |- + fsType is filesystem type of the volume that you want to mount. + Tip: Ensure that the filesystem type is supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + type: string + partition: + description: |- + partition is the partition in the volume that you want to mount. + If omitted, the default is to mount by volume name. + Examples: For volume /dev/sda1, you specify the partition as "1". + Similarly, the volume partition for /dev/sda is "0" (or you can leave the property empty). + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + format: int32 + type: integer + pdName: + description: |- + pdName is unique name of the PD resource in GCE. Used to identify the disk in GCE. + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + type: string + readOnly: + description: |- + readOnly here will force the ReadOnly setting in VolumeMounts. + Defaults to false. + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + type: boolean + required: + - pdName + type: object + gitRepo: + description: |- + gitRepo represents a git repository at a particular revision. + DEPRECATED: GitRepo is deprecated. To provision a container with a git repo, mount an + EmptyDir into an InitContainer that clones the repo using git, then mount the EmptyDir + into the Pod's container. + properties: + directory: + description: |- + directory is the target directory name. + Must not contain or start with '..'. If '.' is supplied, the volume directory will be the + git repository. Otherwise, if specified, the volume will contain the git repository in + the subdirectory with the given name. + type: string + repository: + description: repository is the URL + type: string + revision: + description: revision is the commit + hash for the specified revision. + type: string + required: + - repository + type: object + glusterfs: + description: |- + glusterfs represents a Glusterfs mount on the host that shares a pod's lifetime. + More info: https://examples.k8s.io/volumes/glusterfs/README.md + properties: + endpoints: + description: |- + endpoints is the endpoint name that details Glusterfs topology. + More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod + type: string + path: + description: |- + path is the Glusterfs volume path. + More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod + type: string + readOnly: + description: |- + readOnly here will force the Glusterfs volume to be mounted with read-only permissions. + Defaults to false. + More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod + type: boolean + required: + - endpoints + - path + type: object + hostPath: + description: |- + hostPath represents a pre-existing file or directory on the host + machine that is directly exposed to the container. This is generally + used for system agents or other privileged things that are allowed + to see the host machine. Most containers will NOT need this. + More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath + properties: + path: + description: |- + path of the directory on the host. + If the path is a symlink, it will follow the link to the real path. + More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath + type: string + type: + description: |- + type for HostPath Volume + Defaults to "" + More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath + type: string + required: + - path + type: object + image: + description: |- + image represents an OCI object (a container image or artifact) pulled and mounted on the kubelet's host machine. + The volume is resolved at pod startup depending on which PullPolicy value is provided: + + - Always: the kubelet always attempts to pull the reference. Container creation will fail If the pull fails. + - Never: the kubelet never pulls the reference and only uses a local image or artifact. Container creation will fail if the reference isn't present. + - IfNotPresent: the kubelet pulls if the reference isn't already present on disk. Container creation will fail if the reference isn't present and the pull fails. + + The volume gets re-resolved if the pod gets deleted and recreated, which means that new remote content will become available on pod recreation. + A failure to resolve or pull the image during pod startup will block containers from starting and may add significant latency. Failures will be retried using normal volume backoff and will be reported on the pod reason and message. + The types of objects that may be mounted by this volume are defined by the container runtime implementation on a host machine and at minimum must include all valid types supported by the container image field. + The OCI object gets mounted in a single directory (spec.containers[*].volumeMounts.mountPath) by merging the manifest layers in the same way as for container images. + The volume will be mounted read-only (ro) and non-executable files (noexec). + Sub path mounts for containers are not supported (spec.containers[*].volumeMounts.subpath). + The field spec.securityContext.fsGroupChangePolicy has no effect on this volume type. + properties: + pullPolicy: + description: |- + Policy for pulling OCI objects. Possible values are: + Always: the kubelet always attempts to pull the reference. Container creation will fail If the pull fails. + Never: the kubelet never pulls the reference and only uses a local image or artifact. Container creation will fail if the reference isn't present. + IfNotPresent: the kubelet pulls if the reference isn't already present on disk. Container creation will fail if the reference isn't present and the pull fails. + Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. + type: string + reference: + description: |- + Required: Image or artifact reference to be used. + Behaves in the same way as pod.spec.containers[*].image. + Pull secrets will be assembled in the same way as for the container image by looking up node credentials, SA image pull secrets, and pod spec image pull secrets. + More info: https://kubernetes.io/docs/concepts/containers/images + This field is optional to allow higher level config management to default or override + container images in workload controllers like Deployments and StatefulSets. + type: string + type: object + iscsi: + description: |- + iscsi represents an ISCSI Disk resource that is attached to a + kubelet's host machine and then exposed to the pod. + More info: https://examples.k8s.io/volumes/iscsi/README.md + properties: + chapAuthDiscovery: + description: chapAuthDiscovery defines + whether support iSCSI Discovery + CHAP authentication + type: boolean + chapAuthSession: + description: chapAuthSession defines + whether support iSCSI Session CHAP + authentication + type: boolean + fsType: + description: |- + fsType is the filesystem type of the volume that you want to mount. + Tip: Ensure that the filesystem type is supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://kubernetes.io/docs/concepts/storage/volumes#iscsi + type: string + initiatorName: + description: |- + initiatorName is the custom iSCSI Initiator Name. + If initiatorName is specified with iscsiInterface simultaneously, new iSCSI interface + : will be created for the connection. + type: string + iqn: + description: iqn is the target iSCSI + Qualified Name. + type: string + iscsiInterface: + default: default + description: |- + iscsiInterface is the interface Name that uses an iSCSI transport. + Defaults to 'default' (tcp). + type: string + lun: + description: lun represents iSCSI + Target Lun number. + format: int32 + type: integer + portals: + description: |- + portals is the iSCSI Target Portal List. The portal is either an IP or ip_addr:port if the port + is other than default (typically TCP ports 860 and 3260). + items: + type: string + type: array + x-kubernetes-list-type: atomic + readOnly: + description: |- + readOnly here will force the ReadOnly setting in VolumeMounts. + Defaults to false. + type: boolean + secretRef: + description: secretRef is the CHAP + Secret for iSCSI target and initiator + authentication + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + targetPortal: + description: |- + targetPortal is iSCSI Target Portal. The Portal is either an IP or ip_addr:port if the port + is other than default (typically TCP ports 860 and 3260). + type: string + required: + - iqn + - lun + - targetPortal + type: object + name: + description: |- + name of the volume. + Must be a DNS_LABEL and unique within the pod. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + nfs: + description: |- + nfs represents an NFS mount on the host that shares a pod's lifetime + More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs + properties: + path: + description: |- + path that is exported by the NFS server. + More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs + type: string + readOnly: + description: |- + readOnly here will force the NFS export to be mounted with read-only permissions. + Defaults to false. + More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs + type: boolean + server: + description: |- + server is the hostname or IP address of the NFS server. + More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs + type: string + required: + - path + - server + type: object + persistentVolumeClaim: + description: |- + persistentVolumeClaimVolumeSource represents a reference to a + PersistentVolumeClaim in the same namespace. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims + properties: + claimName: + description: |- + claimName is the name of a PersistentVolumeClaim in the same namespace as the pod using this volume. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims + type: string + readOnly: + description: |- + readOnly Will force the ReadOnly setting in VolumeMounts. + Default false. + type: boolean + required: + - claimName + type: object + photonPersistentDisk: + description: photonPersistentDisk represents + a PhotonController persistent disk attached + and mounted on kubelets host machine + properties: + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + pdID: + description: pdID is the ID that identifies + Photon Controller persistent disk + type: string + required: + - pdID + type: object + portworxVolume: + description: portworxVolume represents + a portworx volume attached and mounted + on kubelets host machine + properties: + fsType: + description: |- + fSType represents the filesystem type to mount + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs". Implicitly inferred to be "ext4" if unspecified. + type: string + readOnly: + description: |- + readOnly defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + volumeID: + description: volumeID uniquely identifies + a Portworx volume + type: string + required: + - volumeID + type: object + projected: + description: projected items for all in + one resources secrets, configmaps, and + downward API + properties: + defaultMode: + description: |- + defaultMode are the mode bits used to set permissions on created files by default. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + Directories within the path are not affected by this setting. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + sources: + description: |- + sources is the list of volume projections. Each entry in this list + handles one source. + items: + description: |- + Projection that may be projected along with other supported volume types. + Exactly one of these fields must be set. + properties: + clusterTrustBundle: + description: |- + ClusterTrustBundle allows a pod to access the `.spec.trustBundle` field + of ClusterTrustBundle objects in an auto-updating file. + + Alpha, gated by the ClusterTrustBundleProjection feature gate. + + ClusterTrustBundle objects can either be selected by name, or by the + combination of signer name and a label selector. + + Kubelet performs aggressive normalization of the PEM contents written + into the pod filesystem. Esoteric PEM features such as inter-block + comments and block headers are stripped. Certificates are deduplicated. + The ordering of certificates within the file is arbitrary, and Kubelet + may change the order over time. + properties: + labelSelector: + description: |- + Select all ClusterTrustBundles that match this label selector. Only has + effect if signerName is set. Mutually-exclusive with name. If unset, + interpreted as "match nothing". If set but empty, interpreted as "match + everything". + properties: + matchExpressions: + description: matchExpressions + is a list of label + selector requirements. + The requirements are + ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key + is the label + key that the + selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + name: + description: |- + Select a single ClusterTrustBundle by object name. Mutually-exclusive + with signerName and labelSelector. + type: string + optional: + description: |- + If true, don't block pod startup if the referenced ClusterTrustBundle(s) + aren't available. If using name, then the named ClusterTrustBundle is + allowed not to exist. If using signerName, then the combination of + signerName and labelSelector is allowed to match zero + ClusterTrustBundles. + type: boolean + path: + description: Relative path + from the volume root to + write the bundle. + type: string + signerName: + description: |- + Select all ClusterTrustBundles that match this signer name. + Mutually-exclusive with name. The contents of all selected + ClusterTrustBundles will be unified and deduplicated. + type: string + required: + - path + type: object + configMap: + description: configMap information + about the configMap data to + project + properties: + items: + description: |- + items if unspecified, each key-value pair in the Data field of the referenced + ConfigMap will be projected into the volume as a file whose name is the + key and content is the value. If specified, the listed keys will be + projected into the specified paths, and unlisted keys will not be + present. If a key is specified which is not present in the ConfigMap, + the volume setup will error unless it is marked optional. Paths must be + relative and may not contain the '..' path or start with '..'. + items: + description: Maps a string + key to a path within + a volume. + properties: + key: + description: key is + the key to project. + type: string + mode: + description: |- + mode is Optional: mode bits used to set permissions on this file. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: |- + path is the relative path of the file to map the key to. + May not be an absolute path. + May not contain the path element '..'. + May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: optional specify + whether the ConfigMap + or its keys must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + downwardAPI: + description: downwardAPI information + about the downwardAPI data + to project + properties: + items: + description: Items is a + list of DownwardAPIVolume + file + items: + description: DownwardAPIVolumeFile + represents information + to create the file containing + the pod field + properties: + fieldRef: + description: 'Required: + Selects a field + of the pod: only + annotations, labels, + name, namespace + and uid are supported.' + properties: + apiVersion: + description: Version + of the schema + the FieldPath + is written in + terms of, defaults + to "v1". + type: string + fieldPath: + description: Path + of the field + to select in + the specified + API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + mode: + description: |- + Optional: mode bits used to set permissions on this file, must be an octal value + between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: 'Required: + Path is the relative + path name of the + file to be created. + Must not be absolute + or contain the ''..'' + path. Must be utf-8 + encoded. The first + item of the relative + path must not start + with ''..''' + type: string + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, requests.cpu and requests.memory) are currently supported. + properties: + containerName: + description: 'Container + name: required + for volumes, + optional for + env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies + the output format + of the exposed + resources, defaults + to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: + resource to + select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + required: + - path + type: object + type: array + x-kubernetes-list-type: atomic + type: object + secret: + description: secret information + about the secret data to project + properties: + items: + description: |- + items if unspecified, each key-value pair in the Data field of the referenced + Secret will be projected into the volume as a file whose name is the + key and content is the value. If specified, the listed keys will be + projected into the specified paths, and unlisted keys will not be + present. If a key is specified which is not present in the Secret, + the volume setup will error unless it is marked optional. Paths must be + relative and may not contain the '..' path or start with '..'. + items: + description: Maps a string + key to a path within + a volume. + properties: + key: + description: key is + the key to project. + type: string + mode: + description: |- + mode is Optional: mode bits used to set permissions on this file. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: |- + path is the relative path of the file to map the key to. + May not be an absolute path. + May not contain the path element '..'. + May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: optional field + specify whether the Secret + or its key must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + serviceAccountToken: + description: serviceAccountToken + is information about the serviceAccountToken + data to project + properties: + audience: + description: |- + audience is the intended audience of the token. A recipient of a token + must identify itself with an identifier specified in the audience of the + token, and otherwise should reject the token. The audience defaults to the + identifier of the apiserver. + type: string + expirationSeconds: + description: |- + expirationSeconds is the requested duration of validity of the service + account token. As the token approaches expiration, the kubelet volume + plugin will proactively rotate the service account token. The kubelet will + start trying to rotate the token if the token is older than 80 percent of + its time to live or if the token is older than 24 hours.Defaults to 1 hour + and must be at least 10 minutes. + format: int64 + type: integer + path: + description: |- + path is the path relative to the mount point of the file to project the + token into. + type: string + required: + - path + type: object + type: object + type: array + x-kubernetes-list-type: atomic + type: object + quobyte: + description: quobyte represents a Quobyte + mount on the host that shares a pod's + lifetime + properties: + group: + description: |- + group to map volume access to + Default is no group + type: string + readOnly: + description: |- + readOnly here will force the Quobyte volume to be mounted with read-only permissions. + Defaults to false. + type: boolean + registry: + description: |- + registry represents a single or multiple Quobyte Registry services + specified as a string as host:port pair (multiple entries are separated with commas) + which acts as the central registry for volumes + type: string + tenant: + description: |- + tenant owning the given Quobyte volume in the Backend + Used with dynamically provisioned Quobyte volumes, value is set by the plugin + type: string + user: + description: |- + user to map volume access to + Defaults to serivceaccount user + type: string + volume: + description: volume is a string that + references an already created Quobyte + volume by name. + type: string + required: + - registry + - volume + type: object + rbd: + description: |- + rbd represents a Rados Block Device mount on the host that shares a pod's lifetime. + More info: https://examples.k8s.io/volumes/rbd/README.md + properties: + fsType: + description: |- + fsType is the filesystem type of the volume that you want to mount. + Tip: Ensure that the filesystem type is supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://kubernetes.io/docs/concepts/storage/volumes#rbd + type: string + image: + description: |- + image is the rados image name. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: string + keyring: + default: /etc/ceph/keyring + description: |- + keyring is the path to key ring for RBDUser. + Default is /etc/ceph/keyring. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: string + monitors: + description: |- + monitors is a collection of Ceph monitors. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + items: + type: string + type: array + x-kubernetes-list-type: atomic + pool: + default: rbd + description: |- + pool is the rados pool name. + Default is rbd. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: string + readOnly: + description: |- + readOnly here will force the ReadOnly setting in VolumeMounts. + Defaults to false. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: boolean + secretRef: + description: |- + secretRef is name of the authentication secret for RBDUser. If provided + overrides keyring. + Default is nil. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + user: + default: admin + description: |- + user is the rados user name. + Default is admin. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: string + required: + - image + - monitors + type: object + scaleIO: + description: scaleIO represents a ScaleIO + persistent volume attached and mounted + on Kubernetes nodes. + properties: + fsType: + default: xfs + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". + Default is "xfs". + type: string + gateway: + description: gateway is the host address + of the ScaleIO API Gateway. + type: string + protectionDomain: + description: protectionDomain is the + name of the ScaleIO Protection Domain + for the configured storage. + type: string + readOnly: + description: |- + readOnly Defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + secretRef: + description: |- + secretRef references to the secret for ScaleIO user and other + sensitive information. If this is not provided, Login operation will fail. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + sslEnabled: + description: sslEnabled Flag enable/disable + SSL communication with Gateway, + default false + type: boolean + storageMode: + default: ThinProvisioned + description: |- + storageMode indicates whether the storage for a volume should be ThickProvisioned or ThinProvisioned. + Default is ThinProvisioned. + type: string + storagePool: + description: storagePool is the ScaleIO + Storage Pool associated with the + protection domain. + type: string + system: + description: system is the name of + the storage system as configured + in ScaleIO. + type: string + volumeName: + description: |- + volumeName is the name of a volume already created in the ScaleIO system + that is associated with this volume source. + type: string + required: + - gateway + - secretRef + - system + type: object + secret: + description: |- + secret represents a secret that should populate this volume. + More info: https://kubernetes.io/docs/concepts/storage/volumes#secret + properties: + defaultMode: + description: |- + defaultMode is Optional: mode bits used to set permissions on created files by default. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values + for mode bits. Defaults to 0644. + Directories within the path are not affected by this setting. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + items: + description: |- + items If unspecified, each key-value pair in the Data field of the referenced + Secret will be projected into the volume as a file whose name is the + key and content is the value. If specified, the listed keys will be + projected into the specified paths, and unlisted keys will not be + present. If a key is specified which is not present in the Secret, + the volume setup will error unless it is marked optional. Paths must be + relative and may not contain the '..' path or start with '..'. + items: + description: Maps a string key to + a path within a volume. + properties: + key: + description: key is the key + to project. + type: string + mode: + description: |- + mode is Optional: mode bits used to set permissions on this file. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: |- + path is the relative path of the file to map the key to. + May not be an absolute path. + May not contain the path element '..'. + May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + optional: + description: optional field specify + whether the Secret or its keys must + be defined + type: boolean + secretName: + description: |- + secretName is the name of the secret in the pod's namespace to use. + More info: https://kubernetes.io/docs/concepts/storage/volumes#secret + type: string + type: object + storageos: + description: storageOS represents a StorageOS + volume attached and mounted on Kubernetes + nodes. + properties: + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + readOnly: + description: |- + readOnly defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + secretRef: + description: |- + secretRef specifies the secret to use for obtaining the StorageOS API + credentials. If not specified, default values will be attempted. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + volumeName: + description: |- + volumeName is the human-readable name of the StorageOS volume. Volume + names are only unique within a namespace. + type: string + volumeNamespace: + description: |- + volumeNamespace specifies the scope of the volume within StorageOS. If no + namespace is specified then the Pod's namespace will be used. This allows the + Kubernetes name scoping to be mirrored within StorageOS for tighter integration. + Set VolumeName to any name to override the default behaviour. + Set to "default" if you are not using namespaces within StorageOS. + Namespaces that do not pre-exist within StorageOS will be created. + type: string + type: object + vsphereVolume: + description: vsphereVolume represents + a vSphere volume attached and mounted + on kubelets host machine + properties: + fsType: + description: |- + fsType is filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + storagePolicyID: + description: storagePolicyID is the + storage Policy Based Management + (SPBM) profile ID associated with + the StoragePolicyName. + type: string + storagePolicyName: + description: storagePolicyName is + the storage Policy Based Management + (SPBM) profile name. + type: string + volumePath: + description: volumePath is the path + that identifies vSphere volume vmdk + type: string + required: + - volumePath + type: object + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + required: + - containers + type: object + type: object + required: + - groupName + - maxReplicas + - minReplicas + - rayStartParams + - template + type: object + type: array + required: + - headGroupSpec + type: object + runtimeEnvYAML: + description: |- + RuntimeEnvYAML represents the runtime environment configuration + provided as a multi-line YAML string. + type: string + shutdownAfterJobFinishes: + description: ShutdownAfterJobFinishes will determine whether to + delete the ray cluster once rayJob succeed or failed. + type: boolean + submissionMode: + default: K8sJobMode + description: |- + SubmissionMode specifies how RayJob submits the Ray job to the RayCluster. + In "K8sJobMode", the KubeRay operator creates a submitter Kubernetes Job to submit the Ray job. + In "HTTPMode", the KubeRay operator sends a request to the RayCluster to create a Ray job. + type: string + submitterConfig: + description: Configurations of submitter k8s job. + properties: + backoffLimit: + description: BackoffLimit of the submitter k8s job. + format: int32 + type: integer + type: object + submitterPodTemplate: + description: SubmitterPodTemplate is the template for the pod + that will run `ray job submit`. + properties: + metadata: + description: |- + Standard object's metadata. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata + type: object + spec: + description: |- + Specification of the desired behavior of the pod. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status + properties: + activeDeadlineSeconds: + description: |- + Optional duration in seconds the pod may be active on the node relative to + StartTime before the system will actively try to mark it failed and kill associated containers. + Value must be a positive integer. + format: int64 + type: integer + affinity: + description: If specified, the pod's scheduling constraints + properties: + nodeAffinity: + description: Describes node affinity scheduling rules + for the pod. + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node matches the corresponding matchExpressions; the + node(s) with the highest sum are the most preferred. + items: + description: |- + An empty preferred scheduling term matches all objects with implicit weight 0 + (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op). + properties: + preference: + description: A node selector term, associated + with the corresponding weight. + properties: + matchExpressions: + description: A list of node selector + requirements by node's labels. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that + the selector applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + description: A list of node selector + requirements by node's fields. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that + the selector applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + weight: + description: Weight associated with matching + the corresponding nodeSelectorTerm, in + the range 1-100. + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to an update), the system + may or may not try to eventually evict the pod from its node. + properties: + nodeSelectorTerms: + description: Required. A list of node selector + terms. The terms are ORed. + items: + description: |- + A null or empty node selector term matches no objects. The requirements of + them are ANDed. + The TopologySelectorTerm type implements a subset of the NodeSelectorTerm. + properties: + matchExpressions: + description: A list of node selector + requirements by node's labels. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that + the selector applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + description: A list of node selector + requirements by node's fields. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that + the selector applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + type: array + x-kubernetes-list-type: atomic + required: + - nodeSelectorTerms + type: object + x-kubernetes-map-type: atomic + type: object + podAffinity: + description: Describes pod affinity scheduling rules + (e.g. co-locate this pod in the same node, zone, + etc. as some other pod(s)). + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the + node(s) with the highest sum are the most preferred. + items: + description: The weights of all of the matched + WeightedPodAffinityTerm fields are added per-node + to find the most preferred node(s) + properties: + podAffinityTerm: + description: Required. A pod affinity term, + associated with the corresponding weight. + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions is + a list of label selector requirements. + The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is + a list of label selector requirements. + The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + weight: + description: |- + weight associated with matching the corresponding podAffinityTerm, + in the range 1-100. + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to a pod label update), the + system may or may not try to eventually evict the pod from its node. + When there are multiple elements, the lists of nodes corresponding to each + podAffinityTerm are intersected, i.e. all terms must be satisfied. + items: + description: |- + Defines a set of pods (namely those matching the labelSelector + relative to the given namespace(s)) that this pod should be + co-located (affinity) or not co-located (anti-affinity) with, + where co-located is defined as running on a node whose value of + the label with key matches that of any node on which + a pod of the set of pods is running + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The + requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The + requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + podAntiAffinity: + description: Describes pod anti-affinity scheduling + rules (e.g. avoid putting this pod in the same node, + zone, etc. as some other pod(s)). + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the anti-affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling anti-affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the + node(s) with the highest sum are the most preferred. + items: + description: The weights of all of the matched + WeightedPodAffinityTerm fields are added per-node + to find the most preferred node(s) + properties: + podAffinityTerm: + description: Required. A pod affinity term, + associated with the corresponding weight. + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions is + a list of label selector requirements. + The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is + a list of label selector requirements. + The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + weight: + description: |- + weight associated with matching the corresponding podAffinityTerm, + in the range 1-100. + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the anti-affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the anti-affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to a pod label update), the + system may or may not try to eventually evict the pod from its node. + When there are multiple elements, the lists of nodes corresponding to each + podAffinityTerm are intersected, i.e. all terms must be satisfied. + items: + description: |- + Defines a set of pods (namely those matching the labelSelector + relative to the given namespace(s)) that this pod should be + co-located (affinity) or not co-located (anti-affinity) with, + where co-located is defined as running on a node whose value of + the label with key matches that of any node on which + a pod of the set of pods is running + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The + requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + This is a beta field and requires enabling MatchLabelKeysInPodAffinity feature gate (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The + requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + type: object + automountServiceAccountToken: + description: AutomountServiceAccountToken indicates whether + a service account token should be automatically mounted. + type: boolean + containers: + description: |- + List of containers belonging to the pod. + Containers cannot currently be added or removed. + There must be at least one container in a Pod. + Cannot be updated. + items: + description: A single application container that you + want to run within a pod. + properties: + args: + description: |- + Arguments to the entrypoint. + The container image's CMD is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell + items: + type: string + type: array + x-kubernetes-list-type: atomic + command: + description: |- + Entrypoint array. Not executed within a shell. + The container image's ENTRYPOINT is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell + items: + type: string + type: array + x-kubernetes-list-type: atomic + env: + description: |- + List of environment variables to set in the container. + Cannot be updated. + items: + description: EnvVar represents an environment + variable present in a Container. + properties: + name: + description: Name of the environment variable. + Must be a C_IDENTIFIER. + type: string + value: + description: |- + Variable references $(VAR_NAME) are expanded + using the previously defined environment variables in the container and + any service environment variables. If a variable cannot be resolved, + the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. + "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless of whether the variable + exists or not. + Defaults to "". + type: string + valueFrom: + description: Source for the environment variable's + value. Cannot be used if value is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the ConfigMap + or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + description: |- + Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, + spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. + properties: + apiVersion: + description: Version of the schema + the FieldPath is written in terms + of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to + select in the specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. + properties: + containerName: + description: 'Container name: required + for volumes, optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output + format of the exposed resources, + defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to + select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + description: Selects a key of a secret + in the pod's namespace + properties: + key: + description: The key of the secret + to select from. Must be a valid + secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret + or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + envFrom: + description: |- + List of sources to populate environment variables in the container. + The keys defined within a source must be a C_IDENTIFIER. All invalid keys + will be reported as an event when the container is starting. When a key exists in multiple + sources, the value associated with the last source will take precedence. + Values defined by an Env with a duplicate key will take precedence. + Cannot be updated. + items: + description: EnvFromSource represents the source + of a set of ConfigMaps + properties: + configMapRef: + description: The ConfigMap to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the ConfigMap + must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + description: An optional identifier to prepend + to each key in the ConfigMap. Must be a + C_IDENTIFIER. + type: string + secretRef: + description: The Secret to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret + must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array + x-kubernetes-list-type: atomic + image: + description: |- + Container image name. + More info: https://kubernetes.io/docs/concepts/containers/images + This field is optional to allow higher level config management to default or override + container images in workload controllers like Deployments and StatefulSets. + type: string + imagePullPolicy: + description: |- + Image pull policy. + One of Always, Never, IfNotPresent. + Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/containers/images#updating-images + type: string + lifecycle: + description: |- + Actions that the management system should take in response to container lifecycle events. + Cannot be updated. + properties: + postStart: + description: |- + PostStart is called immediately after a container is created. If the handler fails, + the container is terminated and restarted according to its restart policy. + Other management of the container blocks until the hook completes. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies the action to + take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies the http + request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in + the request. HTTP allows repeated + headers. + items: + description: HTTPHeader describes + a custom header to be used in HTTP + probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field + value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP + server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents the duration + that the container should sleep before + being terminated. + properties: + seconds: + description: Seconds is the number of + seconds to sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for the backward compatibility. There are no validation of this field and + lifecycle hooks will fail in runtime when tcp handler is specified. + properties: + host: + description: 'Optional: Host name to + connect to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + preStop: + description: |- + PreStop is called immediately before a container is terminated due to an + API request or management event such as liveness/startup probe failure, + preemption, resource contention, etc. The handler is not called if the + container crashes or exits. The Pod's termination grace period countdown begins before the + PreStop hook is executed. Regardless of the outcome of the handler, the + container will eventually terminate within the Pod's termination grace + period (unless delayed by finalizers). Other management of the container blocks until the hook completes + or until the termination grace period is reached. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies the action to + take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies the http + request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in + the request. HTTP allows repeated + headers. + items: + description: HTTPHeader describes + a custom header to be used in HTTP + probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field + value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP + server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents the duration + that the container should sleep before + being terminated. + properties: + seconds: + description: Seconds is the number of + seconds to sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for the backward compatibility. There are no validation of this field and + lifecycle hooks will fail in runtime when tcp handler is specified. + properties: + host: + description: 'Optional: Host name to + connect to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + type: object + livenessProbe: + description: |- + Periodic probe of container liveness. + Container will be restarted if the probe fails. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies the action to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving + a GRPC port. + properties: + port: + description: Port number of the gRPC service. + Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http request + to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the + request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom + header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP + server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action involving + a TCP port. + properties: + host: + description: 'Optional: Host name to connect + to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + name: + description: |- + Name of the container specified as a DNS_LABEL. + Each container in a pod must have a unique name (DNS_LABEL). + Cannot be updated. + type: string + ports: + description: |- + List of ports to expose from the container. Not specifying a port here + DOES NOT prevent that port from being exposed. Any port which is + listening on the default "0.0.0.0" address inside a container will be + accessible from the network. + Modifying this array with strategic merge patch may corrupt the data. + For more information See https://github.com/kubernetes/kubernetes/issues/108255. + Cannot be updated. + items: + description: ContainerPort represents a network + port in a single container. + properties: + containerPort: + description: |- + Number of port to expose on the pod's IP address. + This must be a valid port number, 0 < x < 65536. + format: int32 + type: integer + hostIP: + description: What host IP to bind the external + port to. + type: string + hostPort: + description: |- + Number of port to expose on the host. + If specified, this must be a valid port number, 0 < x < 65536. + If HostNetwork is specified, this must match ContainerPort. + Most containers do not need this. + format: int32 + type: integer + name: + description: |- + If specified, this must be an IANA_SVC_NAME and unique within the pod. Each + named port in a pod must have a unique name. Name for the port that can be + referred to by services. + type: string + protocol: + default: TCP + description: |- + Protocol for port. Must be UDP, TCP, or SCTP. + Defaults to "TCP". + type: string + required: + - containerPort + type: object + type: array + x-kubernetes-list-map-keys: + - containerPort + - protocol + x-kubernetes-list-type: map + readinessProbe: + description: |- + Periodic probe of container service readiness. + Container will be removed from service endpoints if the probe fails. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies the action to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving + a GRPC port. + properties: + port: + description: Port number of the gRPC service. + Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http request + to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the + request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom + header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP + server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action involving + a TCP port. + properties: + host: + description: 'Optional: Host name to connect + to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + resizePolicy: + description: Resources resize policy for the container. + items: + description: ContainerResizePolicy represents + resource resize policy for the container. + properties: + resourceName: + description: |- + Name of the resource to which this resource resize policy applies. + Supported values: cpu, memory. + type: string + restartPolicy: + description: |- + Restart policy to apply when specified resource is resized. + If not specified, it defaults to NotRequired. + type: string + required: + - resourceName + - restartPolicy + type: object + type: array + x-kubernetes-list-type: atomic + resources: + description: |- + Compute Resources required by this container. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one + entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + restartPolicy: + description: |- + RestartPolicy defines the restart behavior of individual containers in a pod. + This field may only be set for init containers, and the only allowed value is "Always". + For non-init containers or when this field is not specified, + the restart behavior is defined by the Pod's restart policy and the container type. + Setting the RestartPolicy as "Always" for the init container will have the following effect: + this init container will be continually restarted on + exit until all regular containers have terminated. Once all regular + containers have completed, all init containers with restartPolicy "Always" + will be shut down. This lifecycle differs from normal init containers and + is often referred to as a "sidecar" container. Although this init + container still starts in the init container sequence, it does not wait + for the container to complete before proceeding to the next init + container. Instead, the next init container starts immediately after this + init container is started, or after any startupProbe has successfully + completed. + type: string + securityContext: + description: |- + SecurityContext defines the security options the container should be run with. + If set, the fields of SecurityContext override the equivalent fields of PodSecurityContext. + More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ + properties: + allowPrivilegeEscalation: + description: |- + AllowPrivilegeEscalation controls whether a process can gain more + privileges than its parent process. This bool directly controls if + the no_new_privs flag will be set on the container process. + AllowPrivilegeEscalation is true always when the container is: + 1) run as Privileged + 2) has CAP_SYS_ADMIN + Note that this field cannot be set when spec.os.name is windows. + type: boolean + appArmorProfile: + description: |- + appArmorProfile is the AppArmor options to use by this container. If set, this profile + overrides the pod's appArmorProfile. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile loaded on the node that should be used. + The profile must be preconfigured on the node to work. + Must match the loaded name of the profile. + Must be set if and only if type is "Localhost". + type: string + type: + description: |- + type indicates which kind of AppArmor profile will be applied. + Valid options are: + Localhost - a profile pre-loaded on the node. + RuntimeDefault - the container runtime's default profile. + Unconfined - no AppArmor enforcement. + type: string + required: + - type + type: object + capabilities: + description: |- + The capabilities to add/drop when running containers. + Defaults to the default set of capabilities granted by the container runtime. + Note that this field cannot be set when spec.os.name is windows. + properties: + add: + description: Added capabilities + items: + description: Capability represent POSIX + capabilities type + type: string + type: array + x-kubernetes-list-type: atomic + drop: + description: Removed capabilities + items: + description: Capability represent POSIX + capabilities type + type: string + type: array + x-kubernetes-list-type: atomic + type: object + privileged: + description: |- + Run container in privileged mode. + Processes in privileged containers are essentially equivalent to root on the host. + Defaults to false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + procMount: + description: |- + procMount denotes the type of proc mount to use for the containers. + The default value is Default which uses the container runtime defaults for + readonly paths and masked paths. + This requires the ProcMountType feature flag to be enabled. + Note that this field cannot be set when spec.os.name is windows. + type: string + readOnlyRootFilesystem: + description: |- + Whether this container has a read-only root filesystem. + Default is false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + runAsGroup: + description: |- + The GID to run the entrypoint of the container process. + Uses runtime default if unset. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + runAsNonRoot: + description: |- + Indicates that the container must run as a non-root user. + If true, the Kubelet will validate the image at runtime to ensure that it + does not run as UID 0 (root) and fail to start the container if it does. + If unset or false, no such validation will be performed. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: |- + The UID to run the entrypoint of the container process. + Defaults to user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + seLinuxOptions: + description: |- + The SELinux context to be applied to the container. + If unspecified, the container runtime will allocate a random SELinux context for each + container. May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + properties: + level: + description: Level is SELinux level label + that applies to the container. + type: string + role: + description: Role is a SELinux role label + that applies to the container. + type: string + type: + description: Type is a SELinux type label + that applies to the container. + type: string + user: + description: User is a SELinux user label + that applies to the container. + type: string + type: object + seccompProfile: + description: |- + The seccomp options to use by this container. If seccomp options are + provided at both the pod & container level, the container options + override the pod options. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile defined in a file on the node should be used. + The profile must be preconfigured on the node to work. + Must be a descending path, relative to the kubelet's configured seccomp profile location. + Must be set if type is "Localhost". Must NOT be set for any other type. + type: string + type: + description: |- + type indicates which kind of seccomp profile will be applied. + Valid options are: + + Localhost - a profile defined in a file on the node should be used. + RuntimeDefault - the container runtime default profile should be used. + Unconfined - no profile should be applied. + type: string + required: + - type + type: object + windowsOptions: + description: |- + The Windows specific settings applied to all containers. + If unspecified, the options from the PodSecurityContext will be used. + If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is linux. + properties: + gmsaCredentialSpec: + description: |- + GMSACredentialSpec is where the GMSA admission webhook + (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the + GMSA credential spec named by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName is the + name of the GMSA credential spec to use. + type: string + hostProcess: + description: |- + HostProcess determines if a container should be run as a 'Host Process' container. + All of a Pod's containers must have the same effective HostProcess value + (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). + In addition, if HostProcess is true then HostNetwork must also be set to true. + type: boolean + runAsUserName: + description: |- + The UserName in Windows to run the entrypoint of the container process. + Defaults to the user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: string + type: object + type: object + startupProbe: + description: |- + StartupProbe indicates that the Pod has successfully initialized. + If specified, no other probes are executed until this completes successfully. + If this probe fails, the Pod will be restarted, just as if the livenessProbe failed. + This can be used to provide different probe parameters at the beginning of a Pod's lifecycle, + when it might take a long time to load data or warm a cache, than during steady-state operation. + This cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies the action to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving + a GRPC port. + properties: + port: + description: Port number of the gRPC service. + Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http request + to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the + request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom + header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP + server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action involving + a TCP port. + properties: + host: + description: 'Optional: Host name to connect + to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + stdin: + description: |- + Whether this container should allocate a buffer for stdin in the container runtime. If this + is not set, reads from stdin in the container will always result in EOF. + Default is false. + type: boolean + stdinOnce: + description: |- + Whether the container runtime should close the stdin channel after it has been opened by + a single attach. When stdin is true the stdin stream will remain open across multiple attach + sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the + first client attaches to stdin, and then remains open and accepts data until the client disconnects, + at which time stdin is closed and remains closed until the container is restarted. If this + flag is false, a container processes that reads from stdin will never receive an EOF. + Default is false + type: boolean + terminationMessagePath: + description: |- + Optional: Path at which the file to which the container's termination message + will be written is mounted into the container's filesystem. + Message written is intended to be brief final status, such as an assertion failure message. + Will be truncated by the node if greater than 4096 bytes. The total message length across + all containers will be limited to 12kb. + Defaults to /dev/termination-log. + Cannot be updated. + type: string + terminationMessagePolicy: + description: |- + Indicate how the termination message should be populated. File will use the contents of + terminationMessagePath to populate the container status message on both success and failure. + FallbackToLogsOnError will use the last chunk of container log output if the termination + message file is empty and the container exited with an error. + The log output is limited to 2048 bytes or 80 lines, whichever is smaller. + Defaults to File. + Cannot be updated. + type: string + tty: + description: |- + Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. + Default is false. + type: boolean + volumeDevices: + description: volumeDevices is the list of block + devices to be used by the container. + items: + description: volumeDevice describes a mapping + of a raw block device within a container. + properties: + devicePath: + description: devicePath is the path inside + of the container that the device will be + mapped to. + type: string + name: + description: name must match the name of a + persistentVolumeClaim in the pod + type: string + required: + - devicePath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - devicePath + x-kubernetes-list-type: map + volumeMounts: + description: |- + Pod volumes to mount into the container's filesystem. + Cannot be updated. + items: + description: VolumeMount describes a mounting + of a Volume within a container. + properties: + mountPath: + description: |- + Path within the container at which the volume should be mounted. Must + not contain ':'. + type: string + mountPropagation: + description: |- + mountPropagation determines how mounts are propagated from the host + to container and the other way around. + When not set, MountPropagationNone is used. + This field is beta in 1.10. + When RecursiveReadOnly is set to IfPossible or to Enabled, MountPropagation must be None or unspecified + (which defaults to None). + type: string + name: + description: This must match the Name of a + Volume. + type: string + readOnly: + description: |- + Mounted read-only if true, read-write otherwise (false or unspecified). + Defaults to false. + type: boolean + recursiveReadOnly: + description: |- + RecursiveReadOnly specifies whether read-only mounts should be handled + recursively. + + If ReadOnly is false, this field has no meaning and must be unspecified. + + If ReadOnly is true, and this field is set to Disabled, the mount is not made + recursively read-only. If this field is set to IfPossible, the mount is made + recursively read-only, if it is supported by the container runtime. If this + field is set to Enabled, the mount is made recursively read-only if it is + supported by the container runtime, otherwise the pod will not be started and + an error will be generated to indicate the reason. + + If this field is set to IfPossible or Enabled, MountPropagation must be set to + None (or be unspecified, which defaults to None). + + If this field is not specified, it is treated as an equivalent of Disabled. + type: string + subPath: + description: |- + Path within the volume from which the container's volume should be mounted. + Defaults to "" (volume's root). + type: string + subPathExpr: + description: |- + Expanded path within the volume from which the container's volume should be mounted. + Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. + Defaults to "" (volume's root). + SubPathExpr and SubPath are mutually exclusive. + type: string + required: + - mountPath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - mountPath + x-kubernetes-list-type: map + workingDir: + description: |- + Container's working directory. + If not specified, the container runtime's default will be used, which + might be configured in the container image. + Cannot be updated. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + dnsConfig: + description: |- + Specifies the DNS parameters of a pod. + Parameters specified here will be merged to the generated DNS + configuration based on DNSPolicy. + properties: + nameservers: + description: |- + A list of DNS name server IP addresses. + This will be appended to the base nameservers generated from DNSPolicy. + Duplicated nameservers will be removed. + items: + type: string + type: array + x-kubernetes-list-type: atomic + options: + description: |- + A list of DNS resolver options. + This will be merged with the base options generated from DNSPolicy. + Duplicated entries will be removed. Resolution options given in Options + will override those that appear in the base DNSPolicy. + items: + description: PodDNSConfigOption defines DNS resolver + options of a pod. + properties: + name: + description: Required. + type: string + value: + type: string + type: object + type: array + x-kubernetes-list-type: atomic + searches: + description: |- + A list of DNS search domains for host-name lookup. + This will be appended to the base search paths generated from DNSPolicy. + Duplicated search paths will be removed. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + dnsPolicy: + description: |- + Set DNS policy for the pod. + Defaults to "ClusterFirst". + Valid values are 'ClusterFirstWithHostNet', 'ClusterFirst', 'Default' or 'None'. + DNS parameters given in DNSConfig will be merged with the policy selected with DNSPolicy. + To have DNS options set along with hostNetwork, you have to specify DNS policy + explicitly to 'ClusterFirstWithHostNet'. + type: string + enableServiceLinks: + description: |- + EnableServiceLinks indicates whether information about services should be injected into pod's + environment variables, matching the syntax of Docker links. + Optional: Defaults to true. + type: boolean + ephemeralContainers: + description: |- + List of ephemeral containers run in this pod. Ephemeral containers may be run in an existing + pod to perform user-initiated actions such as debugging. This list cannot be specified when + creating a pod, and it cannot be modified by updating the pod spec. In order to add an + ephemeral container to an existing pod, use the pod's ephemeralcontainers subresource. + items: + description: |- + An EphemeralContainer is a temporary container that you may add to an existing Pod for + user-initiated activities such as debugging. Ephemeral containers have no resource or + scheduling guarantees, and they will not be restarted when they exit or when a Pod is + removed or restarted. The kubelet may evict a Pod if an ephemeral container causes the + Pod to exceed its resource allocation. + + To add an ephemeral container, use the ephemeralcontainers subresource of an existing + Pod. Ephemeral containers may not be removed or restarted. + properties: + args: + description: |- + Arguments to the entrypoint. + The image's CMD is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell + items: + type: string + type: array + x-kubernetes-list-type: atomic + command: + description: |- + Entrypoint array. Not executed within a shell. + The image's ENTRYPOINT is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell + items: + type: string + type: array + x-kubernetes-list-type: atomic + env: + description: |- + List of environment variables to set in the container. + Cannot be updated. + items: + description: EnvVar represents an environment + variable present in a Container. + properties: + name: + description: Name of the environment variable. + Must be a C_IDENTIFIER. + type: string + value: + description: |- + Variable references $(VAR_NAME) are expanded + using the previously defined environment variables in the container and + any service environment variables. If a variable cannot be resolved, + the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. + "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless of whether the variable + exists or not. + Defaults to "". + type: string + valueFrom: + description: Source for the environment variable's + value. Cannot be used if value is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the ConfigMap + or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + description: |- + Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, + spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. + properties: + apiVersion: + description: Version of the schema + the FieldPath is written in terms + of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to + select in the specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. + properties: + containerName: + description: 'Container name: required + for volumes, optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output + format of the exposed resources, + defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to + select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + description: Selects a key of a secret + in the pod's namespace + properties: + key: + description: The key of the secret + to select from. Must be a valid + secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret + or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + envFrom: + description: |- + List of sources to populate environment variables in the container. + The keys defined within a source must be a C_IDENTIFIER. All invalid keys + will be reported as an event when the container is starting. When a key exists in multiple + sources, the value associated with the last source will take precedence. + Values defined by an Env with a duplicate key will take precedence. + Cannot be updated. + items: + description: EnvFromSource represents the source + of a set of ConfigMaps + properties: + configMapRef: + description: The ConfigMap to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the ConfigMap + must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + description: An optional identifier to prepend + to each key in the ConfigMap. Must be a + C_IDENTIFIER. + type: string + secretRef: + description: The Secret to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret + must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array + x-kubernetes-list-type: atomic + image: + description: |- + Container image name. + More info: https://kubernetes.io/docs/concepts/containers/images + type: string + imagePullPolicy: + description: |- + Image pull policy. + One of Always, Never, IfNotPresent. + Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/containers/images#updating-images + type: string + lifecycle: + description: Lifecycle is not allowed for ephemeral + containers. + properties: + postStart: + description: |- + PostStart is called immediately after a container is created. If the handler fails, + the container is terminated and restarted according to its restart policy. + Other management of the container blocks until the hook completes. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies the action to + take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies the http + request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in + the request. HTTP allows repeated + headers. + items: + description: HTTPHeader describes + a custom header to be used in HTTP + probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field + value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP + server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents the duration + that the container should sleep before + being terminated. + properties: + seconds: + description: Seconds is the number of + seconds to sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for the backward compatibility. There are no validation of this field and + lifecycle hooks will fail in runtime when tcp handler is specified. + properties: + host: + description: 'Optional: Host name to + connect to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + preStop: + description: |- + PreStop is called immediately before a container is terminated due to an + API request or management event such as liveness/startup probe failure, + preemption, resource contention, etc. The handler is not called if the + container crashes or exits. The Pod's termination grace period countdown begins before the + PreStop hook is executed. Regardless of the outcome of the handler, the + container will eventually terminate within the Pod's termination grace + period (unless delayed by finalizers). Other management of the container blocks until the hook completes + or until the termination grace period is reached. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies the action to + take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies the http + request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in + the request. HTTP allows repeated + headers. + items: + description: HTTPHeader describes + a custom header to be used in HTTP + probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field + value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP + server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents the duration + that the container should sleep before + being terminated. + properties: + seconds: + description: Seconds is the number of + seconds to sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for the backward compatibility. There are no validation of this field and + lifecycle hooks will fail in runtime when tcp handler is specified. + properties: + host: + description: 'Optional: Host name to + connect to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + type: object + livenessProbe: + description: Probes are not allowed for ephemeral + containers. + properties: + exec: + description: Exec specifies the action to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving + a GRPC port. + properties: + port: + description: Port number of the gRPC service. + Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http request + to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the + request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom + header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP + server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action involving + a TCP port. + properties: + host: + description: 'Optional: Host name to connect + to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + name: + description: |- + Name of the ephemeral container specified as a DNS_LABEL. + This name must be unique among all containers, init containers and ephemeral containers. + type: string + ports: + description: Ports are not allowed for ephemeral + containers. + items: + description: ContainerPort represents a network + port in a single container. + properties: + containerPort: + description: |- + Number of port to expose on the pod's IP address. + This must be a valid port number, 0 < x < 65536. + format: int32 + type: integer + hostIP: + description: What host IP to bind the external + port to. + type: string + hostPort: + description: |- + Number of port to expose on the host. + If specified, this must be a valid port number, 0 < x < 65536. + If HostNetwork is specified, this must match ContainerPort. + Most containers do not need this. + format: int32 + type: integer + name: + description: |- + If specified, this must be an IANA_SVC_NAME and unique within the pod. Each + named port in a pod must have a unique name. Name for the port that can be + referred to by services. + type: string + protocol: + default: TCP + description: |- + Protocol for port. Must be UDP, TCP, or SCTP. + Defaults to "TCP". + type: string + required: + - containerPort + type: object + type: array + x-kubernetes-list-map-keys: + - containerPort + - protocol + x-kubernetes-list-type: map + readinessProbe: + description: Probes are not allowed for ephemeral + containers. + properties: + exec: + description: Exec specifies the action to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving + a GRPC port. + properties: + port: + description: Port number of the gRPC service. + Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http request + to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the + request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom + header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP + server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action involving + a TCP port. + properties: + host: + description: 'Optional: Host name to connect + to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + resizePolicy: + description: Resources resize policy for the container. + items: + description: ContainerResizePolicy represents + resource resize policy for the container. + properties: + resourceName: + description: |- + Name of the resource to which this resource resize policy applies. + Supported values: cpu, memory. + type: string + restartPolicy: + description: |- + Restart policy to apply when specified resource is resized. + If not specified, it defaults to NotRequired. + type: string + required: + - resourceName + - restartPolicy + type: object + type: array + x-kubernetes-list-type: atomic + resources: + description: |- + Resources are not allowed for ephemeral containers. Ephemeral containers use spare resources + already allocated to the pod. + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one + entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + restartPolicy: + description: |- + Restart policy for the container to manage the restart behavior of each + container within a pod. + This may only be set for init containers. You cannot set this field on + ephemeral containers. + type: string + securityContext: + description: |- + Optional: SecurityContext defines the security options the ephemeral container should be run with. + If set, the fields of SecurityContext override the equivalent fields of PodSecurityContext. + properties: + allowPrivilegeEscalation: + description: |- + AllowPrivilegeEscalation controls whether a process can gain more + privileges than its parent process. This bool directly controls if + the no_new_privs flag will be set on the container process. + AllowPrivilegeEscalation is true always when the container is: + 1) run as Privileged + 2) has CAP_SYS_ADMIN + Note that this field cannot be set when spec.os.name is windows. + type: boolean + appArmorProfile: + description: |- + appArmorProfile is the AppArmor options to use by this container. If set, this profile + overrides the pod's appArmorProfile. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile loaded on the node that should be used. + The profile must be preconfigured on the node to work. + Must match the loaded name of the profile. + Must be set if and only if type is "Localhost". + type: string + type: + description: |- + type indicates which kind of AppArmor profile will be applied. + Valid options are: + Localhost - a profile pre-loaded on the node. + RuntimeDefault - the container runtime's default profile. + Unconfined - no AppArmor enforcement. + type: string + required: + - type + type: object + capabilities: + description: |- + The capabilities to add/drop when running containers. + Defaults to the default set of capabilities granted by the container runtime. + Note that this field cannot be set when spec.os.name is windows. + properties: + add: + description: Added capabilities + items: + description: Capability represent POSIX + capabilities type + type: string + type: array + x-kubernetes-list-type: atomic + drop: + description: Removed capabilities + items: + description: Capability represent POSIX + capabilities type + type: string + type: array + x-kubernetes-list-type: atomic + type: object + privileged: + description: |- + Run container in privileged mode. + Processes in privileged containers are essentially equivalent to root on the host. + Defaults to false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + procMount: + description: |- + procMount denotes the type of proc mount to use for the containers. + The default value is Default which uses the container runtime defaults for + readonly paths and masked paths. + This requires the ProcMountType feature flag to be enabled. + Note that this field cannot be set when spec.os.name is windows. + type: string + readOnlyRootFilesystem: + description: |- + Whether this container has a read-only root filesystem. + Default is false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + runAsGroup: + description: |- + The GID to run the entrypoint of the container process. + Uses runtime default if unset. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + runAsNonRoot: + description: |- + Indicates that the container must run as a non-root user. + If true, the Kubelet will validate the image at runtime to ensure that it + does not run as UID 0 (root) and fail to start the container if it does. + If unset or false, no such validation will be performed. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: |- + The UID to run the entrypoint of the container process. + Defaults to user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + seLinuxOptions: + description: |- + The SELinux context to be applied to the container. + If unspecified, the container runtime will allocate a random SELinux context for each + container. May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + properties: + level: + description: Level is SELinux level label + that applies to the container. + type: string + role: + description: Role is a SELinux role label + that applies to the container. + type: string + type: + description: Type is a SELinux type label + that applies to the container. + type: string + user: + description: User is a SELinux user label + that applies to the container. + type: string + type: object + seccompProfile: + description: |- + The seccomp options to use by this container. If seccomp options are + provided at both the pod & container level, the container options + override the pod options. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile defined in a file on the node should be used. + The profile must be preconfigured on the node to work. + Must be a descending path, relative to the kubelet's configured seccomp profile location. + Must be set if type is "Localhost". Must NOT be set for any other type. + type: string + type: + description: |- + type indicates which kind of seccomp profile will be applied. + Valid options are: + + Localhost - a profile defined in a file on the node should be used. + RuntimeDefault - the container runtime default profile should be used. + Unconfined - no profile should be applied. + type: string + required: + - type + type: object + windowsOptions: + description: |- + The Windows specific settings applied to all containers. + If unspecified, the options from the PodSecurityContext will be used. + If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is linux. + properties: + gmsaCredentialSpec: + description: |- + GMSACredentialSpec is where the GMSA admission webhook + (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the + GMSA credential spec named by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName is the + name of the GMSA credential spec to use. + type: string + hostProcess: + description: |- + HostProcess determines if a container should be run as a 'Host Process' container. + All of a Pod's containers must have the same effective HostProcess value + (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). + In addition, if HostProcess is true then HostNetwork must also be set to true. + type: boolean + runAsUserName: + description: |- + The UserName in Windows to run the entrypoint of the container process. + Defaults to the user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: string + type: object + type: object + startupProbe: + description: Probes are not allowed for ephemeral + containers. + properties: + exec: + description: Exec specifies the action to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving + a GRPC port. + properties: + port: + description: Port number of the gRPC service. + Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http request + to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the + request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom + header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP + server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action involving + a TCP port. + properties: + host: + description: 'Optional: Host name to connect + to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + stdin: + description: |- + Whether this container should allocate a buffer for stdin in the container runtime. If this + is not set, reads from stdin in the container will always result in EOF. + Default is false. + type: boolean + stdinOnce: + description: |- + Whether the container runtime should close the stdin channel after it has been opened by + a single attach. When stdin is true the stdin stream will remain open across multiple attach + sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the + first client attaches to stdin, and then remains open and accepts data until the client disconnects, + at which time stdin is closed and remains closed until the container is restarted. If this + flag is false, a container processes that reads from stdin will never receive an EOF. + Default is false + type: boolean + targetContainerName: + description: |- + If set, the name of the container from PodSpec that this ephemeral container targets. + The ephemeral container will be run in the namespaces (IPC, PID, etc) of this container. + If not set then the ephemeral container uses the namespaces configured in the Pod spec. + + The container runtime must implement support for this feature. If the runtime does not + support namespace targeting then the result of setting this field is undefined. + type: string + terminationMessagePath: + description: |- + Optional: Path at which the file to which the container's termination message + will be written is mounted into the container's filesystem. + Message written is intended to be brief final status, such as an assertion failure message. + Will be truncated by the node if greater than 4096 bytes. The total message length across + all containers will be limited to 12kb. + Defaults to /dev/termination-log. + Cannot be updated. + type: string + terminationMessagePolicy: + description: |- + Indicate how the termination message should be populated. File will use the contents of + terminationMessagePath to populate the container status message on both success and failure. + FallbackToLogsOnError will use the last chunk of container log output if the termination + message file is empty and the container exited with an error. + The log output is limited to 2048 bytes or 80 lines, whichever is smaller. + Defaults to File. + Cannot be updated. + type: string + tty: + description: |- + Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. + Default is false. + type: boolean + volumeDevices: + description: volumeDevices is the list of block + devices to be used by the container. + items: + description: volumeDevice describes a mapping + of a raw block device within a container. + properties: + devicePath: + description: devicePath is the path inside + of the container that the device will be + mapped to. + type: string + name: + description: name must match the name of a + persistentVolumeClaim in the pod + type: string + required: + - devicePath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - devicePath + x-kubernetes-list-type: map + volumeMounts: + description: |- + Pod volumes to mount into the container's filesystem. Subpath mounts are not allowed for ephemeral containers. + Cannot be updated. + items: + description: VolumeMount describes a mounting + of a Volume within a container. + properties: + mountPath: + description: |- + Path within the container at which the volume should be mounted. Must + not contain ':'. + type: string + mountPropagation: + description: |- + mountPropagation determines how mounts are propagated from the host + to container and the other way around. + When not set, MountPropagationNone is used. + This field is beta in 1.10. + When RecursiveReadOnly is set to IfPossible or to Enabled, MountPropagation must be None or unspecified + (which defaults to None). + type: string + name: + description: This must match the Name of a + Volume. + type: string + readOnly: + description: |- + Mounted read-only if true, read-write otherwise (false or unspecified). + Defaults to false. + type: boolean + recursiveReadOnly: + description: |- + RecursiveReadOnly specifies whether read-only mounts should be handled + recursively. + + If ReadOnly is false, this field has no meaning and must be unspecified. + + If ReadOnly is true, and this field is set to Disabled, the mount is not made + recursively read-only. If this field is set to IfPossible, the mount is made + recursively read-only, if it is supported by the container runtime. If this + field is set to Enabled, the mount is made recursively read-only if it is + supported by the container runtime, otherwise the pod will not be started and + an error will be generated to indicate the reason. + + If this field is set to IfPossible or Enabled, MountPropagation must be set to + None (or be unspecified, which defaults to None). + + If this field is not specified, it is treated as an equivalent of Disabled. + type: string + subPath: + description: |- + Path within the volume from which the container's volume should be mounted. + Defaults to "" (volume's root). + type: string + subPathExpr: + description: |- + Expanded path within the volume from which the container's volume should be mounted. + Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. + Defaults to "" (volume's root). + SubPathExpr and SubPath are mutually exclusive. + type: string + required: + - mountPath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - mountPath + x-kubernetes-list-type: map + workingDir: + description: |- + Container's working directory. + If not specified, the container runtime's default will be used, which + might be configured in the container image. + Cannot be updated. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + hostAliases: + description: |- + HostAliases is an optional list of hosts and IPs that will be injected into the pod's hosts + file if specified. + items: + description: |- + HostAlias holds the mapping between IP and hostnames that will be injected as an entry in the + pod's hosts file. + properties: + hostnames: + description: Hostnames for the above IP address. + items: + type: string + type: array + x-kubernetes-list-type: atomic + ip: + description: IP address of the host file entry. + type: string + required: + - ip + type: object + type: array + x-kubernetes-list-map-keys: + - ip + x-kubernetes-list-type: map + hostIPC: + description: |- + Use the host's ipc namespace. + Optional: Default to false. + type: boolean + hostNetwork: + description: |- + Host networking requested for this pod. Use the host's network namespace. + If this option is set, the ports that will be used must be specified. + Default to false. + type: boolean + hostPID: + description: |- + Use the host's pid namespace. + Optional: Default to false. + type: boolean + hostUsers: + description: |- + Use the host's user namespace. + Optional: Default to true. + If set to true or not present, the pod will be run in the host user namespace, useful + for when the pod needs a feature only available to the host user namespace, such as + loading a kernel module with CAP_SYS_MODULE. + When set to false, a new userns is created for the pod. Setting false is useful for + mitigating container breakout vulnerabilities even allowing users to run their + containers as root without actually having root privileges on the host. + This field is alpha-level and is only honored by servers that enable the UserNamespacesSupport feature. + type: boolean + hostname: + description: |- + Specifies the hostname of the Pod + If not specified, the pod's hostname will be set to a system-defined value. + type: string + imagePullSecrets: + description: |- + ImagePullSecrets is an optional list of references to secrets in the same namespace to use for pulling any of the images used by this PodSpec. + If specified, these secrets will be passed to individual puller implementations for them to use. + More info: https://kubernetes.io/docs/concepts/containers/images#specifying-imagepullsecrets-on-a-pod + items: + description: |- + LocalObjectReference contains enough information to let you locate the + referenced object inside the same namespace. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + initContainers: + description: |- + List of initialization containers belonging to the pod. + Init containers are executed in order prior to containers being started. If any + init container fails, the pod is considered to have failed and is handled according + to its restartPolicy. The name for an init container or normal container must be + unique among all containers. + Init containers may not have Lifecycle actions, Readiness probes, Liveness probes, or Startup probes. + The resourceRequirements of an init container are taken into account during scheduling + by finding the highest request/limit for each resource type, and then using the max of + of that value or the sum of the normal containers. Limits are applied to init containers + in a similar fashion. + Init containers cannot currently be added or removed. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/init-containers/ + items: + description: A single application container that you + want to run within a pod. + properties: + args: + description: |- + Arguments to the entrypoint. + The container image's CMD is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell + items: + type: string + type: array + x-kubernetes-list-type: atomic + command: + description: |- + Entrypoint array. Not executed within a shell. + The container image's ENTRYPOINT is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell + items: + type: string + type: array + x-kubernetes-list-type: atomic + env: + description: |- + List of environment variables to set in the container. + Cannot be updated. + items: + description: EnvVar represents an environment + variable present in a Container. + properties: + name: + description: Name of the environment variable. + Must be a C_IDENTIFIER. + type: string + value: + description: |- + Variable references $(VAR_NAME) are expanded + using the previously defined environment variables in the container and + any service environment variables. If a variable cannot be resolved, + the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. + "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless of whether the variable + exists or not. + Defaults to "". + type: string + valueFrom: + description: Source for the environment variable's + value. Cannot be used if value is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the ConfigMap + or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + description: |- + Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, + spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. + properties: + apiVersion: + description: Version of the schema + the FieldPath is written in terms + of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to + select in the specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. + properties: + containerName: + description: 'Container name: required + for volumes, optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output + format of the exposed resources, + defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to + select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + description: Selects a key of a secret + in the pod's namespace + properties: + key: + description: The key of the secret + to select from. Must be a valid + secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret + or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + envFrom: + description: |- + List of sources to populate environment variables in the container. + The keys defined within a source must be a C_IDENTIFIER. All invalid keys + will be reported as an event when the container is starting. When a key exists in multiple + sources, the value associated with the last source will take precedence. + Values defined by an Env with a duplicate key will take precedence. + Cannot be updated. + items: + description: EnvFromSource represents the source + of a set of ConfigMaps + properties: + configMapRef: + description: The ConfigMap to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the ConfigMap + must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + description: An optional identifier to prepend + to each key in the ConfigMap. Must be a + C_IDENTIFIER. + type: string + secretRef: + description: The Secret to select from + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret + must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array + x-kubernetes-list-type: atomic + image: + description: |- + Container image name. + More info: https://kubernetes.io/docs/concepts/containers/images + This field is optional to allow higher level config management to default or override + container images in workload controllers like Deployments and StatefulSets. + type: string + imagePullPolicy: + description: |- + Image pull policy. + One of Always, Never, IfNotPresent. + Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/containers/images#updating-images + type: string + lifecycle: + description: |- + Actions that the management system should take in response to container lifecycle events. + Cannot be updated. + properties: + postStart: + description: |- + PostStart is called immediately after a container is created. If the handler fails, + the container is terminated and restarted according to its restart policy. + Other management of the container blocks until the hook completes. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies the action to + take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies the http + request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in + the request. HTTP allows repeated + headers. + items: + description: HTTPHeader describes + a custom header to be used in HTTP + probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field + value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP + server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents the duration + that the container should sleep before + being terminated. + properties: + seconds: + description: Seconds is the number of + seconds to sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for the backward compatibility. There are no validation of this field and + lifecycle hooks will fail in runtime when tcp handler is specified. + properties: + host: + description: 'Optional: Host name to + connect to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + preStop: + description: |- + PreStop is called immediately before a container is terminated due to an + API request or management event such as liveness/startup probe failure, + preemption, resource contention, etc. The handler is not called if the + container crashes or exits. The Pod's termination grace period countdown begins before the + PreStop hook is executed. Regardless of the outcome of the handler, the + container will eventually terminate within the Pod's termination grace + period (unless delayed by finalizers). Other management of the container blocks until the hook completes + or until the termination grace period is reached. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + properties: + exec: + description: Exec specifies the action to + take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + description: HTTPGet specifies the http + request to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in + the request. HTTP allows repeated + headers. + items: + description: HTTPHeader describes + a custom header to be used in HTTP + probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field + value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP + server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + sleep: + description: Sleep represents the duration + that the container should sleep before + being terminated. + properties: + seconds: + description: Seconds is the number of + seconds to sleep. + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + description: |- + Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept + for the backward compatibility. There are no validation of this field and + lifecycle hooks will fail in runtime when tcp handler is specified. + properties: + host: + description: 'Optional: Host name to + connect to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + type: object + livenessProbe: + description: |- + Periodic probe of container liveness. + Container will be restarted if the probe fails. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies the action to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving + a GRPC port. + properties: + port: + description: Port number of the gRPC service. + Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http request + to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the + request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom + header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP + server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action involving + a TCP port. + properties: + host: + description: 'Optional: Host name to connect + to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + name: + description: |- + Name of the container specified as a DNS_LABEL. + Each container in a pod must have a unique name (DNS_LABEL). + Cannot be updated. + type: string + ports: + description: |- + List of ports to expose from the container. Not specifying a port here + DOES NOT prevent that port from being exposed. Any port which is + listening on the default "0.0.0.0" address inside a container will be + accessible from the network. + Modifying this array with strategic merge patch may corrupt the data. + For more information See https://github.com/kubernetes/kubernetes/issues/108255. + Cannot be updated. + items: + description: ContainerPort represents a network + port in a single container. + properties: + containerPort: + description: |- + Number of port to expose on the pod's IP address. + This must be a valid port number, 0 < x < 65536. + format: int32 + type: integer + hostIP: + description: What host IP to bind the external + port to. + type: string + hostPort: + description: |- + Number of port to expose on the host. + If specified, this must be a valid port number, 0 < x < 65536. + If HostNetwork is specified, this must match ContainerPort. + Most containers do not need this. + format: int32 + type: integer + name: + description: |- + If specified, this must be an IANA_SVC_NAME and unique within the pod. Each + named port in a pod must have a unique name. Name for the port that can be + referred to by services. + type: string + protocol: + default: TCP + description: |- + Protocol for port. Must be UDP, TCP, or SCTP. + Defaults to "TCP". + type: string + required: + - containerPort + type: object + type: array + x-kubernetes-list-map-keys: + - containerPort + - protocol + x-kubernetes-list-type: map + readinessProbe: + description: |- + Periodic probe of container service readiness. + Container will be removed from service endpoints if the probe fails. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies the action to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving + a GRPC port. + properties: + port: + description: Port number of the gRPC service. + Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http request + to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the + request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom + header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP + server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action involving + a TCP port. + properties: + host: + description: 'Optional: Host name to connect + to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + resizePolicy: + description: Resources resize policy for the container. + items: + description: ContainerResizePolicy represents + resource resize policy for the container. + properties: + resourceName: + description: |- + Name of the resource to which this resource resize policy applies. + Supported values: cpu, memory. + type: string + restartPolicy: + description: |- + Restart policy to apply when specified resource is resized. + If not specified, it defaults to NotRequired. + type: string + required: + - resourceName + - restartPolicy + type: object + type: array + x-kubernetes-list-type: atomic + resources: + description: |- + Compute Resources required by this container. + Cannot be updated. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one + entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + restartPolicy: + description: |- + RestartPolicy defines the restart behavior of individual containers in a pod. + This field may only be set for init containers, and the only allowed value is "Always". + For non-init containers or when this field is not specified, + the restart behavior is defined by the Pod's restart policy and the container type. + Setting the RestartPolicy as "Always" for the init container will have the following effect: + this init container will be continually restarted on + exit until all regular containers have terminated. Once all regular + containers have completed, all init containers with restartPolicy "Always" + will be shut down. This lifecycle differs from normal init containers and + is often referred to as a "sidecar" container. Although this init + container still starts in the init container sequence, it does not wait + for the container to complete before proceeding to the next init + container. Instead, the next init container starts immediately after this + init container is started, or after any startupProbe has successfully + completed. + type: string + securityContext: + description: |- + SecurityContext defines the security options the container should be run with. + If set, the fields of SecurityContext override the equivalent fields of PodSecurityContext. + More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ + properties: + allowPrivilegeEscalation: + description: |- + AllowPrivilegeEscalation controls whether a process can gain more + privileges than its parent process. This bool directly controls if + the no_new_privs flag will be set on the container process. + AllowPrivilegeEscalation is true always when the container is: + 1) run as Privileged + 2) has CAP_SYS_ADMIN + Note that this field cannot be set when spec.os.name is windows. + type: boolean + appArmorProfile: + description: |- + appArmorProfile is the AppArmor options to use by this container. If set, this profile + overrides the pod's appArmorProfile. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile loaded on the node that should be used. + The profile must be preconfigured on the node to work. + Must match the loaded name of the profile. + Must be set if and only if type is "Localhost". + type: string + type: + description: |- + type indicates which kind of AppArmor profile will be applied. + Valid options are: + Localhost - a profile pre-loaded on the node. + RuntimeDefault - the container runtime's default profile. + Unconfined - no AppArmor enforcement. + type: string + required: + - type + type: object + capabilities: + description: |- + The capabilities to add/drop when running containers. + Defaults to the default set of capabilities granted by the container runtime. + Note that this field cannot be set when spec.os.name is windows. + properties: + add: + description: Added capabilities + items: + description: Capability represent POSIX + capabilities type + type: string + type: array + x-kubernetes-list-type: atomic + drop: + description: Removed capabilities + items: + description: Capability represent POSIX + capabilities type + type: string + type: array + x-kubernetes-list-type: atomic + type: object + privileged: + description: |- + Run container in privileged mode. + Processes in privileged containers are essentially equivalent to root on the host. + Defaults to false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + procMount: + description: |- + procMount denotes the type of proc mount to use for the containers. + The default value is Default which uses the container runtime defaults for + readonly paths and masked paths. + This requires the ProcMountType feature flag to be enabled. + Note that this field cannot be set when spec.os.name is windows. + type: string + readOnlyRootFilesystem: + description: |- + Whether this container has a read-only root filesystem. + Default is false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + runAsGroup: + description: |- + The GID to run the entrypoint of the container process. + Uses runtime default if unset. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + runAsNonRoot: + description: |- + Indicates that the container must run as a non-root user. + If true, the Kubelet will validate the image at runtime to ensure that it + does not run as UID 0 (root) and fail to start the container if it does. + If unset or false, no such validation will be performed. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: |- + The UID to run the entrypoint of the container process. + Defaults to user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + seLinuxOptions: + description: |- + The SELinux context to be applied to the container. + If unspecified, the container runtime will allocate a random SELinux context for each + container. May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + properties: + level: + description: Level is SELinux level label + that applies to the container. + type: string + role: + description: Role is a SELinux role label + that applies to the container. + type: string + type: + description: Type is a SELinux type label + that applies to the container. + type: string + user: + description: User is a SELinux user label + that applies to the container. + type: string + type: object + seccompProfile: + description: |- + The seccomp options to use by this container. If seccomp options are + provided at both the pod & container level, the container options + override the pod options. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile defined in a file on the node should be used. + The profile must be preconfigured on the node to work. + Must be a descending path, relative to the kubelet's configured seccomp profile location. + Must be set if type is "Localhost". Must NOT be set for any other type. + type: string + type: + description: |- + type indicates which kind of seccomp profile will be applied. + Valid options are: + + Localhost - a profile defined in a file on the node should be used. + RuntimeDefault - the container runtime default profile should be used. + Unconfined - no profile should be applied. + type: string + required: + - type + type: object + windowsOptions: + description: |- + The Windows specific settings applied to all containers. + If unspecified, the options from the PodSecurityContext will be used. + If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is linux. + properties: + gmsaCredentialSpec: + description: |- + GMSACredentialSpec is where the GMSA admission webhook + (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the + GMSA credential spec named by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName is the + name of the GMSA credential spec to use. + type: string + hostProcess: + description: |- + HostProcess determines if a container should be run as a 'Host Process' container. + All of a Pod's containers must have the same effective HostProcess value + (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). + In addition, if HostProcess is true then HostNetwork must also be set to true. + type: boolean + runAsUserName: + description: |- + The UserName in Windows to run the entrypoint of the container process. + Defaults to the user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: string + type: object + type: object + startupProbe: + description: |- + StartupProbe indicates that the Pod has successfully initialized. + If specified, no other probes are executed until this completes successfully. + If this probe fails, the Pod will be restarted, just as if the livenessProbe failed. + This can be used to provide different probe parameters at the beginning of a Pod's lifecycle, + when it might take a long time to load data or warm a cache, than during steady-state operation. + This cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + properties: + exec: + description: Exec specifies the action to take. + properties: + command: + description: |- + Command is the command line to execute inside the container, the working directory for the + command is root ('/') in the container's filesystem. The command is simply exec'd, it is + not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use + a shell, you need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving + a GRPC port. + properties: + port: + description: Port number of the gRPC service. + Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + default: "" + description: |- + Service is the name of the service to place in the gRPC HealthCheckRequest + (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + + If this is not specified, the default behavior is defined by gRPC. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http request + to perform. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the + request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom + header to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP + server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action involving + a TCP port. + properties: + host: + description: 'Optional: Host name to connect + to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this + value overrides the value provided by the pod spec. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + type: object + stdin: + description: |- + Whether this container should allocate a buffer for stdin in the container runtime. If this + is not set, reads from stdin in the container will always result in EOF. + Default is false. + type: boolean + stdinOnce: + description: |- + Whether the container runtime should close the stdin channel after it has been opened by + a single attach. When stdin is true the stdin stream will remain open across multiple attach + sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the + first client attaches to stdin, and then remains open and accepts data until the client disconnects, + at which time stdin is closed and remains closed until the container is restarted. If this + flag is false, a container processes that reads from stdin will never receive an EOF. + Default is false + type: boolean + terminationMessagePath: + description: |- + Optional: Path at which the file to which the container's termination message + will be written is mounted into the container's filesystem. + Message written is intended to be brief final status, such as an assertion failure message. + Will be truncated by the node if greater than 4096 bytes. The total message length across + all containers will be limited to 12kb. + Defaults to /dev/termination-log. + Cannot be updated. + type: string + terminationMessagePolicy: + description: |- + Indicate how the termination message should be populated. File will use the contents of + terminationMessagePath to populate the container status message on both success and failure. + FallbackToLogsOnError will use the last chunk of container log output if the termination + message file is empty and the container exited with an error. + The log output is limited to 2048 bytes or 80 lines, whichever is smaller. + Defaults to File. + Cannot be updated. + type: string + tty: + description: |- + Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. + Default is false. + type: boolean + volumeDevices: + description: volumeDevices is the list of block + devices to be used by the container. + items: + description: volumeDevice describes a mapping + of a raw block device within a container. + properties: + devicePath: + description: devicePath is the path inside + of the container that the device will be + mapped to. + type: string + name: + description: name must match the name of a + persistentVolumeClaim in the pod + type: string + required: + - devicePath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - devicePath + x-kubernetes-list-type: map + volumeMounts: + description: |- + Pod volumes to mount into the container's filesystem. + Cannot be updated. + items: + description: VolumeMount describes a mounting + of a Volume within a container. + properties: + mountPath: + description: |- + Path within the container at which the volume should be mounted. Must + not contain ':'. + type: string + mountPropagation: + description: |- + mountPropagation determines how mounts are propagated from the host + to container and the other way around. + When not set, MountPropagationNone is used. + This field is beta in 1.10. + When RecursiveReadOnly is set to IfPossible or to Enabled, MountPropagation must be None or unspecified + (which defaults to None). + type: string + name: + description: This must match the Name of a + Volume. + type: string + readOnly: + description: |- + Mounted read-only if true, read-write otherwise (false or unspecified). + Defaults to false. + type: boolean + recursiveReadOnly: + description: |- + RecursiveReadOnly specifies whether read-only mounts should be handled + recursively. + + If ReadOnly is false, this field has no meaning and must be unspecified. + + If ReadOnly is true, and this field is set to Disabled, the mount is not made + recursively read-only. If this field is set to IfPossible, the mount is made + recursively read-only, if it is supported by the container runtime. If this + field is set to Enabled, the mount is made recursively read-only if it is + supported by the container runtime, otherwise the pod will not be started and + an error will be generated to indicate the reason. + + If this field is set to IfPossible or Enabled, MountPropagation must be set to + None (or be unspecified, which defaults to None). + + If this field is not specified, it is treated as an equivalent of Disabled. + type: string + subPath: + description: |- + Path within the volume from which the container's volume should be mounted. + Defaults to "" (volume's root). + type: string + subPathExpr: + description: |- + Expanded path within the volume from which the container's volume should be mounted. + Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. + Defaults to "" (volume's root). + SubPathExpr and SubPath are mutually exclusive. + type: string + required: + - mountPath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - mountPath + x-kubernetes-list-type: map + workingDir: + description: |- + Container's working directory. + If not specified, the container runtime's default will be used, which + might be configured in the container image. + Cannot be updated. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + nodeName: + description: |- + NodeName indicates in which node this pod is scheduled. + If empty, this pod is a candidate for scheduling by the scheduler defined in schedulerName. + Once this field is set, the kubelet for this node becomes responsible for the lifecycle of this pod. + This field should not be used to express a desire for the pod to be scheduled on a specific node. + https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodename + type: string + nodeSelector: + additionalProperties: + type: string + description: |- + NodeSelector is a selector which must be true for the pod to fit on a node. + Selector which must match a node's labels for the pod to be scheduled on that node. + More info: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ + type: object + x-kubernetes-map-type: atomic + os: + description: |- + Specifies the OS of the containers in the pod. + Some pod and container fields are restricted if this is set. + + If the OS field is set to linux, the following fields must be unset: + -securityContext.windowsOptions + + If the OS field is set to windows, following fields must be unset: + - spec.hostPID + - spec.hostIPC + - spec.hostUsers + - spec.securityContext.appArmorProfile + - spec.securityContext.seLinuxOptions + - spec.securityContext.seccompProfile + - spec.securityContext.fsGroup + - spec.securityContext.fsGroupChangePolicy + - spec.securityContext.sysctls + - spec.shareProcessNamespace + - spec.securityContext.runAsUser + - spec.securityContext.runAsGroup + - spec.securityContext.supplementalGroups + - spec.securityContext.supplementalGroupsPolicy + - spec.containers[*].securityContext.appArmorProfile + - spec.containers[*].securityContext.seLinuxOptions + - spec.containers[*].securityContext.seccompProfile + - spec.containers[*].securityContext.capabilities + - spec.containers[*].securityContext.readOnlyRootFilesystem + - spec.containers[*].securityContext.privileged + - spec.containers[*].securityContext.allowPrivilegeEscalation + - spec.containers[*].securityContext.procMount + - spec.containers[*].securityContext.runAsUser + - spec.containers[*].securityContext.runAsGroup + properties: + name: + description: |- + Name is the name of the operating system. The currently supported values are linux and windows. + Additional value may be defined in future and can be one of: + https://github.com/opencontainers/runtime-spec/blob/master/config.md#platform-specific-configuration + Clients should expect to handle additional values and treat unrecognized values in this field as os: null + type: string + required: + - name + type: object + overhead: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Overhead represents the resource overhead associated with running a pod for a given RuntimeClass. + This field will be autopopulated at admission time by the RuntimeClass admission controller. If + the RuntimeClass admission controller is enabled, overhead must not be set in Pod create requests. + The RuntimeClass admission controller will reject Pod create requests which have the overhead already + set. If RuntimeClass is configured and selected in the PodSpec, Overhead will be set to the value + defined in the corresponding RuntimeClass, otherwise it will remain unset and treated as zero. + More info: https://git.k8s.io/enhancements/keps/sig-node/688-pod-overhead/README.md + type: object + preemptionPolicy: + description: |- + PreemptionPolicy is the Policy for preempting pods with lower priority. + One of Never, PreemptLowerPriority. + Defaults to PreemptLowerPriority if unset. + type: string + priority: + description: |- + The priority value. Various system components use this field to find the + priority of the pod. When Priority Admission Controller is enabled, it + prevents users from setting this field. The admission controller populates + this field from PriorityClassName. + The higher the value, the higher the priority. + format: int32 + type: integer + priorityClassName: + description: |- + If specified, indicates the pod's priority. "system-node-critical" and + "system-cluster-critical" are two special keywords which indicate the + highest priorities with the former being the highest priority. Any other + name must be defined by creating a PriorityClass object with that name. + If not specified, the pod priority will be default or zero if there is no + default. + type: string + readinessGates: + description: |- + If specified, all readiness gates will be evaluated for pod readiness. + A pod is ready when all its containers are ready AND + all conditions specified in the readiness gates have status equal to "True" + More info: https://git.k8s.io/enhancements/keps/sig-network/580-pod-readiness-gates + items: + description: PodReadinessGate contains the reference + to a pod condition + properties: + conditionType: + description: ConditionType refers to a condition + in the pod's condition list with matching type. + type: string + required: + - conditionType + type: object + type: array + x-kubernetes-list-type: atomic + resourceClaims: + description: |- + ResourceClaims defines which ResourceClaims must be allocated + and reserved before the Pod is allowed to start. The resources + will be made available to those containers which consume them + by name. + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + This field is immutable. + items: + description: |- + PodResourceClaim references exactly one ResourceClaim, either directly + or by naming a ResourceClaimTemplate which is then turned into a ResourceClaim + for the pod. + + It adds a name to it that uniquely identifies the ResourceClaim inside the Pod. + Containers that need access to the ResourceClaim reference it with this name. + properties: + name: + description: |- + Name uniquely identifies this resource claim inside the pod. + This must be a DNS_LABEL. + type: string + resourceClaimName: + description: |- + ResourceClaimName is the name of a ResourceClaim object in the same + namespace as this pod. + + Exactly one of ResourceClaimName and ResourceClaimTemplateName must + be set. + type: string + resourceClaimTemplateName: + description: |- + ResourceClaimTemplateName is the name of a ResourceClaimTemplate + object in the same namespace as this pod. + + The template will be used to create a new ResourceClaim, which will + be bound to this pod. When this pod is deleted, the ResourceClaim + will also be deleted. The pod name and resource name, along with a + generated component, will be used to form a unique name for the + ResourceClaim, which will be recorded in pod.status.resourceClaimStatuses. + + This field is immutable and no changes will be made to the + corresponding ResourceClaim by the control plane after creating the + ResourceClaim. + + Exactly one of ResourceClaimName and ResourceClaimTemplateName must + be set. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + restartPolicy: + description: |- + Restart policy for all containers within the pod. + One of Always, OnFailure, Never. In some contexts, only a subset of those values may be permitted. + Default to Always. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#restart-policy + type: string + runtimeClassName: + description: |- + RuntimeClassName refers to a RuntimeClass object in the node.k8s.io group, which should be used + to run this pod. If no RuntimeClass resource matches the named class, the pod will not be run. + If unset or empty, the "legacy" RuntimeClass will be used, which is an implicit class with an + empty definition that uses the default runtime handler. + More info: https://git.k8s.io/enhancements/keps/sig-node/585-runtime-class + type: string + schedulerName: + description: |- + If specified, the pod will be dispatched by specified scheduler. + If not specified, the pod will be dispatched by default scheduler. + type: string + schedulingGates: + description: |- + SchedulingGates is an opaque list of values that if specified will block scheduling the pod. + If schedulingGates is not empty, the pod will stay in the SchedulingGated state and the + scheduler will not attempt to schedule the pod. + + SchedulingGates can only be set at pod creation time, and be removed only afterwards. + items: + description: PodSchedulingGate is associated to a Pod + to guard its scheduling. + properties: + name: + description: |- + Name of the scheduling gate. + Each scheduling gate must have a unique name field. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + securityContext: + description: |- + SecurityContext holds pod-level security attributes and common container settings. + Optional: Defaults to empty. See type description for default values of each field. + properties: + appArmorProfile: + description: |- + appArmorProfile is the AppArmor options to use by the containers in this pod. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile loaded on the node that should be used. + The profile must be preconfigured on the node to work. + Must match the loaded name of the profile. + Must be set if and only if type is "Localhost". + type: string + type: + description: |- + type indicates which kind of AppArmor profile will be applied. + Valid options are: + Localhost - a profile pre-loaded on the node. + RuntimeDefault - the container runtime's default profile. + Unconfined - no AppArmor enforcement. + type: string + required: + - type + type: object + fsGroup: + description: |- + A special supplemental group that applies to all containers in a pod. + Some volume types allow the Kubelet to change the ownership of that volume + to be owned by the pod: + + 1. The owning GID will be the FSGroup + 2. The setgid bit is set (new files created in the volume will be owned by FSGroup) + 3. The permission bits are OR'd with rw-rw---- + + If unset, the Kubelet will not modify the ownership and permissions of any volume. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + fsGroupChangePolicy: + description: |- + fsGroupChangePolicy defines behavior of changing ownership and permission of the volume + before being exposed inside Pod. This field will only apply to + volume types which support fsGroup based ownership(and permissions). + It will have no effect on ephemeral volume types such as: secret, configmaps + and emptydir. + Valid values are "OnRootMismatch" and "Always". If not specified, "Always" is used. + Note that this field cannot be set when spec.os.name is windows. + type: string + runAsGroup: + description: |- + The GID to run the entrypoint of the container process. + Uses runtime default if unset. + May also be set in SecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence + for that container. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + runAsNonRoot: + description: |- + Indicates that the container must run as a non-root user. + If true, the Kubelet will validate the image at runtime to ensure that it + does not run as UID 0 (root) and fail to start the container if it does. + If unset or false, no such validation will be performed. + May also be set in SecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: |- + The UID to run the entrypoint of the container process. + Defaults to user specified in image metadata if unspecified. + May also be set in SecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence + for that container. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + seLinuxOptions: + description: |- + The SELinux context to be applied to all containers. + If unspecified, the container runtime will allocate a random SELinux context for each + container. May also be set in SecurityContext. If set in + both SecurityContext and PodSecurityContext, the value specified in SecurityContext + takes precedence for that container. + Note that this field cannot be set when spec.os.name is windows. + properties: + level: + description: Level is SELinux level label that + applies to the container. + type: string + role: + description: Role is a SELinux role label that + applies to the container. + type: string + type: + description: Type is a SELinux type label that + applies to the container. + type: string + user: + description: User is a SELinux user label that + applies to the container. + type: string + type: object + seccompProfile: + description: |- + The seccomp options to use by the containers in this pod. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile defined in a file on the node should be used. + The profile must be preconfigured on the node to work. + Must be a descending path, relative to the kubelet's configured seccomp profile location. + Must be set if type is "Localhost". Must NOT be set for any other type. + type: string + type: + description: |- + type indicates which kind of seccomp profile will be applied. + Valid options are: + + Localhost - a profile defined in a file on the node should be used. + RuntimeDefault - the container runtime default profile should be used. + Unconfined - no profile should be applied. + type: string + required: + - type + type: object + supplementalGroups: + description: |- + A list of groups applied to the first process run in each container, in + addition to the container's primary GID and fsGroup (if specified). If + the SupplementalGroupsPolicy feature is enabled, the + supplementalGroupsPolicy field determines whether these are in addition + to or instead of any group memberships defined in the container image. + If unspecified, no additional groups are added, though group memberships + defined in the container image may still be used, depending on the + supplementalGroupsPolicy field. + Note that this field cannot be set when spec.os.name is windows. + items: + format: int64 + type: integer + type: array + x-kubernetes-list-type: atomic + supplementalGroupsPolicy: + description: |- + Defines how supplemental groups of the first container processes are calculated. + Valid values are "Merge" and "Strict". If not specified, "Merge" is used. + (Alpha) Using the field requires the SupplementalGroupsPolicy feature gate to be enabled + and the container runtime must implement support for this feature. + Note that this field cannot be set when spec.os.name is windows. + type: string + sysctls: + description: |- + Sysctls hold a list of namespaced sysctls used for the pod. Pods with unsupported + sysctls (by the container runtime) might fail to launch. + Note that this field cannot be set when spec.os.name is windows. + items: + description: Sysctl defines a kernel parameter to + be set + properties: + name: + description: Name of a property to set + type: string + value: + description: Value of a property to set + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + windowsOptions: + description: |- + The Windows specific settings applied to all containers. + If unspecified, the options within a container's SecurityContext will be used. + If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is linux. + properties: + gmsaCredentialSpec: + description: |- + GMSACredentialSpec is where the GMSA admission webhook + (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the + GMSA credential spec named by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName is the name + of the GMSA credential spec to use. + type: string + hostProcess: + description: |- + HostProcess determines if a container should be run as a 'Host Process' container. + All of a Pod's containers must have the same effective HostProcess value + (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). + In addition, if HostProcess is true then HostNetwork must also be set to true. + type: boolean + runAsUserName: + description: |- + The UserName in Windows to run the entrypoint of the container process. + Defaults to the user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: string + type: object + type: object + serviceAccount: + description: |- + DeprecatedServiceAccount is a deprecated alias for ServiceAccountName. + Deprecated: Use serviceAccountName instead. + type: string + serviceAccountName: + description: |- + ServiceAccountName is the name of the ServiceAccount to use to run this pod. + More info: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ + type: string + setHostnameAsFQDN: + description: |- + If true the pod's hostname will be configured as the pod's FQDN, rather than the leaf name (the default). + In Linux containers, this means setting the FQDN in the hostname field of the kernel (the nodename field of struct utsname). + In Windows containers, this means setting the registry value of hostname for the registry key HKEY_LOCAL_MACHINE\\SYSTEM\\CurrentControlSet\\Services\\Tcpip\\Parameters to FQDN. + If a pod does not have FQDN, this has no effect. + Default to false. + type: boolean + shareProcessNamespace: + description: |- + Share a single process namespace between all of the containers in a pod. + When this is set containers will be able to view and signal processes from other containers + in the same pod, and the first process in each container will not be assigned PID 1. + HostPID and ShareProcessNamespace cannot both be set. + Optional: Default to false. + type: boolean + subdomain: + description: |- + If specified, the fully qualified Pod hostname will be "...svc.". + If not specified, the pod will not have a domainname at all. + type: string + terminationGracePeriodSeconds: + description: |- + Optional duration in seconds the pod needs to terminate gracefully. May be decreased in delete request. + Value must be non-negative integer. The value zero indicates stop immediately via + the kill signal (no opportunity to shut down). + If this value is nil, the default grace period will be used instead. + The grace period is the duration in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly halted with a kill signal. + Set this value longer than the expected cleanup time for your process. + Defaults to 30 seconds. + format: int64 + type: integer + tolerations: + description: If specified, the pod's tolerations. + items: + description: |- + The pod this Toleration is attached to tolerates any taint that matches + the triple using the matching operator . + properties: + effect: + description: |- + Effect indicates the taint effect to match. Empty means match all taint effects. + When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: |- + Key is the taint key that the toleration applies to. Empty means match all taint keys. + If the key is empty, operator must be Exists; this combination means to match all values and all keys. + type: string + operator: + description: |- + Operator represents a key's relationship to the value. + Valid operators are Exists and Equal. Defaults to Equal. + Exists is equivalent to wildcard for value, so that a pod can + tolerate all taints of a particular category. + type: string + tolerationSeconds: + description: |- + TolerationSeconds represents the period of time the toleration (which must be + of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, + it is not set, which means tolerate the taint forever (do not evict). Zero and + negative values will be treated as 0 (evict immediately) by the system. + format: int64 + type: integer + value: + description: |- + Value is the taint value the toleration matches to. + If the operator is Exists, the value should be empty, otherwise just a regular string. + type: string + type: object + type: array + x-kubernetes-list-type: atomic + topologySpreadConstraints: + description: |- + TopologySpreadConstraints describes how a group of pods ought to spread across topology + domains. Scheduler will schedule pods in a way which abides by the constraints. + All topologySpreadConstraints are ANDed. + items: + description: TopologySpreadConstraint specifies how + to spread matching pods among the given topology. + properties: + labelSelector: + description: |- + LabelSelector is used to find matching pods. + Pods that match this label selector are counted to determine the number of pods + in their corresponding topology domain. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that + the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select the pods over which + spreading will be calculated. The keys are used to lookup values from the + incoming pod labels, those key-value labels are ANDed with labelSelector + to select the group of existing pods over which spreading will be calculated + for the incoming pod. The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + MatchLabelKeys cannot be set when LabelSelector isn't set. + Keys that don't exist in the incoming pod labels will + be ignored. A null or empty list means only match against labelSelector. + + This is a beta field and requires the MatchLabelKeysInPodTopologySpread feature gate to be enabled (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + maxSkew: + description: |- + MaxSkew describes the degree to which pods may be unevenly distributed. + When `whenUnsatisfiable=DoNotSchedule`, it is the maximum permitted difference + between the number of matching pods in the target topology and the global minimum. + The global minimum is the minimum number of matching pods in an eligible domain + or zero if the number of eligible domains is less than MinDomains. + For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same + labelSelector spread as 2/2/1: + In this case, the global minimum is 1. + | zone1 | zone2 | zone3 | + | P P | P P | P | + - if MaxSkew is 1, incoming pod can only be scheduled to zone3 to become 2/2/2; + scheduling it onto zone1(zone2) would make the ActualSkew(3-1) on zone1(zone2) + violate MaxSkew(1). + - if MaxSkew is 2, incoming pod can be scheduled onto any zone. + When `whenUnsatisfiable=ScheduleAnyway`, it is used to give higher precedence + to topologies that satisfy it. + It's a required field. Default value is 1 and 0 is not allowed. + format: int32 + type: integer + minDomains: + description: |- + MinDomains indicates a minimum number of eligible domains. + When the number of eligible domains with matching topology keys is less than minDomains, + Pod Topology Spread treats "global minimum" as 0, and then the calculation of Skew is performed. + And when the number of eligible domains with matching topology keys equals or greater than minDomains, + this value has no effect on scheduling. + As a result, when the number of eligible domains is less than minDomains, + scheduler won't schedule more than maxSkew Pods to those domains. + If value is nil, the constraint behaves as if MinDomains is equal to 1. + Valid values are integers greater than 0. + When value is not nil, WhenUnsatisfiable must be DoNotSchedule. + + For example, in a 3-zone cluster, MaxSkew is set to 2, MinDomains is set to 5 and pods with the same + labelSelector spread as 2/2/2: + | zone1 | zone2 | zone3 | + | P P | P P | P P | + The number of domains is less than 5(MinDomains), so "global minimum" is treated as 0. + In this situation, new pod with the same labelSelector cannot be scheduled, + because computed skew will be 3(3 - 0) if new Pod is scheduled to any of the three zones, + it will violate MaxSkew. + format: int32 + type: integer + nodeAffinityPolicy: + description: |- + NodeAffinityPolicy indicates how we will treat Pod's nodeAffinity/nodeSelector + when calculating pod topology spread skew. Options are: + - Honor: only nodes matching nodeAffinity/nodeSelector are included in the calculations. + - Ignore: nodeAffinity/nodeSelector are ignored. All nodes are included in the calculations. + + If this value is nil, the behavior is equivalent to the Honor policy. + This is a beta-level feature default enabled by the NodeInclusionPolicyInPodTopologySpread feature flag. + type: string + nodeTaintsPolicy: + description: |- + NodeTaintsPolicy indicates how we will treat node taints when calculating + pod topology spread skew. Options are: + - Honor: nodes without taints, along with tainted nodes for which the incoming pod + has a toleration, are included. + - Ignore: node taints are ignored. All nodes are included. + + If this value is nil, the behavior is equivalent to the Ignore policy. + This is a beta-level feature default enabled by the NodeInclusionPolicyInPodTopologySpread feature flag. + type: string + topologyKey: + description: |- + TopologyKey is the key of node labels. Nodes that have a label with this key + and identical values are considered to be in the same topology. + We consider each as a "bucket", and try to put balanced number + of pods into each bucket. + We define a domain as a particular instance of a topology. + Also, we define an eligible domain as a domain whose nodes meet the requirements of + nodeAffinityPolicy and nodeTaintsPolicy. + e.g. If TopologyKey is "kubernetes.io/hostname", each Node is a domain of that topology. + And, if TopologyKey is "topology.kubernetes.io/zone", each zone is a domain of that topology. + It's a required field. + type: string + whenUnsatisfiable: + description: |- + WhenUnsatisfiable indicates how to deal with a pod if it doesn't satisfy + the spread constraint. + - DoNotSchedule (default) tells the scheduler not to schedule it. + - ScheduleAnyway tells the scheduler to schedule the pod in any location, + but giving higher precedence to topologies that would help reduce the + skew. + A constraint is considered "Unsatisfiable" for an incoming pod + if and only if every possible node assignment for that pod would violate + "MaxSkew" on some topology. + For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same + labelSelector spread as 3/1/1: + | zone1 | zone2 | zone3 | + | P P P | P | P | + If WhenUnsatisfiable is set to DoNotSchedule, incoming pod can only be scheduled + to zone2(zone3) to become 3/2/1(3/1/2) as ActualSkew(2-1) on zone2(zone3) satisfies + MaxSkew(1). In other words, the cluster can still be imbalanced, but scheduler + won't make it *more* imbalanced. + It's a required field. + type: string + required: + - maxSkew + - topologyKey + - whenUnsatisfiable + type: object + type: array + x-kubernetes-list-map-keys: + - topologyKey + - whenUnsatisfiable + x-kubernetes-list-type: map + volumes: + description: |- + List of volumes that can be mounted by containers belonging to the pod. + More info: https://kubernetes.io/docs/concepts/storage/volumes + items: + description: Volume represents a named volume in a pod + that may be accessed by any container in the pod. + properties: + awsElasticBlockStore: + description: |- + awsElasticBlockStore represents an AWS Disk resource that is attached to a + kubelet's host machine and then exposed to the pod. + More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore + properties: + fsType: + description: |- + fsType is the filesystem type of the volume that you want to mount. + Tip: Ensure that the filesystem type is supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore + type: string + partition: + description: |- + partition is the partition in the volume that you want to mount. + If omitted, the default is to mount by volume name. + Examples: For volume /dev/sda1, you specify the partition as "1". + Similarly, the volume partition for /dev/sda is "0" (or you can leave the property empty). + format: int32 + type: integer + readOnly: + description: |- + readOnly value true will force the readOnly setting in VolumeMounts. + More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore + type: boolean + volumeID: + description: |- + volumeID is unique ID of the persistent disk resource in AWS (Amazon EBS volume). + More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore + type: string + required: + - volumeID + type: object + azureDisk: + description: azureDisk represents an Azure Data + Disk mount on the host and bind mount to the pod. + properties: + cachingMode: + description: 'cachingMode is the Host Caching + mode: None, Read Only, Read Write.' + type: string + diskName: + description: diskName is the Name of the data + disk in the blob storage + type: string + diskURI: + description: diskURI is the URI of data disk + in the blob storage + type: string + fsType: + default: ext4 + description: |- + fsType is Filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + kind: + description: 'kind expected values are Shared: + multiple blob disks per storage account Dedicated: + single blob disk per storage account Managed: + azure managed data disk (only in managed availability + set). defaults to shared' + type: string + readOnly: + default: false + description: |- + readOnly Defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + required: + - diskName + - diskURI + type: object + azureFile: + description: azureFile represents an Azure File + Service mount on the host and bind mount to the + pod. + properties: + readOnly: + description: |- + readOnly defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + secretName: + description: secretName is the name of secret + that contains Azure Storage Account Name and + Key + type: string + shareName: + description: shareName is the azure share Name + type: string + required: + - secretName + - shareName + type: object + cephfs: + description: cephFS represents a Ceph FS mount on + the host that shares a pod's lifetime + properties: + monitors: + description: |- + monitors is Required: Monitors is a collection of Ceph monitors + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + items: + type: string + type: array + x-kubernetes-list-type: atomic + path: + description: 'path is Optional: Used as the + mounted root, rather than the full Ceph tree, + default is /' + type: string + readOnly: + description: |- + readOnly is Optional: Defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + type: boolean + secretFile: + description: |- + secretFile is Optional: SecretFile is the path to key ring for User, default is /etc/ceph/user.secret + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + type: string + secretRef: + description: |- + secretRef is Optional: SecretRef is reference to the authentication secret for User, default is empty. + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + user: + description: |- + user is optional: User is the rados user name, default is admin + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + type: string + required: + - monitors + type: object + cinder: + description: |- + cinder represents a cinder volume attached and mounted on kubelets host machine. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md + properties: + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md + type: string + readOnly: + description: |- + readOnly defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md + type: boolean + secretRef: + description: |- + secretRef is optional: points to a secret object containing parameters used to connect + to OpenStack. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + volumeID: + description: |- + volumeID used to identify the volume in cinder. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md + type: string + required: + - volumeID + type: object + configMap: + description: configMap represents a configMap that + should populate this volume + properties: + defaultMode: + description: |- + defaultMode is optional: mode bits used to set permissions on created files by default. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + Defaults to 0644. + Directories within the path are not affected by this setting. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + items: + description: |- + items if unspecified, each key-value pair in the Data field of the referenced + ConfigMap will be projected into the volume as a file whose name is the + key and content is the value. If specified, the listed keys will be + projected into the specified paths, and unlisted keys will not be + present. If a key is specified which is not present in the ConfigMap, + the volume setup will error unless it is marked optional. Paths must be + relative and may not contain the '..' path or start with '..'. + items: + description: Maps a string key to a path within + a volume. + properties: + key: + description: key is the key to project. + type: string + mode: + description: |- + mode is Optional: mode bits used to set permissions on this file. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: |- + path is the relative path of the file to map the key to. + May not be an absolute path. + May not contain the path element '..'. + May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: optional specify whether the ConfigMap + or its keys must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + csi: + description: csi (Container Storage Interface) represents + ephemeral storage that is handled by certain external + CSI drivers (Beta feature). + properties: + driver: + description: |- + driver is the name of the CSI driver that handles this volume. + Consult with your admin for the correct name as registered in the cluster. + type: string + fsType: + description: |- + fsType to mount. Ex. "ext4", "xfs", "ntfs". + If not provided, the empty value is passed to the associated CSI driver + which will determine the default filesystem to apply. + type: string + nodePublishSecretRef: + description: |- + nodePublishSecretRef is a reference to the secret object containing + sensitive information to pass to the CSI driver to complete the CSI + NodePublishVolume and NodeUnpublishVolume calls. + This field is optional, and may be empty if no secret is required. If the + secret object contains more than one secret, all secret references are passed. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + readOnly: + description: |- + readOnly specifies a read-only configuration for the volume. + Defaults to false (read/write). + type: boolean + volumeAttributes: + additionalProperties: + type: string + description: |- + volumeAttributes stores driver-specific properties that are passed to the CSI + driver. Consult your driver's documentation for supported values. + type: object + required: + - driver + type: object + downwardAPI: + description: downwardAPI represents downward API + about the pod that should populate this volume + properties: + defaultMode: + description: |- + Optional: mode bits to use on created files by default. Must be a + Optional: mode bits used to set permissions on created files by default. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + Defaults to 0644. + Directories within the path are not affected by this setting. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + items: + description: Items is a list of downward API + volume file + items: + description: DownwardAPIVolumeFile represents + information to create the file containing + the pod field + properties: + fieldRef: + description: 'Required: Selects a field + of the pod: only annotations, labels, + name, namespace and uid are supported.' + properties: + apiVersion: + description: Version of the schema + the FieldPath is written in terms + of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to + select in the specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + mode: + description: |- + Optional: mode bits used to set permissions on this file, must be an octal value + between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: 'Required: Path is the relative + path name of the file to be created. + Must not be absolute or contain the + ''..'' path. Must be utf-8 encoded. + The first item of the relative path + must not start with ''..''' + type: string + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, requests.cpu and requests.memory) are currently supported. + properties: + containerName: + description: 'Container name: required + for volumes, optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output + format of the exposed resources, + defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to + select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + required: + - path + type: object + type: array + x-kubernetes-list-type: atomic + type: object + emptyDir: + description: |- + emptyDir represents a temporary directory that shares a pod's lifetime. + More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir + properties: + medium: + description: |- + medium represents what type of storage medium should back this directory. + The default is "" which means to use the node's default medium. + Must be an empty string (default) or Memory. + More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir + type: string + sizeLimit: + anyOf: + - type: integer + - type: string + description: |- + sizeLimit is the total amount of local storage required for this EmptyDir volume. + The size limit is also applicable for memory medium. + The maximum usage on memory medium EmptyDir would be the minimum value between + the SizeLimit specified here and the sum of memory limits of all containers in a pod. + The default is nil which means that the limit is undefined. + More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + ephemeral: + description: |- + ephemeral represents a volume that is handled by a cluster storage driver. + The volume's lifecycle is tied to the pod that defines it - it will be created before the pod starts, + and deleted when the pod is removed. + + Use this if: + a) the volume is only needed while the pod runs, + b) features of normal volumes like restoring from snapshot or capacity + tracking are needed, + c) the storage driver is specified through a storage class, and + d) the storage driver supports dynamic volume provisioning through + a PersistentVolumeClaim (see EphemeralVolumeSource for more + information on the connection between this volume type + and PersistentVolumeClaim). + + Use PersistentVolumeClaim or one of the vendor-specific + APIs for volumes that persist for longer than the lifecycle + of an individual pod. + + Use CSI for light-weight local ephemeral volumes if the CSI driver is meant to + be used that way - see the documentation of the driver for + more information. + + A pod can use both types of ephemeral volumes and + persistent volumes at the same time. + properties: + volumeClaimTemplate: + description: |- + Will be used to create a stand-alone PVC to provision the volume. + The pod in which this EphemeralVolumeSource is embedded will be the + owner of the PVC, i.e. the PVC will be deleted together with the + pod. The name of the PVC will be `-` where + `` is the name from the `PodSpec.Volumes` array + entry. Pod validation will reject the pod if the concatenated name + is not valid for a PVC (for example, too long). + + An existing PVC with that name that is not owned by the pod + will *not* be used for the pod to avoid using an unrelated + volume by mistake. Starting the pod is then blocked until + the unrelated PVC is removed. If such a pre-created PVC is + meant to be used by the pod, the PVC has to updated with an + owner reference to the pod once the pod exists. Normally + this should not be necessary, but it may be useful when + manually reconstructing a broken cluster. + + This field is read-only and no changes will be made by Kubernetes + to the PVC after it has been created. + + Required, must not be nil. + properties: + metadata: + description: |- + May contain labels and annotations that will be copied into the PVC + when creating it. No other fields are allowed and will be rejected during + validation. + type: object + spec: + description: |- + The specification for the PersistentVolumeClaim. The entire content is + copied unchanged into the PVC that gets created from this + template. The same fields as in a PersistentVolumeClaim + are also valid here. + properties: + accessModes: + description: |- + accessModes contains the desired access modes the volume should have. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1 + items: + type: string + type: array + x-kubernetes-list-type: atomic + dataSource: + description: |- + dataSource field can be used to specify either: + * An existing VolumeSnapshot object (snapshot.storage.k8s.io/VolumeSnapshot) + * An existing PVC (PersistentVolumeClaim) + If the provisioner or an external controller can support the specified data source, + it will create a new volume based on the contents of the specified data source. + When the AnyVolumeDataSource feature gate is enabled, dataSource contents will be copied to dataSourceRef, + and dataSourceRef contents will be copied to dataSource when dataSourceRef.namespace is not specified. + If the namespace is specified, then dataSourceRef will not be copied to dataSource. + properties: + apiGroup: + description: |- + APIGroup is the group for the resource being referenced. + If APIGroup is not specified, the specified Kind must be in the core API group. + For any other third-party types, APIGroup is required. + type: string + kind: + description: Kind is the type of + resource being referenced + type: string + name: + description: Name is the name of + resource being referenced + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + dataSourceRef: + description: |- + dataSourceRef specifies the object from which to populate the volume with data, if a non-empty + volume is desired. This may be any object from a non-empty API group (non + core object) or a PersistentVolumeClaim object. + When this field is specified, volume binding will only succeed if the type of + the specified object matches some installed volume populator or dynamic + provisioner. + This field will replace the functionality of the dataSource field and as such + if both fields are non-empty, they must have the same value. For backwards + compatibility, when namespace isn't specified in dataSourceRef, + both fields (dataSource and dataSourceRef) will be set to the same + value automatically if one of them is empty and the other is non-empty. + When namespace is specified in dataSourceRef, + dataSource isn't set to the same value and must be empty. + There are three important differences between dataSource and dataSourceRef: + * While dataSource only allows two specific types of objects, dataSourceRef + allows any non-core object, as well as PersistentVolumeClaim objects. + * While dataSource ignores disallowed values (dropping them), dataSourceRef + preserves all values, and generates an error if a disallowed value is + specified. + * While dataSource only allows local objects, dataSourceRef allows objects + in any namespaces. + (Beta) Using this field requires the AnyVolumeDataSource feature gate to be enabled. + (Alpha) Using the namespace field of dataSourceRef requires the CrossNamespaceVolumeDataSource feature gate to be enabled. + properties: + apiGroup: + description: |- + APIGroup is the group for the resource being referenced. + If APIGroup is not specified, the specified Kind must be in the core API group. + For any other third-party types, APIGroup is required. + type: string + kind: + description: Kind is the type of + resource being referenced + type: string + name: + description: Name is the name of + resource being referenced + type: string + namespace: + description: |- + Namespace is the namespace of resource being referenced + Note that when a namespace is specified, a gateway.networking.k8s.io/ReferenceGrant object is required in the referent namespace to allow that namespace's owner to accept the reference. See the ReferenceGrant documentation for details. + (Alpha) This field requires the CrossNamespaceVolumeDataSource feature gate to be enabled. + type: string + required: + - kind + - name + type: object + resources: + description: |- + resources represents the minimum resources the volume should have. + If RecoverVolumeExpansionFailure feature is enabled users are allowed to specify resource requirements + that are lower than previous value but must still be higher than capacity recorded in the + status field of the claim. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + selector: + description: selector is a label query + over volumes to consider for binding. + properties: + matchExpressions: + description: matchExpressions is + a list of label selector requirements. + The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + storageClassName: + description: |- + storageClassName is the name of the StorageClass required by the claim. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1 + type: string + volumeAttributesClassName: + description: |- + volumeAttributesClassName may be used to set the VolumeAttributesClass used by this claim. + If specified, the CSI driver will create or update the volume with the attributes defined + in the corresponding VolumeAttributesClass. This has a different purpose than storageClassName, + it can be changed after the claim is created. An empty string value means that no VolumeAttributesClass + will be applied to the claim but it's not allowed to reset this field to empty string once it is set. + If unspecified and the PersistentVolumeClaim is unbound, the default VolumeAttributesClass + will be set by the persistentvolume controller if it exists. + If the resource referred to by volumeAttributesClass does not exist, this PersistentVolumeClaim will be + set to a Pending state, as reflected by the modifyVolumeStatus field, until such as a resource + exists. + More info: https://kubernetes.io/docs/concepts/storage/volume-attributes-classes/ + (Beta) Using this field requires the VolumeAttributesClass feature gate to be enabled (off by default). + type: string + volumeMode: + description: |- + volumeMode defines what type of volume is required by the claim. + Value of Filesystem is implied when not included in claim spec. + type: string + volumeName: + description: volumeName is the binding + reference to the PersistentVolume + backing this claim. + type: string + type: object + required: + - spec + type: object + type: object + fc: + description: fc represents a Fibre Channel resource + that is attached to a kubelet's host machine and + then exposed to the pod. + properties: + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + lun: + description: 'lun is Optional: FC target lun + number' + format: int32 + type: integer + readOnly: + description: |- + readOnly is Optional: Defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + targetWWNs: + description: 'targetWWNs is Optional: FC target + worldwide names (WWNs)' + items: + type: string + type: array + x-kubernetes-list-type: atomic + wwids: + description: |- + wwids Optional: FC volume world wide identifiers (wwids) + Either wwids or combination of targetWWNs and lun must be set, but not both simultaneously. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + flexVolume: + description: |- + flexVolume represents a generic volume resource that is + provisioned/attached using an exec based plugin. + properties: + driver: + description: driver is the name of the driver + to use for this volume. + type: string + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". The default filesystem depends on FlexVolume script. + type: string + options: + additionalProperties: + type: string + description: 'options is Optional: this field + holds extra command options if any.' + type: object + readOnly: + description: |- + readOnly is Optional: defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + secretRef: + description: |- + secretRef is Optional: secretRef is reference to the secret object containing + sensitive information to pass to the plugin scripts. This may be + empty if no secret object is specified. If the secret object + contains more than one secret, all secrets are passed to the plugin + scripts. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + required: + - driver + type: object + flocker: + description: flocker represents a Flocker volume + attached to a kubelet's host machine. This depends + on the Flocker control service being running + properties: + datasetName: + description: |- + datasetName is Name of the dataset stored as metadata -> name on the dataset for Flocker + should be considered as deprecated + type: string + datasetUUID: + description: datasetUUID is the UUID of the + dataset. This is unique identifier of a Flocker + dataset + type: string + type: object + gcePersistentDisk: + description: |- + gcePersistentDisk represents a GCE Disk resource that is attached to a + kubelet's host machine and then exposed to the pod. + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + properties: + fsType: + description: |- + fsType is filesystem type of the volume that you want to mount. + Tip: Ensure that the filesystem type is supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + type: string + partition: + description: |- + partition is the partition in the volume that you want to mount. + If omitted, the default is to mount by volume name. + Examples: For volume /dev/sda1, you specify the partition as "1". + Similarly, the volume partition for /dev/sda is "0" (or you can leave the property empty). + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + format: int32 + type: integer + pdName: + description: |- + pdName is unique name of the PD resource in GCE. Used to identify the disk in GCE. + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + type: string + readOnly: + description: |- + readOnly here will force the ReadOnly setting in VolumeMounts. + Defaults to false. + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + type: boolean + required: + - pdName + type: object + gitRepo: + description: |- + gitRepo represents a git repository at a particular revision. + DEPRECATED: GitRepo is deprecated. To provision a container with a git repo, mount an + EmptyDir into an InitContainer that clones the repo using git, then mount the EmptyDir + into the Pod's container. + properties: + directory: + description: |- + directory is the target directory name. + Must not contain or start with '..'. If '.' is supplied, the volume directory will be the + git repository. Otherwise, if specified, the volume will contain the git repository in + the subdirectory with the given name. + type: string + repository: + description: repository is the URL + type: string + revision: + description: revision is the commit hash for + the specified revision. + type: string + required: + - repository + type: object + glusterfs: + description: |- + glusterfs represents a Glusterfs mount on the host that shares a pod's lifetime. + More info: https://examples.k8s.io/volumes/glusterfs/README.md + properties: + endpoints: + description: |- + endpoints is the endpoint name that details Glusterfs topology. + More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod + type: string + path: + description: |- + path is the Glusterfs volume path. + More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod + type: string + readOnly: + description: |- + readOnly here will force the Glusterfs volume to be mounted with read-only permissions. + Defaults to false. + More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod + type: boolean + required: + - endpoints + - path + type: object + hostPath: + description: |- + hostPath represents a pre-existing file or directory on the host + machine that is directly exposed to the container. This is generally + used for system agents or other privileged things that are allowed + to see the host machine. Most containers will NOT need this. + More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath + properties: + path: + description: |- + path of the directory on the host. + If the path is a symlink, it will follow the link to the real path. + More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath + type: string + type: + description: |- + type for HostPath Volume + Defaults to "" + More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath + type: string + required: + - path + type: object + image: + description: |- + image represents an OCI object (a container image or artifact) pulled and mounted on the kubelet's host machine. + The volume is resolved at pod startup depending on which PullPolicy value is provided: + + - Always: the kubelet always attempts to pull the reference. Container creation will fail If the pull fails. + - Never: the kubelet never pulls the reference and only uses a local image or artifact. Container creation will fail if the reference isn't present. + - IfNotPresent: the kubelet pulls if the reference isn't already present on disk. Container creation will fail if the reference isn't present and the pull fails. + + The volume gets re-resolved if the pod gets deleted and recreated, which means that new remote content will become available on pod recreation. + A failure to resolve or pull the image during pod startup will block containers from starting and may add significant latency. Failures will be retried using normal volume backoff and will be reported on the pod reason and message. + The types of objects that may be mounted by this volume are defined by the container runtime implementation on a host machine and at minimum must include all valid types supported by the container image field. + The OCI object gets mounted in a single directory (spec.containers[*].volumeMounts.mountPath) by merging the manifest layers in the same way as for container images. + The volume will be mounted read-only (ro) and non-executable files (noexec). + Sub path mounts for containers are not supported (spec.containers[*].volumeMounts.subpath). + The field spec.securityContext.fsGroupChangePolicy has no effect on this volume type. + properties: + pullPolicy: + description: |- + Policy for pulling OCI objects. Possible values are: + Always: the kubelet always attempts to pull the reference. Container creation will fail If the pull fails. + Never: the kubelet never pulls the reference and only uses a local image or artifact. Container creation will fail if the reference isn't present. + IfNotPresent: the kubelet pulls if the reference isn't already present on disk. Container creation will fail if the reference isn't present and the pull fails. + Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. + type: string + reference: + description: |- + Required: Image or artifact reference to be used. + Behaves in the same way as pod.spec.containers[*].image. + Pull secrets will be assembled in the same way as for the container image by looking up node credentials, SA image pull secrets, and pod spec image pull secrets. + More info: https://kubernetes.io/docs/concepts/containers/images + This field is optional to allow higher level config management to default or override + container images in workload controllers like Deployments and StatefulSets. + type: string + type: object + iscsi: + description: |- + iscsi represents an ISCSI Disk resource that is attached to a + kubelet's host machine and then exposed to the pod. + More info: https://examples.k8s.io/volumes/iscsi/README.md + properties: + chapAuthDiscovery: + description: chapAuthDiscovery defines whether + support iSCSI Discovery CHAP authentication + type: boolean + chapAuthSession: + description: chapAuthSession defines whether + support iSCSI Session CHAP authentication + type: boolean + fsType: + description: |- + fsType is the filesystem type of the volume that you want to mount. + Tip: Ensure that the filesystem type is supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://kubernetes.io/docs/concepts/storage/volumes#iscsi + type: string + initiatorName: + description: |- + initiatorName is the custom iSCSI Initiator Name. + If initiatorName is specified with iscsiInterface simultaneously, new iSCSI interface + : will be created for the connection. + type: string + iqn: + description: iqn is the target iSCSI Qualified + Name. + type: string + iscsiInterface: + default: default + description: |- + iscsiInterface is the interface Name that uses an iSCSI transport. + Defaults to 'default' (tcp). + type: string + lun: + description: lun represents iSCSI Target Lun + number. + format: int32 + type: integer + portals: + description: |- + portals is the iSCSI Target Portal List. The portal is either an IP or ip_addr:port if the port + is other than default (typically TCP ports 860 and 3260). + items: + type: string + type: array + x-kubernetes-list-type: atomic + readOnly: + description: |- + readOnly here will force the ReadOnly setting in VolumeMounts. + Defaults to false. + type: boolean + secretRef: + description: secretRef is the CHAP Secret for + iSCSI target and initiator authentication + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + targetPortal: + description: |- + targetPortal is iSCSI Target Portal. The Portal is either an IP or ip_addr:port if the port + is other than default (typically TCP ports 860 and 3260). + type: string + required: + - iqn + - lun + - targetPortal + type: object + name: + description: |- + name of the volume. + Must be a DNS_LABEL and unique within the pod. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + nfs: + description: |- + nfs represents an NFS mount on the host that shares a pod's lifetime + More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs + properties: + path: + description: |- + path that is exported by the NFS server. + More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs + type: string + readOnly: + description: |- + readOnly here will force the NFS export to be mounted with read-only permissions. + Defaults to false. + More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs + type: boolean + server: + description: |- + server is the hostname or IP address of the NFS server. + More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs + type: string + required: + - path + - server + type: object + persistentVolumeClaim: + description: |- + persistentVolumeClaimVolumeSource represents a reference to a + PersistentVolumeClaim in the same namespace. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims + properties: + claimName: + description: |- + claimName is the name of a PersistentVolumeClaim in the same namespace as the pod using this volume. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims + type: string + readOnly: + description: |- + readOnly Will force the ReadOnly setting in VolumeMounts. + Default false. + type: boolean + required: + - claimName + type: object + photonPersistentDisk: + description: photonPersistentDisk represents a PhotonController + persistent disk attached and mounted on kubelets + host machine + properties: + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + pdID: + description: pdID is the ID that identifies + Photon Controller persistent disk + type: string + required: + - pdID + type: object + portworxVolume: + description: portworxVolume represents a portworx + volume attached and mounted on kubelets host machine + properties: + fsType: + description: |- + fSType represents the filesystem type to mount + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs". Implicitly inferred to be "ext4" if unspecified. + type: string + readOnly: + description: |- + readOnly defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + volumeID: + description: volumeID uniquely identifies a + Portworx volume + type: string + required: + - volumeID + type: object + projected: + description: projected items for all in one resources + secrets, configmaps, and downward API + properties: + defaultMode: + description: |- + defaultMode are the mode bits used to set permissions on created files by default. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + Directories within the path are not affected by this setting. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + sources: + description: |- + sources is the list of volume projections. Each entry in this list + handles one source. + items: + description: |- + Projection that may be projected along with other supported volume types. + Exactly one of these fields must be set. + properties: + clusterTrustBundle: + description: |- + ClusterTrustBundle allows a pod to access the `.spec.trustBundle` field + of ClusterTrustBundle objects in an auto-updating file. + + Alpha, gated by the ClusterTrustBundleProjection feature gate. + + ClusterTrustBundle objects can either be selected by name, or by the + combination of signer name and a label selector. + + Kubelet performs aggressive normalization of the PEM contents written + into the pod filesystem. Esoteric PEM features such as inter-block + comments and block headers are stripped. Certificates are deduplicated. + The ordering of certificates within the file is arbitrary, and Kubelet + may change the order over time. + properties: + labelSelector: + description: |- + Select all ClusterTrustBundles that match this label selector. Only has + effect if signerName is set. Mutually-exclusive with name. If unset, + interpreted as "match nothing". If set but empty, interpreted as "match + everything". + properties: + matchExpressions: + description: matchExpressions + is a list of label selector + requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the + label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + name: + description: |- + Select a single ClusterTrustBundle by object name. Mutually-exclusive + with signerName and labelSelector. + type: string + optional: + description: |- + If true, don't block pod startup if the referenced ClusterTrustBundle(s) + aren't available. If using name, then the named ClusterTrustBundle is + allowed not to exist. If using signerName, then the combination of + signerName and labelSelector is allowed to match zero + ClusterTrustBundles. + type: boolean + path: + description: Relative path from the + volume root to write the bundle. + type: string + signerName: + description: |- + Select all ClusterTrustBundles that match this signer name. + Mutually-exclusive with name. The contents of all selected + ClusterTrustBundles will be unified and deduplicated. + type: string + required: + - path + type: object + configMap: + description: configMap information about + the configMap data to project + properties: + items: + description: |- + items if unspecified, each key-value pair in the Data field of the referenced + ConfigMap will be projected into the volume as a file whose name is the + key and content is the value. If specified, the listed keys will be + projected into the specified paths, and unlisted keys will not be + present. If a key is specified which is not present in the ConfigMap, + the volume setup will error unless it is marked optional. Paths must be + relative and may not contain the '..' path or start with '..'. + items: + description: Maps a string key to + a path within a volume. + properties: + key: + description: key is the key + to project. + type: string + mode: + description: |- + mode is Optional: mode bits used to set permissions on this file. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: |- + path is the relative path of the file to map the key to. + May not be an absolute path. + May not contain the path element '..'. + May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: optional specify whether + the ConfigMap or its keys must be + defined + type: boolean + type: object + x-kubernetes-map-type: atomic + downwardAPI: + description: downwardAPI information about + the downwardAPI data to project + properties: + items: + description: Items is a list of DownwardAPIVolume + file + items: + description: DownwardAPIVolumeFile + represents information to create + the file containing the pod field + properties: + fieldRef: + description: 'Required: Selects + a field of the pod: only annotations, + labels, name, namespace and + uid are supported.' + properties: + apiVersion: + description: Version of + the schema the FieldPath + is written in terms of, + defaults to "v1". + type: string + fieldPath: + description: Path of the + field to select in the + specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + mode: + description: |- + Optional: mode bits used to set permissions on this file, must be an octal value + between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: 'Required: Path + is the relative path name + of the file to be created. + Must not be absolute or contain + the ''..'' path. Must be utf-8 + encoded. The first item of + the relative path must not + start with ''..''' + type: string + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, requests.cpu and requests.memory) are currently supported. + properties: + containerName: + description: 'Container + name: required for volumes, + optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the + output format of the exposed + resources, defaults to + "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: + resource to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + required: + - path + type: object + type: array + x-kubernetes-list-type: atomic + type: object + secret: + description: secret information about + the secret data to project + properties: + items: + description: |- + items if unspecified, each key-value pair in the Data field of the referenced + Secret will be projected into the volume as a file whose name is the + key and content is the value. If specified, the listed keys will be + projected into the specified paths, and unlisted keys will not be + present. If a key is specified which is not present in the Secret, + the volume setup will error unless it is marked optional. Paths must be + relative and may not contain the '..' path or start with '..'. + items: + description: Maps a string key to + a path within a volume. + properties: + key: + description: key is the key + to project. + type: string + mode: + description: |- + mode is Optional: mode bits used to set permissions on this file. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: |- + path is the relative path of the file to map the key to. + May not be an absolute path. + May not contain the path element '..'. + May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: optional field specify + whether the Secret or its key must + be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + serviceAccountToken: + description: serviceAccountToken is information + about the serviceAccountToken data to + project + properties: + audience: + description: |- + audience is the intended audience of the token. A recipient of a token + must identify itself with an identifier specified in the audience of the + token, and otherwise should reject the token. The audience defaults to the + identifier of the apiserver. + type: string + expirationSeconds: + description: |- + expirationSeconds is the requested duration of validity of the service + account token. As the token approaches expiration, the kubelet volume + plugin will proactively rotate the service account token. The kubelet will + start trying to rotate the token if the token is older than 80 percent of + its time to live or if the token is older than 24 hours.Defaults to 1 hour + and must be at least 10 minutes. + format: int64 + type: integer + path: + description: |- + path is the path relative to the mount point of the file to project the + token into. + type: string + required: + - path + type: object + type: object + type: array + x-kubernetes-list-type: atomic + type: object + quobyte: + description: quobyte represents a Quobyte mount + on the host that shares a pod's lifetime + properties: + group: + description: |- + group to map volume access to + Default is no group + type: string + readOnly: + description: |- + readOnly here will force the Quobyte volume to be mounted with read-only permissions. + Defaults to false. + type: boolean + registry: + description: |- + registry represents a single or multiple Quobyte Registry services + specified as a string as host:port pair (multiple entries are separated with commas) + which acts as the central registry for volumes + type: string + tenant: + description: |- + tenant owning the given Quobyte volume in the Backend + Used with dynamically provisioned Quobyte volumes, value is set by the plugin + type: string + user: + description: |- + user to map volume access to + Defaults to serivceaccount user + type: string + volume: + description: volume is a string that references + an already created Quobyte volume by name. + type: string + required: + - registry + - volume + type: object + rbd: + description: |- + rbd represents a Rados Block Device mount on the host that shares a pod's lifetime. + More info: https://examples.k8s.io/volumes/rbd/README.md + properties: + fsType: + description: |- + fsType is the filesystem type of the volume that you want to mount. + Tip: Ensure that the filesystem type is supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://kubernetes.io/docs/concepts/storage/volumes#rbd + type: string + image: + description: |- + image is the rados image name. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: string + keyring: + default: /etc/ceph/keyring + description: |- + keyring is the path to key ring for RBDUser. + Default is /etc/ceph/keyring. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: string + monitors: + description: |- + monitors is a collection of Ceph monitors. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + items: + type: string + type: array + x-kubernetes-list-type: atomic + pool: + default: rbd + description: |- + pool is the rados pool name. + Default is rbd. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: string + readOnly: + description: |- + readOnly here will force the ReadOnly setting in VolumeMounts. + Defaults to false. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: boolean + secretRef: + description: |- + secretRef is name of the authentication secret for RBDUser. If provided + overrides keyring. + Default is nil. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + user: + default: admin + description: |- + user is the rados user name. + Default is admin. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: string + required: + - image + - monitors + type: object + scaleIO: + description: scaleIO represents a ScaleIO persistent + volume attached and mounted on Kubernetes nodes. + properties: + fsType: + default: xfs + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". + Default is "xfs". + type: string + gateway: + description: gateway is the host address of + the ScaleIO API Gateway. + type: string + protectionDomain: + description: protectionDomain is the name of + the ScaleIO Protection Domain for the configured + storage. + type: string + readOnly: + description: |- + readOnly Defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + secretRef: + description: |- + secretRef references to the secret for ScaleIO user and other + sensitive information. If this is not provided, Login operation will fail. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + sslEnabled: + description: sslEnabled Flag enable/disable + SSL communication with Gateway, default false + type: boolean + storageMode: + default: ThinProvisioned + description: |- + storageMode indicates whether the storage for a volume should be ThickProvisioned or ThinProvisioned. + Default is ThinProvisioned. + type: string + storagePool: + description: storagePool is the ScaleIO Storage + Pool associated with the protection domain. + type: string + system: + description: system is the name of the storage + system as configured in ScaleIO. + type: string + volumeName: + description: |- + volumeName is the name of a volume already created in the ScaleIO system + that is associated with this volume source. + type: string + required: + - gateway + - secretRef + - system + type: object + secret: + description: |- + secret represents a secret that should populate this volume. + More info: https://kubernetes.io/docs/concepts/storage/volumes#secret + properties: + defaultMode: + description: |- + defaultMode is Optional: mode bits used to set permissions on created files by default. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values + for mode bits. Defaults to 0644. + Directories within the path are not affected by this setting. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + items: + description: |- + items If unspecified, each key-value pair in the Data field of the referenced + Secret will be projected into the volume as a file whose name is the + key and content is the value. If specified, the listed keys will be + projected into the specified paths, and unlisted keys will not be + present. If a key is specified which is not present in the Secret, + the volume setup will error unless it is marked optional. Paths must be + relative and may not contain the '..' path or start with '..'. + items: + description: Maps a string key to a path within + a volume. + properties: + key: + description: key is the key to project. + type: string + mode: + description: |- + mode is Optional: mode bits used to set permissions on this file. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: |- + path is the relative path of the file to map the key to. + May not be an absolute path. + May not contain the path element '..'. + May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + optional: + description: optional field specify whether + the Secret or its keys must be defined + type: boolean + secretName: + description: |- + secretName is the name of the secret in the pod's namespace to use. + More info: https://kubernetes.io/docs/concepts/storage/volumes#secret + type: string + type: object + storageos: + description: storageOS represents a StorageOS volume + attached and mounted on Kubernetes nodes. + properties: + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + readOnly: + description: |- + readOnly defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + secretRef: + description: |- + secretRef specifies the secret to use for obtaining the StorageOS API + credentials. If not specified, default values will be attempted. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + volumeName: + description: |- + volumeName is the human-readable name of the StorageOS volume. Volume + names are only unique within a namespace. + type: string + volumeNamespace: + description: |- + volumeNamespace specifies the scope of the volume within StorageOS. If no + namespace is specified then the Pod's namespace will be used. This allows the + Kubernetes name scoping to be mirrored within StorageOS for tighter integration. + Set VolumeName to any name to override the default behaviour. + Set to "default" if you are not using namespaces within StorageOS. + Namespaces that do not pre-exist within StorageOS will be created. + type: string + type: object + vsphereVolume: + description: vsphereVolume represents a vSphere + volume attached and mounted on kubelets host machine + properties: + fsType: + description: |- + fsType is filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + storagePolicyID: + description: storagePolicyID is the storage + Policy Based Management (SPBM) profile ID + associated with the StoragePolicyName. + type: string + storagePolicyName: + description: storagePolicyName is the storage + Policy Based Management (SPBM) profile name. + type: string + volumePath: + description: volumePath is the path that identifies + vSphere volume vmdk + type: string + required: + - volumePath + type: object + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + required: + - containers + type: object + type: object + suspend: + description: |- + suspend specifies whether the RayJob controller should create a RayCluster instance + If a job is applied with the suspend field set to true, + the RayCluster will not be created and will wait for the transition to false. + If the RayCluster is already created, it will be deleted. + In case of transition to false a new RayCluster will be created. + type: boolean + ttlSecondsAfterFinished: + default: 0 + description: |- + TTLSecondsAfterFinished is the TTL to clean up RayCluster. + It's only working when ShutdownAfterJobFinishes set to true. + format: int32 + type: integer + type: object + required: + - spec + type: object + required: + - template + type: object + served: true + storage: true diff --git a/config/crd/bases/kjobctl.x-k8s.io_volumebundles.yaml b/config/crd/bases/kjobctl.x-k8s.io_volumebundles.yaml new file mode 100644 index 0000000..8e1d4b4 --- /dev/null +++ b/config/crd/bases/kjobctl.x-k8s.io_volumebundles.yaml @@ -0,0 +1,2007 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.16.5 + name: volumebundles.kjobctl.x-k8s.io +spec: + group: kjobctl.x-k8s.io + names: + kind: VolumeBundle + listKind: VolumeBundleList + plural: volumebundles + singular: volumebundle + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: VolumeBundle is the Schema for the volumebundles API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: VolumeBundleSpec defines the desired state of VolumeBundle + properties: + containerVolumeMounts: + description: containerVolumeMounts is a list of locations in each + container of a pod where the volumes will be mounted. + items: + description: VolumeMount describes a mounting of a Volume within + a container. + properties: + mountPath: + description: |- + Path within the container at which the volume should be mounted. Must + not contain ':'. + type: string + mountPropagation: + description: |- + mountPropagation determines how mounts are propagated from the host + to container and the other way around. + When not set, MountPropagationNone is used. + This field is beta in 1.10. + When RecursiveReadOnly is set to IfPossible or to Enabled, MountPropagation must be None or unspecified + (which defaults to None). + type: string + name: + description: This must match the Name of a Volume. + type: string + readOnly: + description: |- + Mounted read-only if true, read-write otherwise (false or unspecified). + Defaults to false. + type: boolean + recursiveReadOnly: + description: |- + RecursiveReadOnly specifies whether read-only mounts should be handled + recursively. + + If ReadOnly is false, this field has no meaning and must be unspecified. + + If ReadOnly is true, and this field is set to Disabled, the mount is not made + recursively read-only. If this field is set to IfPossible, the mount is made + recursively read-only, if it is supported by the container runtime. If this + field is set to Enabled, the mount is made recursively read-only if it is + supported by the container runtime, otherwise the pod will not be started and + an error will be generated to indicate the reason. + + If this field is set to IfPossible or Enabled, MountPropagation must be set to + None (or be unspecified, which defaults to None). + + If this field is not specified, it is treated as an equivalent of Disabled. + type: string + subPath: + description: |- + Path within the volume from which the container's volume should be mounted. + Defaults to "" (volume's root). + type: string + subPathExpr: + description: |- + Expanded path within the volume from which the container's volume should be mounted. + Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. + Defaults to "" (volume's root). + SubPathExpr and SubPath are mutually exclusive. + type: string + required: + - mountPath + - name + type: object + minItems: 1 + type: array + x-kubernetes-list-map-keys: + - mountPath + x-kubernetes-list-type: map + envVars: + description: |- + envVars are environment variables that refer to absolute paths in the container filesystem. + These key/value pairs will be available in containers as environment variables. + items: + description: EnvVar represents an environment variable present in + a Container. + properties: + name: + description: Name of the environment variable. Must be a C_IDENTIFIER. + type: string + value: + description: |- + Variable references $(VAR_NAME) are expanded + using the previously defined environment variables in the container and + any service environment variables. If a variable cannot be resolved, + the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. + "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless of whether the variable + exists or not. + Defaults to "". + type: string + valueFrom: + description: Source for the environment variable's value. Cannot + be used if value is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the ConfigMap or its key + must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + description: |- + Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, + spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. + properties: + apiVersion: + description: Version of the schema the FieldPath is + written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select in the specified + API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. + properties: + containerName: + description: 'Container name: required for volumes, + optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format of the exposed + resources, defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + description: Selects a key of a secret in the pod's namespace + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret or its key must + be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + volumes: + description: volumes is a set of volumes that will be added to all + pods of the job. + items: + description: Volume represents a named volume in a pod that may + be accessed by any container in the pod. + properties: + awsElasticBlockStore: + description: |- + awsElasticBlockStore represents an AWS Disk resource that is attached to a + kubelet's host machine and then exposed to the pod. + More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore + properties: + fsType: + description: |- + fsType is the filesystem type of the volume that you want to mount. + Tip: Ensure that the filesystem type is supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore + type: string + partition: + description: |- + partition is the partition in the volume that you want to mount. + If omitted, the default is to mount by volume name. + Examples: For volume /dev/sda1, you specify the partition as "1". + Similarly, the volume partition for /dev/sda is "0" (or you can leave the property empty). + format: int32 + type: integer + readOnly: + description: |- + readOnly value true will force the readOnly setting in VolumeMounts. + More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore + type: boolean + volumeID: + description: |- + volumeID is unique ID of the persistent disk resource in AWS (Amazon EBS volume). + More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore + type: string + required: + - volumeID + type: object + azureDisk: + description: azureDisk represents an Azure Data Disk mount on + the host and bind mount to the pod. + properties: + cachingMode: + description: 'cachingMode is the Host Caching mode: None, + Read Only, Read Write.' + type: string + diskName: + description: diskName is the Name of the data disk in the + blob storage + type: string + diskURI: + description: diskURI is the URI of data disk in the blob + storage + type: string + fsType: + default: ext4 + description: |- + fsType is Filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + kind: + description: 'kind expected values are Shared: multiple + blob disks per storage account Dedicated: single blob + disk per storage account Managed: azure managed data + disk (only in managed availability set). defaults to shared' + type: string + readOnly: + default: false + description: |- + readOnly Defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + required: + - diskName + - diskURI + type: object + azureFile: + description: azureFile represents an Azure File Service mount + on the host and bind mount to the pod. + properties: + readOnly: + description: |- + readOnly defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + secretName: + description: secretName is the name of secret that contains + Azure Storage Account Name and Key + type: string + shareName: + description: shareName is the azure share Name + type: string + required: + - secretName + - shareName + type: object + cephfs: + description: cephFS represents a Ceph FS mount on the host that + shares a pod's lifetime + properties: + monitors: + description: |- + monitors is Required: Monitors is a collection of Ceph monitors + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + items: + type: string + type: array + x-kubernetes-list-type: atomic + path: + description: 'path is Optional: Used as the mounted root, + rather than the full Ceph tree, default is /' + type: string + readOnly: + description: |- + readOnly is Optional: Defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + type: boolean + secretFile: + description: |- + secretFile is Optional: SecretFile is the path to key ring for User, default is /etc/ceph/user.secret + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + type: string + secretRef: + description: |- + secretRef is Optional: SecretRef is reference to the authentication secret for User, default is empty. + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + user: + description: |- + user is optional: User is the rados user name, default is admin + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it + type: string + required: + - monitors + type: object + cinder: + description: |- + cinder represents a cinder volume attached and mounted on kubelets host machine. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md + properties: + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md + type: string + readOnly: + description: |- + readOnly defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md + type: boolean + secretRef: + description: |- + secretRef is optional: points to a secret object containing parameters used to connect + to OpenStack. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + volumeID: + description: |- + volumeID used to identify the volume in cinder. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md + type: string + required: + - volumeID + type: object + configMap: + description: configMap represents a configMap that should populate + this volume + properties: + defaultMode: + description: |- + defaultMode is optional: mode bits used to set permissions on created files by default. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + Defaults to 0644. + Directories within the path are not affected by this setting. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + items: + description: |- + items if unspecified, each key-value pair in the Data field of the referenced + ConfigMap will be projected into the volume as a file whose name is the + key and content is the value. If specified, the listed keys will be + projected into the specified paths, and unlisted keys will not be + present. If a key is specified which is not present in the ConfigMap, + the volume setup will error unless it is marked optional. Paths must be + relative and may not contain the '..' path or start with '..'. + items: + description: Maps a string key to a path within a volume. + properties: + key: + description: key is the key to project. + type: string + mode: + description: |- + mode is Optional: mode bits used to set permissions on this file. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: |- + path is the relative path of the file to map the key to. + May not be an absolute path. + May not contain the path element '..'. + May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: optional specify whether the ConfigMap or its + keys must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + csi: + description: csi (Container Storage Interface) represents ephemeral + storage that is handled by certain external CSI drivers (Beta + feature). + properties: + driver: + description: |- + driver is the name of the CSI driver that handles this volume. + Consult with your admin for the correct name as registered in the cluster. + type: string + fsType: + description: |- + fsType to mount. Ex. "ext4", "xfs", "ntfs". + If not provided, the empty value is passed to the associated CSI driver + which will determine the default filesystem to apply. + type: string + nodePublishSecretRef: + description: |- + nodePublishSecretRef is a reference to the secret object containing + sensitive information to pass to the CSI driver to complete the CSI + NodePublishVolume and NodeUnpublishVolume calls. + This field is optional, and may be empty if no secret is required. If the + secret object contains more than one secret, all secret references are passed. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + readOnly: + description: |- + readOnly specifies a read-only configuration for the volume. + Defaults to false (read/write). + type: boolean + volumeAttributes: + additionalProperties: + type: string + description: |- + volumeAttributes stores driver-specific properties that are passed to the CSI + driver. Consult your driver's documentation for supported values. + type: object + required: + - driver + type: object + downwardAPI: + description: downwardAPI represents downward API about the pod + that should populate this volume + properties: + defaultMode: + description: |- + Optional: mode bits to use on created files by default. Must be a + Optional: mode bits used to set permissions on created files by default. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + Defaults to 0644. + Directories within the path are not affected by this setting. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + items: + description: Items is a list of downward API volume file + items: + description: DownwardAPIVolumeFile represents information + to create the file containing the pod field + properties: + fieldRef: + description: 'Required: Selects a field of the pod: + only annotations, labels, name, namespace and uid + are supported.' + properties: + apiVersion: + description: Version of the schema the FieldPath + is written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select in the + specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + mode: + description: |- + Optional: mode bits used to set permissions on this file, must be an octal value + between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: 'Required: Path is the relative path + name of the file to be created. Must not be absolute + or contain the ''..'' path. Must be utf-8 encoded. + The first item of the relative path must not start + with ''..''' + type: string + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, requests.cpu and requests.memory) are currently supported. + properties: + containerName: + description: 'Container name: required for volumes, + optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format of the + exposed resources, defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + required: + - path + type: object + type: array + x-kubernetes-list-type: atomic + type: object + emptyDir: + description: |- + emptyDir represents a temporary directory that shares a pod's lifetime. + More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir + properties: + medium: + description: |- + medium represents what type of storage medium should back this directory. + The default is "" which means to use the node's default medium. + Must be an empty string (default) or Memory. + More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir + type: string + sizeLimit: + anyOf: + - type: integer + - type: string + description: |- + sizeLimit is the total amount of local storage required for this EmptyDir volume. + The size limit is also applicable for memory medium. + The maximum usage on memory medium EmptyDir would be the minimum value between + the SizeLimit specified here and the sum of memory limits of all containers in a pod. + The default is nil which means that the limit is undefined. + More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + ephemeral: + description: |- + ephemeral represents a volume that is handled by a cluster storage driver. + The volume's lifecycle is tied to the pod that defines it - it will be created before the pod starts, + and deleted when the pod is removed. + + Use this if: + a) the volume is only needed while the pod runs, + b) features of normal volumes like restoring from snapshot or capacity + tracking are needed, + c) the storage driver is specified through a storage class, and + d) the storage driver supports dynamic volume provisioning through + a PersistentVolumeClaim (see EphemeralVolumeSource for more + information on the connection between this volume type + and PersistentVolumeClaim). + + Use PersistentVolumeClaim or one of the vendor-specific + APIs for volumes that persist for longer than the lifecycle + of an individual pod. + + Use CSI for light-weight local ephemeral volumes if the CSI driver is meant to + be used that way - see the documentation of the driver for + more information. + + A pod can use both types of ephemeral volumes and + persistent volumes at the same time. + properties: + volumeClaimTemplate: + description: |- + Will be used to create a stand-alone PVC to provision the volume. + The pod in which this EphemeralVolumeSource is embedded will be the + owner of the PVC, i.e. the PVC will be deleted together with the + pod. The name of the PVC will be `-` where + `` is the name from the `PodSpec.Volumes` array + entry. Pod validation will reject the pod if the concatenated name + is not valid for a PVC (for example, too long). + + An existing PVC with that name that is not owned by the pod + will *not* be used for the pod to avoid using an unrelated + volume by mistake. Starting the pod is then blocked until + the unrelated PVC is removed. If such a pre-created PVC is + meant to be used by the pod, the PVC has to updated with an + owner reference to the pod once the pod exists. Normally + this should not be necessary, but it may be useful when + manually reconstructing a broken cluster. + + This field is read-only and no changes will be made by Kubernetes + to the PVC after it has been created. + + Required, must not be nil. + properties: + metadata: + description: |- + May contain labels and annotations that will be copied into the PVC + when creating it. No other fields are allowed and will be rejected during + validation. + type: object + spec: + description: |- + The specification for the PersistentVolumeClaim. The entire content is + copied unchanged into the PVC that gets created from this + template. The same fields as in a PersistentVolumeClaim + are also valid here. + properties: + accessModes: + description: |- + accessModes contains the desired access modes the volume should have. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1 + items: + type: string + type: array + x-kubernetes-list-type: atomic + dataSource: + description: |- + dataSource field can be used to specify either: + * An existing VolumeSnapshot object (snapshot.storage.k8s.io/VolumeSnapshot) + * An existing PVC (PersistentVolumeClaim) + If the provisioner or an external controller can support the specified data source, + it will create a new volume based on the contents of the specified data source. + When the AnyVolumeDataSource feature gate is enabled, dataSource contents will be copied to dataSourceRef, + and dataSourceRef contents will be copied to dataSource when dataSourceRef.namespace is not specified. + If the namespace is specified, then dataSourceRef will not be copied to dataSource. + properties: + apiGroup: + description: |- + APIGroup is the group for the resource being referenced. + If APIGroup is not specified, the specified Kind must be in the core API group. + For any other third-party types, APIGroup is required. + type: string + kind: + description: Kind is the type of resource being + referenced + type: string + name: + description: Name is the name of resource being + referenced + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + dataSourceRef: + description: |- + dataSourceRef specifies the object from which to populate the volume with data, if a non-empty + volume is desired. This may be any object from a non-empty API group (non + core object) or a PersistentVolumeClaim object. + When this field is specified, volume binding will only succeed if the type of + the specified object matches some installed volume populator or dynamic + provisioner. + This field will replace the functionality of the dataSource field and as such + if both fields are non-empty, they must have the same value. For backwards + compatibility, when namespace isn't specified in dataSourceRef, + both fields (dataSource and dataSourceRef) will be set to the same + value automatically if one of them is empty and the other is non-empty. + When namespace is specified in dataSourceRef, + dataSource isn't set to the same value and must be empty. + There are three important differences between dataSource and dataSourceRef: + * While dataSource only allows two specific types of objects, dataSourceRef + allows any non-core object, as well as PersistentVolumeClaim objects. + * While dataSource ignores disallowed values (dropping them), dataSourceRef + preserves all values, and generates an error if a disallowed value is + specified. + * While dataSource only allows local objects, dataSourceRef allows objects + in any namespaces. + (Beta) Using this field requires the AnyVolumeDataSource feature gate to be enabled. + (Alpha) Using the namespace field of dataSourceRef requires the CrossNamespaceVolumeDataSource feature gate to be enabled. + properties: + apiGroup: + description: |- + APIGroup is the group for the resource being referenced. + If APIGroup is not specified, the specified Kind must be in the core API group. + For any other third-party types, APIGroup is required. + type: string + kind: + description: Kind is the type of resource being + referenced + type: string + name: + description: Name is the name of resource being + referenced + type: string + namespace: + description: |- + Namespace is the namespace of resource being referenced + Note that when a namespace is specified, a gateway.networking.k8s.io/ReferenceGrant object is required in the referent namespace to allow that namespace's owner to accept the reference. See the ReferenceGrant documentation for details. + (Alpha) This field requires the CrossNamespaceVolumeDataSource feature gate to be enabled. + type: string + required: + - kind + - name + type: object + resources: + description: |- + resources represents the minimum resources the volume should have. + If RecoverVolumeExpansionFailure feature is enabled users are allowed to specify resource requirements + that are lower than previous value but must still be higher than capacity recorded in the + status field of the claim. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + selector: + description: selector is a label query over volumes + to consider for binding. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that + the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + storageClassName: + description: |- + storageClassName is the name of the StorageClass required by the claim. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1 + type: string + volumeAttributesClassName: + description: |- + volumeAttributesClassName may be used to set the VolumeAttributesClass used by this claim. + If specified, the CSI driver will create or update the volume with the attributes defined + in the corresponding VolumeAttributesClass. This has a different purpose than storageClassName, + it can be changed after the claim is created. An empty string value means that no VolumeAttributesClass + will be applied to the claim but it's not allowed to reset this field to empty string once it is set. + If unspecified and the PersistentVolumeClaim is unbound, the default VolumeAttributesClass + will be set by the persistentvolume controller if it exists. + If the resource referred to by volumeAttributesClass does not exist, this PersistentVolumeClaim will be + set to a Pending state, as reflected by the modifyVolumeStatus field, until such as a resource + exists. + More info: https://kubernetes.io/docs/concepts/storage/volume-attributes-classes/ + (Beta) Using this field requires the VolumeAttributesClass feature gate to be enabled (off by default). + type: string + volumeMode: + description: |- + volumeMode defines what type of volume is required by the claim. + Value of Filesystem is implied when not included in claim spec. + type: string + volumeName: + description: volumeName is the binding reference + to the PersistentVolume backing this claim. + type: string + type: object + required: + - spec + type: object + type: object + fc: + description: fc represents a Fibre Channel resource that is + attached to a kubelet's host machine and then exposed to the + pod. + properties: + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + lun: + description: 'lun is Optional: FC target lun number' + format: int32 + type: integer + readOnly: + description: |- + readOnly is Optional: Defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + targetWWNs: + description: 'targetWWNs is Optional: FC target worldwide + names (WWNs)' + items: + type: string + type: array + x-kubernetes-list-type: atomic + wwids: + description: |- + wwids Optional: FC volume world wide identifiers (wwids) + Either wwids or combination of targetWWNs and lun must be set, but not both simultaneously. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + flexVolume: + description: |- + flexVolume represents a generic volume resource that is + provisioned/attached using an exec based plugin. + properties: + driver: + description: driver is the name of the driver to use for + this volume. + type: string + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". The default filesystem depends on FlexVolume script. + type: string + options: + additionalProperties: + type: string + description: 'options is Optional: this field holds extra + command options if any.' + type: object + readOnly: + description: |- + readOnly is Optional: defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + secretRef: + description: |- + secretRef is Optional: secretRef is reference to the secret object containing + sensitive information to pass to the plugin scripts. This may be + empty if no secret object is specified. If the secret object + contains more than one secret, all secrets are passed to the plugin + scripts. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + required: + - driver + type: object + flocker: + description: flocker represents a Flocker volume attached to + a kubelet's host machine. This depends on the Flocker control + service being running + properties: + datasetName: + description: |- + datasetName is Name of the dataset stored as metadata -> name on the dataset for Flocker + should be considered as deprecated + type: string + datasetUUID: + description: datasetUUID is the UUID of the dataset. This + is unique identifier of a Flocker dataset + type: string + type: object + gcePersistentDisk: + description: |- + gcePersistentDisk represents a GCE Disk resource that is attached to a + kubelet's host machine and then exposed to the pod. + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + properties: + fsType: + description: |- + fsType is filesystem type of the volume that you want to mount. + Tip: Ensure that the filesystem type is supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + type: string + partition: + description: |- + partition is the partition in the volume that you want to mount. + If omitted, the default is to mount by volume name. + Examples: For volume /dev/sda1, you specify the partition as "1". + Similarly, the volume partition for /dev/sda is "0" (or you can leave the property empty). + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + format: int32 + type: integer + pdName: + description: |- + pdName is unique name of the PD resource in GCE. Used to identify the disk in GCE. + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + type: string + readOnly: + description: |- + readOnly here will force the ReadOnly setting in VolumeMounts. + Defaults to false. + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + type: boolean + required: + - pdName + type: object + gitRepo: + description: |- + gitRepo represents a git repository at a particular revision. + DEPRECATED: GitRepo is deprecated. To provision a container with a git repo, mount an + EmptyDir into an InitContainer that clones the repo using git, then mount the EmptyDir + into the Pod's container. + properties: + directory: + description: |- + directory is the target directory name. + Must not contain or start with '..'. If '.' is supplied, the volume directory will be the + git repository. Otherwise, if specified, the volume will contain the git repository in + the subdirectory with the given name. + type: string + repository: + description: repository is the URL + type: string + revision: + description: revision is the commit hash for the specified + revision. + type: string + required: + - repository + type: object + glusterfs: + description: |- + glusterfs represents a Glusterfs mount on the host that shares a pod's lifetime. + More info: https://examples.k8s.io/volumes/glusterfs/README.md + properties: + endpoints: + description: |- + endpoints is the endpoint name that details Glusterfs topology. + More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod + type: string + path: + description: |- + path is the Glusterfs volume path. + More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod + type: string + readOnly: + description: |- + readOnly here will force the Glusterfs volume to be mounted with read-only permissions. + Defaults to false. + More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod + type: boolean + required: + - endpoints + - path + type: object + hostPath: + description: |- + hostPath represents a pre-existing file or directory on the host + machine that is directly exposed to the container. This is generally + used for system agents or other privileged things that are allowed + to see the host machine. Most containers will NOT need this. + More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath + properties: + path: + description: |- + path of the directory on the host. + If the path is a symlink, it will follow the link to the real path. + More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath + type: string + type: + description: |- + type for HostPath Volume + Defaults to "" + More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath + type: string + required: + - path + type: object + image: + description: |- + image represents an OCI object (a container image or artifact) pulled and mounted on the kubelet's host machine. + The volume is resolved at pod startup depending on which PullPolicy value is provided: + + - Always: the kubelet always attempts to pull the reference. Container creation will fail If the pull fails. + - Never: the kubelet never pulls the reference and only uses a local image or artifact. Container creation will fail if the reference isn't present. + - IfNotPresent: the kubelet pulls if the reference isn't already present on disk. Container creation will fail if the reference isn't present and the pull fails. + + The volume gets re-resolved if the pod gets deleted and recreated, which means that new remote content will become available on pod recreation. + A failure to resolve or pull the image during pod startup will block containers from starting and may add significant latency. Failures will be retried using normal volume backoff and will be reported on the pod reason and message. + The types of objects that may be mounted by this volume are defined by the container runtime implementation on a host machine and at minimum must include all valid types supported by the container image field. + The OCI object gets mounted in a single directory (spec.containers[*].volumeMounts.mountPath) by merging the manifest layers in the same way as for container images. + The volume will be mounted read-only (ro) and non-executable files (noexec). + Sub path mounts for containers are not supported (spec.containers[*].volumeMounts.subpath). + The field spec.securityContext.fsGroupChangePolicy has no effect on this volume type. + properties: + pullPolicy: + description: |- + Policy for pulling OCI objects. Possible values are: + Always: the kubelet always attempts to pull the reference. Container creation will fail If the pull fails. + Never: the kubelet never pulls the reference and only uses a local image or artifact. Container creation will fail if the reference isn't present. + IfNotPresent: the kubelet pulls if the reference isn't already present on disk. Container creation will fail if the reference isn't present and the pull fails. + Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. + type: string + reference: + description: |- + Required: Image or artifact reference to be used. + Behaves in the same way as pod.spec.containers[*].image. + Pull secrets will be assembled in the same way as for the container image by looking up node credentials, SA image pull secrets, and pod spec image pull secrets. + More info: https://kubernetes.io/docs/concepts/containers/images + This field is optional to allow higher level config management to default or override + container images in workload controllers like Deployments and StatefulSets. + type: string + type: object + iscsi: + description: |- + iscsi represents an ISCSI Disk resource that is attached to a + kubelet's host machine and then exposed to the pod. + More info: https://examples.k8s.io/volumes/iscsi/README.md + properties: + chapAuthDiscovery: + description: chapAuthDiscovery defines whether support iSCSI + Discovery CHAP authentication + type: boolean + chapAuthSession: + description: chapAuthSession defines whether support iSCSI + Session CHAP authentication + type: boolean + fsType: + description: |- + fsType is the filesystem type of the volume that you want to mount. + Tip: Ensure that the filesystem type is supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://kubernetes.io/docs/concepts/storage/volumes#iscsi + type: string + initiatorName: + description: |- + initiatorName is the custom iSCSI Initiator Name. + If initiatorName is specified with iscsiInterface simultaneously, new iSCSI interface + : will be created for the connection. + type: string + iqn: + description: iqn is the target iSCSI Qualified Name. + type: string + iscsiInterface: + default: default + description: |- + iscsiInterface is the interface Name that uses an iSCSI transport. + Defaults to 'default' (tcp). + type: string + lun: + description: lun represents iSCSI Target Lun number. + format: int32 + type: integer + portals: + description: |- + portals is the iSCSI Target Portal List. The portal is either an IP or ip_addr:port if the port + is other than default (typically TCP ports 860 and 3260). + items: + type: string + type: array + x-kubernetes-list-type: atomic + readOnly: + description: |- + readOnly here will force the ReadOnly setting in VolumeMounts. + Defaults to false. + type: boolean + secretRef: + description: secretRef is the CHAP Secret for iSCSI target + and initiator authentication + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + targetPortal: + description: |- + targetPortal is iSCSI Target Portal. The Portal is either an IP or ip_addr:port if the port + is other than default (typically TCP ports 860 and 3260). + type: string + required: + - iqn + - lun + - targetPortal + type: object + name: + description: |- + name of the volume. + Must be a DNS_LABEL and unique within the pod. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + nfs: + description: |- + nfs represents an NFS mount on the host that shares a pod's lifetime + More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs + properties: + path: + description: |- + path that is exported by the NFS server. + More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs + type: string + readOnly: + description: |- + readOnly here will force the NFS export to be mounted with read-only permissions. + Defaults to false. + More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs + type: boolean + server: + description: |- + server is the hostname or IP address of the NFS server. + More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs + type: string + required: + - path + - server + type: object + persistentVolumeClaim: + description: |- + persistentVolumeClaimVolumeSource represents a reference to a + PersistentVolumeClaim in the same namespace. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims + properties: + claimName: + description: |- + claimName is the name of a PersistentVolumeClaim in the same namespace as the pod using this volume. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims + type: string + readOnly: + description: |- + readOnly Will force the ReadOnly setting in VolumeMounts. + Default false. + type: boolean + required: + - claimName + type: object + photonPersistentDisk: + description: photonPersistentDisk represents a PhotonController + persistent disk attached and mounted on kubelets host machine + properties: + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + pdID: + description: pdID is the ID that identifies Photon Controller + persistent disk + type: string + required: + - pdID + type: object + portworxVolume: + description: portworxVolume represents a portworx volume attached + and mounted on kubelets host machine + properties: + fsType: + description: |- + fSType represents the filesystem type to mount + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs". Implicitly inferred to be "ext4" if unspecified. + type: string + readOnly: + description: |- + readOnly defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + volumeID: + description: volumeID uniquely identifies a Portworx volume + type: string + required: + - volumeID + type: object + projected: + description: projected items for all in one resources secrets, + configmaps, and downward API + properties: + defaultMode: + description: |- + defaultMode are the mode bits used to set permissions on created files by default. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + Directories within the path are not affected by this setting. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + sources: + description: |- + sources is the list of volume projections. Each entry in this list + handles one source. + items: + description: |- + Projection that may be projected along with other supported volume types. + Exactly one of these fields must be set. + properties: + clusterTrustBundle: + description: |- + ClusterTrustBundle allows a pod to access the `.spec.trustBundle` field + of ClusterTrustBundle objects in an auto-updating file. + + Alpha, gated by the ClusterTrustBundleProjection feature gate. + + ClusterTrustBundle objects can either be selected by name, or by the + combination of signer name and a label selector. + + Kubelet performs aggressive normalization of the PEM contents written + into the pod filesystem. Esoteric PEM features such as inter-block + comments and block headers are stripped. Certificates are deduplicated. + The ordering of certificates within the file is arbitrary, and Kubelet + may change the order over time. + properties: + labelSelector: + description: |- + Select all ClusterTrustBundles that match this label selector. Only has + effect if signerName is set. Mutually-exclusive with name. If unset, + interpreted as "match nothing". If set but empty, interpreted as "match + everything". + properties: + matchExpressions: + description: matchExpressions is a list of + label selector requirements. The requirements + are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that + the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + name: + description: |- + Select a single ClusterTrustBundle by object name. Mutually-exclusive + with signerName and labelSelector. + type: string + optional: + description: |- + If true, don't block pod startup if the referenced ClusterTrustBundle(s) + aren't available. If using name, then the named ClusterTrustBundle is + allowed not to exist. If using signerName, then the combination of + signerName and labelSelector is allowed to match zero + ClusterTrustBundles. + type: boolean + path: + description: Relative path from the volume root + to write the bundle. + type: string + signerName: + description: |- + Select all ClusterTrustBundles that match this signer name. + Mutually-exclusive with name. The contents of all selected + ClusterTrustBundles will be unified and deduplicated. + type: string + required: + - path + type: object + configMap: + description: configMap information about the configMap + data to project + properties: + items: + description: |- + items if unspecified, each key-value pair in the Data field of the referenced + ConfigMap will be projected into the volume as a file whose name is the + key and content is the value. If specified, the listed keys will be + projected into the specified paths, and unlisted keys will not be + present. If a key is specified which is not present in the ConfigMap, + the volume setup will error unless it is marked optional. Paths must be + relative and may not contain the '..' path or start with '..'. + items: + description: Maps a string key to a path within + a volume. + properties: + key: + description: key is the key to project. + type: string + mode: + description: |- + mode is Optional: mode bits used to set permissions on this file. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: |- + path is the relative path of the file to map the key to. + May not be an absolute path. + May not contain the path element '..'. + May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: optional specify whether the ConfigMap + or its keys must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + downwardAPI: + description: downwardAPI information about the downwardAPI + data to project + properties: + items: + description: Items is a list of DownwardAPIVolume + file + items: + description: DownwardAPIVolumeFile represents + information to create the file containing + the pod field + properties: + fieldRef: + description: 'Required: Selects a field + of the pod: only annotations, labels, + name, namespace and uid are supported.' + properties: + apiVersion: + description: Version of the schema the + FieldPath is written in terms of, + defaults to "v1". + type: string + fieldPath: + description: Path of the field to select + in the specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + mode: + description: |- + Optional: mode bits used to set permissions on this file, must be an octal value + between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: 'Required: Path is the relative + path name of the file to be created. Must + not be absolute or contain the ''..'' + path. Must be utf-8 encoded. The first + item of the relative path must not start + with ''..''' + type: string + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, requests.cpu and requests.memory) are currently supported. + properties: + containerName: + description: 'Container name: required + for volumes, optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format + of the exposed resources, defaults + to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to + select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + required: + - path + type: object + type: array + x-kubernetes-list-type: atomic + type: object + secret: + description: secret information about the secret data + to project + properties: + items: + description: |- + items if unspecified, each key-value pair in the Data field of the referenced + Secret will be projected into the volume as a file whose name is the + key and content is the value. If specified, the listed keys will be + projected into the specified paths, and unlisted keys will not be + present. If a key is specified which is not present in the Secret, + the volume setup will error unless it is marked optional. Paths must be + relative and may not contain the '..' path or start with '..'. + items: + description: Maps a string key to a path within + a volume. + properties: + key: + description: key is the key to project. + type: string + mode: + description: |- + mode is Optional: mode bits used to set permissions on this file. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: |- + path is the relative path of the file to map the key to. + May not be an absolute path. + May not contain the path element '..'. + May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: optional field specify whether the + Secret or its key must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + serviceAccountToken: + description: serviceAccountToken is information about + the serviceAccountToken data to project + properties: + audience: + description: |- + audience is the intended audience of the token. A recipient of a token + must identify itself with an identifier specified in the audience of the + token, and otherwise should reject the token. The audience defaults to the + identifier of the apiserver. + type: string + expirationSeconds: + description: |- + expirationSeconds is the requested duration of validity of the service + account token. As the token approaches expiration, the kubelet volume + plugin will proactively rotate the service account token. The kubelet will + start trying to rotate the token if the token is older than 80 percent of + its time to live or if the token is older than 24 hours.Defaults to 1 hour + and must be at least 10 minutes. + format: int64 + type: integer + path: + description: |- + path is the path relative to the mount point of the file to project the + token into. + type: string + required: + - path + type: object + type: object + type: array + x-kubernetes-list-type: atomic + type: object + quobyte: + description: quobyte represents a Quobyte mount on the host + that shares a pod's lifetime + properties: + group: + description: |- + group to map volume access to + Default is no group + type: string + readOnly: + description: |- + readOnly here will force the Quobyte volume to be mounted with read-only permissions. + Defaults to false. + type: boolean + registry: + description: |- + registry represents a single or multiple Quobyte Registry services + specified as a string as host:port pair (multiple entries are separated with commas) + which acts as the central registry for volumes + type: string + tenant: + description: |- + tenant owning the given Quobyte volume in the Backend + Used with dynamically provisioned Quobyte volumes, value is set by the plugin + type: string + user: + description: |- + user to map volume access to + Defaults to serivceaccount user + type: string + volume: + description: volume is a string that references an already + created Quobyte volume by name. + type: string + required: + - registry + - volume + type: object + rbd: + description: |- + rbd represents a Rados Block Device mount on the host that shares a pod's lifetime. + More info: https://examples.k8s.io/volumes/rbd/README.md + properties: + fsType: + description: |- + fsType is the filesystem type of the volume that you want to mount. + Tip: Ensure that the filesystem type is supported by the host operating system. + Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://kubernetes.io/docs/concepts/storage/volumes#rbd + type: string + image: + description: |- + image is the rados image name. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: string + keyring: + default: /etc/ceph/keyring + description: |- + keyring is the path to key ring for RBDUser. + Default is /etc/ceph/keyring. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: string + monitors: + description: |- + monitors is a collection of Ceph monitors. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + items: + type: string + type: array + x-kubernetes-list-type: atomic + pool: + default: rbd + description: |- + pool is the rados pool name. + Default is rbd. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: string + readOnly: + description: |- + readOnly here will force the ReadOnly setting in VolumeMounts. + Defaults to false. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: boolean + secretRef: + description: |- + secretRef is name of the authentication secret for RBDUser. If provided + overrides keyring. + Default is nil. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + user: + default: admin + description: |- + user is the rados user name. + Default is admin. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it + type: string + required: + - image + - monitors + type: object + scaleIO: + description: scaleIO represents a ScaleIO persistent volume + attached and mounted on Kubernetes nodes. + properties: + fsType: + default: xfs + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". + Default is "xfs". + type: string + gateway: + description: gateway is the host address of the ScaleIO + API Gateway. + type: string + protectionDomain: + description: protectionDomain is the name of the ScaleIO + Protection Domain for the configured storage. + type: string + readOnly: + description: |- + readOnly Defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + secretRef: + description: |- + secretRef references to the secret for ScaleIO user and other + sensitive information. If this is not provided, Login operation will fail. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + sslEnabled: + description: sslEnabled Flag enable/disable SSL communication + with Gateway, default false + type: boolean + storageMode: + default: ThinProvisioned + description: |- + storageMode indicates whether the storage for a volume should be ThickProvisioned or ThinProvisioned. + Default is ThinProvisioned. + type: string + storagePool: + description: storagePool is the ScaleIO Storage Pool associated + with the protection domain. + type: string + system: + description: system is the name of the storage system as + configured in ScaleIO. + type: string + volumeName: + description: |- + volumeName is the name of a volume already created in the ScaleIO system + that is associated with this volume source. + type: string + required: + - gateway + - secretRef + - system + type: object + secret: + description: |- + secret represents a secret that should populate this volume. + More info: https://kubernetes.io/docs/concepts/storage/volumes#secret + properties: + defaultMode: + description: |- + defaultMode is Optional: mode bits used to set permissions on created files by default. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values + for mode bits. Defaults to 0644. + Directories within the path are not affected by this setting. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + items: + description: |- + items If unspecified, each key-value pair in the Data field of the referenced + Secret will be projected into the volume as a file whose name is the + key and content is the value. If specified, the listed keys will be + projected into the specified paths, and unlisted keys will not be + present. If a key is specified which is not present in the Secret, + the volume setup will error unless it is marked optional. Paths must be + relative and may not contain the '..' path or start with '..'. + items: + description: Maps a string key to a path within a volume. + properties: + key: + description: key is the key to project. + type: string + mode: + description: |- + mode is Optional: mode bits used to set permissions on this file. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: |- + path is the relative path of the file to map the key to. + May not be an absolute path. + May not contain the path element '..'. + May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + optional: + description: optional field specify whether the Secret or + its keys must be defined + type: boolean + secretName: + description: |- + secretName is the name of the secret in the pod's namespace to use. + More info: https://kubernetes.io/docs/concepts/storage/volumes#secret + type: string + type: object + storageos: + description: storageOS represents a StorageOS volume attached + and mounted on Kubernetes nodes. + properties: + fsType: + description: |- + fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + readOnly: + description: |- + readOnly defaults to false (read/write). ReadOnly here will force + the ReadOnly setting in VolumeMounts. + type: boolean + secretRef: + description: |- + secretRef specifies the secret to use for obtaining the StorageOS API + credentials. If not specified, default values will be attempted. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + volumeName: + description: |- + volumeName is the human-readable name of the StorageOS volume. Volume + names are only unique within a namespace. + type: string + volumeNamespace: + description: |- + volumeNamespace specifies the scope of the volume within StorageOS. If no + namespace is specified then the Pod's namespace will be used. This allows the + Kubernetes name scoping to be mirrored within StorageOS for tighter integration. + Set VolumeName to any name to override the default behaviour. + Set to "default" if you are not using namespaces within StorageOS. + Namespaces that do not pre-exist within StorageOS will be created. + type: string + type: object + vsphereVolume: + description: vsphereVolume represents a vSphere volume attached + and mounted on kubelets host machine + properties: + fsType: + description: |- + fsType is filesystem type to mount. + Must be a filesystem type supported by the host operating system. + Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + storagePolicyID: + description: storagePolicyID is the storage Policy Based + Management (SPBM) profile ID associated with the StoragePolicyName. + type: string + storagePolicyName: + description: storagePolicyName is the storage Policy Based + Management (SPBM) profile name. + type: string + volumePath: + description: volumePath is the path that identifies vSphere + volume vmdk + type: string + required: + - volumePath + type: object + required: + - name + type: object + minItems: 1 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + required: + - containerVolumeMounts + - volumes + type: object + type: object + served: true + storage: true diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml new file mode 100644 index 0000000..d3e935f --- /dev/null +++ b/config/crd/kustomization.yaml @@ -0,0 +1,15 @@ +# This kustomization.yaml is not intended to be run by itself, +# since it depends on service name and namespace that are out of this kustomize package. +# It should be run by config/default +resources: +- bases/kjobctl.x-k8s.io_volumebundles.yaml +- bases/kjobctl.x-k8s.io_jobtemplates.yaml +- bases/kjobctl.x-k8s.io_rayjobtemplates.yaml +- bases/kjobctl.x-k8s.io_applicationprofiles.yaml +- bases/kjobctl.x-k8s.io_rayclustertemplates.yaml +# +kubebuilder:scaffold:crdkustomizeresource + +patches: [] + +#configurations: +#- kustomizeconfig.yaml diff --git a/config/default/kustomization.yaml b/config/default/kustomization.yaml new file mode 100644 index 0000000..4c2bb18 --- /dev/null +++ b/config/default/kustomization.yaml @@ -0,0 +1,18 @@ +# Adds namespace to all resources. +namespace: kjobctl-system + +# Value of this field is prepended to the +# names of all resources, e.g. a deployment named +# "wordpress" becomes "alices-wordpress". +# Note that it should also match with the prefix (text before '-') of the namespace +# field above. +namePrefix: kjobctl- + +# Labels to add to all resources and selectors. +#labels: +#- includeSelectors: true +# pairs: +# someName: someValue + +resources: +- ../crd diff --git a/config/samples/README.md b/config/samples/README.md new file mode 100644 index 0000000..ab35943 --- /dev/null +++ b/config/samples/README.md @@ -0,0 +1,3 @@ +RayJob sample are straight from KubeRay (https://raw.githubusercontent.com/ray-project/kuberay/v1.1.1/ray-operator/config/samples/ray-job.sample.yaml). + +RayCluster sample are straight from KubeRay (https://raw.githubusercontent.com/ray-project/kuberay/master/ray-operator/config/samples/ray-cluster.sample.yaml). diff --git a/config/samples/interactive-sample.yaml b/config/samples/interactive-sample.yaml new file mode 100644 index 0000000..aa6c50b --- /dev/null +++ b/config/samples/interactive-sample.yaml @@ -0,0 +1,22 @@ +--- +apiVersion: kjobctl.x-k8s.io/v1alpha1 +kind: ApplicationProfile +metadata: + name: interactive-profile + namespace: default +spec: + supportedModes: + - name: Interactive + template: interactive-template +--- +apiVersion: v1 +kind: PodTemplate +metadata: + name: interactive-template + namespace: default +template: + spec: + containers: + - name: sample-container + image: busybox:1.28 + command: ['/bin/sh'] \ No newline at end of file diff --git a/config/samples/job-sample.yaml b/config/samples/job-sample.yaml new file mode 100644 index 0000000..674a379 --- /dev/null +++ b/config/samples/job-sample.yaml @@ -0,0 +1,31 @@ +--- +apiVersion: kjobctl.x-k8s.io/v1alpha1 +kind: ApplicationProfile +metadata: + name: job-profile + namespace: default +spec: + supportedModes: + - name: Job + template: job-template +--- +apiVersion: kjobctl.x-k8s.io/v1alpha1 +kind: JobTemplate +metadata: + name: job-template + namespace: default +template: + spec: + parallelism: 3 + completions: 3 + template: + spec: + containers: + - name: sample-container + image: gcr.io/k8s-staging-perf-tests/sleep:v0.1.0 + args: [ "30s" ] + resources: + requests: + cpu: "1" + memory: "200Mi" + restartPolicy: Never diff --git a/config/samples/ray-cluster-sample.yaml b/config/samples/ray-cluster-sample.yaml new file mode 100644 index 0000000..a263931 --- /dev/null +++ b/config/samples/ray-cluster-sample.yaml @@ -0,0 +1,120 @@ +--- +apiVersion: kjobctl.x-k8s.io/v1alpha1 +kind: ApplicationProfile +metadata: + name: ray-cluster-profile + namespace: default +spec: + supportedModes: + - name: RayCluster + template: ray-cluster-template + volumeBundles: ["ray-cluster-volume-bundle"] +--- +apiVersion: kjobctl.x-k8s.io/v1alpha1 +kind: RayClusterTemplate +metadata: + name: ray-cluster-template + namespace: default +template: + spec: + rayVersion: '2.9.0' # should match the Ray version in the image of the containers + # Ray head pod template + headGroupSpec: + rayStartParams: { } + #pod template + template: + spec: + containers: + - name: ray-head + image: rayproject/ray:2.9.0 + resources: + limits: + cpu: 1 + memory: 2Gi + requests: + cpu: 500m + memory: 2Gi + ports: + - containerPort: 6379 + name: gcs-server + - containerPort: 8265 # Ray dashboard + name: dashboard + - containerPort: 10001 + name: client + workerGroupSpecs: + # the pod replicas in this group typed worker + - replicas: 1 + minReplicas: 1 + maxReplicas: 5 + # logical group name, for this called small-group, also can be functional + groupName: small-group + rayStartParams: { } + #pod template + template: + spec: + containers: + - name: ray-worker # must consist of lower case alphanumeric characters or '-', and must start and end with an alphanumeric character (e.g. 'my-name', or '123-abc' + image: rayproject/ray:2.9.0 + resources: + limits: + cpu: 1 + memory: 1Gi + requests: + cpu: 500m + memory: 1Gi +--- +apiVersion: kjobctl.x-k8s.io/v1alpha1 +kind: VolumeBundle +metadata: + name: ray-cluster-volume-bundle + namespace: default +spec: + volumes: + - name: ray-cluster-code-sample + configMap: + name: ray-cluster-code-sample + items: + - key: sample_code.py + path: sample_code.py + containerVolumeMounts: + - name: ray-cluster-code-sample + mountPath: /home/ray/samples + envVars: + - name: ENTRYPOINT_PATH + value: /home/ray/samples +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: ray-cluster-code-sample + namespace: default +data: + sample_code.py: | + import ray + import os + import requests + + ray.init() + + @ray.remote + class Counter: + def __init__(self): + # Used to verify runtimeEnv + self.name = os.getenv("counter_name") + assert self.name == "test_counter" + self.counter = 0 + + def inc(self): + self.counter += 1 + + def get_counter(self): + return "{} got {}".format(self.name, self.counter) + + counter = Counter.remote() + + for _ in range(5): + ray.get(counter.inc.remote()) + print(ray.get(counter.get_counter.remote())) + + # Verify that the correct runtime env was used for the job. + assert requests.__version__ == "2.26.0" \ No newline at end of file diff --git a/config/samples/ray-job-sample.yaml b/config/samples/ray-job-sample.yaml new file mode 100644 index 0000000..3d731fb --- /dev/null +++ b/config/samples/ray-job-sample.yaml @@ -0,0 +1,159 @@ +--- +apiVersion: kjobctl.x-k8s.io/v1alpha1 +kind: ApplicationProfile +metadata: + name: ray-job-profile + namespace: default +spec: + supportedModes: + - name: RayJob + template: ray-job-template + volumeBundles: ["ray-job-volume-bundle"] +--- +apiVersion: kjobctl.x-k8s.io/v1alpha1 +kind: RayJobTemplate +metadata: + name: ray-job-template + namespace: default +template: + spec: + # submissionMode specifies how RayJob submits the Ray job to the RayCluster. + # The default value is "K8sJobMode", meaning RayJob will submit the Ray job via a submitter Kubernetes Job. + # The alternative value is "HTTPMode", indicating that KubeRay will submit the Ray job by sending an HTTP request to the RayCluster. + # submissionMode: "K8sJobMode" + entrypoint: python ${ENTRYPOINT_PATH}/sample_code.py + # shutdownAfterJobFinishes specifies whether the RayCluster should be deleted after the RayJob finishes. Default is false. + # shutdownAfterJobFinishes: false + + # ttlSecondsAfterFinished specifies the number of seconds after which the RayCluster will be deleted after the RayJob finishes. + # ttlSecondsAfterFinished: 10 + + # activeDeadlineSeconds is the duration in seconds that the RayJob may be active before + # KubeRay actively tries to terminate the RayJob; value must be positive integer. + # activeDeadlineSeconds: 120 + + # RuntimeEnvYAML represents the runtime environment configuration provided as a multi-line YAML string. + # See https://docs.ray.io/en/latest/ray-core/handling-dependencies.html for details. + # (New in KubeRay version 1.0.) + runtimeEnvYAML: | + pip: + - requests==2.26.0 + - pendulum==2.1.2 + env_vars: + counter_name: "test_counter" + + # Suspend specifies whether the RayJob controller should create a RayCluster instance. + # If a job is applied with the suspend field set to true, the RayCluster will not be created and we will wait for the transition to false. + # If the RayCluster is already created, it will be deleted. In the case of transition to false, a new RayCluster will be created. + # suspend: false + + # rayClusterSpec specifies the RayCluster instance to be created by the RayJob controller. + rayClusterSpec: + rayVersion: '2.9.0' # should match the Ray version in the image of the containers + # Ray head pod template + headGroupSpec: + # The `rayStartParams` are used to configure the `ray start` command. + # See https://github.com/ray-project/kuberay/blob/master/docs/guidance/rayStartParams.md for the default settings of `rayStartParams` in KubeRay. + # See https://docs.ray.io/en/latest/cluster/cli.html#ray-start for all available options in `rayStartParams`. + rayStartParams: + dashboard-host: '0.0.0.0' + #pod template + template: + spec: + containers: + - name: ray-head + image: rayproject/ray:2.9.0 + ports: + - containerPort: 6379 + name: gcs-server + - containerPort: 8265 # Ray dashboard + name: dashboard + - containerPort: 10001 + name: client + resources: + limits: + cpu: "1" + requests: + cpu: "200m" + workerGroupSpecs: + # the pod replicas in this group typed worker + - replicas: 1 + minReplicas: 1 + maxReplicas: 5 + # logical group name, for this called small-group, also can be functional + groupName: small-group + # The `rayStartParams` are used to configure the `ray start` command. + # See https://github.com/ray-project/kuberay/blob/master/docs/guidance/rayStartParams.md for the default settings of `rayStartParams` in KubeRay. + # See https://docs.ray.io/en/latest/cluster/cli.html#ray-start for all available options in `rayStartParams`. + rayStartParams: {} + #pod template + template: + spec: + containers: + - name: ray-worker # must consist of lower case alphanumeric characters or '-', and must start and end with an alphanumeric character (e.g. 'my-name', or '123-abc' + image: rayproject/ray:2.9.0 + lifecycle: + preStop: + exec: + command: [ "/bin/sh","-c","ray stop" ] + resources: + limits: + cpu: "1" + requests: + cpu: "200m" +--- +apiVersion: kjobctl.x-k8s.io/v1alpha1 +kind: VolumeBundle +metadata: + name: ray-job-volume-bundle + namespace: default +spec: + volumes: + - name: ray-job-code-sample + configMap: + name: ray-job-code-sample + items: + - key: sample_code.py + path: sample_code.py + containerVolumeMounts: + - name: ray-job-code-sample + mountPath: /home/ray/samples + envVars: + - name: ENTRYPOINT_PATH + value: /home/ray/samples +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: ray-job-code-sample + namespace: default +data: + sample_code.py: | + import ray + import os + import requests + + ray.init() + + @ray.remote + class Counter: + def __init__(self): + # Used to verify runtimeEnv + self.name = os.getenv("counter_name") + assert self.name == "test_counter" + self.counter = 0 + + def inc(self): + self.counter += 1 + + def get_counter(self): + return "{} got {}".format(self.name, self.counter) + + counter = Counter.remote() + + for _ in range(5): + ray.get(counter.inc.remote()) + print(ray.get(counter.get_counter.remote())) + + # Verify that the correct runtime env was used for the job. + assert requests.__version__ == "2.26.0" \ No newline at end of file diff --git a/config/samples/slurm/script.sh b/config/samples/slurm/script.sh new file mode 100644 index 0000000..176956a --- /dev/null +++ b/config/samples/slurm/script.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +#SBATCH --array=1-3%2 + +echo "now processing task id: ${SLURM_ARRAY_TASK_ID}" +python /home/slurm/samples/sample_code.py diff --git a/config/samples/slurm/slurm-sample.yaml b/config/samples/slurm/slurm-sample.yaml new file mode 100644 index 0000000..674fcda --- /dev/null +++ b/config/samples/slurm/slurm-sample.yaml @@ -0,0 +1,64 @@ +--- +apiVersion: kjobctl.x-k8s.io/v1alpha1 +kind: ApplicationProfile +metadata: + name: slurm-profile + namespace: default +spec: + supportedModes: + - name: Slurm + template: slurm-template + volumeBundles: ["slurm-volume-bundle"] +--- +apiVersion: kjobctl.x-k8s.io/v1alpha1 +kind: JobTemplate +metadata: + name: slurm-template + namespace: default +template: + spec: + parallelism: 3 + completions: 3 + completionMode: Indexed + template: + spec: + containers: + - name: sample-container + image: python:3-slim + restartPolicy: OnFailure +--- +apiVersion: kjobctl.x-k8s.io/v1alpha1 +kind: VolumeBundle +metadata: + name: slurm-volume-bundle + namespace: default +spec: + volumes: + - name: slurm-code-sample + configMap: + name: slurm-code-sample + items: + - key: sample_code.py + path: sample_code.py + containerVolumeMounts: + - name: slurm-code-sample + mountPath: /home/slurm/samples + envVars: + - name: ENTRYPOINT_PATH + value: /home/slurm/samples +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: slurm-code-sample + namespace: default +data: + sample_code.py: | + import time + + print('start at ' + time.strftime('%H:%M:%S')) + + print('sleep for 10 seconds ...') + time.sleep(10) + + print('stop at ' + time.strftime('%H:%M:%S')) diff --git a/docs/_index.md b/docs/_index.md new file mode 100644 index 0000000..b45f998 --- /dev/null +++ b/docs/_index.md @@ -0,0 +1,24 @@ +# Kubectl Kjob Plugin + +The `kubectl-kjob` plugin, `kjobctl`, allows you to list, describe and create jobs. + +## Syntax + +Use the following syntax to run `kubectl kjob` commands from your terminal window: + +```shell +kubectl kjob [OPERATION] [TYPE] [NAME] [flags] +``` + +or with shorter syntax `kjobctl`: + +```shell +kjobctl [OPERATION] [TYPE] [NAME] [flags] +``` + +You can go to the [commands](commands/_index.md) or run `kubectl kjob help` in the terminal to get the full list of commands, along with all possible flags. + +## See Also + +* [installation](installation.md) - Installation guide for the `kubectl-kjob` plugin, `kjobctl`. +* [commands](commands/kjobctl.md) - Full list of commands, along with all possible flags. \ No newline at end of file diff --git a/docs/commands/kjobctl.md b/docs/commands/kjobctl.md new file mode 100644 index 0000000..a449321 --- /dev/null +++ b/docs/commands/kjobctl.md @@ -0,0 +1,207 @@ + + +# kjobctl + + +## Synopsis + + +ML/AI/Batch Jobs Made Easy + + +## Options + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
--as string
+

Username to impersonate for the operation. User could be a regular user or a service account in a namespace.

+
--as-group strings
+

Group to impersonate for the operation, this flag can be repeated to specify multiple groups.

+
--as-uid string
+

UID to impersonate for the operation.

+
--cache-dir string     Default: "$HOME/.kube/cache"
+

Default cache directory

+
--certificate-authority string
+

Path to a cert file for the certificate authority

+
--client-certificate string
+

Path to a client certificate file for TLS

+
--client-key string
+

Path to a client key file for TLS

+
--cluster string
+

The name of the kubeconfig cluster to use

+
--context string
+

The name of the kubeconfig context to use

+
--disable-compression
+

If true, opt-out of response compression for all requests to the server

+
-h, --help
+

help for kjobctl

+
--insecure-skip-tls-verify
+

If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure

+
--kubeconfig string
+

Path to the kubeconfig file to use for CLI requests.

+
-n, --namespace string
+

If present, the namespace scope for this CLI request

+
--request-timeout string     Default: "0"
+

The length of time to wait before giving up on a single server request. Non-zero values should contain a corresponding time unit (e.g. 1s, 2m, 3h). A value of zero means don't timeout requests.

+
-s, --server string
+

The address and port of the Kubernetes API server

+
--tls-server-name string
+

Server name to use for server certificate validation. If it is not provided, the hostname used to contact the server is used

+
--token string
+

Bearer token for authentication to the API server

+
--user string
+

The name of the kubeconfig user to use

+
+ + + +## See Also + +* [kjobctl create](kjobctl_create/_index.md) - Create a task +* [kjobctl delete](kjobctl_delete/_index.md) - Delete resources +* [kjobctl describe](kjobctl_describe/_index.md) - Show details of a specific resource or group of resources. +* [kjobctl list](kjobctl_list/_index.md) - Display resources +* [kjobctl printcrds](kjobctl_printcrds/_index.md) - Print the kjobctl CRDs + diff --git a/docs/commands/kjobctl_create/_index.md b/docs/commands/kjobctl_create/_index.md new file mode 100644 index 0000000..20c6ced --- /dev/null +++ b/docs/commands/kjobctl_create/_index.md @@ -0,0 +1,261 @@ + + +# kjobctl create + + +## Synopsis + + +Create a task + + +## Examples + +``` + # Create job + kjobctl create job \ + --profile my-application-profile \ + --cmd "sleep 5" \ + --parallelism 4 \ + --completions 4 \ + --request cpu=500m,memory=4Gi \ + --localqueue my-local-queue-name + + # Create interactive + kjobctl create interactive \ + --profile my-application-profile \ + --pod-running-timeout 30s \ + --rm + + # Create rayjob + kjobctl create rayjob \ + --profile my-application-profile \ + --cmd "python /home/ray/samples/sample_code.py" \ + --replicas small-group=1 \ + --min-replicas small-group=1 \ + --max-replicas small-group=5 \ + --localqueue my-local-queue-name + + # Create raycluster + kjobctl create raycluster \ + --profile my-application-profile \ + --replicas small-group=1 \ + --min-replicas small-group=1 \ + --max-replicas small-group=5 \ + --localqueue my-local-queue-name + + # Create slurm + kjobctl create slurm --profile my-application-profile -- \ + --array 0-5 --nodes 3 --ntasks 1 ./script.sh +``` + + +## Options + + + + + + + + + + + + + + + + +
-h, --help
+

help for create

+
+ + + +## Options inherited from parent commands + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
--as string
+

Username to impersonate for the operation. User could be a regular user or a service account in a namespace.

+
--as-group strings
+

Group to impersonate for the operation, this flag can be repeated to specify multiple groups.

+
--as-uid string
+

UID to impersonate for the operation.

+
--cache-dir string     Default: "$HOME/.kube/cache"
+

Default cache directory

+
--certificate-authority string
+

Path to a cert file for the certificate authority

+
--client-certificate string
+

Path to a client certificate file for TLS

+
--client-key string
+

Path to a client key file for TLS

+
--cluster string
+

The name of the kubeconfig cluster to use

+
--context string
+

The name of the kubeconfig context to use

+
--disable-compression
+

If true, opt-out of response compression for all requests to the server

+
--insecure-skip-tls-verify
+

If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure

+
--kubeconfig string
+

Path to the kubeconfig file to use for CLI requests.

+
-n, --namespace string
+

If present, the namespace scope for this CLI request

+
--request-timeout string     Default: "0"
+

The length of time to wait before giving up on a single server request. Non-zero values should contain a corresponding time unit (e.g. 1s, 2m, 3h). A value of zero means don't timeout requests.

+
-s, --server string
+

The address and port of the Kubernetes API server

+
--tls-server-name string
+

Server name to use for server certificate validation. If it is not provided, the hostname used to contact the server is used

+
--token string
+

Bearer token for authentication to the API server

+
--user string
+

The name of the kubeconfig user to use

+
+ + + +## See Also + +* [kjobctl](../kjobctl.md) - ML/AI/Batch Jobs Made Easy +* [kjobctl create interactive](kjobctl_create_interactive.md) - Create an interactive shell +* [kjobctl create job](kjobctl_create_job.md) - Create a job +* [kjobctl create raycluster](kjobctl_create_raycluster.md) - Create a raycluster +* [kjobctl create rayjob](kjobctl_create_rayjob.md) - Create a rayjob +* [kjobctl create slurm](kjobctl_create_slurm.md) - Create a slurm job + diff --git a/docs/commands/kjobctl_create/kjobctl_create_interactive.md b/docs/commands/kjobctl_create/kjobctl_create_interactive.md new file mode 100644 index 0000000..a4b3c58 --- /dev/null +++ b/docs/commands/kjobctl_create/kjobctl_create_interactive.md @@ -0,0 +1,369 @@ + + +# kjobctl create interactive + + +## Synopsis + + +Create an interactive shell + +``` +kjobctl create interactive --profile APPLICATION_PROFILE_NAME [--localqueue LOCAL_QUEUE_NAME] [--skip-localqueue-validation] [--priority NAME] [--skip-priority-validation] [--cmd COMMAND] [--request RESOURCE_NAME=QUANTITY] [--pod-running-timeout DURATION] [--time TIME_LIMIT] [--rm] +``` + + +## Examples + +``` + # Create interactive + kjobctl create interactive \ + --profile my-application-profile \ + --pod-running-timeout 30s \ + --rm +``` + + +## Options + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
--allow-missing-template-keys     Default: true
+

If true, ignore any errors in templates when a field or map key is missing in the template. Only applies to golang and jsonpath output formats.

+
--cmd string
+

Command which is associated with the resource.

+
--dry-run string     Default: "none"
+

Must be "none", "server", or "client". If client strategy, only print the object that would be sent, without sending it. If server strategy, submit server-side request without persisting the resource.

+
-h, --help
+

help for interactive

+
--localqueue string
+

Kueue localqueue name which is associated with the resource.

+
-o, --output string
+

Output format. One of: (json, yaml, name, go-template, go-template-file, template, templatefile, jsonpath, jsonpath-as-json, jsonpath-file).

+
--pod-running-timeout duration     Default: 1m0s
+

The length of time (like 5s, 2m, or 3h, higher than zero) to wait until at least one pod is running.

+
--priority string
+

Apply priority for the entire workload.

+
-p, --profile string
+

Application profile contains a template (with defaults set) for running a specific type of application.

+
--request <comma-separated 'key=value' pairs>     Default: []
+

Request is a set of (resource name, quantity) pairs.

+
--rm
+

Remove pod when interactive session exits.

+
--show-managed-fields
+

If true, keep the managedFields when printing objects in JSON or YAML format.

+
--skip-localqueue-validation
+

Skip local queue validation. Add local queue even if the queue does not exist.

+
--skip-priority-validation
+

Skip workload priority class validation. Add priority class label even if the class does not exist.

+
--template string
+

Template string or path to template file to use when -o=go-template, -o=go-template-file. The template format is golang templates [http://golang.org/pkg/text/template/#pkg-overview].

+
-t, --time string
+

Set a limit on the total run time of the job. +A time limit of zero requests that no time limit be imposed. +Acceptable time formats include "minutes", "minutes:seconds", +"hours:minutes:seconds", "days-hours", "days-hours:minutes" +and "days-hours:minutes:seconds".

+
+ + + +## Options inherited from parent commands + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
--as string
+

Username to impersonate for the operation. User could be a regular user or a service account in a namespace.

+
--as-group strings
+

Group to impersonate for the operation, this flag can be repeated to specify multiple groups.

+
--as-uid string
+

UID to impersonate for the operation.

+
--cache-dir string     Default: "$HOME/.kube/cache"
+

Default cache directory

+
--certificate-authority string
+

Path to a cert file for the certificate authority

+
--client-certificate string
+

Path to a client certificate file for TLS

+
--client-key string
+

Path to a client key file for TLS

+
--cluster string
+

The name of the kubeconfig cluster to use

+
--context string
+

The name of the kubeconfig context to use

+
--disable-compression
+

If true, opt-out of response compression for all requests to the server

+
--insecure-skip-tls-verify
+

If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure

+
--kubeconfig string
+

Path to the kubeconfig file to use for CLI requests.

+
-n, --namespace string
+

If present, the namespace scope for this CLI request

+
--request-timeout string     Default: "0"
+

The length of time to wait before giving up on a single server request. Non-zero values should contain a corresponding time unit (e.g. 1s, 2m, 3h). A value of zero means don't timeout requests.

+
-s, --server string
+

The address and port of the Kubernetes API server

+
--tls-server-name string
+

Server name to use for server certificate validation. If it is not provided, the hostname used to contact the server is used

+
--token string
+

Bearer token for authentication to the API server

+
--user string
+

The name of the kubeconfig user to use

+
+ + + +## See Also + +* [kjobctl_create](_index.md) - Create a task + diff --git a/docs/commands/kjobctl_create/kjobctl_create_job.md b/docs/commands/kjobctl_create/kjobctl_create_job.md new file mode 100644 index 0000000..b55a153 --- /dev/null +++ b/docs/commands/kjobctl_create/kjobctl_create_job.md @@ -0,0 +1,372 @@ + + +# kjobctl create job + + +## Synopsis + + +Create a job + +``` +kjobctl create job --profile APPLICATION_PROFILE_NAME [--localqueue LOCAL_QUEUE_NAME] [--skip-localqueue-validation] [--priority NAME] [--skip-priority-validation] [--cmd COMMAND] [--request RESOURCE_NAME=QUANTITY] [--parallelism PARALLELISM] [--completions COMPLETIONS] [--time TIME_LIMIT] +``` + + +## Examples + +``` + # Create job + kjobctl create job \ + --profile my-application-profile \ + --cmd "sleep 5" \ + --parallelism 4 \ + --completions 4 \ + --request cpu=500m,memory=4Gi \ + --localqueue my-local-queue-name +``` + + +## Options + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
--allow-missing-template-keys     Default: true
+

If true, ignore any errors in templates when a field or map key is missing in the template. Only applies to golang and jsonpath output formats.

+
--cmd string
+

Command which is associated with the resource.

+
--completions int
+

Completions specifies the desired number of successfully finished pods.

+
--dry-run string     Default: "none"
+

Must be "none", "server", or "client". If client strategy, only print the object that would be sent, without sending it. If server strategy, submit server-side request without persisting the resource.

+
-h, --help
+

help for job

+
--localqueue string
+

Kueue localqueue name which is associated with the resource.

+
-o, --output string
+

Output format. One of: (json, yaml, name, go-template, go-template-file, template, templatefile, jsonpath, jsonpath-as-json, jsonpath-file).

+
--parallelism int
+

Parallelism specifies the maximum desired number of pods the job should run at any given time.

+
--priority string
+

Apply priority for the entire workload.

+
-p, --profile string
+

Application profile contains a template (with defaults set) for running a specific type of application.

+
--request <comma-separated 'key=value' pairs>     Default: []
+

Request is a set of (resource name, quantity) pairs.

+
--show-managed-fields
+

If true, keep the managedFields when printing objects in JSON or YAML format.

+
--skip-localqueue-validation
+

Skip local queue validation. Add local queue even if the queue does not exist.

+
--skip-priority-validation
+

Skip workload priority class validation. Add priority class label even if the class does not exist.

+
--template string
+

Template string or path to template file to use when -o=go-template, -o=go-template-file. The template format is golang templates [http://golang.org/pkg/text/template/#pkg-overview].

+
-t, --time string
+

Set a limit on the total run time of the job. +A time limit of zero requests that no time limit be imposed. +Acceptable time formats include "minutes", "minutes:seconds", +"hours:minutes:seconds", "days-hours", "days-hours:minutes" +and "days-hours:minutes:seconds".

+
+ + + +## Options inherited from parent commands + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
--as string
+

Username to impersonate for the operation. User could be a regular user or a service account in a namespace.

+
--as-group strings
+

Group to impersonate for the operation, this flag can be repeated to specify multiple groups.

+
--as-uid string
+

UID to impersonate for the operation.

+
--cache-dir string     Default: "$HOME/.kube/cache"
+

Default cache directory

+
--certificate-authority string
+

Path to a cert file for the certificate authority

+
--client-certificate string
+

Path to a client certificate file for TLS

+
--client-key string
+

Path to a client key file for TLS

+
--cluster string
+

The name of the kubeconfig cluster to use

+
--context string
+

The name of the kubeconfig context to use

+
--disable-compression
+

If true, opt-out of response compression for all requests to the server

+
--insecure-skip-tls-verify
+

If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure

+
--kubeconfig string
+

Path to the kubeconfig file to use for CLI requests.

+
-n, --namespace string
+

If present, the namespace scope for this CLI request

+
--request-timeout string     Default: "0"
+

The length of time to wait before giving up on a single server request. Non-zero values should contain a corresponding time unit (e.g. 1s, 2m, 3h). A value of zero means don't timeout requests.

+
-s, --server string
+

The address and port of the Kubernetes API server

+
--tls-server-name string
+

Server name to use for server certificate validation. If it is not provided, the hostname used to contact the server is used

+
--token string
+

Bearer token for authentication to the API server

+
--user string
+

The name of the kubeconfig user to use

+
+ + + +## See Also + +* [kjobctl_create](_index.md) - Create a task + diff --git a/docs/commands/kjobctl_create/kjobctl_create_raycluster.md b/docs/commands/kjobctl_create/kjobctl_create_raycluster.md new file mode 100644 index 0000000..8015b5d --- /dev/null +++ b/docs/commands/kjobctl_create/kjobctl_create_raycluster.md @@ -0,0 +1,364 @@ + + +# kjobctl create raycluster + + +## Synopsis + + +Create a raycluster. + + KubeRay operator is required for RayCluster. How to install KubeRay operator you can find here https://ray-project.github.io/kuberay/deploy/installation/. + +``` +kjobctl create raycluster --profile APPLICATION_PROFILE_NAME [--localqueue LOCAL_QUEUE_NAME] [--skip-localqueue-validation] [--priority NAME] [--skip-priority-validation] [--replicas [WORKER_GROUP]=REPLICAS] [--min-replicas [WORKER_GROUP]=MIN_REPLICAS] [--max-replicas [WORKER_GROUP]=MAX_REPLICAS] [--time TIME_LIMIT] +``` + + +## Examples + +``` + # Create raycluster + kjobctl create raycluster \ + --profile my-application-profile \ + --replicas small-group=1 \ + --min-replicas small-group=1 \ + --max-replicas small-group=5 \ + --localqueue my-local-queue-name +``` + + +## Options + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
--allow-missing-template-keys     Default: true
+

If true, ignore any errors in templates when a field or map key is missing in the template. Only applies to golang and jsonpath output formats.

+
--dry-run string     Default: "none"
+

Must be "none", "server", or "client". If client strategy, only print the object that would be sent, without sending it. If server strategy, submit server-side request without persisting the resource.

+
-h, --help
+

help for raycluster

+
--localqueue string
+

Kueue localqueue name which is associated with the resource.

+
--max-replicas <comma-separated 'key=int' pairs>     Default: []
+

MaxReplicas denotes the maximum number of desired Pods for this worker group, and the default value is maxInt32.

+
--min-replicas <comma-separated 'key=int' pairs>     Default: []
+

MinReplicas denotes the minimum number of desired Pods for this worker group.

+
-o, --output string
+

Output format. One of: (json, yaml, name, go-template, go-template-file, template, templatefile, jsonpath, jsonpath-as-json, jsonpath-file).

+
--priority string
+

Apply priority for the entire workload.

+
-p, --profile string
+

Application profile contains a template (with defaults set) for running a specific type of application.

+
--replicas <comma-separated 'key=int' pairs>     Default: []
+

Replicas is the number of desired Pods for this worker group.

+
--show-managed-fields
+

If true, keep the managedFields when printing objects in JSON or YAML format.

+
--skip-localqueue-validation
+

Skip local queue validation. Add local queue even if the queue does not exist.

+
--skip-priority-validation
+

Skip workload priority class validation. Add priority class label even if the class does not exist.

+
--template string
+

Template string or path to template file to use when -o=go-template, -o=go-template-file. The template format is golang templates [http://golang.org/pkg/text/template/#pkg-overview].

+
-t, --time string
+

Set a limit on the total run time of the job. +A time limit of zero requests that no time limit be imposed. +Acceptable time formats include "minutes", "minutes:seconds", +"hours:minutes:seconds", "days-hours", "days-hours:minutes" +and "days-hours:minutes:seconds".

+
+ + + +## Options inherited from parent commands + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
--as string
+

Username to impersonate for the operation. User could be a regular user or a service account in a namespace.

+
--as-group strings
+

Group to impersonate for the operation, this flag can be repeated to specify multiple groups.

+
--as-uid string
+

UID to impersonate for the operation.

+
--cache-dir string     Default: "$HOME/.kube/cache"
+

Default cache directory

+
--certificate-authority string
+

Path to a cert file for the certificate authority

+
--client-certificate string
+

Path to a client certificate file for TLS

+
--client-key string
+

Path to a client key file for TLS

+
--cluster string
+

The name of the kubeconfig cluster to use

+
--context string
+

The name of the kubeconfig context to use

+
--disable-compression
+

If true, opt-out of response compression for all requests to the server

+
--insecure-skip-tls-verify
+

If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure

+
--kubeconfig string
+

Path to the kubeconfig file to use for CLI requests.

+
-n, --namespace string
+

If present, the namespace scope for this CLI request

+
--request-timeout string     Default: "0"
+

The length of time to wait before giving up on a single server request. Non-zero values should contain a corresponding time unit (e.g. 1s, 2m, 3h). A value of zero means don't timeout requests.

+
-s, --server string
+

The address and port of the Kubernetes API server

+
--tls-server-name string
+

Server name to use for server certificate validation. If it is not provided, the hostname used to contact the server is used

+
--token string
+

Bearer token for authentication to the API server

+
--user string
+

The name of the kubeconfig user to use

+
+ + + +## See Also + +* [kjobctl_create](_index.md) - Create a task + diff --git a/docs/commands/kjobctl_create/kjobctl_create_rayjob.md b/docs/commands/kjobctl_create/kjobctl_create_rayjob.md new file mode 100644 index 0000000..a192549 --- /dev/null +++ b/docs/commands/kjobctl_create/kjobctl_create_rayjob.md @@ -0,0 +1,383 @@ + + +# kjobctl create rayjob + + +## Synopsis + + +Create a rayjob. + + KubeRay operator is required for RayJob. How to install KubeRay operator you can find here https://ray-project.github.io/kuberay/deploy/installation/. + +``` +kjobctl create rayjob --profile APPLICATION_PROFILE_NAME [--localqueue LOCAL_QUEUE_NAME] [--skip-localqueue-validation] [--priority NAME] [--skip-priority-validation] [--cmd COMMAND] [--replicas [WORKER_GROUP]=REPLICAS] [--min-replicas [WORKER_GROUP]=MIN_REPLICAS] [--max-replicas [WORKER_GROUP]=MAX_REPLICAS] [--time TIME_LIMIT] +``` + + +## Examples + +``` + # Create rayjob + kjobctl create rayjob \ + --profile my-application-profile \ + --cmd "python /home/ray/samples/sample_code.py" \ + --replicas small-group=1 \ + --min-replicas small-group=1 \ + --max-replicas small-group=5 \ + --localqueue my-local-queue-name +``` + + +## Options + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
--allow-missing-template-keys     Default: true
+

If true, ignore any errors in templates when a field or map key is missing in the template. Only applies to golang and jsonpath output formats.

+
--cmd string
+

Command which is associated with the resource.

+
--dry-run string     Default: "none"
+

Must be "none", "server", or "client". If client strategy, only print the object that would be sent, without sending it. If server strategy, submit server-side request without persisting the resource.

+
-h, --help
+

help for rayjob

+
--localqueue string
+

Kueue localqueue name which is associated with the resource.

+
--max-replicas <comma-separated 'key=int' pairs>     Default: []
+

MaxReplicas denotes the maximum number of desired Pods for this worker group, and the default value is maxInt32.

+
--min-replicas <comma-separated 'key=int' pairs>     Default: []
+

MinReplicas denotes the minimum number of desired Pods for this worker group.

+
-o, --output string
+

Output format. One of: (json, yaml, name, go-template, go-template-file, template, templatefile, jsonpath, jsonpath-as-json, jsonpath-file).

+
--priority string
+

Apply priority for the entire workload.

+
-p, --profile string
+

Application profile contains a template (with defaults set) for running a specific type of application.

+
--raycluster string
+

Existing ray cluster on which the job will be created.

+
--replicas <comma-separated 'key=int' pairs>     Default: []
+

Replicas is the number of desired Pods for this worker group.

+
--show-managed-fields
+

If true, keep the managedFields when printing objects in JSON or YAML format.

+
--skip-localqueue-validation
+

Skip local queue validation. Add local queue even if the queue does not exist.

+
--skip-priority-validation
+

Skip workload priority class validation. Add priority class label even if the class does not exist.

+
--template string
+

Template string or path to template file to use when -o=go-template, -o=go-template-file. The template format is golang templates [http://golang.org/pkg/text/template/#pkg-overview].

+
-t, --time string
+

Set a limit on the total run time of the job. +A time limit of zero requests that no time limit be imposed. +Acceptable time formats include "minutes", "minutes:seconds", +"hours:minutes:seconds", "days-hours", "days-hours:minutes" +and "days-hours:minutes:seconds".

+
+ + + +## Options inherited from parent commands + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
--as string
+

Username to impersonate for the operation. User could be a regular user or a service account in a namespace.

+
--as-group strings
+

Group to impersonate for the operation, this flag can be repeated to specify multiple groups.

+
--as-uid string
+

UID to impersonate for the operation.

+
--cache-dir string     Default: "$HOME/.kube/cache"
+

Default cache directory

+
--certificate-authority string
+

Path to a cert file for the certificate authority

+
--client-certificate string
+

Path to a client certificate file for TLS

+
--client-key string
+

Path to a client key file for TLS

+
--cluster string
+

The name of the kubeconfig cluster to use

+
--context string
+

The name of the kubeconfig context to use

+
--disable-compression
+

If true, opt-out of response compression for all requests to the server

+
--insecure-skip-tls-verify
+

If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure

+
--kubeconfig string
+

Path to the kubeconfig file to use for CLI requests.

+
-n, --namespace string
+

If present, the namespace scope for this CLI request

+
--request-timeout string     Default: "0"
+

The length of time to wait before giving up on a single server request. Non-zero values should contain a corresponding time unit (e.g. 1s, 2m, 3h). A value of zero means don't timeout requests.

+
-s, --server string
+

The address and port of the Kubernetes API server

+
--tls-server-name string
+

Server name to use for server certificate validation. If it is not provided, the hostname used to contact the server is used

+
--token string
+

Bearer token for authentication to the API server

+
--user string
+

The name of the kubeconfig user to use

+
+ + + +## See Also + +* [kjobctl_create](_index.md) - Create a task + diff --git a/docs/commands/kjobctl_create/kjobctl_create_slurm.md b/docs/commands/kjobctl_create/kjobctl_create_slurm.md new file mode 100644 index 0000000..6f80852 --- /dev/null +++ b/docs/commands/kjobctl_create/kjobctl_create_slurm.md @@ -0,0 +1,354 @@ + + +# kjobctl create slurm + + +## Synopsis + + +Create a slurm job + +``` +kjobctl create slurm --profile APPLICATION_PROFILE_NAME [--localqueue LOCAL_QUEUE_NAME] [--skip-localqueue-validation] [--priority NAME] [--skip-priority-validation] [--ignore-unknown-flags] [--init-image IMAGE] [--first-node-ip] [--first-node-ip-timeout DURATION] -- [--array ARRAY] [--cpus-per-task QUANTITY] [--gpus-per-task QUANTITY] [--mem QUANTITY] [--mem-per-task QUANTITY] [--mem-per-cpu QUANTITY] [--mem-per-gpu QUANTITY] [--nodes COUNT] [--ntasks COUNT] [--output FILENAME_PATTERN] [--error FILENAME_PATTERN] [--input FILENAME_PATTERN] [--job-name NAME] [--partition NAME] SCRIPT +``` + + +## Examples + +``` + # Create slurm + kjobctl create slurm --profile my-application-profile -- \ + --array 0-5 --nodes 3 --ntasks 1 ./script.sh +``` + + +## Options + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
--allow-missing-template-keys     Default: true
+

If true, ignore any errors in templates when a field or map key is missing in the template. Only applies to golang and jsonpath output formats.

+
--dry-run string     Default: "none"
+

Must be "none", "server", or "client". If client strategy, only print the object that would be sent, without sending it. If server strategy, submit server-side request without persisting the resource.

+
--first-node-ip
+

Enable the retrieval of the first node's IP address.

+
--first-node-ip-timeout duration     Default: 1m0s
+

The timeout for the retrieval of the first node's IP address.

+
-h, --help
+

help for slurm

+
--ignore-unknown-flags
+

Ignore all the unsupported flags in the bash script.

+
--init-image string     Default: "registry.k8s.io/busybox:1.27.2"
+

The image used for the init container.

+
--localqueue string
+

Kueue localqueue name which is associated with the resource.

+
-o, --output string
+

Output format. One of: (json, yaml, name, go-template, go-template-file, template, templatefile, jsonpath, jsonpath-as-json, jsonpath-file).

+
--priority string
+

Apply priority for the entire workload.

+
-p, --profile string
+

Application profile contains a template (with defaults set) for running a specific type of application.

+
--show-managed-fields
+

If true, keep the managedFields when printing objects in JSON or YAML format.

+
--skip-localqueue-validation
+

Skip local queue validation. Add local queue even if the queue does not exist.

+
--skip-priority-validation
+

Skip workload priority class validation. Add priority class label even if the class does not exist.

+
--template string
+

Template string or path to template file to use when -o=go-template, -o=go-template-file. The template format is golang templates [http://golang.org/pkg/text/template/#pkg-overview].

+
+ + + +## Options inherited from parent commands + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
--as string
+

Username to impersonate for the operation. User could be a regular user or a service account in a namespace.

+
--as-group strings
+

Group to impersonate for the operation, this flag can be repeated to specify multiple groups.

+
--as-uid string
+

UID to impersonate for the operation.

+
--cache-dir string     Default: "$HOME/.kube/cache"
+

Default cache directory

+
--certificate-authority string
+

Path to a cert file for the certificate authority

+
--client-certificate string
+

Path to a client certificate file for TLS

+
--client-key string
+

Path to a client key file for TLS

+
--cluster string
+

The name of the kubeconfig cluster to use

+
--context string
+

The name of the kubeconfig context to use

+
--disable-compression
+

If true, opt-out of response compression for all requests to the server

+
--insecure-skip-tls-verify
+

If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure

+
--kubeconfig string
+

Path to the kubeconfig file to use for CLI requests.

+
-n, --namespace string
+

If present, the namespace scope for this CLI request

+
--request-timeout string     Default: "0"
+

The length of time to wait before giving up on a single server request. Non-zero values should contain a corresponding time unit (e.g. 1s, 2m, 3h). A value of zero means don't timeout requests.

+
-s, --server string
+

The address and port of the Kubernetes API server

+
--tls-server-name string
+

Server name to use for server certificate validation. If it is not provided, the hostname used to contact the server is used

+
--token string
+

Bearer token for authentication to the API server

+
--user string
+

The name of the kubeconfig user to use

+
+ + + +## See Also + +* [kjobctl_create](_index.md) - Create a task + diff --git a/docs/commands/kjobctl_delete/_index.md b/docs/commands/kjobctl_delete/_index.md new file mode 100644 index 0000000..0197832 --- /dev/null +++ b/docs/commands/kjobctl_delete/_index.md @@ -0,0 +1,237 @@ + + +# kjobctl delete + + +## Synopsis + + +Delete resources + + +## Examples + +``` + # Delete interactive shell + kjobctl delete interactive my-application-profile-interactive-k2wzd + + # Delete Job + kjobctl delete job my-application-profile-job-k2wzd + + # Delete RayJob + kjobctl delete rayjob my-application-profile-rayjob-k2wzd + + # Delete RayCluster + kjobctl delete raycluster my-application-profile-raycluster-k2wzd +``` + + +## Options + + + + + + + + + + + + + + + + +
-h, --help
+

help for delete

+
+ + + +## Options inherited from parent commands + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
--as string
+

Username to impersonate for the operation. User could be a regular user or a service account in a namespace.

+
--as-group strings
+

Group to impersonate for the operation, this flag can be repeated to specify multiple groups.

+
--as-uid string
+

UID to impersonate for the operation.

+
--cache-dir string     Default: "$HOME/.kube/cache"
+

Default cache directory

+
--certificate-authority string
+

Path to a cert file for the certificate authority

+
--client-certificate string
+

Path to a client certificate file for TLS

+
--client-key string
+

Path to a client key file for TLS

+
--cluster string
+

The name of the kubeconfig cluster to use

+
--context string
+

The name of the kubeconfig context to use

+
--disable-compression
+

If true, opt-out of response compression for all requests to the server

+
--insecure-skip-tls-verify
+

If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure

+
--kubeconfig string
+

Path to the kubeconfig file to use for CLI requests.

+
-n, --namespace string
+

If present, the namespace scope for this CLI request

+
--request-timeout string     Default: "0"
+

The length of time to wait before giving up on a single server request. Non-zero values should contain a corresponding time unit (e.g. 1s, 2m, 3h). A value of zero means don't timeout requests.

+
-s, --server string
+

The address and port of the Kubernetes API server

+
--tls-server-name string
+

Server name to use for server certificate validation. If it is not provided, the hostname used to contact the server is used

+
--token string
+

Bearer token for authentication to the API server

+
--user string
+

The name of the kubeconfig user to use

+
+ + + +## See Also + +* [kjobctl](../kjobctl.md) - ML/AI/Batch Jobs Made Easy +* [kjobctl delete interactive](kjobctl_delete_interactive.md) - Delete interactive shell +* [kjobctl delete job](kjobctl_delete_job.md) - Delete Job +* [kjobctl delete raycluster](kjobctl_delete_raycluster.md) - Delete RayCluster +* [kjobctl delete rayjob](kjobctl_delete_rayjob.md) - Delete RayJob +* [kjobctl delete slurm](kjobctl_delete_slurm.md) - Delete Slurm + diff --git a/docs/commands/kjobctl_delete/kjobctl_delete_interactive.md b/docs/commands/kjobctl_delete/kjobctl_delete_interactive.md new file mode 100644 index 0000000..3499a37 --- /dev/null +++ b/docs/commands/kjobctl_delete/kjobctl_delete_interactive.md @@ -0,0 +1,281 @@ + + +# kjobctl delete interactive + + +## Synopsis + + +Delete interactive shell + +``` +kjobctl delete interactive NAME [--cascade STRATEGY] [--dry-run STRATEGY] +``` + + +## Examples + +``` + # Delete interactive shell + kjobctl delete interactive my-application-profile-interactive-k2wzd +``` + + +## Options + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
--allow-missing-template-keys     Default: true
+

If true, ignore any errors in templates when a field or map key is missing in the template. Only applies to golang and jsonpath output formats.

+
--cascade string     Default: "background"
+

Must be "background", "orphan", or "foreground". Defaults to background.

+
--dry-run string     Default: "none"
+

Must be "none", "server", or "client". If client strategy, only print the object that would be sent, without sending it. If server strategy, submit server-side request without persisting the resource.

+
-h, --help
+

help for interactive

+
-o, --output string
+

Output format. One of: (json, yaml, name, go-template, go-template-file, template, templatefile, jsonpath, jsonpath-as-json, jsonpath-file).

+
--show-managed-fields
+

If true, keep the managedFields when printing objects in JSON or YAML format.

+
--template string
+

Template string or path to template file to use when -o=go-template, -o=go-template-file. The template format is golang templates [http://golang.org/pkg/text/template/#pkg-overview].

+
+ + + +## Options inherited from parent commands + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
--as string
+

Username to impersonate for the operation. User could be a regular user or a service account in a namespace.

+
--as-group strings
+

Group to impersonate for the operation, this flag can be repeated to specify multiple groups.

+
--as-uid string
+

UID to impersonate for the operation.

+
--cache-dir string     Default: "$HOME/.kube/cache"
+

Default cache directory

+
--certificate-authority string
+

Path to a cert file for the certificate authority

+
--client-certificate string
+

Path to a client certificate file for TLS

+
--client-key string
+

Path to a client key file for TLS

+
--cluster string
+

The name of the kubeconfig cluster to use

+
--context string
+

The name of the kubeconfig context to use

+
--disable-compression
+

If true, opt-out of response compression for all requests to the server

+
--insecure-skip-tls-verify
+

If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure

+
--kubeconfig string
+

Path to the kubeconfig file to use for CLI requests.

+
-n, --namespace string
+

If present, the namespace scope for this CLI request

+
--request-timeout string     Default: "0"
+

The length of time to wait before giving up on a single server request. Non-zero values should contain a corresponding time unit (e.g. 1s, 2m, 3h). A value of zero means don't timeout requests.

+
-s, --server string
+

The address and port of the Kubernetes API server

+
--tls-server-name string
+

Server name to use for server certificate validation. If it is not provided, the hostname used to contact the server is used

+
--token string
+

Bearer token for authentication to the API server

+
--user string
+

The name of the kubeconfig user to use

+
+ + + +## See Also + +* [kjobctl_delete](_index.md) - Delete resources + diff --git a/docs/commands/kjobctl_delete/kjobctl_delete_job.md b/docs/commands/kjobctl_delete/kjobctl_delete_job.md new file mode 100644 index 0000000..752252d --- /dev/null +++ b/docs/commands/kjobctl_delete/kjobctl_delete_job.md @@ -0,0 +1,281 @@ + + +# kjobctl delete job + + +## Synopsis + + +Delete Job + +``` +kjobctl delete job NAME [--cascade STRATEGY] [--dry-run STRATEGY] +``` + + +## Examples + +``` + # Delete Job + kjobctl delete job my-application-profile-job-k2wzd +``` + + +## Options + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
--allow-missing-template-keys     Default: true
+

If true, ignore any errors in templates when a field or map key is missing in the template. Only applies to golang and jsonpath output formats.

+
--cascade string     Default: "background"
+

Must be "background", "orphan", or "foreground". Defaults to background.

+
--dry-run string     Default: "none"
+

Must be "none", "server", or "client". If client strategy, only print the object that would be sent, without sending it. If server strategy, submit server-side request without persisting the resource.

+
-h, --help
+

help for job

+
-o, --output string
+

Output format. One of: (json, yaml, name, go-template, go-template-file, template, templatefile, jsonpath, jsonpath-as-json, jsonpath-file).

+
--show-managed-fields
+

If true, keep the managedFields when printing objects in JSON or YAML format.

+
--template string
+

Template string or path to template file to use when -o=go-template, -o=go-template-file. The template format is golang templates [http://golang.org/pkg/text/template/#pkg-overview].

+
+ + + +## Options inherited from parent commands + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
--as string
+

Username to impersonate for the operation. User could be a regular user or a service account in a namespace.

+
--as-group strings
+

Group to impersonate for the operation, this flag can be repeated to specify multiple groups.

+
--as-uid string
+

UID to impersonate for the operation.

+
--cache-dir string     Default: "$HOME/.kube/cache"
+

Default cache directory

+
--certificate-authority string
+

Path to a cert file for the certificate authority

+
--client-certificate string
+

Path to a client certificate file for TLS

+
--client-key string
+

Path to a client key file for TLS

+
--cluster string
+

The name of the kubeconfig cluster to use

+
--context string
+

The name of the kubeconfig context to use

+
--disable-compression
+

If true, opt-out of response compression for all requests to the server

+
--insecure-skip-tls-verify
+

If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure

+
--kubeconfig string
+

Path to the kubeconfig file to use for CLI requests.

+
-n, --namespace string
+

If present, the namespace scope for this CLI request

+
--request-timeout string     Default: "0"
+

The length of time to wait before giving up on a single server request. Non-zero values should contain a corresponding time unit (e.g. 1s, 2m, 3h). A value of zero means don't timeout requests.

+
-s, --server string
+

The address and port of the Kubernetes API server

+
--tls-server-name string
+

Server name to use for server certificate validation. If it is not provided, the hostname used to contact the server is used

+
--token string
+

Bearer token for authentication to the API server

+
--user string
+

The name of the kubeconfig user to use

+
+ + + +## See Also + +* [kjobctl_delete](_index.md) - Delete resources + diff --git a/docs/commands/kjobctl_delete/kjobctl_delete_raycluster.md b/docs/commands/kjobctl_delete/kjobctl_delete_raycluster.md new file mode 100644 index 0000000..8ddca7e --- /dev/null +++ b/docs/commands/kjobctl_delete/kjobctl_delete_raycluster.md @@ -0,0 +1,281 @@ + + +# kjobctl delete raycluster + + +## Synopsis + + +Delete RayCluster + +``` +kjobctl delete raycluster NAME [--cascade STRATEGY] [--dry-run STRATEGY] +``` + + +## Examples + +``` + # Delete RayCluster + kjobctl delete raycluster my-application-profile-raycluster-k2wzd +``` + + +## Options + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
--allow-missing-template-keys     Default: true
+

If true, ignore any errors in templates when a field or map key is missing in the template. Only applies to golang and jsonpath output formats.

+
--cascade string     Default: "background"
+

Must be "background", "orphan", or "foreground". Defaults to background.

+
--dry-run string     Default: "none"
+

Must be "none", "server", or "client". If client strategy, only print the object that would be sent, without sending it. If server strategy, submit server-side request without persisting the resource.

+
-h, --help
+

help for raycluster

+
-o, --output string
+

Output format. One of: (json, yaml, name, go-template, go-template-file, template, templatefile, jsonpath, jsonpath-as-json, jsonpath-file).

+
--show-managed-fields
+

If true, keep the managedFields when printing objects in JSON or YAML format.

+
--template string
+

Template string or path to template file to use when -o=go-template, -o=go-template-file. The template format is golang templates [http://golang.org/pkg/text/template/#pkg-overview].

+
+ + + +## Options inherited from parent commands + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
--as string
+

Username to impersonate for the operation. User could be a regular user or a service account in a namespace.

+
--as-group strings
+

Group to impersonate for the operation, this flag can be repeated to specify multiple groups.

+
--as-uid string
+

UID to impersonate for the operation.

+
--cache-dir string     Default: "$HOME/.kube/cache"
+

Default cache directory

+
--certificate-authority string
+

Path to a cert file for the certificate authority

+
--client-certificate string
+

Path to a client certificate file for TLS

+
--client-key string
+

Path to a client key file for TLS

+
--cluster string
+

The name of the kubeconfig cluster to use

+
--context string
+

The name of the kubeconfig context to use

+
--disable-compression
+

If true, opt-out of response compression for all requests to the server

+
--insecure-skip-tls-verify
+

If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure

+
--kubeconfig string
+

Path to the kubeconfig file to use for CLI requests.

+
-n, --namespace string
+

If present, the namespace scope for this CLI request

+
--request-timeout string     Default: "0"
+

The length of time to wait before giving up on a single server request. Non-zero values should contain a corresponding time unit (e.g. 1s, 2m, 3h). A value of zero means don't timeout requests.

+
-s, --server string
+

The address and port of the Kubernetes API server

+
--tls-server-name string
+

Server name to use for server certificate validation. If it is not provided, the hostname used to contact the server is used

+
--token string
+

Bearer token for authentication to the API server

+
--user string
+

The name of the kubeconfig user to use

+
+ + + +## See Also + +* [kjobctl_delete](_index.md) - Delete resources + diff --git a/docs/commands/kjobctl_delete/kjobctl_delete_rayjob.md b/docs/commands/kjobctl_delete/kjobctl_delete_rayjob.md new file mode 100644 index 0000000..1f09351 --- /dev/null +++ b/docs/commands/kjobctl_delete/kjobctl_delete_rayjob.md @@ -0,0 +1,281 @@ + + +# kjobctl delete rayjob + + +## Synopsis + + +Delete RayJob + +``` +kjobctl delete rayjob NAME [--cascade STRATEGY] [--dry-run STRATEGY] +``` + + +## Examples + +``` + # Delete RayJob + kjobctl delete rayjob my-application-profile-rayjob-k2wzd +``` + + +## Options + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
--allow-missing-template-keys     Default: true
+

If true, ignore any errors in templates when a field or map key is missing in the template. Only applies to golang and jsonpath output formats.

+
--cascade string     Default: "background"
+

Must be "background", "orphan", or "foreground". Defaults to background.

+
--dry-run string     Default: "none"
+

Must be "none", "server", or "client". If client strategy, only print the object that would be sent, without sending it. If server strategy, submit server-side request without persisting the resource.

+
-h, --help
+

help for rayjob

+
-o, --output string
+

Output format. One of: (json, yaml, name, go-template, go-template-file, template, templatefile, jsonpath, jsonpath-as-json, jsonpath-file).

+
--show-managed-fields
+

If true, keep the managedFields when printing objects in JSON or YAML format.

+
--template string
+

Template string or path to template file to use when -o=go-template, -o=go-template-file. The template format is golang templates [http://golang.org/pkg/text/template/#pkg-overview].

+
+ + + +## Options inherited from parent commands + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
--as string
+

Username to impersonate for the operation. User could be a regular user or a service account in a namespace.

+
--as-group strings
+

Group to impersonate for the operation, this flag can be repeated to specify multiple groups.

+
--as-uid string
+

UID to impersonate for the operation.

+
--cache-dir string     Default: "$HOME/.kube/cache"
+

Default cache directory

+
--certificate-authority string
+

Path to a cert file for the certificate authority

+
--client-certificate string
+

Path to a client certificate file for TLS

+
--client-key string
+

Path to a client key file for TLS

+
--cluster string
+

The name of the kubeconfig cluster to use

+
--context string
+

The name of the kubeconfig context to use

+
--disable-compression
+

If true, opt-out of response compression for all requests to the server

+
--insecure-skip-tls-verify
+

If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure

+
--kubeconfig string
+

Path to the kubeconfig file to use for CLI requests.

+
-n, --namespace string
+

If present, the namespace scope for this CLI request

+
--request-timeout string     Default: "0"
+

The length of time to wait before giving up on a single server request. Non-zero values should contain a corresponding time unit (e.g. 1s, 2m, 3h). A value of zero means don't timeout requests.

+
-s, --server string
+

The address and port of the Kubernetes API server

+
--tls-server-name string
+

Server name to use for server certificate validation. If it is not provided, the hostname used to contact the server is used

+
--token string
+

Bearer token for authentication to the API server

+
--user string
+

The name of the kubeconfig user to use

+
+ + + +## See Also + +* [kjobctl_delete](_index.md) - Delete resources + diff --git a/docs/commands/kjobctl_delete/kjobctl_delete_slurm.md b/docs/commands/kjobctl_delete/kjobctl_delete_slurm.md new file mode 100644 index 0000000..0c46583 --- /dev/null +++ b/docs/commands/kjobctl_delete/kjobctl_delete_slurm.md @@ -0,0 +1,281 @@ + + +# kjobctl delete slurm + + +## Synopsis + + +Delete Slurm + +``` +kjobctl delete slurm NAME [--cascade STRATEGY] [--dry-run STRATEGY] +``` + + +## Examples + +``` + # Delete Slurm + kjobctl delete slurm my-application-profile-slurm-k2wzd +``` + + +## Options + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
--allow-missing-template-keys     Default: true
+

If true, ignore any errors in templates when a field or map key is missing in the template. Only applies to golang and jsonpath output formats.

+
--cascade string     Default: "background"
+

Must be "background", "orphan", or "foreground". Defaults to background.

+
--dry-run string     Default: "none"
+

Must be "none", "server", or "client". If client strategy, only print the object that would be sent, without sending it. If server strategy, submit server-side request without persisting the resource.

+
-h, --help
+

help for slurm

+
-o, --output string
+

Output format. One of: (json, yaml, name, go-template, go-template-file, template, templatefile, jsonpath, jsonpath-as-json, jsonpath-file).

+
--show-managed-fields
+

If true, keep the managedFields when printing objects in JSON or YAML format.

+
--template string
+

Template string or path to template file to use when -o=go-template, -o=go-template-file. The template format is golang templates [http://golang.org/pkg/text/template/#pkg-overview].

+
+ + + +## Options inherited from parent commands + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
--as string
+

Username to impersonate for the operation. User could be a regular user or a service account in a namespace.

+
--as-group strings
+

Group to impersonate for the operation, this flag can be repeated to specify multiple groups.

+
--as-uid string
+

UID to impersonate for the operation.

+
--cache-dir string     Default: "$HOME/.kube/cache"
+

Default cache directory

+
--certificate-authority string
+

Path to a cert file for the certificate authority

+
--client-certificate string
+

Path to a client certificate file for TLS

+
--client-key string
+

Path to a client key file for TLS

+
--cluster string
+

The name of the kubeconfig cluster to use

+
--context string
+

The name of the kubeconfig context to use

+
--disable-compression
+

If true, opt-out of response compression for all requests to the server

+
--insecure-skip-tls-verify
+

If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure

+
--kubeconfig string
+

Path to the kubeconfig file to use for CLI requests.

+
-n, --namespace string
+

If present, the namespace scope for this CLI request

+
--request-timeout string     Default: "0"
+

The length of time to wait before giving up on a single server request. Non-zero values should contain a corresponding time unit (e.g. 1s, 2m, 3h). A value of zero means don't timeout requests.

+
-s, --server string
+

The address and port of the Kubernetes API server

+
--tls-server-name string
+

Server name to use for server certificate validation. If it is not provided, the hostname used to contact the server is used

+
--token string
+

Bearer token for authentication to the API server

+
--user string
+

The name of the kubeconfig user to use

+
+ + + +## See Also + +* [kjobctl_delete](_index.md) - Delete resources + diff --git a/docs/commands/kjobctl_describe/_index.md b/docs/commands/kjobctl_describe/_index.md new file mode 100644 index 0000000..f6ad10c --- /dev/null +++ b/docs/commands/kjobctl_describe/_index.md @@ -0,0 +1,263 @@ + + +# kjobctl describe + + +## Synopsis + + +Show details of a specific resource or group of resources. + +``` +kjobctl describe MODE NAME +``` + + +## Examples + +``` + # Describe a task with job mode + kjobctl describe job sample-job + + # Describe a task with job mode + kjobctl describe job/sample-job + + # Describe all tasks with job mode + kjobctl describe job + + # Describe tasks by label name=myLabel + kjobctl describe job -l name=myLabel +``` + + +## Options + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
-A, --all-namespaces
+

If present, list the requested object(s) across all namespaces. Namespace in current context is ignored even if specified with --namespace.

+
-h, --help
+

help for describe

+
-p, --profile string
+

Filter by profile name which is associated with the resource.

+
-l, --selector string
+

Selector (label query) to filter on, supports '=', '==', and '!='.(e.g. -l key1=value1,key2=value2). Matching objects must satisfy all of the specified label constraints.

+
+ + + +## Options inherited from parent commands + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
--as string
+

Username to impersonate for the operation. User could be a regular user or a service account in a namespace.

+
--as-group strings
+

Group to impersonate for the operation, this flag can be repeated to specify multiple groups.

+
--as-uid string
+

UID to impersonate for the operation.

+
--cache-dir string     Default: "$HOME/.kube/cache"
+

Default cache directory

+
--certificate-authority string
+

Path to a cert file for the certificate authority

+
--client-certificate string
+

Path to a client certificate file for TLS

+
--client-key string
+

Path to a client key file for TLS

+
--cluster string
+

The name of the kubeconfig cluster to use

+
--context string
+

The name of the kubeconfig context to use

+
--disable-compression
+

If true, opt-out of response compression for all requests to the server

+
--insecure-skip-tls-verify
+

If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure

+
--kubeconfig string
+

Path to the kubeconfig file to use for CLI requests.

+
-n, --namespace string
+

If present, the namespace scope for this CLI request

+
--request-timeout string     Default: "0"
+

The length of time to wait before giving up on a single server request. Non-zero values should contain a corresponding time unit (e.g. 1s, 2m, 3h). A value of zero means don't timeout requests.

+
-s, --server string
+

The address and port of the Kubernetes API server

+
--tls-server-name string
+

Server name to use for server certificate validation. If it is not provided, the hostname used to contact the server is used

+
--token string
+

Bearer token for authentication to the API server

+
--user string
+

The name of the kubeconfig user to use

+
+ + + +## See Also + +* [kjobctl](../kjobctl.md) - ML/AI/Batch Jobs Made Easy + diff --git a/docs/commands/kjobctl_list/_index.md b/docs/commands/kjobctl_list/_index.md new file mode 100644 index 0000000..593d582 --- /dev/null +++ b/docs/commands/kjobctl_list/_index.md @@ -0,0 +1,228 @@ + + +# kjobctl list + + +## Synopsis + + +Display resources + + +## Examples + +``` + # List Job + kjobctl list job +``` + + +## Options + + + + + + + + + + + + + + + + +
-h, --help
+

help for list

+
+ + + +## Options inherited from parent commands + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
--as string
+

Username to impersonate for the operation. User could be a regular user or a service account in a namespace.

+
--as-group strings
+

Group to impersonate for the operation, this flag can be repeated to specify multiple groups.

+
--as-uid string
+

UID to impersonate for the operation.

+
--cache-dir string     Default: "$HOME/.kube/cache"
+

Default cache directory

+
--certificate-authority string
+

Path to a cert file for the certificate authority

+
--client-certificate string
+

Path to a client certificate file for TLS

+
--client-key string
+

Path to a client key file for TLS

+
--cluster string
+

The name of the kubeconfig cluster to use

+
--context string
+

The name of the kubeconfig context to use

+
--disable-compression
+

If true, opt-out of response compression for all requests to the server

+
--insecure-skip-tls-verify
+

If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure

+
--kubeconfig string
+

Path to the kubeconfig file to use for CLI requests.

+
-n, --namespace string
+

If present, the namespace scope for this CLI request

+
--request-timeout string     Default: "0"
+

The length of time to wait before giving up on a single server request. Non-zero values should contain a corresponding time unit (e.g. 1s, 2m, 3h). A value of zero means don't timeout requests.

+
-s, --server string
+

The address and port of the Kubernetes API server

+
--tls-server-name string
+

Server name to use for server certificate validation. If it is not provided, the hostname used to contact the server is used

+
--token string
+

Bearer token for authentication to the API server

+
--user string
+

The name of the kubeconfig user to use

+
+ + + +## See Also + +* [kjobctl](../kjobctl.md) - ML/AI/Batch Jobs Made Easy +* [kjobctl list interactive](kjobctl_list_interactive.md) - List Interactive +* [kjobctl list job](kjobctl_list_job.md) - List Job +* [kjobctl list raycluster](kjobctl_list_raycluster.md) - List RayCluster +* [kjobctl list rayjob](kjobctl_list_rayjob.md) - List RayJob +* [kjobctl list slurm](kjobctl_list_slurm.md) - List Slurm + diff --git a/docs/commands/kjobctl_list/kjobctl_list_interactive.md b/docs/commands/kjobctl_list/kjobctl_list_interactive.md new file mode 100644 index 0000000..b0a3cba --- /dev/null +++ b/docs/commands/kjobctl_list/kjobctl_list_interactive.md @@ -0,0 +1,311 @@ + + +# kjobctl list interactive + + +## Synopsis + + +List Interactive + +``` +kjobctl list interactive [--profile PROFILE_NAME] [--localqueue LOCALQUEUE_NAME] [--selector key1=value1] [--field-selector key1=value1] [--all-namespaces] +``` + + +## Examples + +``` + # List Interactive + kjobctl list interactive + + # List Interactive with profile filter + kjobctl list interactive --profile my-profile +``` + + +## Options + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
-A, --all-namespaces
+

If present, list the requested object(s) across all namespaces. Namespace in current context is ignored even if specified with --namespace.

+
--allow-missing-template-keys     Default: true
+

If true, ignore any errors in templates when a field or map key is missing in the template. Only applies to golang and jsonpath output formats.

+
--field-selector string
+

Selector (field query) to filter on, supports '=', '==', and '!='.(e.g. --field-selector key1=value1,key2=value2). The server only supports a limited number of field queries per type.

+
-h, --help
+

help for interactive

+
-q, --localqueue string
+

Filter by localqueue which is associated with the resource.

+
-o, --output string
+

Output format. One of: (json, yaml, name, go-template, go-template-file, template, templatefile, jsonpath, jsonpath-as-json, jsonpath-file).

+
-p, --profile string
+

Filter by profile name which is associated with the resource.

+
-l, --selector string
+

Selector (label query) to filter on, supports '=', '==', and '!='.(e.g. -l key1=value1,key2=value2). Matching objects must satisfy all of the specified label constraints.

+
--show-managed-fields
+

If true, keep the managedFields when printing objects in JSON or YAML format.

+
--template string
+

Template string or path to template file to use when -o=go-template, -o=go-template-file. The template format is golang templates [http://golang.org/pkg/text/template/#pkg-overview].

+
+ + + +## Options inherited from parent commands + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
--as string
+

Username to impersonate for the operation. User could be a regular user or a service account in a namespace.

+
--as-group strings
+

Group to impersonate for the operation, this flag can be repeated to specify multiple groups.

+
--as-uid string
+

UID to impersonate for the operation.

+
--cache-dir string     Default: "$HOME/.kube/cache"
+

Default cache directory

+
--certificate-authority string
+

Path to a cert file for the certificate authority

+
--client-certificate string
+

Path to a client certificate file for TLS

+
--client-key string
+

Path to a client key file for TLS

+
--cluster string
+

The name of the kubeconfig cluster to use

+
--context string
+

The name of the kubeconfig context to use

+
--disable-compression
+

If true, opt-out of response compression for all requests to the server

+
--insecure-skip-tls-verify
+

If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure

+
--kubeconfig string
+

Path to the kubeconfig file to use for CLI requests.

+
-n, --namespace string
+

If present, the namespace scope for this CLI request

+
--request-timeout string     Default: "0"
+

The length of time to wait before giving up on a single server request. Non-zero values should contain a corresponding time unit (e.g. 1s, 2m, 3h). A value of zero means don't timeout requests.

+
-s, --server string
+

The address and port of the Kubernetes API server

+
--tls-server-name string
+

Server name to use for server certificate validation. If it is not provided, the hostname used to contact the server is used

+
--token string
+

Bearer token for authentication to the API server

+
--user string
+

The name of the kubeconfig user to use

+
+ + + +## See Also + +* [kjobctl_list](_index.md) - Display resources + diff --git a/docs/commands/kjobctl_list/kjobctl_list_job.md b/docs/commands/kjobctl_list/kjobctl_list_job.md new file mode 100644 index 0000000..10c3300 --- /dev/null +++ b/docs/commands/kjobctl_list/kjobctl_list_job.md @@ -0,0 +1,311 @@ + + +# kjobctl list job + + +## Synopsis + + +List Job + +``` +kjobctl list job [--profile PROFILE_NAME] [--localqueue LOCALQUEUE_NAME] [--selector key1=value1] [--field-selector key1=value1] [--all-namespaces] +``` + + +## Examples + +``` + # List Job + kjobctl list job + + # List Job with profile filter + kjobctl list job --profile my-profile +``` + + +## Options + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
-A, --all-namespaces
+

If present, list the requested object(s) across all namespaces. Namespace in current context is ignored even if specified with --namespace.

+
--allow-missing-template-keys     Default: true
+

If true, ignore any errors in templates when a field or map key is missing in the template. Only applies to golang and jsonpath output formats.

+
--field-selector string
+

Selector (field query) to filter on, supports '=', '==', and '!='.(e.g. --field-selector key1=value1,key2=value2). The server only supports a limited number of field queries per type.

+
-h, --help
+

help for job

+
-q, --localqueue string
+

Filter by localqueue which is associated with the resource.

+
-o, --output string
+

Output format. One of: (json, yaml, name, go-template, go-template-file, template, templatefile, jsonpath, jsonpath-as-json, jsonpath-file).

+
-p, --profile string
+

Filter by profile name which is associated with the resource.

+
-l, --selector string
+

Selector (label query) to filter on, supports '=', '==', and '!='.(e.g. -l key1=value1,key2=value2). Matching objects must satisfy all of the specified label constraints.

+
--show-managed-fields
+

If true, keep the managedFields when printing objects in JSON or YAML format.

+
--template string
+

Template string or path to template file to use when -o=go-template, -o=go-template-file. The template format is golang templates [http://golang.org/pkg/text/template/#pkg-overview].

+
+ + + +## Options inherited from parent commands + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
--as string
+

Username to impersonate for the operation. User could be a regular user or a service account in a namespace.

+
--as-group strings
+

Group to impersonate for the operation, this flag can be repeated to specify multiple groups.

+
--as-uid string
+

UID to impersonate for the operation.

+
--cache-dir string     Default: "$HOME/.kube/cache"
+

Default cache directory

+
--certificate-authority string
+

Path to a cert file for the certificate authority

+
--client-certificate string
+

Path to a client certificate file for TLS

+
--client-key string
+

Path to a client key file for TLS

+
--cluster string
+

The name of the kubeconfig cluster to use

+
--context string
+

The name of the kubeconfig context to use

+
--disable-compression
+

If true, opt-out of response compression for all requests to the server

+
--insecure-skip-tls-verify
+

If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure

+
--kubeconfig string
+

Path to the kubeconfig file to use for CLI requests.

+
-n, --namespace string
+

If present, the namespace scope for this CLI request

+
--request-timeout string     Default: "0"
+

The length of time to wait before giving up on a single server request. Non-zero values should contain a corresponding time unit (e.g. 1s, 2m, 3h). A value of zero means don't timeout requests.

+
-s, --server string
+

The address and port of the Kubernetes API server

+
--tls-server-name string
+

Server name to use for server certificate validation. If it is not provided, the hostname used to contact the server is used

+
--token string
+

Bearer token for authentication to the API server

+
--user string
+

The name of the kubeconfig user to use

+
+ + + +## See Also + +* [kjobctl_list](_index.md) - Display resources + diff --git a/docs/commands/kjobctl_list/kjobctl_list_raycluster.md b/docs/commands/kjobctl_list/kjobctl_list_raycluster.md new file mode 100644 index 0000000..49d981f --- /dev/null +++ b/docs/commands/kjobctl_list/kjobctl_list_raycluster.md @@ -0,0 +1,311 @@ + + +# kjobctl list raycluster + + +## Synopsis + + +List RayCluster + +``` +kjobctl list raycluster [--profile PROFILE_NAME] [--localqueue LOCALQUEUE_NAME] [--selector key1=value1] [--field-selector key1=value1] [--all-namespaces] +``` + + +## Examples + +``` + # List RayCluster + kjobctl list raycluster + + # List RayCluster with profile filter + kjobctl list raycluster --profile my-profile +``` + + +## Options + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
-A, --all-namespaces
+

If present, list the requested object(s) across all namespaces. Namespace in current context is ignored even if specified with --namespace.

+
--allow-missing-template-keys     Default: true
+

If true, ignore any errors in templates when a field or map key is missing in the template. Only applies to golang and jsonpath output formats.

+
--field-selector string
+

Selector (field query) to filter on, supports '=', '==', and '!='.(e.g. --field-selector key1=value1,key2=value2). The server only supports a limited number of field queries per type.

+
-h, --help
+

help for raycluster

+
-q, --localqueue string
+

Filter by localqueue which is associated with the resource.

+
-o, --output string
+

Output format. One of: (json, yaml, name, go-template, go-template-file, template, templatefile, jsonpath, jsonpath-as-json, jsonpath-file).

+
-p, --profile string
+

Filter by profile name which is associated with the resource.

+
-l, --selector string
+

Selector (label query) to filter on, supports '=', '==', and '!='.(e.g. -l key1=value1,key2=value2). Matching objects must satisfy all of the specified label constraints.

+
--show-managed-fields
+

If true, keep the managedFields when printing objects in JSON or YAML format.

+
--template string
+

Template string or path to template file to use when -o=go-template, -o=go-template-file. The template format is golang templates [http://golang.org/pkg/text/template/#pkg-overview].

+
+ + + +## Options inherited from parent commands + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
--as string
+

Username to impersonate for the operation. User could be a regular user or a service account in a namespace.

+
--as-group strings
+

Group to impersonate for the operation, this flag can be repeated to specify multiple groups.

+
--as-uid string
+

UID to impersonate for the operation.

+
--cache-dir string     Default: "$HOME/.kube/cache"
+

Default cache directory

+
--certificate-authority string
+

Path to a cert file for the certificate authority

+
--client-certificate string
+

Path to a client certificate file for TLS

+
--client-key string
+

Path to a client key file for TLS

+
--cluster string
+

The name of the kubeconfig cluster to use

+
--context string
+

The name of the kubeconfig context to use

+
--disable-compression
+

If true, opt-out of response compression for all requests to the server

+
--insecure-skip-tls-verify
+

If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure

+
--kubeconfig string
+

Path to the kubeconfig file to use for CLI requests.

+
-n, --namespace string
+

If present, the namespace scope for this CLI request

+
--request-timeout string     Default: "0"
+

The length of time to wait before giving up on a single server request. Non-zero values should contain a corresponding time unit (e.g. 1s, 2m, 3h). A value of zero means don't timeout requests.

+
-s, --server string
+

The address and port of the Kubernetes API server

+
--tls-server-name string
+

Server name to use for server certificate validation. If it is not provided, the hostname used to contact the server is used

+
--token string
+

Bearer token for authentication to the API server

+
--user string
+

The name of the kubeconfig user to use

+
+ + + +## See Also + +* [kjobctl_list](_index.md) - Display resources + diff --git a/docs/commands/kjobctl_list/kjobctl_list_rayjob.md b/docs/commands/kjobctl_list/kjobctl_list_rayjob.md new file mode 100644 index 0000000..5d28f86 --- /dev/null +++ b/docs/commands/kjobctl_list/kjobctl_list_rayjob.md @@ -0,0 +1,311 @@ + + +# kjobctl list rayjob + + +## Synopsis + + +List RayJob + +``` +kjobctl list rayjob [--profile PROFILE_NAME] [--localqueue LOCALQUEUE_NAME] [--selector key1=value1] [--field-selector key1=value1] [--all-namespaces] +``` + + +## Examples + +``` + # List RayJob + kjobctl list rayjob + + # List RayJob with profile filter + kjobctl list rayjob --profile my-profile +``` + + +## Options + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
-A, --all-namespaces
+

If present, list the requested object(s) across all namespaces. Namespace in current context is ignored even if specified with --namespace.

+
--allow-missing-template-keys     Default: true
+

If true, ignore any errors in templates when a field or map key is missing in the template. Only applies to golang and jsonpath output formats.

+
--field-selector string
+

Selector (field query) to filter on, supports '=', '==', and '!='.(e.g. --field-selector key1=value1,key2=value2). The server only supports a limited number of field queries per type.

+
-h, --help
+

help for rayjob

+
-q, --localqueue string
+

Filter by localqueue which is associated with the resource.

+
-o, --output string
+

Output format. One of: (json, yaml, name, go-template, go-template-file, template, templatefile, jsonpath, jsonpath-as-json, jsonpath-file).

+
-p, --profile string
+

Filter by profile name which is associated with the resource.

+
-l, --selector string
+

Selector (label query) to filter on, supports '=', '==', and '!='.(e.g. -l key1=value1,key2=value2). Matching objects must satisfy all of the specified label constraints.

+
--show-managed-fields
+

If true, keep the managedFields when printing objects in JSON or YAML format.

+
--template string
+

Template string or path to template file to use when -o=go-template, -o=go-template-file. The template format is golang templates [http://golang.org/pkg/text/template/#pkg-overview].

+
+ + + +## Options inherited from parent commands + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
--as string
+

Username to impersonate for the operation. User could be a regular user or a service account in a namespace.

+
--as-group strings
+

Group to impersonate for the operation, this flag can be repeated to specify multiple groups.

+
--as-uid string
+

UID to impersonate for the operation.

+
--cache-dir string     Default: "$HOME/.kube/cache"
+

Default cache directory

+
--certificate-authority string
+

Path to a cert file for the certificate authority

+
--client-certificate string
+

Path to a client certificate file for TLS

+
--client-key string
+

Path to a client key file for TLS

+
--cluster string
+

The name of the kubeconfig cluster to use

+
--context string
+

The name of the kubeconfig context to use

+
--disable-compression
+

If true, opt-out of response compression for all requests to the server

+
--insecure-skip-tls-verify
+

If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure

+
--kubeconfig string
+

Path to the kubeconfig file to use for CLI requests.

+
-n, --namespace string
+

If present, the namespace scope for this CLI request

+
--request-timeout string     Default: "0"
+

The length of time to wait before giving up on a single server request. Non-zero values should contain a corresponding time unit (e.g. 1s, 2m, 3h). A value of zero means don't timeout requests.

+
-s, --server string
+

The address and port of the Kubernetes API server

+
--tls-server-name string
+

Server name to use for server certificate validation. If it is not provided, the hostname used to contact the server is used

+
--token string
+

Bearer token for authentication to the API server

+
--user string
+

The name of the kubeconfig user to use

+
+ + + +## See Also + +* [kjobctl_list](_index.md) - Display resources + diff --git a/docs/commands/kjobctl_list/kjobctl_list_slurm.md b/docs/commands/kjobctl_list/kjobctl_list_slurm.md new file mode 100644 index 0000000..a00088e --- /dev/null +++ b/docs/commands/kjobctl_list/kjobctl_list_slurm.md @@ -0,0 +1,311 @@ + + +# kjobctl list slurm + + +## Synopsis + + +List Slurm + +``` +kjobctl list slurm [--profile PROFILE_NAME] [--localqueue LOCALQUEUE_NAME] [--selector key1=value1] [--field-selector key1=value1] [--all-namespaces] +``` + + +## Examples + +``` + # List Slurm + kjobctl list slurm + + # List Slurm with profile filter + kjobctl list slurm --profile my-profile +``` + + +## Options + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
-A, --all-namespaces
+

If present, list the requested object(s) across all namespaces. Namespace in current context is ignored even if specified with --namespace.

+
--allow-missing-template-keys     Default: true
+

If true, ignore any errors in templates when a field or map key is missing in the template. Only applies to golang and jsonpath output formats.

+
--field-selector string
+

Selector (field query) to filter on, supports '=', '==', and '!='.(e.g. --field-selector key1=value1,key2=value2). The server only supports a limited number of field queries per type.

+
-h, --help
+

help for slurm

+
-q, --localqueue string
+

Filter by localqueue which is associated with the resource.

+
-o, --output string
+

Output format. One of: (json, yaml, name, go-template, go-template-file, template, templatefile, jsonpath, jsonpath-as-json, jsonpath-file).

+
-p, --profile string
+

Filter by profile name which is associated with the resource.

+
-l, --selector string
+

Selector (label query) to filter on, supports '=', '==', and '!='.(e.g. -l key1=value1,key2=value2). Matching objects must satisfy all of the specified label constraints.

+
--show-managed-fields
+

If true, keep the managedFields when printing objects in JSON or YAML format.

+
--template string
+

Template string or path to template file to use when -o=go-template, -o=go-template-file. The template format is golang templates [http://golang.org/pkg/text/template/#pkg-overview].

+
+ + + +## Options inherited from parent commands + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
--as string
+

Username to impersonate for the operation. User could be a regular user or a service account in a namespace.

+
--as-group strings
+

Group to impersonate for the operation, this flag can be repeated to specify multiple groups.

+
--as-uid string
+

UID to impersonate for the operation.

+
--cache-dir string     Default: "$HOME/.kube/cache"
+

Default cache directory

+
--certificate-authority string
+

Path to a cert file for the certificate authority

+
--client-certificate string
+

Path to a client certificate file for TLS

+
--client-key string
+

Path to a client key file for TLS

+
--cluster string
+

The name of the kubeconfig cluster to use

+
--context string
+

The name of the kubeconfig context to use

+
--disable-compression
+

If true, opt-out of response compression for all requests to the server

+
--insecure-skip-tls-verify
+

If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure

+
--kubeconfig string
+

Path to the kubeconfig file to use for CLI requests.

+
-n, --namespace string
+

If present, the namespace scope for this CLI request

+
--request-timeout string     Default: "0"
+

The length of time to wait before giving up on a single server request. Non-zero values should contain a corresponding time unit (e.g. 1s, 2m, 3h). A value of zero means don't timeout requests.

+
-s, --server string
+

The address and port of the Kubernetes API server

+
--tls-server-name string
+

Server name to use for server certificate validation. If it is not provided, the hostname used to contact the server is used

+
--token string
+

Bearer token for authentication to the API server

+
--user string
+

The name of the kubeconfig user to use

+
+ + + +## See Also + +* [kjobctl_list](_index.md) - Display resources + diff --git a/docs/commands/kjobctl_printcrds/_index.md b/docs/commands/kjobctl_printcrds/_index.md new file mode 100644 index 0000000..b42e234 --- /dev/null +++ b/docs/commands/kjobctl_printcrds/_index.md @@ -0,0 +1,230 @@ + + +# kjobctl printcrds + + +## Synopsis + + +Print the kjobctl CRDs + +``` +kjobctl printcrds [flags] +``` + + +## Examples + +``` + # Install or update the kjobctl CRDs + kjobctl printcrds | kubectl apply --server-side -f - + + # Remove the kjobctl CRDs + kjobctl printcrds | kubectl delete --ignore-not-found=true -f - +``` + + +## Options + + + + + + + + + + + + + + + + +
-h, --help
+

help for printcrds

+
+ + + +## Options inherited from parent commands + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
--as string
+

Username to impersonate for the operation. User could be a regular user or a service account in a namespace.

+
--as-group strings
+

Group to impersonate for the operation, this flag can be repeated to specify multiple groups.

+
--as-uid string
+

UID to impersonate for the operation.

+
--cache-dir string     Default: "$HOME/.kube/cache"
+

Default cache directory

+
--certificate-authority string
+

Path to a cert file for the certificate authority

+
--client-certificate string
+

Path to a client certificate file for TLS

+
--client-key string
+

Path to a client key file for TLS

+
--cluster string
+

The name of the kubeconfig cluster to use

+
--context string
+

The name of the kubeconfig context to use

+
--disable-compression
+

If true, opt-out of response compression for all requests to the server

+
--insecure-skip-tls-verify
+

If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure

+
--kubeconfig string
+

Path to the kubeconfig file to use for CLI requests.

+
-n, --namespace string
+

If present, the namespace scope for this CLI request

+
--request-timeout string     Default: "0"
+

The length of time to wait before giving up on a single server request. Non-zero values should contain a corresponding time unit (e.g. 1s, 2m, 3h). A value of zero means don't timeout requests.

+
-s, --server string
+

The address and port of the Kubernetes API server

+
--tls-server-name string
+

Server name to use for server certificate validation. If it is not provided, the hostname used to contact the server is used

+
--token string
+

Bearer token for authentication to the API server

+
--user string
+

The name of the kubeconfig user to use

+
+ + + +## See Also + +* [kjobctl](../kjobctl.md) - ML/AI/Batch Jobs Made Easy + diff --git a/docs/installation.md b/docs/installation.md new file mode 100644 index 0000000..4b874d6 --- /dev/null +++ b/docs/installation.md @@ -0,0 +1,59 @@ +# Installation + +Installing the `kubectl-kjob` plugin, `kjobctl`. + +## Installing + +### From source + +```bash +make kubectl-kjob +sudo mv ./bin/kubectl-kjob /usr/local/bin/kubectl-kjob +``` + +## Installing CRDs + +### Using printcrds command + +```bash +kubectl-kjob printcrds | kubectl apply --server-side -f - +``` + +### From source + +```bash +make install +``` + +## Kjobctl + +Additionally, you can create an alias `kjobctl` to allow shorter syntax. + +```bash +echo 'alias kjobctl="kubectl kjob"' >> ~/.bashrc +# Or if you are using ZSH +echo 'alias kjobctl="kubectl kjob"' >> ~/.zshrc +``` + +## Autocompletion + +```bash +echo '[[ $commands[kubectl-kjob] ]] && source <(kubectl-kjob completion bash)' >> ~/.bashrc +# Or if you are using ZSH +echo '[[ $commands[kubectl-kjob] ]] && source <(kubectl-kjob completion zsh)' >> ~/.zshrc + +cat <kubectl_complete-kjob +#!/usr/bin/env sh + +# Call the __complete command passing it all arguments +kubectl kjob __complete "\$@" +EOF + +chmod u+x kubectl_complete-kjob +sudo mv kubectl_complete-kjob /usr/local/bin/kubectl_complete-kjob +``` + +## See Also + +* [overview](_index.md) - `kubectl-kjob` plugin, `kjobctl` overview. +* [commands](commands/kjobctl.md) - Full list of commands, along with all possible flags. \ No newline at end of file diff --git a/docs/run_slurm.md b/docs/run_slurm.md new file mode 100644 index 0000000..fc4c7b7 --- /dev/null +++ b/docs/run_slurm.md @@ -0,0 +1,246 @@ +# Run a Slurm Job on Kubernetes + +This page demonstrates how to run a Slurm job on a Kubernetes cluster using Kjob. + +## Before You Begin + +Ensure the following prerequisites are met: + +- A Kubernetes cluster is operational. +- The [kjobctl](installation.md) plugin is installed. + +## Slurm Mode in Kjob + +This mode is specifically designed to offer a compatibility mode for [Slurm](https://slurm.schedmd.com/) jobs. Kjob does not directly incorporate Slurm's components. + +Instead, it simulates Slurm behavior to facilitate a smooth transition for users. + +By translating Slurm concepts such as nodes into Kubernetes-compatible terms like pods, Kjob ensures that users familiar with Slurm will find the Kjob environment intuitive and easy to use. + +The addition of the Slurm mode to the ApplicationProfile in Kjob aims to replicate the experience of using the sbatch command in Slurm as closely as possible, thereby enhancing user satisfaction and efficiency in managing cluster jobs within a Kubernetes context. + +### Supported Options + +Kjob provides support for executing Slurm scripts by offering several options that can be specified either through the command line interface, `Kjobctl`, or via the `#SBATCH` directive embedded within the bash script. These options are detailed in the table below: + +| Option | Description | +|---------------------|-------------| +| -a, --array | See [array option](https://slurm.schedmd.com/sbatch.html#OPT_array) for the specification. | +| -c, --cpus-per-task | Specifies how many CPUs a container inside a pod requires. | +| -e, --error | Specifies where to redirect the standard error stream of a task. If not passed, it proceeds to stdout and is available via `kubectl logs`. | +| --gpus-per-task | Specifies how many GPUs a container inside a pod requires. | +| -i, --input | Specifies what to pipe into the script. | +| -J, --job-name= | Specifies the job name. | +| --mem | Specifies how much memory a pod requires. | +| --mem-per-cpu | Specifies how much memory a container requires, multiplying the number of requested CPUs per task by mem-per-cpu. | +| --mem-per-gpu | Specifies how much memory a container requires, multiplying the number of requested GPUs per task by mem-per-gpu. | +| --mem-per-task | Specifies how much memory a container requires. | +| -N, --nodes | Specifies the number of pods to be used at a time - parallelism in indexed jobs. | +| -n, --ntasks | Specifies the number of identical containers inside of a pod, usually 1. | +| -o, --output | Specifies where to redirect the standard output stream of a task. If not passed, it proceeds to stdout and is available via `kubectl logs`. | +| --partition | Specifies the local queue name. See [Local Queue](https://kueue.sigs.k8s.io/docs/concepts/local_queue/) for more information. | +| -D, --chdir | Change directory before executing the script. | +| -t, --time | Set a limit on the total run time of the job. A time limit of zero requests that no time limit be imposed. Acceptable time formats include "minutes", "minutes:seconds", "hours:minutes:seconds", "days-hours", "days-hours:minutes" and "days-hours:minutes:seconds". | + +If an unsupported flag is passed in the script, the command will fail with an error unless `--ignore-unknown-flags` is given. + +### Supported Input Environment Variables + +> NOTE: Environment variables will override any options set in a batch script, and command line +> options will override any environment variables. + +| Name | Description | +|-----------------------|-------------------------| +| $SBATCH_ARRAY_INX | Same as -a, --array | +| $SBATCH_GPUS_PER_TASK | Same as --gpus-per-task | +| $SBATCH_MEM_PER_NODE | Same as --mem | +| $SBATCH_MEM_PER_CPU | Same as --mem-per-cpu | +| $SBATCH_MEM_PER_GPU | Same as --mem-per-gpu | +| $SBATCH_OUTPUT | Same as -o, --output | +| $SBATCH_ERROR | Same as -e, --error | +| $SBATCH_INPUT | Same as -i, --input | +| $SBATCH_JOB_NAME | Same as -J, --job-name | +| $SBATCH_PARTITION | Same as -p, --partition | +| $SBATCH_TIMELIMIT | Same as -t, --time | + +### Supported Output Environment Variables + +| Name | Description | +|------------------------------|-------------| +| $SLURM_ARRAY_TASK_ID | Job array ID (index) number. | +| $SLURM_JOB_ID | The Job ID. | +| $SLURM_JOBID | Deprecated. Same as $SLURM_JOB_ID. | +| $SLURM_SUBMIT_DIR | The path of the job submission directory. | +| $SLURM_SUBMIT_HOST | The hostname of the node used for job submission. | +| $SLURM_JOB_NODELIST | Contains the definition (list) of the nodes (actually pods) that is assigned to the job. | +| $SLURM_JOB_FIRST_NODE | First element of SLURM_JOB_NODELIST. | +| $SLURM_JOB_FIRST_NODE_IP | IP of the first element, obtained via nslookup. | +| $SLURM_CPUS_PER_TASK | Number of CPUs per task. | +| $SLURM_CPUS_ON_NODE | Number of CPUs on the allocated node (actually pod). | +| $SLURM_JOB_CPUS_PER_NODE | Count of processors available to the job on this node. | +| $SLURM_CPUS_PER_GPU | Number of CPUs requested per allocated GPU. | +| $SLURM_MEM_PER_CPU | Memory per CPU. Same as --mem-per-cpu. | +| $SLURM_MEM_PER_GPU | Memory per GPU. | +| $SLURM_MEM_PER_NODE | Memory per node. Same as --mem. | +| $SLURM_GPUS | Number of GPUs requested (in total). | +| $SLURM_NTASKS | Same as -n, --ntasks. The number of tasks. | +| $SLURM_NTASKS_PER_NODE | Number of tasks requested per node. | +| $SLURM_NTASKS_PER_SOCKET | Number of tasks requested per socket. To be supported later. | +| $SLURM_NTASKS_PER_CORE | Number of tasks requested per core. To be supported later. | +| $SLURM_NTASKS_PER_GPU | Number of tasks requested per GPU. To be supported later. | +| $SLURM_NPROCS | Same as -n, --ntasks. See $SLURM_NTASKS. | +| $SLURM_NNODES | Total number of nodes (actually pods) in the job’s resource allocation. | +| $SLURM_TASKS_PER_NODE | Number of tasks to be initiated on each node. | +| $SLURM_ARRAY_JOB_ID | Job array’s master job ID number. For now, same as $SLURM_JOB_ID. | +| $SLURM_ARRAY_TASK_COUNT | Total number of tasks in a job array. | +| $SLURM_ARRAY_TASK_MAX | Job array’s maximum ID (index) number. | +| $SLURM_ARRAY_TASK_MIN | Job array’s minimum ID (index) number. | + +## Example + +The following example demonstrates a use case for running a Python script. + +See [config/samples/slurm](../config/samples/slurm/) for the full samples used in this section. + +### 1. Create Slurm Template + +First, you need to create a template for your job using the `JobTemplate` kind. This template should specify the container image with all the dependencies to run your task. Below is an example of a template to run a Python script. + +```yaml +apiVersion: kjobctl.x-k8s.io/v1alpha1 +kind: JobTemplate +metadata: + name: slurm-template + namespace: default +template: + spec: + parallelism: 3 + completions: 3 + completionMode: Indexed + template: + spec: + containers: + - name: sample-container + image: python:3-slim + restartPolicy: OnFailure +``` + +Once you have created your template, you need an `ApplicationProfile` containing `Slurm` and your new template in the supported modes as shown below: + +```yaml +apiVersion: kjobctl.x-k8s.io/v1alpha1 +kind: ApplicationProfile +metadata: + name: slurm-profile + namespace: default +spec: + supportedModes: + - name: Slurm + template: slurm-template + volumeBundles: ["slurm-volume-bundle"] +``` + +Then, save the file as `application_profile.yaml` and create the ApplicationProfile by running: + +```bash +kubectl create -f application_profile.yaml +``` + +> Note: This setup process, including the creation of the JobTemplate and ApplicationProfile, only needs to be completed once and can be applied to subsequent Python script executions. + +### 2. Prepare the Python Script for the Job + +Before loading the Python script for use with the pods, it's crucial to consider how the script will be made accessible. + +For the purposes of this tutorial, we'll utilize a ConfigMap to store and pass the Python script. While this method simplifies the setup and is suitable for demonstration purposes, for a scalable setup, using NFS or a similar shared file system is necessary. + +#### Creating a ConfigMap with Your Python Script + +Create a file named `sample_code.py` and add your Python script. For example: + +```python +import time + +print('Start at ' + time.strftime('%H:%M:%S')) + +print('Sleep for 10 seconds...') +time.sleep(10) + +print('Stop at ' + time.strftime('%H:%M:%S')) +``` + +To create a `ConfigMap` from this Python script, save it and then run the following command: + +```bash +kubectl create configmap slurm-code-sample --from-file=sample_code.py +``` + +This command packages your script into a `ConfigMap` named `slurm-code-sample`. + +### 3. Integrate the Script with a VolumeBundle + +With your Python script now encapsulated in a `ConfigMap`, the next step is to make it accessible to your Slurm pods through a `VolumeBundle`. + +#### Creating a VolumeBundle + +Define a `VolumeBundle` object to specify how the containers should mount the volumes containing your script: + +```yaml +apiVersion: kjobctl.x-k8s.io/v1alpha1 +kind: VolumeBundle +metadata: + name: slurm-volume-bundle + namespace: default +spec: + volumes: + - name: slurm-code-sample + configMap: + name: slurm-code-sample + items: + - key: sample_code.py + path: sample_code.py + containerVolumeMounts: + - name: slurm-code-sample + mountPath: /home/slurm/samples + envVars: + - name: ENTRYPOINT_PATH + value: /home/slurm/samples +``` + +Save this definition as a file named `slurm_volume_bundle.yaml`. And then, apply this configuration to your Kubernetes cluster by running: + +```bash +kubectl create -f slurm_volume_bundle.yaml +``` + +This setup mounts the `ConfigMap` as a volume at `/home/slurm/samples` on the pods, making your Python script readily accessible to be executed as part of your Slurm job. + +### 4. Submit Job + +After setting up the `ApplicationProfile` and `VolumeBundle`, you're ready to run tasks using your template. To submit a job, you need to pass a batch script to the `create slurm` command. You can run your script with custom configurations by the supported options via the `#SBATCH` directive or flags. + +Below is an example of a simple batch script which wraps the Python script loaded in the previous step and executes it. Additionally, it sets the array option to run 3 multiple jobs and the maximum number of simultaneously running tasks to 2. + +```bash +#!/bin/bash + +#SBATCH --array=1-3%2 + +echo "Now processing task ID: ${SLURM_ARRAY_TASK_ID}" +python /home/slurm/samples/sample_code.py +``` + +To run this batch script, save it as `slurm-sample.sh` and execute: + +```bash +kubectl-kjob create slurm --profile slurm-profile -- slurm-sample.sh +``` + +Now, after submitting the job, you should be able to see something similar to the following output in the logs: + +``` +Now processing task ID: 1 +Start at 17:08:04 +Sleep for 10 seconds... +Stop at 17:08:14 +``` diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..6effa8d --- /dev/null +++ b/go.mod @@ -0,0 +1,102 @@ +module sigs.k8s.io/kueue/cmd/experimental/kjobctl + +go 1.23.0 + +require ( + github.com/google/go-cmp v0.6.0 + github.com/onsi/ginkgo/v2 v2.22.0 + github.com/onsi/gomega v1.36.0 + github.com/ray-project/kuberay/ray-operator v1.2.2 + github.com/spf13/cobra v1.8.1 + github.com/spf13/pflag v1.0.5 + k8s.io/api v0.31.3 + k8s.io/apimachinery v0.31.3 + k8s.io/cli-runtime v0.31.3 + k8s.io/client-go v0.31.3 + k8s.io/component-base v0.31.3 + k8s.io/klog/v2 v2.130.1 + k8s.io/kubectl v0.31.3 + k8s.io/utils v0.0.0-20240902221715-702e33fdd3c3 + sigs.k8s.io/controller-runtime v0.19.2 + sigs.k8s.io/kueue v0.9.1 +) + +require ( + github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect + github.com/MakeNowJust/heredoc v1.0.0 // indirect + github.com/beorn7/perks v1.0.1 // indirect + github.com/blang/semver/v4 v4.0.0 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/chai2010/gettext-go v1.0.2 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/emicklei/go-restful/v3 v3.12.1 // indirect + github.com/evanphx/json-patch/v5 v5.9.0 // indirect + github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d // indirect + github.com/fatih/camelcase v1.0.0 // indirect + github.com/fsnotify/fsnotify v1.8.0 // indirect + github.com/fxamacker/cbor/v2 v2.7.0 // indirect + github.com/go-errors/errors v1.4.2 // indirect + github.com/go-logr/logr v1.4.2 // indirect + github.com/go-openapi/jsonpointer v0.21.0 // indirect + github.com/go-openapi/jsonreference v0.21.0 // indirect + github.com/go-openapi/swag v0.23.0 // indirect + github.com/go-task/slim-sprig/v3 v3.0.0 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect + github.com/golang/protobuf v1.5.4 // indirect + github.com/google/btree v1.0.1 // indirect + github.com/google/gnostic-models v0.6.8 // indirect + github.com/google/gofuzz v1.2.0 // indirect + github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db // indirect + github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/gorilla/websocket v1.5.1 // indirect + github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7 // indirect + github.com/imdario/mergo v0.3.16 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/josharian/intern v1.0.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/klauspost/compress v1.17.9 // indirect + github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect + github.com/mailru/easyjson v0.7.7 // indirect + github.com/mitchellh/go-wordwrap v1.0.1 // indirect + github.com/moby/spdystream v0.4.0 // indirect + github.com/moby/term v0.5.0 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect + github.com/peterbourgon/diskv v2.0.1+incompatible // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/prometheus/client_golang v1.20.5 // indirect + github.com/prometheus/client_model v0.6.1 // indirect + github.com/prometheus/common v0.57.0 // indirect + github.com/prometheus/procfs v0.15.1 // indirect + github.com/russross/blackfriday/v2 v2.1.0 // indirect + github.com/x448/float16 v0.8.4 // indirect + github.com/xlab/treeprint v1.2.0 // indirect + go.starlark.net v0.0.0-20230525235612-a134d8f9ddca // indirect + golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect + golang.org/x/net v0.30.0 // indirect + golang.org/x/oauth2 v0.21.0 // indirect + golang.org/x/sync v0.8.0 // indirect + golang.org/x/sys v0.26.0 // indirect + golang.org/x/term v0.25.0 // indirect + golang.org/x/text v0.19.0 // indirect + golang.org/x/time v0.5.0 // indirect + golang.org/x/tools v0.26.0 // indirect + gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect + google.golang.org/protobuf v1.35.1 // indirect + gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect + k8s.io/apiextensions-apiserver v0.31.1 // indirect + k8s.io/kube-openapi v0.0.0-20240812233141-91dab695df6f // indirect + sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect + sigs.k8s.io/kustomize/api v0.17.3 // indirect + sigs.k8s.io/kustomize/kyaml v0.17.2 // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect + sigs.k8s.io/yaml v1.4.0 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..0c29ac5 --- /dev/null +++ b/go.sum @@ -0,0 +1,328 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8= +github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/MakeNowJust/heredoc v1.0.0 h1:cXCdzVdstXyiTqTvfqk9SDHpKNjxuom+DOlyEeQ4pzQ= +github.com/MakeNowJust/heredoc v1.0.0/go.mod h1:mG5amYoWBHf8vpLOuehzbGGw0EHxpZZ6lCpQ4fNJ8LE= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= +github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/chai2010/gettext-go v1.0.2 h1:1Lwwip6Q2QGsAdl/ZKPCwTe9fe0CjlUbqj5bFNSjIRk= +github.com/chai2010/gettext-go v1.0.2/go.mod h1:y+wnP2cHYaVj19NZhYKAwEMH2CI1gNHeQQ+5AjwawxA= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY= +github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/emicklei/go-restful/v3 v3.12.1 h1:PJMDIM/ak7btuL8Ex0iYET9hxM3CI2sjZtzpL63nKAU= +github.com/emicklei/go-restful/v3 v3.12.1/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/evanphx/json-patch v5.9.0+incompatible h1:fBXyNpNMuTTDdquAq/uisOr2lShz4oaXpDTX2bLe7ls= +github.com/evanphx/json-patch v5.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/evanphx/json-patch/v5 v5.9.0 h1:kcBlZQbplgElYIlo/n1hJbls2z/1awpXxpRi0/FOJfg= +github.com/evanphx/json-patch/v5 v5.9.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ= +github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d h1:105gxyaGwCFad8crR9dcMQWvV9Hvulu6hwUh4tWPJnM= +github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d/go.mod h1:ZZMPRZwes7CROmyNKgQzC3XPs6L/G2EJLHddWejkmf4= +github.com/fatih/camelcase v1.0.0 h1:hxNvNX/xYBp0ovncs8WyWZrOrpBNub/JfaMvbURyft8= +github.com/fatih/camelcase v1.0.0/go.mod h1:yN2Sb0lFhZJUdVvtELVWefmrXpuZESvPmqwoZc+/fpc= +github.com/fsnotify/fsnotify v1.8.0 h1:dAwr6QBTBZIkG8roQaJjGof0pp0EeF+tNV7YBP3F/8M= +github.com/fsnotify/fsnotify v1.8.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= +github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E= +github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= +github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= +github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ= +github.com/go-logr/zapr v1.3.0/go.mod h1:YKepepNBd1u/oyhd/yQmtjVXmm9uML4IXUgMOwR8/Gg= +github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ= +github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY= +github.com/go-openapi/jsonreference v0.21.0 h1:Rs+Y7hSXT83Jacb7kFyjn4ijOuVGSvOdF2+tg1TRrwQ= +github.com/go-openapi/jsonreference v0.21.0/go.mod h1:LmZmgsrTkVg9LG4EaHeY8cBDslNPMo06cago5JNLkm4= +github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE= +github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ= +github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= +github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/btree v1.0.1 h1:gK4Kx5IaGY9CD5sPJ36FHiBJ6ZXl0kilRiiCj+jdYp4= +github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= +github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= +github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= +github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db h1:097atOisP2aRj7vFgYQBbFN4U4JNXUNYpxael3UzMyo= +github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= +github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= +github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gorilla/websocket v1.5.1 h1:gmztn0JnHVt9JZquRuzLw3g4wouNVzKL15iLr/zn/QY= +github.com/gorilla/websocket v1.5.1/go.mod h1:x3kM2JMyaluk02fnUJpQuwD2dCS5NDG2ZHL0uE0tcaY= +github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7 h1:pdN6V1QBWetyv/0+wjACpqVH+eVULgEjkurDLq3goeM= +github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= +github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4= +github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/jarcoal/httpmock v1.2.0 h1:gSvTxxFR/MEMfsGrvRbdfpRUMBStovlSRLw0Ep1bwwc= +github.com/jarcoal/httpmock v1.2.0/go.mod h1:oCoTsnAz4+UoOUIf5lJOWV2QQIW5UoeUI6aM2YnWAZk= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= +github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de h1:9TO3cAIGXtEhnIaL+V+BEER86oLrvS+kWobKpbJuye0= +github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de/go.mod h1:zAbeS9B/r2mtpb6U+EI2rYA5OAXxsYw6wTamcNW+zcE= +github.com/lithammer/dedent v1.1.0 h1:VNzHMVCBNG1j0fh3OrsFRkVUwStdDArbgBWoPAffktY= +github.com/lithammer/dedent v1.1.0/go.mod h1:jrXYCQtgg0nJiN+StA2KgR7w6CiQNv9Fd/Z9BP0jIOc= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/mitchellh/go-wordwrap v1.0.1 h1:TLuKupo69TCn6TQSyGxwI1EblZZEsQ0vMlAFQflz0v0= +github.com/mitchellh/go-wordwrap v1.0.1/go.mod h1:R62XHJLzvMFRBbcrT7m7WgmE1eOyTSsCt+hzestvNj0= +github.com/moby/spdystream v0.4.0 h1:Vy79D6mHeJJjiPdFEL2yku1kl0chZpJfZcPpb16BRl8= +github.com/moby/spdystream v0.4.0/go.mod h1:xBAYlnt/ay+11ShkdFKNAG7LsyK/tmNBVvVOwrfMgdI= +github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= +github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 h1:n6/2gBQ3RWajuToeY6ZtZTIKv2v7ThUy5KKusIT0yc0= +github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00/go.mod h1:Pm3mSP3c5uWn86xMLZ5Sa7JB9GsEZySvHYXCTK4E9q4= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f h1:y5//uYreIhSUg3J1GEMiLbxo1LJaP8RfCpH6pymGZus= +github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= +github.com/onsi/ginkgo/v2 v2.22.0 h1:Yed107/8DjTr0lKCNt7Dn8yQ6ybuDRQoMGrNFKzMfHg= +github.com/onsi/ginkgo/v2 v2.22.0/go.mod h1:7Du3c42kxCUegi0IImZ1wUQzMBVecgIHjR1C+NkhLQo= +github.com/onsi/gomega v1.36.0 h1:Pb12RlruUtj4XUuPUqeEWc6j5DkVVVA49Uf6YLfC95Y= +github.com/onsi/gomega v1.36.0/go.mod h1:PvZbdDc8J6XJEpDK4HCuRBm8a6Fzp9/DmhC9C7yFlog= +github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI= +github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.20.5 h1:cxppBPuYhUnsO6yo/aoRol4L7q7UFfdm+bR9r+8l63Y= +github.com/prometheus/client_golang v1.20.5/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= +github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= +github.com/prometheus/common v0.57.0 h1:Ro/rKjwdq9mZn1K5QPctzh+MA4Lp0BuYk5ZZEVhoNcY= +github.com/prometheus/common v0.57.0/go.mod h1:7uRPFSUTbfZWsJ7MHY56sqt7hLQu3bxXHDnNhl8E9qI= +github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= +github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= +github.com/ray-project/kuberay/ray-operator v1.2.2 h1:wj4qe9SmJfD1ubgEaVPuAsnU/WFDvremzR8j3JslBdk= +github.com/ray-project/kuberay/ray-operator v1.2.2/go.mod h1:osTiIyaDoWi5IN1f0tOOtZ4TzVf+5kJXZor8VFvcEiI= +github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= +github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= +github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ= +github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= +github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= +github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= +github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= +github.com/xlab/treeprint v1.2.0 h1:HzHnuAF1plUN2zGlAFHbSQP2qJ0ZAD3XF5XD7OesXRQ= +github.com/xlab/treeprint v1.2.0/go.mod h1:gj5Gd3gPdKtR1ikdDK6fnFLdmIS0X30kTTuNd/WEJu0= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.starlark.net v0.0.0-20230525235612-a134d8f9ddca h1:VdD38733bfYv5tUZwEIskMM93VanwNIi5bIKnDrJdEY= +go.starlark.net v0.0.0-20230525235612-a134d8f9ddca/go.mod h1:jxU+3+j+71eXOW14274+SmmuW82qJzl6iZSeqEtTGds= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8= +golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4= +golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.21.0 h1:tsimM75w1tF/uws5rbeHzIWxEqElMehnc+iW793zsZs= +golang.org/x/oauth2 v0.21.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= +golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo= +golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.0.0-20220526004731-065cf7ba2467/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.25.0 h1:WtHI/ltw4NvSUig5KARz9h521QvRC8RmF/cuYqifU24= +golang.org/x/term v0.25.0/go.mod h1:RPyXicDX+6vLxogjjRxjgD2TKtmAO6NZBsBRfrOLu7M= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM= +golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= +golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.26.0 h1:v/60pFQmzmT9ExmjDv2gGIfi3OqfKoEP6I5+umXlbnQ= +golang.org/x/tools v0.26.0/go.mod h1:TPVVj70c7JJ3WCazhD8OdXcZg/og+b9+tH/KxylGwH0= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw= +gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +google.golang.org/protobuf v1.35.1 h1:m3LfL6/Ca+fqnjnlqQXNpFPABW1UD7mjh8KO2mKFytA= +google.golang.org/protobuf v1.35.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/evanphx/json-patch.v4 v4.12.0 h1:n6jtcsulIzXPJaxegRbvFNNrZDjbij7ny3gmSPG+6V4= +gopkg.in/evanphx/json-patch.v4 v4.12.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +k8s.io/api v0.31.3 h1:umzm5o8lFbdN/hIXbrK9oRpOproJO62CV1zqxXrLgk8= +k8s.io/api v0.31.3/go.mod h1:UJrkIp9pnMOI9K2nlL6vwpxRzzEX5sWgn8kGQe92kCE= +k8s.io/apiextensions-apiserver v0.31.1 h1:L+hwULvXx+nvTYX/MKM3kKMZyei+UiSXQWciX/N6E40= +k8s.io/apiextensions-apiserver v0.31.1/go.mod h1:tWMPR3sgW+jsl2xm9v7lAyRF1rYEK71i9G5dRtkknoQ= +k8s.io/apimachinery v0.31.3 h1:6l0WhcYgasZ/wk9ktLq5vLaoXJJr5ts6lkaQzgeYPq4= +k8s.io/apimachinery v0.31.3/go.mod h1:rsPdaZJfTfLsNJSQzNHQvYoTmxhoOEofxtOsF3rtsMo= +k8s.io/cli-runtime v0.31.3 h1:fEQD9Xokir78y7pVK/fCJN090/iYNrLHpFbGU4ul9TI= +k8s.io/cli-runtime v0.31.3/go.mod h1:Q2jkyTpl+f6AtodQvgDI8io3jrfr+Z0LyQBPJJ2Btq8= +k8s.io/client-go v0.31.3 h1:CAlZuM+PH2cm+86LOBemaJI/lQ5linJ6UFxKX/SoG+4= +k8s.io/client-go v0.31.3/go.mod h1:2CgjPUTpv3fE5dNygAr2NcM8nhHzXvxB8KL5gYc3kJs= +k8s.io/component-base v0.31.3 h1:DMCXXVx546Rfvhj+3cOm2EUxhS+EyztH423j+8sOwhQ= +k8s.io/component-base v0.31.3/go.mod h1:xME6BHfUOafRgT0rGVBGl7TuSg8Z9/deT7qq6w7qjIU= +k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= +k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= +k8s.io/kube-openapi v0.0.0-20240812233141-91dab695df6f h1:bnWtxXWdAl5bVOCEPoNdvMkyj6cTW3zxHuwKIakuV9w= +k8s.io/kube-openapi v0.0.0-20240812233141-91dab695df6f/go.mod h1:G0W3eI9gG219NHRq3h5uQaRBl4pj4ZpwzRP5ti8y770= +k8s.io/kubectl v0.31.3 h1:3r111pCjPsvnR98oLLxDMwAeM6OPGmPty6gSKaLTQes= +k8s.io/kubectl v0.31.3/go.mod h1:lhMECDCbJN8He12qcKqs2QfmVo9Pue30geovBVpH5fs= +k8s.io/utils v0.0.0-20240902221715-702e33fdd3c3 h1:b2FmK8YH+QEwq/Sy2uAEhmqL5nPfGYbJOcaqjeYYZoA= +k8s.io/utils v0.0.0-20240902221715-702e33fdd3c3/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +sigs.k8s.io/controller-runtime v0.19.2 h1:3sPrF58XQEPzbE8T81TN6selQIMGbtYwuaJ6eDssDF8= +sigs.k8s.io/controller-runtime v0.19.2/go.mod h1:iRmWllt8IlaLjvTTDLhRBXIEtkCK6hwVBJJsYS9Ajf4= +sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= +sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= +sigs.k8s.io/kueue v0.9.1 h1:4y8YCoWyZj/C1rOWLvjn9T3Hmjfe+YD7Syhii1B92Fo= +sigs.k8s.io/kueue v0.9.1/go.mod h1:9t5SwZ0XH/xK0+XSuLea4stzCC/PIja5JpFZzip2/Oo= +sigs.k8s.io/kustomize/api v0.17.3 h1:6GCuHSsxq7fN5yhF2XrC+AAr8gxQwhexgHflOAD/JJU= +sigs.k8s.io/kustomize/api v0.17.3/go.mod h1:TuDH4mdx7jTfK61SQ/j1QZM/QWR+5rmEiNjvYlhzFhc= +sigs.k8s.io/kustomize/kyaml v0.17.2 h1:+AzvoJUY0kq4QAhH/ydPHHMRLijtUKiyVyh7fOSshr0= +sigs.k8s.io/kustomize/kyaml v0.17.2/go.mod h1:9V0mCjIEYjlXuCdYsSXvyoy2BTsLESH7TlGV81S282U= +sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4= +sigs.k8s.io/structured-merge-diff/v4 v4.4.1/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08= +sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= +sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= diff --git a/hack/boilerplate.go.txt b/hack/boilerplate.go.txt new file mode 100644 index 0000000..4ad4385 --- /dev/null +++ b/hack/boilerplate.go.txt @@ -0,0 +1,15 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ \ No newline at end of file diff --git a/hack/e2e-test.sh b/hack/e2e-test.sh new file mode 100755 index 0000000..d3d2f7d --- /dev/null +++ b/hack/e2e-test.sh @@ -0,0 +1,99 @@ +#!/usr/bin/env bash + +# Copyright 2024 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +SOURCE_DIR="$(cd "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" +ROOT_DIR="$SOURCE_DIR/.." + +export GINKGO="$ROOT_DIR"/bin/ginkgo +export KIND="$ROOT_DIR"/bin/kind + +export E2E_TEST_BASH_IMAGE=registry.k8s.io/alpine-with-bash:1.0@sha256:0955672451201896cf9e2e5ce30bec0c7f10757af33bf78b7a6afa5672c596f5 + +# $1 cluster name +function cluster_create { + $KIND create cluster --name "$1" --image "$E2E_KIND_VERSION" --wait 1m -v 5 > "$ARTIFACTS/$1-create.log" 2>&1 \ + || { echo "unable to start the $1 cluster "; cat "$ARTIFACTS/$1-create.log" ; } + kubectl config use-context "kind-$1" + kubectl get nodes > "$ARTIFACTS/$1-nodes.log" || true + kubectl describe pods -n kube-system > "$ARTIFACTS/$1-system-pods.log" || true +} + +# $1 - cluster name +function cluster_cleanup { + kubectl config use-context "kind-$1" + $KIND export logs "$ARTIFACTS" --name "$1" || true + kubectl describe pods -n kueue-system > "$ARTIFACTS/$1-kueue-system-pods.log" || true + kubectl describe pods > "$ARTIFACTS/$1-default-pods.log" || true + $KIND delete cluster --name "$1" +} + +function startup { + if [ "$CREATE_KIND_CLUSTER" == 'true' ] + then + if [ ! -d "$ARTIFACTS" ]; then + mkdir -p "$ARTIFACTS" + fi + cluster_create "$KIND_CLUSTER_NAME" + fi +} + +function kind_load { + if [ "$CREATE_KIND_CLUSTER" == 'true' ] + then + cluster_kind_load "$KIND_CLUSTER_NAME" + fi +} + +# $1 cluster +function cluster_kind_load { + docker pull "${E2E_TEST_BASH_IMAGE}" + e2e_test_bash_image_without_sha=${E2E_TEST_BASH_IMAGE%%@*} + # We can load image by a digest but we cannot reference it by the digest that we pulled. + # For more information https://github.com/kubernetes-sigs/kind/issues/2394#issuecomment-888713831. + # Manually create tag for image with digest which is already pulled + docker tag $E2E_TEST_BASH_IMAGE "$e2e_test_bash_image_without_sha" + $KIND load docker-image "${e2e_test_bash_image_without_sha}" --name "$1" +} + +function cleanup { + if [ "$CREATE_KIND_CLUSTER" == 'true' ] + then + if [ ! -d "$ARTIFACTS" ]; then + mkdir -p "$ARTIFACTS" + fi + uninstall_kjobctl + cluster_cleanup "$KIND_CLUSTER_NAME" + fi +} + +function install_kjobctl { + cd "$SOURCE_DIR"/.. && make install +} + +function uninstall_kjobctl { + cd "$SOURCE_DIR"/.. && make uninstall +} + +trap cleanup EXIT +startup +kind_load +install_kjobctl +# shellcheck disable=SC2086 +$GINKGO $GINKGO_ARGS --junit-report=junit.xml --json-report=e2e.json --output-dir="$ARTIFACTS" -v ./test/e2e/... diff --git a/hack/multiplatform-build.sh b/hack/multiplatform-build.sh new file mode 100755 index 0000000..7c39952 --- /dev/null +++ b/hack/multiplatform-build.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash + +# Copyright 2024 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +GO_CMD=${GO_CMD:-go} +LD_FLAGS=${LD_FLAGS:-} + +BUILD_NAME=${BUILD_NAME:-kueuectl} +PLATFORMS=${PLATFORMS:-linux/amd64} + +CURRENT_DIR=$(dirname "${BASH_SOURCE[0]}") +ROOT_PATH=$(realpath "${CURRENT_DIR}/..") +BUILD_PATH=${ROOT_PATH}/${BUILD_DIR} + +mkdir -p "${BUILD_PATH}" + +IFS="," +for PLATFORM in ${PLATFORMS} ; do + export GOOS="${PLATFORM%/*}" + export GOARCH="${PLATFORM#*/}" + EXTENSION="" + + if [ "${GOOS}" == "windows" ]; then + EXTENSION=".exe" + fi + + echo "Building for $PLATFORM platform" + FULL_NAME=${BUILD_NAME}-${GOOS}-${GOARCH} + "${GO_CMD}" build -ldflags="${LD_FLAGS}" -o "${BUILD_PATH}/${FULL_NAME}${EXTENSION}" "$1" + + mkdir -p "${BUILD_PATH}/tmp" + cp "${ROOT_PATH}/LICENSE" "${BUILD_PATH}/tmp" + cp "${BUILD_PATH}/${FULL_NAME}${EXTENSION}" "${BUILD_PATH}/tmp/${BUILD_NAME}${EXTENSION}" + (cd "${BUILD_PATH}/tmp" && tar -czf "${BUILD_PATH}/${FULL_NAME}.tar.gz" ./*) + rm -R "${BUILD_PATH}/tmp" +done diff --git a/hack/shellcheck/Dockerfile b/hack/shellcheck/Dockerfile new file mode 100644 index 0000000..c5f0692 --- /dev/null +++ b/hack/shellcheck/Dockerfile @@ -0,0 +1 @@ +FROM docker.io/koalaman/shellcheck-alpine:v0.10.0 \ No newline at end of file diff --git a/hack/shellcheck/verify.sh b/hack/shellcheck/verify.sh new file mode 100755 index 0000000..aedb0df --- /dev/null +++ b/hack/shellcheck/verify.sh @@ -0,0 +1,59 @@ +#!/bin/bash + +# Copyright 2024 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +# allow overriding docker cli, which should work fine for this script +DOCKER="${DOCKER:-docker}" +CURRENT_DIR=$(dirname "${BASH_SOURCE[0]}") + +SHELLCHECK_IMAGE=$(grep '^FROM' "${CURRENT_DIR}/Dockerfile" | awk '{print $2}') + +# Initialize an empty array for scripts to check +scripts_to_check=() + +if [[ "$#" == 0 ]]; then + # Find all shell scripts excluding certain directories and patterns + while IFS=$'\n' read -r script; do + if ! git check-ignore -q "$script"; then + scripts_to_check+=("$script") + fi + done < <(find . -name "*.sh" \ + -not \( \ + -path ./_\* -o \ + -path ./.git\* -o \ + -path ./vendor\* -o \ + \( -path ./third_party\* -a -not -path ./third_party/forked\* \) \ + \)) +fi + +# Download shellcheck-alpine from Docker Hub +echo "Downloading ShellCheck Docker image..." +"${DOCKER}" pull "${SHELLCHECK_IMAGE}" + +# Run ShellCheck on all shell script files, excluding those in the 'vendor' directory. +# Configuration loaded from the .shelcheckrc file. +echo "Running ShellCheck..." +if [ "${#scripts_to_check[@]}" -ne 0 ]; then + "${DOCKER}" run --rm -v "$(pwd)":/mnt -w /mnt "${SHELLCHECK_IMAGE}" shellcheck "${scripts_to_check[@]}" >&2 +else + echo "No scripts to check" +fi + +echo "Shellcheck ran successfully" + diff --git a/hack/tools/go.mod b/hack/tools/go.mod new file mode 100644 index 0000000..6c0afbe --- /dev/null +++ b/hack/tools/go.mod @@ -0,0 +1,76 @@ +module sigs.k8s.io/kueue/cmd/experimental/kjobctl/hack/tools + +go 1.23 + +require ( + github.com/golangci/golangci-lint v1.62.2 + github.com/onsi/ginkgo/v2 v2.22.0 + gotest.tools/gotestsum v1.12.0 + k8s.io/code-generator v0.31.3 + sigs.k8s.io/controller-runtime/tools/setup-envtest v0.0.0-20240813183042-b901db121e1f + sigs.k8s.io/controller-tools v0.16.5 + sigs.k8s.io/kind v0.25.0 + sigs.k8s.io/kustomize/kustomize/v5 v5.5.0 +) + +require ( + github.com/bitfield/gotestdox v0.2.2 // indirect + github.com/blang/semver/v4 v4.0.0 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/dnephin/pflag v1.0.7 // indirect + github.com/emicklei/go-restful/v3 v3.12.1 // indirect + github.com/fatih/color v1.18.0 // indirect + github.com/fsnotify/fsnotify v1.7.0 // indirect + github.com/fxamacker/cbor/v2 v2.7.0 // indirect + github.com/go-errors/errors v1.4.2 // indirect + github.com/go-logr/logr v1.4.2 // indirect + github.com/go-openapi/jsonpointer v0.21.0 // indirect + github.com/go-openapi/jsonreference v0.21.0 // indirect + github.com/go-openapi/swag v0.23.0 // indirect + github.com/gobuffalo/flect v1.0.3 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/protobuf v1.5.4 // indirect + github.com/google/gnostic-models v0.6.8 // indirect + github.com/google/gofuzz v1.2.0 // indirect + github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db // indirect + github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/josharian/intern v1.0.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/mailru/easyjson v0.7.7 // indirect + github.com/mattn/go-colorable v0.1.13 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/spf13/afero v1.11.0 // indirect + github.com/spf13/cobra v1.8.1 // indirect + github.com/spf13/pflag v1.0.5 // indirect + github.com/x448/float16 v0.8.4 // indirect + github.com/xlab/treeprint v1.2.0 // indirect + go.uber.org/zap v1.27.0 // indirect + golang.org/x/mod v0.22.0 // indirect + golang.org/x/net v0.31.0 // indirect + golang.org/x/sync v0.9.0 // indirect + golang.org/x/sys v0.27.0 // indirect + golang.org/x/term v0.26.0 // indirect + golang.org/x/text v0.20.0 // indirect + golang.org/x/tools v0.27.0 // indirect + google.golang.org/protobuf v1.34.2 // indirect + gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect + k8s.io/apiextensions-apiserver v0.31.2 // indirect + k8s.io/apimachinery v0.31.3 // indirect + k8s.io/gengo/v2 v2.0.0-20240404160639-a0386bf69313 // indirect + k8s.io/klog/v2 v2.130.1 // indirect + k8s.io/kube-openapi v0.0.0-20240709000822-3c01b740850f // indirect + k8s.io/utils v0.0.0-20240902221715-702e33fdd3c3 // indirect + sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect + sigs.k8s.io/kustomize/api v0.18.0 // indirect + sigs.k8s.io/kustomize/kyaml v0.18.1 // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect + sigs.k8s.io/yaml v1.4.0 // indirect +) diff --git a/hack/tools/go.sum b/hack/tools/go.sum new file mode 100644 index 0000000..95170e3 --- /dev/null +++ b/hack/tools/go.sum @@ -0,0 +1,289 @@ +github.com/bitfield/gotestdox v0.2.2 h1:x6RcPAbBbErKLnapz1QeAlf3ospg8efBsedU93CDsnE= +github.com/bitfield/gotestdox v0.2.2/go.mod h1:D+gwtS0urjBrzguAkTM2wodsTQYFHdpx8eqRJ3N+9pY= +github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= +github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= +github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dnephin/pflag v1.0.7 h1:oxONGlWxhmUct0YzKTgrpQv9AUA1wtPBn7zuSjJqptk= +github.com/dnephin/pflag v1.0.7/go.mod h1:uxE91IoWURlOiTUIA8Mq5ZZkAv3dPUfZNaT80Zm7OQE= +github.com/emicklei/go-restful/v3 v3.12.1 h1:PJMDIM/ak7btuL8Ex0iYET9hxM3CI2sjZtzpL63nKAU= +github.com/emicklei/go-restful/v3 v3.12.1/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw= +github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE= +github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM= +github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU= +github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= +github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= +github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E= +github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= +github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= +github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ= +github.com/go-logr/zapr v1.3.0/go.mod h1:YKepepNBd1u/oyhd/yQmtjVXmm9uML4IXUgMOwR8/Gg= +github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ= +github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY= +github.com/go-openapi/jsonreference v0.21.0 h1:Rs+Y7hSXT83Jacb7kFyjn4ijOuVGSvOdF2+tg1TRrwQ= +github.com/go-openapi/jsonreference v0.21.0/go.mod h1:LmZmgsrTkVg9LG4EaHeY8cBDslNPMo06cago5JNLkm4= +github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE= +github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ= +github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= +github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= +github.com/gobuffalo/flect v1.0.3 h1:xeWBM2nui+qnVvNM4S3foBhCAL2XgPU+a7FdpelbTq4= +github.com/gobuffalo/flect v1.0.3/go.mod h1:A5msMlrHtLqh9umBSnvabjsMrCcCpAyzglnDvkbYKHs= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/golangci/golangci-lint v1.62.2 h1:b8K5K9PN+rZN1+mKLtsZHz2XXS9aYKzQ9i25x3Qnxxw= +github.com/golangci/golangci-lint v1.62.2/go.mod h1:ILWWyeFUrctpHVGMa1dg2xZPKoMUTc5OIMgW7HZr34g= +github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= +github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= +github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db h1:097atOisP2aRj7vFgYQBbFN4U4JNXUNYpxael3UzMyo= +github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= +github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= +github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 h1:n6/2gBQ3RWajuToeY6ZtZTIKv2v7ThUy5KKusIT0yc0= +github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00/go.mod h1:Pm3mSP3c5uWn86xMLZ5Sa7JB9GsEZySvHYXCTK4E9q4= +github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= +github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= +github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= +github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= +github.com/onsi/ginkgo/v2 v2.22.0 h1:Yed107/8DjTr0lKCNt7Dn8yQ6ybuDRQoMGrNFKzMfHg= +github.com/onsi/ginkgo/v2 v2.22.0/go.mod h1:7Du3c42kxCUegi0IImZ1wUQzMBVecgIHjR1C+NkhLQo= +github.com/onsi/gomega v1.34.2 h1:pNCwDkzrsv7MS9kpaQvVb1aVLahQXyJ/Tv5oAZMI3i8= +github.com/onsi/gomega v1.34.2/go.mod h1:v1xfxRgk0KIsG+QOdm7p8UosrOzPYRo60fd3B/1Dukc= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= +github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= +github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ= +github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= +github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8= +github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY= +github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= +github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= +github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= +github.com/xlab/treeprint v1.2.0 h1:HzHnuAF1plUN2zGlAFHbSQP2qJ0ZAD3XF5XD7OesXRQ= +github.com/xlab/treeprint v1.2.0/go.mod h1:gj5Gd3gPdKtR1ikdDK6fnFLdmIS0X30kTTuNd/WEJu0= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= +golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio= +golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= +golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= +golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.9.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.16.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.22.0 h1:D4nJWe9zXqHOmWqj4VMOJhvzj7bEZg4wEYa759z1pH4= +golang.org/x/mod v0.22.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= +golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= +golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= +golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= +golang.org/x/net v0.31.0 h1:68CPQngjLL0r2AlUKiSxtQFKvzRVbnzLwMUn5SzcLHo= +golang.org/x/net v0.31.0/go.mod h1:P4fl1q7dY2hnZFxEk4pPSkDHF+QqjitcnDjUQyMM+pM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.9.0 h1:fEo0HyrW1GIgZdpbhCRO0PkJajUS5H9IFUztCgEo2jQ= +golang.org/x/sync v0.9.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.27.0 h1:wBqf8DvsY9Y/2P8gAfPDEYNuS30J4lPHJxXSb/nJZ+s= +golang.org/x/sys v0.27.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/term v0.10.0/go.mod h1:lpqdcUyK/oCiQxvxVrppt5ggO2KCZ5QblwqPnfZ6d5o= +golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= +golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= +golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= +golang.org/x/term v0.26.0 h1:WEQa6V3Gja/BhNxg540hBip/kkaYtRg3cxg4oXSw4AU= +golang.org/x/term v0.26.0/go.mod h1:Si5m1o57C5nBNQo5z1iq+XDijt21BDBDp2bK0QI8e3E= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.20.0 h1:gK/Kv2otX8gz+wn7Rmb3vT96ZwuoxnQlY+HlJVj7Qug= +golang.org/x/text v0.20.0/go.mod h1:D4IsuqiFMhST5bX19pQ9ikHC2GsaKyk/oF+pn3ducp4= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.2.0/go.mod h1:y4OqIKeOV/fWJetJ8bXPU1sEVniLMIyDAZWeHdV+NTA= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/tools v0.11.0/go.mod h1:anzJrxPjNtfgiYQYirP2CPGzGLxrH2u2QBhn6Bf3qY8= +golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= +golang.org/x/tools v0.19.0/go.mod h1:qoJWxmGSIBmAeriMx19ogtrEPrGtDbPK634QFIcLAhc= +golang.org/x/tools v0.27.0 h1:qEKojBykQkQ4EynWy4S8Weg69NumxKdn40Fce3uc/8o= +golang.org/x/tools v0.27.0/go.mod h1:sUi0ZgbwW9ZPAq26Ekut+weQPR5eIM6GQLQ1Yjm1H0Q= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/evanphx/json-patch.v4 v4.12.0 h1:n6jtcsulIzXPJaxegRbvFNNrZDjbij7ny3gmSPG+6V4= +gopkg.in/evanphx/json-patch.v4 v4.12.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gotest.tools/gotestsum v1.12.0 h1:CmwtaGDkHxrZm4Ib0Vob89MTfpc3GrEFMJKovliPwGk= +gotest.tools/gotestsum v1.12.0/go.mod h1:fAvqkSptospfSbQw26CTYzNwnsE/ztqLeyhP0h67ARY= +gotest.tools/v3 v3.5.1 h1:EENdUnS3pdur5nybKYIh2Vfgc8IUNBjxDPSjtiJcOzU= +gotest.tools/v3 v3.5.1/go.mod h1:isy3WKz7GK6uNw/sbHzfKBLvlvXwUyV06n6brMxxopU= +k8s.io/apiextensions-apiserver v0.31.2 h1:W8EwUb8+WXBLu56ser5IudT2cOho0gAKeTOnywBLxd0= +k8s.io/apiextensions-apiserver v0.31.2/go.mod h1:i+Geh+nGCJEGiCGR3MlBDkS7koHIIKWVfWeRFiOsUcM= +k8s.io/apimachinery v0.31.3 h1:6l0WhcYgasZ/wk9ktLq5vLaoXJJr5ts6lkaQzgeYPq4= +k8s.io/apimachinery v0.31.3/go.mod h1:rsPdaZJfTfLsNJSQzNHQvYoTmxhoOEofxtOsF3rtsMo= +k8s.io/code-generator v0.31.3 h1:Pj0fYOBms+ZrsulLi4DMsCEx1jG8fWKRLy44onHsLBI= +k8s.io/code-generator v0.31.3/go.mod h1:/umCIlT84g1+Yu5ZXtP1KGSRTnGiIzzX5AzUAxsNlts= +k8s.io/gengo/v2 v2.0.0-20240404160639-a0386bf69313 h1:bKcdZJOPICVmIIuaM9+MXmapE94dn5AYv5ODs1jA43o= +k8s.io/gengo/v2 v2.0.0-20240404160639-a0386bf69313/go.mod h1:VH3AT8AaQOqiGjMF9p0/IM1Dj+82ZwjfxUP1IxaHE+8= +k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= +k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= +k8s.io/kube-openapi v0.0.0-20240709000822-3c01b740850f h1:2sXuKesAYbRHxL3aE2PN6zX/gcJr22cjrsej+W784Tc= +k8s.io/kube-openapi v0.0.0-20240709000822-3c01b740850f/go.mod h1:UxDHUPsUwTOOxSU+oXURfFBcAS6JwiRXTYqYwfuGowc= +k8s.io/utils v0.0.0-20240902221715-702e33fdd3c3 h1:b2FmK8YH+QEwq/Sy2uAEhmqL5nPfGYbJOcaqjeYYZoA= +k8s.io/utils v0.0.0-20240902221715-702e33fdd3c3/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +sigs.k8s.io/controller-runtime/tools/setup-envtest v0.0.0-20240813183042-b901db121e1f h1:RIqUbZQO5yizUt9nozxQdzyLRD1sG6s2oi/QZrlg/hs= +sigs.k8s.io/controller-runtime/tools/setup-envtest v0.0.0-20240813183042-b901db121e1f/go.mod h1:IaDsO8xSPRxRG1/rm9CP7+jPmj0nMNAuNi/yiHnLX8k= +sigs.k8s.io/controller-tools v0.16.5 h1:5k9FNRqziBPwqr17AMEPPV/En39ZBplLAdOwwQHruP4= +sigs.k8s.io/controller-tools v0.16.5/go.mod h1:8vztuRVzs8IuuJqKqbXCSlXcw+lkAv/M2sTpg55qjMY= +sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= +sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= +sigs.k8s.io/kind v0.25.0 h1:ugUvgesHKKA0yKmD6QtYTiEev+kPUpGxdTPbMGf8VTU= +sigs.k8s.io/kind v0.25.0/go.mod h1:t7ueEpzPYJvHA8aeLtI52rtFftNgUYUaCwvxjk7phfw= +sigs.k8s.io/kustomize/api v0.18.0 h1:hTzp67k+3NEVInwz5BHyzc9rGxIauoXferXyjv5lWPo= +sigs.k8s.io/kustomize/api v0.18.0/go.mod h1:f8isXnX+8b+SGLHQ6yO4JG1rdkZlvhaCf/uZbLVMb0U= +sigs.k8s.io/kustomize/kustomize/v5 v5.5.0 h1:o1mtt6vpxsxDYaZKrw3BnEtc+pAjLz7UffnIvHNbvW0= +sigs.k8s.io/kustomize/kustomize/v5 v5.5.0/go.mod h1:AeFCmgCrXzmvjWWaeZCyBp6XzG1Y0w1svYus8GhJEOE= +sigs.k8s.io/kustomize/kyaml v0.18.1 h1:WvBo56Wzw3fjS+7vBjN6TeivvpbW9GmRaWZ9CIVmt4E= +sigs.k8s.io/kustomize/kyaml v0.18.1/go.mod h1:C3L2BFVU1jgcddNBE1TxuVLgS46TjObMwW5FT9FcjYo= +sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4= +sigs.k8s.io/structured-merge-diff/v4 v4.4.1/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08= +sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= +sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= diff --git a/hack/tools/gzip/gzip.go b/hack/tools/gzip/gzip.go new file mode 100644 index 0000000..34945b2 --- /dev/null +++ b/hack/tools/gzip/gzip.go @@ -0,0 +1,37 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "compress/gzip" + "io" + "os" +) + +// Simple gzip encoder used to have the same output generated regardless of the OS. +func main() { + writer, err := gzip.NewWriterLevel(os.Stdout, gzip.BestCompression) + if err != nil { + panic(err) + } + // The header is initialized with Unknown OS and no additional fields, we should not change it. + _, err = io.Copy(writer, os.Stdin) + if err != nil { + panic(err) + } + writer.Close() +} diff --git a/hack/tools/pinversion.go b/hack/tools/pinversion.go new file mode 100644 index 0000000..ed79056 --- /dev/null +++ b/hack/tools/pinversion.go @@ -0,0 +1,34 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tools + +// Keep a reference to the code generators, so they are not removed by go mod tidy +import ( + _ "github.com/golangci/golangci-lint/pkg/exitcodes" + _ "github.com/onsi/ginkgo/v2/ginkgo/command" + _ "github.com/onsi/ginkgo/v2/ginkgo/run" + _ "gotest.tools/gotestsum/cmd" + _ "k8s.io/code-generator" + _ "sigs.k8s.io/controller-runtime/tools/setup-envtest/env" + + // since verify will error when referencing a cmd package + // we need to reference individual dependencies used by it + _ "sigs.k8s.io/controller-tools/pkg/crd" + _ "sigs.k8s.io/controller-tools/pkg/genall/help/pretty" + _ "sigs.k8s.io/kind/pkg/cmd" + _ "sigs.k8s.io/kustomize/kustomize/v5/commands/edit/listbuiltin" +) diff --git a/hack/update-codegen.sh b/hack/update-codegen.sh new file mode 100755 index 0000000..bc9656b --- /dev/null +++ b/hack/update-codegen.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash + +# Copyright 2024 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +GO_CMD=${1:-go} +CURRENT_DIR=$(dirname "${BASH_SOURCE[0]}") +KJOBCTL_ROOT=$(realpath "$CURRENT_DIR/..") +KJOBCTL_PKG="sigs.k8s.io/kueue/cmd/experimental/kjobctl" +CODEGEN_PKG=$(cd "$TOOLS_DIR" && go mod download && $GO_CMD list -m -f "{{.Dir}}" k8s.io/code-generator) + +cd "$CURRENT_DIR/.." + +# shellcheck source=/dev/null +source "${CODEGEN_PKG}/kube_codegen.sh" + +# Generating conversion and defaults functions +kube::codegen::gen_helpers \ + --boilerplate "${KJOBCTL_ROOT}/hack/boilerplate.go.txt" \ + "${KJOBCTL_ROOT}" + +kube::codegen::gen_client \ + --boilerplate "${KJOBCTL_ROOT}/hack/boilerplate.go.txt" \ + --output-dir "${KJOBCTL_ROOT}/client-go" \ + --output-pkg "${KJOBCTL_PKG}/client-go" \ + "${KJOBCTL_ROOT}" + +# We need to clean up the go.mod file since code-generator adds temporary library to the go.mod file. +"${GO_CMD}" mod tidy diff --git a/pkg/builder/builder.go b/pkg/builder/builder.go new file mode 100644 index 0000000..dae217c --- /dev/null +++ b/pkg/builder/builder.go @@ -0,0 +1,697 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package builder + +import ( + "context" + "errors" + "fmt" + "os" + "slices" + "strings" + "time" + + rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + utilrand "k8s.io/apimachinery/pkg/util/rand" + k8s "k8s.io/client-go/kubernetes" + "k8s.io/utils/ptr" + kueueversioned "sigs.k8s.io/kueue/client-go/clientset/versioned" + kueueconstants "sigs.k8s.io/kueue/pkg/controller/constants" + + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/apis/v1alpha1" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/client-go/clientset/versioned" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/cmd/util" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/constants" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/parser" +) + +var ( + noNamespaceSpecifiedErr = errors.New("no namespace specified") + noApplicationProfileSpecifiedErr = errors.New("no application profile specified") + noApplicationProfileModeSpecifiedErr = errors.New("no application profile mode specified") + invalidApplicationProfileModeErr = errors.New("invalid application profile mode") + applicationProfileModeNotConfiguredErr = errors.New("application profile mode not configured") + noCommandSpecifiedErr = errors.New("no command specified") + noParallelismSpecifiedErr = errors.New("no parallelism specified") + noCompletionsSpecifiedErr = errors.New("no completions specified") + noReplicasSpecifiedErr = errors.New("no replicas specified") + noMinReplicasSpecifiedErr = errors.New("no min-replicas specified") + noMaxReplicasSpecifiedErr = errors.New("no max-replicas specified") + noRequestsSpecifiedErr = errors.New("no requests specified") + noLocalQueueSpecifiedErr = errors.New("no local queue specified") + noRayClusterSpecifiedErr = errors.New("no raycluster specified") + noArraySpecifiedErr = errors.New("no array specified") + noCpusPerTaskSpecifiedErr = errors.New("no cpus-per-task specified") + noErrorSpecifiedErr = errors.New("no error specified") + noGpusPerTaskSpecifiedErr = errors.New("no gpus-per-task specified") + noInputSpecifiedErr = errors.New("no input specified") + noJobNameSpecifiedErr = errors.New("no job-name specified") + noMemPerCPUSpecifiedErr = errors.New("no mem-per-cpu specified") + noMemPerGPUSpecifiedErr = errors.New("no mem-per-gpu specified") + noMemPerTaskSpecifiedErr = errors.New("no mem-per-task specified") + noNodesSpecifiedErr = errors.New("no nodes specified") + noNTasksSpecifiedErr = errors.New("no ntasks specified") + noOutputSpecifiedErr = errors.New("no output specified") + noPartitionSpecifiedErr = errors.New("no partition specified") + noPrioritySpecifiedErr = errors.New("no priority specified") + noTimeSpecifiedErr = errors.New("no time specified") +) + +type builder interface { + build(ctx context.Context) (rootObj runtime.Object, childObjs []runtime.Object, err error) +} + +type Builder struct { + clientGetter util.ClientGetter + kjobctlClientset versioned.Interface + k8sClientset k8s.Interface + kueueClientset kueueversioned.Interface + + namespace string + profileName string + modeName v1alpha1.ApplicationProfileMode + + command []string + parallelism *int32 + completions *int32 + replicas map[string]int + minReplicas map[string]int + maxReplicas map[string]int + requests corev1.ResourceList + localQueue string + rayCluster string + script string + array string + cpusPerTask *resource.Quantity + error string + gpusPerTask map[string]*resource.Quantity + input string + jobName string + memPerNode *resource.Quantity + memPerCPU *resource.Quantity + memPerGPU *resource.Quantity + memPerTask *resource.Quantity + nodes *int32 + nTasks *int32 + output string + partition string + priority string + initImage string + ignoreUnknown bool + skipLocalQueueValidation bool + skipPriorityValidation bool + firstNodeIP bool + firstNodeIPTimeout time.Duration + changeDir string + timeLimit string + + profile *v1alpha1.ApplicationProfile + mode *v1alpha1.SupportedMode + volumeBundles []v1alpha1.VolumeBundle + + buildTime time.Time +} + +func NewBuilder(clientGetter util.ClientGetter, buildTime time.Time) *Builder { + return &Builder{clientGetter: clientGetter, buildTime: buildTime} +} + +func (b *Builder) WithNamespace(namespace string) *Builder { + b.namespace = namespace + return b +} + +func (b *Builder) WithProfileName(profileName string) *Builder { + b.profileName = profileName + return b +} + +func (b *Builder) WithModeName(modeName v1alpha1.ApplicationProfileMode) *Builder { + b.modeName = modeName + return b +} + +func (b *Builder) WithCommand(command []string) *Builder { + b.command = command + return b +} + +func (b *Builder) WithParallelism(parallelism *int32) *Builder { + b.parallelism = parallelism + return b +} + +func (b *Builder) WithCompletions(completions *int32) *Builder { + b.completions = completions + return b +} + +func (b *Builder) WithReplicas(replicas map[string]int) *Builder { + b.replicas = replicas + return b +} + +func (b *Builder) WithMinReplicas(minReplicas map[string]int) *Builder { + b.minReplicas = minReplicas + return b +} + +func (b *Builder) WithMaxReplicas(maxReplicas map[string]int) *Builder { + b.maxReplicas = maxReplicas + return b +} + +func (b *Builder) WithRequests(requests corev1.ResourceList) *Builder { + b.requests = requests + return b +} + +func (b *Builder) WithLocalQueue(localQueue string) *Builder { + b.localQueue = localQueue + return b +} + +func (b *Builder) WithRayCluster(rayCluster string) *Builder { + b.rayCluster = rayCluster + return b +} + +func (b *Builder) WithScript(script string) *Builder { + b.script = script + return b +} + +func (b *Builder) WithArray(array string) *Builder { + b.array = array + return b +} + +func (b *Builder) WithCpusPerTask(cpusPerTask *resource.Quantity) *Builder { + b.cpusPerTask = cpusPerTask + return b +} + +func (b *Builder) WithError(error string) *Builder { + b.error = error + return b +} + +func (b *Builder) WithGpusPerTask(gpusPerTask map[string]*resource.Quantity) *Builder { + b.gpusPerTask = gpusPerTask + return b +} + +func (b *Builder) WithInput(input string) *Builder { + b.input = input + return b +} + +func (b *Builder) WithJobName(jobName string) *Builder { + b.jobName = jobName + return b +} + +func (b *Builder) WithMemPerNode(memPerNode *resource.Quantity) *Builder { + b.memPerNode = memPerNode + return b +} + +func (b *Builder) WithMemPerCPU(memPerCPU *resource.Quantity) *Builder { + b.memPerCPU = memPerCPU + return b +} + +func (b *Builder) WithMemPerGPU(memPerGPU *resource.Quantity) *Builder { + b.memPerGPU = memPerGPU + return b +} + +func (b *Builder) WithMemPerTask(memPerTask *resource.Quantity) *Builder { + b.memPerTask = memPerTask + return b +} + +func (b *Builder) WithNodes(nodes *int32) *Builder { + b.nodes = nodes + return b +} + +func (b *Builder) WithNTasks(nTasks *int32) *Builder { + b.nTasks = nTasks + return b +} + +func (b *Builder) WithOutput(output string) *Builder { + b.output = output + return b +} + +func (b *Builder) WithPartition(partition string) *Builder { + b.partition = partition + return b +} + +func (b *Builder) WithPriority(priority string) *Builder { + b.priority = priority + return b +} + +func (b *Builder) WithInitImage(initImage string) *Builder { + b.initImage = initImage + return b +} + +func (b *Builder) WithIgnoreUnknown(ignoreUnknown bool) *Builder { + b.ignoreUnknown = ignoreUnknown + return b +} + +func (b *Builder) WithChangeDir(chdir string) *Builder { + b.changeDir = chdir + return b +} + +func (b *Builder) WithSkipLocalQueueValidation(skip bool) *Builder { + b.skipLocalQueueValidation = skip + return b +} + +func (b *Builder) WithSkipPriorityValidation(skip bool) *Builder { + b.skipPriorityValidation = skip + return b +} + +func (b *Builder) WithFirstNodeIP(firstNodeIP bool) *Builder { + b.firstNodeIP = firstNodeIP + return b +} + +func (b *Builder) WithFirstNodeIPTimeout(timeout time.Duration) *Builder { + b.firstNodeIPTimeout = timeout + return b +} + +func (b *Builder) WithTimeLimit(timeLimit string) *Builder { + b.timeLimit = timeLimit + return b +} + +func (b *Builder) validateGeneral(ctx context.Context) error { + if b.namespace == "" { + return noNamespaceSpecifiedErr + } + + if b.profileName == "" { + return noApplicationProfileSpecifiedErr + } + + if b.modeName == "" { + return noApplicationProfileModeSpecifiedErr + } + + // check that local queue exists + if len(b.localQueue) != 0 && !b.skipLocalQueueValidation { + _, err := b.kueueClientset.KueueV1beta1().LocalQueues(b.namespace).Get(ctx, b.localQueue, metav1.GetOptions{}) + if err != nil { + return err + } + } + + // check that priority class exists + if len(b.priority) != 0 && !b.skipPriorityValidation { + _, err := b.kueueClientset.KueueV1beta1().WorkloadPriorityClasses().Get(ctx, b.priority, metav1.GetOptions{}) + if err != nil { + return err + } + } + + return nil +} + +func (b *Builder) complete(ctx context.Context) error { + var err error + + b.profile, err = b.kjobctlClientset.KjobctlV1alpha1().ApplicationProfiles(b.namespace).Get(ctx, b.profileName, metav1.GetOptions{}) + if err != nil { + return err + } + + for i, mode := range b.profile.Spec.SupportedModes { + if mode.Name == b.modeName { + b.mode = &b.profile.Spec.SupportedModes[i] + } + } + + if b.mode == nil { + return applicationProfileModeNotConfiguredErr + } + + for _, name := range b.profile.Spec.VolumeBundles { + volumeBundle, err := b.kjobctlClientset.KjobctlV1alpha1().VolumeBundles(b.profile.Namespace).Get(ctx, string(name), metav1.GetOptions{}) + if err != nil { + return err + } + b.volumeBundles = append(b.volumeBundles, *volumeBundle) + } + + return nil +} + +func (b *Builder) validateFlags() error { + if slices.Contains(b.mode.RequiredFlags, v1alpha1.CmdFlag) && len(b.command) == 0 { + return noCommandSpecifiedErr + } + + if slices.Contains(b.mode.RequiredFlags, v1alpha1.ParallelismFlag) && b.parallelism == nil { + return noParallelismSpecifiedErr + } + + if slices.Contains(b.mode.RequiredFlags, v1alpha1.CompletionsFlag) && b.completions == nil { + return noCompletionsSpecifiedErr + } + + if slices.Contains(b.mode.RequiredFlags, v1alpha1.ReplicasFlag) && b.replicas == nil { + return noReplicasSpecifiedErr + } + + if slices.Contains(b.mode.RequiredFlags, v1alpha1.MinReplicasFlag) && b.minReplicas == nil { + return noMinReplicasSpecifiedErr + } + + if slices.Contains(b.mode.RequiredFlags, v1alpha1.MaxReplicasFlag) && b.maxReplicas == nil { + return noMaxReplicasSpecifiedErr + } + + if slices.Contains(b.mode.RequiredFlags, v1alpha1.RequestFlag) && b.requests == nil { + return noRequestsSpecifiedErr + } + + if slices.Contains(b.mode.RequiredFlags, v1alpha1.LocalQueueFlag) && b.localQueue == "" { + return noLocalQueueSpecifiedErr + } + + if slices.Contains(b.mode.RequiredFlags, v1alpha1.RayClusterFlag) && b.rayCluster == "" { + return noRayClusterSpecifiedErr + } + + if slices.Contains(b.mode.RequiredFlags, v1alpha1.ArrayFlag) && b.array == "" { + return noArraySpecifiedErr + } + + if slices.Contains(b.mode.RequiredFlags, v1alpha1.CpusPerTaskFlag) && b.cpusPerTask == nil { + return noCpusPerTaskSpecifiedErr + } + + if slices.Contains(b.mode.RequiredFlags, v1alpha1.ErrorFlag) && b.error == "" { + return noErrorSpecifiedErr + } + + if slices.Contains(b.mode.RequiredFlags, v1alpha1.GpusPerTaskFlag) && b.gpusPerTask == nil { + return noGpusPerTaskSpecifiedErr + } + + if slices.Contains(b.mode.RequiredFlags, v1alpha1.InputFlag) && b.input == "" { + return noInputSpecifiedErr + } + + if slices.Contains(b.mode.RequiredFlags, v1alpha1.JobNameFlag) && b.jobName == "" { + return noJobNameSpecifiedErr + } + + if slices.Contains(b.mode.RequiredFlags, v1alpha1.MemPerCPUFlag) && b.memPerCPU == nil { + return noMemPerCPUSpecifiedErr + } + + if slices.Contains(b.mode.RequiredFlags, v1alpha1.MemPerGPUFlag) && b.memPerGPU == nil { + return noMemPerGPUSpecifiedErr + } + + if slices.Contains(b.mode.RequiredFlags, v1alpha1.MemPerTaskFlag) && b.memPerTask == nil { + return noMemPerTaskSpecifiedErr + } + + if slices.Contains(b.mode.RequiredFlags, v1alpha1.NodesFlag) && b.nodes == nil { + return noNodesSpecifiedErr + } + + if slices.Contains(b.mode.RequiredFlags, v1alpha1.NTasksFlag) && b.nTasks == nil { + return noNTasksSpecifiedErr + } + + if slices.Contains(b.mode.RequiredFlags, v1alpha1.OutputFlag) && b.output == "" { + return noOutputSpecifiedErr + } + + if slices.Contains(b.mode.RequiredFlags, v1alpha1.PartitionFlag) && b.partition == "" { + return noPartitionSpecifiedErr + } + + if slices.Contains(b.mode.RequiredFlags, v1alpha1.PriorityFlag) && b.priority == "" { + return noPrioritySpecifiedErr + } + + if slices.Contains(b.mode.RequiredFlags, v1alpha1.TimeFlag) && b.timeLimit == "" { + return noTimeSpecifiedErr + } + + return nil +} + +func (b *Builder) Do(ctx context.Context) (runtime.Object, []runtime.Object, error) { + if err := b.setClients(); err != nil { + return nil, nil, err + } + + if err := b.validateGeneral(ctx); err != nil { + return nil, nil, err + } + + var bImpl builder + + switch b.modeName { + case v1alpha1.JobMode: + bImpl = newJobBuilder(b) + case v1alpha1.InteractiveMode: + bImpl = newInteractiveBuilder(b) + case v1alpha1.RayJobMode: + bImpl = newRayJobBuilder(b) + case v1alpha1.RayClusterMode: + bImpl = newRayClusterBuilder(b) + case v1alpha1.SlurmMode: + bImpl = newSlurmBuilder(b) + } + + if bImpl == nil { + return nil, nil, invalidApplicationProfileModeErr + } + + if err := b.complete(ctx); err != nil { + return nil, nil, err + } + + if err := b.validateFlags(); err != nil { + return nil, nil, err + } + + return bImpl.build(ctx) +} + +func (b *Builder) setClients() error { + var err error + + b.kjobctlClientset, err = b.clientGetter.KjobctlClientset() + if err != nil { + return err + } + + b.k8sClientset, err = b.clientGetter.K8sClientset() + if err != nil { + return err + } + + b.kueueClientset, err = b.clientGetter.KueueClientset() + if err != nil { + return err + } + + return nil +} + +func (b *Builder) buildObjectMeta(templateObjectMeta metav1.ObjectMeta, strictNaming bool) (metav1.ObjectMeta, error) { + objectMeta := metav1.ObjectMeta{ + Namespace: b.profile.Namespace, + Labels: templateObjectMeta.Labels, + Annotations: templateObjectMeta.Annotations, + } + + if strictNaming { + objectMeta.Name = b.generatePrefixName() + utilrand.String(5) + } else { + objectMeta.GenerateName = b.generatePrefixName() + } + + b.withKjobLabels(&objectMeta) + if err := b.withKueueLabels(&objectMeta); err != nil { + return metav1.ObjectMeta{}, err + } + + return objectMeta, nil +} + +func (b *Builder) buildChildObjectMeta(name string) metav1.ObjectMeta { + objectMeta := metav1.ObjectMeta{ + Name: name, + Namespace: b.profile.Namespace, + } + b.withKjobLabels(&objectMeta) + return objectMeta +} + +func (b *Builder) withKjobLabels(objectMeta *metav1.ObjectMeta) { + if objectMeta.Labels == nil { + objectMeta.Labels = map[string]string{} + } + + if b.profile != nil { + objectMeta.Labels[constants.ProfileLabel] = b.profile.Name + } + + if b.mode != nil { + objectMeta.Labels[constants.ModeLabel] = string(b.mode.Name) + } +} + +func (b *Builder) withKueueLabels(objectMeta *metav1.ObjectMeta) error { + if objectMeta.Labels == nil { + objectMeta.Labels = map[string]string{} + } + + if len(b.localQueue) > 0 { + objectMeta.Labels[kueueconstants.QueueLabel] = b.localQueue + } + + if len(b.priority) != 0 { + objectMeta.Labels[kueueconstants.WorkloadPriorityClassLabel] = b.priority + } + + if b.timeLimit != "" { + maxExecutionTimeSeconds, err := parser.TimeLimitToSeconds(b.timeLimit) + if err != nil { + return fmt.Errorf("cannot parse '%s': %w", b.timeLimit, err) + } + + if ptr.Deref(maxExecutionTimeSeconds, 0) > 0 { + objectMeta.Labels[kueueconstants.MaxExecTimeSecondsLabel] = fmt.Sprint(*maxExecutionTimeSeconds) + } + } + + return nil +} + +func (b *Builder) buildPodSpec(templateSpec corev1.PodSpec) corev1.PodSpec { + b.buildPodSpecVolumesAndEnv(&templateSpec) + + for i := range templateSpec.Containers { + container := &templateSpec.Containers[i] + + if i == 0 && len(b.command) > 0 { + container.Command = b.command + } + + if i == 0 && len(b.requests) > 0 { + container.Resources.Requests = b.requests + } + } + + return templateSpec +} + +func (b *Builder) buildPodSpecVolumesAndEnv(templateSpec *corev1.PodSpec) { + bundle := mergeBundles(b.volumeBundles) + + templateSpec.Volumes = append(templateSpec.Volumes, bundle.Spec.Volumes...) + for i := range templateSpec.Containers { + container := &templateSpec.Containers[i] + + container.VolumeMounts = append(container.VolumeMounts, bundle.Spec.ContainerVolumeMounts...) + container.Env = append(container.Env, bundle.Spec.EnvVars...) + container.Env = append(container.Env, b.additionalEnvironmentVariables()...) + } + + for i := range templateSpec.InitContainers { + initContainer := &templateSpec.InitContainers[i] + + initContainer.VolumeMounts = append(initContainer.VolumeMounts, bundle.Spec.ContainerVolumeMounts...) + initContainer.Env = append(initContainer.Env, bundle.Spec.EnvVars...) + initContainer.Env = append(initContainer.Env, b.additionalEnvironmentVariables()...) + } +} + +func (b *Builder) buildRayClusterSpec(spec *rayv1.RayClusterSpec) { + b.buildPodSpecVolumesAndEnv(&spec.HeadGroupSpec.Template.Spec) + + for index := range spec.WorkerGroupSpecs { + workerGroupSpec := &spec.WorkerGroupSpecs[index] + + if replicas, ok := b.replicas[workerGroupSpec.GroupName]; ok { + workerGroupSpec.Replicas = ptr.To(int32(replicas)) + } + if minReplicas, ok := b.minReplicas[workerGroupSpec.GroupName]; ok { + workerGroupSpec.MinReplicas = ptr.To(int32(minReplicas)) + } + if maxReplicas, ok := b.maxReplicas[workerGroupSpec.GroupName]; ok { + workerGroupSpec.MaxReplicas = ptr.To(int32(maxReplicas)) + } + + b.buildPodSpecVolumesAndEnv(&workerGroupSpec.Template.Spec) + } +} + +func (b *Builder) additionalEnvironmentVariables() []corev1.EnvVar { + userID := os.Getenv(constants.SystemEnvVarNameUser) + timestamp := b.buildTime.Format(time.RFC3339) + taskName := fmt.Sprintf("%s_%s", b.namespace, b.profileName) + + envVars := []corev1.EnvVar{ + {Name: constants.EnvVarNameUserID, Value: userID}, + {Name: constants.EnvVarTaskName, Value: taskName}, + {Name: constants.EnvVarTaskID, Value: fmt.Sprintf("%s_%s_%s", userID, timestamp, taskName)}, + {Name: constants.EnvVarNameProfile, Value: fmt.Sprintf("%s_%s", b.namespace, b.profileName)}, + {Name: constants.EnvVarNameTimestamp, Value: timestamp}, + } + + return envVars +} + +func mergeBundles(bundles []v1alpha1.VolumeBundle) v1alpha1.VolumeBundle { + var volumeBundle v1alpha1.VolumeBundle + for _, b := range bundles { + volumeBundle.Spec.Volumes = append(volumeBundle.Spec.Volumes, b.Spec.Volumes...) + volumeBundle.Spec.ContainerVolumeMounts = append(volumeBundle.Spec.ContainerVolumeMounts, b.Spec.ContainerVolumeMounts...) + volumeBundle.Spec.EnvVars = append(volumeBundle.Spec.EnvVars, b.Spec.EnvVars...) + } + + return volumeBundle +} + +func (b *Builder) generatePrefixName() string { + return strings.ToLower(fmt.Sprintf("%s-%s-", b.profile.Name, b.modeName)) +} diff --git a/pkg/builder/builder_test.go b/pkg/builder/builder_test.go new file mode 100644 index 0000000..527d809 --- /dev/null +++ b/pkg/builder/builder_test.go @@ -0,0 +1,472 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package builder + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/apis/v1alpha1" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/client-go/clientset/versioned/fake" + cmdtesting "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/cmd/testing" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/testing/wrappers" +) + +func TestBuilder(t *testing.T) { + testStartTime := time.Now() + + testCases := map[string]struct { + namespace string + profile string + mode v1alpha1.ApplicationProfileMode + kjobctlObjs []runtime.Object + wantRootObj runtime.Object + wantErr error + }{ + "shouldn't build job because no namespace specified": { + wantErr: noNamespaceSpecifiedErr, + }, + "shouldn't build job because no application profile specified": { + namespace: metav1.NamespaceDefault, + wantErr: noApplicationProfileSpecifiedErr, + }, + "shouldn't build job because application profile not found": { + namespace: metav1.NamespaceDefault, + profile: "profile", + mode: v1alpha1.JobMode, + wantErr: apierrors.NewNotFound(schema.GroupResource{Group: "kjobctl.x-k8s.io", Resource: "applicationprofiles"}, "profile"), + }, + "shouldn't build job because no application profile mode specified": { + namespace: metav1.NamespaceDefault, + profile: "profile", + kjobctlObjs: []runtime.Object{ + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(v1alpha1.SupportedMode{Name: v1alpha1.JobMode}). + Obj(), + }, + wantErr: noApplicationProfileModeSpecifiedErr, + }, + "shouldn't build job because application profile mode not configured": { + namespace: metav1.NamespaceDefault, + profile: "profile", + mode: v1alpha1.JobMode, + kjobctlObjs: []runtime.Object{ + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(v1alpha1.SupportedMode{Name: v1alpha1.InteractiveMode}). + Obj(), + }, + wantErr: applicationProfileModeNotConfiguredErr, + }, + "shouldn't build job because invalid application profile mode": { + namespace: metav1.NamespaceDefault, + profile: "profile", + mode: "Invalid", + kjobctlObjs: []runtime.Object{ + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(v1alpha1.SupportedMode{Name: v1alpha1.InteractiveMode}). + Obj(), + }, + wantErr: invalidApplicationProfileModeErr, + }, + "shouldn't build job because command not specified with required flags": { + namespace: metav1.NamespaceDefault, + profile: "profile", + mode: v1alpha1.JobMode, + kjobctlObjs: []runtime.Object{ + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(v1alpha1.SupportedMode{ + Name: v1alpha1.JobMode, + RequiredFlags: []v1alpha1.Flag{v1alpha1.CmdFlag}, + }). + Obj(), + }, + wantErr: noCommandSpecifiedErr, + }, + "shouldn't build job because parallelism not specified with required flags": { + namespace: metav1.NamespaceDefault, + profile: "profile", + mode: v1alpha1.JobMode, + kjobctlObjs: []runtime.Object{ + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(v1alpha1.SupportedMode{ + Name: v1alpha1.JobMode, + RequiredFlags: []v1alpha1.Flag{v1alpha1.ParallelismFlag}, + }). + Obj(), + }, + wantErr: noParallelismSpecifiedErr, + }, + "shouldn't build job because completions not specified with required flags": { + namespace: metav1.NamespaceDefault, + profile: "profile", + mode: v1alpha1.JobMode, + kjobctlObjs: []runtime.Object{ + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(v1alpha1.SupportedMode{ + Name: v1alpha1.JobMode, + RequiredFlags: []v1alpha1.Flag{v1alpha1.CompletionsFlag}, + }). + Obj(), + }, + wantErr: noCompletionsSpecifiedErr, + }, + "shouldn't build job because replicas not specified with required flags": { + namespace: metav1.NamespaceDefault, + profile: "profile", + mode: v1alpha1.JobMode, + kjobctlObjs: []runtime.Object{ + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(v1alpha1.SupportedMode{ + Name: v1alpha1.JobMode, + RequiredFlags: []v1alpha1.Flag{v1alpha1.ReplicasFlag}, + }). + Obj(), + }, + wantErr: noReplicasSpecifiedErr, + }, + "shouldn't build job because min-replicas not specified with required flags": { + namespace: metav1.NamespaceDefault, + profile: "profile", + mode: v1alpha1.JobMode, + kjobctlObjs: []runtime.Object{ + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(v1alpha1.SupportedMode{ + Name: v1alpha1.JobMode, + RequiredFlags: []v1alpha1.Flag{v1alpha1.MinReplicasFlag}, + }). + Obj(), + }, + wantErr: noMinReplicasSpecifiedErr, + }, + "shouldn't build job because max-replicas not specified with required flags": { + namespace: metav1.NamespaceDefault, + profile: "profile", + mode: v1alpha1.JobMode, + kjobctlObjs: []runtime.Object{ + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(v1alpha1.SupportedMode{ + Name: v1alpha1.JobMode, + RequiredFlags: []v1alpha1.Flag{v1alpha1.MaxReplicasFlag}, + }). + Obj(), + }, + wantErr: noMaxReplicasSpecifiedErr, + }, + "shouldn't build job because request not specified with required flags": { + namespace: metav1.NamespaceDefault, + profile: "profile", + mode: v1alpha1.JobMode, + kjobctlObjs: []runtime.Object{ + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(v1alpha1.SupportedMode{ + Name: v1alpha1.JobMode, + RequiredFlags: []v1alpha1.Flag{v1alpha1.RequestFlag}, + }). + Obj(), + }, + wantErr: noRequestsSpecifiedErr, + }, + "shouldn't build job because local queue not specified with required flags": { + namespace: metav1.NamespaceDefault, + profile: "profile", + mode: v1alpha1.JobMode, + kjobctlObjs: []runtime.Object{ + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(v1alpha1.SupportedMode{ + Name: v1alpha1.JobMode, + RequiredFlags: []v1alpha1.Flag{v1alpha1.LocalQueueFlag}, + }). + Obj(), + }, + wantErr: noLocalQueueSpecifiedErr, + }, + "shouldn't build job because raycluster not specified with required flags": { + namespace: metav1.NamespaceDefault, + profile: "profile", + mode: v1alpha1.RayJobMode, + kjobctlObjs: []runtime.Object{ + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(v1alpha1.SupportedMode{ + Name: v1alpha1.RayJobMode, + RequiredFlags: []v1alpha1.Flag{v1alpha1.RayClusterFlag}, + }). + Obj(), + }, + wantErr: noRayClusterSpecifiedErr, + }, + "shouldn't build job because array not specified with required flags": { + namespace: metav1.NamespaceDefault, + profile: "profile", + mode: v1alpha1.SlurmMode, + kjobctlObjs: []runtime.Object{ + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(v1alpha1.SupportedMode{ + Name: v1alpha1.SlurmMode, + RequiredFlags: []v1alpha1.Flag{v1alpha1.ArrayFlag}, + }). + Obj(), + }, + wantErr: noArraySpecifiedErr, + }, + "shouldn't build job because cpusPerTask not specified with required flags": { + namespace: metav1.NamespaceDefault, + profile: "profile", + mode: v1alpha1.SlurmMode, + kjobctlObjs: []runtime.Object{ + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(v1alpha1.SupportedMode{ + Name: v1alpha1.SlurmMode, + RequiredFlags: []v1alpha1.Flag{v1alpha1.CpusPerTaskFlag}, + }). + Obj(), + }, + wantErr: noCpusPerTaskSpecifiedErr, + }, + "shouldn't build job because error not specified with required flags": { + namespace: metav1.NamespaceDefault, + profile: "profile", + mode: v1alpha1.SlurmMode, + kjobctlObjs: []runtime.Object{ + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(v1alpha1.SupportedMode{ + Name: v1alpha1.SlurmMode, + RequiredFlags: []v1alpha1.Flag{v1alpha1.ErrorFlag}, + }). + Obj(), + }, + wantErr: noErrorSpecifiedErr, + }, + "shouldn't build job because gpusPerTask not specified with required flags": { + namespace: metav1.NamespaceDefault, + profile: "profile", + mode: v1alpha1.SlurmMode, + kjobctlObjs: []runtime.Object{ + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(v1alpha1.SupportedMode{ + Name: v1alpha1.SlurmMode, + RequiredFlags: []v1alpha1.Flag{v1alpha1.GpusPerTaskFlag}, + }). + Obj(), + }, + wantErr: noGpusPerTaskSpecifiedErr, + }, + "shouldn't build job because input not specified with required flags": { + namespace: metav1.NamespaceDefault, + profile: "profile", + mode: v1alpha1.SlurmMode, + kjobctlObjs: []runtime.Object{ + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(v1alpha1.SupportedMode{ + Name: v1alpha1.SlurmMode, + RequiredFlags: []v1alpha1.Flag{v1alpha1.InputFlag}, + }). + Obj(), + }, + wantErr: noInputSpecifiedErr, + }, + "shouldn't build job because jobName not specified with required flags": { + namespace: metav1.NamespaceDefault, + profile: "profile", + mode: v1alpha1.SlurmMode, + kjobctlObjs: []runtime.Object{ + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(v1alpha1.SupportedMode{ + Name: v1alpha1.SlurmMode, + RequiredFlags: []v1alpha1.Flag{v1alpha1.JobNameFlag}, + }). + Obj(), + }, + wantErr: noJobNameSpecifiedErr, + }, + "shouldn't build job because memPerCPU not specified with required flags": { + namespace: metav1.NamespaceDefault, + profile: "profile", + mode: v1alpha1.SlurmMode, + kjobctlObjs: []runtime.Object{ + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(v1alpha1.SupportedMode{ + Name: v1alpha1.SlurmMode, + RequiredFlags: []v1alpha1.Flag{v1alpha1.MemPerCPUFlag}, + }). + Obj(), + }, + wantErr: noMemPerCPUSpecifiedErr, + }, + "shouldn't build job because memPerGPU not specified with required flags": { + namespace: metav1.NamespaceDefault, + profile: "profile", + mode: v1alpha1.SlurmMode, + kjobctlObjs: []runtime.Object{ + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(v1alpha1.SupportedMode{ + Name: v1alpha1.SlurmMode, + RequiredFlags: []v1alpha1.Flag{v1alpha1.MemPerGPUFlag}, + }). + Obj(), + }, + wantErr: noMemPerGPUSpecifiedErr, + }, + "shouldn't build job because memPerTask not specified with required flags": { + namespace: metav1.NamespaceDefault, + profile: "profile", + mode: v1alpha1.SlurmMode, + kjobctlObjs: []runtime.Object{ + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(v1alpha1.SupportedMode{ + Name: v1alpha1.SlurmMode, + RequiredFlags: []v1alpha1.Flag{v1alpha1.MemPerTaskFlag}, + }). + Obj(), + }, + wantErr: noMemPerTaskSpecifiedErr, + }, + "shouldn't build job because nodes not specified with required flags": { + namespace: metav1.NamespaceDefault, + profile: "profile", + mode: v1alpha1.SlurmMode, + kjobctlObjs: []runtime.Object{ + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(v1alpha1.SupportedMode{ + Name: v1alpha1.SlurmMode, + RequiredFlags: []v1alpha1.Flag{v1alpha1.NodesFlag}, + }). + Obj(), + }, + wantErr: noNodesSpecifiedErr, + }, + "shouldn't build job because nTasks not specified with required flags": { + namespace: metav1.NamespaceDefault, + profile: "profile", + mode: v1alpha1.SlurmMode, + kjobctlObjs: []runtime.Object{ + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(v1alpha1.SupportedMode{ + Name: v1alpha1.SlurmMode, + RequiredFlags: []v1alpha1.Flag{v1alpha1.NTasksFlag}, + }). + Obj(), + }, + wantErr: noNTasksSpecifiedErr, + }, + "shouldn't build job because output not specified with required flags": { + namespace: metav1.NamespaceDefault, + profile: "profile", + mode: v1alpha1.SlurmMode, + kjobctlObjs: []runtime.Object{ + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(v1alpha1.SupportedMode{ + Name: v1alpha1.SlurmMode, + RequiredFlags: []v1alpha1.Flag{v1alpha1.OutputFlag}, + }). + Obj(), + }, + wantErr: noOutputSpecifiedErr, + }, + "shouldn't build job because partition not specified with required flags": { + namespace: metav1.NamespaceDefault, + profile: "profile", + mode: v1alpha1.SlurmMode, + kjobctlObjs: []runtime.Object{ + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(v1alpha1.SupportedMode{ + Name: v1alpha1.SlurmMode, + RequiredFlags: []v1alpha1.Flag{v1alpha1.PartitionFlag}, + }). + Obj(), + }, + wantErr: noPartitionSpecifiedErr, + }, + "shouldn't build job because priority not specified with required flags": { + namespace: metav1.NamespaceDefault, + profile: "profile", + mode: v1alpha1.SlurmMode, + kjobctlObjs: []runtime.Object{ + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(v1alpha1.SupportedMode{ + Name: v1alpha1.SlurmMode, + RequiredFlags: []v1alpha1.Flag{v1alpha1.PriorityFlag}, + }). + Obj(), + }, + wantErr: noPrioritySpecifiedErr, + }, + "should build job": { + namespace: metav1.NamespaceDefault, + profile: "profile", + mode: v1alpha1.JobMode, + kjobctlObjs: []runtime.Object{ + wrappers.MakeJobTemplate("job-template", metav1.NamespaceDefault). + Label("foo", "bar"). + Annotation("foo", "baz"). + Obj(), + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(v1alpha1.SupportedMode{ + Name: v1alpha1.JobMode, + Template: "job-template", + }). + Obj(), + }, + wantRootObj: wrappers.MakeJob("", metav1.NamespaceDefault).GenerateName("profile-job-"). + Annotation("foo", "baz"). + Label("foo", "bar"). + Profile("profile"). + Mode(v1alpha1.JobMode). + Obj(), + }, + } + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + tcg := cmdtesting.NewTestClientGetter(). + WithKjobctlClientset(fake.NewSimpleClientset(tc.kjobctlObjs...)) + gotRootObj, gotChildObjs, gotErr := NewBuilder(tcg, testStartTime). + WithNamespace(tc.namespace). + WithProfileName(tc.profile). + WithModeName(tc.mode). + Do(ctx) + + var opts []cmp.Option + var statusError *apierrors.StatusError + if !errors.As(tc.wantErr, &statusError) { + opts = append(opts, cmpopts.EquateErrors()) + } + if diff := cmp.Diff(tc.wantErr, gotErr, opts...); diff != "" { + t.Errorf("Unexpected error (-want/+got)\n%s", diff) + return + } + + if diff := cmp.Diff(tc.wantRootObj, gotRootObj, opts...); diff != "" { + t.Errorf("Root object after build (-want,+got):\n%s", diff) + } + + if diff := cmp.Diff([]runtime.Object(nil), gotChildObjs, opts...); diff != "" { + t.Errorf("Child objects after build (-want,+got):\n%s", diff) + } + }) + } +} diff --git a/pkg/builder/interactive_builder.go b/pkg/builder/interactive_builder.go new file mode 100644 index 0000000..0c8b302 --- /dev/null +++ b/pkg/builder/interactive_builder.go @@ -0,0 +1,65 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package builder + +import ( + "context" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" +) + +type interactiveBuilder struct { + *Builder +} + +var _ builder = (*interactiveBuilder)(nil) + +func (b *interactiveBuilder) build(ctx context.Context) (runtime.Object, []runtime.Object, error) { + template, err := b.k8sClientset.CoreV1().PodTemplates(b.profile.Namespace).Get(ctx, string(b.mode.Template), metav1.GetOptions{}) + if err != nil { + return nil, nil, err + } + + objectMeta, err := b.buildObjectMeta(template.Template.ObjectMeta, false) + if err != nil { + return nil, nil, err + } + + pod := &corev1.Pod{ + TypeMeta: metav1.TypeMeta{ + Kind: "Pod", + APIVersion: "v1", + }, + ObjectMeta: objectMeta, + Spec: template.Template.Spec, + } + + pod.Spec = b.buildPodSpec(pod.Spec) + + if len(pod.Spec.Containers) > 0 { + pod.Spec.Containers[0].TTY = true + pod.Spec.Containers[0].Stdin = true + } + + return pod, nil, nil +} + +func newInteractiveBuilder(b *Builder) *interactiveBuilder { + return &interactiveBuilder{Builder: b} +} diff --git a/pkg/builder/interactive_builder_test.go b/pkg/builder/interactive_builder_test.go new file mode 100644 index 0000000..8971aa5 --- /dev/null +++ b/pkg/builder/interactive_builder_test.go @@ -0,0 +1,218 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package builder + +import ( + "context" + "errors" + "fmt" + "os" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + k8sfake "k8s.io/client-go/kubernetes/fake" + kueueconstants "sigs.k8s.io/kueue/pkg/controller/constants" + + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/apis/v1alpha1" + kjobctlfake "sigs.k8s.io/kueue/cmd/experimental/kjobctl/client-go/clientset/versioned/fake" + cmdtesting "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/cmd/testing" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/constants" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/testing/wrappers" +) + +func TestInteractiveBuilder(t *testing.T) { + testStartTime := time.Now() + userID := os.Getenv(constants.SystemEnvVarNameUser) + + testPodTemplateWrapper := wrappers.MakePodTemplate("pod-template", metav1.NamespaceDefault). + Label("foo", "bar"). + Annotation("foo", "baz"). + WithInitContainer( + *wrappers.MakeContainer("ic1", ""). + WithEnvVar(corev1.EnvVar{Name: "e0", Value: "default-value0"}). + WithVolumeMount(corev1.VolumeMount{Name: "vm0", MountPath: "/etc/default-config0"}). + Obj(), + ). + WithContainer( + *wrappers.MakeContainer("c1", ""). + WithRequest(corev1.ResourceCPU, resource.MustParse("1")). + WithEnvVar(corev1.EnvVar{Name: "e1", Value: "default-value1"}). + WithEnvVar(corev1.EnvVar{Name: "e2", Value: "default-value2"}). + WithVolumeMount(corev1.VolumeMount{Name: "vm1", MountPath: "/etc/default-config1"}). + WithVolumeMount(corev1.VolumeMount{Name: "vm2", MountPath: "/etc/default-config2"}). + Obj(), + ). + WithContainer(*wrappers.MakeContainer("c2", ""). + WithRequest(corev1.ResourceCPU, resource.MustParse("2")). + WithEnvVar(corev1.EnvVar{Name: "e1", Value: "default-value1"}). + WithEnvVar(corev1.EnvVar{Name: "e2", Value: "default-value2"}). + WithVolumeMount(corev1.VolumeMount{Name: "vm1", MountPath: "/etc/default-config1"}). + WithVolumeMount(corev1.VolumeMount{Name: "vm2", MountPath: "/etc/default-config2"}). + Obj(), + ). + WithVolume("v1", "default-config1"). + WithVolume("v2", "default-config2") + + testCases := map[string]struct { + namespace string + profile string + mode v1alpha1.ApplicationProfileMode + command []string + requests corev1.ResourceList + localQueue string + k8sObjs []runtime.Object + kjobctlObjs []runtime.Object + wantRootObj runtime.Object + wantErr error + }{ + "shouldn't build pod because template not found": { + namespace: metav1.NamespaceDefault, + profile: "profile", + mode: v1alpha1.InteractiveMode, + kjobctlObjs: []runtime.Object{ + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(v1alpha1.SupportedMode{Name: v1alpha1.InteractiveMode, Template: "pod-template"}). + Obj(), + }, + wantErr: apierrors.NewNotFound(schema.GroupResource{Resource: "podtemplates"}, "pod-template"), + }, + "should build job without replacements": { + namespace: metav1.NamespaceDefault, + profile: "profile", + mode: v1alpha1.InteractiveMode, + k8sObjs: []runtime.Object{testPodTemplateWrapper.Clone().Obj()}, + kjobctlObjs: []runtime.Object{ + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(v1alpha1.SupportedMode{Name: v1alpha1.InteractiveMode, Template: "pod-template"}). + Obj(), + }, + wantRootObj: wrappers.MakePod("", metav1.NamespaceDefault).GenerateName("profile-interactive-"). + Annotation("foo", "baz"). + Label("foo", "bar"). + Profile("profile"). + Mode(v1alpha1.InteractiveMode). + Spec( + testPodTemplateWrapper.Clone(). + WithEnvVar(corev1.EnvVar{Name: constants.EnvVarNameUserID, Value: userID}). + WithEnvVar(corev1.EnvVar{Name: constants.EnvVarTaskName, Value: "default_profile"}). + WithEnvVar(corev1.EnvVar{ + Name: constants.EnvVarTaskID, + Value: fmt.Sprintf("%s_%s_default_profile", userID, testStartTime.Format(time.RFC3339)), + }). + WithEnvVar(corev1.EnvVar{Name: "PROFILE", Value: "default_profile"}). + WithEnvVar(corev1.EnvVar{Name: "TIMESTAMP", Value: testStartTime.Format(time.RFC3339)}). + TTY(). + Stdin(). + Obj().Template.Spec, + ). + Obj(), + }, + "should build job with replacements": { + namespace: metav1.NamespaceDefault, + profile: "profile", + mode: v1alpha1.InteractiveMode, + command: []string{"sleep"}, + requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("3")}, + localQueue: "lq1", + k8sObjs: []runtime.Object{testPodTemplateWrapper.Clone().Obj()}, + kjobctlObjs: []runtime.Object{ + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(v1alpha1.SupportedMode{Name: v1alpha1.InteractiveMode, Template: "pod-template"}). + WithVolumeBundleReferences("vb1", "vb2"). + Obj(), + wrappers.MakeVolumeBundle("vb1", metav1.NamespaceDefault). + WithVolume("v3", "config3"). + WithVolumeMount(corev1.VolumeMount{Name: "vm3", MountPath: "/etc/config3"}). + WithEnvVar(corev1.EnvVar{Name: "e3", Value: "value3"}). + Obj(), + wrappers.MakeVolumeBundle("vb2", metav1.NamespaceDefault).Obj(), + }, + wantRootObj: wrappers.MakePod("", metav1.NamespaceDefault).GenerateName("profile-interactive-"). + Annotation("foo", "baz"). + Label("foo", "bar"). + Profile("profile"). + Mode(v1alpha1.InteractiveMode). + Label(kueueconstants.QueueLabel, "lq1"). + Spec( + testPodTemplateWrapper.Clone(). + Command([]string{"sleep"}). + WithRequest(corev1.ResourceCPU, resource.MustParse("3")). + WithVolume("v3", "config3"). + WithVolumeMount(corev1.VolumeMount{Name: "vm3", MountPath: "/etc/config3"}). + WithEnvVar(corev1.EnvVar{Name: "e3", Value: "value3"}). + WithEnvVar(corev1.EnvVar{Name: constants.EnvVarNameUserID, Value: userID}). + WithEnvVar(corev1.EnvVar{Name: constants.EnvVarTaskName, Value: "default_profile"}). + WithEnvVar(corev1.EnvVar{ + Name: constants.EnvVarTaskID, + Value: fmt.Sprintf("%s_%s_default_profile", userID, testStartTime.Format(time.RFC3339)), + }). + WithEnvVar(corev1.EnvVar{Name: "PROFILE", Value: "default_profile"}). + WithEnvVar(corev1.EnvVar{Name: "TIMESTAMP", Value: testStartTime.Format(time.RFC3339)}). + TTY(). + Stdin(). + Obj().Template.Spec, + ). + Obj(), + }, + } + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + tcg := cmdtesting.NewTestClientGetter(). + WithKjobctlClientset(kjobctlfake.NewSimpleClientset(tc.kjobctlObjs...)). + WithK8sClientset(k8sfake.NewSimpleClientset(tc.k8sObjs...)) + + gotRootObj, gotChildObjs, gotErr := NewBuilder(tcg, testStartTime). + WithNamespace(tc.namespace). + WithProfileName(tc.profile). + WithModeName(tc.mode). + WithCommand(tc.command). + WithRequests(tc.requests). + WithLocalQueue(tc.localQueue). + WithSkipLocalQueueValidation(true). + Do(ctx) + + var opts []cmp.Option + var statusError *apierrors.StatusError + if !errors.As(tc.wantErr, &statusError) { + opts = append(opts, cmpopts.EquateErrors()) + } + if diff := cmp.Diff(tc.wantErr, gotErr, opts...); diff != "" { + t.Errorf("Unexpected error (-want/+got)\n%s", diff) + return + } + + if diff := cmp.Diff(tc.wantRootObj, gotRootObj, opts...); diff != "" { + t.Errorf("Root object after build (-want,+got):\n%s", diff) + } + + if diff := cmp.Diff([]runtime.Object(nil), gotChildObjs, opts...); diff != "" { + t.Errorf("Child objects after build (-want,+got):\n%s", diff) + } + }) + } +} diff --git a/pkg/builder/job_builder.go b/pkg/builder/job_builder.go new file mode 100644 index 0000000..4bc543b --- /dev/null +++ b/pkg/builder/job_builder.go @@ -0,0 +1,69 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package builder + +import ( + "context" + + batchv1 "k8s.io/api/batch/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" +) + +type jobBuilder struct { + *Builder +} + +var _ builder = (*jobBuilder)(nil) + +func (b *jobBuilder) build(ctx context.Context) (runtime.Object, []runtime.Object, error) { + template, err := b.kjobctlClientset.KjobctlV1alpha1().JobTemplates(b.profile.Namespace). + Get(ctx, string(b.mode.Template), metav1.GetOptions{}) + if err != nil { + return nil, nil, err + } + + objectMeta, err := b.buildObjectMeta(template.Template.ObjectMeta, false) + if err != nil { + return nil, nil, err + } + + job := &batchv1.Job{ + TypeMeta: metav1.TypeMeta{ + Kind: "Job", + APIVersion: "batch/v1", + }, + ObjectMeta: objectMeta, + Spec: template.Template.Spec, + } + + job.Spec.Template.Spec = b.buildPodSpec(job.Spec.Template.Spec) + + if b.parallelism != nil { + job.Spec.Parallelism = b.parallelism + } + + if b.completions != nil { + job.Spec.Completions = b.completions + } + + return job, nil, nil +} + +func newJobBuilder(b *Builder) *jobBuilder { + return &jobBuilder{Builder: b} +} diff --git a/pkg/builder/job_builder_test.go b/pkg/builder/job_builder_test.go new file mode 100644 index 0000000..71e9b9b --- /dev/null +++ b/pkg/builder/job_builder_test.go @@ -0,0 +1,222 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package builder + +import ( + "context" + "errors" + "fmt" + "os" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/utils/ptr" + kueueconstants "sigs.k8s.io/kueue/pkg/controller/constants" + + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/apis/v1alpha1" + kjobctlfake "sigs.k8s.io/kueue/cmd/experimental/kjobctl/client-go/clientset/versioned/fake" + cmdtesting "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/cmd/testing" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/constants" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/testing/wrappers" +) + +func TestJobBuilder(t *testing.T) { + testStartTime := time.Now() + userID := os.Getenv(constants.SystemEnvVarNameUser) + + testJobTemplateWrapper := wrappers.MakeJobTemplate("job-template", metav1.NamespaceDefault). + Label("foo", "bar"). + Annotation("foo", "baz"). + Parallelism(1). + Completions(1). + WithInitContainer( + *wrappers.MakeContainer("ic1", ""). + WithEnvVar(corev1.EnvVar{Name: "e0", Value: "default-value0"}). + WithVolumeMount(corev1.VolumeMount{Name: "vm0", MountPath: "/etc/default-config0"}). + Obj(), + ). + WithContainer( + *wrappers.MakeContainer("c1", ""). + WithRequest(corev1.ResourceCPU, resource.MustParse("1")). + WithEnvVar(corev1.EnvVar{Name: "e1", Value: "default-value1"}). + WithEnvVar(corev1.EnvVar{Name: "e2", Value: "default-value2"}). + WithVolumeMount(corev1.VolumeMount{Name: "vm1", MountPath: "/etc/default-config1"}). + WithVolumeMount(corev1.VolumeMount{Name: "vm2", MountPath: "/etc/default-config2"}). + Obj(), + ). + WithContainer(*wrappers.MakeContainer("c2", ""). + WithRequest(corev1.ResourceCPU, resource.MustParse("2")). + WithEnvVar(corev1.EnvVar{Name: "e1", Value: "default-value1"}). + WithEnvVar(corev1.EnvVar{Name: "e2", Value: "default-value2"}). + WithVolumeMount(corev1.VolumeMount{Name: "vm1", MountPath: "/etc/default-config1"}). + WithVolumeMount(corev1.VolumeMount{Name: "vm2", MountPath: "/etc/default-config2"}). + Obj(), + ). + WithVolume("v1", "default-config1"). + WithVolume("v2", "default-config2") + + testCases := map[string]struct { + namespace string + profile string + mode v1alpha1.ApplicationProfileMode + command []string + parallelism *int32 + completions *int32 + requests corev1.ResourceList + localQueue string + kjobctlObjs []runtime.Object + wantRootObj runtime.Object + wantErr error + }{ + "shouldn't build job because template not found": { + namespace: metav1.NamespaceDefault, + profile: "profile", + mode: v1alpha1.JobMode, + kjobctlObjs: []runtime.Object{ + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(v1alpha1.SupportedMode{Name: v1alpha1.JobMode, Template: "job-template"}). + Obj(), + }, + wantErr: apierrors.NewNotFound(schema.GroupResource{Group: "kjobctl.x-k8s.io", Resource: "jobtemplates"}, "job-template"), + }, + "should build job without replacements": { + namespace: metav1.NamespaceDefault, + profile: "profile", + mode: v1alpha1.JobMode, + kjobctlObjs: []runtime.Object{ + testJobTemplateWrapper.Clone().Obj(), + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(v1alpha1.SupportedMode{Name: v1alpha1.JobMode, Template: "job-template"}). + Obj(), + }, + wantRootObj: wrappers.MakeJob("", metav1.NamespaceDefault).GenerateName("profile-job-"). + Annotation("foo", "baz"). + Label("foo", "bar"). + Profile("profile"). + Mode(v1alpha1.JobMode). + Spec( + testJobTemplateWrapper.Clone(). + WithEnvVar(corev1.EnvVar{Name: constants.EnvVarNameUserID, Value: userID}). + WithEnvVar(corev1.EnvVar{Name: constants.EnvVarTaskName, Value: "default_profile"}). + WithEnvVar(corev1.EnvVar{ + Name: constants.EnvVarTaskID, + Value: fmt.Sprintf("%s_%s_default_profile", userID, testStartTime.Format(time.RFC3339)), + }). + WithEnvVar(corev1.EnvVar{Name: "PROFILE", Value: "default_profile"}). + WithEnvVar(corev1.EnvVar{Name: "TIMESTAMP", Value: testStartTime.Format(time.RFC3339)}). + Obj().Template.Spec, + ). + Obj(), + }, + "should build job with replacements": { + namespace: metav1.NamespaceDefault, + profile: "profile", + mode: v1alpha1.JobMode, + command: []string{"sleep"}, + parallelism: ptr.To[int32](2), + completions: ptr.To[int32](3), + requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("3")}, + localQueue: "lq1", + kjobctlObjs: []runtime.Object{ + testJobTemplateWrapper.Clone().Obj(), + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(v1alpha1.SupportedMode{Name: v1alpha1.JobMode, Template: "job-template"}). + WithVolumeBundleReferences("vb1", "vb2"). + Obj(), + wrappers.MakeVolumeBundle("vb1", metav1.NamespaceDefault). + WithVolume("v3", "config3"). + WithVolumeMount(corev1.VolumeMount{Name: "vm3", MountPath: "/etc/config3"}). + WithEnvVar(corev1.EnvVar{Name: "e3", Value: "value3"}). + Obj(), + wrappers.MakeVolumeBundle("vb2", metav1.NamespaceDefault).Obj(), + }, + wantRootObj: wrappers.MakeJob("", metav1.NamespaceDefault).GenerateName("profile-job-"). + Annotation("foo", "baz"). + Label("foo", "bar"). + Profile("profile"). + Mode(v1alpha1.JobMode). + Label(kueueconstants.QueueLabel, "lq1"). + Spec( + testJobTemplateWrapper.Clone(). + Command([]string{"sleep"}). + Parallelism(2). + Completions(3). + WithRequest(corev1.ResourceCPU, resource.MustParse("3")). + WithVolume("v3", "config3"). + WithVolumeMount(corev1.VolumeMount{Name: "vm3", MountPath: "/etc/config3"}). + WithEnvVar(corev1.EnvVar{Name: "e3", Value: "value3"}). + WithEnvVar(corev1.EnvVar{Name: constants.EnvVarNameUserID, Value: userID}). + WithEnvVar(corev1.EnvVar{Name: constants.EnvVarTaskName, Value: "default_profile"}). + WithEnvVar(corev1.EnvVar{ + Name: constants.EnvVarTaskID, + Value: fmt.Sprintf("%s_%s_default_profile", userID, testStartTime.Format(time.RFC3339)), + }). + WithEnvVar(corev1.EnvVar{Name: "PROFILE", Value: "default_profile"}). + WithEnvVar(corev1.EnvVar{Name: "TIMESTAMP", Value: testStartTime.Format(time.RFC3339)}). + Obj().Template.Spec, + ). + Obj(), + }, + } + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + tcg := cmdtesting.NewTestClientGetter(). + WithKjobctlClientset(kjobctlfake.NewSimpleClientset(tc.kjobctlObjs...)) + + gotRootObj, gotChildObjs, gotErr := NewBuilder(tcg, testStartTime). + WithNamespace(tc.namespace). + WithProfileName(tc.profile). + WithModeName(tc.mode). + WithCommand(tc.command). + WithParallelism(tc.parallelism). + WithCompletions(tc.completions). + WithRequests(tc.requests). + WithLocalQueue(tc.localQueue). + WithSkipLocalQueueValidation(true). + Do(ctx) + + var opts []cmp.Option + var statusError *apierrors.StatusError + if !errors.As(tc.wantErr, &statusError) { + opts = append(opts, cmpopts.EquateErrors()) + } + if diff := cmp.Diff(tc.wantErr, gotErr, opts...); diff != "" { + t.Errorf("Unexpected error (-want/+got)\n%s", diff) + return + } + + if diff := cmp.Diff(tc.wantRootObj, gotRootObj, opts...); diff != "" { + t.Errorf("Root object after build (-want,+got):\n%s", diff) + } + + if diff := cmp.Diff([]runtime.Object(nil), gotChildObjs, opts...); diff != "" { + t.Errorf("Child objects after build (-want,+got):\n%s", diff) + } + }) + } +} diff --git a/pkg/builder/ray_cluster_builder.go b/pkg/builder/ray_cluster_builder.go new file mode 100644 index 0000000..f84b871 --- /dev/null +++ b/pkg/builder/ray_cluster_builder.go @@ -0,0 +1,61 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package builder + +import ( + "context" + + rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" +) + +type rayClusterBuilder struct { + *Builder +} + +var _ builder = (*rayClusterBuilder)(nil) + +func (b *rayClusterBuilder) build(ctx context.Context) (runtime.Object, []runtime.Object, error) { + template, err := b.kjobctlClientset.KjobctlV1alpha1().RayClusterTemplates(b.profile.Namespace). + Get(ctx, string(b.mode.Template), metav1.GetOptions{}) + if err != nil { + return nil, nil, err + } + + objectMeta, err := b.buildObjectMeta(template.Template.ObjectMeta, false) + if err != nil { + return nil, nil, err + } + + rayCluster := &rayv1.RayCluster{ + TypeMeta: metav1.TypeMeta{ + Kind: "RayCluster", + APIVersion: "ray.io/v1", + }, + ObjectMeta: objectMeta, + Spec: template.Template.Spec, + } + + b.buildRayClusterSpec(&rayCluster.Spec) + + return rayCluster, nil, nil +} + +func newRayClusterBuilder(b *Builder) *rayClusterBuilder { + return &rayClusterBuilder{Builder: b} +} diff --git a/pkg/builder/ray_cluster_builder_test.go b/pkg/builder/ray_cluster_builder_test.go new file mode 100644 index 0000000..30ca75e --- /dev/null +++ b/pkg/builder/ray_cluster_builder_test.go @@ -0,0 +1,267 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package builder + +import ( + "context" + "errors" + "fmt" + "os" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + kueueconstants "sigs.k8s.io/kueue/pkg/controller/constants" + + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/apis/v1alpha1" + kjobctlfake "sigs.k8s.io/kueue/cmd/experimental/kjobctl/client-go/clientset/versioned/fake" + cmdtesting "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/cmd/testing" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/constants" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/testing/wrappers" +) + +func TestRayClusterBuilder(t *testing.T) { + testStartTime := time.Now() + userID := os.Getenv(constants.SystemEnvVarNameUser) + + testRayClusterTemplateWrapper := wrappers.MakeRayClusterTemplate("ray-cluster-template", metav1.NamespaceDefault). + Label("foo", "bar"). + Annotation("foo", "baz"). + Spec(*wrappers.MakeRayClusterSpec(). + WithWorkerGroupSpec( + *wrappers.MakeWorkerGroupSpec("g1"). + WithInitContainer( + *wrappers.MakeContainer("ic1", ""). + WithEnvVar(corev1.EnvVar{Name: "e0", Value: "default-value0"}). + WithVolumeMount(corev1.VolumeMount{Name: "vm0", MountPath: "/etc/default-config0"}). + Obj(), + ). + WithContainer( + *wrappers.MakeContainer("c1", ""). + WithRequest(corev1.ResourceCPU, resource.MustParse("1")). + WithEnvVar(corev1.EnvVar{Name: "e1", Value: "default-value1"}). + WithEnvVar(corev1.EnvVar{Name: "e2", Value: "default-value2"}). + WithVolumeMount(corev1.VolumeMount{Name: "vm1", MountPath: "/etc/default-config1"}). + WithVolumeMount(corev1.VolumeMount{Name: "vm2", MountPath: "/etc/default-config2"}). + Obj(), + ). + WithContainer(*wrappers.MakeContainer("c2", ""). + WithRequest(corev1.ResourceCPU, resource.MustParse("2")). + WithEnvVar(corev1.EnvVar{Name: "e1", Value: "default-value1"}). + WithEnvVar(corev1.EnvVar{Name: "e2", Value: "default-value2"}). + WithVolumeMount(corev1.VolumeMount{Name: "vm1", MountPath: "/etc/default-config1"}). + WithVolumeMount(corev1.VolumeMount{Name: "vm2", MountPath: "/etc/default-config2"}). + Obj(), + ). + WithVolume("v1", "default-config1"). + WithVolume("v2", "default-config2"). + Obj(), + ). + WithWorkerGroupSpec( + *wrappers.MakeWorkerGroupSpec("g2"). + WithInitContainer( + *wrappers.MakeContainer("ic1", ""). + WithEnvVar(corev1.EnvVar{Name: "e0", Value: "default-value0"}). + WithVolumeMount(corev1.VolumeMount{Name: "vm0", MountPath: "/etc/default-config0"}). + Obj(), + ). + WithContainer( + *wrappers.MakeContainer("c1", ""). + WithRequest(corev1.ResourceCPU, resource.MustParse("1")). + WithEnvVar(corev1.EnvVar{Name: "e1", Value: "default-value1"}). + WithEnvVar(corev1.EnvVar{Name: "e2", Value: "default-value2"}). + WithVolumeMount(corev1.VolumeMount{Name: "vm1", MountPath: "/etc/default-config1"}). + WithVolumeMount(corev1.VolumeMount{Name: "vm2", MountPath: "/etc/default-config2"}). + Obj(), + ). + WithContainer(*wrappers.MakeContainer("c2", ""). + WithRequest(corev1.ResourceCPU, resource.MustParse("2")). + WithEnvVar(corev1.EnvVar{Name: "e1", Value: "default-value1"}). + WithEnvVar(corev1.EnvVar{Name: "e2", Value: "default-value2"}). + WithVolumeMount(corev1.VolumeMount{Name: "vm1", MountPath: "/etc/default-config1"}). + WithVolumeMount(corev1.VolumeMount{Name: "vm2", MountPath: "/etc/default-config2"}). + Obj(), + ). + WithVolume("v1", "default-config1"). + WithVolume("v2", "default-config2"). + Obj(), + ). + Obj(), + ) + + testCases := map[string]struct { + namespace string + profile string + mode v1alpha1.ApplicationProfileMode + command []string + replicas map[string]int + minReplicas map[string]int + maxReplicas map[string]int + requests corev1.ResourceList + localQueue string + kjobctlObjs []runtime.Object + wantRootObj runtime.Object + wantErr error + }{ + "shouldn't build ray cluster because template not found": { + namespace: metav1.NamespaceDefault, + profile: "profile", + mode: v1alpha1.RayClusterMode, + kjobctlObjs: []runtime.Object{ + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(v1alpha1.SupportedMode{Name: v1alpha1.RayClusterMode, Template: "ray-cluster-template"}). + Obj(), + }, + wantErr: apierrors.NewNotFound(schema.GroupResource{Group: "kjobctl.x-k8s.io", Resource: "rayclustertemplates"}, "ray-cluster-template"), + }, + "should build ray cluster without replacements": { + namespace: metav1.NamespaceDefault, + profile: "profile", + mode: v1alpha1.RayClusterMode, + kjobctlObjs: []runtime.Object{ + testRayClusterTemplateWrapper.Clone().Obj(), + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(v1alpha1.SupportedMode{Name: v1alpha1.RayClusterMode, Template: "ray-cluster-template"}). + Obj(), + }, + wantRootObj: wrappers.MakeRayCluster("", metav1.NamespaceDefault).GenerateName("profile-raycluster-"). + Annotation("foo", "baz"). + Label("foo", "bar"). + Profile("profile"). + Mode(v1alpha1.RayClusterMode). + Spec( + testRayClusterTemplateWrapper.Clone(). + Spec( + *wrappers.FromRayClusterSpec(testRayClusterTemplateWrapper.Clone().Template.Spec). + WithEnvVar(corev1.EnvVar{Name: constants.EnvVarNameUserID, Value: userID}). + WithEnvVar(corev1.EnvVar{Name: constants.EnvVarTaskName, Value: "default_profile"}). + WithEnvVar(corev1.EnvVar{ + Name: constants.EnvVarTaskID, + Value: fmt.Sprintf("%s_%s_default_profile", userID, testStartTime.Format(time.RFC3339)), + }). + WithEnvVar(corev1.EnvVar{Name: "PROFILE", Value: "default_profile"}). + WithEnvVar(corev1.EnvVar{Name: "TIMESTAMP", Value: testStartTime.Format(time.RFC3339)}). + Obj(), + ). + Template.Spec, + ). + Obj(), + }, + "should build ray cluster with replacements": { + namespace: metav1.NamespaceDefault, + profile: "profile", + mode: v1alpha1.RayClusterMode, + command: []string{"sleep"}, + replicas: map[string]int{"g1": 10, "g2": 20}, + minReplicas: map[string]int{"g1": 10, "g2": 20}, + maxReplicas: map[string]int{"g1": 15, "g2": 25}, + requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("3")}, + localQueue: "lq1", + kjobctlObjs: []runtime.Object{ + testRayClusterTemplateWrapper.Clone().Obj(), + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(v1alpha1.SupportedMode{Name: v1alpha1.RayClusterMode, Template: "ray-cluster-template"}). + WithVolumeBundleReferences("vb1", "vb2"). + Obj(), + wrappers.MakeVolumeBundle("vb1", metav1.NamespaceDefault). + WithVolume("v3", "config3"). + WithVolumeMount(corev1.VolumeMount{Name: "vm3", MountPath: "/etc/config3"}). + WithEnvVar(corev1.EnvVar{Name: "e3", Value: "value3"}). + Obj(), + wrappers.MakeVolumeBundle("vb2", metav1.NamespaceDefault).Obj(), + }, + wantRootObj: wrappers.MakeRayCluster("", metav1.NamespaceDefault).GenerateName("profile-raycluster-"). + Annotation("foo", "baz"). + Label("foo", "bar"). + Profile("profile"). + Mode(v1alpha1.RayClusterMode). + Label(kueueconstants.QueueLabel, "lq1"). + Spec( + testRayClusterTemplateWrapper.Clone(). + Spec( + *wrappers.FromRayClusterSpec(testRayClusterTemplateWrapper.Clone().Template.Spec). + Replicas("g1", 10). + Replicas("g2", 20). + MinReplicas("g1", 10). + MinReplicas("g2", 20). + MaxReplicas("g1", 15). + MaxReplicas("g2", 25). + WithVolume("v3", "config3"). + WithVolumeMount(corev1.VolumeMount{Name: "vm3", MountPath: "/etc/config3"}). + WithEnvVar(corev1.EnvVar{Name: "e3", Value: "value3"}). + WithEnvVar(corev1.EnvVar{Name: constants.EnvVarNameUserID, Value: userID}). + WithEnvVar(corev1.EnvVar{Name: constants.EnvVarTaskName, Value: "default_profile"}). + WithEnvVar(corev1.EnvVar{ + Name: constants.EnvVarTaskID, + Value: fmt.Sprintf("%s_%s_default_profile", userID, testStartTime.Format(time.RFC3339)), + }). + WithEnvVar(corev1.EnvVar{Name: "PROFILE", Value: "default_profile"}). + WithEnvVar(corev1.EnvVar{Name: "TIMESTAMP", Value: testStartTime.Format(time.RFC3339)}). + Obj(), + ). + Obj().Template.Spec, + ). + Obj(), + }, + } + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + tcg := cmdtesting.NewTestClientGetter(). + WithKjobctlClientset(kjobctlfake.NewSimpleClientset(tc.kjobctlObjs...)) + + gotRootObj, gotChildObjs, gotErr := NewBuilder(tcg, testStartTime). + WithNamespace(tc.namespace). + WithProfileName(tc.profile). + WithModeName(tc.mode). + WithCommand(tc.command). + WithReplicas(tc.replicas). + WithMinReplicas(tc.minReplicas). + WithMaxReplicas(tc.maxReplicas). + WithLocalQueue(tc.localQueue). + WithSkipLocalQueueValidation(true). + Do(ctx) + + var opts []cmp.Option + var statusError *apierrors.StatusError + if !errors.As(tc.wantErr, &statusError) { + opts = append(opts, cmpopts.EquateErrors()) + } + if diff := cmp.Diff(tc.wantErr, gotErr, opts...); diff != "" { + t.Errorf("Unexpected error (-want/+got)\n%s", diff) + return + } + + if diff := cmp.Diff(tc.wantRootObj, gotRootObj, opts...); diff != "" { + t.Errorf("Root object after build (-want,+got):\n%s", diff) + } + + if diff := cmp.Diff([]runtime.Object(nil), gotChildObjs, opts...); diff != "" { + t.Errorf("Child objects after build (-want,+got):\n%s", diff) + } + }) + } +} diff --git a/pkg/builder/ray_job_builder.go b/pkg/builder/ray_job_builder.go new file mode 100644 index 0000000..1969e13 --- /dev/null +++ b/pkg/builder/ray_job_builder.go @@ -0,0 +1,77 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package builder + +import ( + "context" + "strings" + + rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1" + rayutil "github.com/ray-project/kuberay/ray-operator/controllers/ray/utils" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + kueueconstants "sigs.k8s.io/kueue/pkg/controller/constants" +) + +type rayJobBuilder struct { + *Builder +} + +var _ builder = (*rayJobBuilder)(nil) + +func (b *rayJobBuilder) build(ctx context.Context) (runtime.Object, []runtime.Object, error) { + template, err := b.kjobctlClientset.KjobctlV1alpha1().RayJobTemplates(b.profile.Namespace). + Get(ctx, string(b.mode.Template), metav1.GetOptions{}) + if err != nil { + return nil, nil, err + } + + objectMeta, err := b.buildObjectMeta(template.Template.ObjectMeta, false) + if err != nil { + return nil, nil, err + } + + rayJob := &rayv1.RayJob{ + TypeMeta: metav1.TypeMeta{ + Kind: "RayJob", + APIVersion: "ray.io/v1", + }, + ObjectMeta: objectMeta, + Spec: template.Template.Spec, + } + + if b.command != nil { + rayJob.Spec.Entrypoint = strings.Join(b.command, " ") + } + + if len(b.rayCluster) != 0 { + delete(rayJob.ObjectMeta.Labels, kueueconstants.QueueLabel) + rayJob.Spec.RayClusterSpec = nil + if rayJob.Spec.ClusterSelector == nil { + rayJob.Spec.ClusterSelector = map[string]string{} + } + rayJob.Spec.ClusterSelector[rayutil.RayClusterLabelKey] = b.rayCluster + } else if rayJob.Spec.RayClusterSpec != nil { + b.buildRayClusterSpec(rayJob.Spec.RayClusterSpec) + } + + return rayJob, nil, nil +} + +func newRayJobBuilder(b *Builder) *rayJobBuilder { + return &rayJobBuilder{Builder: b} +} diff --git a/pkg/builder/ray_job_builder_test.go b/pkg/builder/ray_job_builder_test.go new file mode 100644 index 0000000..272bee0 --- /dev/null +++ b/pkg/builder/ray_job_builder_test.go @@ -0,0 +1,300 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package builder + +import ( + "context" + "errors" + "fmt" + "os" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + kueueconstants "sigs.k8s.io/kueue/pkg/controller/constants" + + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/apis/v1alpha1" + kjobctlfake "sigs.k8s.io/kueue/cmd/experimental/kjobctl/client-go/clientset/versioned/fake" + cmdtesting "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/cmd/testing" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/constants" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/testing/wrappers" +) + +func TestRayJobBuilder(t *testing.T) { + testStartTime := time.Now() + userID := os.Getenv(constants.SystemEnvVarNameUser) + + testRayJobTemplateWrapper := wrappers.MakeRayJobTemplate("ray-job-template", metav1.NamespaceDefault). + Label("foo", "bar"). + Annotation("foo", "baz"). + WithRayClusterSpec(wrappers.MakeRayClusterSpec(). + WithWorkerGroupSpec( + *wrappers.MakeWorkerGroupSpec("g1"). + WithInitContainer( + *wrappers.MakeContainer("ic1", ""). + WithEnvVar(corev1.EnvVar{Name: "e0", Value: "default-value0"}). + WithVolumeMount(corev1.VolumeMount{Name: "vm0", MountPath: "/etc/default-config0"}). + Obj(), + ). + WithContainer( + *wrappers.MakeContainer("c1", ""). + WithRequest(corev1.ResourceCPU, resource.MustParse("1")). + WithEnvVar(corev1.EnvVar{Name: "e1", Value: "default-value1"}). + WithEnvVar(corev1.EnvVar{Name: "e2", Value: "default-value2"}). + WithVolumeMount(corev1.VolumeMount{Name: "vm1", MountPath: "/etc/default-config1"}). + WithVolumeMount(corev1.VolumeMount{Name: "vm2", MountPath: "/etc/default-config2"}). + Obj(), + ). + WithContainer(*wrappers.MakeContainer("c2", ""). + WithRequest(corev1.ResourceCPU, resource.MustParse("2")). + WithEnvVar(corev1.EnvVar{Name: "e1", Value: "default-value1"}). + WithEnvVar(corev1.EnvVar{Name: "e2", Value: "default-value2"}). + WithVolumeMount(corev1.VolumeMount{Name: "vm1", MountPath: "/etc/default-config1"}). + WithVolumeMount(corev1.VolumeMount{Name: "vm2", MountPath: "/etc/default-config2"}). + Obj(), + ). + WithVolume("v1", "default-config1"). + WithVolume("v2", "default-config2"). + Obj(), + ). + WithWorkerGroupSpec( + *wrappers.MakeWorkerGroupSpec("g2"). + WithInitContainer( + *wrappers.MakeContainer("ic1", ""). + WithEnvVar(corev1.EnvVar{Name: "e0", Value: "default-value0"}). + WithVolumeMount(corev1.VolumeMount{Name: "vm0", MountPath: "/etc/default-config0"}). + Obj(), + ). + WithContainer( + *wrappers.MakeContainer("c1", ""). + WithRequest(corev1.ResourceCPU, resource.MustParse("1")). + WithEnvVar(corev1.EnvVar{Name: "e1", Value: "default-value1"}). + WithEnvVar(corev1.EnvVar{Name: "e2", Value: "default-value2"}). + WithVolumeMount(corev1.VolumeMount{Name: "vm1", MountPath: "/etc/default-config1"}). + WithVolumeMount(corev1.VolumeMount{Name: "vm2", MountPath: "/etc/default-config2"}). + Obj(), + ). + WithContainer(*wrappers.MakeContainer("c2", ""). + WithRequest(corev1.ResourceCPU, resource.MustParse("2")). + WithEnvVar(corev1.EnvVar{Name: "e1", Value: "default-value1"}). + WithEnvVar(corev1.EnvVar{Name: "e2", Value: "default-value2"}). + WithVolumeMount(corev1.VolumeMount{Name: "vm1", MountPath: "/etc/default-config1"}). + WithVolumeMount(corev1.VolumeMount{Name: "vm2", MountPath: "/etc/default-config2"}). + Obj(), + ). + WithVolume("v1", "default-config1"). + WithVolume("v2", "default-config2"). + Obj(), + ). + Obj(), + ) + + testCases := map[string]struct { + namespace string + profile string + mode v1alpha1.ApplicationProfileMode + command []string + replicas map[string]int + minReplicas map[string]int + maxReplicas map[string]int + requests corev1.ResourceList + localQueue string + rayCluster string + kjobctlObjs []runtime.Object + wantRootObj runtime.Object + wantErr error + }{ + "shouldn't build ray job because template not found": { + namespace: metav1.NamespaceDefault, + profile: "profile", + mode: v1alpha1.RayJobMode, + kjobctlObjs: []runtime.Object{ + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(v1alpha1.SupportedMode{Name: v1alpha1.RayJobMode, Template: "ray-job-template"}). + Obj(), + }, + wantErr: apierrors.NewNotFound(schema.GroupResource{Group: "kjobctl.x-k8s.io", Resource: "rayjobtemplates"}, "ray-job-template"), + }, + "should build ray job without replacements": { + namespace: metav1.NamespaceDefault, + profile: "profile", + mode: v1alpha1.RayJobMode, + kjobctlObjs: []runtime.Object{ + testRayJobTemplateWrapper.Clone().Obj(), + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(v1alpha1.SupportedMode{Name: v1alpha1.RayJobMode, Template: "ray-job-template"}). + Obj(), + }, + wantRootObj: wrappers.MakeRayJob("", metav1.NamespaceDefault).GenerateName("profile-rayjob-"). + Annotation("foo", "baz"). + Label("foo", "bar"). + Profile("profile"). + Mode(v1alpha1.RayJobMode). + Spec( + testRayJobTemplateWrapper.Clone(). + WithRayClusterSpec( + wrappers.FromRayClusterSpec(*testRayJobTemplateWrapper.Clone().Template.Spec.RayClusterSpec). + WithEnvVar(corev1.EnvVar{Name: constants.EnvVarNameUserID, Value: userID}). + WithEnvVar(corev1.EnvVar{Name: constants.EnvVarTaskName, Value: "default_profile"}). + WithEnvVar(corev1.EnvVar{ + Name: constants.EnvVarTaskID, + Value: fmt.Sprintf("%s_%s_default_profile", userID, testStartTime.Format(time.RFC3339)), + }). + WithEnvVar(corev1.EnvVar{Name: "PROFILE", Value: "default_profile"}). + WithEnvVar(corev1.EnvVar{Name: "TIMESTAMP", Value: testStartTime.Format(time.RFC3339)}). + Obj(), + ). + Template.Spec, + ). + Obj(), + }, + "should build ray job with replacements": { + namespace: metav1.NamespaceDefault, + profile: "profile", + mode: v1alpha1.RayJobMode, + command: []string{"sleep"}, + replicas: map[string]int{"g1": 10, "g2": 20}, + minReplicas: map[string]int{"g1": 10, "g2": 20}, + maxReplicas: map[string]int{"g1": 15, "g2": 25}, + requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("3")}, + localQueue: "lq1", + kjobctlObjs: []runtime.Object{ + testRayJobTemplateWrapper.Clone().Obj(), + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(v1alpha1.SupportedMode{Name: v1alpha1.RayJobMode, Template: "ray-job-template"}). + WithVolumeBundleReferences("vb1", "vb2"). + Obj(), + wrappers.MakeVolumeBundle("vb1", metav1.NamespaceDefault). + WithVolume("v3", "config3"). + WithVolumeMount(corev1.VolumeMount{Name: "vm3", MountPath: "/etc/config3"}). + WithEnvVar(corev1.EnvVar{Name: "e3", Value: "value3"}). + Obj(), + wrappers.MakeVolumeBundle("vb2", metav1.NamespaceDefault).Obj(), + }, + wantRootObj: wrappers.MakeRayJob("", metav1.NamespaceDefault).GenerateName("profile-rayjob-"). + Annotation("foo", "baz"). + Label("foo", "bar"). + Profile("profile"). + Mode(v1alpha1.RayJobMode). + Label(kueueconstants.QueueLabel, "lq1"). + Spec( + testRayJobTemplateWrapper.Clone(). + Entrypoint("sleep"). + WithRayClusterSpec( + wrappers.FromRayClusterSpec(*testRayJobTemplateWrapper.Clone().Template.Spec.RayClusterSpec). + Replicas("g1", 10). + Replicas("g2", 20). + MinReplicas("g1", 10). + MinReplicas("g2", 20). + MaxReplicas("g1", 15). + MaxReplicas("g2", 25). + WithVolume("v3", "config3"). + WithVolumeMount(corev1.VolumeMount{Name: "vm3", MountPath: "/etc/config3"}). + WithEnvVar(corev1.EnvVar{Name: "e3", Value: "value3"}). + WithEnvVar(corev1.EnvVar{Name: constants.EnvVarNameUserID, Value: userID}). + WithEnvVar(corev1.EnvVar{Name: constants.EnvVarTaskName, Value: "default_profile"}). + WithEnvVar(corev1.EnvVar{ + Name: constants.EnvVarTaskID, + Value: fmt.Sprintf("%s_%s_default_profile", userID, testStartTime.Format(time.RFC3339)), + }). + WithEnvVar(corev1.EnvVar{Name: "PROFILE", Value: "default_profile"}). + WithEnvVar(corev1.EnvVar{Name: "TIMESTAMP", Value: testStartTime.Format(time.RFC3339)}). + Obj(), + ). + Obj().Template.Spec, + ). + Obj(), + }, + "should build ray job with raycluster replacement": { + namespace: metav1.NamespaceDefault, + profile: "profile", + mode: v1alpha1.RayJobMode, + command: []string{"python /home/ray/samples/sample_code.py"}, + requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("3")}, + localQueue: "lq1", + rayCluster: "rc1", + kjobctlObjs: []runtime.Object{ + testRayJobTemplateWrapper.Clone().Obj(), + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(v1alpha1.SupportedMode{Name: v1alpha1.RayJobMode, Template: "ray-job-template"}). + WithVolumeBundleReferences("vb1", "vb2"). + Obj(), + wrappers.MakeVolumeBundle("vb1", metav1.NamespaceDefault). + WithVolume("v3", "config3"). + WithVolumeMount(corev1.VolumeMount{Name: "vm3", MountPath: "/etc/config3"}). + WithEnvVar(corev1.EnvVar{Name: "e3", Value: "value3"}). + Obj(), + wrappers.MakeVolumeBundle("vb2", metav1.NamespaceDefault).Obj(), + }, + wantRootObj: wrappers.MakeRayJob("", metav1.NamespaceDefault).GenerateName("profile-rayjob-"). + Annotation("foo", "baz"). + Label("foo", "bar"). + Profile("profile"). + Mode(v1alpha1.RayJobMode). + WithRayClusterLabelSelector("rc1"). + Entrypoint("python /home/ray/samples/sample_code.py"). + Obj(), + }, + } + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + tcg := cmdtesting.NewTestClientGetter(). + WithKjobctlClientset(kjobctlfake.NewSimpleClientset(tc.kjobctlObjs...)) + + gotRootObj, gotChildObjs, gotErr := NewBuilder(tcg, testStartTime). + WithNamespace(tc.namespace). + WithProfileName(tc.profile). + WithModeName(tc.mode). + WithCommand(tc.command). + WithReplicas(tc.replicas). + WithMinReplicas(tc.minReplicas). + WithMaxReplicas(tc.maxReplicas). + WithLocalQueue(tc.localQueue). + WithSkipLocalQueueValidation(true). + WithRayCluster(tc.rayCluster). + Do(ctx) + + var opts []cmp.Option + var statusError *apierrors.StatusError + if !errors.As(tc.wantErr, &statusError) { + opts = append(opts, cmpopts.EquateErrors()) + } + if diff := cmp.Diff(tc.wantErr, gotErr, opts...); diff != "" { + t.Errorf("Unexpected error (-want/+got)\n%s", diff) + return + } + + if diff := cmp.Diff(tc.wantRootObj, gotRootObj, opts...); diff != "" { + t.Errorf("Root object after build (-want,+got):\n%s", diff) + } + + if diff := cmp.Diff([]runtime.Object(nil), gotChildObjs, opts...); diff != "" { + t.Errorf("Child objects after build (-want,+got):\n%s", diff) + } + }) + } +} diff --git a/pkg/builder/slurm_builder.go b/pkg/builder/slurm_builder.go new file mode 100644 index 0000000..fd1b643 --- /dev/null +++ b/pkg/builder/slurm_builder.go @@ -0,0 +1,775 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package builder + +import ( + "bytes" + "context" + "embed" + "errors" + "fmt" + "math" + "os" + "strconv" + "strings" + "text/template" + + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/utils/ptr" + + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/apis/v1alpha1" + kjobctlconstants "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/constants" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/parser" +) + +//go:embed templates/slurm_* +var slurmTemplates embed.FS + +const ( + // Note that the first job ID will always be 1. + slurmArrayJobID = 1 + + slurmScriptsPath = "/slurm/scripts" + slurmInitEntrypointFilename = "init-entrypoint.sh" + slurmEntrypointFilename = "entrypoint.sh" + slurmScriptFilename = "script" + + slurmEnvsPath = "/slurm/env" + slurmSlurmEnvFilename = "slurm.env" +) + +var ( + noScriptSpecifiedErr = errors.New("no script specified") +) + +var ( + slurmInitEntrypointFilenamePath = fmt.Sprintf("%s/%s", slurmScriptsPath, slurmInitEntrypointFilename) + slurmEntrypointFilenamePath = fmt.Sprintf("%s/%s", slurmScriptsPath, slurmEntrypointFilename) + slurmScriptFilenamePath = fmt.Sprintf("%s/%s", slurmScriptsPath, slurmScriptFilename) + + unmaskReplacer = strings.NewReplacer( + "%%", "%", + "%A", "${SLURM_ARRAY_JOB_ID}", + "%a", "${SLURM_ARRAY_TASK_ID}", + "%j", "${SLURM_JOB_ID}", + "%N", "${HOSTNAME}", + "%n", "${JOB_COMPLETION_INDEX}", + "%t", "${SLURM_ARRAY_TASK_ID}", + "%u", "${USER_ID}", + "%x", "${SBATCH_JOB_NAME}", + ) +) + +type slurmBuilder struct { + *Builder + + scriptContent string + template *template.Template + arrayIndexes parser.ArrayIndexes + cpusOnNode *resource.Quantity + cpusPerGpu *resource.Quantity + totalMemPerNode *resource.Quantity + totalGpus *resource.Quantity +} + +var _ builder = (*slurmBuilder)(nil) + +func (b *slurmBuilder) validateGeneral() error { + if len(b.script) == 0 { + return noScriptSpecifiedErr + } + + if b.memPerCPU != nil && b.cpusPerTask == nil { + return noCpusPerTaskSpecifiedErr + } + + if b.memPerGPU != nil && b.gpusPerTask == nil { + return noGpusPerTaskSpecifiedErr + } + + return nil +} + +func (b *slurmBuilder) complete() error { + content, err := os.ReadFile(b.script) + if err != nil { + return err + } + b.scriptContent = string(content) + + t, err := template.ParseFS(slurmTemplates, "templates/*") + if err != nil { + return err + } + b.template = t + + if err := b.getSbatchEnvs(); err != nil { + return err + } + + if err := b.replaceScriptFlags(); err != nil { + return err + } + + if err := b.validateMutuallyExclusiveFlags(); err != nil { + return err + } + + if b.array == "" { + b.arrayIndexes = parser.GenerateArrayIndexes(ptr.Deref(b.nodes, 1) * ptr.Deref(b.nTasks, 1)) + } else { + b.arrayIndexes, err = parser.ParseArrayIndexes(b.array) + if err != nil { + return err + } + if b.arrayIndexes.Parallelism != nil { + b.nodes = b.arrayIndexes.Parallelism + } + } + + return nil +} + +func (b *slurmBuilder) validateMutuallyExclusiveFlags() error { + flags := map[string]bool{ + string(v1alpha1.MemPerNodeFlag): b.memPerNode != nil, + string(v1alpha1.MemPerTaskFlag): b.memPerTask != nil, + string(v1alpha1.MemPerCPUFlag): b.memPerCPU != nil, + string(v1alpha1.MemPerGPUFlag): b.memPerGPU != nil, + } + + var setFlagsCount int + setFlags := make([]string, 0) + for f, isSet := range flags { + if isSet { + setFlagsCount++ + setFlags = append(setFlags, f) + } + } + + if setFlagsCount > 1 { + return fmt.Errorf( + "if any flags in the group [%s %s %s] are set none of the others can be; [%s] were all set", + v1alpha1.MemPerTaskFlag, + v1alpha1.MemPerGPUFlag, + v1alpha1.MemPerGPUFlag, + strings.Join(setFlags, " "), + ) + } + + return nil +} + +func (b *slurmBuilder) build(ctx context.Context) (runtime.Object, []runtime.Object, error) { + if err := b.validateGeneral(); err != nil { + return nil, nil, err + } + + if err := b.complete(); err != nil { + return nil, nil, err + } + + template, err := b.kjobctlClientset.KjobctlV1alpha1().JobTemplates(b.profile.Namespace). + Get(ctx, string(b.mode.Template), metav1.GetOptions{}) + if err != nil { + return nil, nil, err + } + + objectMeta, err := b.buildObjectMeta(template.Template.ObjectMeta, true) + if err != nil { + return nil, nil, err + } + + if b.script != "" { + if objectMeta.Annotations == nil { + objectMeta.Annotations = make(map[string]string, 1) + } + objectMeta.Annotations[kjobctlconstants.ScriptAnnotation] = b.script + } + + job := &batchv1.Job{ + TypeMeta: metav1.TypeMeta{Kind: "Job", APIVersion: "batch/v1"}, + ObjectMeta: objectMeta, + Spec: template.Template.Spec, + } + + job.Spec.CompletionMode = ptr.To(batchv1.IndexedCompletion) + job.Spec.Template.Spec.Subdomain = job.Name + + b.buildPodSpecVolumesAndEnv(&job.Spec.Template.Spec) + job.Spec.Template.Spec.Volumes = append(job.Spec.Template.Spec.Volumes, + corev1.Volume{ + Name: "slurm-scripts", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: job.Name, + }, + Items: []corev1.KeyToPath{ + { + Key: slurmInitEntrypointFilename, + Path: slurmInitEntrypointFilename, + }, + { + Key: slurmEntrypointFilename, + Path: slurmEntrypointFilename, + }, + { + Key: slurmScriptFilename, + Path: slurmScriptFilename, + Mode: ptr.To[int32](0755), + }, + }, + }, + }, + }, + corev1.Volume{ + Name: "slurm-env", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }, + ) + + job.Spec.Template.Spec.InitContainers = append(job.Spec.Template.Spec.InitContainers, corev1.Container{ + Name: "slurm-init-env", + Image: b.initImage, + Command: []string{"sh", slurmInitEntrypointFilenamePath}, + Env: []corev1.EnvVar{ + { + Name: "POD_IP", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{FieldPath: "status.podIP"}, + }, + }, + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "slurm-scripts", + MountPath: slurmScriptsPath, + }, + { + Name: "slurm-env", + MountPath: slurmEnvsPath, + }, + }, + }) + + var totalGPUsPerTask resource.Quantity + for _, number := range b.gpusPerTask { + totalGPUsPerTask.Add(*number) + } + + var memPerCPU, memPerGPU, memPerContainer resource.Quantity + if b.memPerCPU != nil && b.cpusPerTask != nil { + memPerCPU = *b.memPerCPU + memPerCPU.Mul(b.cpusPerTask.Value()) + } + + if b.memPerGPU != nil && b.gpusPerTask != nil { + memPerGPU = *b.memPerGPU + memPerGPU.Mul(totalGPUsPerTask.Value()) + } + + if b.memPerNode != nil { + mem := b.memPerNode.MilliValue() / int64(len(job.Spec.Template.Spec.Containers)) + memPerContainer = *resource.NewMilliQuantity(mem, b.memPerNode.Format) + } + + var totalCpus, totalGpus, totalMem resource.Quantity + for i := range job.Spec.Template.Spec.Containers { + container := &job.Spec.Template.Spec.Containers[i] + + container.Command = []string{"bash", slurmEntrypointFilenamePath} + + var requests corev1.ResourceList + if b.requests != nil { + requests = b.requests + } else { + requests = corev1.ResourceList{} + } + + if b.cpusPerTask != nil { + requests[corev1.ResourceCPU] = *b.cpusPerTask + totalCpus.Add(*b.cpusPerTask) + } + + if b.gpusPerTask != nil { + for name, number := range b.gpusPerTask { + requests[corev1.ResourceName(name)] = *number + } + totalGpus.Add(totalGPUsPerTask) + } + + if b.memPerTask != nil { + requests[corev1.ResourceMemory] = *b.memPerTask + totalMem.Add(*b.memPerTask) + } + + if !memPerCPU.IsZero() { + requests[corev1.ResourceMemory] = memPerCPU + totalMem.Add(memPerCPU) + } + + if !memPerGPU.IsZero() { + requests[corev1.ResourceMemory] = memPerGPU + totalMem.Add(memPerGPU) + } + + if len(requests) > 0 { + container.Resources.Requests = requests + } + + limits := corev1.ResourceList{} + if !memPerContainer.IsZero() { + limits[corev1.ResourceMemory] = memPerContainer + } + + if len(limits) > 0 { + container.Resources.Limits = limits + } + + container.VolumeMounts = append(container.VolumeMounts, + corev1.VolumeMount{ + Name: "slurm-scripts", + MountPath: slurmScriptsPath, + }, + corev1.VolumeMount{ + Name: "slurm-env", + MountPath: slurmEnvsPath, + }, + ) + } + + nTasks := ptr.Deref(b.nTasks, 1) + completions := int32(math.Ceil(float64(b.arrayIndexes.Count()) / float64(nTasks))) + + job.Spec.Completions = ptr.To(completions) + job.Spec.Parallelism = b.nodes + + if nTasks > 1 { + for i := 1; i < int(nTasks); i++ { + replica := job.Spec.Template.Spec.Containers[0].DeepCopy() + replica.Name = fmt.Sprintf("%s-%d", job.Spec.Template.Spec.Containers[0].Name, i) + job.Spec.Template.Spec.Containers = append(job.Spec.Template.Spec.Containers, *replica) + + if b.cpusPerTask != nil { + totalCpus.Add(*b.cpusPerTask) + } + + if !memPerCPU.IsZero() { + totalMem.Add(memPerCPU) + } + + if !memPerGPU.IsZero() { + totalMem.Add(memPerGPU) + } + } + + job.Spec.Template.Spec.Containers[0].Name = fmt.Sprintf("%s-0", job.Spec.Template.Spec.Containers[0].Name) + } + + for i := range job.Spec.Template.Spec.Containers { + job.Spec.Template.Spec.Containers[i].Env = append(job.Spec.Template.Spec.Containers[i].Env, corev1.EnvVar{ + Name: "JOB_CONTAINER_INDEX", + Value: strconv.FormatInt(int64(i), 10), + }) + } + + if b.nodes != nil { + job.Spec.Parallelism = b.nodes + } + + if !totalCpus.IsZero() { + b.cpusOnNode = &totalCpus + } + + if !totalGpus.IsZero() { + b.totalGpus = &totalGpus + } + + if b.memPerNode != nil { + b.totalMemPerNode = b.memPerNode + } else if !totalMem.IsZero() { + b.totalMemPerNode = &totalMem + } + + if b.cpusOnNode != nil && b.totalGpus != nil && !totalGpus.IsZero() { + cpusPerGpu := totalCpus.MilliValue() / totalGpus.MilliValue() + b.cpusPerGpu = resource.NewQuantity(cpusPerGpu, b.cpusOnNode.Format) + } + + initEntrypointScript, err := b.buildInitEntrypointScript(job.Name) + if err != nil { + return nil, nil, err + } + + entrypointScript, err := b.buildEntrypointScript() + if err != nil { + return nil, nil, err + } + + configMap := &corev1.ConfigMap{ + TypeMeta: metav1.TypeMeta{Kind: "ConfigMap", APIVersion: "v1"}, + ObjectMeta: b.buildChildObjectMeta(job.Name), + Data: map[string]string{ + slurmInitEntrypointFilename: initEntrypointScript, + slurmEntrypointFilename: entrypointScript, + slurmScriptFilename: b.scriptContent, + }, + } + + service := &corev1.Service{ + TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: "v1"}, + ObjectMeta: b.buildChildObjectMeta(job.Name), + Spec: corev1.ServiceSpec{ + ClusterIP: "None", + Selector: map[string]string{ + "job-name": job.Name, + }, + }, + } + + return job, []runtime.Object{configMap, service}, nil +} + +func (b *slurmBuilder) buildArrayIndexes() string { + nTasks := ptr.Deref(b.nTasks, 1) + length := int64(math.Ceil(float64(len(b.arrayIndexes.Indexes)) / float64(nTasks))) + containerIndexes := make([][]string, length) + + var ( + completionIndex int32 + containerIndex int32 + ) + for _, index := range b.arrayIndexes.Indexes { + containerIndexes[completionIndex] = append(containerIndexes[completionIndex], fmt.Sprint(index)) + containerIndex++ + if containerIndex >= nTasks { + containerIndex = 0 + completionIndex++ + } + } + + completionIndexes := make([]string, length) + for completionIndex, containerIndexes := range containerIndexes { + completionIndexes[completionIndex] = strings.Join(containerIndexes, ",") + } + + return strings.Join(completionIndexes, ";") +} + +type slurmInitEntrypointScript struct { + ArrayIndexes string + + JobName string + Namespace string + + EnvsPath string + SlurmEnvFilename string + + SlurmArrayJobID int32 + SlurmArrayTaskCount int32 + SlurmArrayTaskMax int32 + SlurmArrayTaskMin int32 + SlurmTasksPerNode int32 + SlurmCPUsPerTask string + SlurmCPUsOnNode string + SlurmJobCPUsPerNode string + SlurmCPUsPerGPU string + SlurmMemPerCPU string + SlurmMemPerGPU string + SlurmMemPerNode string + SlurmGPUs string + SlurmNTasks int32 + SlurmNTasksPerNode int32 + SlurmNProcs int32 + SlurmNNodes int32 + SlurmSubmitDir string + SlurmJobNodeList string + SlurmJobFirstNode string + + FirstNodeIP bool + FirstNodeIPTimeoutSeconds int32 +} + +func (b *slurmBuilder) buildInitEntrypointScript(jobName string) (string, error) { + nTasks := ptr.Deref(b.nTasks, 1) + nodes := ptr.Deref(b.nodes, 1) + + nodeList := make([]string, nodes) + for i := int32(0); i < nodes; i++ { + nodeList[i] = fmt.Sprintf("%s-%d.%s", jobName, i, jobName) + } + + scriptValues := slurmInitEntrypointScript{ + ArrayIndexes: b.buildArrayIndexes(), + + JobName: jobName, + Namespace: b.namespace, + + EnvsPath: slurmEnvsPath, + SlurmEnvFilename: slurmSlurmEnvFilename, + + SlurmArrayJobID: slurmArrayJobID, + SlurmArrayTaskCount: int32(b.arrayIndexes.Count()), + SlurmArrayTaskMax: b.arrayIndexes.Max(), + SlurmArrayTaskMin: b.arrayIndexes.Min(), + SlurmTasksPerNode: nTasks, + SlurmCPUsPerTask: getValueOrEmpty(b.cpusPerTask), + SlurmCPUsOnNode: getValueOrEmpty(b.cpusOnNode), + SlurmJobCPUsPerNode: getValueOrEmpty(b.cpusOnNode), + SlurmCPUsPerGPU: getValueOrEmpty(b.cpusPerGpu), + SlurmMemPerCPU: getValueOrEmpty(b.memPerCPU), + SlurmMemPerGPU: getValueOrEmpty(b.memPerGPU), + SlurmMemPerNode: getValueOrEmpty(b.totalMemPerNode), + SlurmGPUs: getValueOrEmpty(b.totalGpus), + SlurmNTasks: nTasks, + SlurmNTasksPerNode: nTasks, + SlurmNProcs: nTasks, + SlurmNNodes: nodes, + SlurmSubmitDir: slurmScriptsPath, + SlurmJobNodeList: strings.Join(nodeList, ","), + SlurmJobFirstNode: nodeList[0], + + FirstNodeIP: b.firstNodeIP, + FirstNodeIPTimeoutSeconds: int32(b.firstNodeIPTimeout.Seconds()), + } + + var script bytes.Buffer + + if err := b.template.ExecuteTemplate(&script, "slurm_init_entrypoint_script.sh.tmpl", scriptValues); err != nil { + return "", err + } + + return script.String(), nil +} + +type slurmEntrypointScript struct { + EnvsPath string + SbatchJobName string + SlurmEnvFilename string + ChangeDir string + BuildEntrypointCommand string +} + +func (b *slurmBuilder) buildEntrypointScript() (string, error) { + scriptValues := slurmEntrypointScript{ + EnvsPath: slurmEnvsPath, + SbatchJobName: b.jobName, + SlurmEnvFilename: slurmSlurmEnvFilename, + ChangeDir: b.changeDir, + BuildEntrypointCommand: b.buildEntrypointCommand(), + } + + var script bytes.Buffer + + if err := b.template.ExecuteTemplate(&script, "slurm_entrypoint_script.sh.tmpl", scriptValues); err != nil { + return "", err + } + + return script.String(), nil +} + +func getValueOrEmpty(ptr *resource.Quantity) string { + if ptr != nil { + return ptr.String() + } + + return "" +} + +// unmaskFilename unmasks a filename based on the filename pattern. +// For more details, see https://slurm.schedmd.com/sbatch.html#SECTION_FILENAME-PATTERN. +func unmaskFilename(filename string) string { + if strings.Contains(filename, "\\\\") { + return strings.ReplaceAll(filename, "\\\\", "") + } + return unmaskReplacer.Replace(filename) +} + +func (b *slurmBuilder) buildEntrypointCommand() string { + strBuilder := strings.Builder{} + + strBuilder.WriteString(slurmScriptFilenamePath) + + if b.input != "" { + strBuilder.WriteString(" <") + strBuilder.WriteString(unmaskFilename(b.input)) + } + + if b.output != "" { + strBuilder.WriteString(" 1>") + strBuilder.WriteString(unmaskFilename(b.output)) + } + + if b.error != "" { + strBuilder.WriteString(" 2>") + strBuilder.WriteString(unmaskFilename(b.error)) + } + + return strBuilder.String() +} + +func (b *slurmBuilder) getSbatchEnvs() error { + if len(b.array) == 0 { + b.array = os.Getenv("SBATCH_ARRAY_INX") + } + + if b.gpusPerTask == nil { + if env, ok := os.LookupEnv("SBATCH_GPUS_PER_TASK"); ok { + val, err := parser.GpusFlag(env) + if err != nil { + return fmt.Errorf("cannot parse '%s': %w", env, err) + } + b.gpusPerTask = val + } + } + + if b.memPerNode == nil { + if env, ok := os.LookupEnv("SBATCH_MEM_PER_NODE"); ok { + val, err := resource.ParseQuantity(env) + if err != nil { + return fmt.Errorf("cannot parse '%s': %w", env, err) + } + b.memPerNode = ptr.To(val) + } + } + + if b.memPerTask == nil { + if env, ok := os.LookupEnv("SBATCH_MEM_PER_CPU"); ok { + val, err := resource.ParseQuantity(env) + if err != nil { + return fmt.Errorf("cannot parse '%s': %w", env, err) + } + b.memPerTask = ptr.To(val) + } + } + + if b.memPerGPU == nil { + if env, ok := os.LookupEnv("SBATCH_MEM_PER_GPU"); ok { + val, err := resource.ParseQuantity(env) + if err != nil { + return fmt.Errorf("cannot parse '%s': %w", env, err) + } + b.memPerGPU = ptr.To(val) + } + } + + if len(b.output) == 0 { + b.output = os.Getenv("SBATCH_OUTPUT") + } + + if len(b.error) == 0 { + b.error = os.Getenv("SBATCH_ERROR") + } + + if len(b.input) == 0 { + b.input = os.Getenv("SBATCH_INPUT") + } + + if len(b.jobName) == 0 { + b.jobName = os.Getenv("SBATCH_JOB_NAME") + } + + if len(b.partition) == 0 { + b.partition = os.Getenv("SBATCH_PARTITION") + } + + if b.timeLimit == "" { + b.timeLimit = os.Getenv("SBATCH_TIMELIMIT") + } + + return nil +} + +func (b *slurmBuilder) replaceScriptFlags() error { + scriptFlags, err := parser.SlurmFlags(b.scriptContent, b.ignoreUnknown) + if err != nil { + return err + } + + if len(b.array) == 0 { + b.array = scriptFlags.Array + } + + if b.cpusPerTask == nil { + b.cpusPerTask = scriptFlags.CpusPerTask + } + + if b.gpusPerTask == nil { + b.gpusPerTask = scriptFlags.GpusPerTask + } + + if b.memPerNode == nil { + b.memPerNode = scriptFlags.MemPerNode + } + + if b.memPerTask == nil { + b.memPerTask = scriptFlags.MemPerTask + } + + if b.memPerCPU == nil { + b.memPerCPU = scriptFlags.MemPerCPU + } + + if b.memPerGPU == nil { + b.memPerGPU = scriptFlags.MemPerGPU + } + + if b.nodes == nil { + b.nodes = scriptFlags.Nodes + } + + if b.nTasks == nil { + b.nTasks = scriptFlags.NTasks + } + + if len(b.output) == 0 { + b.output = scriptFlags.Output + } + + if len(b.error) == 0 { + b.error = scriptFlags.Error + } + + if len(b.input) == 0 { + b.input = scriptFlags.Input + } + + if len(b.jobName) == 0 { + b.jobName = scriptFlags.JobName + } + + if len(b.partition) == 0 { + b.partition = scriptFlags.Partition + } + + if b.timeLimit == "" { + b.timeLimit = scriptFlags.TimeLimit + } + + return nil +} + +func newSlurmBuilder(b *Builder) *slurmBuilder { + return &slurmBuilder{Builder: b} +} diff --git a/pkg/builder/slurm_builder_test.go b/pkg/builder/slurm_builder_test.go new file mode 100644 index 0000000..78fe933 --- /dev/null +++ b/pkg/builder/slurm_builder_test.go @@ -0,0 +1,413 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package builder + +import ( + "context" + "errors" + "fmt" + "os" + "regexp" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/utils/ptr" + + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/apis/v1alpha1" + kjobctlfake "sigs.k8s.io/kueue/cmd/experimental/kjobctl/client-go/clientset/versioned/fake" + cmdtesting "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/cmd/testing" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/constants" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/testing/wrappers" +) + +type slurmBuilderTestCase struct { + beforeTest func(t *testing.T, tc *slurmBuilderTestCase) + tempFile string + namespace string + profile string + mode v1alpha1.ApplicationProfileMode + array string + cpusPerTask *resource.Quantity + gpusPerTask map[string]*resource.Quantity + memPerTask *resource.Quantity + memPerCPU *resource.Quantity + memPerGPU *resource.Quantity + nodes *int32 + nTasks *int32 + output string + err string + input string + jobName string + partition string + initImage string + firstNodeTimeout time.Duration + kjobctlObjs []runtime.Object + wantRootObj runtime.Object + wantChildObjs []runtime.Object + wantErr error + cmpopts []cmp.Option +} + +func beforeSlurmTest(t *testing.T, tc *slurmBuilderTestCase) { + file, err := os.CreateTemp("", "slurm") + if err != nil { + t.Fatal(err) + } + defer file.Close() + t.Cleanup(func() { + if err := os.Remove(file.Name()); err != nil { + t.Fatal(err) + } + }) + + if _, err := file.WriteString("#!/bin/bash\nsleep 300'"); err != nil { + t.Fatal(err) + } + + tc.tempFile = file.Name() +} + +func TestSlurmBuilderDo(t *testing.T) { + testStartTime := time.Now() + userID := os.Getenv(constants.SystemEnvVarNameUser) + + testCases := map[string]slurmBuilderTestCase{ + "shouldn't build slurm job because script not specified": { + namespace: metav1.NamespaceDefault, + profile: "profile", + mode: v1alpha1.SlurmMode, + kjobctlObjs: []runtime.Object{ + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(v1alpha1.SupportedMode{Name: v1alpha1.SlurmMode, Template: "slurm-template"}). + Obj(), + }, + wantErr: noScriptSpecifiedErr, + }, + "shouldn't build slurm job because template not found": { + beforeTest: beforeSlurmTest, + namespace: metav1.NamespaceDefault, + profile: "profile", + mode: v1alpha1.SlurmMode, + kjobctlObjs: []runtime.Object{ + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(v1alpha1.SupportedMode{Name: v1alpha1.SlurmMode, Template: "slurm-template"}). + Obj(), + }, + wantErr: apierrors.NewNotFound(schema.GroupResource{Group: "kjobctl.x-k8s.io", Resource: "jobtemplates"}, "slurm-template"), + }, + "should build slurm job": { + beforeTest: beforeSlurmTest, + namespace: metav1.NamespaceDefault, + profile: "profile", + mode: v1alpha1.SlurmMode, + array: "1-5%2", + initImage: "bash:latest", + kjobctlObjs: []runtime.Object{ + wrappers.MakeJobTemplate("slurm-job-template", metav1.NamespaceDefault). + WithContainer(*wrappers.MakeContainer("c1", "bash:4.4").Obj()). + Obj(), + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(*wrappers.MakeSupportedMode(v1alpha1.SlurmMode, "slurm-job-template").Obj()). + Obj(), + }, + wantRootObj: wrappers.MakeJob("", metav1.NamespaceDefault). + Parallelism(2). + Completions(5). + CompletionMode(batchv1.IndexedCompletion). + Profile("profile"). + Mode(v1alpha1.SlurmMode). + Subdomain("profile-slurm"). + WithInitContainer(*wrappers.MakeContainer("slurm-init-env", "bash:latest"). + Command("sh", "/slurm/scripts/init-entrypoint.sh"). + WithVolumeMount(corev1.VolumeMount{Name: "slurm-scripts", MountPath: "/slurm/scripts"}). + WithVolumeMount(corev1.VolumeMount{Name: "slurm-env", MountPath: "/slurm/env"}). + WithEnvVar(corev1.EnvVar{Name: "POD_IP", ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{FieldPath: "status.podIP"}, + }}). + Obj()). + WithContainer(*wrappers.MakeContainer("c1", "bash:4.4"). + Command("bash", "/slurm/scripts/entrypoint.sh"). + WithVolumeMount(corev1.VolumeMount{Name: "slurm-scripts", MountPath: "/slurm/scripts"}). + WithVolumeMount(corev1.VolumeMount{Name: "slurm-env", MountPath: "/slurm/env"}). + WithEnvVar(corev1.EnvVar{Name: constants.EnvVarNameUserID, Value: userID}). + WithEnvVar(corev1.EnvVar{Name: constants.EnvVarTaskName, Value: "default_profile"}). + WithEnvVar(corev1.EnvVar{ + Name: constants.EnvVarTaskID, + Value: fmt.Sprintf("%s_%s_default_profile", userID, testStartTime.Format(time.RFC3339)), + }). + WithEnvVar(corev1.EnvVar{Name: "PROFILE", Value: "default_profile"}). + WithEnvVar(corev1.EnvVar{Name: "TIMESTAMP", Value: testStartTime.Format(time.RFC3339)}). + WithEnvVar(corev1.EnvVar{Name: "JOB_CONTAINER_INDEX", Value: "0"}). + Obj()). + WithVolume(corev1.Volume{ + Name: "slurm-scripts", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{Name: "profile-slurm"}, + Items: []corev1.KeyToPath{ + {Key: "init-entrypoint.sh", Path: "init-entrypoint.sh"}, + {Key: "entrypoint.sh", Path: "entrypoint.sh"}, + {Key: "script", Path: "script", Mode: ptr.To[int32](0755)}, + }, + }, + }, + }). + WithVolume(corev1.Volume{ + Name: "slurm-env", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }). + Obj(), + wantChildObjs: []runtime.Object{ + wrappers.MakeConfigMap("", metav1.NamespaceDefault). + Profile("profile"). + Mode(v1alpha1.SlurmMode). + Data(map[string]string{ + "script": "#!/bin/bash\nsleep 300'", + "init-entrypoint.sh": `#!/bin/sh + +set -o errexit +set -o nounset +set -o pipefail +set -x + +# External variables +# JOB_COMPLETION_INDEX - completion index of the job. +# POD_IP - current pod IP + +array_indexes="1;2;3;4;5" +container_indexes=$(echo "$array_indexes" | awk -F';' -v idx="$JOB_COMPLETION_INDEX" '{print $((idx + 1))}') + +for i in $(seq 0 1) +do + container_index=$(echo "$container_indexes" | awk -F',' -v idx="$i" '{print $((idx + 1))}') + + if [ -z "$container_index" ]; then + break + fi + + mkdir -p /slurm/env/$i + + + cat << EOF > /slurm/env/$i/slurm.env +SLURM_ARRAY_JOB_ID=1 +SLURM_ARRAY_TASK_COUNT=5 +SLURM_ARRAY_TASK_MAX=5 +SLURM_ARRAY_TASK_MIN=1 +SLURM_TASKS_PER_NODE=1 +SLURM_CPUS_PER_TASK= +SLURM_CPUS_ON_NODE= +SLURM_JOB_CPUS_PER_NODE= +SLURM_CPUS_PER_GPU= +SLURM_MEM_PER_CPU= +SLURM_MEM_PER_GPU= +SLURM_MEM_PER_NODE= +SLURM_GPUS= +SLURM_NTASKS=1 +SLURM_NTASKS_PER_NODE=1 +SLURM_NPROCS=1 +SLURM_NNODES=2 +SLURM_SUBMIT_DIR=/slurm/scripts +SLURM_SUBMIT_HOST=$HOSTNAME +SLURM_JOB_NODELIST=profile-slurm-0.profile-slurm,profile-slurm-1.profile-slurm +SLURM_JOB_FIRST_NODE=profile-slurm-0.profile-slurm +SLURM_JOB_ID=$(expr $JOB_COMPLETION_INDEX \* 1 + $i + 1) +SLURM_JOBID=$(expr $JOB_COMPLETION_INDEX \* 1 + $i + 1) +SLURM_ARRAY_TASK_ID=$container_index +SLURM_JOB_FIRST_NODE_IP=${SLURM_JOB_FIRST_NODE_IP:-""} +EOF + +done +`, + "entrypoint.sh": `#!/usr/local/bin/bash + +set -o errexit +set -o nounset +set -o pipefail + +# External variables +# JOB_CONTAINER_INDEX - container index in the container template. + +if [ ! -d "/slurm/env/$JOB_CONTAINER_INDEX" ]; then + exit 0 +fi + +SBATCH_JOB_NAME= + +export $(cat /slurm/env/$JOB_CONTAINER_INDEX/slurm.env | xargs) + +/slurm/scripts/script +`, + }). + Obj(), + wrappers.MakeService("profile-slurm", metav1.NamespaceDefault). + Profile("profile"). + Mode(v1alpha1.SlurmMode). + ClusterIP("None"). + Selector("job-name", "profile-slurm"). + Obj(), + }, + cmpopts: []cmp.Option{ + cmpopts.AcyclicTransformer("RemoveGeneratedNameSuffixInString", func(val string) string { + return regexp.MustCompile("(profile-slurm)(-.{5})").ReplaceAllString(val, "$1") + }), + cmpopts.AcyclicTransformer("RemoveGeneratedNameSuffixInMap", func(m map[string]string) map[string]string { + for key, val := range m { + m[key] = regexp.MustCompile("(profile-slurm)(-.{5})").ReplaceAllString(val, "$1") + } + return m + }), + }, + }, + } + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + if tc.beforeTest != nil { + tc.beforeTest(t, &tc) + } + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + tcg := cmdtesting.NewTestClientGetter(). + WithKjobctlClientset(kjobctlfake.NewSimpleClientset(tc.kjobctlObjs...)) + + gotRootObj, gotChildObjs, gotErr := NewBuilder(tcg, testStartTime). + WithNamespace(tc.namespace). + WithProfileName(tc.profile). + WithModeName(tc.mode). + WithScript(tc.tempFile). + WithArray(tc.array). + WithCpusPerTask(tc.cpusPerTask). + WithGpusPerTask(tc.gpusPerTask). + WithMemPerTask(tc.memPerTask). + WithMemPerCPU(tc.memPerCPU). + WithMemPerGPU(tc.memPerGPU). + WithNodes(tc.nodes). + WithNTasks(tc.nTasks). + WithOutput(tc.output). + WithError(tc.err). + WithInput(tc.input). + WithJobName(tc.jobName). + WithPartition(tc.partition). + WithInitImage(tc.initImage). + WithFirstNodeIPTimeout(tc.firstNodeTimeout). + Do(ctx) + + var opts []cmp.Option + var statusError *apierrors.StatusError + if !errors.As(tc.wantErr, &statusError) { + opts = append(opts, cmpopts.EquateErrors()) + } + if diff := cmp.Diff(tc.wantErr, gotErr, opts...); diff != "" { + t.Errorf("Unexpected error (-want/+got)\n%s", diff) + return + } + + defaultCmpOpts := []cmp.Option{cmpopts.IgnoreFields(metav1.ObjectMeta{}, "Name")} + opts = append(defaultCmpOpts, tc.cmpopts...) + + if job, ok := tc.wantRootObj.(*batchv1.Job); ok { + if job.Annotations == nil { + job.Annotations = make(map[string]string, 1) + } + job.Annotations[constants.ScriptAnnotation] = tc.tempFile + } + + if diff := cmp.Diff(tc.wantRootObj, gotRootObj, opts...); diff != "" { + t.Errorf("Root object after build (-want,+got):\n%s", diff) + } + + if diff := cmp.Diff(tc.wantChildObjs, gotChildObjs, opts...); diff != "" { + t.Errorf("Child objects after build (-want,+got):\n%s", diff) + } + }) + } +} + +func TestSlurmBuilderBuildEntrypointCommand(t *testing.T) { + testStartTime := time.Now() + + testCases := map[string]struct { + input string + output string + error string + wantEntrypointCommand string + }{ + "should build entrypoint command": { + wantEntrypointCommand: "/slurm/scripts/script", + }, + "should build entrypoint command with output": { + output: "/home/test/stdout.out", + wantEntrypointCommand: "/slurm/scripts/script 1>/home/test/stdout.out", + }, + "should build entrypoint command with error": { + error: "/home/test/stderr.out", + wantEntrypointCommand: "/slurm/scripts/script 2>/home/test/stderr.out", + }, + "should build entrypoint command with output and error": { + output: "/home/test/stdout.out", + error: "/home/test/stderr.out", + wantEntrypointCommand: "/slurm/scripts/script 1>/home/test/stdout.out 2>/home/test/stderr.out", + }, + "should build entrypoint command with input": { + input: "/home/test/script.sh", + wantEntrypointCommand: "/slurm/scripts/script /home/test/stdout.out", + }, + "should build entrypoint command with input and error": { + input: "/home/test/script.sh", + error: "/home/test/stderr.out", + wantEntrypointCommand: "/slurm/scripts/script /home/test/stderr.out", + }, + "should build entrypoint command with input, output and error": { + input: "/home/test/script.sh", + output: "/home/test/stdout.out", + error: "/home/test/stderr.out", + wantEntrypointCommand: "/slurm/scripts/script /home/test/stdout.out 2>/home/test/stderr.out", + }, + } + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + tcg := cmdtesting.NewTestClientGetter() + + newBuilder := NewBuilder(tcg, testStartTime). + WithInput(tc.input). + WithOutput(tc.output). + WithError(tc.error) + gotEntrypointCommand := newSlurmBuilder(newBuilder).buildEntrypointCommand() + if diff := cmp.Diff(tc.wantEntrypointCommand, gotEntrypointCommand); diff != "" { + t.Errorf("Unexpected entrypoint command (-want/+got)\n%s", diff) + return + } + }) + } +} diff --git a/pkg/builder/templates/slurm_entrypoint_script.sh.tmpl b/pkg/builder/templates/slurm_entrypoint_script.sh.tmpl new file mode 100644 index 0000000..b4a47f7 --- /dev/null +++ b/pkg/builder/templates/slurm_entrypoint_script.sh.tmpl @@ -0,0 +1,20 @@ +#!/usr/local/bin/bash + +set -o errexit +set -o nounset +set -o pipefail + +# External variables +# JOB_CONTAINER_INDEX - container index in the container template. + +if [ ! -d "{{.EnvsPath}}/$JOB_CONTAINER_INDEX" ]; then + exit 0 +fi + +SBATCH_JOB_NAME={{.SbatchJobName}} + +export $(cat {{.EnvsPath}}/$JOB_CONTAINER_INDEX/{{.SlurmEnvFilename}} | xargs) + +{{- if .ChangeDir }}cd {{.ChangeDir}}{{end}} + +{{.BuildEntrypointCommand}} diff --git a/pkg/builder/templates/slurm_init_entrypoint_script.sh.tmpl b/pkg/builder/templates/slurm_init_entrypoint_script.sh.tmpl new file mode 100644 index 0000000..12c2be4 --- /dev/null +++ b/pkg/builder/templates/slurm_init_entrypoint_script.sh.tmpl @@ -0,0 +1,77 @@ +#!/bin/sh + +set -o errexit +set -o nounset +set -o pipefail +set -x + +# External variables +# JOB_COMPLETION_INDEX - completion index of the job. +# POD_IP - current pod IP + +array_indexes="{{.ArrayIndexes}}" +container_indexes=$(echo "$array_indexes" | awk -F';' -v idx="$JOB_COMPLETION_INDEX" '{print $((idx + 1))}') + +for i in $(seq 0 {{.SlurmNTasksPerNode}}) +do + container_index=$(echo "$container_indexes" | awk -F',' -v idx="$i" '{print $((idx + 1))}') + + if [ -z "$container_index" ]; then + break + fi + + mkdir -p {{.EnvsPath}}/$i + +{{if .FirstNodeIP}} + if [[ "$JOB_COMPLETION_INDEX" -eq 0 ]]; then + SLURM_JOB_FIRST_NODE_IP=${POD_IP} + else + timeout={{.FirstNodeIPTimeoutSeconds}} + start_time=$(date +%s) + while true; do + ip=$(nslookup "{{.SlurmJobFirstNode}}" | grep "Address 1" | awk 'NR==2 {print $3}') || true + if [[ -n "$ip" ]]; then + SLURM_JOB_FIRST_NODE_IP=$ip + break + else + current_time=$(date +%s) + elapsed_time=$((current_time - start_time)) + if [ "$elapsed_time" -ge "$timeout" ]; then + echo "Timeout reached, IP address for the first node ({{.SlurmJobFirstNode}}) not found." + break + fi + echo "IP Address for the first node ({{.SlurmJobFirstNode}}) not found, retrying..." + sleep 1 + fi + done + fi +{{end}} + cat << EOF > {{.EnvsPath}}/$i/{{.SlurmEnvFilename}} +SLURM_ARRAY_JOB_ID={{.SlurmArrayJobID}} +SLURM_ARRAY_TASK_COUNT={{.SlurmArrayTaskCount}} +SLURM_ARRAY_TASK_MAX={{.SlurmArrayTaskMax}} +SLURM_ARRAY_TASK_MIN={{.SlurmArrayTaskMin}} +SLURM_TASKS_PER_NODE={{.SlurmTasksPerNode}} +SLURM_CPUS_PER_TASK={{.SlurmCPUsPerTask}} +SLURM_CPUS_ON_NODE={{.SlurmCPUsOnNode}} +SLURM_JOB_CPUS_PER_NODE={{.SlurmJobCPUsPerNode}} +SLURM_CPUS_PER_GPU={{.SlurmCPUsPerGPU}} +SLURM_MEM_PER_CPU={{.SlurmMemPerCPU}} +SLURM_MEM_PER_GPU={{.SlurmMemPerGPU}} +SLURM_MEM_PER_NODE={{.SlurmMemPerNode}} +SLURM_GPUS={{.SlurmGPUs}} +SLURM_NTASKS={{.SlurmNTasks}} +SLURM_NTASKS_PER_NODE={{.SlurmNTasksPerNode}} +SLURM_NPROCS={{.SlurmNProcs}} +SLURM_NNODES={{.SlurmNNodes}} +SLURM_SUBMIT_DIR={{.SlurmSubmitDir}} +SLURM_SUBMIT_HOST=$HOSTNAME +SLURM_JOB_NODELIST={{.SlurmJobNodeList}} +SLURM_JOB_FIRST_NODE={{.SlurmJobFirstNode}} +SLURM_JOB_ID=$(expr $JOB_COMPLETION_INDEX \* {{.SlurmNTasksPerNode}} + $i + 1) +SLURM_JOBID=$(expr $JOB_COMPLETION_INDEX \* {{.SlurmNTasksPerNode}} + $i + 1) +SLURM_ARRAY_TASK_ID=$container_index +SLURM_JOB_FIRST_NODE_IP=${SLURM_JOB_FIRST_NODE_IP:-""} +EOF + +done diff --git a/pkg/cmd/cmd.go b/pkg/cmd/cmd.go new file mode 100644 index 0000000..c16e429 --- /dev/null +++ b/pkg/cmd/cmd.go @@ -0,0 +1,87 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cmd + +import ( + "os" + + "github.com/spf13/cobra" + "k8s.io/cli-runtime/pkg/genericclioptions" + "k8s.io/cli-runtime/pkg/genericiooptions" + "k8s.io/utils/clock" + + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/cmd/completion" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/cmd/create" + deletecmd "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/cmd/delete" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/cmd/describe" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/cmd/list" + crds "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/cmd/printcrds" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/cmd/util" +) + +type KjobctlOptions struct { + Clock clock.Clock + ConfigFlags *genericclioptions.ConfigFlags + + genericiooptions.IOStreams +} + +func defaultConfigFlags() *genericclioptions.ConfigFlags { + return genericclioptions.NewConfigFlags(true).WithDiscoveryQPS(50.0) +} + +func NewDefaultKjobctlCmd() *cobra.Command { + ioStreams := genericiooptions.IOStreams{In: os.Stdin, Out: os.Stdout, ErrOut: os.Stderr} + return NewKjobctlCmd(KjobctlOptions{ + ConfigFlags: defaultConfigFlags().WithWarningPrinter(ioStreams), + IOStreams: ioStreams, + }) +} + +func NewKjobctlCmd(o KjobctlOptions) *cobra.Command { + cmd := &cobra.Command{ + Use: "kjobctl", + Short: "ML/AI/Batch Jobs Made Easy", + } + + if o.Clock == nil { + o.Clock = clock.RealClock{} + } + + flags := cmd.PersistentFlags() + + configFlags := o.ConfigFlags + if configFlags == nil { + configFlags = defaultConfigFlags().WithWarningPrinter(o.IOStreams) + } + configFlags.AddFlags(flags) + + clientGetter := util.NewClientGetter(configFlags) + + cobra.CheckErr(cmd.RegisterFlagCompletionFunc("namespace", completion.NamespaceNameFunc(clientGetter))) + cobra.CheckErr(cmd.RegisterFlagCompletionFunc("context", completion.ContextsFunc(clientGetter))) + cobra.CheckErr(cmd.RegisterFlagCompletionFunc("cluster", completion.ClustersFunc(clientGetter))) + cobra.CheckErr(cmd.RegisterFlagCompletionFunc("user", completion.UsersFunc(clientGetter))) + + cmd.AddCommand(create.NewCreateCmd(clientGetter, o.IOStreams, o.Clock)) + cmd.AddCommand(describe.NewDescribeCmd(clientGetter, o.IOStreams)) + cmd.AddCommand(list.NewListCmd(clientGetter, o.IOStreams, o.Clock)) + cmd.AddCommand(deletecmd.NewDeleteCmd(clientGetter, o.IOStreams)) + cmd.AddCommand(crds.NewCmd()) + + return cmd +} diff --git a/pkg/cmd/completion/completion.go b/pkg/cmd/completion/completion.go new file mode 100644 index 0000000..91feaa3 --- /dev/null +++ b/pkg/cmd/completion/completion.go @@ -0,0 +1,276 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package completion + +import ( + "fmt" + "slices" + "strings" + + "github.com/spf13/cobra" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/apis/v1alpha1" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/cmd/util" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/constants" +) + +const completionLimit = 100 + +func NamespaceNameFunc(clientGetter util.ClientGetter) func(*cobra.Command, []string, string) ([]string, cobra.ShellCompDirective) { + return func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { + clientSet, err := clientGetter.K8sClientset() + if err != nil { + return []string{}, cobra.ShellCompDirectiveError + } + + list, err := clientSet.CoreV1().Namespaces().List(cmd.Context(), metav1.ListOptions{Limit: completionLimit}) + if err != nil { + return []string{}, cobra.ShellCompDirectiveError + } + + var validArgs []string + for _, ns := range list.Items { + validArgs = append(validArgs, ns.Name) + } + + return validArgs, cobra.ShellCompDirectiveNoFileComp + } +} + +func ContextsFunc(clientGetter util.ClientGetter) func(*cobra.Command, []string, string) ([]string, cobra.ShellCompDirective) { + return func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { + config, err := clientGetter.ToRawKubeConfigLoader().RawConfig() + if err != nil { + return nil, cobra.ShellCompDirectiveNoFileComp + } + var validArgs []string + for name := range config.Contexts { + if strings.HasPrefix(name, toComplete) { + validArgs = append(validArgs, name) + } + } + return validArgs, cobra.ShellCompDirectiveNoFileComp + } +} + +func ClustersFunc(clientGetter util.ClientGetter) func(*cobra.Command, []string, string) ([]string, cobra.ShellCompDirective) { + return func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { + config, err := clientGetter.ToRawKubeConfigLoader().RawConfig() + if err != nil { + return nil, cobra.ShellCompDirectiveNoFileComp + } + var validArgs []string + for name := range config.Clusters { + if strings.HasPrefix(name, toComplete) { + validArgs = append(validArgs, name) + } + } + return validArgs, cobra.ShellCompDirectiveNoFileComp + } +} + +func UsersFunc(clientGetter util.ClientGetter) func(*cobra.Command, []string, string) ([]string, cobra.ShellCompDirective) { + return func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { + config, err := clientGetter.ToRawKubeConfigLoader().RawConfig() + if err != nil { + return nil, cobra.ShellCompDirectiveNoFileComp + } + var validArgs []string + for name := range config.AuthInfos { + if strings.HasPrefix(name, toComplete) { + validArgs = append(validArgs, name) + } + } + return validArgs, cobra.ShellCompDirectiveNoFileComp + } +} + +func ApplicationProfileNameFunc(clientGetter util.ClientGetter) func(*cobra.Command, []string, string) ([]string, cobra.ShellCompDirective) { + return func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { + clientSet, err := clientGetter.KjobctlClientset() + if err != nil { + return []string{}, cobra.ShellCompDirectiveError + } + + namespace, _, err := clientGetter.ToRawKubeConfigLoader().Namespace() + if err != nil { + return []string{}, cobra.ShellCompDirectiveError + } + + list, err := clientSet.KjobctlV1alpha1().ApplicationProfiles(namespace).List(cmd.Context(), metav1.ListOptions{Limit: completionLimit}) + if err != nil { + return []string{}, cobra.ShellCompDirectiveError + } + + var validArgs []string + for _, ap := range list.Items { + if !slices.Contains(args, ap.Name) { + validArgs = append(validArgs, ap.Name) + } + } + + return validArgs, cobra.ShellCompDirectiveNoFileComp + } +} + +func LocalQueueNameFunc(clientGetter util.ClientGetter) func(*cobra.Command, []string, string) ([]string, cobra.ShellCompDirective) { + return func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { + clientSet, err := clientGetter.KueueClientset() + if err != nil { + return []string{}, cobra.ShellCompDirectiveError + } + + namespace, _, err := clientGetter.ToRawKubeConfigLoader().Namespace() + if err != nil { + return []string{}, cobra.ShellCompDirectiveError + } + + list, err := clientSet.KueueV1beta1().LocalQueues(namespace).List(cmd.Context(), metav1.ListOptions{Limit: completionLimit}) + if err != nil { + return []string{}, cobra.ShellCompDirectiveError + } + + var validArgs []string + for _, lq := range list.Items { + if !slices.Contains(args, lq.Name) { + validArgs = append(validArgs, lq.Name) + } + } + + return validArgs, cobra.ShellCompDirectiveNoFileComp + } +} + +func JobNameFunc(clientGetter util.ClientGetter, mode v1alpha1.ApplicationProfileMode) func(*cobra.Command, []string, string) ([]string, cobra.ShellCompDirective) { + return func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { + clientset, err := clientGetter.K8sClientset() + if err != nil { + return []string{}, cobra.ShellCompDirectiveError + } + + namespace, _, err := clientGetter.ToRawKubeConfigLoader().Namespace() + if err != nil { + return []string{}, cobra.ShellCompDirectiveError + } + + opts := metav1.ListOptions{ + LabelSelector: fmt.Sprintf("%s,%s=%s", constants.ProfileLabel, constants.ModeLabel, mode), + Limit: completionLimit, + } + list, err := clientset.BatchV1().Jobs(namespace).List(cmd.Context(), opts) + if err != nil { + return []string{}, cobra.ShellCompDirectiveError + } + + var validArgs []string + for _, job := range list.Items { + if !slices.Contains(args, job.Name) { + validArgs = append(validArgs, job.Name) + } + } + + return validArgs, cobra.ShellCompDirectiveNoFileComp + } +} + +func RayJobNameFunc(clientGetter util.ClientGetter) func(*cobra.Command, []string, string) ([]string, cobra.ShellCompDirective) { + return func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { + clientset, err := clientGetter.RayClientset() + if err != nil { + return []string{}, cobra.ShellCompDirectiveError + } + + namespace, _, err := clientGetter.ToRawKubeConfigLoader().Namespace() + if err != nil { + return []string{}, cobra.ShellCompDirectiveError + } + + opts := metav1.ListOptions{LabelSelector: constants.ProfileLabel, Limit: completionLimit} + list, err := clientset.RayV1().RayJobs(namespace).List(cmd.Context(), opts) + if err != nil { + return []string{}, cobra.ShellCompDirectiveError + } + + var validArgs []string + for _, rayJob := range list.Items { + if !slices.Contains(args, rayJob.Name) { + validArgs = append(validArgs, rayJob.Name) + } + } + + return validArgs, cobra.ShellCompDirectiveNoFileComp + } +} + +func RayClusterNameFunc(clientGetter util.ClientGetter) func(*cobra.Command, []string, string) ([]string, cobra.ShellCompDirective) { + return func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { + clientset, err := clientGetter.RayClientset() + if err != nil { + return []string{}, cobra.ShellCompDirectiveError + } + + namespace, _, err := clientGetter.ToRawKubeConfigLoader().Namespace() + if err != nil { + return []string{}, cobra.ShellCompDirectiveError + } + + opts := metav1.ListOptions{LabelSelector: constants.ProfileLabel, Limit: completionLimit} + list, err := clientset.RayV1().RayClusters(namespace).List(cmd.Context(), opts) + if err != nil { + return []string{}, cobra.ShellCompDirectiveError + } + + var validArgs []string + for _, rayCluster := range list.Items { + if !slices.Contains(args, rayCluster.Name) { + validArgs = append(validArgs, rayCluster.Name) + } + } + + return validArgs, cobra.ShellCompDirectiveNoFileComp + } +} + +func PodNameFunc(clientGetter util.ClientGetter) func(*cobra.Command, []string, string) ([]string, cobra.ShellCompDirective) { + return func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { + clientset, err := clientGetter.K8sClientset() + if err != nil { + return []string{}, cobra.ShellCompDirectiveError + } + + namespace, _, err := clientGetter.ToRawKubeConfigLoader().Namespace() + if err != nil { + return []string{}, cobra.ShellCompDirectiveError + } + + opts := metav1.ListOptions{LabelSelector: constants.ProfileLabel, Limit: completionLimit} + list, err := clientset.CoreV1().Pods(namespace).List(cmd.Context(), opts) + if err != nil { + return []string{}, cobra.ShellCompDirectiveError + } + + var validArgs []string + for _, pod := range list.Items { + if !slices.Contains(args, pod.Name) { + validArgs = append(validArgs, pod.Name) + } + } + + return validArgs, cobra.ShellCompDirectiveNoFileComp + } +} diff --git a/pkg/cmd/completion/completion_test.go b/pkg/cmd/completion/completion_test.go new file mode 100644 index 0000000..5251923 --- /dev/null +++ b/pkg/cmd/completion/completion_test.go @@ -0,0 +1,216 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package completion + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + rayfake "github.com/ray-project/kuberay/ray-operator/pkg/client/clientset/versioned/fake" + "github.com/spf13/cobra" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + k8sfake "k8s.io/client-go/kubernetes/fake" + "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueuefake "sigs.k8s.io/kueue/client-go/clientset/versioned/fake" + + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/apis/v1alpha1" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/client-go/clientset/versioned/fake" + cmdtesting "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/cmd/testing" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/testing/wrappers" +) + +func TestNamespaceNameFunc(t *testing.T) { + objs := []runtime.Object{ + &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "ns1"}}, + &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "ns2"}}, + } + + wantNames := []string{"ns1", "ns2"} + wantDirective := cobra.ShellCompDirectiveNoFileComp + + tcg := cmdtesting.NewTestClientGetter() + tcg.WithK8sClientset(k8sfake.NewSimpleClientset(objs...)) + + complFn := NamespaceNameFunc(tcg) + names, directive := complFn(&cobra.Command{}, []string{}, "") + if diff := cmp.Diff(wantNames, names); diff != "" { + t.Errorf("Unexpected names (-want/+got)\n%s", diff) + } + + if diff := cmp.Diff(wantDirective, directive); diff != "" { + t.Errorf("Unexpected directive (-want/+got)\n%s", diff) + } +} + +func TestApplicationProfileNameFunc(t *testing.T) { + args := []string{"ap1"} + objs := []runtime.Object{ + wrappers.MakeApplicationProfile("ap1", metav1.NamespaceDefault).Obj(), + wrappers.MakeApplicationProfile("ap2", metav1.NamespaceDefault).Obj(), + wrappers.MakeApplicationProfile("ap3", "test").Obj(), + } + + wantNames := []string{"ap2"} + wantDirective := cobra.ShellCompDirectiveNoFileComp + + tcg := cmdtesting.NewTestClientGetter() + tcg.WithKjobctlClientset(fake.NewSimpleClientset(objs...)) + + complFn := ApplicationProfileNameFunc(tcg) + names, directive := complFn(&cobra.Command{}, args, "") + if diff := cmp.Diff(wantNames, names); diff != "" { + t.Errorf("Unexpected names (-want/+got)\n%s", diff) + } + + if diff := cmp.Diff(wantDirective, directive); diff != "" { + t.Errorf("Unexpected directive (-want/+got)\n%s", diff) + } +} + +func TestLocalQueueNameFunc(t *testing.T) { + args := []string{"lq1"} + objs := []runtime.Object{ + &v1beta1.LocalQueue{ObjectMeta: metav1.ObjectMeta{Name: "lq1", Namespace: metav1.NamespaceDefault}}, + &v1beta1.LocalQueue{ObjectMeta: metav1.ObjectMeta{Name: "lq2", Namespace: metav1.NamespaceDefault}}, + &v1beta1.LocalQueue{ObjectMeta: metav1.ObjectMeta{Name: "lq3", Namespace: "test"}}, + } + + wantNames := []string{"lq2"} + wantDirective := cobra.ShellCompDirectiveNoFileComp + + tcg := cmdtesting.NewTestClientGetter() + tcg.WithKueueClientset(kueuefake.NewSimpleClientset(objs...)) + + complFn := LocalQueueNameFunc(tcg) + names, directive := complFn(&cobra.Command{}, args, "") + if diff := cmp.Diff(wantNames, names); diff != "" { + t.Errorf("Unexpected names (-want/+got)\n%s", diff) + } + + if diff := cmp.Diff(wantDirective, directive); diff != "" { + t.Errorf("Unexpected directive (-want/+got)\n%s", diff) + } +} + +func TestJobNameCompletionFunc(t *testing.T) { + args := []string{"job1"} + objs := []runtime.Object{ + wrappers.MakeJob("job1", metav1.NamespaceDefault).Profile("p1").Mode("jJb").Obj(), + wrappers.MakeJob("job2", metav1.NamespaceDefault).Profile("p1").Mode("Job").Obj(), + wrappers.MakeJob("job3", metav1.NamespaceDefault).Profile("p1").Mode("Slurm").Obj(), + wrappers.MakeJob("job4", "test").Profile("p1").Mode("Job").Obj(), + wrappers.MakeJob("job5", metav1.NamespaceDefault).Profile("p1").Obj(), + wrappers.MakeJob("job6", metav1.NamespaceDefault).Mode("Job").Obj(), + } + mode := v1alpha1.JobMode + + wantNames := []string{"job2"} + wantDirective := cobra.ShellCompDirectiveNoFileComp + + tcg := cmdtesting.NewTestClientGetter() + tcg.WithK8sClientset(k8sfake.NewSimpleClientset(objs...)) + + complFn := JobNameFunc(tcg, mode) + names, directive := complFn(&cobra.Command{}, args, "") + if diff := cmp.Diff(wantNames, names); diff != "" { + t.Errorf("Unexpected names (-want/+got)\n%s", diff) + } + + if diff := cmp.Diff(wantDirective, directive); diff != "" { + t.Errorf("Unexpected directive (-want/+got)\n%s", diff) + } +} + +func TestRayJobNameCompletionFunc(t *testing.T) { + args := []string{"ray-job1"} + objs := []runtime.Object{ + wrappers.MakeRayJob("ray-job1", metav1.NamespaceDefault).Profile("p1").Obj(), + wrappers.MakeRayJob("ray-job2", metav1.NamespaceDefault).Profile("p1").Obj(), + wrappers.MakeRayJob("ray-job3", "test").Profile("p1").Obj(), + wrappers.MakeRayJob("ray-job4", metav1.NamespaceDefault).Obj(), + } + + wantNames := []string{"ray-job2"} + wantDirective := cobra.ShellCompDirectiveNoFileComp + + tcg := cmdtesting.NewTestClientGetter() + tcg.WithRayClientset(rayfake.NewSimpleClientset(objs...)) + + complFn := RayJobNameFunc(tcg) + names, directive := complFn(&cobra.Command{}, args, "") + if diff := cmp.Diff(wantNames, names); diff != "" { + t.Errorf("Unexpected names (-want/+got)\n%s", diff) + } + + if diff := cmp.Diff(wantDirective, directive); diff != "" { + t.Errorf("Unexpected directive (-want/+got)\n%s", diff) + } +} + +func TestRayClusterNameCompletionFunc(t *testing.T) { + args := []string{"ray-cluster1"} + objs := []runtime.Object{ + wrappers.MakeRayCluster("ray-cluster1", metav1.NamespaceDefault).Profile("p1").Obj(), + wrappers.MakeRayCluster("ray-cluster2", metav1.NamespaceDefault).Profile("p1").Obj(), + wrappers.MakeRayCluster("ray-cluster3", "test").Profile("p1").Obj(), + wrappers.MakeRayCluster("ray-cluster4", metav1.NamespaceDefault).Obj(), + } + + wantNames := []string{"ray-cluster2"} + wantDirective := cobra.ShellCompDirectiveNoFileComp + + tcg := cmdtesting.NewTestClientGetter() + tcg.WithRayClientset(rayfake.NewSimpleClientset(objs...)) + + complFn := RayClusterNameFunc(tcg) + names, directive := complFn(&cobra.Command{}, args, "") + if diff := cmp.Diff(wantNames, names); diff != "" { + t.Errorf("Unexpected names (-want/+got)\n%s", diff) + } + + if diff := cmp.Diff(wantDirective, directive); diff != "" { + t.Errorf("Unexpected directive (-want/+got)\n%s", diff) + } +} + +func TestPodNameCompletionFunc(t *testing.T) { + args := []string{"pod1"} + objs := []runtime.Object{ + wrappers.MakePod("pod1", metav1.NamespaceDefault).Profile("p1").Obj(), + wrappers.MakePod("pod2", metav1.NamespaceDefault).Profile("p1").Obj(), + wrappers.MakePod("pod3", "test").Profile("p1").Obj(), + wrappers.MakePod("pod4", metav1.NamespaceDefault).Obj(), + } + + wantNames := []string{"pod2"} + wantDirective := cobra.ShellCompDirectiveNoFileComp + + tcg := cmdtesting.NewTestClientGetter() + tcg.WithK8sClientset(k8sfake.NewSimpleClientset(objs...)) + + complFn := PodNameFunc(tcg) + names, directive := complFn(&cobra.Command{}, args, "") + if diff := cmp.Diff(wantNames, names); diff != "" { + t.Errorf("Unexpected names (-want/+got)\n%s", diff) + } + + if diff := cmp.Diff(wantDirective, directive); diff != "" { + t.Errorf("Unexpected directive (-want/+got)\n%s", diff) + } +} diff --git a/pkg/cmd/create/create.go b/pkg/cmd/create/create.go new file mode 100644 index 0000000..7d55670 --- /dev/null +++ b/pkg/cmd/create/create.go @@ -0,0 +1,861 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package create + +import ( + "context" + "errors" + "fmt" + "strings" + "time" + + rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1" + "github.com/spf13/cobra" + "github.com/spf13/pflag" + corev1 "k8s.io/api/core/v1" + apiresource "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/cli-runtime/pkg/genericclioptions" + "k8s.io/cli-runtime/pkg/genericiooptions" + "k8s.io/cli-runtime/pkg/printers" + k8sscheme "k8s.io/client-go/kubernetes/scheme" + restclient "k8s.io/client-go/rest" + "k8s.io/client-go/tools/remotecommand" + "k8s.io/kubectl/pkg/cmd/attach" + "k8s.io/kubectl/pkg/cmd/exec" + "k8s.io/kubectl/pkg/util/templates" + "k8s.io/utils/clock" + "k8s.io/utils/ptr" + + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/apis/v1alpha1" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/builder" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/cmd/completion" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/cmd/util" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/parser" +) + +const ( + profileFlagName = "profile" + podRunningTimeoutFlagName = "pod-running-timeout" + removeFlagName = "rm" + ignoreUnknownFlagName = "ignore-unknown-flags" + initImageFlagName = "init-image" + skipLocalQueueValidationFlagName = "skip-localqueue-validation" + skipPriorityValidationFlagName = "skip-priority-validation" + changeDirFlagName = "chdir" + firstNodeIPFlagName = "first-node-ip" + firstNodeIPTimeoutFlagName = "first-node-ip-timeout" + + commandFlagName = string(v1alpha1.CmdFlag) + parallelismFlagName = string(v1alpha1.ParallelismFlag) + completionsFlagName = string(v1alpha1.CompletionsFlag) + replicasFlagName = string(v1alpha1.ReplicasFlag) + minReplicasFlagName = string(v1alpha1.MinReplicasFlag) + maxReplicasFlagName = string(v1alpha1.MaxReplicasFlag) + requestFlagName = string(v1alpha1.RequestFlag) + localQueueFlagName = string(v1alpha1.LocalQueueFlag) + rayClusterFlagName = string(v1alpha1.RayClusterFlag) + arrayFlagName = string(v1alpha1.ArrayFlag) + cpusPerTaskFlagName = string(v1alpha1.CpusPerTaskFlag) + gpusPerTaskFlagName = string(v1alpha1.GpusPerTaskFlag) + memPerNodeFlagName = string(v1alpha1.MemPerNodeFlag) + memPerTaskFlagName = string(v1alpha1.MemPerTaskFlag) + memPerCPUFlagName = string(v1alpha1.MemPerCPUFlag) + memPerGPUFlagName = string(v1alpha1.MemPerGPUFlag) + nodesFlagName = string(v1alpha1.NodesFlag) + nTasksFlagName = string(v1alpha1.NTasksFlag) + outputFlagName = string(v1alpha1.OutputFlag) + errorFlagName = string(v1alpha1.ErrorFlag) + inputFlagName = string(v1alpha1.InputFlag) + jobNameFlagName = string(v1alpha1.JobNameFlag) + partitionFlagName = string(v1alpha1.PartitionFlag) + priorityFlagName = string(v1alpha1.PriorityFlag) + timeFlagName = string(v1alpha1.TimeFlag) +) + +func withTimeFlag(f *pflag.FlagSet, p *string) { + f.StringVarP(p, timeFlagName, "t", "", + `Set a limit on the total run time of the job. +A time limit of zero requests that no time limit be imposed. +Acceptable time formats include "minutes", "minutes:seconds", +"hours:minutes:seconds", "days-hours", "days-hours:minutes" +and "days-hours:minutes:seconds".`) +} + +var ( + createJobExample = templates.Examples(` + # Create job + kjobctl create job \ + --profile my-application-profile \ + --cmd "sleep 5" \ + --parallelism 4 \ + --completions 4 \ + --request cpu=500m,memory=4Gi \ + --localqueue my-local-queue-name + `) + createInteractiveExample = templates.Examples(` + # Create interactive + kjobctl create interactive \ + --profile my-application-profile \ + --pod-running-timeout 30s \ + --rm + `) + createRayJobLong = templates.LongDesc(` + Create a rayjob. + + KubeRay operator is required for RayJob. + How to install KubeRay operator you can find here https://ray-project.github.io/kuberay/deploy/installation/. + `) + createRayJobExample = templates.Examples(` + # Create rayjob + kjobctl create rayjob \ + --profile my-application-profile \ + --cmd "python /home/ray/samples/sample_code.py" \ + --replicas small-group=1 \ + --min-replicas small-group=1 \ + --max-replicas small-group=5 \ + --localqueue my-local-queue-name + `) + createRayClusterLong = templates.LongDesc(` + Create a raycluster. + + KubeRay operator is required for RayCluster. + How to install KubeRay operator you can find here https://ray-project.github.io/kuberay/deploy/installation/. + `) + createRayClusterExample = templates.Examples(` + # Create raycluster + kjobctl create raycluster \ + --profile my-application-profile \ + --replicas small-group=1 \ + --min-replicas small-group=1 \ + --max-replicas small-group=5 \ + --localqueue my-local-queue-name + `) + createSlurmExample = templates.Examples(` + # Create slurm + kjobctl create slurm --profile my-application-profile -- \ + --array 0-5 --nodes 3 --ntasks 1 ./script.sh + `) +) + +var ( + podRunningTimeoutDefault = 1 * time.Minute +) + +type CreateOptions struct { + exec.StreamOptions + + PrintFlags *genericclioptions.PrintFlags + Config *restclient.Config + Attach attach.RemoteAttach + AttachFunc func(*CreateOptions, *corev1.Container, remotecommand.TerminalSizeQueue, *corev1.Pod) func() error + + DryRunStrategy util.DryRunStrategy + + Namespace string + ProfileName string + ModeName v1alpha1.ApplicationProfileMode + Script string + InitImage string + PodRunningTimeout time.Duration + FirstNodeIPTimeout time.Duration + FirstNodeIP bool + RemoveInteractivePod bool + ChangeDir string + + SlurmFlagSet *pflag.FlagSet + + Command []string + Parallelism *int32 + Completions *int32 + Replicas map[string]int + MinReplicas map[string]int + MaxReplicas map[string]int + Requests corev1.ResourceList + LocalQueue string + RayCluster string + Array string + CpusPerTask *apiresource.Quantity + GpusPerTask map[string]*apiresource.Quantity + MemPerNode *apiresource.Quantity + MemPerTask *apiresource.Quantity + MemPerCPU *apiresource.Quantity + MemPerGPU *apiresource.Quantity + Nodes *int32 + NTasks *int32 + Output string + Error string + Input string + JobName string + Partition string + Priority string + TimeLimit string + IgnoreUnknown bool + SkipLocalQueueValidation bool + SkipPriorityValidation bool + + UserSpecifiedCommand string + UserSpecifiedParallelism int32 + UserSpecifiedCompletions int32 + UserSpecifiedRequest map[string]string + UserSpecifiedCpusPerTask string + UserSpecifiedGpusPerTask string + UserSpecifiedMemPerNode string + UserSpecifiedMemPerTask string + UserSpecifiedMemPerCPU string + UserSpecifiedMemPerGPU string + UserSpecifiedNodes int32 + UserSpecifiedNTasks int32 + + PrintObj printers.ResourcePrinterFunc + + genericiooptions.IOStreams +} + +func NewCreateOptions(streams genericiooptions.IOStreams) *CreateOptions { + scheme := runtime.NewScheme() + utilruntime.Must(k8sscheme.AddToScheme(scheme)) + utilruntime.Must(rayv1.AddToScheme(scheme)) + + return &CreateOptions{ + PrintFlags: genericclioptions.NewPrintFlags("created").WithTypeSetter(scheme), + IOStreams: streams, + StreamOptions: exec.StreamOptions{ + IOStreams: streams, + }, + Attach: &attach.DefaultRemoteAttach{}, + AttachFunc: defaultAttachFunc, + } +} + +type modeSubcommand struct { + ModeName v1alpha1.ApplicationProfileMode + Setup func(clientGetter util.ClientGetter, subcmd *cobra.Command, o *CreateOptions) +} + +var createModeSubcommands = map[string]modeSubcommand{ + "job": { + ModeName: v1alpha1.JobMode, + Setup: func(clientGetter util.ClientGetter, subcmd *cobra.Command, o *CreateOptions) { + subcmd.Use += " [--cmd COMMAND]" + + " [--request RESOURCE_NAME=QUANTITY]" + + " [--parallelism PARALLELISM]" + + " [--completions COMPLETIONS]" + + " [--time TIME_LIMIT]" + subcmd.Short = "Create a job" + subcmd.Example = createJobExample + + subcmd.Flags().StringVar(&o.UserSpecifiedCommand, commandFlagName, "", + "Command which is associated with the resource.") + subcmd.Flags().StringToStringVar(&o.UserSpecifiedRequest, requestFlagName, nil, + "Request is a set of (resource name, quantity) pairs.") + subcmd.Flags().Int32Var(&o.UserSpecifiedParallelism, parallelismFlagName, 0, + "Parallelism specifies the maximum desired number of pods the job should run at any given time.") + subcmd.Flags().Int32Var(&o.UserSpecifiedCompletions, completionsFlagName, 0, + "Completions specifies the desired number of successfully finished pods.") + + withTimeFlag(subcmd.Flags(), &o.TimeLimit) + }, + }, + "interactive": { + ModeName: v1alpha1.InteractiveMode, + Setup: func(clientGetter util.ClientGetter, subcmd *cobra.Command, o *CreateOptions) { + subcmd.Use += " [--cmd COMMAND]" + + " [--request RESOURCE_NAME=QUANTITY]" + + " [--pod-running-timeout DURATION]" + + " [--time TIME_LIMIT]" + + " [--rm]" + subcmd.Short = "Create an interactive shell" + subcmd.Example = createInteractiveExample + + subcmd.Flags().StringVar(&o.UserSpecifiedCommand, commandFlagName, "", + "Command which is associated with the resource.") + subcmd.Flags().StringToStringVar(&o.UserSpecifiedRequest, requestFlagName, nil, + "Request is a set of (resource name, quantity) pairs.") + subcmd.Flags().DurationVar(&o.PodRunningTimeout, podRunningTimeoutFlagName, podRunningTimeoutDefault, + "The length of time (like 5s, 2m, or 3h, higher than zero) to wait until at least one pod is running.") + subcmd.Flags().BoolVar(&o.RemoveInteractivePod, removeFlagName, false, + "Remove pod when interactive session exits.") + + withTimeFlag(subcmd.Flags(), &o.TimeLimit) + }, + }, + "rayjob": { + ModeName: v1alpha1.RayJobMode, + Setup: func(clientGetter util.ClientGetter, subcmd *cobra.Command, o *CreateOptions) { + subcmd.Use += " [--cmd COMMAND]" + + " [--replicas [WORKER_GROUP]=REPLICAS]" + + " [--min-replicas [WORKER_GROUP]=MIN_REPLICAS]" + + " [--max-replicas [WORKER_GROUP]=MAX_REPLICAS]" + + " [--time TIME_LIMIT]" + subcmd.Short = "Create a rayjob" + subcmd.Long = createRayJobLong + subcmd.Example = createRayJobExample + + subcmd.Flags().StringVar(&o.UserSpecifiedCommand, commandFlagName, "", + "Command which is associated with the resource.") + subcmd.Flags().StringToIntVar(&o.Replicas, replicasFlagName, nil, + "Replicas is the number of desired Pods for this worker group.") + subcmd.Flags().StringToIntVar(&o.MinReplicas, minReplicasFlagName, nil, + "MinReplicas denotes the minimum number of desired Pods for this worker group.") + subcmd.Flags().StringToIntVar(&o.MaxReplicas, maxReplicasFlagName, nil, + "MaxReplicas denotes the maximum number of desired Pods for this worker group, and the default value is maxInt32.") + subcmd.Flags().StringVar(&o.RayCluster, rayClusterFlagName, "", + "Existing ray cluster on which the job will be created.") + + withTimeFlag(subcmd.Flags(), &o.TimeLimit) + + subcmd.MarkFlagsMutuallyExclusive(rayClusterFlagName, replicasFlagName) + subcmd.MarkFlagsMutuallyExclusive(rayClusterFlagName, minReplicasFlagName) + subcmd.MarkFlagsMutuallyExclusive(rayClusterFlagName, maxReplicasFlagName) + subcmd.MarkFlagsMutuallyExclusive(rayClusterFlagName, localQueueFlagName) + }, + }, + "raycluster": { + ModeName: v1alpha1.RayClusterMode, + Setup: func(clientGetter util.ClientGetter, subcmd *cobra.Command, o *CreateOptions) { + subcmd.Use += " [--replicas [WORKER_GROUP]=REPLICAS]" + + " [--min-replicas [WORKER_GROUP]=MIN_REPLICAS]" + + " [--max-replicas [WORKER_GROUP]=MAX_REPLICAS]" + + " [--time TIME_LIMIT]" + subcmd.Short = "Create a raycluster" + subcmd.Long = createRayClusterLong + subcmd.Example = createRayClusterExample + + subcmd.Flags().StringToIntVar(&o.Replicas, replicasFlagName, nil, + "Replicas is the number of desired Pods for this worker group.") + subcmd.Flags().StringToIntVar(&o.MinReplicas, minReplicasFlagName, nil, + "MinReplicas denotes the minimum number of desired Pods for this worker group.") + subcmd.Flags().StringToIntVar(&o.MaxReplicas, maxReplicasFlagName, nil, + "MaxReplicas denotes the maximum number of desired Pods for this worker group, and the default value is maxInt32.") + + withTimeFlag(subcmd.Flags(), &o.TimeLimit) + }, + }, + "slurm": { + ModeName: v1alpha1.SlurmMode, + Setup: func(clientGetter util.ClientGetter, subcmd *cobra.Command, o *CreateOptions) { + subcmd.Use += " [--ignore-unknown-flags]" + + " [--init-image IMAGE]" + + " [--first-node-ip]" + + " [--first-node-ip-timeout DURATION]" + + " -- " + + " [--array ARRAY]" + + " [--cpus-per-task QUANTITY]" + + " [--gpus-per-task QUANTITY]" + + " [--mem QUANTITY]" + + " [--mem-per-task QUANTITY]" + + " [--mem-per-cpu QUANTITY]" + + " [--mem-per-gpu QUANTITY]" + + " [--nodes COUNT]" + + " [--ntasks COUNT]" + + " [--output FILENAME_PATTERN]" + + " [--error FILENAME_PATTERN]" + + " [--input FILENAME_PATTERN]" + + " [--job-name NAME]" + + " [--partition NAME]" + + " SCRIPT" + + subcmd.Short = "Create a slurm job" + subcmd.Example = createSlurmExample + subcmd.Args = cobra.MinimumNArgs(1) + + subcmd.Flags().BoolVar(&o.IgnoreUnknown, ignoreUnknownFlagName, false, + "Ignore all the unsupported flags in the bash script.") + subcmd.Flags().StringVar(&o.InitImage, initImageFlagName, "registry.k8s.io/busybox:1.27.2", + "The image used for the init container.") + subcmd.Flags().BoolVar(&o.FirstNodeIP, firstNodeIPFlagName, false, + "Enable the retrieval of the first node's IP address.") + subcmd.Flags().DurationVar(&o.FirstNodeIPTimeout, firstNodeIPTimeoutFlagName, time.Minute, + "The timeout for the retrieval of the first node's IP address.") + + o.SlurmFlagSet = pflag.NewFlagSet("slurm", pflag.ExitOnError) + o.SlurmFlagSet.StringVarP(&o.Array, arrayFlagName, "a", "", + `Submit a job array, multiple jobs to be executed with identical parameters. +The indexes specification identifies what array index values should be used. +Multiple values may be specified using a comma separated list and/or a range of values with a "-" separator. For example, "--array=0-15" or "--array=0,6,16-32". +A maximum number of simultaneously running tasks from the job array may be specified using a "%" separator. For example "--array=0-15%4" will limit the number of simultaneously running tasks from this job array to 4. +The minimum index value is 0. The maximum index value is 2147483647.`) + o.SlurmFlagSet.StringVarP(&o.UserSpecifiedCpusPerTask, cpusPerTaskFlagName, "c", "", + "How much cpus a container inside a pod requires.") + o.SlurmFlagSet.StringVar(&o.UserSpecifiedGpusPerTask, gpusPerTaskFlagName, "", + "How much gpus a container inside a pod requires.") + o.SlurmFlagSet.StringVar(&o.UserSpecifiedMemPerNode, memPerNodeFlagName, "", + "How much memory a pod requires.") + o.SlurmFlagSet.StringVar(&o.UserSpecifiedMemPerTask, memPerTaskFlagName, "", + "How much memory a container requires.") + o.SlurmFlagSet.StringVar(&o.UserSpecifiedMemPerCPU, memPerCPUFlagName, "", + "How much memory a container requires, it multiplies the number of requested cpus per task by mem-per-cpu.") + o.SlurmFlagSet.StringVar(&o.UserSpecifiedMemPerGPU, memPerGPUFlagName, "", + "How much memory a container requires, it multiplies the number of requested gpus per task by mem-per-gpu.") + o.SlurmFlagSet.Int32VarP(&o.UserSpecifiedNodes, nodesFlagName, "N", 0, + "Number of pods to be used at a time.") + o.SlurmFlagSet.Int32VarP(&o.UserSpecifiedNTasks, nTasksFlagName, "n", 1, + "Number of identical containers inside of a pod, usually 1.") + o.SlurmFlagSet.StringVarP(&o.Output, outputFlagName, "o", "", + "Where to redirect the standard output stream of a task. If not passed it proceeds to stdout, and is available via kubectl logs.") + o.SlurmFlagSet.StringVarP(&o.Error, errorFlagName, "e", "", + "Where to redirect std error stream of a task. If not passed it proceeds to stdout, and is available via kubectl logs.") + o.SlurmFlagSet.StringVar(&o.Input, inputFlagName, "", + "What to pipe into the script.") + o.SlurmFlagSet.StringVarP(&o.JobName, jobNameFlagName, "J", "", + "What is the job name.") + o.SlurmFlagSet.StringVar(&o.Partition, partitionFlagName, "", + "Local queue name.") + o.SlurmFlagSet.StringVarP(&o.ChangeDir, changeDirFlagName, "D", "", + "Change directory before executing the script.") + + withTimeFlag(o.SlurmFlagSet, &o.TimeLimit) + }, + }, +} + +func NewCreateCmd(clientGetter util.ClientGetter, streams genericiooptions.IOStreams, clock clock.Clock) *cobra.Command { + o := NewCreateOptions(streams) + + cmd := &cobra.Command{ + Use: "create", + Short: "Create a task", + Example: fmt.Sprintf("%s\n\n%s\n\n%s\n\n%s\n\n%s", + createJobExample, + createInteractiveExample, + createRayJobExample, + createRayClusterExample, + createSlurmExample, + ), + } + + for modeName, modeSubcommand := range createModeSubcommands { + subcmd := &cobra.Command{ + Use: modeName + + " --profile APPLICATION_PROFILE_NAME" + + " [--localqueue LOCAL_QUEUE_NAME]" + + " [--skip-localqueue-validation]" + + " [--priority NAME]" + + " [--skip-priority-validation]", + DisableFlagsInUseLine: true, + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + cmd.SilenceUsage = true + + err := o.Complete(clientGetter, cmd, args) + if err != nil { + return err + } + + return o.Run(cmd.Context(), clientGetter, clock.Now()) + }, + } + + o.PrintFlags.AddFlags(subcmd) + + subcmd.Flags().StringVarP(&o.ProfileName, profileFlagName, "p", "", + "Application profile contains a template (with defaults set) for running a specific type of application.") + subcmd.Flags().StringVar(&o.LocalQueue, localQueueFlagName, "", + "Kueue localqueue name which is associated with the resource.") + subcmd.Flags().BoolVar(&o.SkipLocalQueueValidation, skipLocalQueueValidationFlagName, false, + "Skip local queue validation. Add local queue even if the queue does not exist.") + subcmd.Flags().StringVar(&o.Priority, priorityFlagName, "", + "Apply priority for the entire workload.") + subcmd.Flags().BoolVar(&o.SkipPriorityValidation, skipPriorityValidationFlagName, false, + "Skip workload priority class validation. Add priority class label even if the class does not exist.") + + modeSubcommand.Setup(clientGetter, subcmd, o) + + util.AddDryRunFlag(subcmd) + + _ = subcmd.MarkFlagRequired(profileFlagName) + + cobra.CheckErr(subcmd.RegisterFlagCompletionFunc(profileFlagName, completion.ApplicationProfileNameFunc(clientGetter))) + cobra.CheckErr(subcmd.RegisterFlagCompletionFunc(localQueueFlagName, completion.LocalQueueNameFunc(clientGetter))) + + cmd.AddCommand(subcmd) + } + + return cmd +} + +func (o *CreateOptions) Complete(clientGetter util.ClientGetter, cmd *cobra.Command, args []string) error { + currentSubcommand := createModeSubcommands[cmd.Name()] + o.ModeName = currentSubcommand.ModeName + + var err error + + if o.ModeName == v1alpha1.SlurmMode { + argsLenAtDash := cmd.ArgsLenAtDash() + + if argsLenAtDash == -1 || argsLenAtDash > 0 { + return fmt.Errorf("unknown command \"%s\" for \"%s\"", args[0], cmd.CommandPath()) + } + + if err := o.SlurmFlagSet.Parse(args[argsLenAtDash:]); err != nil { + return err + } + + slurmArgs := o.SlurmFlagSet.Args() + + if len(slurmArgs) == 0 { + return errors.New("must specify script") + } + + if len(slurmArgs) > 1 { + return errors.New("must specify only one script") + } + + o.Script = slurmArgs[0] + } + + o.Namespace, _, err = clientGetter.ToRawKubeConfigLoader().Namespace() + if err != nil { + return err + } + + if o.UserSpecifiedCommand != "" { + o.Command = strings.Fields(o.UserSpecifiedCommand) + } + + if cmd.Flags().Changed(parallelismFlagName) { + o.Parallelism = ptr.To(o.UserSpecifiedParallelism) + } + + if cmd.Flags().Changed(completionsFlagName) { + o.Completions = ptr.To(o.UserSpecifiedCompletions) + } + + if len(o.UserSpecifiedRequest) > 0 { + o.Requests = make(corev1.ResourceList) + for key, value := range o.UserSpecifiedRequest { + quantity, err := apiresource.ParseQuantity(value) + if err != nil { + return err + } + o.Requests[corev1.ResourceName(key)] = quantity + } + } + + if cmd.Flags().Changed(podRunningTimeoutFlagName) && o.PodRunningTimeout <= 0 { + return errors.New("--pod-running-timeout must be higher than zero") + } + + if o.SlurmFlagSet.Changed(cpusPerTaskFlagName) { + quantity, err := apiresource.ParseQuantity(o.UserSpecifiedCpusPerTask) + if err != nil { + return fmt.Errorf("cannot parse '%s': %w", o.UserSpecifiedCpusPerTask, err) + } + o.CpusPerTask = &quantity + } + + if o.SlurmFlagSet.Changed(gpusPerTaskFlagName) { + gpusPerTask, err := parser.GpusFlag(o.UserSpecifiedGpusPerTask) + if err != nil { + return fmt.Errorf("cannot parse '%s': %w", o.UserSpecifiedGpusPerTask, err) + } + o.GpusPerTask = gpusPerTask + } + + if o.SlurmFlagSet.Changed(memPerNodeFlagName) { + quantity, err := apiresource.ParseQuantity(o.UserSpecifiedMemPerNode) + if err != nil { + return fmt.Errorf("cannot parse '%s': %w", o.UserSpecifiedMemPerNode, err) + } + o.MemPerNode = &quantity + } + + if o.SlurmFlagSet.Changed(memPerTaskFlagName) { + quantity, err := apiresource.ParseQuantity(o.UserSpecifiedMemPerTask) + if err != nil { + return fmt.Errorf("cannot parse '%s': %w", o.UserSpecifiedMemPerTask, err) + } + o.MemPerTask = &quantity + } + + if o.SlurmFlagSet.Changed(memPerCPUFlagName) { + quantity, err := apiresource.ParseQuantity(o.UserSpecifiedMemPerCPU) + if err != nil { + return fmt.Errorf("cannot parse '%s': %w", o.UserSpecifiedMemPerCPU, err) + } + o.MemPerCPU = &quantity + } + + if o.SlurmFlagSet.Changed(memPerGPUFlagName) { + quantity, err := apiresource.ParseQuantity(o.UserSpecifiedMemPerGPU) + if err != nil { + return fmt.Errorf("cannot parse '%s': %w", o.UserSpecifiedMemPerGPU, err) + } + o.MemPerGPU = &quantity + } + + if o.SlurmFlagSet.Changed(nodesFlagName) { + o.Nodes = &o.UserSpecifiedNodes + } + + if o.SlurmFlagSet.Changed(nTasksFlagName) { + if o.UserSpecifiedNTasks <= 0 { + return errors.New("--nTasks must be greater than 0") + } + + o.NTasks = &o.UserSpecifiedNTasks + } + + o.DryRunStrategy, err = util.GetDryRunStrategy(cmd) + if err != nil { + return err + } + + err = util.PrintFlagsWithDryRunStrategy(o.PrintFlags, o.DryRunStrategy) + if err != nil { + return err + } + + printer, err := o.PrintFlags.ToPrinter() + if err != nil { + return err + } + + o.PrintObj = printer.PrintObj + + o.Config, err = clientGetter.ToRESTConfig() + if err != nil { + return err + } + + err = setKubernetesDefaults(o.Config) + if err != nil { + return err + } + + return nil +} + +func (o *CreateOptions) Run(ctx context.Context, clientGetter util.ClientGetter, runTime time.Time) error { + rootObj, childObjs, err := builder.NewBuilder(clientGetter, runTime). + WithNamespace(o.Namespace). + WithProfileName(o.ProfileName). + WithModeName(o.ModeName). + WithCommand(o.Command). + WithParallelism(o.Parallelism). + WithCompletions(o.Completions). + WithReplicas(o.Replicas). + WithMinReplicas(o.MinReplicas). + WithMaxReplicas(o.MaxReplicas). + WithRequests(o.Requests). + WithLocalQueue(o.LocalQueue). + WithRayCluster(o.RayCluster). + WithScript(o.Script). + WithArray(o.Array). + WithCpusPerTask(o.CpusPerTask). + WithGpusPerTask(o.GpusPerTask). + WithMemPerNode(o.MemPerNode). + WithMemPerTask(o.MemPerTask). + WithMemPerCPU(o.MemPerCPU). + WithMemPerGPU(o.MemPerGPU). + WithNodes(o.Nodes). + WithNTasks(o.NTasks). + WithOutput(o.Output). + WithError(o.Error). + WithInput(o.Input). + WithJobName(o.JobName). + WithPartition(o.Partition). + WithPriority(o.Priority). + WithInitImage(o.InitImage). + WithIgnoreUnknown(o.IgnoreUnknown). + WithSkipLocalQueueValidation(o.SkipLocalQueueValidation). + WithSkipPriorityValidation(o.SkipPriorityValidation). + WithChangeDir(o.ChangeDir). + WithFirstNodeIP(o.FirstNodeIP). + WithFirstNodeIPTimeout(o.FirstNodeIPTimeout). + WithTimeLimit(o.TimeLimit). + Do(ctx) + if err != nil { + return err + } + + if o.DryRunStrategy != util.DryRunClient { + rootObj, err = o.createObject(ctx, clientGetter, rootObj, nil) + if err != nil { + return err + } + } + + err = o.PrintObj(rootObj, o.Out) + if err != nil { + return err + } + + for i := range childObjs { + if o.DryRunStrategy != util.DryRunClient { + childObjs[i], err = o.createObject(ctx, clientGetter, childObjs[i], rootObj) + if err != nil { + return err + } + } + + err = o.PrintObj(childObjs[i], o.Out) + if err != nil { + return err + } + } + + if o.DryRunStrategy == util.DryRunNone && o.ModeName == v1alpha1.InteractiveMode { + pod := rootObj.(*corev1.Pod) + return o.RunInteractivePod(ctx, clientGetter, pod.Name) + } + + return nil +} + +func (o *CreateOptions) createObject(ctx context.Context, clientGetter util.ClientGetter, obj runtime.Object, owner runtime.Object) (runtime.Object, error) { + options := metav1.CreateOptions{} + if o.DryRunStrategy == util.DryRunServer { + options.DryRun = []string{metav1.DryRunAll} + } + + dynamicClient, err := clientGetter.DynamicClient() + if err != nil { + return nil, err + } + + restMapper, err := clientGetter.ToRESTMapper() + if err != nil { + return nil, err + } + + gvk := obj.GetObjectKind().GroupVersionKind() + mapping, err := restMapper.RESTMapping(gvk.GroupKind(), gvk.Version) + if err != nil { + return nil, err + } + gvr := mapping.Resource + + unstructuredObj := &unstructured.Unstructured{} + unstructuredObj.Object, err = runtime.DefaultUnstructuredConverter.ToUnstructured(obj) + if err != nil { + return nil, err + } + + if owner != nil { + unstructuredOwner := &unstructured.Unstructured{} + unstructuredOwner.Object, err = runtime.DefaultUnstructuredConverter.ToUnstructured(owner) + if err != nil { + return nil, err + } + unstructuredObj.SetOwnerReferences(append(unstructuredOwner.GetOwnerReferences(), metav1.OwnerReference{ + APIVersion: unstructuredOwner.GetAPIVersion(), + Kind: unstructuredOwner.GetKind(), + Name: unstructuredOwner.GetName(), + UID: unstructuredOwner.GetUID(), + })) + } + + unstructuredObj, err = dynamicClient.Resource(gvr).Namespace(o.Namespace).Create(ctx, unstructuredObj, options) + if err != nil { + return nil, err + } + + createdObj := obj.DeepCopyObject() + + err = runtime.DefaultUnstructuredConverter.FromUnstructured(unstructuredObj.UnstructuredContent(), createdObj) + if err != nil { + return nil, err + } + + return createdObj, nil +} + +func (o *CreateOptions) RunInteractivePod(ctx context.Context, clientGetter util.ClientGetter, podName string) error { + k8sclient, err := clientGetter.K8sClientset() + if err != nil { + return err + } + + if o.RemoveInteractivePod { + defer func() { + err = k8sclient.CoreV1().Pods(o.Namespace).Delete(ctx, podName, metav1.DeleteOptions{}) + if err != nil { + fmt.Fprintln(o.ErrOut, err.Error()) + } + fmt.Fprintf(o.Out, "pod \"%s\" deleted\n", podName) + }() + } + + fmt.Fprintf(o.Out, "waiting for pod \"%s\" to be running...\n", podName) + err = waitForPodRunning(ctx, k8sclient, o.Namespace, podName, o.PodRunningTimeout) + if err != nil { + return err + } + + pod, err := k8sclient.CoreV1().Pods(o.Namespace).Get(ctx, podName, metav1.GetOptions{}) + if err != nil { + return err + } + + err = attachTTY(o, pod) + if err != nil { + return err + } + + return nil +} + +func attachTTY(o *CreateOptions, pod *corev1.Pod) error { + o.Stdin = true + o.TTY = true + containerToAttach := &pod.Spec.Containers[0] + if !containerToAttach.TTY { + return fmt.Errorf("error: Unable to use a TTY - container %s did not allocate one", containerToAttach.Name) + } + + tty := o.SetupTTY() + + var sizeQueue remotecommand.TerminalSizeQueue + if tty.Raw { + // this call spawns a goroutine to monitor/update the terminal size + sizeQueue = tty.MonitorSize(tty.GetSize()) + + // unset p.Err if it was previously set because both stdout and stderr go over p.Out when tty is true + o.ErrOut = nil + } + + return tty.Safe(o.AttachFunc(o, containerToAttach, sizeQueue, pod)) +} + +func defaultAttachFunc(o *CreateOptions, containerToAttach *corev1.Container, sizeQueue remotecommand.TerminalSizeQueue, pod *corev1.Pod) func() error { + return func() error { + restClient, err := restclient.RESTClientFor(o.Config) + if err != nil { + return err + } + + req := restClient.Post(). + Resource("pods"). + Name(pod.Name). + Namespace(pod.Namespace). + SubResource("attach") + req.VersionedParams(&corev1.PodAttachOptions{ + Container: containerToAttach.Name, + Stdin: o.Stdin, + Stdout: o.Out != nil, + Stderr: o.ErrOut != nil, + TTY: o.TTY, + }, k8sscheme.ParameterCodec) + + return o.Attach.Attach(req.URL(), o.Config, o.In, o.Out, o.ErrOut, o.TTY, sizeQueue) + } +} diff --git a/pkg/cmd/create/create_test.go b/pkg/cmd/create/create_test.go new file mode 100644 index 0000000..fd93421 --- /dev/null +++ b/pkg/cmd/create/create_test.go @@ -0,0 +1,2340 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package create + +import ( + "context" + "fmt" + "io" + "net/url" + "os" + "regexp" + "strings" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + utilrand "k8s.io/apimachinery/pkg/util/rand" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/cli-runtime/pkg/genericclioptions" + "k8s.io/cli-runtime/pkg/genericiooptions" + "k8s.io/client-go/dynamic/fake" + k8sfake "k8s.io/client-go/kubernetes/fake" + k8sscheme "k8s.io/client-go/kubernetes/scheme" + restclient "k8s.io/client-go/rest" + kubetesting "k8s.io/client-go/testing" + "k8s.io/client-go/tools/remotecommand" + clocktesting "k8s.io/utils/clock/testing" + "k8s.io/utils/ptr" + kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" + kueuefake "sigs.k8s.io/kueue/client-go/clientset/versioned/fake" + + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/apis/v1alpha1" + kjobctlfake "sigs.k8s.io/kueue/cmd/experimental/kjobctl/client-go/clientset/versioned/fake" + cmdtesting "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/cmd/testing" + cmdutil "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/cmd/util" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/constants" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/testing/wrappers" +) + +func TestCreateOptions_Complete(t *testing.T) { + testStartTime := time.Now() + + testCases := map[string]struct { + args []string + options *CreateOptions + wantOptions *CreateOptions + wantErr string + }{ + "invalid request": { + args: []string{"job"}, + options: &CreateOptions{ + Namespace: metav1.NamespaceDefault, + ModeName: v1alpha1.JobMode, + UserSpecifiedRequest: map[string]string{"cpu": "invalid"}, + }, + wantOptions: &CreateOptions{}, + wantErr: "quantities must match the regular expression '^([+-]?[0-9.]+)([eEinumkKMGTP]*[-+]?[0-9]*)$'", + }, + } + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + streams, _, out, outErr := genericiooptions.NewTestIOStreams() + + tcg := cmdtesting.NewTestClientGetter() + + cmd := NewCreateCmd(tcg, streams, clocktesting.NewFakeClock(testStartTime)) + cmd.SetOut(out) + cmd.SetErr(outErr) + cmd.SetArgs(tc.args) + + gotErr := tc.options.Complete(tcg, cmd.Commands()[0], nil) + + var gotErrStr string + if gotErr != nil { + gotErrStr = gotErr.Error() + } + + if diff := cmp.Diff(tc.wantErr, gotErrStr); diff != "" { + t.Errorf("Unexpected error (-want/+got)\n%s", diff) + } + + if gotErr != nil { + return + } + }) + } +} + +type createCmdTestCase struct { + beforeTest func(t *testing.T, tc *createCmdTestCase) + tempFile string + ns string + args func(tc *createCmdTestCase) []string + kjobctlObjs []runtime.Object + kueueObjs []runtime.Object + gvks []schema.GroupVersionKind + wantLists []runtime.Object + cmpopts []cmp.Option + wantOut string + wantOutPattern string + wantOutErr string + wantErr string +} + +func beforeSlurmTest(t *testing.T, tc *createCmdTestCase) { + file, err := os.CreateTemp("", "slurm") + if err != nil { + t.Fatal(err) + } + defer file.Close() + t.Cleanup(func() { + if err := os.Remove(tc.tempFile); err != nil { + t.Fatal(err) + } + }) + + if _, err := file.WriteString("#!/bin/bash\nsleep 300'"); err != nil { + t.Fatal(err) + } + + tc.tempFile = file.Name() +} + +func TestCreateCmd(t *testing.T) { + testStartTime := time.Now() + userID := os.Getenv(constants.SystemEnvVarNameUser) + + testCases := map[string]createCmdTestCase{ + "should create job": { + args: func(tc *createCmdTestCase) []string { return []string{"job", "--profile", "profile"} }, + kjobctlObjs: []runtime.Object{ + wrappers.MakeJobTemplate("job-template", metav1.NamespaceDefault).Obj(), + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(*wrappers.MakeSupportedMode(v1alpha1.JobMode, "job-template").Obj()). + Obj(), + }, + gvks: []schema.GroupVersionKind{{Group: "batch", Version: "v1", Kind: "Job"}}, + wantLists: []runtime.Object{ + &batchv1.JobList{ + TypeMeta: metav1.TypeMeta{Kind: "JobList", APIVersion: "batch/v1"}, + Items: []batchv1.Job{ + *wrappers.MakeJob("", metav1.NamespaceDefault). + GenerateName("profile-job-"). + Profile("profile"). + Mode(v1alpha1.JobMode). + Obj(), + }, + }, + }, + // Fake dynamic client not generating name. That's why we have . + wantOut: "job.batch/ created\n", + }, + "should create rayjob": { + args: func(tc *createCmdTestCase) []string { return []string{"rayjob", "--profile", "profile"} }, + kjobctlObjs: []runtime.Object{ + wrappers.MakeRayJobTemplate("ray-job-template", metav1.NamespaceDefault).Obj(), + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(*wrappers.MakeSupportedMode(v1alpha1.RayJobMode, "ray-job-template").Obj()). + Obj(), + }, + gvks: []schema.GroupVersionKind{{Group: "ray.io", Version: "v1", Kind: "RayJob"}}, + wantLists: []runtime.Object{ + &rayv1.RayJobList{ + TypeMeta: metav1.TypeMeta{Kind: "RayJobList", APIVersion: "ray.io/v1"}, + Items: []rayv1.RayJob{ + *wrappers.MakeRayJob("", metav1.NamespaceDefault). + GenerateName("profile-rayjob-"). + Profile("profile"). + Mode(v1alpha1.RayJobMode). + Obj(), + }, + }, + }, + // Fake dynamic client not generating name. That's why we have . + wantOut: "rayjob.ray.io/ created\n", + }, + "should create raycluster": { + args: func(tc *createCmdTestCase) []string { return []string{"raycluster", "--profile", "profile"} }, + kjobctlObjs: []runtime.Object{ + wrappers.MakeRayClusterTemplate("ray-cluster-template", metav1.NamespaceDefault).Obj(), + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(*wrappers.MakeSupportedMode(v1alpha1.RayClusterMode, "ray-cluster-template").Obj()). + Obj(), + }, + gvks: []schema.GroupVersionKind{{Group: "ray.io", Version: "v1", Kind: "RayCluster"}}, + wantLists: []runtime.Object{ + &rayv1.RayClusterList{ + TypeMeta: metav1.TypeMeta{Kind: "RayClusterList", APIVersion: "ray.io/v1"}, + Items: []rayv1.RayCluster{ + *wrappers.MakeRayCluster("", metav1.NamespaceDefault). + GenerateName("profile-raycluster-"). + Profile("profile"). + Mode(v1alpha1.RayClusterMode). + Obj(), + }, + }, + }, + // Fake dynamic client not generating name. That's why we have . + wantOut: "raycluster.ray.io/ created\n", + }, + "should create job with short profile flag": { + args: func(tc *createCmdTestCase) []string { return []string{"job", "-p", "profile"} }, + kjobctlObjs: []runtime.Object{ + wrappers.MakeJobTemplate("job-template", metav1.NamespaceDefault).Obj(), + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(*wrappers.MakeSupportedMode(v1alpha1.JobMode, "job-template").Obj()). + Obj(), + }, + gvks: []schema.GroupVersionKind{{Group: "batch", Version: "v1", Kind: "Job"}}, + wantLists: []runtime.Object{ + &batchv1.JobList{ + TypeMeta: metav1.TypeMeta{Kind: "JobList", APIVersion: "batch/v1"}, + Items: []batchv1.Job{ + *wrappers.MakeJob("", metav1.NamespaceDefault). + GenerateName("profile-job-"). + Profile("profile"). + Mode(v1alpha1.JobMode). + Obj(), + }, + }, + }, + // Fake dynamic client not generating name. That's why we have . + wantOut: "job.batch/ created\n", + }, + "should create job with localqueue replacement": { + args: func(tc *createCmdTestCase) []string { + return []string{"job", "--profile", "profile", "--localqueue", "lq1"} + }, + kjobctlObjs: []runtime.Object{ + wrappers.MakeJobTemplate("job-template", metav1.NamespaceDefault).Obj(), + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(*wrappers.MakeSupportedMode(v1alpha1.JobMode, "job-template").Obj()). + Obj(), + }, + kueueObjs: []runtime.Object{ + &kueue.LocalQueue{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: metav1.NamespaceDefault, + Name: "lq1", + }, + }, + }, + gvks: []schema.GroupVersionKind{{Group: "batch", Version: "v1", Kind: "Job"}}, + wantLists: []runtime.Object{ + &batchv1.JobList{ + TypeMeta: metav1.TypeMeta{Kind: "JobList", APIVersion: "batch/v1"}, + Items: []batchv1.Job{ + *wrappers.MakeJob("", metav1.NamespaceDefault). + GenerateName("profile-job-"). + Profile("profile"). + Mode(v1alpha1.JobMode). + LocalQueue("lq1"). + Obj(), + }, + }, + }, + // Fake dynamic client not generating name. That's why we have . + wantOut: "job.batch/ created\n", + }, + "should create job with localqueue and skip local queue validation": { + args: func(tc *createCmdTestCase) []string { + return []string{"job", "--profile", "profile", "--localqueue", "lq1", "--skip-localqueue-validation"} + }, + kjobctlObjs: []runtime.Object{ + wrappers.MakeJobTemplate("job-template", metav1.NamespaceDefault).Obj(), + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(*wrappers.MakeSupportedMode(v1alpha1.JobMode, "job-template").Obj()). + Obj(), + }, + gvks: []schema.GroupVersionKind{{Group: "batch", Version: "v1", Kind: "Job"}}, + wantLists: []runtime.Object{ + &batchv1.JobList{ + TypeMeta: metav1.TypeMeta{Kind: "JobList", APIVersion: "batch/v1"}, + Items: []batchv1.Job{ + *wrappers.MakeJob("", metav1.NamespaceDefault). + GenerateName("profile-job-"). + Profile("profile"). + Mode(v1alpha1.JobMode). + LocalQueue("lq1"). + Obj(), + }, + }, + }, + // Fake dynamic client not generating name. That's why we have . + wantOut: "job.batch/ created\n", + }, + "should create job with parallelism replacement": { + args: func(tc *createCmdTestCase) []string { + return []string{"job", "--profile", "profile", "--parallelism", "5"} + }, + kjobctlObjs: []runtime.Object{ + wrappers.MakeJobTemplate("job-template", metav1.NamespaceDefault). + Parallelism(1). + Completions(1). + Obj(), + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(*wrappers.MakeSupportedMode(v1alpha1.JobMode, "job-template").Obj()). + Obj(), + }, + gvks: []schema.GroupVersionKind{{Group: "batch", Version: "v1", Kind: "Job"}}, + wantLists: []runtime.Object{ + &batchv1.JobList{ + TypeMeta: metav1.TypeMeta{Kind: "JobList", APIVersion: "batch/v1"}, + Items: []batchv1.Job{ + *wrappers.MakeJob("", metav1.NamespaceDefault). + GenerateName("profile-job-"). + Profile("profile"). + Mode(v1alpha1.JobMode). + Parallelism(5). + Completions(1). + Obj(), + }, + }, + }, + // Fake dynamic client not generating name. That's why we have . + wantOut: "job.batch/ created\n", + }, + "should create job with --priority flag": { + args: func(tc *createCmdTestCase) []string { + return []string{"job", "--profile", "profile", "--priority", "sample-priority"} + }, + kjobctlObjs: []runtime.Object{ + wrappers.MakeJobTemplate("job-template", metav1.NamespaceDefault). + Obj(), + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(*wrappers.MakeSupportedMode(v1alpha1.JobMode, "job-template").Obj()). + Obj(), + }, + kueueObjs: []runtime.Object{ + &kueue.WorkloadPriorityClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: "sample-priority", + }, + }, + }, + gvks: []schema.GroupVersionKind{{Group: "batch", Version: "v1", Kind: "Job"}}, + wantLists: []runtime.Object{ + &batchv1.JobList{ + TypeMeta: metav1.TypeMeta{Kind: "JobList", APIVersion: "batch/v1"}, + Items: []batchv1.Job{ + *wrappers.MakeJob("", metav1.NamespaceDefault). + Priority("sample-priority"). + GenerateName("profile-job-"). + Profile("profile"). + Mode(v1alpha1.JobMode). + Obj(), + }, + }, + }, + // Fake dynamic client not generating name. That's why we have . + wantOut: "job.batch/ created\n", + }, + "should create job with --priority flag and skip workload priority class validation": { + args: func(tc *createCmdTestCase) []string { + return []string{"job", "--profile", "profile", "--skip-priority-validation", "--priority", "sample-priority"} + }, + kjobctlObjs: []runtime.Object{ + wrappers.MakeJobTemplate("job-template", metav1.NamespaceDefault). + Obj(), + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(*wrappers.MakeSupportedMode(v1alpha1.JobMode, "job-template").Obj()). + Obj(), + }, + gvks: []schema.GroupVersionKind{{Group: "batch", Version: "v1", Kind: "Job"}}, + wantLists: []runtime.Object{ + &batchv1.JobList{ + TypeMeta: metav1.TypeMeta{Kind: "JobList", APIVersion: "batch/v1"}, + Items: []batchv1.Job{ + *wrappers.MakeJob("", metav1.NamespaceDefault). + Priority("sample-priority"). + GenerateName("profile-job-"). + Profile("profile"). + Mode(v1alpha1.JobMode). + Obj(), + }, + }, + }, + // Fake dynamic client not generating name. That's why we have . + wantOut: "job.batch/ created\n", + }, + "should create job with completions replacement": { + args: func(tc *createCmdTestCase) []string { + return []string{"job", "--profile", "profile", "--completions", "5"} + }, + kjobctlObjs: []runtime.Object{ + wrappers.MakeJobTemplate("job-template", metav1.NamespaceDefault). + Parallelism(1). + Completions(1). + Obj(), + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(*wrappers.MakeSupportedMode(v1alpha1.JobMode, "job-template").Obj()). + Obj(), + }, + gvks: []schema.GroupVersionKind{{Group: "batch", Version: "v1", Kind: "Job"}}, + wantLists: []runtime.Object{ + &batchv1.JobList{ + TypeMeta: metav1.TypeMeta{Kind: "JobList", APIVersion: "batch/v1"}, + Items: []batchv1.Job{ + *wrappers.MakeJob("", metav1.NamespaceDefault). + GenerateName("profile-job-"). + Profile("profile"). + Mode(v1alpha1.JobMode). + Parallelism(1). + Completions(5). + Obj(), + }, + }, + }, + // Fake dynamic client not generating name. That's why we have . + wantOut: "job.batch/ created\n", + }, + "should create job with command replacement": { + args: func(tc *createCmdTestCase) []string { + return []string{"job", "--profile", "profile", "--cmd", "sleep 15s"} + }, + kjobctlObjs: []runtime.Object{ + wrappers.MakeJobTemplate("job-template", metav1.NamespaceDefault). + Parallelism(1). + Completions(1). + WithContainer(*wrappers.MakeContainer("c1", "sleep").Obj()). + WithContainer(*wrappers.MakeContainer("c2", "sleep").Obj()). + Obj(), + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(*wrappers.MakeSupportedMode(v1alpha1.JobMode, "job-template").Obj()). + Obj(), + }, + gvks: []schema.GroupVersionKind{{Group: "batch", Version: "v1", Kind: "Job"}}, + wantLists: []runtime.Object{ + &batchv1.JobList{ + TypeMeta: metav1.TypeMeta{Kind: "JobList", APIVersion: "batch/v1"}, + Items: []batchv1.Job{ + *wrappers.MakeJob("", metav1.NamespaceDefault). + GenerateName("profile-job-"). + Profile("profile"). + Mode(v1alpha1.JobMode). + Parallelism(1). + Completions(1). + WithContainer(*wrappers.MakeContainer("c1", "sleep").Command("sleep", "15s").Obj()). + WithContainer(*wrappers.MakeContainer("c2", "sleep").Obj()). + WithEnvVar(corev1.EnvVar{Name: constants.EnvVarNameUserID, Value: userID}). + WithEnvVar(corev1.EnvVar{Name: constants.EnvVarTaskName, Value: "default_profile"}). + WithEnvVar(corev1.EnvVar{ + Name: constants.EnvVarTaskID, + Value: fmt.Sprintf("%s_%s_default_profile", userID, testStartTime.Format(time.RFC3339)), + }). + WithEnvVar(corev1.EnvVar{Name: "PROFILE", Value: "default_profile"}). + WithEnvVar(corev1.EnvVar{Name: "TIMESTAMP", Value: testStartTime.Format(time.RFC3339)}). + Obj(), + }, + }, + }, + // Fake dynamic client not generating name. That's why we have . + wantOut: "job.batch/ created\n", + }, + "should create job with request replacement": { + args: func(tc *createCmdTestCase) []string { + return []string{"job", "--profile", "profile", "--request", "cpu=100m,ram=3Gi"} + }, + kjobctlObjs: []runtime.Object{ + wrappers.MakeJobTemplate("job-template", metav1.NamespaceDefault). + Parallelism(1). + Completions(1). + WithContainer(*wrappers.MakeContainer("c1", "sleep").Obj()). + WithContainer(*wrappers.MakeContainer("c2", "sleep").Obj()). + Obj(), + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(*wrappers.MakeSupportedMode(v1alpha1.JobMode, "job-template").Obj()). + Obj(), + }, + gvks: []schema.GroupVersionKind{{Group: "batch", Version: "v1", Kind: "Job"}}, + wantLists: []runtime.Object{ + &batchv1.JobList{ + TypeMeta: metav1.TypeMeta{Kind: "JobList", APIVersion: "batch/v1"}, + Items: []batchv1.Job{ + *wrappers.MakeJob("", metav1.NamespaceDefault). + GenerateName("profile-job-"). + Profile("profile"). + Mode(v1alpha1.JobMode). + Parallelism(1). + Completions(1). + WithContainer( + *wrappers.MakeContainer("c1", "sleep"). + WithRequest("cpu", resource.MustParse("100m")). + WithRequest("ram", resource.MustParse("3Gi")). + Obj(), + ). + WithContainer(*wrappers.MakeContainer("c2", "sleep").Obj()). + WithEnvVar(corev1.EnvVar{Name: constants.EnvVarNameUserID, Value: userID}). + WithEnvVar(corev1.EnvVar{Name: constants.EnvVarTaskName, Value: "default_profile"}). + WithEnvVar(corev1.EnvVar{ + Name: constants.EnvVarTaskID, + Value: fmt.Sprintf("%s_%s_default_profile", userID, testStartTime.Format(time.RFC3339)), + }). + WithEnvVar(corev1.EnvVar{Name: "PROFILE", Value: "default_profile"}). + WithEnvVar(corev1.EnvVar{Name: "TIMESTAMP", Value: testStartTime.Format(time.RFC3339)}). + Obj(), + }, + }, + }, + // Fake dynamic client not generating name. That's why we have . + wantOut: "job.batch/ created\n", + }, + "should create ray job with replicas replacement": { + args: func(tc *createCmdTestCase) []string { + return []string{"rayjob", "--profile", "profile", "--replicas", "g1=5"} + }, + kjobctlObjs: []runtime.Object{ + wrappers.MakeRayJobTemplate("ray-job-template", metav1.NamespaceDefault). + WithRayClusterSpec( + wrappers.MakeRayClusterSpec(). + WithWorkerGroupSpec(*wrappers.MakeWorkerGroupSpec("g1").Obj()). + Obj(), + ). + Obj(), + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(*wrappers.MakeSupportedMode(v1alpha1.RayJobMode, "ray-job-template").Obj()). + Obj(), + }, + gvks: []schema.GroupVersionKind{{Group: "ray.io", Version: "v1", Kind: "RayJob"}}, + wantLists: []runtime.Object{ + &rayv1.RayJobList{ + TypeMeta: metav1.TypeMeta{Kind: "RayJobList", APIVersion: "ray.io/v1"}, + Items: []rayv1.RayJob{ + *wrappers.MakeRayJob("", metav1.NamespaceDefault). + GenerateName("profile-rayjob-"). + Profile("profile"). + Mode(v1alpha1.RayJobMode). + WithWorkerGroupSpec(*wrappers.MakeWorkerGroupSpec("g1").Replicas(5).Obj()). + Obj(), + }, + }, + }, + // Fake dynamic client not generating name. That's why we have . + wantOut: "rayjob.ray.io/ created\n", + }, + "should create ray job with cmd replacement": { + args: func(tc *createCmdTestCase) []string { + return []string{"rayjob", "--profile", "profile", "--cmd", "sleep 3s"} + }, + kjobctlObjs: []runtime.Object{ + wrappers.MakeRayJobTemplate("ray-job-template", metav1.NamespaceDefault). + Obj(), + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(*wrappers.MakeSupportedMode(v1alpha1.RayJobMode, "ray-job-template").Obj()). + Obj(), + }, + gvks: []schema.GroupVersionKind{{Group: "ray.io", Version: "v1", Kind: "RayJob"}}, + wantLists: []runtime.Object{ + &rayv1.RayJobList{ + TypeMeta: metav1.TypeMeta{Kind: "RayJobList", APIVersion: "ray.io/v1"}, + Items: []rayv1.RayJob{ + *wrappers.MakeRayJob("", metav1.NamespaceDefault). + GenerateName("profile-rayjob-"). + Profile("profile"). + Mode(v1alpha1.RayJobMode). + Entrypoint("sleep 3s"). + Obj(), + }, + }, + }, + // Fake dynamic client not generating name. That's why we have . + wantOut: "rayjob.ray.io/ created\n", + }, + "should create ray job with min-replicas replacement": { + args: func(tc *createCmdTestCase) []string { + return []string{"rayjob", "--profile", "profile", "--min-replicas", "g1=5"} + }, + kjobctlObjs: []runtime.Object{ + wrappers.MakeRayJobTemplate("ray-job-template", metav1.NamespaceDefault). + WithRayClusterSpec( + wrappers.MakeRayClusterSpec(). + WithWorkerGroupSpec(*wrappers.MakeWorkerGroupSpec("g1").Obj()). + Obj(), + ). + Obj(), + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(*wrappers.MakeSupportedMode(v1alpha1.RayJobMode, "ray-job-template").Obj()). + Obj(), + }, + gvks: []schema.GroupVersionKind{{Group: "ray.io", Version: "v1", Kind: "RayJob"}}, + wantLists: []runtime.Object{ + &rayv1.RayJobList{ + TypeMeta: metav1.TypeMeta{Kind: "RayJobList", APIVersion: "ray.io/v1"}, + Items: []rayv1.RayJob{ + *wrappers.MakeRayJob("", metav1.NamespaceDefault). + GenerateName("profile-rayjob-"). + Profile("profile"). + Mode(v1alpha1.RayJobMode). + WithWorkerGroupSpec(*wrappers.MakeWorkerGroupSpec("g1").MinReplicas(5).Obj()). + Obj(), + }, + }, + }, + // Fake dynamic client not generating name. That's why we have . + wantOut: "rayjob.ray.io/ created\n", + }, + "should create ray job with max-replicas replacement": { + args: func(tc *createCmdTestCase) []string { + return []string{"rayjob", "--profile", "profile", "--max-replicas", "g1=5"} + }, + kjobctlObjs: []runtime.Object{ + wrappers.MakeRayJobTemplate("ray-job-template", metav1.NamespaceDefault). + WithRayClusterSpec( + wrappers.MakeRayClusterSpec(). + WithWorkerGroupSpec(*wrappers.MakeWorkerGroupSpec("g1").Obj()). + Obj(), + ). + Obj(), + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(*wrappers.MakeSupportedMode(v1alpha1.RayJobMode, "ray-job-template").Obj()). + Obj(), + }, + gvks: []schema.GroupVersionKind{{Group: "ray.io", Version: "v1", Kind: "RayJob"}}, + wantLists: []runtime.Object{ + &rayv1.RayJobList{ + TypeMeta: metav1.TypeMeta{Kind: "RayJobList", APIVersion: "ray.io/v1"}, + Items: []rayv1.RayJob{ + *wrappers.MakeRayJob("", metav1.NamespaceDefault). + GenerateName("profile-rayjob-"). + Profile("profile"). + Mode(v1alpha1.RayJobMode). + WithWorkerGroupSpec(*wrappers.MakeWorkerGroupSpec("g1").MaxReplicas(5).Obj()). + Obj(), + }, + }, + }, + // Fake dynamic client not generating name. That's why we have . + wantOut: "rayjob.ray.io/ created\n", + }, + "should create ray job with raycluster replacement": { + args: func(tc *createCmdTestCase) []string { + return []string{"rayjob", "--profile", "profile", "--raycluster", "rc1"} + }, + kjobctlObjs: []runtime.Object{ + wrappers.MakeRayJobTemplate("ray-job-template", metav1.NamespaceDefault). + WithRayClusterSpec( + wrappers.MakeRayClusterSpec(). + WithWorkerGroupSpec(*wrappers.MakeWorkerGroupSpec("g1").Obj()). + Obj(), + ). + Obj(), + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(*wrappers.MakeSupportedMode(v1alpha1.RayJobMode, "ray-job-template").Obj()). + Obj(), + }, + gvks: []schema.GroupVersionKind{{Group: "ray.io", Version: "v1", Kind: "RayJob"}}, + wantLists: []runtime.Object{ + &rayv1.RayJobList{ + TypeMeta: metav1.TypeMeta{Kind: "RayJobList", APIVersion: "ray.io/v1"}, + Items: []rayv1.RayJob{ + *wrappers.MakeRayJob("", metav1.NamespaceDefault). + GenerateName("profile-rayjob-"). + Profile("profile"). + Mode(v1alpha1.RayJobMode). + WithRayClusterLabelSelector("rc1"). + Obj(), + }, + }, + }, + // Fake dynamic client not generating name. That's why we have . + wantOut: "rayjob.ray.io/ created\n", + }, + "should create ray job with --priority flag": { + args: func(tc *createCmdTestCase) []string { + return []string{"rayjob", "--profile", "profile", "--priority", "sample-priority"} + }, + kjobctlObjs: []runtime.Object{ + wrappers.MakeRayJobTemplate("ray-job-template", metav1.NamespaceDefault). + Obj(), + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(*wrappers.MakeSupportedMode(v1alpha1.RayJobMode, "ray-job-template").Obj()). + Obj(), + }, + kueueObjs: []runtime.Object{ + &kueue.WorkloadPriorityClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: "sample-priority", + }, + }, + }, + gvks: []schema.GroupVersionKind{{Group: "ray.io", Version: "v1", Kind: "RayJob"}}, + wantLists: []runtime.Object{ + &rayv1.RayJobList{ + TypeMeta: metav1.TypeMeta{Kind: "RayJobList", APIVersion: "ray.io/v1"}, + Items: []rayv1.RayJob{ + *wrappers.MakeRayJob("", metav1.NamespaceDefault). + Priority("sample-priority"). + GenerateName("profile-rayjob-"). + Profile("profile"). + Mode(v1alpha1.RayJobMode). + Obj(), + }, + }, + }, + // Fake dynamic client not generating name. That's why we have . + wantOut: "rayjob.ray.io/ created\n", + }, + "should create ray job with --priority flag and skip workload priority class validation": { + args: func(tc *createCmdTestCase) []string { + return []string{"rayjob", "--profile", "profile", "--skip-priority-validation", "--priority", "sample-priority"} + }, + kjobctlObjs: []runtime.Object{ + wrappers.MakeRayJobTemplate("ray-job-template", metav1.NamespaceDefault). + Obj(), + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(*wrappers.MakeSupportedMode(v1alpha1.RayJobMode, "ray-job-template").Obj()). + Obj(), + }, + gvks: []schema.GroupVersionKind{{Group: "ray.io", Version: "v1", Kind: "RayJob"}}, + wantLists: []runtime.Object{ + &rayv1.RayJobList{ + TypeMeta: metav1.TypeMeta{Kind: "RayJobList", APIVersion: "ray.io/v1"}, + Items: []rayv1.RayJob{ + *wrappers.MakeRayJob("", metav1.NamespaceDefault). + Priority("sample-priority"). + GenerateName("profile-rayjob-"). + Profile("profile"). + Mode(v1alpha1.RayJobMode). + Obj(), + }, + }, + }, + // Fake dynamic client not generating name. That's why we have . + wantOut: "rayjob.ray.io/ created\n", + }, + "shouldn't create ray job with raycluster and localqueue replacements because mutually exclusive": { + args: func(tc *createCmdTestCase) []string { + return []string{"rayjob", "--profile", "profile", "--raycluster", "rc1", "--localqueue", "lq1"} + }, + wantErr: "if any flags in the group [raycluster localqueue] are set none of the others can be; [localqueue raycluster] were all set", + }, + "shouldn't create ray job with raycluster and replicas replacements because mutually exclusive": { + args: func(tc *createCmdTestCase) []string { + return []string{"rayjob", "--profile", "profile", "--raycluster", "rc1", "--replicas", "g1=5"} + }, + wantErr: "if any flags in the group [raycluster replicas] are set none of the others can be; [raycluster replicas] were all set", + }, + "shouldn't create ray job with raycluster and min-replicas replacements because mutually exclusive": { + args: func(tc *createCmdTestCase) []string { + return []string{"rayjob", "--profile", "profile", "--raycluster", "rc1", "--min-replicas", "g1=5"} + }, + wantErr: "if any flags in the group [raycluster min-replicas] are set none of the others can be; [min-replicas raycluster] were all set", + }, + "shouldn't create ray job with raycluster and max-replicas replacements because mutually exclusive": { + args: func(tc *createCmdTestCase) []string { + return []string{"rayjob", "--profile", "profile", "--raycluster", "rc1", "--max-replicas", "g1=5"} + }, + wantErr: "if any flags in the group [raycluster max-replicas] are set none of the others can be; [max-replicas raycluster] were all set", + }, + "should create raycluster with array ": { + args: func(tc *createCmdTestCase) []string { return []string{"raycluster", "--profile", "profile"} }, + kjobctlObjs: []runtime.Object{ + wrappers.MakeRayClusterTemplate("ray-cluster-template", metav1.NamespaceDefault).Obj(), + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(*wrappers.MakeSupportedMode(v1alpha1.RayClusterMode, "ray-cluster-template").Obj()). + Obj(), + }, + gvks: []schema.GroupVersionKind{{Group: "ray.io", Version: "v1", Kind: "RayCluster"}}, + wantLists: []runtime.Object{ + &rayv1.RayClusterList{ + TypeMeta: metav1.TypeMeta{Kind: "RayClusterList", APIVersion: "ray.io/v1"}, + Items: []rayv1.RayCluster{ + *wrappers.MakeRayCluster("", metav1.NamespaceDefault). + GenerateName("profile-raycluster-"). + Profile("profile"). + Mode(v1alpha1.RayClusterMode). + Obj(), + }, + }, + }, + // Fake dynamic client not generating name. That's why we have . + wantOut: "raycluster.ray.io/ created\n", + }, + "should create raycluster with --priority flag": { + args: func(tc *createCmdTestCase) []string { + return []string{ + "raycluster", + "--profile", "profile", + "--priority", "sample-priority", + } + }, + kjobctlObjs: []runtime.Object{ + wrappers.MakeRayClusterTemplate("ray-cluster-template", metav1.NamespaceDefault).Obj(), + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(*wrappers.MakeSupportedMode(v1alpha1.RayClusterMode, "ray-cluster-template").Obj()). + Obj(), + }, + kueueObjs: []runtime.Object{ + &kueue.WorkloadPriorityClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: "sample-priority", + }, + }, + }, + gvks: []schema.GroupVersionKind{{Group: "ray.io", Version: "v1", Kind: "RayCluster"}}, + wantLists: []runtime.Object{ + &rayv1.RayClusterList{ + TypeMeta: metav1.TypeMeta{Kind: "RayClusterList", APIVersion: "ray.io/v1"}, + Items: []rayv1.RayCluster{ + *wrappers.MakeRayCluster("", metav1.NamespaceDefault). + Priority("sample-priority"). + GenerateName("profile-raycluster-"). + Profile("profile"). + Mode(v1alpha1.RayClusterMode). + Obj(), + }, + }, + }, + // Fake dynamic client not generating name. That's why we have . + wantOut: "raycluster.ray.io/ created\n", + }, + "should create raycluster with --priority flag and skip workload priority class validation": { + args: func(tc *createCmdTestCase) []string { + return []string{ + "raycluster", + "--profile", "profile", + "--skip-priority-validation", + "--priority", "sample-priority", + } + }, + kjobctlObjs: []runtime.Object{ + wrappers.MakeRayClusterTemplate("ray-cluster-template", metav1.NamespaceDefault).Obj(), + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(*wrappers.MakeSupportedMode(v1alpha1.RayClusterMode, "ray-cluster-template").Obj()). + Obj(), + }, + gvks: []schema.GroupVersionKind{{Group: "ray.io", Version: "v1", Kind: "RayCluster"}}, + wantLists: []runtime.Object{ + &rayv1.RayClusterList{ + TypeMeta: metav1.TypeMeta{Kind: "RayClusterList", APIVersion: "ray.io/v1"}, + Items: []rayv1.RayCluster{ + *wrappers.MakeRayCluster("", metav1.NamespaceDefault). + Priority("sample-priority"). + GenerateName("profile-raycluster-"). + Profile("profile"). + Mode(v1alpha1.RayClusterMode). + Obj(), + }, + }, + }, + // Fake dynamic client not generating name. That's why we have . + wantOut: "raycluster.ray.io/ created\n", + }, + "shouldn't create slurm because slurm args must be specified": { + args: func(tc *createCmdTestCase) []string { + return []string{"slurm", "--profile", "profile"} + }, + wantErr: "requires at least 1 arg(s), only received 0", + }, + "shouldn't create slurm because script must be specified on slurm args": { + args: func(tc *createCmdTestCase) []string { + return []string{"slurm", "--profile", "profile", "./script.sh"} + }, + wantErr: "unknown command \"./script.sh\" for \"create slurm\"", + }, + "shouldn't create slurm because script must be specified": { + args: func(tc *createCmdTestCase) []string { + return []string{"slurm", "--profile", "profile", "--", "--array", "0-5"} + }, + wantErr: "must specify script", + }, + "shouldn't create slurm because script only one script must be specified": { + args: func(tc *createCmdTestCase) []string { + return []string{"slurm", "--profile", "profile", "--", "./script.sh", "./script.sh"} + }, + wantErr: "must specify only one script", + }, + "should create slurm": { + beforeTest: beforeSlurmTest, + args: func(tc *createCmdTestCase) []string { + return []string{"slurm", "--profile", "profile", "--", tc.tempFile} + }, + kjobctlObjs: []runtime.Object{ + wrappers.MakeJobTemplate("slurm-job-template", metav1.NamespaceDefault). + WithContainer(*wrappers.MakeContainer("c1", "bash:4.4").Obj()). + Obj(), + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(*wrappers.MakeSupportedMode(v1alpha1.SlurmMode, "slurm-job-template").Obj()). + Obj(), + }, + gvks: []schema.GroupVersionKind{ + {Group: "batch", Version: "v1", Kind: "Job"}, + {Group: "", Version: "v1", Kind: "ConfigMap"}, + {Group: "", Version: "v1", Kind: "Service"}, + }, + wantLists: []runtime.Object{ + &batchv1.JobList{ + TypeMeta: metav1.TypeMeta{Kind: "JobList", APIVersion: "batch/v1"}, + Items: []batchv1.Job{ + *wrappers.MakeJob("profile-slurm", metav1.NamespaceDefault). + Completions(1). + CompletionMode(batchv1.IndexedCompletion). + Profile("profile"). + Mode(v1alpha1.SlurmMode). + Subdomain("profile-slurm"). + WithInitContainer(*wrappers.MakeContainer("slurm-init-env", "registry.k8s.io/busybox:1.27.2"). + Command("sh", "/slurm/scripts/init-entrypoint.sh"). + WithVolumeMount(corev1.VolumeMount{Name: "slurm-scripts", MountPath: "/slurm/scripts"}). + WithVolumeMount(corev1.VolumeMount{Name: "slurm-env", MountPath: "/slurm/env"}). + WithEnvVar(corev1.EnvVar{Name: "POD_IP", ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{FieldPath: "status.podIP"}, + }}). + Obj()). + WithContainer(*wrappers.MakeContainer("c1", "bash:4.4"). + Command("bash", "/slurm/scripts/entrypoint.sh"). + WithVolumeMount(corev1.VolumeMount{Name: "slurm-scripts", MountPath: "/slurm/scripts"}). + WithVolumeMount(corev1.VolumeMount{Name: "slurm-env", MountPath: "/slurm/env"}). + Obj()). + WithVolume(corev1.Volume{ + Name: "slurm-scripts", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{Name: "profile-slurm"}, + Items: []corev1.KeyToPath{ + {Key: "init-entrypoint.sh", Path: "init-entrypoint.sh"}, + {Key: "entrypoint.sh", Path: "entrypoint.sh"}, + {Key: "script", Path: "script", Mode: ptr.To[int32](0755)}, + }, + }, + }, + }). + WithVolume(corev1.Volume{ + Name: "slurm-env", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }). + WithEnvVar(corev1.EnvVar{Name: constants.EnvVarNameUserID, Value: userID}). + WithEnvVar(corev1.EnvVar{Name: constants.EnvVarTaskName, Value: "default_profile"}). + WithEnvVar(corev1.EnvVar{ + Name: constants.EnvVarTaskID, + Value: fmt.Sprintf("%s_%s_default_profile", userID, testStartTime.Format(time.RFC3339)), + }). + WithEnvVar(corev1.EnvVar{Name: "PROFILE", Value: "default_profile"}). + WithEnvVar(corev1.EnvVar{Name: "TIMESTAMP", Value: testStartTime.Format(time.RFC3339)}). + WithEnvVar(corev1.EnvVar{Name: "JOB_CONTAINER_INDEX", Value: "0"}). + Obj(), + }, + }, + &corev1.ConfigMapList{ + TypeMeta: metav1.TypeMeta{Kind: "ConfigMapList", APIVersion: "v1"}, + Items: []corev1.ConfigMap{ + *wrappers.MakeConfigMap("profile-slurm", metav1.NamespaceDefault). + WithOwnerReference(metav1.OwnerReference{ + Name: "profile-slurm", + APIVersion: "batch/v1", + Kind: "Job", + }). + Profile("profile"). + Mode(v1alpha1.SlurmMode). + Data(map[string]string{ + "script": "#!/bin/bash\nsleep 300'", + "init-entrypoint.sh": `#!/bin/sh + +set -o errexit +set -o nounset +set -o pipefail +set -x + +# External variables +# JOB_COMPLETION_INDEX - completion index of the job. +# POD_IP - current pod IP + +array_indexes="0" +container_indexes=$(echo "$array_indexes" | awk -F';' -v idx="$JOB_COMPLETION_INDEX" '{print $((idx + 1))}') + +for i in $(seq 0 1) +do + container_index=$(echo "$container_indexes" | awk -F',' -v idx="$i" '{print $((idx + 1))}') + + if [ -z "$container_index" ]; then + break + fi + + mkdir -p /slurm/env/$i + + + cat << EOF > /slurm/env/$i/slurm.env +SLURM_ARRAY_JOB_ID=1 +SLURM_ARRAY_TASK_COUNT=1 +SLURM_ARRAY_TASK_MAX=0 +SLURM_ARRAY_TASK_MIN=0 +SLURM_TASKS_PER_NODE=1 +SLURM_CPUS_PER_TASK= +SLURM_CPUS_ON_NODE= +SLURM_JOB_CPUS_PER_NODE= +SLURM_CPUS_PER_GPU= +SLURM_MEM_PER_CPU= +SLURM_MEM_PER_GPU= +SLURM_MEM_PER_NODE= +SLURM_GPUS= +SLURM_NTASKS=1 +SLURM_NTASKS_PER_NODE=1 +SLURM_NPROCS=1 +SLURM_NNODES=1 +SLURM_SUBMIT_DIR=/slurm/scripts +SLURM_SUBMIT_HOST=$HOSTNAME +SLURM_JOB_NODELIST=profile-slurm-0.profile-slurm +SLURM_JOB_FIRST_NODE=profile-slurm-0.profile-slurm +SLURM_JOB_ID=$(expr $JOB_COMPLETION_INDEX \* 1 + $i + 1) +SLURM_JOBID=$(expr $JOB_COMPLETION_INDEX \* 1 + $i + 1) +SLURM_ARRAY_TASK_ID=$container_index +SLURM_JOB_FIRST_NODE_IP=${SLURM_JOB_FIRST_NODE_IP:-""} +EOF + +done +`, + "entrypoint.sh": `#!/usr/local/bin/bash + +set -o errexit +set -o nounset +set -o pipefail + +# External variables +# JOB_CONTAINER_INDEX - container index in the container template. + +if [ ! -d "/slurm/env/$JOB_CONTAINER_INDEX" ]; then + exit 0 +fi + +SBATCH_JOB_NAME= + +export $(cat /slurm/env/$JOB_CONTAINER_INDEX/slurm.env | xargs) + +/slurm/scripts/script +`, + }). + Obj(), + }, + }, + &corev1.ServiceList{ + TypeMeta: metav1.TypeMeta{Kind: "ServiceList", APIVersion: "v1"}, + Items: []corev1.Service{ + *wrappers.MakeService("profile-slurm", metav1.NamespaceDefault). + Profile("profile"). + Mode(v1alpha1.SlurmMode). + ClusterIP("None"). + Selector("job-name", "profile-slurm"). + WithOwnerReference(metav1.OwnerReference{ + Name: "profile-slurm", + APIVersion: "batch/v1", + Kind: "Job", + }). + Obj(), + }, + }, + }, + cmpopts: []cmp.Option{ + cmpopts.AcyclicTransformer("RemoveGeneratedNameSuffixInString", func(val string) string { + return regexp.MustCompile("(profile-slurm)(-.{5})").ReplaceAllString(val, "$1") + }), + cmpopts.AcyclicTransformer("RemoveGeneratedNameSuffixInMap", func(m map[string]string) map[string]string { + for key, val := range m { + m[key] = regexp.MustCompile("(profile-slurm)(-.{5})").ReplaceAllString(val, "$1") + } + return m + }), + }, + wantOutPattern: `job\.batch\/.+ created\\nconfigmap\/.+ created`, + }, + "should create slurm with flags": { + beforeTest: beforeSlurmTest, + args: func(tc *createCmdTestCase) []string { + return []string{ + "slurm", + "--profile", "profile", + "--localqueue", "lq1", + "--init-image", "bash:latest", + "--first-node-ip", + "--first-node-ip-timeout", "29s", + "--", + "--array", "0-25", + "--nodes", "2", + "--ntasks", "3", + "--input", "\\\\/home/%u/%x/stderr%%-%A-%a-%j-%N-%n-%t.out", + "--output", "/home/%u/%x/stdout%%-%A-%a-%j-%N-%n-%t.out", + "--error", "/home/%u/%x/stderr%%-%A-%a-%j-%N-%n-%t.out", + "--job-name", "job-name", + "--partition", "lq1", + "--chdir", "/mydir", + "--cpus-per-task", "2", + tc.tempFile, + } + }, + kjobctlObjs: []runtime.Object{ + wrappers.MakeJobTemplate("slurm-job-template", metav1.NamespaceDefault). + WithContainer(*wrappers.MakeContainer("c1", "bash:4.4").Obj()). + WithContainer(*wrappers.MakeContainer("c2", "bash:4.4").Obj()). + Obj(), + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(*wrappers.MakeSupportedMode(v1alpha1.SlurmMode, "slurm-job-template").Obj()). + Obj(), + }, + kueueObjs: []runtime.Object{ + &kueue.LocalQueue{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: metav1.NamespaceDefault, + Name: "lq1", + }, + }, + }, + gvks: []schema.GroupVersionKind{ + {Group: "batch", Version: "v1", Kind: "Job"}, + {Group: "", Version: "v1", Kind: "ConfigMap"}, + {Group: "", Version: "v1", Kind: "Service"}, + }, + wantLists: []runtime.Object{ + &batchv1.JobList{ + TypeMeta: metav1.TypeMeta{Kind: "JobList", APIVersion: "batch/v1"}, + Items: []batchv1.Job{ + *wrappers.MakeJob("profile-slurm", metav1.NamespaceDefault). + Parallelism(2). + Completions(9). + CompletionMode(batchv1.IndexedCompletion). + Profile("profile"). + Mode(v1alpha1.SlurmMode). + LocalQueue("lq1"). + Subdomain("profile-slurm"). + WithInitContainer(*wrappers.MakeContainer("slurm-init-env", "bash:latest"). + Command("sh", "/slurm/scripts/init-entrypoint.sh"). + WithVolumeMount(corev1.VolumeMount{Name: "slurm-scripts", MountPath: "/slurm/scripts"}). + WithVolumeMount(corev1.VolumeMount{Name: "slurm-env", MountPath: "/slurm/env"}). + WithEnvVar(corev1.EnvVar{Name: "POD_IP", ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{FieldPath: "status.podIP"}, + }}). + Obj()). + WithContainer(*wrappers.MakeContainer("c1-0", "bash:4.4"). + Command("bash", "/slurm/scripts/entrypoint.sh"). + WithVolumeMount(corev1.VolumeMount{Name: "slurm-scripts", MountPath: "/slurm/scripts"}). + WithVolumeMount(corev1.VolumeMount{Name: "slurm-env", MountPath: "/slurm/env"}). + WithRequest(corev1.ResourceCPU, resource.MustParse("2")). + Obj()). + WithContainer(*wrappers.MakeContainer("c2", "bash:4.4"). + Command("bash", "/slurm/scripts/entrypoint.sh"). + WithVolumeMount(corev1.VolumeMount{Name: "slurm-scripts", MountPath: "/slurm/scripts"}). + WithVolumeMount(corev1.VolumeMount{Name: "slurm-env", MountPath: "/slurm/env"}). + WithRequest(corev1.ResourceCPU, resource.MustParse("2")). + Obj()). + WithContainer(*wrappers.MakeContainer("c1-1", "bash:4.4"). + Command("bash", "/slurm/scripts/entrypoint.sh"). + WithVolumeMount(corev1.VolumeMount{Name: "slurm-scripts", MountPath: "/slurm/scripts"}). + WithVolumeMount(corev1.VolumeMount{Name: "slurm-env", MountPath: "/slurm/env"}). + WithRequest(corev1.ResourceCPU, resource.MustParse("2")). + Obj()). + WithContainer(*wrappers.MakeContainer("c1-2", "bash:4.4"). + Command("bash", "/slurm/scripts/entrypoint.sh"). + WithVolumeMount(corev1.VolumeMount{Name: "slurm-scripts", MountPath: "/slurm/scripts"}). + WithVolumeMount(corev1.VolumeMount{Name: "slurm-env", MountPath: "/slurm/env"}). + WithRequest(corev1.ResourceCPU, resource.MustParse("2")). + Obj()). + WithVolume(corev1.Volume{ + Name: "slurm-scripts", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{Name: "profile-slurm"}, + Items: []corev1.KeyToPath{ + {Key: "init-entrypoint.sh", Path: "init-entrypoint.sh"}, + {Key: "entrypoint.sh", Path: "entrypoint.sh"}, + {Key: "script", Path: "script", Mode: ptr.To[int32](0755)}, + }, + }, + }, + }). + WithVolume(corev1.Volume{ + Name: "slurm-env", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }). + WithEnvVar(corev1.EnvVar{Name: constants.EnvVarNameUserID, Value: userID}). + WithEnvVar(corev1.EnvVar{Name: constants.EnvVarTaskName, Value: "default_profile"}). + WithEnvVar(corev1.EnvVar{ + Name: constants.EnvVarTaskID, + Value: fmt.Sprintf("%s_%s_default_profile", userID, testStartTime.Format(time.RFC3339)), + }). + WithEnvVar(corev1.EnvVar{Name: "PROFILE", Value: "default_profile"}). + WithEnvVar(corev1.EnvVar{Name: "TIMESTAMP", Value: testStartTime.Format(time.RFC3339)}). + WithEnvVarIndexValue("JOB_CONTAINER_INDEX"). + Obj(), + }, + }, + &corev1.ConfigMapList{ + TypeMeta: metav1.TypeMeta{Kind: "ConfigMapList", APIVersion: "v1"}, + Items: []corev1.ConfigMap{ + *wrappers.MakeConfigMap("profile-slurm", metav1.NamespaceDefault). + WithOwnerReference(metav1.OwnerReference{ + Name: "profile-slurm", + APIVersion: "batch/v1", + Kind: "Job", + }). + Profile("profile"). + Mode(v1alpha1.SlurmMode). + Data(map[string]string{ + "script": "#!/bin/bash\nsleep 300'", + "init-entrypoint.sh": `#!/bin/sh + +set -o errexit +set -o nounset +set -o pipefail +set -x + +# External variables +# JOB_COMPLETION_INDEX - completion index of the job. +# POD_IP - current pod IP + +array_indexes="0,1,2;3,4,5;6,7,8;9,10,11;12,13,14;15,16,17;18,19,20;21,22,23;24,25" +container_indexes=$(echo "$array_indexes" | awk -F';' -v idx="$JOB_COMPLETION_INDEX" '{print $((idx + 1))}') + +for i in $(seq 0 3) +do + container_index=$(echo "$container_indexes" | awk -F',' -v idx="$i" '{print $((idx + 1))}') + + if [ -z "$container_index" ]; then + break + fi + + mkdir -p /slurm/env/$i + + + if [[ "$JOB_COMPLETION_INDEX" -eq 0 ]]; then + SLURM_JOB_FIRST_NODE_IP=${POD_IP} + else + timeout=29 + start_time=$(date +%s) + while true; do + ip=$(nslookup "profile-slurm-r8njg-0.profile-slurm-r8njg" | grep "Address 1" | awk 'NR==2 {print $3}') || true + if [[ -n "$ip" ]]; then + SLURM_JOB_FIRST_NODE_IP=$ip + break + else + current_time=$(date +%s) + elapsed_time=$((current_time - start_time)) + if [ "$elapsed_time" -ge "$timeout" ]; then + echo "Timeout reached, IP address for the first node (profile-slurm-r8njg-0.profile-slurm-r8njg) not found." + break + fi + echo "IP Address for the first node (profile-slurm-r8njg-0.profile-slurm-r8njg) not found, retrying..." + sleep 1 + fi + done + fi + + cat << EOF > /slurm/env/$i/slurm.env +SLURM_ARRAY_JOB_ID=1 +SLURM_ARRAY_TASK_COUNT=26 +SLURM_ARRAY_TASK_MAX=25 +SLURM_ARRAY_TASK_MIN=0 +SLURM_TASKS_PER_NODE=3 +SLURM_CPUS_PER_TASK=2 +SLURM_CPUS_ON_NODE=8 +SLURM_JOB_CPUS_PER_NODE=8 +SLURM_CPUS_PER_GPU= +SLURM_MEM_PER_CPU= +SLURM_MEM_PER_GPU= +SLURM_MEM_PER_NODE= +SLURM_GPUS= +SLURM_NTASKS=3 +SLURM_NTASKS_PER_NODE=3 +SLURM_NPROCS=3 +SLURM_NNODES=2 +SLURM_SUBMIT_DIR=/slurm/scripts +SLURM_SUBMIT_HOST=$HOSTNAME +SLURM_JOB_NODELIST=profile-slurm-fpxnj-0.profile-slurm-fpxnj,profile-slurm-fpxnj-1.profile-slurm-fpxnj +SLURM_JOB_FIRST_NODE=profile-slurm-fpxnj-0.profile-slurm-fpxnj +SLURM_JOB_ID=$(expr $JOB_COMPLETION_INDEX \* 3 + $i + 1) +SLURM_JOBID=$(expr $JOB_COMPLETION_INDEX \* 3 + $i + 1) +SLURM_ARRAY_TASK_ID=$container_index +SLURM_JOB_FIRST_NODE_IP=${SLURM_JOB_FIRST_NODE_IP:-""} +EOF + +done +`, + "entrypoint.sh": `#!/usr/local/bin/bash + +set -o errexit +set -o nounset +set -o pipefail + +# External variables +# JOB_CONTAINER_INDEX - container index in the container template. + +if [ ! -d "/slurm/env/$JOB_CONTAINER_INDEX" ]; then + exit 0 +fi + +SBATCH_JOB_NAME=job-name + +export $(cat /slurm/env/$JOB_CONTAINER_INDEX/slurm.env | xargs)cd /mydir + +/slurm/scripts/script /home/${USER_ID}/${SBATCH_JOB_NAME}/stdout%-${SLURM_ARRAY_JOB_ID}-${SLURM_ARRAY_TASK_ID}-${SLURM_JOB_ID}-${HOSTNAME}-${JOB_COMPLETION_INDEX}-${SLURM_ARRAY_TASK_ID}.out 2>/home/${USER_ID}/${SBATCH_JOB_NAME}/stderr%-${SLURM_ARRAY_JOB_ID}-${SLURM_ARRAY_TASK_ID}-${SLURM_JOB_ID}-${HOSTNAME}-${JOB_COMPLETION_INDEX}-${SLURM_ARRAY_TASK_ID}.out +`, + }). + Obj(), + }, + }, + &corev1.ServiceList{ + TypeMeta: metav1.TypeMeta{Kind: "ServiceList", APIVersion: "v1"}, + Items: []corev1.Service{ + *wrappers.MakeService("profile-slurm", metav1.NamespaceDefault). + Profile("profile"). + Mode(v1alpha1.SlurmMode). + ClusterIP("None"). + Selector("job-name", "profile-slurm"). + WithOwnerReference(metav1.OwnerReference{ + Name: "profile-slurm", + APIVersion: "batch/v1", + Kind: "Job", + }). + Obj(), + }, + }, + }, + cmpopts: []cmp.Option{ + cmpopts.AcyclicTransformer("RemoveGeneratedNameSuffixInString", func(val string) string { + return regexp.MustCompile("(profile-slurm)(-.{5})").ReplaceAllString(val, "$1") + }), + cmpopts.AcyclicTransformer("RemoveGeneratedNameSuffixInMap", func(m map[string]string) map[string]string { + for key, val := range m { + m[key] = regexp.MustCompile("(profile-slurm)(-.{5})").ReplaceAllString(val, "$1") + } + return m + }), + }, + wantOutPattern: `job\.batch\/.+ created\\nconfigmap\/.+ created`, + }, + "should create slurm with --ntasks flag": { + beforeTest: beforeSlurmTest, + args: func(tc *createCmdTestCase) []string { + return []string{ + "slurm", + "--profile", "profile", + "--", + "--ntasks", "3", + tc.tempFile, + } + }, + kjobctlObjs: []runtime.Object{ + wrappers.MakeJobTemplate("slurm-job-template", metav1.NamespaceDefault). + WithContainer(*wrappers.MakeContainer("c1", "bash:4.4").Obj()). + Obj(), + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(*wrappers.MakeSupportedMode(v1alpha1.SlurmMode, "slurm-job-template").Obj()). + Obj(), + }, + gvks: []schema.GroupVersionKind{ + {Group: "batch", Version: "v1", Kind: "Job"}, + {Group: "", Version: "v1", Kind: "ConfigMap"}, + {Group: "", Version: "v1", Kind: "Service"}, + }, + wantLists: []runtime.Object{ + &batchv1.JobList{ + TypeMeta: metav1.TypeMeta{Kind: "JobList", APIVersion: "batch/v1"}, + Items: []batchv1.Job{ + *wrappers.MakeJob("", metav1.NamespaceDefault). + Completions(1). + CompletionMode(batchv1.IndexedCompletion). + Profile("profile"). + Mode(v1alpha1.SlurmMode). + Subdomain("profile-slurm"). + WithContainer(*wrappers.MakeContainer("c1-0", "bash:4.4"). + Command("bash", "/slurm/scripts/entrypoint.sh"). + Obj()). + WithContainer(*wrappers.MakeContainer("c1-1", "bash:4.4"). + Command("bash", "/slurm/scripts/entrypoint.sh"). + Obj()). + WithContainer(*wrappers.MakeContainer("c1-2", "bash:4.4"). + Command("bash", "/slurm/scripts/entrypoint.sh"). + Obj()). + Obj(), + }, + }, + &corev1.ConfigMapList{}, + &corev1.ServiceList{}, + }, + cmpopts: []cmp.Option{ + cmpopts.IgnoreFields(corev1.LocalObjectReference{}, "Name"), + cmpopts.IgnoreFields(metav1.ObjectMeta{}, "Name"), + cmpopts.IgnoreFields(metav1.OwnerReference{}, "Name"), + cmpopts.IgnoreFields(corev1.PodSpec{}, "InitContainers", "Subdomain"), + cmpopts.IgnoreTypes([]corev1.EnvVar{}), + cmpopts.IgnoreTypes([]corev1.Volume{}), + cmpopts.IgnoreTypes([]corev1.VolumeMount{}), + cmpopts.IgnoreTypes(corev1.ConfigMapList{}), + cmpopts.IgnoreTypes(corev1.ServiceList{}), + }, + wantOutPattern: `job\.batch\/.+ created\\nconfigmap\/.+ created`, + }, + "should divide --mem exactly across containers": { + beforeTest: beforeSlurmTest, + args: func(tc *createCmdTestCase) []string { + return []string{ + "slurm", + "--profile", "profile", + "--", + "--mem", "2G", + tc.tempFile, + } + }, + kjobctlObjs: []runtime.Object{ + wrappers.MakeJobTemplate("slurm-job-template", metav1.NamespaceDefault). + WithContainer(*wrappers.MakeContainer("c1", "bash:4.4").Obj()). + WithContainer(*wrappers.MakeContainer("c2", "bash:4.4").Obj()). + Obj(), + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(*wrappers.MakeSupportedMode(v1alpha1.SlurmMode, "slurm-job-template").Obj()). + Obj(), + }, + gvks: []schema.GroupVersionKind{ + {Group: "batch", Version: "v1", Kind: "Job"}, + {Group: "", Version: "v1", Kind: "ConfigMap"}, + {Group: "", Version: "v1", Kind: "Service"}, + }, + wantLists: []runtime.Object{ + &batchv1.JobList{ + TypeMeta: metav1.TypeMeta{Kind: "JobList", APIVersion: "batch/v1"}, + Items: []batchv1.Job{ + *wrappers.MakeJob("", metav1.NamespaceDefault). + Completions(1). + CompletionMode(batchv1.IndexedCompletion). + Profile("profile"). + Mode(v1alpha1.SlurmMode). + WithContainer(*wrappers.MakeContainer("c1", "bash:4.4"). + Command("bash", "/slurm/scripts/entrypoint.sh"). + WithResources(corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + corev1.ResourceMemory: resource.MustParse("1G"), + }, + }). + Obj()). + WithContainer(*wrappers.MakeContainer("c2", "bash:4.4"). + Command("bash", "/slurm/scripts/entrypoint.sh"). + WithResources(corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + corev1.ResourceMemory: resource.MustParse("1G"), + }, + }). + Obj()). + Obj(), + }, + }, + &corev1.ConfigMapList{}, + &corev1.ServiceList{}, + }, + cmpopts: []cmp.Option{ + cmpopts.IgnoreFields(corev1.LocalObjectReference{}, "Name"), + cmpopts.IgnoreFields(metav1.ObjectMeta{}, "Name"), + cmpopts.IgnoreFields(metav1.OwnerReference{}, "Name"), + cmpopts.IgnoreFields(corev1.PodSpec{}, "InitContainers", "Subdomain"), + cmpopts.IgnoreTypes([]corev1.EnvVar{}), + cmpopts.IgnoreTypes([]corev1.Volume{}), + cmpopts.IgnoreTypes([]corev1.VolumeMount{}), + cmpopts.IgnoreTypes(corev1.ConfigMapList{}), + cmpopts.IgnoreTypes(corev1.ServiceList{}), + }, + wantOutPattern: `job\.batch\/.+ created\\nconfigmap\/.+ created`, + }, + "should handle non-exact --mem division across containers": { + beforeTest: beforeSlurmTest, + args: func(tc *createCmdTestCase) []string { + return []string{ + "slurm", + "--profile", "profile", + "--", + "--mem", "1G", + tc.tempFile, + } + }, + kjobctlObjs: []runtime.Object{ + wrappers.MakeJobTemplate("slurm-job-template", metav1.NamespaceDefault). + WithContainer(*wrappers.MakeContainer("c1", "bash:4.4").Obj()). + WithContainer(*wrappers.MakeContainer("c2", "bash:4.4").Obj()). + Obj(), + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(*wrappers.MakeSupportedMode(v1alpha1.SlurmMode, "slurm-job-template").Obj()). + Obj(), + }, + gvks: []schema.GroupVersionKind{ + {Group: "batch", Version: "v1", Kind: "Job"}, + {Group: "", Version: "v1", Kind: "ConfigMap"}, + {Group: "", Version: "v1", Kind: "Service"}, + }, + wantLists: []runtime.Object{ + &batchv1.JobList{ + TypeMeta: metav1.TypeMeta{Kind: "JobList", APIVersion: "batch/v1"}, + Items: []batchv1.Job{ + *wrappers.MakeJob("", metav1.NamespaceDefault). + Completions(1). + CompletionMode(batchv1.IndexedCompletion). + Profile("profile"). + Mode(v1alpha1.SlurmMode). + WithContainer(*wrappers.MakeContainer("c1", "bash:4.4"). + Command("bash", "/slurm/scripts/entrypoint.sh"). + WithResources(corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + corev1.ResourceMemory: resource.MustParse("500M"), + }, + }). + Obj()). + WithContainer(*wrappers.MakeContainer("c2", "bash:4.4"). + Command("bash", "/slurm/scripts/entrypoint.sh"). + WithResources(corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + corev1.ResourceMemory: resource.MustParse("500M"), + }, + }). + Obj()). + Obj(), + }, + }, + &corev1.ConfigMapList{}, + &corev1.ServiceList{}, + }, + cmpopts: []cmp.Option{ + cmpopts.IgnoreFields(corev1.LocalObjectReference{}, "Name"), + cmpopts.IgnoreFields(metav1.ObjectMeta{}, "Name"), + cmpopts.IgnoreFields(metav1.OwnerReference{}, "Name"), + cmpopts.IgnoreFields(corev1.PodSpec{}, "InitContainers", "Subdomain"), + cmpopts.IgnoreTypes([]corev1.EnvVar{}), + cmpopts.IgnoreTypes([]corev1.Volume{}), + cmpopts.IgnoreTypes([]corev1.VolumeMount{}), + cmpopts.IgnoreTypes(corev1.ConfigMapList{}), + cmpopts.IgnoreTypes(corev1.ServiceList{}), + }, + wantOutPattern: `job\.batch\/.+ created\\nconfigmap\/.+ created`, + }, + "should create slurm with --mem-per-cpu flag": { + beforeTest: beforeSlurmTest, + args: func(tc *createCmdTestCase) []string { + return []string{ + "slurm", + "--profile", "profile", + "--", + "--cpus-per-task", "2", + "--mem-per-cpu", "500M", + tc.tempFile, + } + }, + kjobctlObjs: []runtime.Object{ + wrappers.MakeJobTemplate("slurm-job-template", metav1.NamespaceDefault). + WithContainer(*wrappers.MakeContainer("c1", "bash:4.4").Obj()). + WithContainer(*wrappers.MakeContainer("c2", "bash:4.4").Obj()). + Obj(), + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(*wrappers.MakeSupportedMode(v1alpha1.SlurmMode, "slurm-job-template").Obj()). + Obj(), + }, + gvks: []schema.GroupVersionKind{ + {Group: "batch", Version: "v1", Kind: "Job"}, + {Group: "", Version: "v1", Kind: "ConfigMap"}, + {Group: "", Version: "v1", Kind: "Service"}, + }, + wantLists: []runtime.Object{ + &batchv1.JobList{ + TypeMeta: metav1.TypeMeta{Kind: "JobList", APIVersion: "batch/v1"}, + Items: []batchv1.Job{ + *wrappers.MakeJob("", metav1.NamespaceDefault). + Completions(1). + CompletionMode(batchv1.IndexedCompletion). + Profile("profile"). + Mode(v1alpha1.SlurmMode). + WithContainer(*wrappers.MakeContainer("c1", "bash:4.4"). + Command("bash", "/slurm/scripts/entrypoint.sh"). + WithResources(corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("2"), + corev1.ResourceMemory: resource.MustParse("1G"), + }, + }). + Obj()). + WithContainer(*wrappers.MakeContainer("c2", "bash:4.4"). + Command("bash", "/slurm/scripts/entrypoint.sh"). + WithResources(corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("2"), + corev1.ResourceMemory: resource.MustParse("1G"), + }, + }). + Obj()). + Obj(), + }, + }, + &corev1.ConfigMapList{}, + &corev1.ServiceList{}, + }, + cmpopts: []cmp.Option{ + cmpopts.IgnoreFields(corev1.LocalObjectReference{}, "Name"), + cmpopts.IgnoreFields(metav1.ObjectMeta{}, "Name"), + cmpopts.IgnoreFields(metav1.OwnerReference{}, "Name"), + cmpopts.IgnoreFields(corev1.PodSpec{}, "InitContainers", "Subdomain"), + cmpopts.IgnoreTypes([]corev1.EnvVar{}), + cmpopts.IgnoreTypes([]corev1.Volume{}), + cmpopts.IgnoreTypes([]corev1.VolumeMount{}), + cmpopts.IgnoreTypes(corev1.ConfigMapList{}), + cmpopts.IgnoreTypes(corev1.ServiceList{}), + }, + wantOutPattern: `job\.batch\/.+ created\\nconfigmap\/.+ created`, + }, + "shouldn't create slurm with --mem-per-cpu flag because --cpus-per-task flag not specified": { + beforeTest: beforeSlurmTest, + args: func(tc *createCmdTestCase) []string { + return []string{ + "slurm", + "--profile", "profile", + "--", + "--mem-per-cpu", "500M", + tc.tempFile, + } + }, + kjobctlObjs: []runtime.Object{ + wrappers.MakeJobTemplate("slurm-job-template", metav1.NamespaceDefault). + WithContainer(*wrappers.MakeContainer("c1", "bash:4.4").Obj()). + WithContainer(*wrappers.MakeContainer("c2", "bash:4.4").Obj()). + Obj(), + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(*wrappers.MakeSupportedMode(v1alpha1.SlurmMode, "slurm-job-template").Obj()). + Obj(), + }, + wantErr: "no cpus-per-task specified", + }, + "should create slurm with --priority flag": { + beforeTest: beforeSlurmTest, + args: func(tc *createCmdTestCase) []string { + return []string{ + "slurm", + "--profile", "profile", + "--priority", "sample-priority", + "--", + tc.tempFile, + } + }, + kjobctlObjs: []runtime.Object{ + wrappers.MakeJobTemplate("slurm-job-template", metav1.NamespaceDefault). + WithContainer(*wrappers.MakeContainer("c1", "bash:4.4").Obj()). + Obj(), + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(*wrappers.MakeSupportedMode(v1alpha1.SlurmMode, "slurm-job-template").Obj()). + Obj(), + }, + kueueObjs: []runtime.Object{ + &kueue.WorkloadPriorityClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: "sample-priority", + }, + }, + }, + gvks: []schema.GroupVersionKind{ + {Group: "batch", Version: "v1", Kind: "Job"}, + {Group: "", Version: "v1", Kind: "ConfigMap"}, + {Group: "", Version: "v1", Kind: "Service"}, + }, + wantLists: []runtime.Object{ + &batchv1.JobList{ + TypeMeta: metav1.TypeMeta{Kind: "JobList", APIVersion: "batch/v1"}, + Items: []batchv1.Job{ + *wrappers.MakeJob("", metav1.NamespaceDefault). + Priority("sample-priority"). + Completions(1). + CompletionMode(batchv1.IndexedCompletion). + Profile("profile"). + Mode(v1alpha1.SlurmMode). + WithContainer(*wrappers.MakeContainer("c1", "bash:4.4"). + Command("bash", "/slurm/scripts/entrypoint.sh"). + Obj()). + Obj(), + }, + }, + &corev1.ConfigMapList{}, + &corev1.ServiceList{}, + }, + cmpopts: []cmp.Option{ + cmpopts.IgnoreFields(corev1.LocalObjectReference{}, "Name"), + cmpopts.IgnoreFields(metav1.ObjectMeta{}, "Name"), + cmpopts.IgnoreFields(metav1.OwnerReference{}, "Name"), + cmpopts.IgnoreFields(corev1.PodSpec{}, "InitContainers", "Subdomain"), + cmpopts.IgnoreTypes([]corev1.EnvVar{}), + cmpopts.IgnoreTypes([]corev1.Volume{}), + cmpopts.IgnoreTypes([]corev1.VolumeMount{}), + cmpopts.IgnoreTypes(corev1.ConfigMapList{}), + cmpopts.IgnoreTypes(corev1.ServiceList{}), + }, + wantOutPattern: `job\.batch\/.+ created\\nconfigmap\/.+ created`, + }, + "should create slurm with --mem-per-gpu flag": { + beforeTest: beforeSlurmTest, + args: func(tc *createCmdTestCase) []string { + return []string{ + "slurm", + "--profile", "profile", + "--", + "--gpus-per-task", "volta:3,kepler:1", + "--mem-per-gpu", "500M", + tc.tempFile, + } + }, + kjobctlObjs: []runtime.Object{ + wrappers.MakeJobTemplate("slurm-job-template", metav1.NamespaceDefault). + WithContainer(*wrappers.MakeContainer("c1", "bash:4.4").Obj()). + WithContainer(*wrappers.MakeContainer("c2", "bash:4.4").Obj()). + Obj(), + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(*wrappers.MakeSupportedMode(v1alpha1.SlurmMode, "slurm-job-template").Obj()). + Obj(), + }, + gvks: []schema.GroupVersionKind{ + {Group: "batch", Version: "v1", Kind: "Job"}, + {Group: "", Version: "v1", Kind: "ConfigMap"}, + {Group: "", Version: "v1", Kind: "Service"}, + }, + wantLists: []runtime.Object{ + &batchv1.JobList{ + TypeMeta: metav1.TypeMeta{Kind: "JobList", APIVersion: "batch/v1"}, + Items: []batchv1.Job{ + *wrappers.MakeJob("", metav1.NamespaceDefault). + Completions(1). + CompletionMode(batchv1.IndexedCompletion). + Profile("profile"). + Mode(v1alpha1.SlurmMode). + WithContainer(*wrappers.MakeContainer("c1", "bash:4.4"). + Command("bash", "/slurm/scripts/entrypoint.sh"). + WithResources(corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceMemory: resource.MustParse("2G"), + "volta": resource.MustParse("3"), + "kepler": resource.MustParse("1"), + }, + }). + Obj()). + WithContainer(*wrappers.MakeContainer("c2", "bash:4.4"). + Command("bash", "/slurm/scripts/entrypoint.sh"). + WithResources(corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceMemory: resource.MustParse("2G"), + "volta": resource.MustParse("3"), + "kepler": resource.MustParse("1"), + }, + }). + Obj()). + Obj(), + }, + }, + &corev1.ConfigMapList{}, + &corev1.ServiceList{}, + }, + cmpopts: []cmp.Option{ + cmpopts.IgnoreFields(corev1.LocalObjectReference{}, "Name"), + cmpopts.IgnoreFields(metav1.ObjectMeta{}, "Name"), + cmpopts.IgnoreFields(metav1.OwnerReference{}, "Name"), + cmpopts.IgnoreFields(corev1.PodSpec{}, "InitContainers", "Subdomain"), + cmpopts.IgnoreTypes([]corev1.EnvVar{}), + cmpopts.IgnoreTypes([]corev1.Volume{}), + cmpopts.IgnoreTypes([]corev1.VolumeMount{}), + cmpopts.IgnoreTypes(corev1.ConfigMapList{}), + cmpopts.IgnoreTypes(corev1.ServiceList{}), + }, + wantOutPattern: `job\.batch\/.+ created\\nconfigmap\/.+ created`, + }, + "shouldn't create slurm with --mem-per-gpu flag because --gpus-per-task flag not specified": { + beforeTest: beforeSlurmTest, + args: func(tc *createCmdTestCase) []string { + return []string{ + "slurm", + "--profile", "profile", + "--", + "--mem-per-gpu", "500M", + tc.tempFile, + } + }, + kjobctlObjs: []runtime.Object{ + wrappers.MakeJobTemplate("slurm-job-template", metav1.NamespaceDefault). + WithContainer(*wrappers.MakeContainer("c1", "bash:4.4").Obj()). + WithContainer(*wrappers.MakeContainer("c2", "bash:4.4").Obj()). + Obj(), + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(*wrappers.MakeSupportedMode(v1alpha1.SlurmMode, "slurm-job-template").Obj()). + Obj(), + }, + wantErr: "no gpus-per-task specified", + }, + "should create slurm with --priority flag and skip workload priority class validation": { + beforeTest: beforeSlurmTest, + args: func(tc *createCmdTestCase) []string { + return []string{ + "slurm", + "--profile", "profile", + "--priority", "sample-priority", + "--skip-priority-validation", + "--", + tc.tempFile, + } + }, + kjobctlObjs: []runtime.Object{ + wrappers.MakeJobTemplate("slurm-job-template", metav1.NamespaceDefault). + WithContainer(*wrappers.MakeContainer("c1", "bash:4.4").Obj()). + Obj(), + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(*wrappers.MakeSupportedMode(v1alpha1.SlurmMode, "slurm-job-template").Obj()). + Obj(), + }, + gvks: []schema.GroupVersionKind{ + {Group: "batch", Version: "v1", Kind: "Job"}, + {Group: "", Version: "v1", Kind: "ConfigMap"}, + {Group: "", Version: "v1", Kind: "Service"}, + }, + wantLists: []runtime.Object{ + &batchv1.JobList{ + TypeMeta: metav1.TypeMeta{Kind: "JobList", APIVersion: "batch/v1"}, + Items: []batchv1.Job{ + *wrappers.MakeJob("", metav1.NamespaceDefault). + Priority("sample-priority"). + Completions(1). + CompletionMode(batchv1.IndexedCompletion). + Profile("profile"). + Mode(v1alpha1.SlurmMode). + Subdomain("profile-slurm"). + WithContainer(*wrappers.MakeContainer("c1", "bash:4.4"). + Command("bash", "/slurm/scripts/entrypoint.sh"). + Obj()). + Obj(), + }, + }, + &corev1.ConfigMapList{}, + &corev1.ServiceList{}, + }, + cmpopts: []cmp.Option{ + cmpopts.IgnoreFields(corev1.LocalObjectReference{}, "Name"), + cmpopts.IgnoreFields(metav1.ObjectMeta{}, "Name"), + cmpopts.IgnoreFields(metav1.OwnerReference{}, "Name"), + cmpopts.IgnoreFields(corev1.PodSpec{}, "InitContainers", "Subdomain"), + cmpopts.IgnoreTypes([]corev1.EnvVar{}), + cmpopts.IgnoreTypes([]corev1.Volume{}), + cmpopts.IgnoreTypes([]corev1.VolumeMount{}), + cmpopts.IgnoreTypes(corev1.ConfigMapList{}), + cmpopts.IgnoreTypes(corev1.ServiceList{}), + }, + wantOutPattern: `job\.batch\/.+ created\\nconfigmap\/.+ created`, + }, + "should create slurm with --time flag": { + beforeTest: beforeSlurmTest, + args: func(tc *createCmdTestCase) []string { + return []string{ + "slurm", + "--profile", "profile", + "--", + tc.tempFile, + "--time", "1", + } + }, + kjobctlObjs: []runtime.Object{ + wrappers.MakeJobTemplate("slurm-job-template", metav1.NamespaceDefault). + WithContainer(*wrappers.MakeContainer("c1", "bash:4.4").Obj()). + Obj(), + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(*wrappers.MakeSupportedMode(v1alpha1.SlurmMode, "slurm-job-template").Obj()). + Obj(), + }, + gvks: []schema.GroupVersionKind{ + {Group: "batch", Version: "v1", Kind: "Job"}, + {Group: "", Version: "v1", Kind: "ConfigMap"}, + {Group: "", Version: "v1", Kind: "Service"}, + }, + wantLists: []runtime.Object{ + &batchv1.JobList{ + TypeMeta: metav1.TypeMeta{Kind: "JobList", APIVersion: "batch/v1"}, + Items: []batchv1.Job{ + *wrappers.MakeJob("", metav1.NamespaceDefault). + MaxExecTimeSecondsLabel("60"). + Completions(1). + CompletionMode(batchv1.IndexedCompletion). + Profile("profile"). + Mode(v1alpha1.SlurmMode). + Subdomain("profile-slurm"). + WithContainer(*wrappers.MakeContainer("c1", "bash:4.4"). + Command("bash", "/slurm/scripts/entrypoint.sh"). + Obj()). + Obj(), + }, + }, + &corev1.ConfigMapList{}, + &corev1.ServiceList{}, + }, + cmpopts: []cmp.Option{ + cmpopts.IgnoreFields(corev1.LocalObjectReference{}, "Name"), + cmpopts.IgnoreFields(metav1.ObjectMeta{}, "Name"), + cmpopts.IgnoreFields(metav1.OwnerReference{}, "Name"), + cmpopts.IgnoreFields(corev1.PodSpec{}, "InitContainers", "Subdomain"), + cmpopts.IgnoreTypes([]corev1.EnvVar{}), + cmpopts.IgnoreTypes([]corev1.Volume{}), + cmpopts.IgnoreTypes([]corev1.VolumeMount{}), + cmpopts.IgnoreTypes(corev1.ConfigMapList{}), + cmpopts.IgnoreTypes(corev1.ServiceList{}), + }, + wantOutPattern: `job\.batch\/.+ created\\nconfigmap\/.+ created`, + }, + "should create slurm with -t flag": { + beforeTest: beforeSlurmTest, + args: func(tc *createCmdTestCase) []string { + return []string{ + "slurm", + "--profile", "profile", + "--", + tc.tempFile, + "-t", "2-12:05:23", + } + }, + kjobctlObjs: []runtime.Object{ + wrappers.MakeJobTemplate("slurm-job-template", metav1.NamespaceDefault). + WithContainer(*wrappers.MakeContainer("c1", "bash:4.4").Obj()). + Obj(), + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(*wrappers.MakeSupportedMode(v1alpha1.SlurmMode, "slurm-job-template").Obj()). + Obj(), + }, + gvks: []schema.GroupVersionKind{ + {Group: "batch", Version: "v1", Kind: "Job"}, + {Group: "", Version: "v1", Kind: "ConfigMap"}, + {Group: "", Version: "v1", Kind: "Service"}, + }, + wantLists: []runtime.Object{ + &batchv1.JobList{ + TypeMeta: metav1.TypeMeta{Kind: "JobList", APIVersion: "batch/v1"}, + Items: []batchv1.Job{ + *wrappers.MakeJob("", metav1.NamespaceDefault). + MaxExecTimeSecondsLabel("216323"). + Completions(1). + CompletionMode(batchv1.IndexedCompletion). + Profile("profile"). + Mode(v1alpha1.SlurmMode). + Subdomain("profile-slurm"). + WithContainer(*wrappers.MakeContainer("c1", "bash:4.4"). + Command("bash", "/slurm/scripts/entrypoint.sh"). + Obj()). + Obj(), + }, + }, + &corev1.ConfigMapList{}, + &corev1.ServiceList{}, + }, + cmpopts: []cmp.Option{ + cmpopts.IgnoreFields(corev1.LocalObjectReference{}, "Name"), + cmpopts.IgnoreFields(metav1.ObjectMeta{}, "Name"), + cmpopts.IgnoreFields(metav1.OwnerReference{}, "Name"), + cmpopts.IgnoreFields(corev1.PodSpec{}, "InitContainers", "Subdomain"), + cmpopts.IgnoreTypes([]corev1.EnvVar{}), + cmpopts.IgnoreTypes([]corev1.Volume{}), + cmpopts.IgnoreTypes([]corev1.VolumeMount{}), + cmpopts.IgnoreTypes(corev1.ConfigMapList{}), + cmpopts.IgnoreTypes(corev1.ServiceList{}), + }, + wantOutPattern: `job\.batch\/.+ created\\nconfigmap\/.+ created`, + }, + "shouldn't create job with client dry run": { + args: func(tc *createCmdTestCase) []string { + return []string{"job", "--profile", "profile", "--dry-run", "client"} + }, + kjobctlObjs: []runtime.Object{ + wrappers.MakeJobTemplate("job-template", metav1.NamespaceDefault).Obj(), + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(*wrappers.MakeSupportedMode(v1alpha1.JobMode, "job-template").Obj()). + Obj(), + }, + gvks: []schema.GroupVersionKind{{Group: "batch", Version: "v1", Kind: "Job"}}, + wantLists: []runtime.Object{ + &batchv1.JobList{ + TypeMeta: metav1.TypeMeta{Kind: "JobList", APIVersion: "batch/v1"}, + Items: []batchv1.Job{}, + }, + }, + // Fake dynamic client not generating name. That's why we have . + wantOut: "job.batch/ created (client dry run)\n", + }, + } + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + if tc.beforeTest != nil { + tc.beforeTest(t, &tc) + } + + streams, _, out, outErr := genericiooptions.NewTestIOStreams() + + scheme := runtime.NewScheme() + utilruntime.Must(k8sscheme.AddToScheme(scheme)) + utilruntime.Must(rayv1.AddToScheme(scheme)) + + clientset := kjobctlfake.NewSimpleClientset(tc.kjobctlObjs...) + dynamicClient := fake.NewSimpleDynamicClient(scheme) + kueueClientset := kueuefake.NewSimpleClientset(tc.kueueObjs...) + restMapper := meta.NewDefaultRESTMapper([]schema.GroupVersion{}) + + for _, gvk := range tc.gvks { + restMapper.Add(gvk, meta.RESTScopeNamespace) + } + + tcg := cmdtesting.NewTestClientGetter(). + WithKjobctlClientset(clientset). + WithDynamicClient(dynamicClient). + WithKueueClientset(kueueClientset). + WithRESTMapper(restMapper) + if tc.ns != "" { + tcg.WithNamespace(tc.ns) + } + + cmd := NewCreateCmd(tcg, streams, clocktesting.NewFakeClock(testStartTime)) + cmd.SetOut(out) + cmd.SetErr(outErr) + cmd.SetArgs(tc.args(&tc)) + + gotErr := cmd.Execute() + + var gotErrStr string + if gotErr != nil { + gotErrStr = gotErr.Error() + } + + if diff := cmp.Diff(tc.wantErr, gotErrStr); diff != "" { + t.Errorf("Unexpected error (-want/+got)\n%s", diff) + } + + if gotErr != nil { + return + } + + gotOut := out.String() + if tc.wantOutPattern != "" { + gotOut = strings.ReplaceAll(gotOut, "\n", "\\n") + match, err := regexp.MatchString(tc.wantOutPattern, gotOut) + if err != nil { + t.Error(err) + return + } + if !match { + t.Errorf("Unexpected output. Not match pattern \"%s\":\n%s", tc.wantOutPattern, gotOut) + } + } else if diff := cmp.Diff(tc.wantOut, gotOut); diff != "" { + t.Errorf("Unexpected output (-want/+got)\n%s", diff) + } + + gotOutErr := outErr.String() + if diff := cmp.Diff(tc.wantOutErr, gotOutErr); diff != "" { + t.Errorf("Unexpected output (-want/+got)\n%s", diff) + } + + for index, gvk := range tc.gvks { + mapping, err := restMapper.RESTMapping(gvk.GroupKind(), gvk.Version) + if err != nil { + t.Error(err) + return + } + + unstructured, err := dynamicClient.Resource(mapping.Resource).Namespace(metav1.NamespaceDefault). + List(context.Background(), metav1.ListOptions{}) + if err != nil { + t.Error(err) + return + } + + gotList := tc.wantLists[index].DeepCopyObject() + + err = runtime.DefaultUnstructuredConverter.FromUnstructured(unstructured.UnstructuredContent(), gotList) + if err != nil { + t.Error(err) + return + } + + if job, ok := tc.wantLists[index].(*batchv1.JobList); ok && len(job.Items) > 0 { + if tc.tempFile != "" { + if job.Items[0].Annotations == nil { + job.Items[0].Annotations = make(map[string]string) + } + job.Items[0].Annotations[constants.ScriptAnnotation] = tc.tempFile + } + } + + if diff := cmp.Diff(tc.wantLists[index], gotList, tc.cmpopts...); diff != "" { + t.Errorf("Unexpected list for %s (-want/+got)\n%s", gvk.String(), diff) + } + } + }) + } +} + +func TestCreateOptionsRunInteractive(t *testing.T) { + testStartTime := time.Now() + userID := os.Getenv(constants.SystemEnvVarNameUser) + + testCases := map[string]struct { + options *CreateOptions + k8sObjs []runtime.Object + kjobctlObjs []runtime.Object + createMutation func(pod *corev1.Pod) + wantPodList *corev1.PodList + wantErr string + }{ + "success": { + options: &CreateOptions{ + Namespace: metav1.NamespaceDefault, + ProfileName: "profile", + ModeName: v1alpha1.InteractiveMode, + Attach: &fakeRemoteAttach{}, + AttachFunc: testAttachFunc, + }, + k8sObjs: []runtime.Object{ + wrappers.MakePodTemplate("pod-template", metav1.NamespaceDefault). + WithContainer(*wrappers.MakeContainer("c1", "sleep").Obj()). + Obj(), + }, + kjobctlObjs: []runtime.Object{ + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(*wrappers.MakeSupportedMode(v1alpha1.InteractiveMode, "pod-template").Obj()). + Obj(), + }, + createMutation: func(pod *corev1.Pod) { + pod.Status.Phase = corev1.PodRunning + }, + wantPodList: &corev1.PodList{ + Items: []corev1.Pod{ + *wrappers.MakePod("", metav1.NamespaceDefault). + GenerateName("profile-interactive-"). + Profile("profile"). + Mode(v1alpha1.InteractiveMode). + WithContainer(*wrappers.MakeContainer("c1", "sleep"). + TTY(). + Stdin(). + WithEnvVar(corev1.EnvVar{Name: constants.EnvVarNameUserID, Value: userID}). + WithEnvVar(corev1.EnvVar{Name: constants.EnvVarTaskName, Value: "default_profile"}). + WithEnvVar(corev1.EnvVar{ + Name: constants.EnvVarTaskID, + Value: fmt.Sprintf("%s_%s_default_profile", userID, testStartTime.Format(time.RFC3339)), + }). + WithEnvVar(corev1.EnvVar{Name: "PROFILE", Value: "default_profile"}). + WithEnvVar(corev1.EnvVar{Name: "TIMESTAMP", Value: testStartTime.Format(time.RFC3339)}). + Obj()). + Phase(corev1.PodRunning). + Obj(), + }, + }, + }, + "success with remove interactive pod": { + options: &CreateOptions{ + Namespace: metav1.NamespaceDefault, + ProfileName: "profile", + ModeName: v1alpha1.InteractiveMode, + RemoveInteractivePod: true, + Attach: &fakeRemoteAttach{}, + AttachFunc: testAttachFunc, + }, + k8sObjs: []runtime.Object{ + wrappers.MakePodTemplate("pod-template", metav1.NamespaceDefault). + WithContainer(*wrappers.MakeContainer("c1", "sleep").Obj()). + Obj(), + }, + kjobctlObjs: []runtime.Object{ + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(*wrappers.MakeSupportedMode(v1alpha1.InteractiveMode, "pod-template").Obj()). + Obj(), + }, + createMutation: func(pod *corev1.Pod) { + pod.Status.Phase = corev1.PodRunning + }, + wantPodList: &corev1.PodList{}, + }, + "success with dry-run client": { + options: &CreateOptions{ + Namespace: metav1.NamespaceDefault, + ProfileName: "profile", + ModeName: v1alpha1.InteractiveMode, + DryRunStrategy: cmdutil.DryRunClient, + Attach: &fakeRemoteAttach{}, + AttachFunc: testAttachFunc, + }, + k8sObjs: []runtime.Object{ + wrappers.MakePodTemplate("pod-template", metav1.NamespaceDefault). + WithContainer(*wrappers.MakeContainer("c1", "sleep").Obj()). + Obj(), + }, + kjobctlObjs: []runtime.Object{ + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(*wrappers.MakeSupportedMode(v1alpha1.InteractiveMode, "pod-template").Obj()). + Obj(), + }, + wantPodList: &corev1.PodList{}, + }, + "timeout waiting for pod": { + options: &CreateOptions{ + Namespace: metav1.NamespaceDefault, + ProfileName: "profile", + ModeName: v1alpha1.InteractiveMode, + Attach: &fakeRemoteAttach{}, + AttachFunc: testAttachFunc, + }, + k8sObjs: []runtime.Object{ + wrappers.MakePodTemplate("pod-template", metav1.NamespaceDefault). + WithContainer(*wrappers.MakeContainer("c1", "sleep").Obj()). + Obj(), + }, + kjobctlObjs: []runtime.Object{ + wrappers.MakeApplicationProfile("profile", metav1.NamespaceDefault). + WithSupportedMode(*wrappers.MakeSupportedMode(v1alpha1.InteractiveMode, "pod-template").Obj()). + Obj(), + }, + wantPodList: &corev1.PodList{ + Items: []corev1.Pod{ + *wrappers.MakePod("", metav1.NamespaceDefault). + GenerateName("profile-interactive-"). + Profile("profile"). + Mode(v1alpha1.InteractiveMode). + WithContainer(*wrappers.MakeContainer("c1", "sleep"). + TTY(). + Stdin(). + WithEnvVar(corev1.EnvVar{Name: constants.EnvVarNameUserID, Value: userID}). + WithEnvVar(corev1.EnvVar{Name: constants.EnvVarTaskName, Value: "default_profile"}). + WithEnvVar(corev1.EnvVar{ + Name: constants.EnvVarTaskID, + Value: fmt.Sprintf("%s_%s_default_profile", userID, testStartTime.Format(time.RFC3339)), + }). + WithEnvVar(corev1.EnvVar{Name: "PROFILE", Value: "default_profile"}). + WithEnvVar(corev1.EnvVar{Name: "TIMESTAMP", Value: testStartTime.Format(time.RFC3339)}). + Obj()). + Obj(), + }, + }, + wantErr: "context deadline exceeded", + }, + } + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + streams, _, out, outErr := genericiooptions.NewTestIOStreams() + tc.options.IOStreams = streams + tc.options.Out = out + tc.options.ErrOut = outErr + tc.options.PrintFlags = genericclioptions.NewPrintFlags("created").WithTypeSetter(k8sscheme.Scheme) + printer, err := tc.options.PrintFlags.ToPrinter() + if err != nil { + t.Fatal(err) + } + tc.options.PrintObj = printer.PrintObj + + k8sClientset := k8sfake.NewSimpleClientset(tc.k8sObjs...) + kjobctlClientset := kjobctlfake.NewSimpleClientset(tc.kjobctlObjs...) + dynamicClient := fake.NewSimpleDynamicClient(k8sscheme.Scheme) + restMapper := meta.NewDefaultRESTMapper([]schema.GroupVersion{}) + restMapper.Add(schema.GroupVersionKind{Group: "", Version: "v1", Kind: "Pod"}, meta.RESTScopeNamespace) + + dynamicClient.PrependReactor("create", "pods", func(action kubetesting.Action) (handled bool, ret runtime.Object, err error) { + createAction := action.(kubetesting.CreateAction) + + unstructuredObj := createAction.GetObject().(*unstructured.Unstructured) + unstructuredObj.SetName(unstructuredObj.GetGenerateName() + utilrand.String(5)) + + pod := &corev1.Pod{} + + if err = runtime.DefaultUnstructuredConverter.FromUnstructured(unstructuredObj.UnstructuredContent(), pod); err != nil { + return true, nil, err + } + + if tc.createMutation != nil { + tc.createMutation(pod) + } + + _, err = k8sClientset.CoreV1().Pods(pod.GetNamespace()).Create(context.Background(), pod, metav1.CreateOptions{}) + if err != nil { + return true, nil, err + } + + return true, unstructuredObj, err + }) + + tcg := cmdtesting.NewTestClientGetter(). + WithK8sClientset(k8sClientset). + WithKjobctlClientset(kjobctlClientset). + WithDynamicClient(dynamicClient). + WithRESTMapper(restMapper) + + gotErr := tc.options.Run(context.Background(), tcg, testStartTime) + + var gotErrStr string + if gotErr != nil { + gotErrStr = gotErr.Error() + } + + if diff := cmp.Diff(tc.wantErr, gotErrStr); diff != "" { + t.Errorf("Unexpected error (-want/+got)\n%s", diff) + } + + gotPodList, err := k8sClientset.CoreV1().Pods(metav1.NamespaceDefault).List(context.Background(), metav1.ListOptions{}) + if err != nil { + t.Fatal(err) + } + + defaultCmpOpts := []cmp.Option{cmpopts.IgnoreFields(metav1.ObjectMeta{}, "Name")} + if diff := cmp.Diff(tc.wantPodList, gotPodList, defaultCmpOpts...); diff != "" { + t.Errorf("Unexpected error (-want/+got)\n%s", diff) + } + }) + } +} + +type fakeRemoteAttach struct { + url *url.URL + err error +} + +func (f *fakeRemoteAttach) Attach(url *url.URL, _ *restclient.Config, _ io.Reader, _, _ io.Writer, _ bool, _ remotecommand.TerminalSizeQueue) error { + f.url = url + return f.err +} + +func testAttachFunc(o *CreateOptions, _ *corev1.Container, sizeQueue remotecommand.TerminalSizeQueue, _ *corev1.Pod) func() error { + return func() error { + u, err := url.Parse("http://kjobctl.test") + if err != nil { + return err + } + + return o.Attach.Attach(u, nil, nil, nil, nil, o.TTY, sizeQueue) + } +} diff --git a/pkg/cmd/create/helpers.go b/pkg/cmd/create/helpers.go new file mode 100644 index 0000000..c59b582 --- /dev/null +++ b/pkg/cmd/create/helpers.go @@ -0,0 +1,68 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package create + +import ( + "context" + "time" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/client-go/clientset/versioned/scheme" +) + +// setKubernetesDefaults sets default values on the provided client config for accessing the +// Kubernetes API or returns an error if any of the defaults are impossible or invalid. +// copied from https://github.com/kubernetes/kubectl/blob/a6de79e3d40575480caa71a8e5efffeea6587251/pkg/cmd/util/kubectl_match_version.go#L115 +func setKubernetesDefaults(config *rest.Config) error { + // This is allowing the GetOptions to be serialized. + config.GroupVersion = &schema.GroupVersion{Group: "", Version: "v1"} + + if config.APIPath == "" { + config.APIPath = "/api" + } + if config.NegotiatedSerializer == nil { + // This codec factory ensures the resources are not converted. Therefore, resources + // will not be round-tripped through internal versions. Defaulting does not happen + // on the client. + config.NegotiatedSerializer = scheme.Codecs.WithoutConversion() + } + return rest.SetKubernetesDefaults(config) +} + +// return a condition function that indicates whether the given pod is currently running +func isPodRunning(ctx context.Context, k8sClient kubernetes.Interface, podName, namespace string) wait.ConditionWithContextFunc { + return func(ct context.Context) (bool, error) { + pod, err := k8sClient.CoreV1().Pods(namespace).Get(ctx, podName, metav1.GetOptions{}) + if err != nil { + return false, err + } + + return pod.Status.Phase == corev1.PodRunning, nil + } +} + +// Poll up to timeout seconds for pod to enter running state. +// Returns an error if the pod never enters the running state. +func waitForPodRunning(ctx context.Context, k8sClient kubernetes.Interface, namespace, podName string, timeout time.Duration) error { + return wait.PollUntilContextTimeout(ctx, time.Second, timeout, true, isPodRunning(ctx, k8sClient, podName, namespace)) +} diff --git a/pkg/cmd/delete/delete.go b/pkg/cmd/delete/delete.go new file mode 100644 index 0000000..d6aba48 --- /dev/null +++ b/pkg/cmd/delete/delete.go @@ -0,0 +1,49 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package delete + +import ( + "fmt" + + "github.com/spf13/cobra" + "k8s.io/cli-runtime/pkg/genericiooptions" + "k8s.io/kubectl/pkg/util/templates" + + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/cmd/util" +) + +var ( + deleteExample = templates.Examples( + fmt.Sprintf("%s\n\n%s\n\n%s\n\n%s", interactiveExample, jobExample, rayJobExample, rayClusterExample), + ) +) + +func NewDeleteCmd(clientGetter util.ClientGetter, streams genericiooptions.IOStreams) *cobra.Command { + cmd := &cobra.Command{ + Use: "delete", + Short: "Delete resources", + Example: deleteExample, + } + + cmd.AddCommand(NewInteractiveCmd(clientGetter, streams)) + cmd.AddCommand(NewJobCmd(clientGetter, streams)) + cmd.AddCommand(NewRayJobCmd(clientGetter, streams)) + cmd.AddCommand(NewRayClusterCmd(clientGetter, streams)) + cmd.AddCommand(NewSlurmCmd(clientGetter, streams)) + + return cmd +} diff --git a/pkg/cmd/delete/delete_interactive.go b/pkg/cmd/delete/delete_interactive.go new file mode 100644 index 0000000..83704d4 --- /dev/null +++ b/pkg/cmd/delete/delete_interactive.go @@ -0,0 +1,176 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package delete + +import ( + "context" + "fmt" + + "github.com/spf13/cobra" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/cli-runtime/pkg/genericclioptions" + "k8s.io/cli-runtime/pkg/genericiooptions" + "k8s.io/cli-runtime/pkg/printers" + "k8s.io/client-go/kubernetes/scheme" + corev1 "k8s.io/client-go/kubernetes/typed/core/v1" + "k8s.io/kubectl/pkg/util/templates" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" + + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/cmd/completion" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/cmd/util" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/constants" +) + +var ( + interactiveExample = templates.Examples(` + # Delete interactive shell + kjobctl delete interactive my-application-profile-interactive-k2wzd + `) +) + +type InteractiveOptions struct { + PrintFlags *genericclioptions.PrintFlags + + InteractiveNames []string + Namespace string + + CascadeStrategy metav1.DeletionPropagation + DryRunStrategy util.DryRunStrategy + + Client corev1.CoreV1Interface + + PrintObj printers.ResourcePrinterFunc + + genericiooptions.IOStreams +} + +func NewInteractiveOptions(streams genericiooptions.IOStreams) *InteractiveOptions { + return &InteractiveOptions{ + PrintFlags: genericclioptions.NewPrintFlags("deleted").WithTypeSetter(scheme.Scheme), + IOStreams: streams, + } +} + +func NewInteractiveCmd(clientGetter util.ClientGetter, streams genericiooptions.IOStreams) *cobra.Command { + o := NewInteractiveOptions(streams) + + cmd := &cobra.Command{ + Use: "interactive NAME [--cascade STRATEGY] [--dry-run STRATEGY]", + DisableFlagsInUseLine: true, + Short: "Delete interactive shell", + Example: interactiveExample, + Args: cobra.MinimumNArgs(1), + ValidArgsFunction: completion.PodNameFunc(clientGetter), + RunE: func(cmd *cobra.Command, args []string) error { + cmd.SilenceUsage = true + + err := o.Complete(clientGetter, cmd, args) + if err != nil { + return err + } + + return o.Run(cmd.Context()) + }, + } + + addCascadingFlag(cmd) + util.AddDryRunFlag(cmd) + + o.PrintFlags.AddFlags(cmd) + + return cmd +} + +func (o *InteractiveOptions) Complete(clientGetter util.ClientGetter, cmd *cobra.Command, args []string) error { + o.InteractiveNames = args + + var err error + + o.Namespace, _, err = clientGetter.ToRawKubeConfigLoader().Namespace() + if err != nil { + return err + } + + o.DryRunStrategy, err = util.GetDryRunStrategy(cmd) + if err != nil { + return err + } + + err = util.PrintFlagsWithDryRunStrategy(o.PrintFlags, o.DryRunStrategy) + if err != nil { + return err + } + + o.CascadeStrategy, err = getCascadingStrategy(cmd) + if err != nil { + return err + } + + printer, err := o.PrintFlags.ToPrinter() + if err != nil { + return err + } + + o.PrintObj = printer.PrintObj + + clientset, err := clientGetter.K8sClientset() + if err != nil { + return err + } + + o.Client = clientset.CoreV1() + + return nil +} + +func (o *InteractiveOptions) Run(ctx context.Context) error { + for _, podName := range o.InteractiveNames { + pod, err := o.Client.Pods(o.Namespace).Get(ctx, podName, metav1.GetOptions{}) + if client.IgnoreNotFound(err) != nil { + return err + } + if err != nil { + fmt.Fprintln(o.ErrOut, err) + continue + } + if _, ok := pod.Labels[constants.ProfileLabel]; !ok { + fmt.Fprintf(o.ErrOut, "pods \"%s\" not created via kjob\n", pod.Name) + continue + } + + if o.DryRunStrategy != util.DryRunClient { + deleteOptions := metav1.DeleteOptions{ + PropagationPolicy: ptr.To(o.CascadeStrategy), + } + + if o.DryRunStrategy == util.DryRunServer { + deleteOptions.DryRun = []string{metav1.DryRunAll} + } + + if err := o.Client.Pods(o.Namespace).Delete(ctx, podName, deleteOptions); err != nil { + return err + } + } + + if err := o.PrintObj(pod, o.Out); err != nil { + return err + } + } + + return nil +} diff --git a/pkg/cmd/delete/delete_interactive_test.go b/pkg/cmd/delete/delete_interactive_test.go new file mode 100644 index 0000000..2899a16 --- /dev/null +++ b/pkg/cmd/delete/delete_interactive_test.go @@ -0,0 +1,190 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package delete + +import ( + "context" + "slices" + "testing" + + "github.com/google/go-cmp/cmp" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/cli-runtime/pkg/genericiooptions" + k8sfake "k8s.io/client-go/kubernetes/fake" + kubetesting "k8s.io/client-go/testing" + + cmdtesting "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/cmd/testing" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/testing/wrappers" +) + +func TestInteractiveCmd(t *testing.T) { + testCases := map[string]struct { + ns string + args []string + objs []runtime.Object + wantPods []corev1.Pod + wantOut string + wantOutErr string + wantErr string + }{ + "shouldn't delete an interactive shell because it is not found": { + args: []string{"pod"}, + objs: []runtime.Object{ + wrappers.MakePod("pod1", metav1.NamespaceDefault).Profile("p1").Obj(), + wrappers.MakePod("pod2", metav1.NamespaceDefault).Profile("p2").Obj(), + }, + wantPods: []corev1.Pod{ + *wrappers.MakePod("pod1", metav1.NamespaceDefault).Profile("p1").Obj(), + *wrappers.MakePod("pod2", metav1.NamespaceDefault).Profile("p2").Obj(), + }, + wantOutErr: "pods \"pod\" not found\n", + }, + "shouldn't delete an interactive shell because it is not created via kjob": { + args: []string{"pod1"}, + objs: []runtime.Object{ + wrappers.MakePod("pod1", metav1.NamespaceDefault).Obj(), + }, + wantPods: []corev1.Pod{ + *wrappers.MakePod("pod1", metav1.NamespaceDefault).Obj(), + }, + wantOutErr: "pods \"pod1\" not created via kjob\n", + }, + "should delete an interactive shell": { + args: []string{"pod1"}, + objs: []runtime.Object{ + wrappers.MakePod("pod1", metav1.NamespaceDefault).Profile("p1").Obj(), + wrappers.MakePod("pod2", metav1.NamespaceDefault).Profile("p2").Obj(), + }, + wantPods: []corev1.Pod{ + *wrappers.MakePod("pod2", metav1.NamespaceDefault).Profile("p2").Obj(), + }, + wantOut: "pod/pod1 deleted\n", + }, + "should delete interactive shells": { + args: []string{"pod1", "pod2"}, + objs: []runtime.Object{ + wrappers.MakePod("pod1", metav1.NamespaceDefault).Profile("p1").Obj(), + wrappers.MakePod("pod2", metav1.NamespaceDefault).Profile("p2").Obj(), + }, + wantOut: "pod/pod1 deleted\npod/pod2 deleted\n", + }, + "should delete only one interactive shell": { + args: []string{"pod1", "pod"}, + objs: []runtime.Object{ + wrappers.MakePod("pod1", metav1.NamespaceDefault).Profile("p1").Obj(), + wrappers.MakePod("pod2", metav1.NamespaceDefault).Profile("p2").Obj(), + }, + wantPods: []corev1.Pod{ + *wrappers.MakePod("pod2", metav1.NamespaceDefault).Profile("p2").Obj(), + }, + wantOut: "pod/pod1 deleted\n", + wantOutErr: "pods \"pod\" not found\n", + }, + "shouldn't delete an interactive shell with client dry run": { + args: []string{"pod1", "--dry-run", "client"}, + objs: []runtime.Object{ + wrappers.MakePod("pod1", metav1.NamespaceDefault).Profile("p1").Obj(), + wrappers.MakePod("pod2", metav1.NamespaceDefault).Profile("p2").Obj(), + }, + wantPods: []corev1.Pod{ + *wrappers.MakePod("pod1", metav1.NamespaceDefault).Profile("p1").Obj(), + *wrappers.MakePod("pod2", metav1.NamespaceDefault).Profile("p2").Obj(), + }, + wantOut: "pod/pod1 deleted (client dry run)\n", + }, + "shouldn't delete an interactive shell with server dry run": { + args: []string{"pod1", "--dry-run", "server"}, + objs: []runtime.Object{ + wrappers.MakePod("pod1", metav1.NamespaceDefault).Profile("p1").Obj(), + wrappers.MakePod("pod2", metav1.NamespaceDefault).Profile("p2").Obj(), + }, + wantPods: []corev1.Pod{ + *wrappers.MakePod("pod1", metav1.NamespaceDefault).Profile("p1").Obj(), + *wrappers.MakePod("pod2", metav1.NamespaceDefault).Profile("p2").Obj(), + }, + wantOut: "pod/pod1 deleted (server dry run)\n", + }, + "no args": { + args: []string{}, + wantErr: "requires at least 1 arg(s), only received 0", + }, + } + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + ns := metav1.NamespaceDefault + if tc.ns != "" { + ns = tc.ns + } + + streams, _, out, outErr := genericiooptions.NewTestIOStreams() + + clientset := k8sfake.NewSimpleClientset(tc.objs...) + clientset.PrependReactor("delete", "pods", func(action kubetesting.Action) (handled bool, ret runtime.Object, err error) { + if slices.Contains(action.(kubetesting.DeleteAction).GetDeleteOptions().DryRun, metav1.DryRunAll) { + handled = true + } + return handled, ret, err + }) + + tcg := cmdtesting.NewTestClientGetter(). + WithK8sClientset(clientset). + WithNamespace(ns) + + cmd := NewInteractiveCmd(tcg, streams) + cmd.SetOut(out) + cmd.SetErr(outErr) + cmd.SetArgs(tc.args) + + gotErr := cmd.Execute() + + var gotErrStr string + if gotErr != nil { + gotErrStr = gotErr.Error() + } + + if diff := cmp.Diff(tc.wantErr, gotErrStr); diff != "" { + t.Errorf("Unexpected error (-want/+got)\n%s", diff) + } + + if gotErr != nil { + return + } + + gotOut := out.String() + if diff := cmp.Diff(tc.wantOut, gotOut); diff != "" { + t.Errorf("Unexpected output (-want/+got)\n%s", diff) + } + + gotOutErr := outErr.String() + if diff := cmp.Diff(tc.wantOutErr, gotOutErr); diff != "" { + t.Errorf("Unexpected error output (-want/+got)\n%s", diff) + } + + gotInteractiveList, err := clientset.CoreV1().Pods(tc.ns).List(context.Background(), metav1.ListOptions{}) + if err != nil { + t.Error(err) + return + } + + if diff := cmp.Diff(tc.wantPods, gotInteractiveList.Items); diff != "" { + t.Errorf("Unexpected pods (-want/+got)\n%s", diff) + } + }) + } +} diff --git a/pkg/cmd/delete/delete_job.go b/pkg/cmd/delete/delete_job.go new file mode 100644 index 0000000..5f0ca09 --- /dev/null +++ b/pkg/cmd/delete/delete_job.go @@ -0,0 +1,182 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package delete + +import ( + "context" + "fmt" + + "github.com/spf13/cobra" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/cli-runtime/pkg/genericclioptions" + "k8s.io/cli-runtime/pkg/genericiooptions" + "k8s.io/cli-runtime/pkg/printers" + "k8s.io/client-go/kubernetes/scheme" + batchv1 "k8s.io/client-go/kubernetes/typed/batch/v1" + "k8s.io/kubectl/pkg/util/templates" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" + + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/apis/v1alpha1" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/cmd/completion" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/cmd/util" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/constants" +) + +var ( + jobExample = templates.Examples(` + # Delete Job + kjobctl delete job my-application-profile-job-k2wzd + `) +) + +type JobOptions struct { + PrintFlags *genericclioptions.PrintFlags + + JobNames []string + Namespace string + + CascadeStrategy metav1.DeletionPropagation + DryRunStrategy util.DryRunStrategy + + Client batchv1.BatchV1Interface + + PrintObj printers.ResourcePrinterFunc + + genericiooptions.IOStreams +} + +func NewJobOptions(streams genericiooptions.IOStreams) *JobOptions { + return &JobOptions{ + PrintFlags: genericclioptions.NewPrintFlags("deleted").WithTypeSetter(scheme.Scheme), + IOStreams: streams, + } +} + +func NewJobCmd(clientGetter util.ClientGetter, streams genericiooptions.IOStreams) *cobra.Command { + o := NewJobOptions(streams) + + cmd := &cobra.Command{ + Use: "job NAME [--cascade STRATEGY] [--dry-run STRATEGY]", + DisableFlagsInUseLine: true, + Short: "Delete Job", + Example: jobExample, + Args: cobra.MinimumNArgs(1), + ValidArgsFunction: completion.JobNameFunc(clientGetter, v1alpha1.JobMode), + RunE: func(cmd *cobra.Command, args []string) error { + cmd.SilenceUsage = true + + err := o.Complete(clientGetter, cmd, args) + if err != nil { + return err + } + + return o.Run(cmd.Context()) + }, + } + + addCascadingFlag(cmd) + util.AddDryRunFlag(cmd) + + o.PrintFlags.AddFlags(cmd) + + return cmd +} + +func (o *JobOptions) Complete(clientGetter util.ClientGetter, cmd *cobra.Command, args []string) error { + o.JobNames = args + + var err error + + o.Namespace, _, err = clientGetter.ToRawKubeConfigLoader().Namespace() + if err != nil { + return err + } + + o.DryRunStrategy, err = util.GetDryRunStrategy(cmd) + if err != nil { + return err + } + + err = util.PrintFlagsWithDryRunStrategy(o.PrintFlags, o.DryRunStrategy) + if err != nil { + return err + } + + o.CascadeStrategy, err = getCascadingStrategy(cmd) + if err != nil { + return err + } + + printer, err := o.PrintFlags.ToPrinter() + if err != nil { + return err + } + + o.PrintObj = printer.PrintObj + + clientset, err := clientGetter.K8sClientset() + if err != nil { + return err + } + + o.Client = clientset.BatchV1() + + return nil +} + +func (o *JobOptions) Run(ctx context.Context) error { + for _, jobName := range o.JobNames { + job, err := o.Client.Jobs(o.Namespace).Get(ctx, jobName, metav1.GetOptions{}) + if client.IgnoreNotFound(err) != nil { + return err + } + if err != nil { + fmt.Fprintln(o.ErrOut, err) + continue + } + if _, ok := job.Labels[constants.ProfileLabel]; !ok { + fmt.Fprintf(o.ErrOut, "jobs.batch \"%s\" not created via kjob\n", job.Name) + continue + } + if job.Labels[constants.ModeLabel] != string(v1alpha1.JobMode) { + fmt.Fprintf(o.ErrOut, "jobs.batch \"%s\" created in \"%s\" mode. Switch to the correct mode to delete it\n", + job.Name, job.Labels[constants.ModeLabel]) + continue + } + + if o.DryRunStrategy != util.DryRunClient { + deleteOptions := metav1.DeleteOptions{ + PropagationPolicy: ptr.To(o.CascadeStrategy), + } + + if o.DryRunStrategy == util.DryRunServer { + deleteOptions.DryRun = []string{metav1.DryRunAll} + } + + if err := o.Client.Jobs(o.Namespace).Delete(ctx, jobName, deleteOptions); err != nil { + return err + } + } + + if err := o.PrintObj(job, o.Out); err != nil { + return err + } + } + + return nil +} diff --git a/pkg/cmd/delete/delete_job_test.go b/pkg/cmd/delete/delete_job_test.go new file mode 100644 index 0000000..531dabb --- /dev/null +++ b/pkg/cmd/delete/delete_job_test.go @@ -0,0 +1,205 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package delete + +import ( + "context" + "slices" + "testing" + + "github.com/google/go-cmp/cmp" + batchv1 "k8s.io/api/batch/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/cli-runtime/pkg/genericiooptions" + k8sfake "k8s.io/client-go/kubernetes/fake" + kubetesting "k8s.io/client-go/testing" + + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/apis/v1alpha1" + cmdtesting "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/cmd/testing" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/testing/wrappers" +) + +func TestJobCmd(t *testing.T) { + testCases := map[string]struct { + ns string + args []string + objs []runtime.Object + wantJobs []batchv1.Job + wantOut string + wantOutErr string + wantErr string + }{ + "shouldn't delete job because it is not found": { + args: []string{"j"}, + objs: []runtime.Object{ + wrappers.MakeJob("j1", metav1.NamespaceDefault).Profile("p1").Mode(v1alpha1.JobMode).Obj(), + wrappers.MakeJob("j2", metav1.NamespaceDefault).Profile("p2").Mode(v1alpha1.JobMode).Obj(), + }, + wantJobs: []batchv1.Job{ + *wrappers.MakeJob("j1", metav1.NamespaceDefault).Profile("p1").Mode(v1alpha1.JobMode).Obj(), + *wrappers.MakeJob("j2", metav1.NamespaceDefault).Profile("p2").Mode(v1alpha1.JobMode).Obj(), + }, + wantOutErr: "jobs.batch \"j\" not found\n", + }, + "shouldn't delete job because it is not created via kjob": { + args: []string{"j1"}, + objs: []runtime.Object{ + wrappers.MakeJob("j1", metav1.NamespaceDefault).Obj(), + }, + wantJobs: []batchv1.Job{ + *wrappers.MakeJob("j1", metav1.NamespaceDefault).Obj(), + }, + wantOutErr: "jobs.batch \"j1\" not created via kjob\n", + }, + "shouldn't delete job because it is not used for Job mode": { + args: []string{"j1", "j2"}, + objs: []runtime.Object{ + wrappers.MakeJob("j1", metav1.NamespaceDefault).Profile("p1").Obj(), + wrappers.MakeJob("j2", metav1.NamespaceDefault).Profile("p1").Mode(v1alpha1.SlurmMode).Obj(), + }, + wantJobs: []batchv1.Job{ + *wrappers.MakeJob("j1", metav1.NamespaceDefault).Profile("p1").Obj(), + *wrappers.MakeJob("j2", metav1.NamespaceDefault).Profile("p1").Mode(v1alpha1.SlurmMode).Obj(), + }, + wantOutErr: `jobs.batch "j1" created in "" mode. Switch to the correct mode to delete it +jobs.batch "j2" created in "Slurm" mode. Switch to the correct mode to delete it +`, + }, + "should delete job": { + args: []string{"j1"}, + objs: []runtime.Object{ + wrappers.MakeJob("j1", metav1.NamespaceDefault).Profile("p1").Mode(v1alpha1.JobMode).Obj(), + wrappers.MakeJob("j2", metav1.NamespaceDefault).Profile("p2").Mode(v1alpha1.JobMode).Obj(), + }, + wantJobs: []batchv1.Job{ + *wrappers.MakeJob("j2", metav1.NamespaceDefault).Profile("p2").Mode(v1alpha1.JobMode).Obj(), + }, + wantOut: "job.batch/j1 deleted\n", + }, + "should delete jobs": { + args: []string{"j1", "j2"}, + objs: []runtime.Object{ + wrappers.MakeJob("j1", metav1.NamespaceDefault).Profile("p1").Mode(v1alpha1.JobMode).Obj(), + wrappers.MakeJob("j2", metav1.NamespaceDefault).Profile("p2").Mode(v1alpha1.JobMode).Obj(), + }, + wantOut: "job.batch/j1 deleted\njob.batch/j2 deleted\n", + }, + "should delete only one job": { + args: []string{"j1", "j"}, + objs: []runtime.Object{ + wrappers.MakeJob("j1", metav1.NamespaceDefault).Profile("p1").Mode(v1alpha1.JobMode).Obj(), + wrappers.MakeJob("j2", metav1.NamespaceDefault).Profile("p2").Mode(v1alpha1.JobMode).Obj(), + }, + wantJobs: []batchv1.Job{ + *wrappers.MakeJob("j2", metav1.NamespaceDefault).Profile("p2").Mode(v1alpha1.JobMode).Obj(), + }, + wantOut: "job.batch/j1 deleted\n", + wantOutErr: "jobs.batch \"j\" not found\n", + }, + "shouldn't delete job with client dry run": { + args: []string{"j1", "--dry-run", "client"}, + objs: []runtime.Object{ + wrappers.MakeJob("j1", metav1.NamespaceDefault).Profile("p1").Mode(v1alpha1.JobMode).Obj(), + wrappers.MakeJob("j2", metav1.NamespaceDefault).Profile("p2").Mode(v1alpha1.JobMode).Obj(), + }, + wantJobs: []batchv1.Job{ + *wrappers.MakeJob("j1", metav1.NamespaceDefault).Profile("p1").Mode(v1alpha1.JobMode).Obj(), + *wrappers.MakeJob("j2", metav1.NamespaceDefault).Profile("p2").Mode(v1alpha1.JobMode).Obj(), + }, + wantOut: "job.batch/j1 deleted (client dry run)\n", + }, + "shouldn't delete job with server dry run": { + args: []string{"j1", "--dry-run", "server"}, + objs: []runtime.Object{ + wrappers.MakeJob("j1", metav1.NamespaceDefault).Profile("p1").Mode(v1alpha1.JobMode).Obj(), + wrappers.MakeJob("j2", metav1.NamespaceDefault).Profile("p2").Mode(v1alpha1.JobMode).Obj(), + }, + wantJobs: []batchv1.Job{ + *wrappers.MakeJob("j1", metav1.NamespaceDefault).Profile("p1").Mode(v1alpha1.JobMode).Obj(), + *wrappers.MakeJob("j2", metav1.NamespaceDefault).Profile("p2").Mode(v1alpha1.JobMode).Obj(), + }, + wantOut: "job.batch/j1 deleted (server dry run)\n", + }, + "no args": { + args: []string{}, + wantErr: "requires at least 1 arg(s), only received 0", + }, + } + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + ns := metav1.NamespaceDefault + if tc.ns != "" { + ns = tc.ns + } + + streams, _, out, outErr := genericiooptions.NewTestIOStreams() + + clientset := k8sfake.NewSimpleClientset(tc.objs...) + clientset.PrependReactor("delete", "jobs", func(action kubetesting.Action) (handled bool, ret runtime.Object, err error) { + if slices.Contains(action.(kubetesting.DeleteAction).GetDeleteOptions().DryRun, metav1.DryRunAll) { + handled = true + } + return handled, ret, err + }) + + tcg := cmdtesting.NewTestClientGetter(). + WithK8sClientset(clientset). + WithNamespace(ns) + + cmd := NewJobCmd(tcg, streams) + cmd.SetOut(out) + cmd.SetErr(outErr) + cmd.SetArgs(tc.args) + + gotErr := cmd.Execute() + + var gotErrStr string + if gotErr != nil { + gotErrStr = gotErr.Error() + } + + if diff := cmp.Diff(tc.wantErr, gotErrStr); diff != "" { + t.Errorf("Unexpected error (-want/+got)\n%s", diff) + } + + if gotErr != nil { + return + } + + gotOut := out.String() + if diff := cmp.Diff(tc.wantOut, gotOut); diff != "" { + t.Errorf("Unexpected output (-want/+got)\n%s", diff) + } + + gotOutErr := outErr.String() + if diff := cmp.Diff(tc.wantOutErr, gotOutErr); diff != "" { + t.Errorf("Unexpected error output (-want/+got)\n%s", diff) + } + + gotJobList, err := clientset.BatchV1().Jobs(tc.ns).List(context.Background(), metav1.ListOptions{}) + if err != nil { + t.Error(err) + return + } + + if diff := cmp.Diff(tc.wantJobs, gotJobList.Items); diff != "" { + t.Errorf("Unexpected jobs (-want/+got)\n%s", diff) + } + }) + } +} diff --git a/pkg/cmd/delete/delete_ray_cluster.go b/pkg/cmd/delete/delete_ray_cluster.go new file mode 100644 index 0000000..7d963ef --- /dev/null +++ b/pkg/cmd/delete/delete_ray_cluster.go @@ -0,0 +1,176 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package delete + +import ( + "context" + "fmt" + + "github.com/ray-project/kuberay/ray-operator/pkg/client/clientset/versioned/scheme" + rayv1 "github.com/ray-project/kuberay/ray-operator/pkg/client/clientset/versioned/typed/ray/v1" + "github.com/spf13/cobra" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/cli-runtime/pkg/genericclioptions" + "k8s.io/cli-runtime/pkg/genericiooptions" + "k8s.io/cli-runtime/pkg/printers" + "k8s.io/kubectl/pkg/util/templates" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" + + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/cmd/completion" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/cmd/util" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/constants" +) + +var ( + rayClusterExample = templates.Examples(` + # Delete RayCluster + kjobctl delete raycluster my-application-profile-raycluster-k2wzd + `) +) + +type RayClusterOptions struct { + PrintFlags *genericclioptions.PrintFlags + + RayClusterNames []string + Namespace string + + CascadeStrategy metav1.DeletionPropagation + DryRunStrategy util.DryRunStrategy + + Client rayv1.RayV1Interface + + PrintObj printers.ResourcePrinterFunc + + genericiooptions.IOStreams +} + +func NewRayClusterOptions(streams genericiooptions.IOStreams) *RayClusterOptions { + return &RayClusterOptions{ + PrintFlags: genericclioptions.NewPrintFlags("deleted").WithTypeSetter(scheme.Scheme), + IOStreams: streams, + } +} + +func NewRayClusterCmd(clientGetter util.ClientGetter, streams genericiooptions.IOStreams) *cobra.Command { + o := NewRayClusterOptions(streams) + + cmd := &cobra.Command{ + Use: "raycluster NAME [--cascade STRATEGY] [--dry-run STRATEGY]", + DisableFlagsInUseLine: true, + Short: "Delete RayCluster", + Example: rayClusterExample, + Args: cobra.MinimumNArgs(1), + ValidArgsFunction: completion.RayClusterNameFunc(clientGetter), + RunE: func(cmd *cobra.Command, args []string) error { + cmd.SilenceUsage = true + + err := o.Complete(clientGetter, cmd, args) + if err != nil { + return err + } + + return o.Run(cmd.Context()) + }, + } + + addCascadingFlag(cmd) + util.AddDryRunFlag(cmd) + + o.PrintFlags.AddFlags(cmd) + + return cmd +} + +func (o *RayClusterOptions) Complete(clientGetter util.ClientGetter, cmd *cobra.Command, args []string) error { + o.RayClusterNames = args + + var err error + + o.Namespace, _, err = clientGetter.ToRawKubeConfigLoader().Namespace() + if err != nil { + return err + } + + o.DryRunStrategy, err = util.GetDryRunStrategy(cmd) + if err != nil { + return err + } + + err = util.PrintFlagsWithDryRunStrategy(o.PrintFlags, o.DryRunStrategy) + if err != nil { + return err + } + + o.CascadeStrategy, err = getCascadingStrategy(cmd) + if err != nil { + return err + } + + printer, err := o.PrintFlags.ToPrinter() + if err != nil { + return err + } + + o.PrintObj = printer.PrintObj + + clientset, err := clientGetter.RayClientset() + if err != nil { + return err + } + + o.Client = clientset.RayV1() + + return nil +} + +func (o *RayClusterOptions) Run(ctx context.Context) error { + for _, rayClusterName := range o.RayClusterNames { + rayCluster, err := o.Client.RayClusters(o.Namespace).Get(ctx, rayClusterName, metav1.GetOptions{}) + if client.IgnoreNotFound(err) != nil { + return err + } + if err != nil { + fmt.Fprintln(o.ErrOut, err) + continue + } + if _, ok := rayCluster.Labels[constants.ProfileLabel]; !ok { + fmt.Fprintf(o.ErrOut, "rayclusters.ray.io \"%s\" not created via kjob\n", rayCluster.Name) + continue + } + + if o.DryRunStrategy != util.DryRunClient { + deleteOptions := metav1.DeleteOptions{ + PropagationPolicy: ptr.To(o.CascadeStrategy), + } + + if o.DryRunStrategy == util.DryRunServer { + deleteOptions.DryRun = []string{metav1.DryRunAll} + } + + if err := o.Client.RayClusters(o.Namespace).Delete(ctx, rayClusterName, deleteOptions); err != nil { + return err + } + } + + if err := o.PrintObj(rayCluster, o.Out); err != nil { + return err + } + } + + return nil +} diff --git a/pkg/cmd/delete/delete_ray_cluster_test.go b/pkg/cmd/delete/delete_ray_cluster_test.go new file mode 100644 index 0000000..0dce881 --- /dev/null +++ b/pkg/cmd/delete/delete_ray_cluster_test.go @@ -0,0 +1,190 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package delete + +import ( + "context" + "slices" + "testing" + + "github.com/google/go-cmp/cmp" + rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1" + "github.com/ray-project/kuberay/ray-operator/pkg/client/clientset/versioned/fake" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/cli-runtime/pkg/genericiooptions" + kubetesting "k8s.io/client-go/testing" + + cmdtesting "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/cmd/testing" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/testing/wrappers" +) + +func TestRayClusterCmd(t *testing.T) { + testCases := map[string]struct { + ns string + args []string + objs []runtime.Object + wantRayClusters []rayv1.RayCluster + wantOut string + wantOutErr string + wantErr string + }{ + "shouldn't delete ray cluster because it is not found": { + args: []string{"rc"}, + objs: []runtime.Object{ + wrappers.MakeRayCluster("rc1", metav1.NamespaceDefault).Profile("p1").Obj(), + wrappers.MakeRayCluster("rc2", metav1.NamespaceDefault).Profile("p2").Obj(), + }, + wantRayClusters: []rayv1.RayCluster{ + *wrappers.MakeRayCluster("rc1", metav1.NamespaceDefault).Profile("p1").Obj(), + *wrappers.MakeRayCluster("rc2", metav1.NamespaceDefault).Profile("p2").Obj(), + }, + wantOutErr: "rayclusters.ray.io \"rc\" not found\n", + }, + "shouldn't delete ray cluster because it is not created via kjob": { + args: []string{"rc1"}, + objs: []runtime.Object{ + wrappers.MakeRayCluster("rc1", metav1.NamespaceDefault).Obj(), + }, + wantRayClusters: []rayv1.RayCluster{ + *wrappers.MakeRayCluster("rc1", metav1.NamespaceDefault).Obj(), + }, + wantOutErr: "rayclusters.ray.io \"rc1\" not created via kjob\n", + }, + "should delete ray cluster": { + args: []string{"rc1"}, + objs: []runtime.Object{ + wrappers.MakeRayCluster("rc1", metav1.NamespaceDefault).Profile("p1").Obj(), + wrappers.MakeRayCluster("rc2", metav1.NamespaceDefault).Profile("p2").Obj(), + }, + wantRayClusters: []rayv1.RayCluster{ + *wrappers.MakeRayCluster("rc2", metav1.NamespaceDefault).Profile("p2").Obj(), + }, + wantOut: "raycluster.ray.io/rc1 deleted\n", + }, + "should delete ray clusters": { + args: []string{"rc1", "rc2"}, + objs: []runtime.Object{ + wrappers.MakeRayCluster("rc1", metav1.NamespaceDefault).Profile("p1").Obj(), + wrappers.MakeRayCluster("rc2", metav1.NamespaceDefault).Profile("p2").Obj(), + }, + wantOut: "raycluster.ray.io/rc1 deleted\nraycluster.ray.io/rc2 deleted\n", + }, + "should delete only one ray cluster": { + args: []string{"rc1", "rc"}, + objs: []runtime.Object{ + wrappers.MakeRayCluster("rc1", metav1.NamespaceDefault).Profile("p1").Obj(), + wrappers.MakeRayCluster("rc2", metav1.NamespaceDefault).Profile("p2").Obj(), + }, + wantRayClusters: []rayv1.RayCluster{ + *wrappers.MakeRayCluster("rc2", metav1.NamespaceDefault).Profile("p2").Obj(), + }, + wantOut: "raycluster.ray.io/rc1 deleted\n", + wantOutErr: "rayclusters.ray.io \"rc\" not found\n", + }, + "shouldn't delete ray cluster with client dry run": { + args: []string{"rc1", "--dry-run", "client"}, + objs: []runtime.Object{ + wrappers.MakeRayCluster("rc1", metav1.NamespaceDefault).Profile("p1").Obj(), + wrappers.MakeRayCluster("rc2", metav1.NamespaceDefault).Profile("p2").Obj(), + }, + wantRayClusters: []rayv1.RayCluster{ + *wrappers.MakeRayCluster("rc1", metav1.NamespaceDefault).Profile("p1").Obj(), + *wrappers.MakeRayCluster("rc2", metav1.NamespaceDefault).Profile("p2").Obj(), + }, + wantOut: "raycluster.ray.io/rc1 deleted (client dry run)\n", + }, + "shouldn't delete ray cluster with server dry run": { + args: []string{"rc1", "--dry-run", "server"}, + objs: []runtime.Object{ + wrappers.MakeRayCluster("rc1", metav1.NamespaceDefault).Profile("p1").Obj(), + wrappers.MakeRayCluster("rc2", metav1.NamespaceDefault).Profile("p2").Obj(), + }, + wantRayClusters: []rayv1.RayCluster{ + *wrappers.MakeRayCluster("rc1", metav1.NamespaceDefault).Profile("p1").Obj(), + *wrappers.MakeRayCluster("rc2", metav1.NamespaceDefault).Profile("p2").Obj(), + }, + wantOut: "raycluster.ray.io/rc1 deleted (server dry run)\n", + }, + "no args": { + args: []string{}, + wantErr: "requires at least 1 arg(s), only received 0", + }, + } + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + ns := metav1.NamespaceDefault + if tc.ns != "" { + ns = tc.ns + } + + streams, _, out, outErr := genericiooptions.NewTestIOStreams() + + clientset := fake.NewSimpleClientset(tc.objs...) + clientset.PrependReactor("delete", "rayclusters", func(action kubetesting.Action) (handled bool, ret runtime.Object, err error) { + if slices.Contains(action.(kubetesting.DeleteAction).GetDeleteOptions().DryRun, metav1.DryRunAll) { + handled = true + } + return handled, ret, err + }) + + tcg := cmdtesting.NewTestClientGetter(). + WithRayClientset(clientset). + WithNamespace(ns) + + cmd := NewRayClusterCmd(tcg, streams) + cmd.SetOut(out) + cmd.SetErr(outErr) + cmd.SetArgs(tc.args) + + gotErr := cmd.Execute() + + var gotErrStr string + if gotErr != nil { + gotErrStr = gotErr.Error() + } + + if diff := cmp.Diff(tc.wantErr, gotErrStr); diff != "" { + t.Errorf("Unexpected error (-want/+got)\n%s", diff) + } + + if gotErr != nil { + return + } + + gotOut := out.String() + if diff := cmp.Diff(tc.wantOut, gotOut); diff != "" { + t.Errorf("Unexpected output (-want/+got)\n%s", diff) + } + + gotOutErr := outErr.String() + if diff := cmp.Diff(tc.wantOutErr, gotOutErr); diff != "" { + t.Errorf("Unexpected error output (-want/+got)\n%s", diff) + } + + gotRayClusterList, err := clientset.RayV1().RayClusters(tc.ns).List(context.Background(), metav1.ListOptions{}) + if err != nil { + t.Error(err) + return + } + + if diff := cmp.Diff(tc.wantRayClusters, gotRayClusterList.Items); diff != "" { + t.Errorf("Unexpected ray clusters (-want/+got)\n%s", diff) + } + }) + } +} diff --git a/pkg/cmd/delete/delete_ray_job.go b/pkg/cmd/delete/delete_ray_job.go new file mode 100644 index 0000000..6dee8a7 --- /dev/null +++ b/pkg/cmd/delete/delete_ray_job.go @@ -0,0 +1,176 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package delete + +import ( + "context" + "fmt" + + "github.com/ray-project/kuberay/ray-operator/pkg/client/clientset/versioned/scheme" + rayv1 "github.com/ray-project/kuberay/ray-operator/pkg/client/clientset/versioned/typed/ray/v1" + "github.com/spf13/cobra" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/cli-runtime/pkg/genericclioptions" + "k8s.io/cli-runtime/pkg/genericiooptions" + "k8s.io/cli-runtime/pkg/printers" + "k8s.io/kubectl/pkg/util/templates" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" + + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/cmd/completion" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/cmd/util" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/constants" +) + +var ( + rayJobExample = templates.Examples(` + # Delete RayJob + kjobctl delete rayjob my-application-profile-rayjob-k2wzd + `) +) + +type RayJobOptions struct { + PrintFlags *genericclioptions.PrintFlags + + RayJobNames []string + Namespace string + + CascadeStrategy metav1.DeletionPropagation + DryRunStrategy util.DryRunStrategy + + Client rayv1.RayV1Interface + + PrintObj printers.ResourcePrinterFunc + + genericiooptions.IOStreams +} + +func NewRayJobOptions(streams genericiooptions.IOStreams) *RayJobOptions { + return &RayJobOptions{ + PrintFlags: genericclioptions.NewPrintFlags("deleted").WithTypeSetter(scheme.Scheme), + IOStreams: streams, + } +} + +func NewRayJobCmd(clientGetter util.ClientGetter, streams genericiooptions.IOStreams) *cobra.Command { + o := NewRayJobOptions(streams) + + cmd := &cobra.Command{ + Use: "rayjob NAME [--cascade STRATEGY] [--dry-run STRATEGY]", + DisableFlagsInUseLine: true, + Short: "Delete RayJob", + Example: rayJobExample, + Args: cobra.MinimumNArgs(1), + ValidArgsFunction: completion.RayJobNameFunc(clientGetter), + RunE: func(cmd *cobra.Command, args []string) error { + cmd.SilenceUsage = true + + err := o.Complete(clientGetter, cmd, args) + if err != nil { + return err + } + + return o.Run(cmd.Context()) + }, + } + + addCascadingFlag(cmd) + util.AddDryRunFlag(cmd) + + o.PrintFlags.AddFlags(cmd) + + return cmd +} + +func (o *RayJobOptions) Complete(clientGetter util.ClientGetter, cmd *cobra.Command, args []string) error { + o.RayJobNames = args + + var err error + + o.Namespace, _, err = clientGetter.ToRawKubeConfigLoader().Namespace() + if err != nil { + return err + } + + o.DryRunStrategy, err = util.GetDryRunStrategy(cmd) + if err != nil { + return err + } + + err = util.PrintFlagsWithDryRunStrategy(o.PrintFlags, o.DryRunStrategy) + if err != nil { + return err + } + + o.CascadeStrategy, err = getCascadingStrategy(cmd) + if err != nil { + return err + } + + printer, err := o.PrintFlags.ToPrinter() + if err != nil { + return err + } + + o.PrintObj = printer.PrintObj + + clientset, err := clientGetter.RayClientset() + if err != nil { + return err + } + + o.Client = clientset.RayV1() + + return nil +} + +func (o *RayJobOptions) Run(ctx context.Context) error { + for _, rayJobName := range o.RayJobNames { + rayJob, err := o.Client.RayJobs(o.Namespace).Get(ctx, rayJobName, metav1.GetOptions{}) + if client.IgnoreNotFound(err) != nil { + return err + } + if err != nil { + fmt.Fprintln(o.ErrOut, err) + continue + } + if _, ok := rayJob.Labels[constants.ProfileLabel]; !ok { + fmt.Fprintf(o.ErrOut, "rayjobs.ray.io \"%s\" not created via kjob\n", rayJob.Name) + continue + } + + if o.DryRunStrategy != util.DryRunClient { + deleteOptions := metav1.DeleteOptions{ + PropagationPolicy: ptr.To(o.CascadeStrategy), + } + + if o.DryRunStrategy == util.DryRunServer { + deleteOptions.DryRun = []string{metav1.DryRunAll} + } + + if err := o.Client.RayJobs(o.Namespace).Delete(ctx, rayJobName, deleteOptions); err != nil { + return err + } + } + + if err := o.PrintObj(rayJob, o.Out); err != nil { + return err + } + } + + return nil +} diff --git a/pkg/cmd/delete/delete_ray_job_test.go b/pkg/cmd/delete/delete_ray_job_test.go new file mode 100644 index 0000000..018d5ac --- /dev/null +++ b/pkg/cmd/delete/delete_ray_job_test.go @@ -0,0 +1,190 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package delete + +import ( + "context" + "slices" + "testing" + + "github.com/google/go-cmp/cmp" + rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1" + "github.com/ray-project/kuberay/ray-operator/pkg/client/clientset/versioned/fake" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/cli-runtime/pkg/genericiooptions" + kubetesting "k8s.io/client-go/testing" + + cmdtesting "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/cmd/testing" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/testing/wrappers" +) + +func TestRayJobCmd(t *testing.T) { + testCases := map[string]struct { + ns string + args []string + objs []runtime.Object + wantRayJobs []rayv1.RayJob + wantOut string + wantOutErr string + wantErr string + }{ + "shouldn't delete ray job because it is not found": { + args: []string{"rj"}, + objs: []runtime.Object{ + wrappers.MakeRayJob("rj1", metav1.NamespaceDefault).Profile("p1").Obj(), + wrappers.MakeRayJob("rj2", metav1.NamespaceDefault).Profile("p2").Obj(), + }, + wantRayJobs: []rayv1.RayJob{ + *wrappers.MakeRayJob("rj1", metav1.NamespaceDefault).Profile("p1").Obj(), + *wrappers.MakeRayJob("rj2", metav1.NamespaceDefault).Profile("p2").Obj(), + }, + wantOutErr: "rayjobs.ray.io \"rj\" not found\n", + }, + "shouldn't delete ray job because it is not created via kjob": { + args: []string{"rj1"}, + objs: []runtime.Object{ + wrappers.MakeRayJob("rj1", metav1.NamespaceDefault).Obj(), + }, + wantRayJobs: []rayv1.RayJob{ + *wrappers.MakeRayJob("rj1", metav1.NamespaceDefault).Obj(), + }, + wantOutErr: "rayjobs.ray.io \"rj1\" not created via kjob\n", + }, + "should delete ray job": { + args: []string{"rj1"}, + objs: []runtime.Object{ + wrappers.MakeRayJob("rj1", metav1.NamespaceDefault).Profile("p1").Obj(), + wrappers.MakeRayJob("rj2", metav1.NamespaceDefault).Profile("p2").Obj(), + }, + wantRayJobs: []rayv1.RayJob{ + *wrappers.MakeRayJob("rj2", metav1.NamespaceDefault).Profile("p2").Obj(), + }, + wantOut: "rayjob.ray.io/rj1 deleted\n", + }, + "should delete ray jobs": { + args: []string{"rj1", "rj2"}, + objs: []runtime.Object{ + wrappers.MakeRayJob("rj1", metav1.NamespaceDefault).Profile("p1").Obj(), + wrappers.MakeRayJob("rj2", metav1.NamespaceDefault).Profile("p2").Obj(), + }, + wantOut: "rayjob.ray.io/rj1 deleted\nrayjob.ray.io/rj2 deleted\n", + }, + "should delete only one ray job": { + args: []string{"rj1", "rj"}, + objs: []runtime.Object{ + wrappers.MakeRayJob("rj1", metav1.NamespaceDefault).Profile("p1").Obj(), + wrappers.MakeRayJob("rj2", metav1.NamespaceDefault).Profile("p2").Obj(), + }, + wantRayJobs: []rayv1.RayJob{ + *wrappers.MakeRayJob("rj2", metav1.NamespaceDefault).Profile("p2").Obj(), + }, + wantOut: "rayjob.ray.io/rj1 deleted\n", + wantOutErr: "rayjobs.ray.io \"rj\" not found\n", + }, + "shouldn't delete ray job with client dry run": { + args: []string{"rj1", "--dry-run", "client"}, + objs: []runtime.Object{ + wrappers.MakeRayJob("rj1", metav1.NamespaceDefault).Profile("p1").Obj(), + wrappers.MakeRayJob("rj2", metav1.NamespaceDefault).Profile("p2").Obj(), + }, + wantRayJobs: []rayv1.RayJob{ + *wrappers.MakeRayJob("rj1", metav1.NamespaceDefault).Profile("p1").Obj(), + *wrappers.MakeRayJob("rj2", metav1.NamespaceDefault).Profile("p2").Obj(), + }, + wantOut: "rayjob.ray.io/rj1 deleted (client dry run)\n", + }, + "shouldn't delete ray job with server dry run": { + args: []string{"rj1", "--dry-run", "server"}, + objs: []runtime.Object{ + wrappers.MakeRayJob("rj1", metav1.NamespaceDefault).Profile("p1").Obj(), + wrappers.MakeRayJob("rj2", metav1.NamespaceDefault).Profile("p2").Obj(), + }, + wantRayJobs: []rayv1.RayJob{ + *wrappers.MakeRayJob("rj1", metav1.NamespaceDefault).Profile("p1").Obj(), + *wrappers.MakeRayJob("rj2", metav1.NamespaceDefault).Profile("p2").Obj(), + }, + wantOut: "rayjob.ray.io/rj1 deleted (server dry run)\n", + }, + "no args": { + args: []string{}, + wantErr: "requires at least 1 arg(s), only received 0", + }, + } + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + ns := metav1.NamespaceDefault + if tc.ns != "" { + ns = tc.ns + } + + streams, _, out, outErr := genericiooptions.NewTestIOStreams() + + clientset := fake.NewSimpleClientset(tc.objs...) + clientset.PrependReactor("delete", "rayjobs", func(action kubetesting.Action) (handled bool, ret runtime.Object, err error) { + if slices.Contains(action.(kubetesting.DeleteAction).GetDeleteOptions().DryRun, metav1.DryRunAll) { + handled = true + } + return handled, ret, err + }) + + tcg := cmdtesting.NewTestClientGetter(). + WithRayClientset(clientset). + WithNamespace(ns) + + cmd := NewRayJobCmd(tcg, streams) + cmd.SetOut(out) + cmd.SetErr(outErr) + cmd.SetArgs(tc.args) + + gotErr := cmd.Execute() + + var gotErrStr string + if gotErr != nil { + gotErrStr = gotErr.Error() + } + + if diff := cmp.Diff(tc.wantErr, gotErrStr); diff != "" { + t.Errorf("Unexpected error (-want/+got)\n%s", diff) + } + + if gotErr != nil { + return + } + + gotOut := out.String() + if diff := cmp.Diff(tc.wantOut, gotOut); diff != "" { + t.Errorf("Unexpected output (-want/+got)\n%s", diff) + } + + gotOutErr := outErr.String() + if diff := cmp.Diff(tc.wantOutErr, gotOutErr); diff != "" { + t.Errorf("Unexpected error output (-want/+got)\n%s", diff) + } + + gotRayJobList, err := clientset.RayV1().RayJobs(tc.ns).List(context.Background(), metav1.ListOptions{}) + if err != nil { + t.Error(err) + return + } + + if diff := cmp.Diff(tc.wantRayJobs, gotRayJobList.Items); diff != "" { + t.Errorf("Unexpected ray jobs (-want/+got)\n%s", diff) + } + }) + } +} diff --git a/pkg/cmd/delete/delete_slurm.go b/pkg/cmd/delete/delete_slurm.go new file mode 100644 index 0000000..4da4896 --- /dev/null +++ b/pkg/cmd/delete/delete_slurm.go @@ -0,0 +1,184 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package delete + +import ( + "context" + "fmt" + + "github.com/spf13/cobra" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/cli-runtime/pkg/genericclioptions" + "k8s.io/cli-runtime/pkg/genericiooptions" + "k8s.io/cli-runtime/pkg/printers" + k8s "k8s.io/client-go/kubernetes" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/kubectl/pkg/util/templates" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" + + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/apis/v1alpha1" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/cmd/completion" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/cmd/util" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/constants" +) + +var ( + slurmExample = templates.Examples(` + # Delete Slurm + kjobctl delete slurm my-application-profile-slurm-k2wzd + `) +) + +type SlurmOptions struct { + PrintFlags *genericclioptions.PrintFlags + + JobNames []string + Namespace string + + CascadeStrategy metav1.DeletionPropagation + DryRunStrategy util.DryRunStrategy + + Clientset k8s.Interface + + PrintObj printers.ResourcePrinterFunc + + genericiooptions.IOStreams +} + +func NewSlurmOptions(streams genericiooptions.IOStreams) *SlurmOptions { + return &SlurmOptions{ + PrintFlags: genericclioptions.NewPrintFlags("deleted").WithTypeSetter(scheme.Scheme), + IOStreams: streams, + } +} + +func NewSlurmCmd(clientGetter util.ClientGetter, streams genericiooptions.IOStreams) *cobra.Command { + o := NewSlurmOptions(streams) + + cmd := &cobra.Command{ + Use: "slurm NAME [--cascade STRATEGY] [--dry-run STRATEGY]", + DisableFlagsInUseLine: true, + Short: "Delete Slurm", + Example: slurmExample, + Args: cobra.MinimumNArgs(1), + ValidArgsFunction: completion.JobNameFunc(clientGetter, v1alpha1.SlurmMode), + RunE: func(cmd *cobra.Command, args []string) error { + cmd.SilenceUsage = true + + err := o.Complete(clientGetter, cmd, args) + if err != nil { + return err + } + + return o.Run(cmd.Context()) + }, + } + + addCascadingFlag(cmd) + util.AddDryRunFlag(cmd) + + o.PrintFlags.AddFlags(cmd) + + return cmd +} + +func (o *SlurmOptions) Complete(clientGetter util.ClientGetter, cmd *cobra.Command, args []string) error { + o.JobNames = args + + var err error + + o.Namespace, _, err = clientGetter.ToRawKubeConfigLoader().Namespace() + if err != nil { + return err + } + + o.DryRunStrategy, err = util.GetDryRunStrategy(cmd) + if err != nil { + return err + } + + err = util.PrintFlagsWithDryRunStrategy(o.PrintFlags, o.DryRunStrategy) + if err != nil { + return err + } + + o.CascadeStrategy, err = getCascadingStrategy(cmd) + if err != nil { + return err + } + + printer, err := o.PrintFlags.ToPrinter() + if err != nil { + return err + } + + o.PrintObj = printer.PrintObj + + o.Clientset, err = clientGetter.K8sClientset() + if err != nil { + return err + } + + return nil +} + +func (o *SlurmOptions) Run(ctx context.Context) error { + for _, jobName := range o.JobNames { + if err := o.deleteJob(ctx, jobName); err != nil { + return err + } + } + + return nil +} + +func (o *SlurmOptions) deleteJob(ctx context.Context, jobName string) error { + job, err := o.Clientset.BatchV1().Jobs(o.Namespace).Get(ctx, jobName, metav1.GetOptions{}) + if client.IgnoreNotFound(err) != nil { + return err + } + if err != nil { + fmt.Fprintln(o.ErrOut, err) + return nil + } + if _, ok := job.Labels[constants.ProfileLabel]; !ok { + fmt.Fprintf(o.ErrOut, "jobs.batch \"%s\" not created via kjob\n", job.Name) + return nil + } + if job.Labels[constants.ModeLabel] != string(v1alpha1.SlurmMode) { + fmt.Fprintf(o.ErrOut, "jobs.batch \"%s\" created in \"%s\" mode. Switch to the correct mode to delete it\n", + job.Name, job.Labels[constants.ModeLabel]) + return nil + } + + if o.DryRunStrategy != util.DryRunClient { + deleteOptions := metav1.DeleteOptions{ + PropagationPolicy: ptr.To(o.CascadeStrategy), + } + + if o.DryRunStrategy == util.DryRunServer { + deleteOptions.DryRun = []string{metav1.DryRunAll} + } + + if err := o.Clientset.BatchV1().Jobs(o.Namespace).Delete(ctx, jobName, deleteOptions); err != nil { + return err + } + } + + return o.PrintObj(job, o.Out) +} diff --git a/pkg/cmd/delete/delete_slurm_test.go b/pkg/cmd/delete/delete_slurm_test.go new file mode 100644 index 0000000..5d711cb --- /dev/null +++ b/pkg/cmd/delete/delete_slurm_test.go @@ -0,0 +1,205 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package delete + +import ( + "context" + "slices" + "testing" + + "github.com/google/go-cmp/cmp" + batchv1 "k8s.io/api/batch/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/cli-runtime/pkg/genericiooptions" + k8sfake "k8s.io/client-go/kubernetes/fake" + kubetesting "k8s.io/client-go/testing" + + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/apis/v1alpha1" + cmdtesting "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/cmd/testing" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/testing/wrappers" +) + +func TestSlurmCmd(t *testing.T) { + testCases := map[string]struct { + ns string + args []string + objs []runtime.Object + wantJobs []batchv1.Job + wantOut string + wantOutErr string + wantErr string + }{ + "shouldn't delete slurm job because it is not found": { + args: []string{"j"}, + objs: []runtime.Object{ + wrappers.MakeJob("j1", metav1.NamespaceDefault).Profile("p1").Mode(v1alpha1.SlurmMode).Obj(), + wrappers.MakeJob("j2", metav1.NamespaceDefault).Profile("p2").Mode(v1alpha1.SlurmMode).Obj(), + }, + wantJobs: []batchv1.Job{ + *wrappers.MakeJob("j1", metav1.NamespaceDefault).Profile("p1").Mode(v1alpha1.SlurmMode).Obj(), + *wrappers.MakeJob("j2", metav1.NamespaceDefault).Profile("p2").Mode(v1alpha1.SlurmMode).Obj(), + }, + wantOutErr: "jobs.batch \"j\" not found\n", + }, + "shouldn't delete slurm job because it is not created via kjob": { + args: []string{"j1"}, + objs: []runtime.Object{ + wrappers.MakeJob("j1", metav1.NamespaceDefault).Obj(), + }, + wantJobs: []batchv1.Job{ + *wrappers.MakeJob("j1", metav1.NamespaceDefault).Obj(), + }, + wantOutErr: "jobs.batch \"j1\" not created via kjob\n", + }, + "shouldn't delete slurm job because it is not used for Job mode": { + args: []string{"j1", "j2"}, + objs: []runtime.Object{ + wrappers.MakeJob("j1", metav1.NamespaceDefault).Profile("p1").Obj(), + wrappers.MakeJob("j2", metav1.NamespaceDefault).Profile("p1").Mode(v1alpha1.JobMode).Obj(), + }, + wantJobs: []batchv1.Job{ + *wrappers.MakeJob("j1", metav1.NamespaceDefault).Profile("p1").Obj(), + *wrappers.MakeJob("j2", metav1.NamespaceDefault).Profile("p1").Mode(v1alpha1.JobMode).Obj(), + }, + wantOutErr: `jobs.batch "j1" created in "" mode. Switch to the correct mode to delete it +jobs.batch "j2" created in "Job" mode. Switch to the correct mode to delete it +`, + }, + "should delete slurm job": { + args: []string{"j1"}, + objs: []runtime.Object{ + wrappers.MakeJob("j1", metav1.NamespaceDefault).Profile("p1").Mode(v1alpha1.SlurmMode).Obj(), + wrappers.MakeJob("j2", metav1.NamespaceDefault).Profile("p2").Mode(v1alpha1.SlurmMode).Obj(), + }, + wantJobs: []batchv1.Job{ + *wrappers.MakeJob("j2", metav1.NamespaceDefault).Profile("p2").Mode(v1alpha1.SlurmMode).Obj(), + }, + wantOut: "job.batch/j1 deleted\n", + }, + "should delete slurm jobs": { + args: []string{"j1", "j2"}, + objs: []runtime.Object{ + wrappers.MakeJob("j1", metav1.NamespaceDefault).Profile("p1").Mode(v1alpha1.SlurmMode).Obj(), + wrappers.MakeJob("j2", metav1.NamespaceDefault).Profile("p2").Mode(v1alpha1.SlurmMode).Obj(), + }, + wantOut: "job.batch/j1 deleted\njob.batch/j2 deleted\n", + }, + "should delete only one slurm job": { + args: []string{"j1", "j"}, + objs: []runtime.Object{ + wrappers.MakeJob("j1", metav1.NamespaceDefault).Profile("p1").Mode(v1alpha1.SlurmMode).Obj(), + wrappers.MakeJob("j2", metav1.NamespaceDefault).Profile("p2").Mode(v1alpha1.SlurmMode).Obj(), + }, + wantJobs: []batchv1.Job{ + *wrappers.MakeJob("j2", metav1.NamespaceDefault).Profile("p2").Mode(v1alpha1.SlurmMode).Obj(), + }, + wantOut: "job.batch/j1 deleted\n", + wantOutErr: "jobs.batch \"j\" not found\n", + }, + "shouldn't delete slurm job with client dry run": { + args: []string{"j1", "--dry-run", "client"}, + objs: []runtime.Object{ + wrappers.MakeJob("j1", metav1.NamespaceDefault).Profile("p1").Mode(v1alpha1.SlurmMode).Obj(), + wrappers.MakeJob("j2", metav1.NamespaceDefault).Profile("p2").Mode(v1alpha1.SlurmMode).Obj(), + }, + wantJobs: []batchv1.Job{ + *wrappers.MakeJob("j1", metav1.NamespaceDefault).Profile("p1").Mode(v1alpha1.SlurmMode).Obj(), + *wrappers.MakeJob("j2", metav1.NamespaceDefault).Profile("p2").Mode(v1alpha1.SlurmMode).Obj(), + }, + wantOut: "job.batch/j1 deleted (client dry run)\n", + }, + "shouldn't delete slurm job with server dry run": { + args: []string{"j1", "--dry-run", "server"}, + objs: []runtime.Object{ + wrappers.MakeJob("j1", metav1.NamespaceDefault).Profile("p1").Mode(v1alpha1.SlurmMode).Obj(), + wrappers.MakeJob("j2", metav1.NamespaceDefault).Profile("p2").Mode(v1alpha1.SlurmMode).Obj(), + }, + wantJobs: []batchv1.Job{ + *wrappers.MakeJob("j1", metav1.NamespaceDefault).Profile("p1").Mode(v1alpha1.SlurmMode).Obj(), + *wrappers.MakeJob("j2", metav1.NamespaceDefault).Profile("p2").Mode(v1alpha1.SlurmMode).Obj(), + }, + wantOut: "job.batch/j1 deleted (server dry run)\n", + }, + "no args": { + args: []string{}, + wantErr: "requires at least 1 arg(s), only received 0", + }, + } + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + ns := metav1.NamespaceDefault + if tc.ns != "" { + ns = tc.ns + } + + streams, _, out, outErr := genericiooptions.NewTestIOStreams() + + clientset := k8sfake.NewSimpleClientset(tc.objs...) + clientset.PrependReactor("delete", "jobs", func(action kubetesting.Action) (handled bool, ret runtime.Object, err error) { + if slices.Contains(action.(kubetesting.DeleteAction).GetDeleteOptions().DryRun, metav1.DryRunAll) { + handled = true + } + return handled, ret, err + }) + + tcg := cmdtesting.NewTestClientGetter(). + WithK8sClientset(clientset). + WithNamespace(ns) + + cmd := NewSlurmCmd(tcg, streams) + cmd.SetOut(out) + cmd.SetErr(outErr) + cmd.SetArgs(tc.args) + + gotErr := cmd.Execute() + + var gotErrStr string + if gotErr != nil { + gotErrStr = gotErr.Error() + } + + if diff := cmp.Diff(tc.wantErr, gotErrStr); diff != "" { + t.Errorf("Unexpected error (-want/+got)\n%s", diff) + } + + if gotErr != nil { + return + } + + gotOut := out.String() + if diff := cmp.Diff(tc.wantOut, gotOut); diff != "" { + t.Errorf("Unexpected output (-want/+got)\n%s", diff) + } + + gotOutErr := outErr.String() + if diff := cmp.Diff(tc.wantOutErr, gotOutErr); diff != "" { + t.Errorf("Unexpected error output (-want/+got)\n%s", diff) + } + + gotJobList, err := clientset.BatchV1().Jobs(tc.ns).List(context.Background(), metav1.ListOptions{}) + if err != nil { + t.Error(err) + return + } + + if diff := cmp.Diff(tc.wantJobs, gotJobList.Items); diff != "" { + t.Errorf("Unexpected jobs (-want/+got)\n%s", diff) + } + }) + } +} diff --git a/pkg/cmd/delete/helpers.go b/pkg/cmd/delete/helpers.go new file mode 100644 index 0000000..b7b93d6 --- /dev/null +++ b/pkg/cmd/delete/helpers.go @@ -0,0 +1,52 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package delete + +import ( + "fmt" + + "github.com/spf13/cobra" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/cmd/util" +) + +const ( + cascadingFlagName = "cascade" +) + +func addCascadingFlag(cmd *cobra.Command) { + cmd.PersistentFlags().String( + cascadingFlagName, + "background", + `Must be "background", "orphan", or "foreground". Defaults to background.`, + ) +} + +func getCascadingStrategy(cmd *cobra.Command) (metav1.DeletionPropagation, error) { + cascadingFlag := util.FlagString(cmd, cascadingFlagName) + switch cascadingFlag { + case "orphan": + return metav1.DeletePropagationOrphan, nil + case "foreground": + return metav1.DeletePropagationForeground, nil + case "background": + return metav1.DeletePropagationBackground, nil + default: + return metav1.DeletePropagationBackground, fmt.Errorf(`invalid cascade value (%v). Must be "background", "foreground", or "orphan"`, cascadingFlag) + } +} diff --git a/pkg/cmd/describe/describe.go b/pkg/cmd/describe/describe.go new file mode 100644 index 0000000..476cb85 --- /dev/null +++ b/pkg/cmd/describe/describe.go @@ -0,0 +1,352 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package describe + +import ( + "context" + "fmt" + "slices" + "strings" + + "github.com/spf13/cobra" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/errors" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/cli-runtime/pkg/genericiooptions" + "k8s.io/cli-runtime/pkg/resource" + "k8s.io/client-go/kubernetes" + "k8s.io/kubectl/pkg/util/templates" + + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/apis/v1alpha1" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/cmd/util" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/constants" +) + +var ( + jobExample = templates.Examples(` + # Describe a task with job mode + kjobctl describe job sample-job + + # Describe a task with job mode + kjobctl describe job/sample-job + + # Describe all tasks with job mode + kjobctl describe job + + # Describe tasks by label name=myLabel + kjobctl describe job -l name=myLabel + `) +) + +const ( + modeTaskArgsFormat = iota + modeSlashTaskArgsFormat + modeArgsFormat +) + +type DescribeOptions struct { + AllNamespaces bool + Namespace string + ProfileName string + ModeName string + TaskName string + LabelSelector string + + UserSpecifiedTask []string + + argsFormat int + ResourceGVK schema.GroupVersionKind + ResourceBuilder *resource.Builder + + Clientset kubernetes.Interface + + genericiooptions.IOStreams +} + +func NewDescribeOptions(streams genericiooptions.IOStreams) *DescribeOptions { + return &DescribeOptions{ + IOStreams: streams, + } +} + +func NewDescribeCmd(clientGetter util.ClientGetter, streams genericiooptions.IOStreams) *cobra.Command { + o := NewDescribeOptions(streams) + + cmd := &cobra.Command{ + Use: "describe MODE NAME", + DisableFlagsInUseLine: true, + Short: "Show details of a specific resource or group of resources.", + Example: jobExample, + Args: cobra.MatchAll(cobra.RangeArgs(1, 2), cobra.OnlyValidArgs), + RunE: func(cmd *cobra.Command, args []string) error { + // stop the usage if we've got this far + cmd.SilenceUsage = true + + err := o.Complete(clientGetter, args) + if err != nil { + return err + } + + return o.Run(cmd.Context()) + }, + } + + util.AddAllNamespacesFlagVar(cmd, &o.AllNamespaces) + util.AddLabelSelectorFlagVar(cmd, &o.LabelSelector) + util.AddProfileFlagVar(cmd, &o.ProfileName) + + return cmd +} + +func (o *DescribeOptions) Complete(clientGetter util.ClientGetter, args []string) error { + var err error + + o.findArgsFormat(args) + + err = o.parseArgs(args) + if err != nil { + return err + } + + resource := resourceFor(o.ModeName) + mapper, err := clientGetter.ToRESTMapper() + if err != nil { + return err + } + o.ResourceGVK, err = gvkFor(mapper, resource) + if err != nil { + return err + } + + o.Namespace, _, err = clientGetter.ToRawKubeConfigLoader().Namespace() + if err != nil { + return err + } + + o.ResourceBuilder = clientGetter.NewResourceBuilder() + + o.Clientset, err = clientGetter.K8sClientset() + if err != nil { + return err + } + + return nil +} + +func (o *DescribeOptions) findArgsFormat(args []string) { + if len(args) == 2 { + o.argsFormat = modeTaskArgsFormat + } else { + if strings.Contains(args[0], "/") { + o.argsFormat = modeSlashTaskArgsFormat + } else { + o.argsFormat = modeArgsFormat + } + } +} + +func (o *DescribeOptions) parseArgs(args []string) error { + switch o.argsFormat { + case modeTaskArgsFormat: + o.ModeName, o.TaskName = args[0], args[1] + case modeSlashTaskArgsFormat: + parsedArg, err := parseAppProfileModeName(args[0]) + if err != nil { + return err + } + o.ModeName, o.TaskName = parsedArg[0], parsedArg[1] + default: + o.ModeName = args[0] + } + + return nil +} + +func (o *DescribeOptions) Run(ctx context.Context) error { + builder := o.customizeResourceBuilder() + + r := builder.Do() + if err := r.Err(); err != nil { + return err + } + infos, err := r.Infos() + if err != nil { + return err + } + + if strings.EqualFold(o.ModeName, string(v1alpha1.SlurmMode)) { + configMapsAsInfos, err := o.getConfigMaps(ctx) + if err != nil { + return err + } + + infos = append(infos, configMapsAsInfos...) + } + + // In Slurm mode, configMaps are created with the same name as the jobs they are mounted on. + // Since the first half of the slice contains jobs and the second half contains configMaps, + // sorting objects by name will interleave jobs with their corresponding configMap. + slices.SortFunc(infos, func(a, b *resource.Info) int { + return strings.Compare(a.Name, b.Name) + }) + + allErrs := []error{} + errs := sets.NewString() + first := true + for _, info := range infos { + obj, ok := info.Object.(*unstructured.Unstructured) + if !ok { + err := fmt.Errorf("invalid object %+v. Unexpected type %T", obj, info.Object) + if errs.Has(err.Error()) { + continue + } + allErrs = append(allErrs, err) + errs.Insert(err.Error()) + continue + } + + labels := obj.GetLabels() + if _, ok := labels[constants.ProfileLabel]; !ok { + continue + } + + mapping := info.ResourceMapping() + describer, err := NewResourceDescriber(mapping) + if err != nil { + if errs.Has(err.Error()) { + continue + } + allErrs = append(allErrs, err) + errs.Insert(err.Error()) + continue + } + + output, err := describer.Describe(obj) + if err != nil { + if errs.Has(err.Error()) { + continue + } + allErrs = append(allErrs, err) + errs.Insert(err.Error()) + continue + } + + if first { + first = false + fmt.Fprint(o.Out, output) + } else { + fmt.Fprintf(o.Out, "\n\n%s", output) + } + } + + if first && len(allErrs) == 0 { + if o.AllNamespaces { + fmt.Fprintln(o.ErrOut, "No resources found") + } else { + fmt.Fprintf(o.ErrOut, "No resources found in %s namespace.\n", o.Namespace) + } + } + + return errors.NewAggregate(allErrs) +} + +func (o *DescribeOptions) customizeResourceBuilder() *resource.Builder { + builder := o.ResourceBuilder. + Unstructured(). + NamespaceParam(o.Namespace). + DefaultNamespace() + switch o.argsFormat { + case modeTaskArgsFormat, modeSlashTaskArgsFormat: + builder = builder.ResourceTypeOrNameArgs(true, o.ResourceGVK.Kind, o.TaskName). + SingleResourceType() + default: + selector := constants.ProfileLabel + if o.ProfileName != "" { + selector += fmt.Sprintf("=%s", o.ProfileName) + } + if o.LabelSelector != "" { + selector += fmt.Sprintf(",%s", o.LabelSelector) + } + + builder = builder.AllNamespaces(o.AllNamespaces). + LabelSelectorParam(selector). + ResourceTypeOrNameArgs(true, o.ResourceGVK.Kind). + ContinueOnError() + } + + return builder.Flatten() +} + +func (o *DescribeOptions) getConfigMaps(ctx context.Context) ([]*resource.Info, error) { + infos := make([]*resource.Info, 0) + + if o.argsFormat == modeTaskArgsFormat || o.argsFormat == modeSlashTaskArgsFormat { + cm, err := o.Clientset.CoreV1().ConfigMaps(o.Namespace).Get(ctx, o.TaskName, metav1.GetOptions{}) + if err != nil { + return nil, err + } + + info, err := configMapToInfo(cm) + if err != nil { + return nil, err + } + + infos = append(infos, info) + } else { + cmList, err := o.Clientset.CoreV1().ConfigMaps(o.Namespace).List(ctx, metav1.ListOptions{ + LabelSelector: fmt.Sprintf("%s,%s=%s", constants.ProfileLabel, constants.ModeLabel, v1alpha1.SlurmMode), + }) + if err != nil { + return nil, err + } + + for _, cm := range cmList.Items { + info, err := configMapToInfo(&cm) + if err != nil { + return nil, err + } + + infos = append(infos, info) + } + } + + return infos, nil +} + +func configMapToInfo(cm *corev1.ConfigMap) (*resource.Info, error) { + u, err := runtime.DefaultUnstructuredConverter.ToUnstructured(cm) + if err != nil { + return nil, err + } + + obj := &unstructured.Unstructured{Object: u} + return &resource.Info{ + Mapping: &meta.RESTMapping{ + GroupVersionKind: schema.GroupVersionKind{ + Version: "v1", + Kind: "ConfigMap", + }, + }, + Name: obj.GetName(), + Object: obj, + }, nil +} diff --git a/pkg/cmd/describe/describe_printer.go b/pkg/cmd/describe/describe_printer.go new file mode 100644 index 0000000..c8c3ba0 --- /dev/null +++ b/pkg/cmd/describe/describe_printer.go @@ -0,0 +1,335 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package describe + +import ( + "fmt" + "io" + "strconv" + "time" + + rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/duration" + describehelper "k8s.io/kubectl/pkg/describe" + "k8s.io/utils/ptr" + utilmaps "sigs.k8s.io/kueue/pkg/util/maps" +) + +// ResourceDescriber generates output for the named resource or an error +// if the output could not be generated. Implementers typically +// abstract the retrieval of the named object from a remote server. +type ResourceDescriber interface { + Describe(object *unstructured.Unstructured) (output string, err error) +} + +// NewResourceDescriber returns a Describer for displaying the specified RESTMapping type or an error. +func NewResourceDescriber(mapping *meta.RESTMapping) (ResourceDescriber, error) { + if describer, ok := DescriberFor(mapping.GroupVersionKind.GroupKind()); ok { + return describer, nil + } + + return nil, fmt.Errorf("no description has been implemented for %s", mapping.GroupVersionKind.String()) +} + +// DescriberFor returns the default describe functions for each of the standard +// Kubernetes types. +func DescriberFor(kind schema.GroupKind) (ResourceDescriber, bool) { + describers := map[schema.GroupKind]ResourceDescriber{ + {Group: batchv1.GroupName, Kind: "Job"}: &JobDescriber{}, + {Group: corev1.GroupName, Kind: "Pod"}: &PodDescriber{}, + {Group: rayv1.GroupVersion.Group, Kind: "RayJob"}: &RayJobDescriber{}, + {Group: rayv1.GroupVersion.Group, Kind: "RayCluster"}: &RayClusterDescriber{}, + {Group: corev1.GroupName, Kind: "ConfigMap"}: &ConfigMapDescriber{}, + } + + f, ok := describers[kind] + return f, ok +} + +// JobDescriber generates information about a job and the pods it has created. +type JobDescriber struct{} + +func (d *JobDescriber) Describe(object *unstructured.Unstructured) (string, error) { + job := &batchv1.Job{} + err := runtime.DefaultUnstructuredConverter.FromUnstructured(object.UnstructuredContent(), job) + if err != nil { + return "", err + } + + return describeJob(job) +} + +func describeJob(job *batchv1.Job) (string, error) { + return tabbedString(func(out io.Writer) error { + w := describehelper.NewPrefixWriter(out) + + w.Write(IndentLevelZero, "Name:\t%s\n", job.Name) + w.Write(IndentLevelZero, "Namespace:\t%s\n", job.Namespace) + printLabelsMultiline(w, "Labels", job.Labels) + printLabelsMultiline(w, "Annotations", job.Annotations) + if job.Spec.Parallelism != nil { + w.Write(IndentLevelZero, "Parallelism:\t%d\n", *job.Spec.Parallelism) + } + if job.Spec.Completions != nil { + w.Write(IndentLevelZero, "Completions:\t%d\n", *job.Spec.Completions) + } else { + w.Write(IndentLevelZero, "Completions:\t\n") + } + if job.Status.StartTime != nil { + w.Write(IndentLevelZero, "Start Time:\t%s\n", job.Status.StartTime.Time.Format(time.RFC1123Z)) + } + if job.Status.CompletionTime != nil { + w.Write(IndentLevelZero, "Completed At:\t%s\n", job.Status.CompletionTime.Time.Format(time.RFC1123Z)) + } + if job.Status.StartTime != nil && job.Status.CompletionTime != nil { + w.Write(IndentLevelZero, "Duration:\t%s\n", duration.HumanDuration(job.Status.CompletionTime.Sub(job.Status.StartTime.Time))) + } + if job.Status.Ready == nil { + w.Write(IndentLevelZero, "Pods Statuses:\t%d Active / %d Succeeded / %d Failed\n", job.Status.Active, job.Status.Succeeded, job.Status.Failed) + } else { + w.Write(IndentLevelZero, "Pods Statuses:\t%d Active (%d Ready) / %d Succeeded / %d Failed\n", job.Status.Active, *job.Status.Ready, job.Status.Succeeded, job.Status.Failed) + } + + describePodTemplate(&job.Spec.Template, w) + + return nil + }) +} + +// PodDescriber generates information about a pod. +type PodDescriber struct{} + +func (d *PodDescriber) Describe(object *unstructured.Unstructured) (string, error) { + pod := &corev1.Pod{} + err := runtime.DefaultUnstructuredConverter.FromUnstructured(object.UnstructuredContent(), pod) + if err != nil { + return "", err + } + + return describePod(pod) +} + +func describePod(pod *corev1.Pod) (string, error) { + return tabbedString(func(out io.Writer) error { + w := describehelper.NewPrefixWriter(out) + + w.Write(IndentLevelZero, "Name:\t%s\n", pod.Name) + w.Write(IndentLevelZero, "Namespace:\t%s\n", pod.Namespace) + if pod.Status.StartTime != nil { + w.Write(IndentLevelZero, "Start Time:\t%s\n", pod.Status.StartTime.Time.Format(time.RFC1123Z)) + } + printLabelsMultiline(w, "Labels", pod.Labels) + + if pod.DeletionTimestamp != nil && pod.Status.Phase != corev1.PodFailed && pod.Status.Phase != corev1.PodSucceeded { + w.Write(IndentLevelZero, "Status:\tTerminating (lasts %s)\n", translateTimestampSince(*pod.DeletionTimestamp)) + w.Write(IndentLevelZero, "Termination Grace Period:\t%ds\n", *pod.DeletionGracePeriodSeconds) + } else { + w.Write(IndentLevelZero, "Status:\t%s\n", string(pod.Status.Phase)) + } + if len(pod.Status.Reason) > 0 { + w.Write(IndentLevelZero, "Reason:\t%s\n", pod.Status.Reason) + } + if len(pod.Status.Message) > 0 { + w.Write(IndentLevelZero, "Message:\t%s\n", pod.Status.Message) + } + + if len(pod.Spec.InitContainers) > 0 { + describeContainers("Init Containers", pod.Spec.InitContainers, w, "") + } + describeContainers("Containers", pod.Spec.Containers, w, "") + if len(pod.Spec.EphemeralContainers) > 0 { + var ec []corev1.Container + for i := range pod.Spec.EphemeralContainers { + ec = append(ec, corev1.Container(pod.Spec.EphemeralContainers[i].EphemeralContainerCommon)) + } + describeContainers("Ephemeral Containers", ec, w, "") + } + + describeVolumes(pod.Spec.Volumes, w, "") + + return nil + }) +} + +// RayJobDescriber generates information about a ray job. +type RayJobDescriber struct{} + +func (d *RayJobDescriber) Describe(object *unstructured.Unstructured) (string, error) { + rayJob := &rayv1.RayJob{} + err := runtime.DefaultUnstructuredConverter.FromUnstructured(object.UnstructuredContent(), rayJob) + if err != nil { + return "", err + } + + return describeRayJob(rayJob) +} + +func describeRayJob(rayJob *rayv1.RayJob) (string, error) { + return tabbedString(func(out io.Writer) error { + w := describehelper.NewPrefixWriter(out) + + w.Write(IndentLevelZero, "Name:\t%s\n", rayJob.Name) + w.Write(IndentLevelZero, "Namespace:\t%s\n", rayJob.Namespace) + if rayJob.Status.StartTime != nil { + w.Write(IndentLevelZero, "Start Time:\t%s\n", rayJob.Status.StartTime.Format(time.RFC1123Z)) + } + if rayJob.Status.EndTime != nil { + w.Write(IndentLevelZero, "End Time:\t%s\n", rayJob.Status.EndTime.Format(time.RFC1123Z)) + } + printLabelsMultiline(w, "Labels", rayJob.Labels) + + if rayJob.DeletionTimestamp != nil { + w.Write(IndentLevelZero, "Job Deployment Status:\tTerminating (lasts %s)\n", translateTimestampSince(*rayJob.DeletionTimestamp)) + w.Write(IndentLevelZero, "Job Status:\tTerminating (lasts %s)\n", translateTimestampSince(*rayJob.DeletionTimestamp)) + w.Write(IndentLevelZero, "Termination Grace Period:\t%ds\n", *rayJob.DeletionGracePeriodSeconds) + } else { + if len(rayJob.Status.JobDeploymentStatus) > 0 { + w.Write(IndentLevelZero, "Job Deployment Status:\t%s\n", string(rayJob.Status.JobDeploymentStatus)) + } + if len(rayJob.Status.JobStatus) > 0 { + w.Write(IndentLevelZero, "Job Status:\t%s\n", string(rayJob.Status.JobStatus)) + } + if len(rayJob.Status.Reason) > 0 { + w.Write(IndentLevelZero, "Reason:\t%s\n", rayJob.Status.Reason) + } + if len(rayJob.Status.Message) > 0 { + w.Write(IndentLevelZero, "Message:\t%s\n", rayJob.Status.Message) + } + } + + if len(rayJob.Status.RayClusterName) > 0 { + w.Write(IndentLevelZero, "Ray Cluster Name:\t%s\n", rayJob.Status.RayClusterName) + } + + w.Write(IndentLevelZero, "Ray Cluster Status:\n") + w.Write(IndentLevelOne, "Desired CPU:\t%s\n", rayJob.Status.RayClusterStatus.DesiredCPU.String()) + w.Write(IndentLevelOne, "Desired GPU:\t%s\n", rayJob.Status.RayClusterStatus.DesiredGPU.String()) + w.Write(IndentLevelOne, "Desired Memory:\t%s\n", rayJob.Status.RayClusterStatus.DesiredMemory.String()) + w.Write(IndentLevelOne, "Desired TPU:\t%s\n", rayJob.Status.RayClusterStatus.DesiredTPU.String()) + + return nil + }) +} + +// RayClusterDescriber generates information about a ray cluster. +type RayClusterDescriber struct{} + +func (d *RayClusterDescriber) Describe(object *unstructured.Unstructured) (string, error) { + rayCluster := &rayv1.RayCluster{} + err := runtime.DefaultUnstructuredConverter.FromUnstructured(object.UnstructuredContent(), rayCluster) + if err != nil { + return "", err + } + + return describeRayCluster(rayCluster) +} + +func describeRayCluster(rayCluster *rayv1.RayCluster) (string, error) { + return tabbedString(func(out io.Writer) error { + w := describehelper.NewPrefixWriter(out) + + w.Write(IndentLevelZero, "Name:\t%s\n", rayCluster.Name) + w.Write(IndentLevelZero, "Namespace:\t%s\n", rayCluster.Namespace) + + printLabelsMultiline(w, "Labels", rayCluster.Labels) + + w.Write(IndentLevelZero, "Suspend:\t%t\n", ptr.Deref(rayCluster.Spec.Suspend, false)) + w.Write(IndentLevelZero, "State:\t%s\n", string(rayCluster.Status.State)) + if len(rayCluster.Status.Reason) > 0 { + w.Write(IndentLevelZero, "Reason:\t%s\n", rayCluster.Status.Reason) + } + + w.Write(IndentLevelZero, "Desired CPU:\t%s\n", rayCluster.Status.DesiredCPU.String()) + w.Write(IndentLevelZero, "Desired GPU:\t%s\n", rayCluster.Status.DesiredGPU.String()) + w.Write(IndentLevelZero, "Desired Memory:\t%s\n", rayCluster.Status.DesiredMemory.String()) + w.Write(IndentLevelZero, "Desired TPU:\t%s\n", rayCluster.Status.DesiredTPU.String()) + + w.Write(IndentLevelZero, "Ready Worker Replicas:\t%d\n", rayCluster.Status.ReadyWorkerReplicas) + w.Write(IndentLevelZero, "Available Worker Replicas:\t%d\n", rayCluster.Status.AvailableWorkerReplicas) + w.Write(IndentLevelZero, "Desired Worker Replicas:\t%d\n", rayCluster.Status.DesiredWorkerReplicas) + w.Write(IndentLevelZero, "Min Worker Replicas:\t%d\n", rayCluster.Status.MinWorkerReplicas) + w.Write(IndentLevelZero, "Max Worker Replicas:\t%d\n", rayCluster.Status.MaxWorkerReplicas) + + w.Write(IndentLevelZero, "Head Group:\n") + headGroupWriter := describehelper.NewNestedPrefixWriter(w, 1) + printLabelsMultiline(headGroupWriter, "Start Params", rayCluster.Spec.HeadGroupSpec.RayStartParams) + describePodTemplate(&rayCluster.Spec.HeadGroupSpec.Template, headGroupWriter) + + w.Write(IndentLevelZero, "Worker Groups:\n") + for _, wg := range rayCluster.Spec.WorkerGroupSpecs { + w.Write(IndentLevelOne, fmt.Sprintf("%s:\n", wg.GroupName)) + workerGroupWriter := describehelper.NewNestedPrefixWriter(w, 2) + if wg.Replicas != nil { + workerGroupWriter.Write(IndentLevelZero, "Replicas:\t%d\n", *wg.Replicas) + } + if wg.MinReplicas != nil { + workerGroupWriter.Write(IndentLevelZero, "Min Replicas:\t%d\n", *wg.MinReplicas) + } + if wg.MaxReplicas != nil { + workerGroupWriter.Write(IndentLevelZero, "Max Replicas:\t%d\n", *wg.MaxReplicas) + } + printLabelsMultiline(workerGroupWriter, "Start Params", wg.RayStartParams) + describePodTemplate(&wg.Template, workerGroupWriter) + } + + return nil + }) +} + +// ConfigMapDescriber generates information about a configMap. +type ConfigMapDescriber struct{} + +func (d *ConfigMapDescriber) Describe(object *unstructured.Unstructured) (string, error) { + configMap := &corev1.ConfigMap{} + err := runtime.DefaultUnstructuredConverter.FromUnstructured(object.UnstructuredContent(), configMap) + if err != nil { + return "", err + } + + return describeConfigMap(configMap) +} + +func describeConfigMap(configMap *corev1.ConfigMap) (string, error) { + return tabbedString(func(out io.Writer) error { + w := describehelper.NewPrefixWriter(out) + + w.Write(IndentLevelZero, "Name:\t%s\n", configMap.Name) + w.Write(IndentLevelZero, "Namespace:\t%s\n", configMap.Namespace) + printLabelsMultiline(w, "Labels", configMap.Labels) + + w.Write(IndentLevelZero, "\nData\n====\n") + for _, k := range utilmaps.SortedKeys(configMap.Data) { + w.Write(IndentLevelZero, "%s:\n----\n", k) + w.Write(IndentLevelZero, "%s\n", configMap.Data[k]) + w.Write(IndentLevelZero, "\n") + } + + w.Write(IndentLevelZero, "\nBinaryData\n====\n") + for _, k := range utilmaps.SortedKeys(configMap.BinaryData) { + w.Write(IndentLevelZero, "%s: %s bytes\n", k, strconv.Itoa(len(configMap.BinaryData[k]))) + } + w.Write(IndentLevelZero, "\n") + + return nil + }) +} diff --git a/pkg/cmd/describe/describe_test.go b/pkg/cmd/describe/describe_test.go new file mode 100644 index 0000000..cced729 --- /dev/null +++ b/pkg/cmd/describe/describe_test.go @@ -0,0 +1,830 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package describe + +import ( + "io" + "net/http" + "strings" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/meta" + apiresource "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/runtime/serializer" + "k8s.io/cli-runtime/pkg/genericiooptions" + "k8s.io/cli-runtime/pkg/resource" + k8sfake "k8s.io/client-go/kubernetes/fake" + restfake "k8s.io/client-go/rest/fake" + "k8s.io/utils/ptr" + + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/apis/v1alpha1" + cmdtesting "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/cmd/testing" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/constants" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/testing/wrappers" +) + +func TestDescribeCmd(t *testing.T) { + testCases := map[string]struct { + args []string + argsFormat int + objs []runtime.Object + withK8sClientSet bool + mapperKinds []schema.GroupVersionKind + wantOut string + wantOutErr string + wantErr error + }{ + "shouldn't describe none kjobctl owned jobs": { + args: []string{"job", "sample-job-8c7zt"}, + argsFormat: modeTaskArgsFormat, + mapperKinds: []schema.GroupVersionKind{ + batchv1.SchemeGroupVersion.WithKind("Job"), + }, + objs: []runtime.Object{ + wrappers.MakeJob("sample-job-8c7zt", metav1.NamespaceDefault).Obj(), + }, + wantOutErr: "No resources found in default namespace.\n", + }, + "describe job with 'mode task' format": { + args: []string{"job", "sample-job-8c7zt"}, + argsFormat: modeTaskArgsFormat, + mapperKinds: []schema.GroupVersionKind{ + batchv1.SchemeGroupVersion.WithKind("Job"), + }, + objs: []runtime.Object{ + getSampleJob("sample-job-8c7zt"), + }, + wantOut: `Name: sample-job-8c7zt +Namespace: default +Labels: kjobctl.x-k8s.io/profile=sample-profile +Annotations: kjobctl.x-k8s.io/script=test.sh +Parallelism: 3 +Completions: 2 +Start Time: Mon, 01 Jan 2024 00:00:00 +0000 +Completed At: Mon, 01 Jan 2024 00:00:33 +0000 +Duration: 33s +Pods Statuses: 0 Active / 2 Succeeded / 0 Failed +Pod Template: + Containers: + sample-container: + Port: + Host Port: + Command: + sleep + 15s + Args: + 30s + Requests: + cpu: 1 + memory: 200Mi + Environment: + Mounts: + Volumes: +`, + }, + "describe slurm with 'mode task' format": { + args: []string{"slurm", "sample-job-8c7zt"}, + argsFormat: modeTaskArgsFormat, + mapperKinds: []schema.GroupVersionKind{ + batchv1.SchemeGroupVersion.WithKind("Job"), + corev1.SchemeGroupVersion.WithKind("ConfigMap"), + }, + withK8sClientSet: true, + objs: []runtime.Object{ + getSampleJob("sample-job-8c7zt"), + wrappers.MakeConfigMap("sample-job-8c7zt", "default"). + Profile("profile"). + Mode(v1alpha1.SlurmMode). + Data(map[string]string{ + "script": "#!/bin/bash\nsleep 300", + "entrypoint.sh": `#!/usr/bin/bash + +set -o errexit +set -o nounset +set -o pipefail + +# External variables +# JOB_COMPLETION_INDEX - completion index of the job. +# JOB_CONTAINER_INDEX - container index in the container template. + +# ["COMPLETION_INDEX"]="CONTAINER_INDEX_1,CONTAINER_INDEX_2" +declare -A array_indexes=(["0"]="0") # Requires bash v4+ + +container_indexes=${array_indexes[${JOB_COMPLETION_INDEX}]} +container_indexes=(${container_indexes//,/ }) + +if [[ ! -v container_indexes[${JOB_CONTAINER_INDEX}] ]]; +then +exit 0 +fi + +SBATCH_ARRAY_INX= +SBATCH_GPUS_PER_TASK= +SBATCH_MEM_PER_CPU= +SBATCH_MEM_PER_GPU= +SBATCH_OUTPUT= +SBATCH_ERROR= +SBATCH_INPUT= +SBATCH_JOB_NAME= +SBATCH_PARTITION= + +export SLURM_ARRAY_JOB_ID=1 # Job array’s master job ID number. +export SLURM_ARRAY_TASK_COUNT=1 # Total number of tasks in a job array. +export SLURM_ARRAY_TASK_MAX=0 # Job array’s maximum ID (index) number. +export SLURM_ARRAY_TASK_MIN=0 # Job array’s minimum ID (index) number. +export SLURM_TASKS_PER_NODE=1 # Number of tasks to be initiated on each node. +export SLURM_CPUS_PER_TASK= # Number of CPUs per task. +export SLURM_CPUS_ON_NODE= # Number of CPUs on the allocated node (actually pod). +export SLURM_JOB_CPUS_PER_NODE= # Count of processors available to the job on this node. +export SLURM_CPUS_PER_GPU= # Number of CPUs requested per allocated GPU. +export SLURM_MEM_PER_CPU= # Memory per CPU. Same as --mem-per-cpu . +export SLURM_MEM_PER_GPU= # Memory per GPU. +export SLURM_MEM_PER_NODE= # Memory per node. Same as --mem. +export SLURM_GPUS=0 # Number of GPUs requested (in total). +export SLURM_NTASKS=1 # Same as -n, –ntasks. The number of tasks. +export SLURM_NTASKS_PER_NODE=1 # Number of tasks requested per node. +export SLURM_NPROCS=$SLURM_NTASKS # Same as -n, --ntasks. See $SLURM_NTASKS. +export SLURM_NNODES=1 # Total number of nodes (actually pods) in the job’s resource allocation. +export SLURM_SUBMIT_DIR=/slurm # The path of the job submission directory. +export SLURM_SUBMIT_HOST=$HOSTNAME # The hostname of the node used for job submission. + +export SLURM_JOB_ID=$(( JOB_COMPLETION_INDEX * SLURM_TASKS_PER_NODE + JOB_CONTAINER_INDEX + SLURM_ARRAY_JOB_ID )) # The Job ID. +export SLURM_JOBID=$SLURM_JOB_ID # Deprecated. Same as $SLURM_JOB_ID +export SLURM_ARRAY_TASK_ID=${container_indexes[${JOB_CONTAINER_INDEX}]} # Task ID. + +unmask_filename () { +replaced="$1" + +if [[ "$replaced" == "\\"* ]]; then +replaced="${replaced//\\/}" +echo "${replaced}" +return 0 +fi + +replaced=$(echo "$replaced" | sed -E "s/(%)(%A)/\1\n\2/g;:a s/(^|[^\n])%A/\1$SLURM_ARRAY_JOB_ID/;ta;s/\n//g") +replaced=$(echo "$replaced" | sed -E "s/(%)(%a)/\1\n\2/g;:a s/(^|[^\n])%a/\1$SLURM_ARRAY_TASK_ID/;ta;s/\n//g") +replaced=$(echo "$replaced" | sed -E "s/(%)(%j)/\1\n\2/g;:a s/(^|[^\n])%j/\1$SLURM_JOB_ID/;ta;s/\n//g") +replaced=$(echo "$replaced" | sed -E "s/(%)(%N)/\1\n\2/g;:a s/(^|[^\n])%N/\1$HOSTNAME/;ta;s/\n//g") +replaced=$(echo "$replaced" | sed -E "s/(%)(%n)/\1\n\2/g;:a s/(^|[^\n])%n/\1$JOB_COMPLETION_INDEX/;ta;s/\n//g") +replaced=$(echo "$replaced" | sed -E "s/(%)(%t)/\1\n\2/g;:a s/(^|[^\n])%t/\1$SLURM_ARRAY_TASK_ID/;ta;s/\n//g") +replaced=$(echo "$replaced" | sed -E "s/(%)(%u)/\1\n\2/g;:a s/(^|[^\n])%u/\1$USER_ID/;ta;s/\n//g") +replaced=$(echo "$replaced" | sed -E "s/(%)(%x)/\1\n\2/g;:a s/(^|[^\n])%x/\1$SBATCH_JOB_NAME/;ta;s/\n//g") + +replaced="${replaced//%%/%}" + +echo "$replaced" +} + +input_file=$(unmask_filename "$SBATCH_INPUT") +output_file=$(unmask_filename "$SBATCH_OUTPUT") +error_path=$(unmask_filename "$SBATCH_ERROR") + +/slurm/script +`, + }). + Obj(), + }, + wantOut: `Name: sample-job-8c7zt +Namespace: default +Labels: kjobctl.x-k8s.io/profile=sample-profile +Annotations: kjobctl.x-k8s.io/script=test.sh +Parallelism: 3 +Completions: 2 +Start Time: Mon, 01 Jan 2024 00:00:00 +0000 +Completed At: Mon, 01 Jan 2024 00:00:33 +0000 +Duration: 33s +Pods Statuses: 0 Active / 2 Succeeded / 0 Failed +Pod Template: + Containers: + sample-container: + Port: + Host Port: + Command: + sleep + 15s + Args: + 30s + Requests: + cpu: 1 + memory: 200Mi + Environment: + Mounts: + Volumes: + + +Name: sample-job-8c7zt +Namespace: default +Labels: kjobctl.x-k8s.io/mode=Slurm + kjobctl.x-k8s.io/profile=profile + +Data +==== +entrypoint.sh: +---- +#!/usr/bin/bash + +set -o errexit +set -o nounset +set -o pipefail + +# External variables +# JOB_COMPLETION_INDEX - completion index of the job. +# JOB_CONTAINER_INDEX - container index in the container template. + +# ["COMPLETION_INDEX"]="CONTAINER_INDEX_1,CONTAINER_INDEX_2" +declare -A array_indexes=(["0"]="0") # Requires bash v4+ + +container_indexes=${array_indexes[${JOB_COMPLETION_INDEX}]} +container_indexes=(${container_indexes//,/ }) + +if [[ ! -v container_indexes[${JOB_CONTAINER_INDEX}] ]]; +then +exit 0 +fi + +SBATCH_ARRAY_INX= +SBATCH_GPUS_PER_TASK= +SBATCH_MEM_PER_CPU= +SBATCH_MEM_PER_GPU= +SBATCH_OUTPUT= +SBATCH_ERROR= +SBATCH_INPUT= +SBATCH_JOB_NAME= +SBATCH_PARTITION= + +export SLURM_ARRAY_JOB_ID=1 # Job array’s master job ID number. +export SLURM_ARRAY_TASK_COUNT=1 # Total number of tasks in a job array. +export SLURM_ARRAY_TASK_MAX=0 # Job array’s maximum ID (index) number. +export SLURM_ARRAY_TASK_MIN=0 # Job array’s minimum ID (index) number. +export SLURM_TASKS_PER_NODE=1 # Number of tasks to be initiated on each node. +export SLURM_CPUS_PER_TASK= # Number of CPUs per task. +export SLURM_CPUS_ON_NODE= # Number of CPUs on the allocated node (actually pod). +export SLURM_JOB_CPUS_PER_NODE= # Count of processors available to the job on this node. +export SLURM_CPUS_PER_GPU= # Number of CPUs requested per allocated GPU. +export SLURM_MEM_PER_CPU= # Memory per CPU. Same as --mem-per-cpu . +export SLURM_MEM_PER_GPU= # Memory per GPU. +export SLURM_MEM_PER_NODE= # Memory per node. Same as --mem. +export SLURM_GPUS=0 # Number of GPUs requested (in total). +export SLURM_NTASKS=1 # Same as -n, –ntasks. The number of tasks. +export SLURM_NTASKS_PER_NODE=1 # Number of tasks requested per node. +export SLURM_NPROCS=$SLURM_NTASKS # Same as -n, --ntasks. See $SLURM_NTASKS. +export SLURM_NNODES=1 # Total number of nodes (actually pods) in the job’s resource allocation. +export SLURM_SUBMIT_DIR=/slurm # The path of the job submission directory. +export SLURM_SUBMIT_HOST=$HOSTNAME # The hostname of the node used for job submission. + +export SLURM_JOB_ID=$(( JOB_COMPLETION_INDEX * SLURM_TASKS_PER_NODE + JOB_CONTAINER_INDEX + SLURM_ARRAY_JOB_ID )) # The Job ID. +export SLURM_JOBID=$SLURM_JOB_ID # Deprecated. Same as $SLURM_JOB_ID +export SLURM_ARRAY_TASK_ID=${container_indexes[${JOB_CONTAINER_INDEX}]} # Task ID. + +unmask_filename () { +replaced="$1" + +if [[ "$replaced" == "\\"* ]]; then +replaced="${replaced//\\/}" +echo "${replaced}" +return 0 +fi + +replaced=$(echo "$replaced" | sed -E "s/(%)(%A)/\1\n\2/g;:a s/(^|[^\n])%A/\1$SLURM_ARRAY_JOB_ID/;ta;s/\n//g") +replaced=$(echo "$replaced" | sed -E "s/(%)(%a)/\1\n\2/g;:a s/(^|[^\n])%a/\1$SLURM_ARRAY_TASK_ID/;ta;s/\n//g") +replaced=$(echo "$replaced" | sed -E "s/(%)(%j)/\1\n\2/g;:a s/(^|[^\n])%j/\1$SLURM_JOB_ID/;ta;s/\n//g") +replaced=$(echo "$replaced" | sed -E "s/(%)(%N)/\1\n\2/g;:a s/(^|[^\n])%N/\1$HOSTNAME/;ta;s/\n//g") +replaced=$(echo "$replaced" | sed -E "s/(%)(%n)/\1\n\2/g;:a s/(^|[^\n])%n/\1$JOB_COMPLETION_INDEX/;ta;s/\n//g") +replaced=$(echo "$replaced" | sed -E "s/(%)(%t)/\1\n\2/g;:a s/(^|[^\n])%t/\1$SLURM_ARRAY_TASK_ID/;ta;s/\n//g") +replaced=$(echo "$replaced" | sed -E "s/(%)(%u)/\1\n\2/g;:a s/(^|[^\n])%u/\1$USER_ID/;ta;s/\n//g") +replaced=$(echo "$replaced" | sed -E "s/(%)(%x)/\1\n\2/g;:a s/(^|[^\n])%x/\1$SBATCH_JOB_NAME/;ta;s/\n//g") + +replaced="${replaced//%%/%}" + +echo "$replaced" +} + +input_file=$(unmask_filename "$SBATCH_INPUT") +output_file=$(unmask_filename "$SBATCH_OUTPUT") +error_path=$(unmask_filename "$SBATCH_ERROR") + +/slurm/script + + +script: +---- +#!/bin/bash +sleep 300 + + +BinaryData +==== + +`, + }, + "describe specific task with 'mode slash task' format": { + args: []string{"job/sample-job-8c7zt"}, + argsFormat: modeTaskArgsFormat, + mapperKinds: []schema.GroupVersionKind{ + batchv1.SchemeGroupVersion.WithKind("Job"), + }, + objs: []runtime.Object{ + getSampleJob("sample-job-8c7zt"), + }, + wantOut: `Name: sample-job-8c7zt +Namespace: default +Labels: kjobctl.x-k8s.io/profile=sample-profile +Annotations: kjobctl.x-k8s.io/script=test.sh +Parallelism: 3 +Completions: 2 +Start Time: Mon, 01 Jan 2024 00:00:00 +0000 +Completed At: Mon, 01 Jan 2024 00:00:33 +0000 +Duration: 33s +Pods Statuses: 0 Active / 2 Succeeded / 0 Failed +Pod Template: + Containers: + sample-container: + Port: + Host Port: + Command: + sleep + 15s + Args: + 30s + Requests: + cpu: 1 + memory: 200Mi + Environment: + Mounts: + Volumes: +`, + }, + "describe all jobs": { + args: []string{"job"}, + argsFormat: modeTaskArgsFormat, + mapperKinds: []schema.GroupVersionKind{ + batchv1.SchemeGroupVersion.WithKind("Job"), + }, + objs: []runtime.Object{ + &batchv1.JobList{ + Items: []batchv1.Job{ + *getSampleJob("sample-job-5zd6r"), + *getSampleJob("sample-job-8c7zt"), + }, + }, + }, + wantOut: `Name: sample-job-5zd6r +Namespace: default +Labels: kjobctl.x-k8s.io/profile=sample-profile +Annotations: kjobctl.x-k8s.io/script=test.sh +Parallelism: 3 +Completions: 2 +Start Time: Mon, 01 Jan 2024 00:00:00 +0000 +Completed At: Mon, 01 Jan 2024 00:00:33 +0000 +Duration: 33s +Pods Statuses: 0 Active / 2 Succeeded / 0 Failed +Pod Template: + Containers: + sample-container: + Port: + Host Port: + Command: + sleep + 15s + Args: + 30s + Requests: + cpu: 1 + memory: 200Mi + Environment: + Mounts: + Volumes: + + +Name: sample-job-8c7zt +Namespace: default +Labels: kjobctl.x-k8s.io/profile=sample-profile +Annotations: kjobctl.x-k8s.io/script=test.sh +Parallelism: 3 +Completions: 2 +Start Time: Mon, 01 Jan 2024 00:00:00 +0000 +Completed At: Mon, 01 Jan 2024 00:00:33 +0000 +Duration: 33s +Pods Statuses: 0 Active / 2 Succeeded / 0 Failed +Pod Template: + Containers: + sample-container: + Port: + Host Port: + Command: + sleep + 15s + Args: + 30s + Requests: + cpu: 1 + memory: 200Mi + Environment: + Mounts: + Volumes: +`, + }, + "describe interactive with 'mode task' format": { + args: []string{"interactive", "sample-interactive-fgnh9"}, + argsFormat: modeTaskArgsFormat, + mapperKinds: []schema.GroupVersionKind{ + corev1.SchemeGroupVersion.WithKind("Pod"), + }, + objs: []runtime.Object{ + getSampleInteractive("sample-interactive-fgnh9"), + }, + wantOut: `Name: sample-interactive-fgnh9 +Namespace: default +Start Time: Mon, 01 Jan 2024 00:00:00 +0000 +Labels: kjobctl.x-k8s.io/profile=sample-profile +Status: Running +Containers: + sample-container: + Port: + Host Port: + Command: + /bin/sh + Environment: + TASK_NAME: sample-interactive + Mounts: + /sample from sample-volume (rw) +Volumes: + sample-volume: + Type: EmptyDir (a temporary directory that shares a pod's lifetime) + Medium: + SizeLimit: +`, + }, + "describe all interactive tasks": { + args: []string{"interactive"}, + argsFormat: modeTaskArgsFormat, + mapperKinds: []schema.GroupVersionKind{ + corev1.SchemeGroupVersion.WithKind("Pod"), + }, + objs: []runtime.Object{ + &corev1.PodList{ + Items: []corev1.Pod{ + *getSampleInteractive("sample-interactive-fgnh9"), + *getSampleInteractive("sample-interactive-hs2b2"), + }, + }, + }, + wantOut: `Name: sample-interactive-fgnh9 +Namespace: default +Start Time: Mon, 01 Jan 2024 00:00:00 +0000 +Labels: kjobctl.x-k8s.io/profile=sample-profile +Status: Running +Containers: + sample-container: + Port: + Host Port: + Command: + /bin/sh + Environment: + TASK_NAME: sample-interactive + Mounts: + /sample from sample-volume (rw) +Volumes: + sample-volume: + Type: EmptyDir (a temporary directory that shares a pod's lifetime) + Medium: + SizeLimit: + + +Name: sample-interactive-hs2b2 +Namespace: default +Start Time: Mon, 01 Jan 2024 00:00:00 +0000 +Labels: kjobctl.x-k8s.io/profile=sample-profile +Status: Running +Containers: + sample-container: + Port: + Host Port: + Command: + /bin/sh + Environment: + TASK_NAME: sample-interactive + Mounts: + /sample from sample-volume (rw) +Volumes: + sample-volume: + Type: EmptyDir (a temporary directory that shares a pod's lifetime) + Medium: + SizeLimit: +`, + }, + "describe ray job with 'mode task' format": { + args: []string{"rayjob", "sample-ray-job"}, + argsFormat: modeTaskArgsFormat, + mapperKinds: []schema.GroupVersionKind{ + rayv1.SchemeGroupVersion.WithKind("RayJob"), + }, + objs: []runtime.Object{ + wrappers.MakeRayJob("sample-ray-job", metav1.NamespaceDefault). + Profile("my-profile"). + LocalQueue("lq"). + StartTime(time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC)). + EndTime(time.Date(2024, 1, 1, 1, 0, 0, 0, time.UTC)). + JobDeploymentStatus(rayv1.JobDeploymentStatusRunning). + JobStatus(rayv1.JobStatusFailed). + Reason(rayv1.DeadlineExceeded). + Message("Error message"). + RayClusterName("my-cluster"). + Obj(), + }, + wantOut: `Name: sample-ray-job +Namespace: default +Start Time: Mon, 01 Jan 2024 00:00:00 +0000 +End Time: Mon, 01 Jan 2024 01:00:00 +0000 +Labels: kjobctl.x-k8s.io/profile=my-profile + kueue.x-k8s.io/queue-name=lq +Job Deployment Status: Running +Job Status: FAILED +Reason: DeadlineExceeded +Message: Error message +Ray Cluster Name: my-cluster +Ray Cluster Status: + Desired CPU: 0 + Desired GPU: 0 + Desired Memory: 0 + Desired TPU: 0 +`, + }, + "describe ray cluster with 'mode task' format": { + args: []string{"raycluster", "sample-ray-cluster"}, + argsFormat: modeTaskArgsFormat, + mapperKinds: []schema.GroupVersionKind{ + rayv1.SchemeGroupVersion.WithKind("RayCluster"), + }, + objs: []runtime.Object{ + wrappers.MakeRayCluster("sample-ray-job", metav1.NamespaceDefault). + Profile("my-profile"). + LocalQueue("lq"). + State(rayv1.Failed). + Reason("Reason message"). + DesiredCPU(apiresource.MustParse("1")). + DesiredGPU(apiresource.MustParse("5")). + DesiredMemory(apiresource.MustParse("2Gi")). + DesiredTPU(apiresource.MustParse("10")). + ReadyWorkerReplicas(1). + AvailableWorkerReplicas(1). + DesiredWorkerReplicas(1). + MinWorkerReplicas(1). + MaxWorkerReplicas(5). + Spec( + *wrappers.MakeRayClusterSpec(). + HeadGroupSpec(rayv1.HeadGroupSpec{ + RayStartParams: map[string]string{"p1": "v1", "p2": "v2"}, + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + *wrappers.MakeContainer("ray-head", "rayproject/ray:2.9.0"). + WithEnvVar(corev1.EnvVar{Name: "TASK_NAME", Value: "sample-interactive"}). + WithVolumeMount(corev1.VolumeMount{Name: "sample-volume", MountPath: "/sample"}). + Obj(), + }, + }, + }, + }). + WithWorkerGroupSpec( + *wrappers.MakeWorkerGroupSpec("group1"). + Replicas(1). + MinReplicas(1). + MaxReplicas(5). + RayStartParams(map[string]string{"p1": "v1", "p2": "v2"}). + WithContainer( + *wrappers.MakeContainer("ray-worker", "rayproject/ray:2.9.0"). + WithEnvVar(corev1.EnvVar{Name: "TASK_NAME", Value: "sample-interactive"}). + WithVolumeMount(corev1.VolumeMount{Name: "sample-volume", MountPath: "/sample"}). + Obj(), + ). + Obj(), + ). + Suspend(true). + Obj(), + ). + Obj(), + }, + wantOut: `Name: sample-ray-job +Namespace: default +Labels: kjobctl.x-k8s.io/profile=my-profile + kueue.x-k8s.io/queue-name=lq +Suspend: true +State: failed +Reason: Reason message +Desired CPU: 1 +Desired GPU: 5 +Desired Memory: 2Gi +Desired TPU: 10 +Ready Worker Replicas: 1 +Available Worker Replicas: 1 +Desired Worker Replicas: 1 +Min Worker Replicas: 1 +Max Worker Replicas: 5 +Head Group: + Start Params: p1=v1 + p2=v2 + Pod Template: + Containers: + ray-head: + Port: + Host Port: + Environment: + TASK_NAME: sample-interactive + Mounts: + /sample from sample-volume (rw) + Volumes: +Worker Groups: + group1: + Replicas: 1 + Min Replicas: 1 + Max Replicas: 5 + Start Params: p1=v1 + p2=v2 + Pod Template: + Containers: + ray-worker: + Port: + Host Port: + Environment: + TASK_NAME: sample-interactive + Mounts: + /sample from sample-volume (rw) + Volumes: +`, + }, + } + + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + t.Setenv("TZ", "") + streams, _, out, outErr := genericiooptions.NewTestIOStreams() + + tcg := cmdtesting.NewTestClientGetter() + if tc.withK8sClientSet { + tcg.WithK8sClientset(k8sfake.NewSimpleClientset(tc.objs...)) + } + + if len(tc.mapperKinds) != 0 { + mapper := meta.NewDefaultRESTMapper([]schema.GroupVersion{}) + for _, k := range tc.mapperKinds { + mapper.Add(k, meta.RESTScopeNamespace) + } + tcg.WithRESTMapper(mapper) + } + + if len(tc.objs) != 0 { + scheme := runtime.NewScheme() + + if err := batchv1.AddToScheme(scheme); err != nil { + t.Errorf("Unexpected error\n%s", err) + } + + if err := corev1.AddToScheme(scheme); err != nil { + t.Errorf("Unexpected error\n%s", err) + } + + if err := rayv1.AddToScheme(scheme); err != nil { + t.Errorf("Unexpected error\n%s", err) + } + + codec := serializer.NewCodecFactory(scheme).LegacyCodec(scheme.PrioritizedVersionsAllGroups()...) + tcg.WithRESTClient(&restfake.RESTClient{ + NegotiatedSerializer: resource.UnstructuredPlusDefaultContentConfig().NegotiatedSerializer, + Resp: &http.Response{ + StatusCode: http.StatusOK, + Body: io.NopCloser(strings.NewReader(runtime.EncodeOrDie(codec, tc.objs[0]))), + }, + }) + } + + cmd := NewDescribeCmd(tcg, streams) + cmd.SetArgs(tc.args) + + gotErr := cmd.Execute() + if diff := cmp.Diff(tc.wantErr, gotErr, cmpopts.EquateErrors()); diff != "" { + t.Errorf("Unexpected error (-want/+got)\n%s", diff) + } + + gotOut := out.String() + if diff := cmp.Diff(tc.wantOut, gotOut); diff != "" { + t.Errorf("Unexpected output (-want/+got)\n%s", diff) + } + + gotOutErr := outErr.String() + if diff := cmp.Diff(tc.wantOutErr, gotOutErr); diff != "" { + t.Errorf("Unexpected output (-want/+got)\n%s", diff) + } + }) + } +} + +func getSampleJob(name string) *batchv1.Job { + return &batchv1.Job{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: "default", + Labels: map[string]string{ + "kjobctl.x-k8s.io/profile": "sample-profile", + }, + Annotations: map[string]string{ + constants.ScriptAnnotation: "test.sh", + }, + }, + Spec: batchv1.JobSpec{ + Parallelism: ptr.To[int32](3), + Completions: ptr.To[int32](2), + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "sample-container", + Command: []string{"sleep", "15s"}, + Args: []string{"30s"}, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: apiresource.MustParse("1"), + corev1.ResourceMemory: apiresource.MustParse("200Mi"), + }, + }, + }, + }, + }, + }, + }, + Status: batchv1.JobStatus{ + Active: 0, + Succeeded: 2, + Failed: 0, + StartTime: ptr.To(metav1.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC)), + CompletionTime: ptr.To(metav1.Date(2024, 1, 1, 0, 0, 33, 0, time.UTC)), + }, + } +} + +func getSampleInteractive(name string) *corev1.Pod { + return &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: "default", + Labels: map[string]string{ + "kjobctl.x-k8s.io/profile": "sample-profile", + }, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "sample-container", + Command: []string{"/bin/sh"}, + Env: []corev1.EnvVar{ + { + Name: "TASK_NAME", + Value: "sample-interactive", + }, + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "sample-volume", + MountPath: "/sample", + }, + }, + }, + }, + Volumes: []corev1.Volume{ + { + Name: "sample-volume", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }, + }, + }, + Status: corev1.PodStatus{ + Phase: corev1.PodRunning, + StartTime: ptr.To(metav1.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC)), + }, + } +} diff --git a/pkg/cmd/describe/helpers.go b/pkg/cmd/describe/helpers.go new file mode 100644 index 0000000..0005df8 --- /dev/null +++ b/pkg/cmd/describe/helpers.go @@ -0,0 +1,868 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package describe + +import ( + "bytes" + "errors" + "fmt" + "io" + "sort" + "strconv" + "strings" + "text/tabwriter" + "time" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/duration" + "k8s.io/apimachinery/pkg/util/sets" + describehelper "k8s.io/kubectl/pkg/describe" + resourcehelper "k8s.io/kubectl/pkg/util/resource" + storageutil "k8s.io/kubectl/pkg/util/storage" + + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/apis/v1alpha1" +) + +// Each level has 2 spaces for describehelper.PrefixWriter +const ( + IndentLevelZero = iota + IndentLevelOne + IndentLevelTwo + IndentLevelThree +) + +var maxAnnotationLen = 140 + +func parseAppProfileModeName(s string) ([]string, error) { + seg := strings.Split(s, "/") + if len(seg) != 2 { + if len(seg) > 2 { + return nil, errors.New("arguments in mode/name form may not have more than one slash") + } + return nil, errors.New("argument must be in mode/name form") + } + + return seg, nil +} + +func resourceFor(mode string) string { + if strings.EqualFold(mode, string(v1alpha1.InteractiveMode)) { + return "pod" + } + + if strings.EqualFold(mode, string(v1alpha1.SlurmMode)) { + return "job" + } + + return strings.ToLower(mode) +} + +func gvkFor(mapper meta.RESTMapper, resource string) (schema.GroupVersionKind, error) { + var gvk schema.GroupVersionKind + var err error + + fullySpecifiedGVR, groupResource := schema.ParseResourceArg(strings.ToLower(resource)) + gvr := schema.GroupVersionResource{} + if fullySpecifiedGVR != nil { + gvr, _ = mapper.ResourceFor(*fullySpecifiedGVR) + } + if gvr.Empty() { + gvr, err = mapper.ResourceFor(groupResource.WithVersion("")) + if err != nil { + return gvk, err + } + } + + gvk, err = mapper.KindFor(gvr) + if err != nil { + return gvk, err + } + + return gvk, err +} + +// below functions copied from https://github.com/kubernetes/kubectl/blob/master/pkg/describe/describe.go + +func tabbedString(f func(io.Writer) error) (string, error) { + out := new(tabwriter.Writer) + buf := &bytes.Buffer{} + out.Init(buf, 0, 8, 2, ' ', 0) + + err := f(out) + if err != nil { + return "", err + } + + out.Flush() + return buf.String(), nil +} + +// printLabelsMultiline prints multiple labels with a proper alignment. +func printLabelsMultiline(w describehelper.PrefixWriter, title string, labels map[string]string) { + printLabelsMultilineWithIndent(w, "", title, "\t", labels, sets.New[string]()) +} + +// printLabelsMultiline prints multiple labels with a user-defined alignment. +func printLabelsMultilineWithIndent(w describehelper.PrefixWriter, initialIndent, title, innerIndent string, labels map[string]string, skip sets.Set[string]) { + w.Write(IndentLevelZero, "%s%s:%s", initialIndent, title, innerIndent) + + if len(labels) == 0 { + w.WriteLine("") + return + } + + // to print labels in the sorted order + keys := make([]string, 0, len(labels)) + for key := range labels { + if skip.Has(key) { + continue + } + keys = append(keys, key) + } + if len(keys) == 0 { + w.WriteLine("") + return + } + sort.Strings(keys) + + for i, key := range keys { + if i != 0 { + w.Write(IndentLevelZero, "%s", initialIndent) + w.Write(IndentLevelZero, "%s", innerIndent) + } + w.Write(IndentLevelZero, "%s=%s\n", key, labels[key]) + } +} + +func describePodTemplate(template *corev1.PodTemplateSpec, w describehelper.PrefixWriter) { + w.Write(IndentLevelZero, "Pod Template:\n") + if template == nil { + w.Write(IndentLevelOne, "") + return + } + + if len(template.Spec.InitContainers) > 0 { + describeContainers("Init Containers", template.Spec.InitContainers, w, " ") + } + describeContainers("Containers", template.Spec.Containers, w, " ") + describeVolumes(template.Spec.Volumes, w, " ") +} + +func describeContainers(label string, containers []corev1.Container, w describehelper.PrefixWriter, space string) { + describeContainersLabel(containers, label, space, w) + + for _, container := range containers { + describeContainerBasicInfo(container, space, w) + describeContainerCommand(container, w) + describeContainerResource(container, w) + describeContainerProbe(container, w) + if len(container.EnvFrom) > 0 { + describeContainerEnvFrom(container, w) + } + describeContainerEnvVars(container, w) + describeContainerVolumes(container, w) + } +} + +func describeContainersLabel(containers []corev1.Container, label, space string, w describehelper.PrefixWriter) { + none := "" + if len(containers) == 0 { + none = " " + } + w.Write(IndentLevelZero, "%s%s:%s\n", space, label, none) +} + +func describeContainerBasicInfo(container corev1.Container, space string, w describehelper.PrefixWriter) { + nameIndent := "" + if len(space) > 0 { + nameIndent = " " + } + w.Write(IndentLevelOne, "%s%v:\n", nameIndent, container.Name) + portString := describeContainerPorts(container.Ports) + if strings.Contains(portString, ",") { + w.Write(IndentLevelTwo, "Ports:\t%s\n", portString) + } else { + w.Write(IndentLevelTwo, "Port:\t%s\n", stringOrNone(portString)) + } + hostPortString := describeContainerHostPorts(container.Ports) + if strings.Contains(hostPortString, ",") { + w.Write(IndentLevelTwo, "Host Ports:\t%s\n", hostPortString) + } else { + w.Write(IndentLevelTwo, "Host Port:\t%s\n", stringOrNone(hostPortString)) + } +} + +func stringOrNone(s string) string { + return stringOrDefaultValue(s, "") +} + +func stringOrDefaultValue(s, defaultValue string) string { + if len(s) > 0 { + return s + } + return defaultValue +} + +func describeContainerPorts(cPorts []corev1.ContainerPort) string { + ports := make([]string, 0, len(cPorts)) + for _, cPort := range cPorts { + ports = append(ports, fmt.Sprintf("%d/%s", cPort.ContainerPort, cPort.Protocol)) + } + return strings.Join(ports, ", ") +} + +func describeContainerHostPorts(cPorts []corev1.ContainerPort) string { + ports := make([]string, 0, len(cPorts)) + for _, cPort := range cPorts { + ports = append(ports, fmt.Sprintf("%d/%s", cPort.HostPort, cPort.Protocol)) + } + return strings.Join(ports, ", ") +} + +func describeContainerCommand(container corev1.Container, w describehelper.PrefixWriter) { + if len(container.Command) > 0 { + w.Write(IndentLevelTwo, "Command:\n") + for _, c := range container.Command { + for _, s := range strings.Split(c, "\n") { + w.Write(IndentLevelThree, "%s\n", s) + } + } + } + if len(container.Args) > 0 { + w.Write(IndentLevelTwo, "Args:\n") + for _, arg := range container.Args { + for _, s := range strings.Split(arg, "\n") { + w.Write(IndentLevelThree, "%s\n", s) + } + } + } +} + +func describeContainerResource(container corev1.Container, w describehelper.PrefixWriter) { + resources := container.Resources + if len(resources.Limits) > 0 { + w.Write(IndentLevelTwo, "Limits:\n") + } + for _, name := range describehelper.SortedResourceNames(resources.Limits) { + quantity := resources.Limits[name] + w.Write(IndentLevelThree, "%s:\t%s\n", name, quantity.String()) + } + + if len(resources.Requests) > 0 { + w.Write(IndentLevelTwo, "Requests:\n") + } + for _, name := range describehelper.SortedResourceNames(resources.Requests) { + quantity := resources.Requests[name] + w.Write(IndentLevelThree, "%s:\t%s\n", name, quantity.String()) + } +} + +func describeContainerProbe(container corev1.Container, w describehelper.PrefixWriter) { + if container.LivenessProbe != nil { + probe := describehelper.DescribeProbe(container.LivenessProbe) + w.Write(IndentLevelTwo, "Liveness:\t%s\n", probe) + } + if container.ReadinessProbe != nil { + probe := describehelper.DescribeProbe(container.ReadinessProbe) + w.Write(IndentLevelTwo, "Readiness:\t%s\n", probe) + } + if container.StartupProbe != nil { + probe := describehelper.DescribeProbe(container.StartupProbe) + w.Write(IndentLevelTwo, "Startup:\t%s\n", probe) + } +} + +func describeContainerVolumes(container corev1.Container, w describehelper.PrefixWriter) { + // Show volumeMounts + none := "" + if len(container.VolumeMounts) == 0 { + none = "\t" + } + w.Write(IndentLevelTwo, "Mounts:%s\n", none) + sort.Sort(describehelper.SortableVolumeMounts(container.VolumeMounts)) + for _, mount := range container.VolumeMounts { + flags := []string{} + if mount.ReadOnly { + flags = append(flags, "ro") + } else { + flags = append(flags, "rw") + } + if len(mount.SubPath) > 0 { + flags = append(flags, fmt.Sprintf("path=%q", mount.SubPath)) + } + w.Write(IndentLevelThree, "%s from %s (%s)\n", mount.MountPath, mount.Name, strings.Join(flags, ",")) + } + // Show volumeDevices if exists + if len(container.VolumeDevices) > 0 { + w.Write(IndentLevelTwo, "Devices:%s\n", none) + sort.Sort(describehelper.SortableVolumeDevices(container.VolumeDevices)) + for _, device := range container.VolumeDevices { + w.Write(IndentLevelThree, "%s from %s\n", device.DevicePath, device.Name) + } + } +} + +func describeContainerEnvVars(container corev1.Container, w describehelper.PrefixWriter) { + none := "" + if len(container.Env) == 0 { + none = "\t" + } + w.Write(IndentLevelTwo, "Environment:%s\n", none) + + for _, e := range container.Env { + if e.ValueFrom == nil { + for i, s := range strings.Split(e.Value, "\n") { + if i == 0 { + w.Write(IndentLevelThree, "%s:\t%s\n", e.Name, s) + } else { + w.Write(IndentLevelThree, "\t%s\n", s) + } + } + continue + } + + switch { + case e.ValueFrom.FieldRef != nil: + w.Write(IndentLevelThree, "%s:\t%s (%s:%s)\n", e.Name, "", e.ValueFrom.FieldRef.APIVersion, e.ValueFrom.FieldRef.FieldPath) + case e.ValueFrom.ResourceFieldRef != nil: + valueFrom, err := resourcehelper.ExtractContainerResourceValue(e.ValueFrom.ResourceFieldRef, &container) + if err != nil { + valueFrom = "" + } + resource := e.ValueFrom.ResourceFieldRef.Resource + if valueFrom == "0" && (resource == "limits.cpu" || resource == "limits.memory") { + valueFrom = "node allocatable" + } + w.Write(IndentLevelThree, "%s:\t%s (%s)\n", e.Name, valueFrom, resource) + case e.ValueFrom.SecretKeyRef != nil: + optional := e.ValueFrom.SecretKeyRef.Optional != nil && *e.ValueFrom.SecretKeyRef.Optional + w.Write(IndentLevelThree, "%s:\t\tOptional: %t\n", e.Name, e.ValueFrom.SecretKeyRef.Key, e.ValueFrom.SecretKeyRef.Name, optional) + case e.ValueFrom.ConfigMapKeyRef != nil: + optional := e.ValueFrom.ConfigMapKeyRef.Optional != nil && *e.ValueFrom.ConfigMapKeyRef.Optional + w.Write(IndentLevelThree, "%s:\t\tOptional: %t\n", e.Name, e.ValueFrom.ConfigMapKeyRef.Key, e.ValueFrom.ConfigMapKeyRef.Name, optional) + } + } +} + +func describeContainerEnvFrom(container corev1.Container, w describehelper.PrefixWriter) { + none := "" + if len(container.EnvFrom) == 0 { + none = "\t" + } + w.Write(IndentLevelTwo, "Environment Variables from:%s\n", none) + + for _, e := range container.EnvFrom { + from := "" + name := "" + optional := false + if e.ConfigMapRef != nil { + from = "ConfigMap" + name = e.ConfigMapRef.Name + optional = e.ConfigMapRef.Optional != nil && *e.ConfigMapRef.Optional + } else if e.SecretRef != nil { + from = "Secret" + name = e.SecretRef.Name + optional = e.SecretRef.Optional != nil && *e.SecretRef.Optional + } + if len(e.Prefix) == 0 { + w.Write(IndentLevelThree, "%s\t%s\tOptional: %t\n", name, from, optional) + } else { + w.Write(IndentLevelThree, "%s\t%s with prefix '%s'\tOptional: %t\n", name, from, e.Prefix, optional) + } + } +} + +func describeVolumes(volumes []corev1.Volume, w describehelper.PrefixWriter, space string) { + if len(volumes) == 0 { + w.Write(IndentLevelZero, "%sVolumes:\t\n", space) + return + } + + w.Write(IndentLevelZero, "%sVolumes:\n", space) + for _, volume := range volumes { + nameIndent := "" + if len(space) > 0 { + nameIndent = " " + } + w.Write(IndentLevelOne, "%s%v:\n", nameIndent, volume.Name) + switch { + case volume.VolumeSource.HostPath != nil: + printHostPathVolumeSource(volume.VolumeSource.HostPath, w) + case volume.VolumeSource.EmptyDir != nil: + printEmptyDirVolumeSource(volume.VolumeSource.EmptyDir, w) + case volume.VolumeSource.GCEPersistentDisk != nil: + printGCEPersistentDiskVolumeSource(volume.VolumeSource.GCEPersistentDisk, w) + case volume.VolumeSource.AWSElasticBlockStore != nil: + printAWSElasticBlockStoreVolumeSource(volume.VolumeSource.AWSElasticBlockStore, w) + case volume.VolumeSource.GitRepo != nil: + printGitRepoVolumeSource(volume.VolumeSource.GitRepo, w) + case volume.VolumeSource.Secret != nil: + printSecretVolumeSource(volume.VolumeSource.Secret, w) + case volume.VolumeSource.ConfigMap != nil: + printConfigMapVolumeSource(volume.VolumeSource.ConfigMap, w) + case volume.VolumeSource.NFS != nil: + printNFSVolumeSource(volume.VolumeSource.NFS, w) + case volume.VolumeSource.ISCSI != nil: + printISCSIVolumeSource(volume.VolumeSource.ISCSI, w) + case volume.VolumeSource.Glusterfs != nil: + printGlusterfsVolumeSource(volume.VolumeSource.Glusterfs, w) + case volume.VolumeSource.PersistentVolumeClaim != nil: + printPersistentVolumeClaimVolumeSource(volume.VolumeSource.PersistentVolumeClaim, w) + case volume.VolumeSource.Ephemeral != nil: + printEphemeralVolumeSource(volume.VolumeSource.Ephemeral, w) + case volume.VolumeSource.RBD != nil: + printRBDVolumeSource(volume.VolumeSource.RBD, w) + case volume.VolumeSource.Quobyte != nil: + printQuobyteVolumeSource(volume.VolumeSource.Quobyte, w) + case volume.VolumeSource.DownwardAPI != nil: + printDownwardAPIVolumeSource(volume.VolumeSource.DownwardAPI, w) + case volume.VolumeSource.AzureDisk != nil: + printAzureDiskVolumeSource(volume.VolumeSource.AzureDisk, w) + case volume.VolumeSource.VsphereVolume != nil: + printVsphereVolumeSource(volume.VolumeSource.VsphereVolume, w) + case volume.VolumeSource.Cinder != nil: + printCinderVolumeSource(volume.VolumeSource.Cinder, w) + case volume.VolumeSource.PhotonPersistentDisk != nil: + printPhotonPersistentDiskVolumeSource(volume.VolumeSource.PhotonPersistentDisk, w) + case volume.VolumeSource.PortworxVolume != nil: + printPortworxVolumeSource(volume.VolumeSource.PortworxVolume, w) + case volume.VolumeSource.ScaleIO != nil: + printScaleIOVolumeSource(volume.VolumeSource.ScaleIO, w) + case volume.VolumeSource.CephFS != nil: + printCephFSVolumeSource(volume.VolumeSource.CephFS, w) + case volume.VolumeSource.StorageOS != nil: + printStorageOSVolumeSource(volume.VolumeSource.StorageOS, w) + case volume.VolumeSource.FC != nil: + printFCVolumeSource(volume.VolumeSource.FC, w) + case volume.VolumeSource.AzureFile != nil: + printAzureFileVolumeSource(volume.VolumeSource.AzureFile, w) + case volume.VolumeSource.FlexVolume != nil: + printFlexVolumeSource(volume.VolumeSource.FlexVolume, w) + case volume.VolumeSource.Flocker != nil: + printFlockerVolumeSource(volume.VolumeSource.Flocker, w) + case volume.VolumeSource.Projected != nil: + printProjectedVolumeSource(volume.VolumeSource.Projected, w) + case volume.VolumeSource.CSI != nil: + printCSIVolumeSource(volume.VolumeSource.CSI, w) + default: + w.Write(IndentLevelOne, "\n") + } + } +} + +func printHostPathVolumeSource(hostPath *corev1.HostPathVolumeSource, w describehelper.PrefixWriter) { + hostPathType := "" + if hostPath.Type != nil { + hostPathType = string(*hostPath.Type) + } + w.Write(IndentLevelTwo, "Type:\tHostPath (bare host directory volume)\n"+ + " Path:\t%v\n"+ + " HostPathType:\t%v\n", + hostPath.Path, hostPathType) +} + +func printEmptyDirVolumeSource(emptyDir *corev1.EmptyDirVolumeSource, w describehelper.PrefixWriter) { + var sizeLimit string + if emptyDir.SizeLimit != nil && emptyDir.SizeLimit.Cmp(resource.Quantity{}) > 0 { + sizeLimit = fmt.Sprintf("%v", emptyDir.SizeLimit) + } else { + sizeLimit = "" + } + w.Write(IndentLevelTwo, "Type:\tEmptyDir (a temporary directory that shares a pod's lifetime)\n"+ + " Medium:\t%v\n"+ + " SizeLimit:\t%v\n", + emptyDir.Medium, sizeLimit) +} + +func printGCEPersistentDiskVolumeSource(gce *corev1.GCEPersistentDiskVolumeSource, w describehelper.PrefixWriter) { + w.Write(IndentLevelTwo, "Type:\tGCEPersistentDisk (a Persistent Disk resource in Google Compute Engine)\n"+ + " PDName:\t%v\n"+ + " FSType:\t%v\n"+ + " Partition:\t%v\n"+ + " ReadOnly:\t%v\n", + gce.PDName, gce.FSType, gce.Partition, gce.ReadOnly) +} + +func printAWSElasticBlockStoreVolumeSource(aws *corev1.AWSElasticBlockStoreVolumeSource, w describehelper.PrefixWriter) { + w.Write(IndentLevelTwo, "Type:\tAWSElasticBlockStore (a Persistent Disk resource in AWS)\n"+ + " VolumeID:\t%v\n"+ + " FSType:\t%v\n"+ + " Partition:\t%v\n"+ + " ReadOnly:\t%v\n", + aws.VolumeID, aws.FSType, aws.Partition, aws.ReadOnly) +} + +func printGitRepoVolumeSource(git *corev1.GitRepoVolumeSource, w describehelper.PrefixWriter) { + w.Write(IndentLevelTwo, "Type:\tGitRepo (a volume that is pulled from git when the pod is created)\n"+ + " Repository:\t%v\n"+ + " Revision:\t%v\n", + git.Repository, git.Revision) +} + +func printSecretVolumeSource(secret *corev1.SecretVolumeSource, w describehelper.PrefixWriter) { + optional := secret.Optional != nil && *secret.Optional + w.Write(IndentLevelTwo, "Type:\tSecret (a volume populated by a Secret)\n"+ + " SecretName:\t%v\n"+ + " Optional:\t%v\n", + secret.SecretName, optional) +} + +func printConfigMapVolumeSource(configMap *corev1.ConfigMapVolumeSource, w describehelper.PrefixWriter) { + optional := configMap.Optional != nil && *configMap.Optional + w.Write(IndentLevelTwo, "Type:\tConfigMap (a volume populated by a ConfigMap)\n"+ + " Name:\t%v\n"+ + " Optional:\t%v\n", + configMap.Name, optional) +} + +func printProjectedVolumeSource(projected *corev1.ProjectedVolumeSource, w describehelper.PrefixWriter) { + w.Write(IndentLevelTwo, "Type:\tProjected (a volume that contains injected data from multiple sources)\n") + for _, source := range projected.Sources { + if source.Secret != nil { + w.Write(IndentLevelTwo, "SecretName:\t%v\n"+ + " SecretOptionalName:\t%v\n", + source.Secret.Name, source.Secret.Optional) + continue + } + if source.DownwardAPI != nil { + w.Write(IndentLevelTwo, "DownwardAPI:\ttrue\n") + continue + } + if source.ConfigMap != nil { + w.Write(IndentLevelTwo, "ConfigMapName:\t%v\n"+ + " ConfigMapOptional:\t%v\n", + source.ConfigMap.Name, source.ConfigMap.Optional) + continue + } + if source.ServiceAccountToken != nil { + w.Write(IndentLevelTwo, "TokenExpirationSeconds:\t%d\n", + *source.ServiceAccountToken.ExpirationSeconds) + } + } +} + +func printNFSVolumeSource(nfs *corev1.NFSVolumeSource, w describehelper.PrefixWriter) { + w.Write(IndentLevelTwo, "Type:\tNFS (an NFS mount that lasts the lifetime of a pod)\n"+ + " Server:\t%v\n"+ + " Path:\t%v\n"+ + " ReadOnly:\t%v\n", + nfs.Server, nfs.Path, nfs.ReadOnly) +} + +func printQuobyteVolumeSource(quobyte *corev1.QuobyteVolumeSource, w describehelper.PrefixWriter) { + w.Write(IndentLevelTwo, "Type:\tQuobyte (a Quobyte mount on the host that shares a pod's lifetime)\n"+ + " Registry:\t%v\n"+ + " Volume:\t%v\n"+ + " ReadOnly:\t%v\n", + quobyte.Registry, quobyte.Volume, quobyte.ReadOnly) +} + +func printPortworxVolumeSource(pwxVolume *corev1.PortworxVolumeSource, w describehelper.PrefixWriter) { + w.Write(IndentLevelTwo, "Type:\tPortworxVolume (a Portworx Volume resource)\n"+ + " VolumeID:\t%v\n", + pwxVolume.VolumeID) +} + +func printISCSIVolumeSource(iscsi *corev1.ISCSIVolumeSource, w describehelper.PrefixWriter) { + initiator := "" + if iscsi.InitiatorName != nil { + initiator = *iscsi.InitiatorName + } + w.Write(IndentLevelTwo, "Type:\tISCSI (an ISCSI Disk resource that is attached to a kubelet's host machine and then exposed to the pod)\n"+ + " TargetPortal:\t%v\n"+ + " IQN:\t%v\n"+ + " Lun:\t%v\n"+ + " ISCSIInterface\t%v\n"+ + " FSType:\t%v\n"+ + " ReadOnly:\t%v\n"+ + " Portals:\t%v\n"+ + " DiscoveryCHAPAuth:\t%v\n"+ + " SessionCHAPAuth:\t%v\n"+ + " SecretRef:\t%v\n"+ + " InitiatorName:\t%v\n", + iscsi.TargetPortal, iscsi.IQN, iscsi.Lun, iscsi.ISCSIInterface, iscsi.FSType, iscsi.ReadOnly, iscsi.Portals, iscsi.DiscoveryCHAPAuth, iscsi.SessionCHAPAuth, iscsi.SecretRef, initiator) +} + +func printGlusterfsVolumeSource(glusterfs *corev1.GlusterfsVolumeSource, w describehelper.PrefixWriter) { + w.Write(IndentLevelTwo, "Type:\tGlusterfs (a Glusterfs mount on the host that shares a pod's lifetime)\n"+ + " EndpointsName:\t%v\n"+ + " Path:\t%v\n"+ + " ReadOnly:\t%v\n", + glusterfs.EndpointsName, glusterfs.Path, glusterfs.ReadOnly) +} + +func printPersistentVolumeClaimVolumeSource(claim *corev1.PersistentVolumeClaimVolumeSource, w describehelper.PrefixWriter) { + w.Write(IndentLevelTwo, "Type:\tPersistentVolumeClaim (a reference to a PersistentVolumeClaim in the same namespace)\n"+ + " ClaimName:\t%v\n"+ + " ReadOnly:\t%v\n", + claim.ClaimName, claim.ReadOnly) +} + +func printEphemeralVolumeSource(ephemeral *corev1.EphemeralVolumeSource, w describehelper.PrefixWriter) { + w.Write(IndentLevelTwo, "Type:\tEphemeralVolume (an inline specification for a volume that gets created and deleted with the pod)\n") + if ephemeral.VolumeClaimTemplate != nil { + printPersistentVolumeClaim(describehelper.NewNestedPrefixWriter(w, IndentLevelTwo), + &corev1.PersistentVolumeClaim{ + ObjectMeta: ephemeral.VolumeClaimTemplate.ObjectMeta, + Spec: ephemeral.VolumeClaimTemplate.Spec, + }, false /* not a full PVC */) + } +} + +// printPersistentVolumeClaim is used for both PVCs and PersistentVolumeClaimTemplate. For the latter, +// we need to skip some fields which have no meaning. +func printPersistentVolumeClaim(w describehelper.PrefixWriter, pvc *corev1.PersistentVolumeClaim, isFullPVC bool) { + if isFullPVC { + w.Write(IndentLevelZero, "Name:\t%s\n", pvc.Name) + w.Write(IndentLevelZero, "Namespace:\t%s\n", pvc.Namespace) + } + w.Write(IndentLevelZero, "StorageClass:\t%s\n", storageutil.GetPersistentVolumeClaimClass(pvc)) + if isFullPVC { + if pvc.ObjectMeta.DeletionTimestamp != nil { + w.Write(IndentLevelZero, "Status:\tTerminating (lasts %s)\n", translateTimestampSince(*pvc.ObjectMeta.DeletionTimestamp)) + } else { + w.Write(IndentLevelZero, "Status:\t%v\n", pvc.Status.Phase) + } + } + w.Write(IndentLevelZero, "Volume:\t%s\n", pvc.Spec.VolumeName) + printLabelsMultiline(w, "Labels", pvc.Labels) + if isFullPVC { + w.Write(IndentLevelZero, "Finalizers:\t%v\n", pvc.ObjectMeta.Finalizers) + } + storage := pvc.Spec.Resources.Requests[corev1.ResourceStorage] + capacity := "" + accessModes := "" + if pvc.Spec.VolumeName != "" { + accessModes = storageutil.GetAccessModesAsString(pvc.Status.AccessModes) + storage = pvc.Status.Capacity[corev1.ResourceStorage] + capacity = storage.String() + } + w.Write(IndentLevelZero, "Capacity:\t%s\n", capacity) + w.Write(IndentLevelZero, "Access Modes:\t%s\n", accessModes) + if pvc.Spec.VolumeMode != nil { + w.Write(IndentLevelZero, "VolumeMode:\t%v\n", *pvc.Spec.VolumeMode) + } + if pvc.Spec.DataSource != nil { + w.Write(IndentLevelZero, "DataSource:\n") + if pvc.Spec.DataSource.APIGroup != nil { + w.Write(IndentLevelOne, "APIGroup:\t%v\n", *pvc.Spec.DataSource.APIGroup) + } + w.Write(IndentLevelOne, "Kind:\t%v\n", pvc.Spec.DataSource.Kind) + w.Write(IndentLevelOne, "Name:\t%v\n", pvc.Spec.DataSource.Name) + } +} + +// translateTimestampSince returns the elapsed time since timestamp in +// human-readable approximation. +func translateTimestampSince(timestamp metav1.Time) string { + if timestamp.IsZero() { + return "" + } + + return duration.HumanDuration(time.Since(timestamp.Time)) +} + +func printRBDVolumeSource(rbd *corev1.RBDVolumeSource, w describehelper.PrefixWriter) { + w.Write(IndentLevelTwo, "Type:\tRBD (a Rados Block Device mount on the host that shares a pod's lifetime)\n"+ + " CephMonitors:\t%v\n"+ + " RBDImage:\t%v\n"+ + " FSType:\t%v\n"+ + " RBDPool:\t%v\n"+ + " RadosUser:\t%v\n"+ + " Keyring:\t%v\n"+ + " SecretRef:\t%v\n"+ + " ReadOnly:\t%v\n", + rbd.CephMonitors, rbd.RBDImage, rbd.FSType, rbd.RBDPool, rbd.RadosUser, rbd.Keyring, rbd.SecretRef, rbd.ReadOnly) +} + +func printDownwardAPIVolumeSource(d *corev1.DownwardAPIVolumeSource, w describehelper.PrefixWriter) { + w.Write(IndentLevelTwo, "Type:\tDownwardAPI (a volume populated by information about the pod)\n Items:\n") + for _, mapping := range d.Items { + if mapping.FieldRef != nil { + w.Write(IndentLevelThree, "%v -> %v\n", mapping.FieldRef.FieldPath, mapping.Path) + } + if mapping.ResourceFieldRef != nil { + w.Write(IndentLevelThree, "%v -> %v\n", mapping.ResourceFieldRef.Resource, mapping.Path) + } + } +} + +func printAzureDiskVolumeSource(d *corev1.AzureDiskVolumeSource, w describehelper.PrefixWriter) { + w.Write(IndentLevelTwo, "Type:\tAzureDisk (an Azure Data Disk mount on the host and bind mount to the pod)\n"+ + " DiskName:\t%v\n"+ + " DiskURI:\t%v\n"+ + " Kind: \t%v\n"+ + " FSType:\t%v\n"+ + " CachingMode:\t%v\n"+ + " ReadOnly:\t%v\n", + d.DiskName, d.DataDiskURI, *d.Kind, *d.FSType, *d.CachingMode, *d.ReadOnly) +} + +func printVsphereVolumeSource(vsphere *corev1.VsphereVirtualDiskVolumeSource, w describehelper.PrefixWriter) { + w.Write(IndentLevelTwo, "Type:\tvSphereVolume (a Persistent Disk resource in vSphere)\n"+ + " VolumePath:\t%v\n"+ + " FSType:\t%v\n"+ + " StoragePolicyName:\t%v\n", + vsphere.VolumePath, vsphere.FSType, vsphere.StoragePolicyName) +} + +func printPhotonPersistentDiskVolumeSource(photon *corev1.PhotonPersistentDiskVolumeSource, w describehelper.PrefixWriter) { + w.Write(IndentLevelTwo, "Type:\tPhotonPersistentDisk (a Persistent Disk resource in photon platform)\n"+ + " PdID:\t%v\n"+ + " FSType:\t%v\n", + photon.PdID, photon.FSType) +} + +func printCinderVolumeSource(cinder *corev1.CinderVolumeSource, w describehelper.PrefixWriter) { + w.Write(IndentLevelTwo, "Type:\tCinder (a Persistent Disk resource in OpenStack)\n"+ + " VolumeID:\t%v\n"+ + " FSType:\t%v\n"+ + " ReadOnly:\t%v\n"+ + " SecretRef:\t%v\n", + cinder.VolumeID, cinder.FSType, cinder.ReadOnly, cinder.SecretRef) +} + +func printScaleIOVolumeSource(sio *corev1.ScaleIOVolumeSource, w describehelper.PrefixWriter) { + w.Write(IndentLevelTwo, "Type:\tScaleIO (a persistent volume backed by a block device in ScaleIO)\n"+ + " Gateway:\t%v\n"+ + " System:\t%v\n"+ + " Protection Domain:\t%v\n"+ + " Storage Pool:\t%v\n"+ + " Storage Mode:\t%v\n"+ + " VolumeName:\t%v\n"+ + " FSType:\t%v\n"+ + " ReadOnly:\t%v\n", + sio.Gateway, sio.System, sio.ProtectionDomain, sio.StoragePool, sio.StorageMode, sio.VolumeName, sio.FSType, sio.ReadOnly) +} + +func printCephFSVolumeSource(cephfs *corev1.CephFSVolumeSource, w describehelper.PrefixWriter) { + w.Write(IndentLevelTwo, "Type:\tCephFS (a CephFS mount on the host that shares a pod's lifetime)\n"+ + " Monitors:\t%v\n"+ + " Path:\t%v\n"+ + " User:\t%v\n"+ + " SecretFile:\t%v\n"+ + " SecretRef:\t%v\n"+ + " ReadOnly:\t%v\n", + cephfs.Monitors, cephfs.Path, cephfs.User, cephfs.SecretFile, cephfs.SecretRef, cephfs.ReadOnly) +} + +func printStorageOSVolumeSource(storageos *corev1.StorageOSVolumeSource, w describehelper.PrefixWriter) { + w.Write(IndentLevelTwo, "Type:\tStorageOS (a StorageOS Persistent Disk resource)\n"+ + " VolumeName:\t%v\n"+ + " VolumeNamespace:\t%v\n"+ + " FSType:\t%v\n"+ + " ReadOnly:\t%v\n", + storageos.VolumeName, storageos.VolumeNamespace, storageos.FSType, storageos.ReadOnly) +} + +func printFCVolumeSource(fc *corev1.FCVolumeSource, w describehelper.PrefixWriter) { + lun := "" + if fc.Lun != nil { + lun = strconv.Itoa(int(*fc.Lun)) + } + w.Write(IndentLevelTwo, "Type:\tFC (a Fibre Channel disk)\n"+ + " TargetWWNs:\t%v\n"+ + " LUN:\t%v\n"+ + " FSType:\t%v\n"+ + " ReadOnly:\t%v\n", + strings.Join(fc.TargetWWNs, ", "), lun, fc.FSType, fc.ReadOnly) +} + +func printAzureFileVolumeSource(azureFile *corev1.AzureFileVolumeSource, w describehelper.PrefixWriter) { + w.Write(IndentLevelTwo, "Type:\tAzureFile (an Azure File Service mount on the host and bind mount to the pod)\n"+ + " SecretName:\t%v\n"+ + " ShareName:\t%v\n"+ + " ReadOnly:\t%v\n", + azureFile.SecretName, azureFile.ShareName, azureFile.ReadOnly) +} + +func printFlexVolumeSource(flex *corev1.FlexVolumeSource, w describehelper.PrefixWriter) { + w.Write(IndentLevelTwo, "Type:\tFlexVolume (a generic volume resource that is provisioned/attached using an exec based plugin)\n"+ + " Driver:\t%v\n"+ + " FSType:\t%v\n"+ + " SecretRef:\t%v\n"+ + " ReadOnly:\t%v\n"+ + " Options:\t%v\n", + flex.Driver, flex.FSType, flex.SecretRef, flex.ReadOnly, flex.Options) +} + +func printFlockerVolumeSource(flocker *corev1.FlockerVolumeSource, w describehelper.PrefixWriter) { + w.Write(IndentLevelTwo, "Type:\tFlocker (a Flocker volume mounted by the Flocker agent)\n"+ + " DatasetName:\t%v\n"+ + " DatasetUUID:\t%v\n", + flocker.DatasetName, flocker.DatasetUUID) +} + +func printCSIVolumeSource(csi *corev1.CSIVolumeSource, w describehelper.PrefixWriter) { + var readOnly bool + var fsType string + if csi.ReadOnly != nil && *csi.ReadOnly { + readOnly = true + } + if csi.FSType != nil { + fsType = *csi.FSType + } + w.Write(IndentLevelTwo, "Type:\tCSI (a Container Storage Interface (CSI) volume source)\n"+ + " Driver:\t%v\n"+ + " FSType:\t%v\n"+ + " ReadOnly:\t%v\n", + csi.Driver, fsType, readOnly) + printCSIPersistentVolumeAttributesMultiline(w, "VolumeAttributes", csi.VolumeAttributes) +} + +func printCSIPersistentVolumeAttributesMultiline(w describehelper.PrefixWriter, title string, annotations map[string]string) { + printCSIPersistentVolumeAttributesMultilineIndent(w, "", title, "\t", annotations, sets.New[string]()) +} + +func printCSIPersistentVolumeAttributesMultilineIndent(w describehelper.PrefixWriter, initialIndent, title, innerIndent string, attributes map[string]string, skip sets.Set[string]) { + w.Write(IndentLevelTwo, "%s%s:%s", initialIndent, title, innerIndent) + + if len(attributes) == 0 { + w.WriteLine("") + return + } + + // to print labels in the sorted order + keys := make([]string, 0, len(attributes)) + for key := range attributes { + if skip.Has(key) { + continue + } + keys = append(keys, key) + } + if len(attributes) == 0 { + w.WriteLine("") + return + } + sort.Strings(keys) + + for i, key := range keys { + if i != 0 { + w.Write(IndentLevelTwo, initialIndent) + w.Write(IndentLevelTwo, innerIndent) + } + line := fmt.Sprintf("%s=%s", key, attributes[key]) + if len(line) > maxAnnotationLen { + w.Write(IndentLevelTwo, "%s...\n", line[:maxAnnotationLen]) + } else { + w.Write(IndentLevelTwo, "%s\n", line) + } + } +} diff --git a/pkg/cmd/list/helpers.go b/pkg/cmd/list/helpers.go new file mode 100644 index 0000000..d27f72f --- /dev/null +++ b/pkg/cmd/list/helpers.go @@ -0,0 +1,47 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package list + +import ( + "errors" + "os" + "strconv" +) + +const ( + defaultListRequestLimit = 100 + KjobctlListRequestLimitEnvName = "KJOBCTL_LIST_REQUEST_LIMIT" +) + +var ( + invalidListRequestLimitError = errors.New("invalid list request limit") +) + +func listRequestLimit() (int64, error) { + listRequestLimitEnv := os.Getenv(KjobctlListRequestLimitEnvName) + + if len(listRequestLimitEnv) == 0 { + return defaultListRequestLimit, nil + } + + limit, err := strconv.ParseInt(listRequestLimitEnv, 10, 64) + if err != nil { + return 0, invalidListRequestLimitError + } + + return limit, nil +} diff --git a/pkg/cmd/list/list.go b/pkg/cmd/list/list.go new file mode 100644 index 0000000..79394c9 --- /dev/null +++ b/pkg/cmd/list/list.go @@ -0,0 +1,50 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package list + +import ( + "github.com/spf13/cobra" + "k8s.io/cli-runtime/pkg/genericiooptions" + "k8s.io/kubectl/pkg/util/templates" + "k8s.io/utils/clock" + + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/cmd/util" +) + +var ( + listExample = templates.Examples(` + # List Job + kjobctl list job + `) +) + +func NewListCmd(clientGetter util.ClientGetter, streams genericiooptions.IOStreams, clock clock.Clock) *cobra.Command { + cmd := &cobra.Command{ + Use: "list", + Short: "Display resources", + Example: listExample, + SuggestFor: []string{"ps"}, + } + + cmd.AddCommand(NewJobCmd(clientGetter, streams, clock)) + cmd.AddCommand(NewInteractiveCmd(clientGetter, streams, clock)) + cmd.AddCommand(NewRayJobCmd(clientGetter, streams, clock)) + cmd.AddCommand(NewRayClusterCmd(clientGetter, streams, clock)) + cmd.AddCommand(NewSlurmCmd(clientGetter, streams, clock)) + + return cmd +} diff --git a/pkg/cmd/list/list_interactive.go b/pkg/cmd/list/list_interactive.go new file mode 100644 index 0000000..fade0ac --- /dev/null +++ b/pkg/cmd/list/list_interactive.go @@ -0,0 +1,220 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package list + +import ( + "context" + "fmt" + + "github.com/spf13/cobra" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/cli-runtime/pkg/genericclioptions" + "k8s.io/cli-runtime/pkg/genericiooptions" + "k8s.io/cli-runtime/pkg/printers" + "k8s.io/client-go/kubernetes/scheme" + corev1 "k8s.io/client-go/kubernetes/typed/core/v1" + "k8s.io/kubectl/pkg/util/templates" + "k8s.io/utils/clock" + kueueconstants "sigs.k8s.io/kueue/pkg/controller/constants" + + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/cmd/completion" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/cmd/util" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/constants" +) + +var ( + interactiveExample = templates.Examples(` + # List Interactive + kjobctl list interactive + + # List Interactive with profile filter + kjobctl list interactive --profile my-profile + `) +) + +type InteractiveOptions struct { + Clock clock.Clock + PrintFlags *genericclioptions.PrintFlags + + Limit int64 + AllNamespaces bool + Namespace string + ProfileFilter string + LocalQueueFilter string + FieldSelector string + LabelSelector string + ClusterQueueFilter string + + Client corev1.CoreV1Interface + + genericiooptions.IOStreams +} + +func NewInteractiveOptions(streams genericiooptions.IOStreams, clock clock.Clock) *InteractiveOptions { + return &InteractiveOptions{ + PrintFlags: genericclioptions.NewPrintFlags("").WithTypeSetter(scheme.Scheme), + IOStreams: streams, + Clock: clock, + } +} + +func NewInteractiveCmd(clientGetter util.ClientGetter, streams genericiooptions.IOStreams, clock clock.Clock) *cobra.Command { + o := NewInteractiveOptions(streams, clock) + + cmd := &cobra.Command{ + Use: "interactive" + + " [--profile PROFILE_NAME]" + + " [--localqueue LOCALQUEUE_NAME]" + + " [--selector key1=value1]" + + " [--field-selector key1=value1]" + + " [--all-namespaces]", + DisableFlagsInUseLine: true, + Short: "List Interactive", + Example: interactiveExample, + RunE: func(cmd *cobra.Command, _ []string) error { + cmd.SilenceUsage = true + err := o.Complete(clientGetter) + if err != nil { + return err + } + return o.Run(cmd.Context()) + }, + } + + o.PrintFlags.AddFlags(cmd) + + util.AddAllNamespacesFlagVar(cmd, &o.AllNamespaces) + util.AddFieldSelectorFlagVar(cmd, &o.FieldSelector) + util.AddLabelSelectorFlagVar(cmd, &o.LabelSelector) + util.AddProfileFlagVar(cmd, &o.ProfileFilter) + util.AddLocalQueueFlagVar(cmd, &o.LocalQueueFilter) + + cobra.CheckErr(cmd.RegisterFlagCompletionFunc("profile", completion.ApplicationProfileNameFunc(clientGetter))) + cobra.CheckErr(cmd.RegisterFlagCompletionFunc("localqueue", completion.LocalQueueNameFunc(clientGetter))) + + return cmd +} + +// Complete completes all the required options +func (o *InteractiveOptions) Complete(clientGetter util.ClientGetter) error { + var err error + + o.Limit, err = listRequestLimit() + if err != nil { + return err + } + + o.Namespace, _, err = clientGetter.ToRawKubeConfigLoader().Namespace() + if err != nil { + return err + } + + clientset, err := clientGetter.K8sClientset() + if err != nil { + return err + } + + o.Client = clientset.CoreV1() + + return nil +} + +func (o *InteractiveOptions) ToPrinter(headers bool) (printers.ResourcePrinterFunc, error) { + if !o.PrintFlags.OutputFlagSpecified() { + printer := newInteractiveTablePrinter(). + WithNamespace(o.AllNamespaces). + WithHeaders(headers). + WithClock(o.Clock) + return printer.PrintObj, nil + } + + printer, err := o.PrintFlags.ToPrinter() + if err != nil { + return nil, err + } + + return printer.PrintObj, nil +} + +// Run performs the list operation. +func (o *InteractiveOptions) Run(ctx context.Context) error { + var totalCount int + + namespace := o.Namespace + if o.AllNamespaces { + namespace = "" + } + + opts := metav1.ListOptions{ + FieldSelector: o.FieldSelector, + Limit: o.Limit, + } + + if len(o.ProfileFilter) > 0 { + opts.LabelSelector = fmt.Sprintf("%s=%s", constants.ProfileLabel, o.ProfileFilter) + } else { + opts.LabelSelector = constants.ProfileLabel + } + if len(o.LocalQueueFilter) > 0 { + opts.LabelSelector = fmt.Sprintf("%s,%s=%s", opts.LabelSelector, kueueconstants.QueueLabel, o.LocalQueueFilter) + } + if len(o.LabelSelector) > 0 { + opts.LabelSelector = fmt.Sprintf("%s,%s", opts.LabelSelector, o.LabelSelector) + } + + tabWriter := printers.GetNewTabWriter(o.Out) + + for { + headers := totalCount == 0 + + list, err := o.Client.Pods(namespace).List(ctx, opts) + if err != nil { + return err + } + + totalCount += len(list.Items) + + printer, err := o.ToPrinter(headers) + if err != nil { + return err + } + + if err := printer.PrintObj(list, tabWriter); err != nil { + return err + } + + if list.Continue != "" { + opts.Continue = list.Continue + continue + } + + if totalCount == 0 { + if !o.AllNamespaces { + fmt.Fprintf(o.ErrOut, "No resources found in %s namespace.\n", o.Namespace) + } else { + fmt.Fprintln(o.ErrOut, "No resources found") + } + return nil + } + + if err := tabWriter.Flush(); err != nil { + return err + } + + return nil + } +} diff --git a/pkg/cmd/list/list_interactive_printer.go b/pkg/cmd/list/list_interactive_printer.go new file mode 100644 index 0000000..475d118 --- /dev/null +++ b/pkg/cmd/list/list_interactive_printer.go @@ -0,0 +1,105 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package list + +import ( + "errors" + "io" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/duration" + "k8s.io/cli-runtime/pkg/printers" + "k8s.io/utils/clock" + kueueconstants "sigs.k8s.io/kueue/pkg/controller/constants" + + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/constants" +) + +type listInteractivePrinter struct { + clock clock.Clock + printOptions printers.PrintOptions +} + +var _ printers.ResourcePrinter = (*listInteractivePrinter)(nil) + +func (p *listInteractivePrinter) PrintObj(obj runtime.Object, out io.Writer) error { + printer := printers.NewTablePrinter(p.printOptions) + + list, ok := obj.(*corev1.PodList) + if !ok { + return errors.New("invalid object type") + } + + table := &metav1.Table{ + ColumnDefinitions: []metav1.TableColumnDefinition{ + {Name: "Name", Type: "string", Format: "name"}, + {Name: "Profile", Type: "string"}, + {Name: "Local Queue", Type: "string"}, + {Name: "Status", Type: "string"}, + {Name: "Age", Type: "string"}, + }, + Rows: p.printPodList(list), + } + + return printer.PrintObj(table, out) +} + +func (p *listInteractivePrinter) WithNamespace(f bool) *listInteractivePrinter { + p.printOptions.WithNamespace = f + return p +} + +func (p *listInteractivePrinter) WithHeaders(f bool) *listInteractivePrinter { + p.printOptions.NoHeaders = !f + return p +} + +func (p *listInteractivePrinter) WithClock(c clock.Clock) *listInteractivePrinter { + p.clock = c + return p +} + +func newInteractiveTablePrinter() *listInteractivePrinter { + return &listInteractivePrinter{ + clock: clock.RealClock{}, + } +} + +func (p *listInteractivePrinter) printPodList(list *corev1.PodList) []metav1.TableRow { + rows := make([]metav1.TableRow, len(list.Items)) + for index := range list.Items { + rows[index] = p.printPod(&list.Items[index]) + } + return rows +} + +func (p *listInteractivePrinter) printPod(pod *corev1.Pod) metav1.TableRow { + row := metav1.TableRow{ + Object: runtime.RawExtension{Object: pod}, + } + + row.Cells = []any{ + pod.Name, + pod.ObjectMeta.Labels[constants.ProfileLabel], + pod.ObjectMeta.Labels[kueueconstants.QueueLabel], + pod.Status.Phase, + duration.HumanDuration(p.clock.Since(pod.CreationTimestamp.Time)), + } + return row +} diff --git a/pkg/cmd/list/list_interactive_test.go b/pkg/cmd/list/list_interactive_test.go new file mode 100644 index 0000000..dbce757 --- /dev/null +++ b/pkg/cmd/list/list_interactive_test.go @@ -0,0 +1,306 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package list + +import ( + "fmt" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/cli-runtime/pkg/genericiooptions" + "k8s.io/client-go/kubernetes/fake" + kubetesting "k8s.io/client-go/testing" + testingclock "k8s.io/utils/clock/testing" + + cmdtesting "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/cmd/testing" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/testing/wrappers" +) + +func TestInteractiveCmd(t *testing.T) { + testStartTime := time.Now() + + testCases := map[string]struct { + ns string + objs []runtime.Object + args []string + wantOut string + wantOutErr string + wantErr error + }{ + "should print only kjobctl pods": { + ns: "ns1", + objs: []runtime.Object{ + wrappers.MakePod("i1", "ns1"). + Profile("profile1"). + LocalQueue("lq1"). + CreationTimestamp(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + StartTime(testStartTime.Add(-2 * time.Hour).Truncate(time.Second)). + Phase(corev1.PodSucceeded). + Obj(), + wrappers.MakePod("j2", "ns2"). + LocalQueue("lq2"). + CreationTimestamp(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + StartTime(testStartTime.Add(-2 * time.Hour).Truncate(time.Second)). + Phase(corev1.PodSucceeded). + Obj(), + }, + wantOut: `NAME PROFILE LOCAL QUEUE STATUS AGE +i1 profile1 lq1 Succeeded 60m +`, + }, + "should print interactive list with namespace filter": { + ns: "ns1", + objs: []runtime.Object{ + wrappers.MakePod("i1", "ns1"). + Profile("profile1"). + LocalQueue("lq1"). + CreationTimestamp(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + StartTime(testStartTime.Add(-2 * time.Hour).Truncate(time.Second)). + Phase(corev1.PodSucceeded). + Obj(), + wrappers.MakePod("j2", "ns2"). + Profile("profile2"). + LocalQueue("lq2"). + CreationTimestamp(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + StartTime(testStartTime.Add(-2 * time.Hour).Truncate(time.Second)). + Phase(corev1.PodSucceeded). + Obj(), + }, + wantOut: `NAME PROFILE LOCAL QUEUE STATUS AGE +i1 profile1 lq1 Succeeded 60m +`, + }, + "should print interactive list with profile filter": { + args: []string{"--profile", "profile1"}, + objs: []runtime.Object{ + wrappers.MakePod("i1", metav1.NamespaceDefault). + Profile("profile1"). + LocalQueue("lq1"). + CreationTimestamp(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + StartTime(testStartTime.Add(-2 * time.Hour).Truncate(time.Second)). + Phase(corev1.PodSucceeded). + Obj(), + wrappers.MakePod("i2", metav1.NamespaceDefault). + Profile("profile2"). + LocalQueue("lq2"). + CreationTimestamp(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + StartTime(testStartTime.Add(-2 * time.Hour).Truncate(time.Second)). + Phase(corev1.PodSucceeded). + Obj(), + }, + wantOut: `NAME PROFILE LOCAL QUEUE STATUS AGE +i1 profile1 lq1 Succeeded 60m +`, + }, + "should print interactive list with profile filter (short flag)": { + args: []string{"-p", "profile1"}, + objs: []runtime.Object{ + wrappers.MakePod("i1", metav1.NamespaceDefault). + Profile("profile1"). + LocalQueue("lq1"). + CreationTimestamp(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + StartTime(testStartTime.Add(-2 * time.Hour).Truncate(time.Second)). + Phase(corev1.PodSucceeded). + Obj(), + wrappers.MakePod("i2", metav1.NamespaceDefault). + Profile("profile2"). + LocalQueue("lq2"). + CreationTimestamp(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + StartTime(testStartTime.Add(-2 * time.Hour).Truncate(time.Second)). + Phase(corev1.PodSucceeded). + Obj(), + }, + wantOut: `NAME PROFILE LOCAL QUEUE STATUS AGE +i1 profile1 lq1 Succeeded 60m +`, + }, + "should print interactive list with localqueue filter": { + args: []string{"--localqueue", "lq1"}, + objs: []runtime.Object{ + wrappers.MakePod("i1", metav1.NamespaceDefault). + Profile("profile1"). + LocalQueue("lq1"). + CreationTimestamp(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + StartTime(testStartTime.Add(-2 * time.Hour).Truncate(time.Second)). + Phase(corev1.PodSucceeded). + Obj(), + wrappers.MakePod("i2", metav1.NamespaceDefault). + Profile("profile1"). + LocalQueue("lq2"). + CreationTimestamp(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + StartTime(testStartTime.Add(-2 * time.Hour).Truncate(time.Second)). + Phase(corev1.PodSucceeded). + Obj(), + }, + wantOut: `NAME PROFILE LOCAL QUEUE STATUS AGE +i1 profile1 lq1 Succeeded 60m +`, + }, + "should print interactive list with localqueue filter (short flag)": { + args: []string{"-q", "lq1"}, + objs: []runtime.Object{ + wrappers.MakePod("i1", metav1.NamespaceDefault). + Profile("profile1"). + LocalQueue("lq1"). + CreationTimestamp(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + StartTime(testStartTime.Add(-2 * time.Hour).Truncate(time.Second)). + Phase(corev1.PodSucceeded). + Obj(), + wrappers.MakePod("i2", metav1.NamespaceDefault). + Profile("profile1"). + LocalQueue("lq2"). + CreationTimestamp(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + StartTime(testStartTime.Add(-2 * time.Hour).Truncate(time.Second)). + Phase(corev1.PodSucceeded). + Obj(), + }, + wantOut: `NAME PROFILE LOCAL QUEUE STATUS AGE +i1 profile1 lq1 Succeeded 60m +`, + }, + "should print interactive list with label selector filter": { + args: []string{"--selector", "foo=bar"}, + objs: []runtime.Object{ + wrappers.MakePod("i1", metav1.NamespaceDefault). + Profile("profile1"). + LocalQueue("lq1"). + Label("foo", "bar"). + CreationTimestamp(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + StartTime(testStartTime.Add(-2 * time.Hour).Truncate(time.Second)). + Phase(corev1.PodSucceeded). + Obj(), + wrappers.MakePod("i2", metav1.NamespaceDefault). + Profile("profile2"). + LocalQueue("lq2"). + CreationTimestamp(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + StartTime(testStartTime.Add(-2 * time.Hour).Truncate(time.Second)). + Phase(corev1.PodSucceeded). + Obj(), + }, + wantOut: `NAME PROFILE LOCAL QUEUE STATUS AGE +i1 profile1 lq1 Succeeded 60m +`, + }, + "should print interactive list with label selector filter (short flag)": { + args: []string{"-l", "foo=bar"}, + objs: []runtime.Object{ + wrappers.MakePod("i1", metav1.NamespaceDefault). + Profile("profile1"). + LocalQueue("lq1"). + Label("foo", "bar"). + CreationTimestamp(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + StartTime(testStartTime.Add(-2 * time.Hour).Truncate(time.Second)). + Phase(corev1.PodSucceeded). + Obj(), + wrappers.MakePod("i2", metav1.NamespaceDefault). + Profile("profile2"). + LocalQueue("lq2"). + CreationTimestamp(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + StartTime(testStartTime.Add(-2 * time.Hour).Truncate(time.Second)). + Phase(corev1.PodSucceeded). + Obj(), + }, + wantOut: `NAME PROFILE LOCAL QUEUE STATUS AGE +i1 profile1 lq1 Succeeded 60m +`, + }, + "should print interactive list with field selector filter": { + args: []string{"--field-selector", "metadata.name=i1"}, + objs: []runtime.Object{ + wrappers.MakePod("i1", metav1.NamespaceDefault). + Profile("profile1"). + LocalQueue("lq1"). + CreationTimestamp(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + StartTime(testStartTime.Add(-2 * time.Hour).Truncate(time.Second)). + Phase(corev1.PodSucceeded). + Obj(), + wrappers.MakePod("i2", metav1.NamespaceDefault). + Profile("profile2"). + LocalQueue("lq2"). + CreationTimestamp(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + StartTime(testStartTime.Add(-2 * time.Hour).Truncate(time.Second)). + Phase(corev1.PodSucceeded). + Obj(), + }, + wantOut: `NAME PROFILE LOCAL QUEUE STATUS AGE +i1 profile1 lq1 Succeeded 60m +`, + }, + "should print not found error": { + wantOutErr: fmt.Sprintf("No resources found in %s namespace.\n", metav1.NamespaceDefault), + }, + "should print not found error with all-namespaces filter": { + args: []string{"-A"}, + wantOutErr: "No resources found\n", + }, + } + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + streams, _, out, outErr := genericiooptions.NewTestIOStreams() + + clientset := fake.NewSimpleClientset(tc.objs...) + clientset.PrependReactor("list", "pods", func(action kubetesting.Action) (handled bool, ret runtime.Object, err error) { + listAction := action.(kubetesting.ListActionImpl) + fieldsSelector := listAction.GetListRestrictions().Fields + + obj, err := clientset.Tracker().List(listAction.GetResource(), listAction.GetKind(), listAction.Namespace) + podList := obj.(*corev1.PodList) + + filtered := make([]corev1.Pod, 0, len(podList.Items)) + for _, item := range podList.Items { + fieldsSet := fields.Set{ + "metadata.name": item.Name, + } + if fieldsSelector.Matches(fieldsSet) { + filtered = append(filtered, item) + } + } + podList.Items = filtered + return true, podList, err + }) + + tcg := cmdtesting.NewTestClientGetter().WithK8sClientset(clientset) + if len(tc.ns) > 0 { + tcg.WithNamespace(tc.ns) + } + + cmd := NewInteractiveCmd(tcg, streams, testingclock.NewFakeClock(testStartTime)) + cmd.SetArgs(tc.args) + + gotErr := cmd.Execute() + if diff := cmp.Diff(tc.wantErr, gotErr, cmpopts.EquateErrors()); diff != "" { + t.Errorf("Unexpected error (-want/+got)\n%s", diff) + } + + gotOut := out.String() + if diff := cmp.Diff(tc.wantOut, gotOut); diff != "" { + t.Errorf("Unexpected output (-want/+got)\n%s", diff) + } + + gotOutErr := outErr.String() + if diff := cmp.Diff(tc.wantOutErr, gotOutErr); diff != "" { + t.Errorf("Unexpected output (-want/+got)\n%s", diff) + } + }) + } +} diff --git a/pkg/cmd/list/list_job.go b/pkg/cmd/list/list_job.go new file mode 100644 index 0000000..7d63438 --- /dev/null +++ b/pkg/cmd/list/list_job.go @@ -0,0 +1,223 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package list + +import ( + "context" + "fmt" + + "github.com/spf13/cobra" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/cli-runtime/pkg/genericclioptions" + "k8s.io/cli-runtime/pkg/genericiooptions" + "k8s.io/cli-runtime/pkg/printers" + "k8s.io/client-go/kubernetes/scheme" + batchv1 "k8s.io/client-go/kubernetes/typed/batch/v1" + "k8s.io/kubectl/pkg/util/templates" + "k8s.io/utils/clock" + kueueconstants "sigs.k8s.io/kueue/pkg/controller/constants" + + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/apis/v1alpha1" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/cmd/completion" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/cmd/util" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/constants" +) + +var ( + jobExample = templates.Examples(` + # List Job + kjobctl list job + + # List Job with profile filter + kjobctl list job --profile my-profile + `) +) + +type JobOptions struct { + Clock clock.Clock + PrintFlags *genericclioptions.PrintFlags + + Limit int64 + AllNamespaces bool + Namespace string + ProfileFilter string + LocalQueueFilter string + FieldSelector string + LabelSelector string + + Client batchv1.BatchV1Interface + + genericiooptions.IOStreams +} + +func NewJobOptions(streams genericiooptions.IOStreams, clock clock.Clock) *JobOptions { + return &JobOptions{ + PrintFlags: genericclioptions.NewPrintFlags("").WithTypeSetter(scheme.Scheme), + IOStreams: streams, + Clock: clock, + } +} + +func NewJobCmd(clientGetter util.ClientGetter, streams genericiooptions.IOStreams, clock clock.Clock) *cobra.Command { + o := NewJobOptions(streams, clock) + + cmd := &cobra.Command{ + Use: "job" + + " [--profile PROFILE_NAME]" + + " [--localqueue LOCALQUEUE_NAME]" + + " [--selector key1=value1]" + + " [--field-selector key1=value1]" + + " [--all-namespaces]", + DisableFlagsInUseLine: true, + Short: "List Job", + Example: jobExample, + RunE: func(cmd *cobra.Command, _ []string) error { + cmd.SilenceUsage = true + err := o.Complete(clientGetter) + if err != nil { + return err + } + return o.Run(cmd.Context()) + }, + } + + o.PrintFlags.AddFlags(cmd) + + util.AddAllNamespacesFlagVar(cmd, &o.AllNamespaces) + util.AddFieldSelectorFlagVar(cmd, &o.FieldSelector) + util.AddLabelSelectorFlagVar(cmd, &o.LabelSelector) + util.AddProfileFlagVar(cmd, &o.ProfileFilter) + util.AddLocalQueueFlagVar(cmd, &o.LocalQueueFilter) + + cobra.CheckErr(cmd.RegisterFlagCompletionFunc("profile", completion.ApplicationProfileNameFunc(clientGetter))) + cobra.CheckErr(cmd.RegisterFlagCompletionFunc("localqueue", completion.LocalQueueNameFunc(clientGetter))) + + return cmd +} + +// Complete completes all the required options +func (o *JobOptions) Complete(clientGetter util.ClientGetter) error { + var err error + + o.Limit, err = listRequestLimit() + if err != nil { + return err + } + + o.Namespace, _, err = clientGetter.ToRawKubeConfigLoader().Namespace() + if err != nil { + return err + } + + clientset, err := clientGetter.K8sClientset() + if err != nil { + return err + } + + o.Client = clientset.BatchV1() + + return nil +} + +func (o *JobOptions) ToPrinter(headers bool) (printers.ResourcePrinterFunc, error) { + if !o.PrintFlags.OutputFlagSpecified() { + printer := newJobTablePrinter(). + WithNamespace(o.AllNamespaces). + WithHeaders(headers). + WithClock(o.Clock) + return printer.PrintObj, nil + } + + printer, err := o.PrintFlags.ToPrinter() + if err != nil { + return nil, err + } + + return printer.PrintObj, nil +} + +// Run performs the list operation. +func (o *JobOptions) Run(ctx context.Context) error { + var totalCount int + + namespace := o.Namespace + if o.AllNamespaces { + namespace = "" + } + + opts := metav1.ListOptions{ + FieldSelector: o.FieldSelector, + Limit: o.Limit, + } + + if len(o.ProfileFilter) > 0 { + opts.LabelSelector = fmt.Sprintf("%s=%s", constants.ProfileLabel, o.ProfileFilter) + } else { + opts.LabelSelector = constants.ProfileLabel + } + + opts.LabelSelector = fmt.Sprintf("%s,%s=%s", opts.LabelSelector, constants.ModeLabel, v1alpha1.JobMode) + + if len(o.LocalQueueFilter) > 0 { + opts.LabelSelector = fmt.Sprintf("%s,%s=%s", opts.LabelSelector, kueueconstants.QueueLabel, o.LocalQueueFilter) + } + if len(o.LabelSelector) > 0 { + opts.LabelSelector = fmt.Sprintf("%s,%s", opts.LabelSelector, o.LabelSelector) + } + + tabWriter := printers.GetNewTabWriter(o.Out) + + for { + headers := totalCount == 0 + + list, err := o.Client.Jobs(namespace).List(ctx, opts) + if err != nil { + return err + } + + totalCount += len(list.Items) + + printer, err := o.ToPrinter(headers) + if err != nil { + return err + } + + if err := printer.PrintObj(list, tabWriter); err != nil { + return err + } + + if list.Continue != "" { + opts.Continue = list.Continue + continue + } + + if totalCount == 0 { + if !o.AllNamespaces { + fmt.Fprintf(o.ErrOut, "No resources found in %s namespace.\n", o.Namespace) + } else { + fmt.Fprintln(o.ErrOut, "No resources found") + } + return nil + } + + if err := tabWriter.Flush(); err != nil { + return err + } + + return nil + } +} diff --git a/pkg/cmd/list/list_job_printer.go b/pkg/cmd/list/list_job_printer.go new file mode 100644 index 0000000..41ae59a --- /dev/null +++ b/pkg/cmd/list/list_job_printer.go @@ -0,0 +1,117 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package list + +import ( + "errors" + "fmt" + "io" + "time" + + batchv1 "k8s.io/api/batch/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/duration" + "k8s.io/cli-runtime/pkg/printers" + "k8s.io/utils/clock" + "k8s.io/utils/ptr" + kueueconstants "sigs.k8s.io/kueue/pkg/controller/constants" + + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/constants" +) + +type listJobPrinter struct { + clock clock.Clock + printOptions printers.PrintOptions +} + +var _ printers.ResourcePrinter = (*listJobPrinter)(nil) + +func (p *listJobPrinter) PrintObj(obj runtime.Object, out io.Writer) error { + printer := printers.NewTablePrinter(p.printOptions) + + list, ok := obj.(*batchv1.JobList) + if !ok { + return errors.New("invalid object type") + } + + table := &metav1.Table{ + ColumnDefinitions: []metav1.TableColumnDefinition{ + {Name: "Name", Type: "string", Format: "name"}, + {Name: "Profile", Type: "string"}, + {Name: "Local Queue", Type: "string"}, + {Name: "Completions", Type: "string"}, + {Name: "Duration", Type: "string"}, + {Name: "Age", Type: "string"}, + }, + Rows: p.printJobList(list), + } + + return printer.PrintObj(table, out) +} + +func (p *listJobPrinter) WithNamespace(f bool) *listJobPrinter { + p.printOptions.WithNamespace = f + return p +} + +func (p *listJobPrinter) WithHeaders(f bool) *listJobPrinter { + p.printOptions.NoHeaders = !f + return p +} + +func (p *listJobPrinter) WithClock(c clock.Clock) *listJobPrinter { + p.clock = c + return p +} + +func newJobTablePrinter() *listJobPrinter { + return &listJobPrinter{ + clock: clock.RealClock{}, + } +} + +func (p *listJobPrinter) printJobList(list *batchv1.JobList) []metav1.TableRow { + rows := make([]metav1.TableRow, len(list.Items)) + for index := range list.Items { + rows[index] = p.printJob(&list.Items[index]) + } + return rows +} + +func (p *listJobPrinter) printJob(job *batchv1.Job) metav1.TableRow { + row := metav1.TableRow{ + Object: runtime.RawExtension{Object: job}, + } + var durationStr string + if job.Status.StartTime != nil { + completionTime := time.Now() + if job.Status.CompletionTime != nil { + completionTime = job.Status.CompletionTime.Time + } + durationStr = duration.HumanDuration(completionTime.Sub(job.Status.StartTime.Time)) + } + row.Cells = []any{ + job.Name, + job.ObjectMeta.Labels[constants.ProfileLabel], + job.ObjectMeta.Labels[kueueconstants.QueueLabel], + fmt.Sprintf("%d/%d", job.Status.Succeeded, ptr.Deref(job.Spec.Completions, 1)), + durationStr, + duration.HumanDuration(p.clock.Since(job.CreationTimestamp.Time)), + } + return row +} diff --git a/pkg/cmd/list/list_job_test.go b/pkg/cmd/list/list_job_test.go new file mode 100644 index 0000000..e4bc324 --- /dev/null +++ b/pkg/cmd/list/list_job_test.go @@ -0,0 +1,390 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package list + +import ( + "fmt" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + batchv1 "k8s.io/api/batch/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/cli-runtime/pkg/genericiooptions" + "k8s.io/client-go/kubernetes/fake" + kubetesting "k8s.io/client-go/testing" + testingclock "k8s.io/utils/clock/testing" + + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/apis/v1alpha1" + cmdtesting "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/cmd/testing" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/testing/wrappers" +) + +func TestJobCmd(t *testing.T) { + testStartTime := time.Now() + + testCases := map[string]struct { + ns string + objs []runtime.Object + args []string + wantOut string + wantOutErr string + wantErr error + }{ + "should print only kjobctl jobs": { + ns: "ns1", + objs: []runtime.Object{ + wrappers.MakeJob("j1", "ns1"). + Profile("profile1"). + Mode(v1alpha1.JobMode). + LocalQueue("lq1"). + Completions(3). + CreationTimestamp(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + StartTime(testStartTime.Add(-2 * time.Hour).Truncate(time.Second)). + CompletionTime(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + Succeeded(3). + Obj(), + wrappers.MakeJob("j2", "ns2"). + LocalQueue("lq2"). + Completions(3). + CreationTimestamp(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + StartTime(testStartTime.Add(-2 * time.Hour).Truncate(time.Second)). + CompletionTime(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + Succeeded(3). + Obj(), + }, + wantOut: `NAME PROFILE LOCAL QUEUE COMPLETIONS DURATION AGE +j1 profile1 lq1 3/3 60m 60m +`, + }, + "should print only Job mode jobs": { + ns: "ns1", + objs: []runtime.Object{ + wrappers.MakeJob("j1", "ns1"). + Profile("profile1"). + Mode(v1alpha1.JobMode). + LocalQueue("lq1"). + Completions(3). + CreationTimestamp(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + StartTime(testStartTime.Add(-2 * time.Hour).Truncate(time.Second)). + CompletionTime(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + Succeeded(3). + Obj(), + wrappers.MakeJob("j2", "ns2"). + LocalQueue("lq2"). + Mode(v1alpha1.SlurmMode). + Completions(3). + CreationTimestamp(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + StartTime(testStartTime.Add(-2 * time.Hour).Truncate(time.Second)). + CompletionTime(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + Succeeded(3). + Obj(), + }, + wantOut: `NAME PROFILE LOCAL QUEUE COMPLETIONS DURATION AGE +j1 profile1 lq1 3/3 60m 60m +`, + }, + "should print job list with namespace filter": { + ns: "ns1", + objs: []runtime.Object{ + wrappers.MakeJob("j1", "ns1"). + Profile("profile1"). + Mode(v1alpha1.JobMode). + LocalQueue("lq1"). + Completions(3). + CreationTimestamp(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + StartTime(testStartTime.Add(-2 * time.Hour).Truncate(time.Second)). + CompletionTime(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + Succeeded(3). + Obj(), + wrappers.MakeJob("j2", "ns2"). + Profile("profile2"). + Mode(v1alpha1.JobMode). + LocalQueue("lq2"). + Completions(3). + CreationTimestamp(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + StartTime(testStartTime.Add(-2 * time.Hour).Truncate(time.Second)). + CompletionTime(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + Succeeded(3). + Obj(), + }, + wantOut: `NAME PROFILE LOCAL QUEUE COMPLETIONS DURATION AGE +j1 profile1 lq1 3/3 60m 60m +`, + }, + "should print job list with profile filter": { + args: []string{"--profile", "profile1"}, + objs: []runtime.Object{ + wrappers.MakeJob("j1", metav1.NamespaceDefault). + Profile("profile1"). + Mode(v1alpha1.JobMode). + LocalQueue("lq1"). + Completions(3). + CreationTimestamp(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + StartTime(testStartTime.Add(-2 * time.Hour).Truncate(time.Second)). + CompletionTime(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + Succeeded(3). + Obj(), + wrappers.MakeJob("j2", metav1.NamespaceDefault). + Profile("profile2"). + Mode(v1alpha1.JobMode). + LocalQueue("lq2"). + Completions(3). + CreationTimestamp(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + StartTime(testStartTime.Add(-2 * time.Hour).Truncate(time.Second)). + CompletionTime(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + Succeeded(3). + Obj(), + }, + wantOut: `NAME PROFILE LOCAL QUEUE COMPLETIONS DURATION AGE +j1 profile1 lq1 3/3 60m 60m +`, + }, + "should print job list with profile filter (short flag)": { + args: []string{"-p", "profile1"}, + objs: []runtime.Object{ + wrappers.MakeJob("j1", metav1.NamespaceDefault). + Profile("profile1"). + Mode(v1alpha1.JobMode). + LocalQueue("lq1"). + Completions(3). + CreationTimestamp(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + StartTime(testStartTime.Add(-2 * time.Hour).Truncate(time.Second)). + CompletionTime(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + Succeeded(3). + Obj(), + wrappers.MakeJob("j2", metav1.NamespaceDefault). + Profile("profile2"). + Mode(v1alpha1.JobMode). + LocalQueue("lq2"). + Completions(3). + CreationTimestamp(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + StartTime(testStartTime.Add(-2 * time.Hour).Truncate(time.Second)). + CompletionTime(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + Succeeded(3). + Obj(), + }, + wantOut: `NAME PROFILE LOCAL QUEUE COMPLETIONS DURATION AGE +j1 profile1 lq1 3/3 60m 60m +`, + }, + "should print job list with localqueue filter": { + args: []string{"--localqueue", "lq1"}, + objs: []runtime.Object{ + wrappers.MakeJob("j1", metav1.NamespaceDefault). + Profile("profile1"). + Mode(v1alpha1.JobMode). + LocalQueue("lq1"). + LocalQueue("lq1"). + Completions(3). + CreationTimestamp(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + StartTime(testStartTime.Add(-2 * time.Hour).Truncate(time.Second)). + CompletionTime(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + Succeeded(3). + Obj(), + wrappers.MakeJob("j2", metav1.NamespaceDefault). + Profile("profile2"). + Mode(v1alpha1.JobMode). + LocalQueue("lq2"). + LocalQueue("lq2"). + Completions(3). + CreationTimestamp(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + StartTime(testStartTime.Add(-2 * time.Hour).Truncate(time.Second)). + CompletionTime(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + Succeeded(3). + Obj(), + }, + wantOut: `NAME PROFILE LOCAL QUEUE COMPLETIONS DURATION AGE +j1 profile1 lq1 3/3 60m 60m +`, + }, + "should print job list with localqueue filter (short flag)": { + args: []string{"-q", "lq1"}, + objs: []runtime.Object{ + wrappers.MakeJob("j1", metav1.NamespaceDefault). + Profile("profile1"). + Mode(v1alpha1.JobMode). + LocalQueue("lq1"). + Completions(3). + CreationTimestamp(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + StartTime(testStartTime.Add(-2 * time.Hour).Truncate(time.Second)). + CompletionTime(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + Succeeded(3). + Obj(), + wrappers.MakeJob("j2", metav1.NamespaceDefault). + Profile("profile2"). + Mode(v1alpha1.JobMode). + LocalQueue("lq2"). + LocalQueue("lq2"). + Completions(3). + CreationTimestamp(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + StartTime(testStartTime.Add(-2 * time.Hour).Truncate(time.Second)). + CompletionTime(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + Succeeded(3). + Obj(), + }, + wantOut: `NAME PROFILE LOCAL QUEUE COMPLETIONS DURATION AGE +j1 profile1 lq1 3/3 60m 60m +`, + }, + "should print job list with label selector filter": { + args: []string{"--selector", "foo=bar"}, + objs: []runtime.Object{ + wrappers.MakeJob("j1", metav1.NamespaceDefault). + Profile("profile1"). + Mode(v1alpha1.JobMode). + LocalQueue("lq1"). + Label("foo", "bar"). + Completions(3). + CreationTimestamp(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + StartTime(testStartTime.Add(-2 * time.Hour).Truncate(time.Second)). + CompletionTime(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + Succeeded(3). + Obj(), + wrappers.MakeJob("j2", metav1.NamespaceDefault). + Profile("profile2"). + Mode(v1alpha1.JobMode). + LocalQueue("lq2"). + Completions(3). + CreationTimestamp(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + StartTime(testStartTime.Add(-2 * time.Hour).Truncate(time.Second)). + CompletionTime(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + Succeeded(3). + Obj(), + }, + wantOut: `NAME PROFILE LOCAL QUEUE COMPLETIONS DURATION AGE +j1 profile1 lq1 3/3 60m 60m +`, + }, + "should print job list with label selector filter (short flag)": { + args: []string{"-l", "foo=bar"}, + objs: []runtime.Object{ + wrappers.MakeJob("j1", metav1.NamespaceDefault). + Profile("profile1"). + Mode(v1alpha1.JobMode). + LocalQueue("lq1"). + Label("foo", "bar"). + Completions(3). + CreationTimestamp(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + StartTime(testStartTime.Add(-2 * time.Hour).Truncate(time.Second)). + CompletionTime(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + Succeeded(3). + Obj(), + wrappers.MakeJob("j2", metav1.NamespaceDefault). + Profile("profile2"). + Mode(v1alpha1.JobMode). + LocalQueue("lq2"). + Completions(3). + CreationTimestamp(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + StartTime(testStartTime.Add(-2 * time.Hour).Truncate(time.Second)). + CompletionTime(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + Succeeded(3). + Obj(), + }, + wantOut: `NAME PROFILE LOCAL QUEUE COMPLETIONS DURATION AGE +j1 profile1 lq1 3/3 60m 60m +`, + }, + "should print job list with field selector filter": { + args: []string{"--field-selector", "metadata.name=j1"}, + objs: []runtime.Object{ + wrappers.MakeJob("j1", metav1.NamespaceDefault). + Profile("profile1"). + Mode(v1alpha1.JobMode). + LocalQueue("lq1"). + Completions(3). + CreationTimestamp(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + StartTime(testStartTime.Add(-2 * time.Hour).Truncate(time.Second)). + CompletionTime(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + Succeeded(3). + Obj(), + wrappers.MakeJob("j2", metav1.NamespaceDefault). + Profile("profile2"). + Mode(v1alpha1.JobMode). + LocalQueue("lq2"). + Completions(3). + CreationTimestamp(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + StartTime(testStartTime.Add(-2 * time.Hour).Truncate(time.Second)). + CompletionTime(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + Succeeded(3). + Obj(), + }, + wantOut: `NAME PROFILE LOCAL QUEUE COMPLETIONS DURATION AGE +j1 profile1 lq1 3/3 60m 60m +`, + }, + "should print not found error": { + wantOutErr: fmt.Sprintf("No resources found in %s namespace.\n", metav1.NamespaceDefault), + }, + "should print not found error with all-namespaces filter": { + args: []string{"-A"}, + wantOutErr: "No resources found\n", + }, + } + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + streams, _, out, outErr := genericiooptions.NewTestIOStreams() + + clientset := fake.NewSimpleClientset(tc.objs...) + clientset.PrependReactor("list", "jobs", func(action kubetesting.Action) (handled bool, ret runtime.Object, err error) { + listAction := action.(kubetesting.ListActionImpl) + fieldsSelector := listAction.GetListRestrictions().Fields + + obj, err := clientset.Tracker().List(listAction.GetResource(), listAction.GetKind(), listAction.Namespace) + jobList := obj.(*batchv1.JobList) + + filtered := make([]batchv1.Job, 0, len(jobList.Items)) + for _, item := range jobList.Items { + fieldsSet := fields.Set{ + "metadata.name": item.Name, + } + if fieldsSelector.Matches(fieldsSet) { + filtered = append(filtered, item) + } + } + jobList.Items = filtered + return true, jobList, err + }) + + tcg := cmdtesting.NewTestClientGetter().WithK8sClientset(clientset) + if len(tc.ns) > 0 { + tcg.WithNamespace(tc.ns) + } + + cmd := NewJobCmd(tcg, streams, testingclock.NewFakeClock(testStartTime)) + cmd.SetArgs(tc.args) + + gotErr := cmd.Execute() + if diff := cmp.Diff(tc.wantErr, gotErr, cmpopts.EquateErrors()); diff != "" { + t.Errorf("Unexpected error (-want/+got)\n%s", diff) + } + + gotOut := out.String() + if diff := cmp.Diff(tc.wantOut, gotOut); diff != "" { + t.Errorf("Unexpected output (-want/+got)\n%s", diff) + } + + gotOutErr := outErr.String() + if diff := cmp.Diff(tc.wantOutErr, gotOutErr); diff != "" { + t.Errorf("Unexpected output (-want/+got)\n%s", diff) + } + }) + } +} diff --git a/pkg/cmd/list/list_ray_cluster.go b/pkg/cmd/list/list_ray_cluster.go new file mode 100644 index 0000000..9bf7b2e --- /dev/null +++ b/pkg/cmd/list/list_ray_cluster.go @@ -0,0 +1,220 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package list + +import ( + "context" + "fmt" + + "github.com/ray-project/kuberay/ray-operator/pkg/client/clientset/versioned/scheme" + rayv1 "github.com/ray-project/kuberay/ray-operator/pkg/client/clientset/versioned/typed/ray/v1" + "github.com/spf13/cobra" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/cli-runtime/pkg/genericclioptions" + "k8s.io/cli-runtime/pkg/genericiooptions" + "k8s.io/cli-runtime/pkg/printers" + "k8s.io/kubectl/pkg/util/templates" + "k8s.io/utils/clock" + kueueconstants "sigs.k8s.io/kueue/pkg/controller/constants" + + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/cmd/completion" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/cmd/util" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/constants" +) + +var ( + rayClusterExample = templates.Examples(` + # List RayCluster + kjobctl list raycluster + + # List RayCluster with profile filter + kjobctl list raycluster --profile my-profile + `) +) + +type RayClusterOptions struct { + Clock clock.Clock + PrintFlags *genericclioptions.PrintFlags + + Limit int64 + AllNamespaces bool + Namespace string + ProfileFilter string + LocalQueueFilter string + FieldSelector string + LabelSelector string + + Client rayv1.RayV1Interface + + genericiooptions.IOStreams +} + +func NewRayClusterOptions(streams genericiooptions.IOStreams, clock clock.Clock) *RayClusterOptions { + return &RayClusterOptions{ + PrintFlags: genericclioptions.NewPrintFlags("").WithTypeSetter(scheme.Scheme), + IOStreams: streams, + Clock: clock, + } +} + +func NewRayClusterCmd(clientGetter util.ClientGetter, streams genericiooptions.IOStreams, clock clock.Clock) *cobra.Command { + o := NewRayClusterOptions(streams, clock) + + cmd := &cobra.Command{ + Use: "raycluster" + + " [--profile PROFILE_NAME]" + + " [--localqueue LOCALQUEUE_NAME]" + + " [--selector key1=value1]" + + " [--field-selector key1=value1]" + + " [--all-namespaces]", + DisableFlagsInUseLine: true, + Short: "List RayCluster", + Example: rayClusterExample, + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, _ []string) error { + cmd.SilenceUsage = true + err := o.Complete(clientGetter) + if err != nil { + return err + } + return o.Run(cmd.Context()) + }, + } + + o.PrintFlags.AddFlags(cmd) + + util.AddAllNamespacesFlagVar(cmd, &o.AllNamespaces) + util.AddFieldSelectorFlagVar(cmd, &o.FieldSelector) + util.AddLabelSelectorFlagVar(cmd, &o.LabelSelector) + util.AddProfileFlagVar(cmd, &o.ProfileFilter) + util.AddLocalQueueFlagVar(cmd, &o.LocalQueueFilter) + + cobra.CheckErr(cmd.RegisterFlagCompletionFunc("profile", completion.ApplicationProfileNameFunc(clientGetter))) + cobra.CheckErr(cmd.RegisterFlagCompletionFunc("localqueue", completion.LocalQueueNameFunc(clientGetter))) + + return cmd +} + +// Complete completes all the required options +func (o *RayClusterOptions) Complete(clientGetter util.ClientGetter) error { + var err error + + o.Limit, err = listRequestLimit() + if err != nil { + return err + } + + o.Namespace, _, err = clientGetter.ToRawKubeConfigLoader().Namespace() + if err != nil { + return err + } + + clientset, err := clientGetter.RayClientset() + if err != nil { + return err + } + + o.Client = clientset.RayV1() + + return nil +} + +func (o *RayClusterOptions) ToPrinter(headers bool) (printers.ResourcePrinterFunc, error) { + if !o.PrintFlags.OutputFlagSpecified() { + printer := newRayClusterTablePrinter(). + WithNamespace(o.AllNamespaces). + WithHeaders(headers). + WithClock(o.Clock) + return printer.PrintObj, nil + } + + printer, err := o.PrintFlags.ToPrinter() + if err != nil { + return nil, err + } + + return printer.PrintObj, nil +} + +// Run performs the list operation. +func (o *RayClusterOptions) Run(ctx context.Context) error { + var totalCount int + + namespace := o.Namespace + if o.AllNamespaces { + namespace = "" + } + + opts := metav1.ListOptions{ + FieldSelector: o.FieldSelector, + Limit: o.Limit, + } + + if len(o.ProfileFilter) > 0 { + opts.LabelSelector = fmt.Sprintf("%s=%s", constants.ProfileLabel, o.ProfileFilter) + } else { + opts.LabelSelector = constants.ProfileLabel + } + if len(o.LocalQueueFilter) > 0 { + opts.LabelSelector = fmt.Sprintf("%s,%s=%s", opts.LabelSelector, kueueconstants.QueueLabel, o.LocalQueueFilter) + } + if len(o.LabelSelector) > 0 { + opts.LabelSelector = fmt.Sprintf("%s,%s", opts.LabelSelector, o.LabelSelector) + } + + tabWriter := printers.GetNewTabWriter(o.Out) + + for { + headers := totalCount == 0 + + list, err := o.Client.RayClusters(namespace).List(ctx, opts) + if err != nil { + return err + } + + totalCount += len(list.Items) + + printer, err := o.ToPrinter(headers) + if err != nil { + return err + } + + if err := printer.PrintObj(list, tabWriter); err != nil { + return err + } + + if list.Continue != "" { + opts.Continue = list.Continue + continue + } + + if totalCount == 0 { + if !o.AllNamespaces { + fmt.Fprintf(o.ErrOut, "No resources found in %s namespace.\n", o.Namespace) + } else { + fmt.Fprintln(o.ErrOut, "No resources found") + } + return nil + } + + if err := tabWriter.Flush(); err != nil { + return err + } + + return nil + } +} diff --git a/pkg/cmd/list/list_ray_cluster_printer.go b/pkg/cmd/list/list_ray_cluster_printer.go new file mode 100644 index 0000000..6b4cc24 --- /dev/null +++ b/pkg/cmd/list/list_ray_cluster_printer.go @@ -0,0 +1,114 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package list + +import ( + "errors" + "io" + + rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/duration" + "k8s.io/cli-runtime/pkg/printers" + "k8s.io/utils/clock" + kueueconstants "sigs.k8s.io/kueue/pkg/controller/constants" + + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/constants" +) + +type listRayClusterPrinter struct { + clock clock.Clock + printOptions printers.PrintOptions +} + +var _ printers.ResourcePrinter = (*listRayClusterPrinter)(nil) + +func (p *listRayClusterPrinter) PrintObj(obj runtime.Object, out io.Writer) error { + printer := printers.NewTablePrinter(p.printOptions) + + list, ok := obj.(*rayv1.RayClusterList) + if !ok { + return errors.New("invalid object type") + } + + table := &metav1.Table{ + ColumnDefinitions: []metav1.TableColumnDefinition{ + {Name: "Name", Type: "string", Format: "name"}, + {Name: "Profile", Type: "string"}, + {Name: "Local Queue", Type: "string"}, + {Name: "Desired Workers", Type: "string"}, + {Name: "Available Workers", Type: "string"}, + {Name: "CPUs", Type: "string"}, + {Name: "Memory", Type: "string"}, + {Name: "GPUs", Type: "string"}, + {Name: "Status", Type: "string"}, + {Name: "Age", Type: "string"}, + }, + Rows: p.printRayClusterList(list), + } + + return printer.PrintObj(table, out) +} + +func (p *listRayClusterPrinter) WithNamespace(f bool) *listRayClusterPrinter { + p.printOptions.WithNamespace = f + return p +} + +func (p *listRayClusterPrinter) WithHeaders(f bool) *listRayClusterPrinter { + p.printOptions.NoHeaders = !f + return p +} + +func (p *listRayClusterPrinter) WithClock(c clock.Clock) *listRayClusterPrinter { + p.clock = c + return p +} + +func newRayClusterTablePrinter() *listRayClusterPrinter { + return &listRayClusterPrinter{ + clock: clock.RealClock{}, + } +} + +func (p *listRayClusterPrinter) printRayClusterList(list *rayv1.RayClusterList) []metav1.TableRow { + rows := make([]metav1.TableRow, len(list.Items)) + for index := range list.Items { + rows[index] = p.printRayCluster(&list.Items[index]) + } + return rows +} + +func (p *listRayClusterPrinter) printRayCluster(rayCluster *rayv1.RayCluster) metav1.TableRow { + row := metav1.TableRow{ + Object: runtime.RawExtension{Object: rayCluster}, + } + row.Cells = []any{ + rayCluster.Name, + rayCluster.ObjectMeta.Labels[constants.ProfileLabel], + rayCluster.ObjectMeta.Labels[kueueconstants.QueueLabel], + rayCluster.Status.DesiredWorkerReplicas, + rayCluster.Status.AvailableWorkerReplicas, + rayCluster.Status.DesiredCPU.String(), + rayCluster.Status.DesiredMemory.String(), + rayCluster.Status.DesiredGPU.String(), + rayCluster.Status.State, + duration.HumanDuration(p.clock.Since(rayCluster.CreationTimestamp.Time)), + } + return row +} diff --git a/pkg/cmd/list/list_ray_cluster_test.go b/pkg/cmd/list/list_ray_cluster_test.go new file mode 100644 index 0000000..de3ab09 --- /dev/null +++ b/pkg/cmd/list/list_ray_cluster_test.go @@ -0,0 +1,381 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package list + +import ( + "fmt" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1" + "github.com/ray-project/kuberay/ray-operator/pkg/client/clientset/versioned/fake" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/cli-runtime/pkg/genericiooptions" + kubetesting "k8s.io/client-go/testing" + testingclock "k8s.io/utils/clock/testing" + + cmdtesting "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/cmd/testing" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/testing/wrappers" +) + +func TestRayClusterCmd(t *testing.T) { + testStartTime := time.Now().Truncate(time.Second) + + testCases := map[string]struct { + ns string + objs []runtime.Object + args []string + wantOut string + wantOutErr string + wantErr error + }{ + "should print only kjobctl ray clusters": { + args: []string{}, + objs: []runtime.Object{ + wrappers.MakeRayCluster("rc1", metav1.NamespaceDefault). + Profile("profile1"). + LocalQueue("localqueue1"). + DesiredWorkerReplicas(2). + AvailableWorkerReplicas(3). + DesiredCPU(resource.MustParse("5")). + DesiredMemory(resource.MustParse("10Gi")). + DesiredGPU(resource.MustParse("10")). + State(rayv1.Ready). + CreationTimestamp(testStartTime.Add(-2 * time.Hour)). + Obj(), + wrappers.MakeRayCluster("rc2", "ns1"). + LocalQueue("localqueue1"). + DesiredWorkerReplicas(2). + AvailableWorkerReplicas(3). + DesiredCPU(resource.MustParse("5")). + DesiredMemory(resource.MustParse("10Gi")). + DesiredGPU(resource.MustParse("10")). + State(rayv1.Ready). + CreationTimestamp(testStartTime.Add(-2 * time.Hour)). + Obj(), + }, + wantOut: `NAME PROFILE LOCAL QUEUE DESIRED WORKERS AVAILABLE WORKERS CPUS MEMORY GPUS STATUS AGE +rc1 profile1 localqueue1 2 3 5 10Gi 10 ready 120m +`, + }, + "should print ray cluster list with namespace filter": { + args: []string{}, + ns: "ns1", + objs: []runtime.Object{ + wrappers.MakeRayCluster("rc1", "ns1"). + Profile("profile1"). + LocalQueue("localqueue1"). + DesiredWorkerReplicas(2). + AvailableWorkerReplicas(3). + DesiredCPU(resource.MustParse("5")). + DesiredMemory(resource.MustParse("10Gi")). + DesiredGPU(resource.MustParse("10")). + State(rayv1.Ready). + CreationTimestamp(testStartTime.Add(-2 * time.Hour)). + Obj(), + wrappers.MakeRayCluster("rc2", "ns2"). + Profile("profile1"). + LocalQueue("localqueue1"). + DesiredWorkerReplicas(2). + AvailableWorkerReplicas(3). + DesiredCPU(resource.MustParse("5")). + DesiredMemory(resource.MustParse("10Gi")). + DesiredGPU(resource.MustParse("10")). + State(rayv1.Ready). + CreationTimestamp(testStartTime.Add(-2 * time.Hour)). + Obj(), + }, + wantOut: `NAME PROFILE LOCAL QUEUE DESIRED WORKERS AVAILABLE WORKERS CPUS MEMORY GPUS STATUS AGE +rc1 profile1 localqueue1 2 3 5 10Gi 10 ready 120m +`, + }, + "should print ray cluster list with profile filter": { + args: []string{"--profile", "profile1"}, + objs: []runtime.Object{ + wrappers.MakeRayCluster("rc1", metav1.NamespaceDefault). + Profile("profile1"). + LocalQueue("localqueue1"). + DesiredWorkerReplicas(2). + AvailableWorkerReplicas(3). + DesiredCPU(resource.MustParse("5")). + DesiredMemory(resource.MustParse("10Gi")). + DesiredGPU(resource.MustParse("10")). + State(rayv1.Ready). + CreationTimestamp(testStartTime.Add(-2 * time.Hour)). + Obj(), + wrappers.MakeRayCluster("rc2", metav1.NamespaceDefault). + Profile("profile2"). + LocalQueue("localqueue1"). + DesiredWorkerReplicas(2). + AvailableWorkerReplicas(3). + DesiredCPU(resource.MustParse("5")). + DesiredMemory(resource.MustParse("10Gi")). + DesiredGPU(resource.MustParse("10")). + State(rayv1.Ready). + CreationTimestamp(testStartTime.Add(-2 * time.Hour)). + Obj(), + }, + wantOut: `NAME PROFILE LOCAL QUEUE DESIRED WORKERS AVAILABLE WORKERS CPUS MEMORY GPUS STATUS AGE +rc1 profile1 localqueue1 2 3 5 10Gi 10 ready 120m +`, + }, + "should print ray cluster list with profile filter (short flag)": { + args: []string{"-p", "profile1"}, + objs: []runtime.Object{ + wrappers.MakeRayCluster("rc1", metav1.NamespaceDefault). + Profile("profile1"). + LocalQueue("localqueue1"). + DesiredWorkerReplicas(2). + AvailableWorkerReplicas(3). + DesiredCPU(resource.MustParse("5")). + DesiredMemory(resource.MustParse("10Gi")). + DesiredGPU(resource.MustParse("10")). + State(rayv1.Ready). + CreationTimestamp(testStartTime.Add(-2 * time.Hour)). + Obj(), + wrappers.MakeRayCluster("rc2", metav1.NamespaceDefault). + Profile("profile2"). + LocalQueue("localqueue1"). + DesiredWorkerReplicas(2). + AvailableWorkerReplicas(3). + DesiredCPU(resource.MustParse("5")). + DesiredMemory(resource.MustParse("10Gi")). + DesiredGPU(resource.MustParse("10")). + State(rayv1.Ready). + CreationTimestamp(testStartTime.Add(-2 * time.Hour)). + Obj(), + }, + wantOut: `NAME PROFILE LOCAL QUEUE DESIRED WORKERS AVAILABLE WORKERS CPUS MEMORY GPUS STATUS AGE +rc1 profile1 localqueue1 2 3 5 10Gi 10 ready 120m +`, + }, + "should print ray cluster list with localqueue filter": { + args: []string{"--localqueue", "localqueue1"}, + objs: []runtime.Object{ + wrappers.MakeRayCluster("rc1", metav1.NamespaceDefault). + Profile("profile1"). + LocalQueue("localqueue1"). + DesiredWorkerReplicas(2). + AvailableWorkerReplicas(3). + DesiredCPU(resource.MustParse("5")). + DesiredMemory(resource.MustParse("10Gi")). + DesiredGPU(resource.MustParse("10")). + State(rayv1.Ready). + CreationTimestamp(testStartTime.Add(-2 * time.Hour)). + Obj(), + wrappers.MakeRayCluster("rc2", metav1.NamespaceDefault). + Profile("profile1"). + LocalQueue("localqueue2"). + DesiredWorkerReplicas(2). + AvailableWorkerReplicas(3). + DesiredCPU(resource.MustParse("5")). + DesiredMemory(resource.MustParse("10Gi")). + DesiredGPU(resource.MustParse("10")). + State(rayv1.Ready). + CreationTimestamp(testStartTime.Add(-2 * time.Hour)). + Obj(), + }, + wantOut: `NAME PROFILE LOCAL QUEUE DESIRED WORKERS AVAILABLE WORKERS CPUS MEMORY GPUS STATUS AGE +rc1 profile1 localqueue1 2 3 5 10Gi 10 ready 120m +`, + }, + "should print ray cluster list with localqueue filter (short filter)": { + args: []string{"-q", "localqueue1"}, + objs: []runtime.Object{ + wrappers.MakeRayCluster("rc1", metav1.NamespaceDefault). + Profile("profile1"). + LocalQueue("localqueue1"). + DesiredWorkerReplicas(2). + AvailableWorkerReplicas(3). + DesiredCPU(resource.MustParse("5")). + DesiredMemory(resource.MustParse("10Gi")). + DesiredGPU(resource.MustParse("10")). + State(rayv1.Ready). + CreationTimestamp(testStartTime.Add(-2 * time.Hour)). + Obj(), + wrappers.MakeRayCluster("rc2", metav1.NamespaceDefault). + Profile("profile1"). + LocalQueue("localqueue2"). + DesiredWorkerReplicas(2). + AvailableWorkerReplicas(3). + DesiredCPU(resource.MustParse("5")). + DesiredMemory(resource.MustParse("10Gi")). + DesiredGPU(resource.MustParse("10")). + State(rayv1.Ready). + CreationTimestamp(testStartTime.Add(-2 * time.Hour)). + Obj(), + }, + wantOut: `NAME PROFILE LOCAL QUEUE DESIRED WORKERS AVAILABLE WORKERS CPUS MEMORY GPUS STATUS AGE +rc1 profile1 localqueue1 2 3 5 10Gi 10 ready 120m +`, + }, + "should print ray cluster list with label selector filter": { + args: []string{"--selector", "foo=bar"}, + objs: []runtime.Object{ + wrappers.MakeRayCluster("rc1", metav1.NamespaceDefault). + Label("foo", "bar"). + Profile("profile1"). + LocalQueue("localqueue1"). + DesiredWorkerReplicas(2). + AvailableWorkerReplicas(3). + DesiredCPU(resource.MustParse("5")). + DesiredMemory(resource.MustParse("10Gi")). + DesiredGPU(resource.MustParse("10")). + State(rayv1.Ready). + CreationTimestamp(testStartTime.Add(-2 * time.Hour)). + Obj(), + wrappers.MakeRayCluster("rc2", metav1.NamespaceDefault). + Profile("profile1"). + LocalQueue("localqueue1"). + DesiredWorkerReplicas(2). + AvailableWorkerReplicas(3). + DesiredCPU(resource.MustParse("5")). + DesiredMemory(resource.MustParse("10Gi")). + DesiredGPU(resource.MustParse("10")). + State(rayv1.Ready). + CreationTimestamp(testStartTime.Add(-2 * time.Hour)). + Obj(), + }, + wantOut: `NAME PROFILE LOCAL QUEUE DESIRED WORKERS AVAILABLE WORKERS CPUS MEMORY GPUS STATUS AGE +rc1 profile1 localqueue1 2 3 5 10Gi 10 ready 120m +`, + }, + "should print ray cluster list with label selector filter (short flag)": { + args: []string{"-l", "foo=bar"}, + objs: []runtime.Object{ + wrappers.MakeRayCluster("rc1", metav1.NamespaceDefault). + Label("foo", "bar"). + Profile("profile1"). + LocalQueue("localqueue1"). + DesiredWorkerReplicas(2). + AvailableWorkerReplicas(3). + DesiredCPU(resource.MustParse("5")). + DesiredMemory(resource.MustParse("10Gi")). + DesiredGPU(resource.MustParse("10")). + State(rayv1.Ready). + CreationTimestamp(testStartTime.Add(-2 * time.Hour)). + Obj(), + wrappers.MakeRayCluster("rc2", metav1.NamespaceDefault). + Profile("profile1"). + LocalQueue("localqueue1"). + DesiredWorkerReplicas(2). + AvailableWorkerReplicas(3). + DesiredCPU(resource.MustParse("5")). + DesiredMemory(resource.MustParse("10Gi")). + DesiredGPU(resource.MustParse("10")). + State(rayv1.Ready). + CreationTimestamp(testStartTime.Add(-2 * time.Hour)). + Obj(), + }, + wantOut: `NAME PROFILE LOCAL QUEUE DESIRED WORKERS AVAILABLE WORKERS CPUS MEMORY GPUS STATUS AGE +rc1 profile1 localqueue1 2 3 5 10Gi 10 ready 120m +`, + }, + "should print ray cluster list with field selector filter": { + args: []string{"--field-selector", "metadata.name=rc1"}, + objs: []runtime.Object{ + wrappers.MakeRayCluster("rc1", metav1.NamespaceDefault). + Profile("profile1"). + LocalQueue("localqueue1"). + DesiredWorkerReplicas(2). + AvailableWorkerReplicas(3). + DesiredCPU(resource.MustParse("5")). + DesiredMemory(resource.MustParse("10Gi")). + DesiredGPU(resource.MustParse("10")). + State(rayv1.Ready). + CreationTimestamp(testStartTime.Add(-2 * time.Hour)). + Obj(), + wrappers.MakeRayCluster("rc2", metav1.NamespaceDefault). + Profile("profile1"). + LocalQueue("localqueue1"). + DesiredWorkerReplicas(2). + AvailableWorkerReplicas(3). + DesiredCPU(resource.MustParse("5")). + DesiredMemory(resource.MustParse("10Gi")). + DesiredGPU(resource.MustParse("10")). + State(rayv1.Ready). + CreationTimestamp(testStartTime.Add(-2 * time.Hour)). + Obj(), + }, + wantOut: `NAME PROFILE LOCAL QUEUE DESIRED WORKERS AVAILABLE WORKERS CPUS MEMORY GPUS STATUS AGE +rc1 profile1 localqueue1 2 3 5 10Gi 10 ready 120m +`, + }, + "should print not found error": { + args: []string{}, + wantOutErr: fmt.Sprintf("No resources found in %s namespace.\n", metav1.NamespaceDefault), + }, + "should print not found error with all-namespaces filter": { + args: []string{"-A"}, + wantOutErr: "No resources found\n", + }, + } + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + streams, _, out, outErr := genericiooptions.NewTestIOStreams() + + clientset := fake.NewSimpleClientset(tc.objs...) + clientset.PrependReactor("list", "rayclusters", func(action kubetesting.Action) (handled bool, ret runtime.Object, err error) { + listAction := action.(kubetesting.ListActionImpl) + fieldsSelector := listAction.GetListRestrictions().Fields + + obj, err := clientset.Tracker().List(listAction.GetResource(), listAction.GetKind(), listAction.Namespace) + rayClusterList := obj.(*rayv1.RayClusterList) + + filtered := make([]rayv1.RayCluster, 0, len(rayClusterList.Items)) + for _, item := range rayClusterList.Items { + fieldsSet := fields.Set{ + "metadata.name": item.Name, + } + if fieldsSelector.Matches(fieldsSet) { + filtered = append(filtered, item) + } + } + rayClusterList.Items = filtered + return true, rayClusterList, err + }) + + tcg := cmdtesting.NewTestClientGetter().WithRayClientset(clientset) + if len(tc.ns) > 0 { + tcg.WithNamespace(tc.ns) + } + + cmd := NewRayClusterCmd(tcg, streams, testingclock.NewFakeClock(testStartTime)) + cmd.SetArgs(tc.args) + + gotErr := cmd.Execute() + if diff := cmp.Diff(tc.wantErr, gotErr, cmpopts.EquateErrors()); diff != "" { + t.Errorf("Unexpected error (-want/+got)\n%s", diff) + } + + gotOut := out.String() + if diff := cmp.Diff(tc.wantOut, gotOut); diff != "" { + t.Errorf("Unexpected output (-want/+got)\n%s", diff) + } + + gotOutErr := outErr.String() + if diff := cmp.Diff(tc.wantOutErr, gotOutErr); diff != "" { + t.Errorf("Unexpected output (-want/+got)\n%s", diff) + } + }) + } +} diff --git a/pkg/cmd/list/list_ray_job.go b/pkg/cmd/list/list_ray_job.go new file mode 100644 index 0000000..7c08304 --- /dev/null +++ b/pkg/cmd/list/list_ray_job.go @@ -0,0 +1,220 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package list + +import ( + "context" + "fmt" + + "github.com/ray-project/kuberay/ray-operator/pkg/client/clientset/versioned/scheme" + rayv1 "github.com/ray-project/kuberay/ray-operator/pkg/client/clientset/versioned/typed/ray/v1" + "github.com/spf13/cobra" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/cli-runtime/pkg/genericclioptions" + "k8s.io/cli-runtime/pkg/genericiooptions" + "k8s.io/cli-runtime/pkg/printers" + "k8s.io/kubectl/pkg/util/templates" + "k8s.io/utils/clock" + kueueconstants "sigs.k8s.io/kueue/pkg/controller/constants" + + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/cmd/completion" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/cmd/util" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/constants" +) + +var ( + rayJobExample = templates.Examples(` + # List RayJob + kjobctl list rayjob + + # List RayJob with profile filter + kjobctl list rayjob --profile my-profile + `) +) + +type RayJobOptions struct { + Clock clock.Clock + PrintFlags *genericclioptions.PrintFlags + + Limit int64 + AllNamespaces bool + Namespace string + ProfileFilter string + LocalQueueFilter string + FieldSelector string + LabelSelector string + + Client rayv1.RayV1Interface + + genericiooptions.IOStreams +} + +func NewRayJobOptions(streams genericiooptions.IOStreams, clock clock.Clock) *RayJobOptions { + return &RayJobOptions{ + PrintFlags: genericclioptions.NewPrintFlags("").WithTypeSetter(scheme.Scheme), + IOStreams: streams, + Clock: clock, + } +} + +func NewRayJobCmd(clientGetter util.ClientGetter, streams genericiooptions.IOStreams, clock clock.Clock) *cobra.Command { + o := NewRayJobOptions(streams, clock) + + cmd := &cobra.Command{ + Use: "rayjob" + + " [--profile PROFILE_NAME]" + + " [--localqueue LOCALQUEUE_NAME]" + + " [--selector key1=value1]" + + " [--field-selector key1=value1]" + + " [--all-namespaces]", + DisableFlagsInUseLine: true, + Short: "List RayJob", + Example: rayJobExample, + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, _ []string) error { + cmd.SilenceUsage = true + err := o.Complete(clientGetter) + if err != nil { + return err + } + return o.Run(cmd.Context()) + }, + } + + o.PrintFlags.AddFlags(cmd) + + util.AddAllNamespacesFlagVar(cmd, &o.AllNamespaces) + util.AddFieldSelectorFlagVar(cmd, &o.FieldSelector) + util.AddLabelSelectorFlagVar(cmd, &o.LabelSelector) + util.AddProfileFlagVar(cmd, &o.ProfileFilter) + util.AddLocalQueueFlagVar(cmd, &o.LocalQueueFilter) + + cobra.CheckErr(cmd.RegisterFlagCompletionFunc("profile", completion.ApplicationProfileNameFunc(clientGetter))) + cobra.CheckErr(cmd.RegisterFlagCompletionFunc("localqueue", completion.LocalQueueNameFunc(clientGetter))) + + return cmd +} + +// Complete completes all the required options +func (o *RayJobOptions) Complete(clientGetter util.ClientGetter) error { + var err error + + o.Limit, err = listRequestLimit() + if err != nil { + return err + } + + o.Namespace, _, err = clientGetter.ToRawKubeConfigLoader().Namespace() + if err != nil { + return err + } + + clientset, err := clientGetter.RayClientset() + if err != nil { + return err + } + + o.Client = clientset.RayV1() + + return nil +} + +func (o *RayJobOptions) ToPrinter(headers bool) (printers.ResourcePrinterFunc, error) { + if !o.PrintFlags.OutputFlagSpecified() { + printer := newRayJobTablePrinter(). + WithNamespace(o.AllNamespaces). + WithHeaders(headers). + WithClock(o.Clock) + return printer.PrintObj, nil + } + + printer, err := o.PrintFlags.ToPrinter() + if err != nil { + return nil, err + } + + return printer.PrintObj, nil +} + +// Run performs the list operation. +func (o *RayJobOptions) Run(ctx context.Context) error { + var totalCount int + + namespace := o.Namespace + if o.AllNamespaces { + namespace = "" + } + + opts := metav1.ListOptions{ + FieldSelector: o.FieldSelector, + Limit: o.Limit, + } + + if len(o.ProfileFilter) > 0 { + opts.LabelSelector = fmt.Sprintf("%s=%s", constants.ProfileLabel, o.ProfileFilter) + } else { + opts.LabelSelector = constants.ProfileLabel + } + if len(o.LocalQueueFilter) > 0 { + opts.LabelSelector = fmt.Sprintf("%s,%s=%s", opts.LabelSelector, kueueconstants.QueueLabel, o.LocalQueueFilter) + } + if len(o.LabelSelector) > 0 { + opts.LabelSelector = fmt.Sprintf("%s,%s", opts.LabelSelector, o.LabelSelector) + } + + tabWriter := printers.GetNewTabWriter(o.Out) + + for { + headers := totalCount == 0 + + list, err := o.Client.RayJobs(namespace).List(ctx, opts) + if err != nil { + return err + } + + totalCount += len(list.Items) + + printer, err := o.ToPrinter(headers) + if err != nil { + return err + } + + if err := printer.PrintObj(list, tabWriter); err != nil { + return err + } + + if list.Continue != "" { + opts.Continue = list.Continue + continue + } + + if totalCount == 0 { + if !o.AllNamespaces { + fmt.Fprintf(o.ErrOut, "No resources found in %s namespace.\n", o.Namespace) + } else { + fmt.Fprintln(o.ErrOut, "No resources found") + } + return nil + } + + if err := tabWriter.Flush(); err != nil { + return err + } + + return nil + } +} diff --git a/pkg/cmd/list/list_ray_job_printer.go b/pkg/cmd/list/list_ray_job_printer.go new file mode 100644 index 0000000..006ad24 --- /dev/null +++ b/pkg/cmd/list/list_ray_job_printer.go @@ -0,0 +1,120 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package list + +import ( + "errors" + "io" + "time" + + rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/duration" + "k8s.io/cli-runtime/pkg/printers" + "k8s.io/utils/clock" + kueueconstants "sigs.k8s.io/kueue/pkg/controller/constants" + + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/constants" +) + +type listRayJobPrinter struct { + clock clock.Clock + printOptions printers.PrintOptions +} + +var _ printers.ResourcePrinter = (*listRayJobPrinter)(nil) + +func (p *listRayJobPrinter) PrintObj(obj runtime.Object, out io.Writer) error { + printer := printers.NewTablePrinter(p.printOptions) + + list, ok := obj.(*rayv1.RayJobList) + if !ok { + return errors.New("invalid object type") + } + + table := &metav1.Table{ + ColumnDefinitions: []metav1.TableColumnDefinition{ + {Name: "Name", Type: "string", Format: "name"}, + {Name: "Profile", Type: "string"}, + {Name: "Local Queue", Type: "string"}, + {Name: "Ray Cluster Name", Type: "string"}, + {Name: "Job Status", Type: "string"}, + {Name: "Deployment Status", Type: "string"}, + {Name: "Start Time", Type: "dateTime"}, + {Name: "End Time", Type: "string"}, + {Name: "Age", Type: "string"}, + }, + Rows: p.printRayJobList(list), + } + + return printer.PrintObj(table, out) +} + +func (p *listRayJobPrinter) WithNamespace(f bool) *listRayJobPrinter { + p.printOptions.WithNamespace = f + return p +} + +func (p *listRayJobPrinter) WithHeaders(f bool) *listRayJobPrinter { + p.printOptions.NoHeaders = !f + return p +} + +func (p *listRayJobPrinter) WithClock(c clock.Clock) *listRayJobPrinter { + p.clock = c + return p +} + +func newRayJobTablePrinter() *listRayJobPrinter { + return &listRayJobPrinter{ + clock: clock.RealClock{}, + } +} + +func (p *listRayJobPrinter) printRayJobList(list *rayv1.RayJobList) []metav1.TableRow { + rows := make([]metav1.TableRow, len(list.Items)) + for index := range list.Items { + rows[index] = p.printRayJob(&list.Items[index]) + } + return rows +} + +func (p *listRayJobPrinter) printRayJob(rayJob *rayv1.RayJob) metav1.TableRow { + row := metav1.TableRow{ + Object: runtime.RawExtension{Object: rayJob}, + } + var startTime, endTime string + if rayJob.Status.StartTime != nil { + startTime = rayJob.Status.StartTime.Format(time.DateTime) + } + if rayJob.Status.EndTime != nil { + endTime = rayJob.Status.EndTime.Format(time.DateTime) + } + row.Cells = []any{ + rayJob.Name, + rayJob.ObjectMeta.Labels[constants.ProfileLabel], + rayJob.ObjectMeta.Labels[kueueconstants.QueueLabel], + rayJob.Status.RayClusterName, + rayJob.Status.JobStatus, + rayJob.Status.JobDeploymentStatus, + startTime, + endTime, + duration.HumanDuration(p.clock.Since(rayJob.CreationTimestamp.Time)), + } + return row +} diff --git a/pkg/cmd/list/list_ray_job_test.go b/pkg/cmd/list/list_ray_job_test.go new file mode 100644 index 0000000..674b795 --- /dev/null +++ b/pkg/cmd/list/list_ray_job_test.go @@ -0,0 +1,387 @@ +/* +Copyright %s Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package list + +import ( + "fmt" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1" + "github.com/ray-project/kuberay/ray-operator/pkg/client/clientset/versioned/fake" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/cli-runtime/pkg/genericiooptions" + kubetesting "k8s.io/client-go/testing" + testingclock "k8s.io/utils/clock/testing" + + cmdtesting "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/cmd/testing" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/testing/wrappers" +) + +func TestRayJobCmd(t *testing.T) { + testStartTime := time.Now().Truncate(time.Second) + + testCases := map[string]struct { + ns string + objs []runtime.Object + args []string + wantOut string + wantOutErr string + wantErr error + }{ + "should print only kjobctl ray jobs": { + args: []string{}, + objs: []runtime.Object{ + wrappers.MakeRayJob("rj1", metav1.NamespaceDefault). + Profile("profile1"). + LocalQueue("localqueue1"). + JobStatus(rayv1.JobStatusSucceeded). + RayClusterName("rc1"). + JobDeploymentStatus(rayv1.JobDeploymentStatusComplete). + CreationTimestamp(testStartTime.Add(-2 * time.Hour)). + StartTime(testStartTime.Add(-2 * time.Hour)). + EndTime(testStartTime.Add(-1 * time.Hour)). + Obj(), + wrappers.MakeRayJob("rj2", "ns1"). + LocalQueue("localqueue1"). + JobStatus(rayv1.JobStatusSucceeded). + RayClusterName("rc1"). + JobDeploymentStatus(rayv1.JobDeploymentStatusComplete). + CreationTimestamp(testStartTime.Add(-2 * time.Hour)). + StartTime(testStartTime.Add(-2 * time.Hour)). + EndTime(testStartTime.Add(-1 * time.Hour)). + Obj(), + }, + wantOut: fmt.Sprintf(`NAME PROFILE LOCAL QUEUE RAY CLUSTER NAME JOB STATUS DEPLOYMENT STATUS START TIME END TIME AGE +rj1 profile1 localqueue1 rc1 SUCCEEDED Complete %s %s 120m +`, + testStartTime.Add(-2*time.Hour).Format(time.DateTime), + testStartTime.Add(-1*time.Hour).Format(time.DateTime), + ), + }, + "should print ray job list with namespace filter": { + args: []string{}, + ns: "ns1", + objs: []runtime.Object{ + wrappers.MakeRayJob("rj1", "ns1"). + Profile("profile1"). + LocalQueue("localqueue1"). + JobStatus(rayv1.JobStatusSucceeded). + RayClusterName("rc1"). + JobDeploymentStatus(rayv1.JobDeploymentStatusComplete). + CreationTimestamp(testStartTime.Add(-2 * time.Hour)). + StartTime(testStartTime.Add(-2 * time.Hour)). + EndTime(testStartTime.Add(-1 * time.Hour)). + Obj(), + wrappers.MakeRayJob("rj2", "ns2"). + Profile("profile1"). + LocalQueue("localqueue1"). + JobStatus(rayv1.JobStatusSucceeded). + RayClusterName("rc1"). + JobDeploymentStatus(rayv1.JobDeploymentStatusComplete). + CreationTimestamp(testStartTime.Add(-2 * time.Hour)). + StartTime(testStartTime.Add(-2 * time.Hour)). + EndTime(testStartTime.Add(-1 * time.Hour)). + Obj(), + }, + wantOut: fmt.Sprintf(`NAME PROFILE LOCAL QUEUE RAY CLUSTER NAME JOB STATUS DEPLOYMENT STATUS START TIME END TIME AGE +rj1 profile1 localqueue1 rc1 SUCCEEDED Complete %s %s 120m +`, + testStartTime.Add(-2*time.Hour).Format(time.DateTime), + testStartTime.Add(-1*time.Hour).Format(time.DateTime), + ), + }, + "should print ray job list with profile filter": { + args: []string{"--profile", "profile1"}, + objs: []runtime.Object{ + wrappers.MakeRayJob("rj1", metav1.NamespaceDefault). + Profile("profile1"). + LocalQueue("localqueue1"). + JobStatus(rayv1.JobStatusSucceeded). + RayClusterName("rc1"). + JobDeploymentStatus(rayv1.JobDeploymentStatusComplete). + CreationTimestamp(testStartTime.Add(-2 * time.Hour)). + StartTime(testStartTime.Add(-2 * time.Hour)). + EndTime(testStartTime.Add(-1 * time.Hour)). + Obj(), + wrappers.MakeRayJob("rj2", metav1.NamespaceDefault). + Profile("profile2"). + LocalQueue("localqueue1"). + JobStatus(rayv1.JobStatusSucceeded). + RayClusterName("rc1"). + JobDeploymentStatus(rayv1.JobDeploymentStatusComplete). + CreationTimestamp(testStartTime.Add(-2 * time.Hour)). + StartTime(testStartTime.Add(-2 * time.Hour)). + EndTime(testStartTime.Add(-1 * time.Hour)). + Obj(), + }, + wantOut: fmt.Sprintf(`NAME PROFILE LOCAL QUEUE RAY CLUSTER NAME JOB STATUS DEPLOYMENT STATUS START TIME END TIME AGE +rj1 profile1 localqueue1 rc1 SUCCEEDED Complete %s %s 120m +`, + testStartTime.Add(-2*time.Hour).Format(time.DateTime), + testStartTime.Add(-1*time.Hour).Format(time.DateTime), + ), + }, + "should print ray job list with profile filter (short flag)": { + args: []string{"-p", "profile1"}, + objs: []runtime.Object{ + wrappers.MakeRayJob("rj1", metav1.NamespaceDefault). + Profile("profile1"). + LocalQueue("localqueue1"). + JobStatus(rayv1.JobStatusSucceeded). + RayClusterName("rc1"). + JobDeploymentStatus(rayv1.JobDeploymentStatusComplete). + CreationTimestamp(testStartTime.Add(-2 * time.Hour)). + StartTime(testStartTime.Add(-2 * time.Hour)). + EndTime(testStartTime.Add(-1 * time.Hour)). + Obj(), + wrappers.MakeRayJob("rj2", metav1.NamespaceDefault). + Profile("profile2"). + LocalQueue("localqueue1"). + JobStatus(rayv1.JobStatusSucceeded). + RayClusterName("rc1"). + JobDeploymentStatus(rayv1.JobDeploymentStatusComplete). + CreationTimestamp(testStartTime.Add(-2 * time.Hour)). + StartTime(testStartTime.Add(-2 * time.Hour)). + EndTime(testStartTime.Add(-1 * time.Hour)). + Obj(), + }, + wantOut: fmt.Sprintf(`NAME PROFILE LOCAL QUEUE RAY CLUSTER NAME JOB STATUS DEPLOYMENT STATUS START TIME END TIME AGE +rj1 profile1 localqueue1 rc1 SUCCEEDED Complete %s %s 120m +`, + testStartTime.Add(-2*time.Hour).Format(time.DateTime), + testStartTime.Add(-1*time.Hour).Format(time.DateTime), + ), + }, + "should print ray job list with localqueue filter": { + args: []string{"--localqueue", "localqueue1"}, + objs: []runtime.Object{ + wrappers.MakeRayJob("rj1", metav1.NamespaceDefault). + Profile("profile1"). + LocalQueue("localqueue1"). + JobStatus(rayv1.JobStatusSucceeded). + RayClusterName("rc1"). + JobDeploymentStatus(rayv1.JobDeploymentStatusComplete). + CreationTimestamp(testStartTime.Add(-2 * time.Hour)). + StartTime(testStartTime.Add(-2 * time.Hour)). + EndTime(testStartTime.Add(-1 * time.Hour)). + Obj(), + wrappers.MakeRayJob("rj2", metav1.NamespaceDefault). + Profile("profile1"). + LocalQueue("localqueue2"). + JobStatus(rayv1.JobStatusSucceeded). + RayClusterName("rc1"). + JobDeploymentStatus(rayv1.JobDeploymentStatusComplete). + CreationTimestamp(testStartTime.Add(-2 * time.Hour)). + StartTime(testStartTime.Add(-2 * time.Hour)). + EndTime(testStartTime.Add(-1 * time.Hour)). + Obj(), + }, + wantOut: fmt.Sprintf(`NAME PROFILE LOCAL QUEUE RAY CLUSTER NAME JOB STATUS DEPLOYMENT STATUS START TIME END TIME AGE +rj1 profile1 localqueue1 rc1 SUCCEEDED Complete %s %s 120m +`, + testStartTime.Add(-2*time.Hour).Format(time.DateTime), + testStartTime.Add(-1*time.Hour).Format(time.DateTime), + ), + }, + "should print ray job list with localqueue filter (short filter)": { + args: []string{"-q", "localqueue1"}, + objs: []runtime.Object{ + wrappers.MakeRayJob("rj1", metav1.NamespaceDefault). + Profile("profile1"). + LocalQueue("localqueue1"). + JobStatus(rayv1.JobStatusSucceeded). + RayClusterName("rc1"). + JobDeploymentStatus(rayv1.JobDeploymentStatusComplete). + CreationTimestamp(testStartTime.Add(-2 * time.Hour)). + StartTime(testStartTime.Add(-2 * time.Hour)). + EndTime(testStartTime.Add(-1 * time.Hour)). + Obj(), + wrappers.MakeRayJob("rj2", metav1.NamespaceDefault). + Profile("profile1"). + LocalQueue("localqueue2"). + JobStatus(rayv1.JobStatusSucceeded). + RayClusterName("rc1"). + JobDeploymentStatus(rayv1.JobDeploymentStatusComplete). + CreationTimestamp(testStartTime.Add(-2 * time.Hour)). + StartTime(testStartTime.Add(-2 * time.Hour)). + EndTime(testStartTime.Add(-1 * time.Hour)). + Obj(), + }, + wantOut: fmt.Sprintf(`NAME PROFILE LOCAL QUEUE RAY CLUSTER NAME JOB STATUS DEPLOYMENT STATUS START TIME END TIME AGE +rj1 profile1 localqueue1 rc1 SUCCEEDED Complete %s %s 120m +`, + testStartTime.Add(-2*time.Hour).Format(time.DateTime), + testStartTime.Add(-1*time.Hour).Format(time.DateTime), + ), + }, + "should print ray job list with label selector filter": { + args: []string{"--selector", "foo=bar"}, + objs: []runtime.Object{ + wrappers.MakeRayJob("rj1", metav1.NamespaceDefault). + Label("foo", "bar"). + Profile("profile1"). + LocalQueue("localqueue1"). + JobStatus(rayv1.JobStatusSucceeded). + RayClusterName("rc1"). + JobDeploymentStatus(rayv1.JobDeploymentStatusComplete). + CreationTimestamp(testStartTime.Add(-2 * time.Hour)). + StartTime(testStartTime.Add(-2 * time.Hour)). + EndTime(testStartTime.Add(-1 * time.Hour)). + Obj(), + wrappers.MakeRayJob("rj2", metav1.NamespaceDefault). + Profile("profile1"). + LocalQueue("localqueue1"). + JobStatus(rayv1.JobStatusSucceeded). + RayClusterName("rc1"). + JobDeploymentStatus(rayv1.JobDeploymentStatusComplete). + CreationTimestamp(testStartTime.Add(-2 * time.Hour)). + StartTime(testStartTime.Add(-2 * time.Hour)). + EndTime(testStartTime.Add(-1 * time.Hour)). + Obj(), + }, + wantOut: fmt.Sprintf(`NAME PROFILE LOCAL QUEUE RAY CLUSTER NAME JOB STATUS DEPLOYMENT STATUS START TIME END TIME AGE +rj1 profile1 localqueue1 rc1 SUCCEEDED Complete %s %s 120m +`, + testStartTime.Add(-2*time.Hour).Format(time.DateTime), + testStartTime.Add(-1*time.Hour).Format(time.DateTime), + ), + }, + "should print ray job list with label selector filter (short flag)": { + args: []string{"-l", "foo=bar"}, + objs: []runtime.Object{ + wrappers.MakeRayJob("rj1", metav1.NamespaceDefault). + Label("foo", "bar"). + Profile("profile1"). + LocalQueue("localqueue1"). + JobStatus(rayv1.JobStatusSucceeded). + RayClusterName("rc1"). + JobDeploymentStatus(rayv1.JobDeploymentStatusComplete). + CreationTimestamp(testStartTime.Add(-2 * time.Hour)). + StartTime(testStartTime.Add(-2 * time.Hour)). + EndTime(testStartTime.Add(-1 * time.Hour)). + Obj(), + wrappers.MakeRayJob("rj2", metav1.NamespaceDefault). + Profile("profile1"). + LocalQueue("localqueue1"). + JobStatus(rayv1.JobStatusSucceeded). + RayClusterName("rc1"). + JobDeploymentStatus(rayv1.JobDeploymentStatusComplete). + CreationTimestamp(testStartTime.Add(-2 * time.Hour)). + StartTime(testStartTime.Add(-2 * time.Hour)). + EndTime(testStartTime.Add(-1 * time.Hour)). + Obj(), + }, + wantOut: fmt.Sprintf(`NAME PROFILE LOCAL QUEUE RAY CLUSTER NAME JOB STATUS DEPLOYMENT STATUS START TIME END TIME AGE +rj1 profile1 localqueue1 rc1 SUCCEEDED Complete %s %s 120m +`, + testStartTime.Add(-2*time.Hour).Format(time.DateTime), + testStartTime.Add(-1*time.Hour).Format(time.DateTime), + ), + }, + "should print ray job list with field selector filter": { + args: []string{"--field-selector", "metadata.name=rj1"}, + objs: []runtime.Object{ + wrappers.MakeRayJob("rj1", metav1.NamespaceDefault). + Profile("profile1"). + LocalQueue("localqueue1"). + JobStatus(rayv1.JobStatusSucceeded). + RayClusterName("rc1"). + JobDeploymentStatus(rayv1.JobDeploymentStatusComplete). + CreationTimestamp(testStartTime.Add(-2 * time.Hour)). + StartTime(testStartTime.Add(-2 * time.Hour)). + EndTime(testStartTime.Add(-1 * time.Hour)). + Obj(), + wrappers.MakeRayJob("rj2", metav1.NamespaceDefault). + Profile("profile1"). + LocalQueue("localqueue1"). + JobStatus(rayv1.JobStatusSucceeded). + RayClusterName("rc1"). + JobDeploymentStatus(rayv1.JobDeploymentStatusComplete). + CreationTimestamp(testStartTime.Add(-2 * time.Hour)). + StartTime(testStartTime.Add(-2 * time.Hour)). + EndTime(testStartTime.Add(-1 * time.Hour)). + Obj(), + }, + wantOut: fmt.Sprintf(`NAME PROFILE LOCAL QUEUE RAY CLUSTER NAME JOB STATUS DEPLOYMENT STATUS START TIME END TIME AGE +rj1 profile1 localqueue1 rc1 SUCCEEDED Complete %s %s 120m +`, + testStartTime.Add(-2*time.Hour).Format(time.DateTime), + testStartTime.Add(-1*time.Hour).Format(time.DateTime), + ), + }, + "should print not found error": { + args: []string{}, + wantOutErr: fmt.Sprintf("No resources found in %s namespace.\n", metav1.NamespaceDefault), + }, + "should print not found error with all-namespaces filter": { + args: []string{"-A"}, + wantOutErr: "No resources found\n", + }, + } + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + streams, _, out, outErr := genericiooptions.NewTestIOStreams() + + clientset := fake.NewSimpleClientset(tc.objs...) + clientset.PrependReactor("list", "rayjobs", func(action kubetesting.Action) (handled bool, ret runtime.Object, err error) { + listAction := action.(kubetesting.ListActionImpl) + fieldsSelector := listAction.GetListRestrictions().Fields + + obj, err := clientset.Tracker().List(listAction.GetResource(), listAction.GetKind(), listAction.Namespace) + rayJobList := obj.(*rayv1.RayJobList) + + filtered := make([]rayv1.RayJob, 0, len(rayJobList.Items)) + for _, item := range rayJobList.Items { + fieldsSet := fields.Set{ + "metadata.name": item.Name, + } + if fieldsSelector.Matches(fieldsSet) { + filtered = append(filtered, item) + } + } + rayJobList.Items = filtered + return true, rayJobList, err + }) + + tcg := cmdtesting.NewTestClientGetter().WithRayClientset(clientset) + if len(tc.ns) > 0 { + tcg.WithNamespace(tc.ns) + } + + cmd := NewRayJobCmd(tcg, streams, testingclock.NewFakeClock(testStartTime)) + cmd.SetArgs(tc.args) + + gotErr := cmd.Execute() + if diff := cmp.Diff(tc.wantErr, gotErr, cmpopts.EquateErrors()); diff != "" { + t.Errorf("Unexpected error (-want/+got)\n%s", diff) + } + + gotOut := out.String() + if diff := cmp.Diff(tc.wantOut, gotOut); diff != "" { + t.Errorf("Unexpected output (-want/+got)\n%s", diff) + } + + gotOutErr := outErr.String() + if diff := cmp.Diff(tc.wantOutErr, gotOutErr); diff != "" { + t.Errorf("Unexpected output (-want/+got)\n%s", diff) + } + }) + } +} diff --git a/pkg/cmd/list/list_slurm.go b/pkg/cmd/list/list_slurm.go new file mode 100644 index 0000000..5ac6756 --- /dev/null +++ b/pkg/cmd/list/list_slurm.go @@ -0,0 +1,223 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package list + +import ( + "context" + "fmt" + + "github.com/spf13/cobra" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/cli-runtime/pkg/genericclioptions" + "k8s.io/cli-runtime/pkg/genericiooptions" + "k8s.io/cli-runtime/pkg/printers" + "k8s.io/client-go/kubernetes/scheme" + batchv1 "k8s.io/client-go/kubernetes/typed/batch/v1" + "k8s.io/kubectl/pkg/util/templates" + "k8s.io/utils/clock" + kueueconstants "sigs.k8s.io/kueue/pkg/controller/constants" + + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/apis/v1alpha1" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/cmd/completion" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/cmd/util" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/constants" +) + +var ( + slurmExample = templates.Examples(` + # List Slurm + kjobctl list slurm + + # List Slurm with profile filter + kjobctl list slurm --profile my-profile + `) +) + +type SlurmOptions struct { + Clock clock.Clock + PrintFlags *genericclioptions.PrintFlags + + Limit int64 + AllNamespaces bool + Namespace string + ProfileFilter string + LocalQueueFilter string + FieldSelector string + LabelSelector string + + Client batchv1.BatchV1Interface + + genericiooptions.IOStreams +} + +func NewSlurmOptions(streams genericiooptions.IOStreams, clock clock.Clock) *SlurmOptions { + return &SlurmOptions{ + PrintFlags: genericclioptions.NewPrintFlags("").WithTypeSetter(scheme.Scheme), + IOStreams: streams, + Clock: clock, + } +} + +func NewSlurmCmd(clientGetter util.ClientGetter, streams genericiooptions.IOStreams, clock clock.Clock) *cobra.Command { + o := NewSlurmOptions(streams, clock) + + cmd := &cobra.Command{ + Use: "slurm" + + " [--profile PROFILE_NAME]" + + " [--localqueue LOCALQUEUE_NAME]" + + " [--selector key1=value1]" + + " [--field-selector key1=value1]" + + " [--all-namespaces]", + DisableFlagsInUseLine: true, + Short: "List Slurm", + Example: slurmExample, + RunE: func(cmd *cobra.Command, _ []string) error { + cmd.SilenceUsage = true + err := o.Complete(clientGetter) + if err != nil { + return err + } + return o.Run(cmd.Context()) + }, + } + + o.PrintFlags.AddFlags(cmd) + + util.AddAllNamespacesFlagVar(cmd, &o.AllNamespaces) + util.AddFieldSelectorFlagVar(cmd, &o.FieldSelector) + util.AddLabelSelectorFlagVar(cmd, &o.LabelSelector) + util.AddProfileFlagVar(cmd, &o.ProfileFilter) + util.AddLocalQueueFlagVar(cmd, &o.LocalQueueFilter) + + cobra.CheckErr(cmd.RegisterFlagCompletionFunc("profile", completion.ApplicationProfileNameFunc(clientGetter))) + cobra.CheckErr(cmd.RegisterFlagCompletionFunc("localqueue", completion.LocalQueueNameFunc(clientGetter))) + + return cmd +} + +// Complete completes all the required options +func (o *SlurmOptions) Complete(clientGetter util.ClientGetter) error { + var err error + + o.Limit, err = listRequestLimit() + if err != nil { + return err + } + + o.Namespace, _, err = clientGetter.ToRawKubeConfigLoader().Namespace() + if err != nil { + return err + } + + clientset, err := clientGetter.K8sClientset() + if err != nil { + return err + } + + o.Client = clientset.BatchV1() + + return nil +} + +func (o *SlurmOptions) ToPrinter(headers bool) (printers.ResourcePrinterFunc, error) { + if !o.PrintFlags.OutputFlagSpecified() { + printer := newJobTablePrinter(). + WithNamespace(o.AllNamespaces). + WithHeaders(headers). + WithClock(o.Clock) + return printer.PrintObj, nil + } + + printer, err := o.PrintFlags.ToPrinter() + if err != nil { + return nil, err + } + + return printer.PrintObj, nil +} + +// Run performs the list operation. +func (o *SlurmOptions) Run(ctx context.Context) error { + var totalCount int + + namespace := o.Namespace + if o.AllNamespaces { + namespace = "" + } + + opts := metav1.ListOptions{ + FieldSelector: o.FieldSelector, + Limit: o.Limit, + } + + if len(o.ProfileFilter) > 0 { + opts.LabelSelector = fmt.Sprintf("%s=%s", constants.ProfileLabel, o.ProfileFilter) + } else { + opts.LabelSelector = constants.ProfileLabel + } + + opts.LabelSelector = fmt.Sprintf("%s,%s=%s", opts.LabelSelector, constants.ModeLabel, v1alpha1.SlurmMode) + + if len(o.LocalQueueFilter) > 0 { + opts.LabelSelector = fmt.Sprintf("%s,%s=%s", opts.LabelSelector, kueueconstants.QueueLabel, o.LocalQueueFilter) + } + if len(o.LabelSelector) > 0 { + opts.LabelSelector = fmt.Sprintf("%s,%s", opts.LabelSelector, o.LabelSelector) + } + + tabWriter := printers.GetNewTabWriter(o.Out) + + for { + headers := totalCount == 0 + + list, err := o.Client.Jobs(namespace).List(ctx, opts) + if err != nil { + return err + } + + totalCount += len(list.Items) + + printer, err := o.ToPrinter(headers) + if err != nil { + return err + } + + if err := printer.PrintObj(list, tabWriter); err != nil { + return err + } + + if list.Continue != "" { + opts.Continue = list.Continue + continue + } + + if totalCount == 0 { + if !o.AllNamespaces { + fmt.Fprintf(o.ErrOut, "No resources found in %s namespace.\n", o.Namespace) + } else { + fmt.Fprintln(o.ErrOut, "No resources found") + } + return nil + } + + if err := tabWriter.Flush(); err != nil { + return err + } + + return nil + } +} diff --git a/pkg/cmd/list/list_slurm_test.go b/pkg/cmd/list/list_slurm_test.go new file mode 100644 index 0000000..0bfc814 --- /dev/null +++ b/pkg/cmd/list/list_slurm_test.go @@ -0,0 +1,389 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package list + +import ( + "fmt" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + batchv1 "k8s.io/api/batch/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/cli-runtime/pkg/genericiooptions" + "k8s.io/client-go/kubernetes/fake" + kubetesting "k8s.io/client-go/testing" + testingclock "k8s.io/utils/clock/testing" + + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/apis/v1alpha1" + cmdtesting "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/cmd/testing" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/testing/wrappers" +) + +func TestSlurmCmd(t *testing.T) { + testStartTime := time.Now() + + testCases := map[string]struct { + ns string + objs []runtime.Object + args []string + wantOut string + wantOutErr string + wantErr error + }{ + "should print only kjobctl jobs": { + ns: "ns1", + objs: []runtime.Object{ + wrappers.MakeJob("j1", "ns1"). + Profile("profile1"). + Mode(v1alpha1.SlurmMode). + LocalQueue("lq1"). + Completions(3). + CreationTimestamp(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + StartTime(testStartTime.Add(-2 * time.Hour).Truncate(time.Second)). + CompletionTime(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + Succeeded(3). + Obj(), + wrappers.MakeJob("j2", "ns2"). + LocalQueue("lq2"). + Completions(3). + CreationTimestamp(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + StartTime(testStartTime.Add(-2 * time.Hour).Truncate(time.Second)). + CompletionTime(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + Succeeded(3). + Obj(), + }, + wantOut: `NAME PROFILE LOCAL QUEUE COMPLETIONS DURATION AGE +j1 profile1 lq1 3/3 60m 60m +`, + }, + "should print only Slurm mode jobs": { + ns: "ns1", + objs: []runtime.Object{ + wrappers.MakeJob("j1", "ns1"). + Profile("profile1"). + Mode(v1alpha1.SlurmMode). + LocalQueue("lq1"). + Completions(3). + CreationTimestamp(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + StartTime(testStartTime.Add(-2 * time.Hour).Truncate(time.Second)). + CompletionTime(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + Succeeded(3). + Obj(), + wrappers.MakeJob("j2", "ns2"). + LocalQueue("lq2"). + Completions(3). + CreationTimestamp(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + StartTime(testStartTime.Add(-2 * time.Hour).Truncate(time.Second)). + CompletionTime(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + Succeeded(3). + Obj(), + }, + wantOut: `NAME PROFILE LOCAL QUEUE COMPLETIONS DURATION AGE +j1 profile1 lq1 3/3 60m 60m +`, + }, + "should print slurm job list with namespace filter": { + ns: "ns1", + objs: []runtime.Object{ + wrappers.MakeJob("j1", "ns1"). + Profile("profile1"). + Mode(v1alpha1.SlurmMode). + LocalQueue("lq1"). + Completions(3). + CreationTimestamp(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + StartTime(testStartTime.Add(-2 * time.Hour).Truncate(time.Second)). + CompletionTime(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + Succeeded(3). + Obj(), + wrappers.MakeJob("j2", "ns2"). + Profile("profile2"). + Mode(v1alpha1.SlurmMode). + LocalQueue("lq2"). + Completions(3). + CreationTimestamp(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + StartTime(testStartTime.Add(-2 * time.Hour).Truncate(time.Second)). + CompletionTime(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + Succeeded(3). + Obj(), + }, + wantOut: `NAME PROFILE LOCAL QUEUE COMPLETIONS DURATION AGE +j1 profile1 lq1 3/3 60m 60m +`, + }, + "should print slurm job list with profile filter": { + args: []string{"--profile", "profile1"}, + objs: []runtime.Object{ + wrappers.MakeJob("j1", metav1.NamespaceDefault). + Profile("profile1"). + Mode(v1alpha1.SlurmMode). + LocalQueue("lq1"). + Completions(3). + CreationTimestamp(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + StartTime(testStartTime.Add(-2 * time.Hour).Truncate(time.Second)). + CompletionTime(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + Succeeded(3). + Obj(), + wrappers.MakeJob("j2", metav1.NamespaceDefault). + Profile("profile2"). + Mode(v1alpha1.SlurmMode). + LocalQueue("lq2"). + Completions(3). + CreationTimestamp(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + StartTime(testStartTime.Add(-2 * time.Hour).Truncate(time.Second)). + CompletionTime(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + Succeeded(3). + Obj(), + }, + wantOut: `NAME PROFILE LOCAL QUEUE COMPLETIONS DURATION AGE +j1 profile1 lq1 3/3 60m 60m +`, + }, + "should print slurm job list with profile filter (short flag)": { + args: []string{"-p", "profile1"}, + objs: []runtime.Object{ + wrappers.MakeJob("j1", metav1.NamespaceDefault). + Profile("profile1"). + Mode(v1alpha1.SlurmMode). + LocalQueue("lq1"). + Completions(3). + CreationTimestamp(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + StartTime(testStartTime.Add(-2 * time.Hour).Truncate(time.Second)). + CompletionTime(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + Succeeded(3). + Obj(), + wrappers.MakeJob("j2", metav1.NamespaceDefault). + Profile("profile2"). + Mode(v1alpha1.SlurmMode). + LocalQueue("lq2"). + Completions(3). + CreationTimestamp(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + StartTime(testStartTime.Add(-2 * time.Hour).Truncate(time.Second)). + CompletionTime(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + Succeeded(3). + Obj(), + }, + wantOut: `NAME PROFILE LOCAL QUEUE COMPLETIONS DURATION AGE +j1 profile1 lq1 3/3 60m 60m +`, + }, + "should print slurm job list with localqueue filter": { + args: []string{"--localqueue", "lq1"}, + objs: []runtime.Object{ + wrappers.MakeJob("j1", metav1.NamespaceDefault). + Profile("profile1"). + Mode(v1alpha1.SlurmMode). + LocalQueue("lq1"). + LocalQueue("lq1"). + Completions(3). + CreationTimestamp(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + StartTime(testStartTime.Add(-2 * time.Hour).Truncate(time.Second)). + CompletionTime(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + Succeeded(3). + Obj(), + wrappers.MakeJob("j2", metav1.NamespaceDefault). + Profile("profile2"). + Mode(v1alpha1.SlurmMode). + LocalQueue("lq2"). + LocalQueue("lq2"). + Completions(3). + CreationTimestamp(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + StartTime(testStartTime.Add(-2 * time.Hour).Truncate(time.Second)). + CompletionTime(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + Succeeded(3). + Obj(), + }, + wantOut: `NAME PROFILE LOCAL QUEUE COMPLETIONS DURATION AGE +j1 profile1 lq1 3/3 60m 60m +`, + }, + "should print slurm job list with localqueue filter (short flag)": { + args: []string{"-q", "lq1"}, + objs: []runtime.Object{ + wrappers.MakeJob("j1", metav1.NamespaceDefault). + Profile("profile1"). + Mode(v1alpha1.SlurmMode). + LocalQueue("lq1"). + Completions(3). + CreationTimestamp(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + StartTime(testStartTime.Add(-2 * time.Hour).Truncate(time.Second)). + CompletionTime(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + Succeeded(3). + Obj(), + wrappers.MakeJob("j2", metav1.NamespaceDefault). + Profile("profile2"). + Mode(v1alpha1.SlurmMode). + LocalQueue("lq2"). + LocalQueue("lq2"). + Completions(3). + CreationTimestamp(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + StartTime(testStartTime.Add(-2 * time.Hour).Truncate(time.Second)). + CompletionTime(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + Succeeded(3). + Obj(), + }, + wantOut: `NAME PROFILE LOCAL QUEUE COMPLETIONS DURATION AGE +j1 profile1 lq1 3/3 60m 60m +`, + }, + "should print slurm job list with label selector filter": { + args: []string{"--selector", "foo=bar"}, + objs: []runtime.Object{ + wrappers.MakeJob("j1", metav1.NamespaceDefault). + Profile("profile1"). + Mode(v1alpha1.SlurmMode). + LocalQueue("lq1"). + Label("foo", "bar"). + Completions(3). + CreationTimestamp(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + StartTime(testStartTime.Add(-2 * time.Hour).Truncate(time.Second)). + CompletionTime(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + Succeeded(3). + Obj(), + wrappers.MakeJob("j2", metav1.NamespaceDefault). + Profile("profile2"). + Mode(v1alpha1.SlurmMode). + LocalQueue("lq2"). + Completions(3). + CreationTimestamp(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + StartTime(testStartTime.Add(-2 * time.Hour).Truncate(time.Second)). + CompletionTime(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + Succeeded(3). + Obj(), + }, + wantOut: `NAME PROFILE LOCAL QUEUE COMPLETIONS DURATION AGE +j1 profile1 lq1 3/3 60m 60m +`, + }, + "should print slurm job list with label selector filter (short flag)": { + args: []string{"-l", "foo=bar"}, + objs: []runtime.Object{ + wrappers.MakeJob("j1", metav1.NamespaceDefault). + Profile("profile1"). + Mode(v1alpha1.SlurmMode). + LocalQueue("lq1"). + Label("foo", "bar"). + Completions(3). + CreationTimestamp(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + StartTime(testStartTime.Add(-2 * time.Hour).Truncate(time.Second)). + CompletionTime(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + Succeeded(3). + Obj(), + wrappers.MakeJob("j2", metav1.NamespaceDefault). + Profile("profile2"). + Mode(v1alpha1.SlurmMode). + LocalQueue("lq2"). + Completions(3). + CreationTimestamp(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + StartTime(testStartTime.Add(-2 * time.Hour).Truncate(time.Second)). + CompletionTime(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + Succeeded(3). + Obj(), + }, + wantOut: `NAME PROFILE LOCAL QUEUE COMPLETIONS DURATION AGE +j1 profile1 lq1 3/3 60m 60m +`, + }, + "should print slurm job list with field selector filter": { + args: []string{"--field-selector", "metadata.name=j1"}, + objs: []runtime.Object{ + wrappers.MakeJob("j1", metav1.NamespaceDefault). + Profile("profile1"). + Mode(v1alpha1.SlurmMode). + LocalQueue("lq1"). + Completions(3). + CreationTimestamp(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + StartTime(testStartTime.Add(-2 * time.Hour).Truncate(time.Second)). + CompletionTime(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + Succeeded(3). + Obj(), + wrappers.MakeJob("j2", metav1.NamespaceDefault). + Profile("profile2"). + Mode(v1alpha1.SlurmMode). + LocalQueue("lq2"). + Completions(3). + CreationTimestamp(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + StartTime(testStartTime.Add(-2 * time.Hour).Truncate(time.Second)). + CompletionTime(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + Succeeded(3). + Obj(), + }, + wantOut: `NAME PROFILE LOCAL QUEUE COMPLETIONS DURATION AGE +j1 profile1 lq1 3/3 60m 60m +`, + }, + "should print not found error": { + wantOutErr: fmt.Sprintf("No resources found in %s namespace.\n", metav1.NamespaceDefault), + }, + "should print not found error with all-namespaces filter": { + args: []string{"-A"}, + wantOutErr: "No resources found\n", + }, + } + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + streams, _, out, outErr := genericiooptions.NewTestIOStreams() + + clientset := fake.NewSimpleClientset(tc.objs...) + clientset.PrependReactor("list", "jobs", func(action kubetesting.Action) (handled bool, ret runtime.Object, err error) { + listAction := action.(kubetesting.ListActionImpl) + fieldsSelector := listAction.GetListRestrictions().Fields + + obj, err := clientset.Tracker().List(listAction.GetResource(), listAction.GetKind(), listAction.Namespace) + jobList := obj.(*batchv1.JobList) + + filtered := make([]batchv1.Job, 0, len(jobList.Items)) + for _, item := range jobList.Items { + fieldsSet := fields.Set{ + "metadata.name": item.Name, + } + if fieldsSelector.Matches(fieldsSet) { + filtered = append(filtered, item) + } + } + jobList.Items = filtered + return true, jobList, err + }) + + tcg := cmdtesting.NewTestClientGetter().WithK8sClientset(clientset) + if len(tc.ns) > 0 { + tcg.WithNamespace(tc.ns) + } + + cmd := NewSlurmCmd(tcg, streams, testingclock.NewFakeClock(testStartTime)) + cmd.SetArgs(tc.args) + + gotErr := cmd.Execute() + if diff := cmp.Diff(tc.wantErr, gotErr, cmpopts.EquateErrors()); diff != "" { + t.Errorf("Unexpected error (-want/+got)\n%s", diff) + } + + gotOut := out.String() + if diff := cmp.Diff(tc.wantOut, gotOut); diff != "" { + t.Errorf("Unexpected output (-want/+got)\n%s", diff) + } + + gotOutErr := outErr.String() + if diff := cmp.Diff(tc.wantOutErr, gotOutErr); diff != "" { + t.Errorf("Unexpected output (-want/+got)\n%s", diff) + } + }) + } +} diff --git a/pkg/cmd/list/list_test.go b/pkg/cmd/list/list_test.go new file mode 100644 index 0000000..f700218 --- /dev/null +++ b/pkg/cmd/list/list_test.go @@ -0,0 +1,206 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package list + +import ( + "fmt" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1" + rayfake "github.com/ray-project/kuberay/ray-operator/pkg/client/clientset/versioned/fake" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/cli-runtime/pkg/genericiooptions" + k8sfake "k8s.io/client-go/kubernetes/fake" + testingclock "k8s.io/utils/clock/testing" + + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/apis/v1alpha1" + cmdtesting "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/cmd/testing" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/testing/wrappers" +) + +func TestListCmd(t *testing.T) { + testStartTime := time.Now() + + testCases := map[string]struct { + ns string + k8sObjs []runtime.Object + rayObjs []runtime.Object + args []string + wantOut string + wantOutErr string + wantErr error + }{ + "should print job list with all namespaces": { + args: []string{"job", "--all-namespaces"}, + k8sObjs: []runtime.Object{ + wrappers.MakeJob("j1", "ns1"). + Profile("profile1"). + Mode(v1alpha1.JobMode). + LocalQueue("lq1"). + Completions(3). + CreationTimestamp(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + StartTime(testStartTime.Add(-2 * time.Hour).Truncate(time.Second)). + CompletionTime(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + Succeeded(3). + Obj(), + wrappers.MakeJob("j2", "ns2"). + Profile("profile2"). + Mode(v1alpha1.JobMode). + LocalQueue("lq1"). + Completions(3). + CreationTimestamp(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + StartTime(testStartTime.Add(-2 * time.Hour).Truncate(time.Second)). + CompletionTime(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + Succeeded(3). + Obj(), + }, + wantOut: `NAMESPACE NAME PROFILE LOCAL QUEUE COMPLETIONS DURATION AGE +ns1 j1 profile1 lq1 3/3 60m 60m +ns2 j2 profile2 lq1 3/3 60m 60m +`, + }, + "should print slurm job list with all namespaces": { + args: []string{"slurm", "--all-namespaces"}, + k8sObjs: []runtime.Object{ + wrappers.MakeJob("j1", "ns1"). + Profile("profile1"). + Mode(v1alpha1.SlurmMode). + LocalQueue("lq1"). + Completions(3). + CreationTimestamp(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + StartTime(testStartTime.Add(-2 * time.Hour).Truncate(time.Second)). + CompletionTime(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + Succeeded(3). + Obj(), + wrappers.MakeJob("j2", "ns2"). + Profile("profile2"). + Mode(v1alpha1.SlurmMode). + LocalQueue("lq1"). + Completions(3). + CreationTimestamp(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + StartTime(testStartTime.Add(-2 * time.Hour).Truncate(time.Second)). + CompletionTime(testStartTime.Add(-1 * time.Hour).Truncate(time.Second)). + Succeeded(3). + Obj(), + }, + wantOut: `NAMESPACE NAME PROFILE LOCAL QUEUE COMPLETIONS DURATION AGE +ns1 j1 profile1 lq1 3/3 60m 60m +ns2 j2 profile2 lq1 3/3 60m 60m +`, + }, + "should print rayjob list with all namespaces": { + args: []string{"rayjob", "--all-namespaces"}, + rayObjs: []runtime.Object{ + wrappers.MakeRayJob("rj1", metav1.NamespaceDefault). + Profile("profile1"). + LocalQueue("lq1"). + JobStatus(rayv1.JobStatusSucceeded). + RayClusterName("rc1"). + JobDeploymentStatus(rayv1.JobDeploymentStatusComplete). + CreationTimestamp(testStartTime.Add(-2 * time.Hour)). + StartTime(testStartTime.Add(-2 * time.Hour)). + EndTime(testStartTime.Add(-1 * time.Hour)). + Obj(), + wrappers.MakeRayJob("rj2", metav1.NamespaceDefault). + Profile("profile2"). + LocalQueue("lq2"). + JobStatus(rayv1.JobStatusSucceeded). + JobDeploymentStatus(rayv1.JobDeploymentStatusComplete). + CreationTimestamp(testStartTime.Add(-2 * time.Hour)). + StartTime(testStartTime.Add(-2 * time.Hour)). + EndTime(testStartTime.Add(-1 * time.Hour)). + Obj(), + }, + wantOut: fmt.Sprintf(`NAMESPACE NAME PROFILE LOCAL QUEUE RAY CLUSTER NAME JOB STATUS DEPLOYMENT STATUS START TIME END TIME AGE +default rj1 profile1 lq1 rc1 SUCCEEDED Complete %s %s 120m +default rj2 profile2 lq2 SUCCEEDED Complete %s %s 120m +`, + testStartTime.Add(-2*time.Hour).Format(time.DateTime), + testStartTime.Add(-1*time.Hour).Format(time.DateTime), + testStartTime.Add(-2*time.Hour).Format(time.DateTime), + testStartTime.Add(-1*time.Hour).Format(time.DateTime), + ), + }, + "should print raycluster list with all namespaces": { + args: []string{"raycluster", "--all-namespaces"}, + rayObjs: []runtime.Object{ + wrappers.MakeRayCluster("rc1", metav1.NamespaceDefault). + Profile("profile1"). + LocalQueue("lq1"). + DesiredWorkerReplicas(2). + AvailableWorkerReplicas(3). + DesiredCPU(resource.MustParse("5")). + DesiredMemory(resource.MustParse("10Gi")). + DesiredGPU(resource.MustParse("10")). + State(rayv1.Ready). + CreationTimestamp(testStartTime.Add(-2 * time.Hour)). + Obj(), + wrappers.MakeRayCluster("rc2", metav1.NamespaceDefault). + Profile("profile2"). + LocalQueue("lq2"). + DesiredWorkerReplicas(2). + AvailableWorkerReplicas(3). + DesiredCPU(resource.MustParse("5")). + DesiredMemory(resource.MustParse("10Gi")). + DesiredGPU(resource.MustParse("10")). + State(rayv1.Ready). + CreationTimestamp(testStartTime.Add(-2 * time.Hour)). + Obj(), + }, + wantOut: `NAMESPACE NAME PROFILE LOCAL QUEUE DESIRED WORKERS AVAILABLE WORKERS CPUS MEMORY GPUS STATUS AGE +default rc1 profile1 lq1 2 3 5 10Gi 10 ready 120m +default rc2 profile2 lq2 2 3 5 10Gi 10 ready 120m +`, + }, + } + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + streams, _, out, outErr := genericiooptions.NewTestIOStreams() + + tcg := cmdtesting.NewTestClientGetter() + if len(tc.ns) > 0 { + tcg.WithNamespace(tc.ns) + } + + tcg.WithK8sClientset(k8sfake.NewSimpleClientset(tc.k8sObjs...)) + tcg.WithRayClientset(rayfake.NewSimpleClientset(tc.rayObjs...)) + + cmd := NewListCmd(tcg, streams, testingclock.NewFakeClock(testStartTime)) + cmd.SetArgs(tc.args) + + gotErr := cmd.Execute() + if diff := cmp.Diff(tc.wantErr, gotErr, cmpopts.EquateErrors()); diff != "" { + t.Errorf("Unexpected error (-want/+got)\n%s", diff) + } + + gotOut := out.String() + if diff := cmp.Diff(tc.wantOut, gotOut); diff != "" { + t.Errorf("Unexpected output (-want/+got)\n%s", diff) + } + + gotOutErr := outErr.String() + if diff := cmp.Diff(tc.wantOutErr, gotOutErr); diff != "" { + t.Errorf("Unexpected output (-want/+got)\n%s", diff) + } + }) + } +} diff --git a/pkg/cmd/printcrds/embed/manifest.gz b/pkg/cmd/printcrds/embed/manifest.gz new file mode 100644 index 0000000000000000000000000000000000000000..8061f42c42b7c22608769cf23e4d490c7306579c GIT binary patch literal 432291 zcmagFb8sfn*EJeD6XS_(Ke5e8Cf3BZZQB#uwr$(CZD*2v^Zwp@>r>sj{m1Fr>#V(Z zSDikmt9PwU7zqRNUl-W-<)^13X-geP%y*5Ne?hBHf}ErUN6Z76XTqQPISXrYohpkW z&Yr}BKai$^sB;KRpiGIWBQu`gm^dkYx7^q|g-sG=5h;fV)zRX*{9DoR@AxPQ4$n|~ z0ST01sb6H!NIgm5<1-~Su>@2yJGm2zG}*pn2`MDRwqs!|=vaMHQTI*!3`tu-)Fv}jKd!Wr@99(AHBMdQ*FSfysocBA9=@3K@yEO$`f*}w* z$W3T<=<3N;cYCQ@oQj+UrBS{%IZ{bbnS_mav@=9pt9L{NjoBBgW#vnW*i9%X--dZg zSCFT##j)V6PWJ{>-AScX`j}D ztLUjoT;?r{4BM4#xV&UsP%PlH#r8MJP~HLpEeS^cPn15pkYY|+DJ1>m>88vB0IKLq zxHM)XG~?)-edkF4(YW|u#B?f^QI3-n!}85=X}9c@sPZ)~ydT_j7+CkbSM6ol=g|fedSAkTPTjgmKjAanU2fNU@A7NK>BZ3R?&9Z6KsvWsK z^D-}Y=hp;n@u^uX8H#)~YFDv3vg6{SsW0LIC$kj#=6%#kx5Q&q`fy_osY*p-ySn7@ z(zaV(4Jp0OvvIlgl653!f5BG}wD&WOQq&W`Z<4k#)>blR&02(YxW8*Pws)aNzE!QR zNC;bLB^7y`sNqWho@%J7*C<_~z-r&*RCKOn1I)#sEI#9=~KVje(zfB%>@W6<;=O ziB72~6NJc^5Zaa$foNhSg`r=Ap?r@hGn>AQ4T>}*xi1}+=x^oCu(S^<6LUx(>Ti=d zuXabaCZZQU&EVQ^FbFhUFqF{(@a+5$P%VlGCtoP4a+zwGYD$WTLB6e&r ze^c=CDR*gAS2qFTzTVumHq||TW_k{@@t-~#JfLp_$l8Gp-jPz-HfB=<>O5BhcYE(p z7*kUqa6)al7A7k!eUlG>apdWG*?|s%zWqtYoMJL5WhCZlKikltT8qH8(wiFmXfg}u z@0-i(-nx*!=UvA-$#q*~Ot2XbDr)I9#;{XbIzo>ym_9M2=!+di)x`49udz_?sVq+_ zO0K_zynoP*ZI4nnMGo>Sx0$L~y3+XuA1V0`6onWeG2xt&ZU+%!RG)KU!D9aW;R*of zJthJLt?4yChZ%-TB&8{w-^ey(20wyUYJO*}?!@2#dQMsW;njYy>@{9)QmD&e`^A;7 z@gON_bwL0xX=fc;`)HY6>f)dc{MxN_hdFiGLJcnLIOfWIWb28=Z3Z_roVvN5t^9p- zPQ>PBv3XjCSP;_Ca_!2QXi30S;>;(MVc2<}7L6LRe3)Imx0tumqeEL#SrAe3CBj7( zfFfojP@wo^J!ACtnpIm{_47bXwPH3De6gnJbtx0t zC330yw>KM6xx9|_>|k-tjIH~*$L3-}N9h3O@f6p|G?T*@CBN0W9gzyFVLKa);pFX8lKk4+}}x-jddt>9Ah3c zTS%aHmQ;b?1B3x*)~WdTRf_K`7k?{U_>4y3lL;D}W}1cD{ViB3aAto^YunJKYu#77 zw~gV~dZZ$99fd?gHW{!DspfC9Z-8c5{iHAC#HJ-$Obq$lmeoW z+Ds2lR)I|Um!2lGV|gZrll&jqi5DmG593nZH`GUD0=rh3M^_*X4%+x0E(ZfsHeRo5 z_yN=*-uMUTOvW4mMS+kKhPW!^c8L57ivcKBd`-?eSz^9&F_B;3Y4psW)wR4e z4zNe9sh%$54J?%3O7*Of!qQ8bqC1e$USy;sFfz&Q!Tr!FZ6WL)V9)1}>KCA48+Atn ztw;PpUwAs!z;&BF8L7dhjnLorkz?kt7DhDK7Ur&D!e&*lJO(EFFBL>*Ja3@RcMD^Ujs2VKo|3f(&J3b`c_4rV?#Wbctg z9E+;6ewc~skw8r|U@*e%6U8P_J#rF59a%xyH^NELO=$IS3@6fu;zzfR)_D|^JZBu) z(TX&@S`8ek4uWo^ylMM4ksmn}B>i>I1Y!8MX>X5ensxktlT>R+XS=kZrr9jRb4Fg=OD+Y~FNumj*(FzJzd_2h1fABTm%|JU zL=`@5t2S2Y(m}?T67{o7kb4jXJ5c=%D50zvbqrHs0VrDpbl3dXp=(xZyKzm~X+#rO z1%0aN?;X%!h4FVLw|I*B4UI5+R8VF4eUMNuR?-6bz$3H8BlEr245|^%W(b%*`G{aU zs@&l334;Cy;Se$Xq-b8IB?A3MNZBqHDZKylr|y7{u$=8n=*L3N>#c7x@=A%yhEQ$y z@Z2igs15C}ErAfAckDU1J^Si{ektQ4`$+CKs#e^w^3T_T?V9vAa7!CTyZhX;)tZxt zWPvR^6Y5)UYNwteYKYCi|AWbIZ{QUsL z*NzeO#`z|3E_gn?FMazoS;Va%?6Ee@;l6~hMET3?L3@UpH`iM*>1J~K9&3C@DybuV zBKd2KKXfZFP4icgfXhUf^%rT*pWPp;<=Ds!xd#px}cuE@u9)AoQtQZxc$-$i203a=>H_YfNErJ86|k{&ztU- zYk^%M$c5Axr%INjv#k!0ZUV)@n?3%)22*jNEG&Pu8%4qL4W)#iC#8U*_H>J1-Xd1(lEc7#KzNMX6UFmb zS&jy5kJ71jb3vz=)P;d0(XxX5}xxX`K~;MqA4{dXvhs$l|c-tJXUqAFX*c9mP`**U$Yh0bugujz2Ye zkc~uhSVej*0_4W@cm+Af!U0(bh( z$Rfs6MfG{?je9W@%(V6{5-bT+49$DkPsy8_lB^U_Ft*nVnVbV634kCbCXU(RJ~)}o)>o?abSe=6B>Bv({wIXtY049uTV98)uYCI62Ji-#YP zKw2MUg(bw<+Bv6KYO(e#sPIixn=lfpNwlSPKov`05!poL!rr71egmdW%wr#35*;Y2 zWGgb6!kU%eEXUvYpW&x@J$I-$DMm%jzjyu$oHo4|4z2b^RXYPrMW90}O-CBgy~{K6 zPZm~0@^3FxmO~>qE>7Ygd=##feTQY{PGT!{ZyRseO{eAv9J1q!k{j>@y1*+^71~{> z29ICEk2&Fk)D3-r<-05*C3B>z;)WS3o*{?~4PpUF>7}6Wy{IgvVyuicKzvm58YuH0 z)!r+u9q1T3D$C%@9wnkCM}C=Qnrv40q6zbwv@`{s4ZM5T97gpR%-w9!`Q#7-Ofjy-c7(3gbedPo(2_O05C{#GehjWf){OeQUKm z^9|GR{K9CHinC0YT}pm{w{!r4nWBmMBeKf0VBRZDV0@cv6;WYtf4sUSuM92w`J$q@ z){hbuL<2E$CPn(QJ!nTY$YsNYzc?AVWAJPj8oyC)p--?FgU5;`S<7(PI*rO*z5~on zik2;ii9_Bvk^93Rjcjp2yH~-h!pH*-k1-t4s@W$f(Gqw$C8}7Li*E@4K|eXlW=Yt8 zDfB<&b@Qp2Zu@mid$b@Jf9`=dBsP2TaTIw2cjwsJ?9+=U1y8|1%;g5^d;}zFXCw7( zZ-2>4hb?RQlWF19_N6}yUH&U-jMqt1=vlW_y`O8ygf4GC_0<+vl?!m2XjM*7nl2#^ zUvzO^I8_l!qr?C0k70gD``%nJ1wJsKIeN24H0RAgA}{Xi<|RhMMIPRzRN- zqsoM~i4(zp(>=6oc8={R&wuhFcT7 z1V0^dGu2H+XtebWOnNx$Qnf=2R%q4q1PNov33iVlBwhO7NgTV)+Cj+$^#cUi>l7C0 zT2di1)U5a-@_Zg!cis6*+ln5p1o``-9Vve)=NWn1jJ4wSw}s?>XRt}Rx3n=m?nvaX zaMo3Tu!QshU#Yodst99Rnpn?sSxT8IReqCdCTT1`H0~Nx$nt5ZK9NC3xBF3e3*=s_ z$1;W4fSi}k_`!>&6{i>TL|N$=0tEMP4*e{JX4w6NcawC&w5{Ipu1qOrjfLU8vv^Ti zT)}7@IFn4LkMP<)**QVq#8N<8p(Mw-3VwOpo-U;wt4Di~{E|{F#<1%2DfX|2rWeEX zw6EiUpHc)+wL1mX3IqByzx8EOY82F9vW}g2LHmdpfnt$Sr^ATSutqvB*3Qa;3CPc^ ze;V~AR6qI3?$Xf;GP2VyC|F_XE=>lCI`+;@+dSHrv@3T(Z|+oF+YY2W=GDhpURvC} zMV=B#ox&ui%}jb2yVX~6FJrj6A!mOIDN|$bLSvOSQvsT4wz^<$zM^29WeGy3{?GaQ{WvIp9X!l{}< zRz0}27~>MMHZDni>8A0pCVpF*WNPE8uvfrBB268D=dcOvhWq!q8-}WrXaV!yo3*W8 z@A;1z=A%EAp@-RjYbd2X@GZ6P^haPj`|JZ=I<@aavdC<%{?f>hK06W!rTZlPOxNN! zTbc)HvN0iM9w!tY#+ZxRX>7YZuHm_f>aLQV@(`WnYgtFHp6SxpZM?p%)@#)vV8I9M zG;8r{*bpIpz>5rBi%CJDXtn0i84&tjBZ94Oj65s<1_w;4+580V$Leesb08h2fUKL{ z<`VQNH0ZOpdKy&ksi-lj@|KLvxm?V)6gL;pQ!W{C(^J&@z#Z^PgiZVnkaIXFd83YIQ5E7pi3QnD~;#YetdP92lA zlQ@cEE}*7RujWr_re&DbP9qB{B;#}J3iTGtkTKOI>jbQvQ5!iqb^$K*ZV%&0ws6Kr4%8TDX(Q{?R zu>TL+6qJ^SYbt&MQ(PY4u-CJ*ruW-9CT3*w)Du#-Lsz=lv zGh_M*6Smq0GU*UdEKT{$I|qqVGQMMK@>me1e5=isboZTFE_WNhiIX0`Z;!(YtME&| zq*jl4oG`5#;w4qzFSb9yL(~)|;L8CHmX(Bw!H``;rASyDwn$Jyt*N+gotVKDx_;@E zpZ!aP=fqJ2qm#p4km7Z0#A-J9EQG9&xoIqx2B22u%)D_LiTf22dK#G;NS|Wle{8o^ zD}K25OkHofBWe@Yfi>-v(GbS1O`vJK_)`54MNa=>XaB+ceb;>;1c!fdMN)USDk3^2m9@IUGCCoILGvlmRu5 zw+p-K(0?+fxe>r$vTeXPW4p(%!>4rF1#F3abuP%V^3e)pPN$_C3Fa^VR@+e)y!uAM zNQL7P#EN@nByHNM?4dFKxRHu8awY;7aIm497mFAXi+hKw%L~|NaaeQfA1p$|Nvp5F_%CTaKW6-!rtmYeiQ^5xVotj{!fZFnKk{ zq3F>RRt`AyYTp(D4^Bk9GG;H7WK5>(E3Bk;512Y9=m6S4j*83W2k~&aO`({7xamRcv4`b34y8CoZ&KiypiOD9oR63J`^=I6>~PTDs<}Q24~EznlYW!YLU^ z8?Js(L+_I>T5%-U>&Kd4#8=MWDv?1HW;gyl%C&pMyZ)hr7ixQjuRCadTqL|xemtHk`tTkQ*xd<>W z4-@;(iU)5hX1ob%vqymU37HEWqQmS4v%sIiv4V-4(CRE38;I>>VGA^RK~EX#Rq>;^SB zvP8Ug`fe1-6!`DT6CVla>Uy|MX}7#1_2w7Vuql6Kfe5Wgbg<_gNP)?fDfZVRZQ0D} zlM%4`Q}4~vQQ4{6cN6qnd#`<$zxc+q7#ALQIA1o-O*P>2Ses>iNO|m8&tG0e zVa5!dF;oOy^GDFkIOepiO%W649kVn+OCG_9+vXMkVdFd zJ5hW2#!iHDecwkzCnJ9dYzV!U%I!UBqPS#pgzn0=7nZ%jP*i!@tR&Hk%Bpgvo&+~W zX4M0bJocN8`+ypD!p`cnbiXZgo!3mvUfDl`V7HBl+HGNX>gKrK zd-yCP!%}bw>uvkKCFrbNZL;=;3yz?elt2 zX?x(0kq%8bEt{KlyLv-P(`*V-5(ufCfSu22r0K#RwYs`(mG&Q^dKVRo3^9LdCr=;E z?F7^x0WBE1(as3{%qEB(paXsz2&jJ57R?>AH9$JzHBO`aOgn~*VR3_o2c}`_BdA0O z6McV&DKNT$A5i+dWlNUI?3eISS3W@eT}jvDhNm=MXGLO-W=D z4s|5J$+8@|50i^9q(atINu&WC88(^(`*EjExt=H15_Ks+NHplI=?D%+o9O@sX_SYESxi(XSjn7*byNFL=j$akpm|#G;rvJoab09?{305B$?~mm-VSjg>8ZP_b?5J|XQ7c5ON3(XxpX-7(*~6NAso-Et8&qYV1L zhSOidFxFh}tLhyX#d@`G&FnJYOX_#vSe&cLr1cR?Z(3VKRl$;KmsbP`+lgWv)J(W& zkL};@OBLAF$T>qyn^VVa2pf1!KD!ELe zy;g~CnXBdH&@JeCLZ|7|(U97iQfj6s&hZ|o_b0j8p15!>9ob>~pmMo_db^9S-Ga$x zo)D;%u7y_?$}INNq+6QNIZc{NUo=vyipvCaBBQ~+3b4ivJVq0lRQBmIh*^rHR2D(* z&6LPVml<@}GEZz#>PNJ=eRIy8ErCJv2iUK1k_G8rud9ooThdYM;ykUlNl?+hJmzHE z2cbL3_`CI4_I6FiG6okJQ_Nu7PatPr-7sx>q}U`M`bSJjG=2o?rDbq=oM|;>RWvsw zIIGrAOD}7Tanov;l3!FBWhGsfSdd@*n?+uS&HKLwkJ}?3RW<93w;R1<`@tn$CkwnQ zW8;?J3t`uA**5rgTDMLWZiWX?GHwC~u&V51)Qj8APi*!!onCy8w$qY-E+R{fKwuBt zHnFDoO>u1V5U&Uds$AAS+%1EhbL!*TTQo`anD-I?=$rHyb{=C>T0&69c> zu+Yx(1}y7kOM3j)CA55HmU7O+v=N{c#?%g~QPTD-K=7+$_$&y4>8^!->6tmgNJ=8YU9 zm@ti-=ALw~y0wwLxce{sXl4j!u7E7M{EUQsqWzaVW=t!Q zO)=t`V)E@KBnW`0hadde=~31g`=P6fkYae6;jzpl^DU@WG)M46{}f$^_XGW=^ijqF zVWSfwTevak@FH@m>bZR>lE80*8JA%){O&!|p!hVcr4kW&7P{1!+LUrUYW|LGMUdBT z{6-WKV0Scl9f%rcV#|5~PJB}uZ4b_?uKEk#p1IIm8e7z?tqh?Jo~|k;s0`{zixppi;?t`N_qUrA>XJ!wvx~QUI{}5kS~cqu+*8a7pqCAv&fa!$C5M+zg~-B<;Y3mizh0xx$wUF5`)6DrtjFYF2KlJz zz$8yY2deS;Su#+jN;zHbDpA&Z?6*QP+Plww!pME!eDhQs@z+4jx=ZpPnV%3}@fK!< zHa^6;5X?`Dtp6XN&oIHfbRS;i~{R0VIZoOE_W&xWJ~p^%zFNjGU&X>^)&VgiaWMZNaFdgoah5SIyykH8TY za1zS5GRzPD0GZUY-c3{t2aeW1d0ccj*&hGMQJi&}_ezRH`}6a1|JMgb=Dppna{t%* z{8?Z@U}i6Sy@LSWb2oTv%Va#hI}zbZ$>KI%DOO3=y5rNrWP-MCDuaqk3eCq!0u2vH zOb}N?Ms+rIV9$5v8qBA&upc$yQa_N^Z`ZpI?@yIY4TV)5!9{znDgu+**?PO7u9!K=|@9Z9KNLbE@$Nk&?kP z4nuD8O4MGtqqQ`p%45i}VQ(u^JNBMC1IQt+z8fk{pDKvK9ca`EUt3I+*z^zOr+|(@ z7*K$6{vJWW@b6GD+mwe-Z@^=#&z<4A|Y z??qIr>rwk2q>|4WPM#AW=PA-z=J#cTBYy$#t6_AZ&6d(D<*Q!C_wDUi&-O%lCtAcC zetj$1xf_J?9o{at2mQo-;?H9xKx+d!-CCq0^arU=@@RbuI{)9wJ);?1X_qjjdQ*5T zezVeB83Y_lV&~@0nhjVU9#p4hIN0q7B|?7mSIU9P78ua$*Qdz6m5v{8d`_*lBTYHc zmbMtn9fr1x`~&Vel!>mk`kgaa{T(g0ThkRQ?oZq`IwwmPogP_gH4o>`I$i%8C?Hj% z(}BEdefrELqMJi{>8wj!GedCw4rHEcLa^RC4(qIpTTVfreis(ja77z#pFm#5q@?91 zW9GMdDjLBOPo6i9xGv7Lagz+R>!G-dGZrt5P8m>=Q&LVGUp~-MQEL%ukausXsOY$9 zZ*TIxiMa4t9&%Q@GFH=fEjV3QS-qmUbir_OHGw*(xxAi5iEjTpr~P$GelM6J(7u-Y z&-&oOE}bC6zf!!umvDV6u)G&a5%Bnxz1kcy z-l{$~_?MuHI=`Z}BWZBsKWX@R3U2#eP-27l; zbkb;h5$rmynHcXPoDoV%>X;bcvVzc7y=I_w{f~uD>*k`-_K*Azt6sj2=&4?|QR}Fy zXxdP*-aPKUwzzVcZ2xEdC$fuH$DXd5Wy^ot{~rteKUcqnqekcdWoy>}o&PXQVNqO$Bz1qNiY zv@jVaaRDwg$6y7^D~y9fLL(WaGnFH84^g|${35hYD8!`2J@MK;mPJA3&a7krQ$t(@ zQ^0mHxRXYgjkkpqHH1D*Mqj<4)K4bBkyra0?>6{}qX&;F6{&jcpLmPhbXNnX4Lz&v zF;ih8j?URrmg0S`bC^Wa@s1)4TSyFbE9x8yHeyU#JP%QJ30kaL6d1IRs z<6tXmKo2=ye`2aor3DbmOQ7t^HfH2dG=Jj7mW>B`7qscfp5BuYB4@sl+V}ehd_-uP zs-n0kauYRA^yo zc1PzE8?`K~N!o6|ET0xWmm!=$7qflch%STxKF3G~dcZ{UtYgy;SLc{8XfcPmV8-9n&ZHwk)ny?mOmTIUvtAWq16J$x=ecM^l zGU0A6ftu?j(4!OTUx#h|_d?l9#f*JE5g6Cdod=sR?w>&?Tqfp(8;(maiEdx;v=mQu zz7h5O*AuYjjc6!15{10zlrC1(POEJYnhTU<8O2-H4GL?QP$$ zEgMKH9547K7o6H7XWVie-x8}%x?0m^AbsI4g$8#;1lp%BMS$?e{K_!=3fY$k*`5LC z8di8p=fy>Di(lc^!>520d$d@IUrd>Qj#cZ{2W{=HtvM&ynTOx+w$HRq?+I6IYizD- zcC>Z9t+s{XFJM=5HRbrm@vt3f#aVxEJ zwz)KJxl_noyZ3u}{$YC)_LaQ(Fm-bq=A*gwTw!w`=5yidGZ<-$xQuNRaB%SH7~mRk zLf_VhU+8tG{%$OBKsL+>e^53ip~~{Y>GtVOhS`l@%dlr9I&x!wXUg&+DQ-WV|4!mR zv?bi!n_R=lvEY`u)Ws)Si@{sV4v+gs5z-prM>u;}M`Z`?u z=+-~&VyUPRD@l01Qo`Gyt2q*)%#q?OqW%4Cm>5k{y^fw(C~JYma&=go^T}$b`w#iy z!v0>#wxzVEX&eDemt7l218tu+RqYcknndKhnoFw?0!Z6Gu1#gziFV0x!efs7OB{Ue zyuOF~LbSpj2Hpn(@h7{Cyx)+2PvR*3u%NCRj+nc_M@}kT+T3jxp$KV-9&(Qnrakq@ z>yjfI))*VQcxJgE4fian?o2wyL6EfcIyR`%lT=nx_JooPFnGNKjWp1;sKOoc?IJJ; zP95hed?9W;ki748(|cQPe))W#RP=nlj3MRp@9Z9PgdOktj}lJ=kbw^?Ie-`V$JQUm zGzw5vN^_;!l~vn*js%oER;h&fqiAFYRGpJe1NFA-C|JaUd+knps`4WuvX#W{U zc8Z_C&V8Llsi!mtOUBuqc88DF(z(EH@emQi7U6KgbVU>}RfgC#%Oh48!52_c94@4o z$hWacM9GB({EkOQOR#L}V8p<3`{~gQd`SN`(y2r=y|XqvD+7UhUJnZ{Yc{R$;j+~{ zN9kk8ahoWdlm6>(Zhzmj&wHeihPL*AnqAjiTc(-iko2u@{uu_8$S%6j#B1a;}X}r;atH* zR9qHHJy7@yJu1%C`c1~vpLY`Wg^0$JJ}T!vSF zq1b1`nn-&3JeQNkO~FJxJ_IK6`1I$f)5$I_#^A!}+M9zLLi;F-C;$d|Rma|jom#y6@?0)(apJLlDrNQTtU98!RZYP|a2rA97(=f6IvJG7DHc1d$ z4RRXzn4Bs}8? z>U?)O#fT)#w_8b}rNi`iW)ujDYsDrxCK({fEX)EO7)!lgr?Ti_qQCPTNMmn$nE*K~ zR&@xE%KRt#5V@a{pB?J#1g{SwA~2g*Qkg{z2;`b36l|Q@^}@fBB0-2L`!?NCGSiI} z&i>AShwWZ9?6dhNZ&{XFtj&F7xxU2V!kt@}aMA6sjL+^YJKS!j0|LG~l^IzC_ zJRleKIO+N|S2GeGVEfY2u{UWBD!*+>IE}#cpHsk9L|6NT!qW{y+n3B@`6To3FhrwTN0IulD7DH(fJ3&e%DL#8=Fvh!^g6M9wUTU3-aLZ7 z2SwSt$Vd%oGw~F^{eUsfDBSqg$&+2Egc=(x&^3c;0?Xk>zm<%YPlddHWFfCXON`NTY{MQ=mf#Q~#xEBYCVNYU|{Pb=$FkB*Fw@{M6^ zT2ZXZTC~@3S%YRMRjc`A1~|UwnhP3zVvU0fG?``{dih_D%l%9`;{NJsNa3X<*=R$?#8c;6pq-$I0-1RkA+CEagiK*1j2kt|Fpzgano#k;*L7L}i>!3kZXGd^Eya%@%FRpP>0ZZ{CP_0o5;fQ`D?dtPlpbRHkEqlnc1 zq?j%k3`;!O6=8%obPa;fMK8nxEg~libv{!GhkA5^0qUYAVPomr1q+^Gb>P`A<(3Ca zLnNbtwMmbmH5Cob(L+>sU@W#As_zEn2EdFzO=+?u-Ajao82wCh;z0&Nv=sTl=FXeG z6XwP?DOa+SOZ3w|8>Zy0IE>qg4X`K6bB&@!L<%ZZsO7$eq$hFH1AdZ`1G3Z9D^ZN~ z#4+xzYWc$Q@BCU}*45h@W!)YQ^Vv5)YkLe_6=` zs-`;dI1TR~&E^5QorZ-NloCJmB$~O6c_IEI{Vx!9T8u>EFwtg&Y`e(sO%gqgD*B))p(YmR+Oba51S{ z1^U1sKra!?oov$Tk}+rQGqxL9h?f+Xs1_!UyN=z1Z_)$Ltp`VQ+^`Ne^Y!&`^)=Ot zbB1Mo@P-HO@w@Qm661i%C+g;6sGS1gCf`>7#De0gnW^Pgcoy4xcEf7hM>vr~e}mJ` zYX3{(T<7GIeZsr?LfOcTR84d0#Ncc9V<0PeK;dm>@sH6gySLmxTwI4YJ!S*4Ljr|4O|i@QpM)K7K_;oOe7R zJnx81-xIJ>1pGLE1(jP$5|9~rW6IJK2ZZ_`9KB~JzY&4_yfbymqGs;%+ajN$FOq9+ zaV4gK1*QBoRr*8wVS&nCdX`P@SfQehZ2wN5+-$$>$6d3U71c^q%pyNAUFrDM(vVgv zXm)cOs;FVnCVh{(=Td9A%?G_)mk?hI^4O+9KDK^du?uX=J2~rH8T!KY=vMMr;Rsf^ zszJIYA)+P;!n!M&e{CN4R|xixBy-}wv2}juM=g^?H4D$;)%KdQ@|PmAmy$D_0-hG# z=w|ut1n;nORUgqRmPL6~*LhSld6b*=Q=og_E(gNvfzadAnUr^}*;R{8UAykhefM6^ z5kHKT6*9a2y}DaN{&MTrhtUE~KhzB_&7BvgAvPksG{M>Sil>c@g1H2dQtvAGh8eHd z{T(?wvcm6pXRCb%2vG>B`)8?*ZA@pt5S!~O#BrsaX=`-b#C5y-(0}$T@>fW0#OOE3 zZt!!`G_-4PthR3ensn8g8g)dzzfKL3NpN|-?_Zd4N>tRad1o!uC~iL{HoAWIwy;1| zfl#zra?2I1B2tz8NpU4+ydiXZbc{>UO+LjU3x3ty4ca77)Uu1}FX&VtXL~J}hGDKf zpfQFvs=-duU8s1C4LT6GP>aXP0g)6 zG5Wc=#Qo9;K~TtCrC}6kJ=d|)bpH`OMViEW2^UO9`KTG>0%C>M0An5<3KV^})UyT< z`Tmb0+%R#73PohrVVeC!hIphYpGrzJnyM?`AZdzl3u0yV7$q}e<@wB8MZSjmPtLqC zGG%x(IkNpC3+mgA@QPSeyDw?fBWwzf|v9()NX&OM;wi>qYWa-g&$Sieb(i8|9 z=9#gX(_wHnlk`3VQu|+$10c;Ugw@wX{Uyj0k@!U9d5EV|5Y1^{3!v?Kz=wi8L`2)g zC}}Z4%=-JDzy!P=oZjyk6gj0l3p4c#}5EqJJ09<7P?D!o5xmH*`6LB!LYOBVtn^wU#LzDD8GuRs&uG5lzyFG!j zMDqBP8r{!lHC^wfkwBEau9^dsO-!&;`moJGn_n^M73(1~lH$6s>2I`Ti83QNL|7`62DGgas|GN-50KA1FS1? z&={7UtS6#hD@@GWA-jAc3W%HF(A^b9nKFk4AJVqgEHQlf7E^>>>#FN76!6(wDaQk2 zy7v^fu*F%_xLVISpc%9eM47ufL)V}9dB~-?h)87{`+~+(+*J4D{wb_Xw^V1=<^*vY zQMKdbn|9AYg8IUAo-XHi#nnkUenj1#d*cf3rel$%vSCnVD%gkyF`-Zlqz!zIcRBbA z+YO;*4_OrriSOuc6X<>0UV-TQ_Z{uxbNjk`ZY&h|;cUC6*ni#2oAP$|bnvKL83(amK*u=ibUxj$iTAn=^S4+RlFfq4E2w2 ze%`~t;{2PhKJw=l@xnju3vKMb`HUDDth9Bp;%gZ!;eS!E(%K)rT;t1Vo2nRCnncz) z@^}8-|7T3L!-)T|{&~2>mk9ve+x|1>Dy{Orz|n{6|8e?k6EB7%IqMmtpD|;J6uEKJrwIJ1$8Acgw%Wg#oVz%qzFtLR`#V2bw7qG*f5_Z}{JJ4tNz%aiI>fi~r8 zO;jy(0J(zPCXa5`kLfC}z=ZsNJ+6mZlg~JX`m_jal(*I^PX%YQ^vTP*5hNJ^{fdb3 zXG@u_Jt@+_ieo)?<)y_n+-iPJ=UqrK(;(69zeyJ0x(0$0iH@`Mz1t)UjP5##5sT^# zYow&kihzmnA7~?ScI^3v6$Du)se8<4A4}(IC_X2iFb0w=KOed#q*G?$2iMiy1bah1 z+@E=AVsK2D?F77>{Nd)!=EwR<+T7q9Md_ayGB{Ev5lJ; zrbnU|qvVjdW@Lgx0f1afG=)(O_5~-paEEO^8D&`wBQk9BvRVipb1+QU1?a>EY$gbO z=}A`i&HG0=CCRF-u-|R?UsMF1jriB4^>t~2d&m+KfU@L*-7~$h@NmYYO3&7#vhG?) z*o#%oE~HWF5v1I8LzS&Vw+QoMScdw6p(va{5eR$r!fj%Xg@80mw^O5ja(z33I@~`T zGwmV?IGFhG=j{Z<30M}(B1Y1KKV^IG4jmhNg$hxRI1ycHLR%aDin>7m9U5OSF7^T# zg@$i_R}Tlal7MPe!RHS9)i%SYM`tgw>(|00oE$2%5#Y#ahizEHL*bplT3kR|oI%pm zDvH)}ocjLy{+b~CF!kl-`}|V*b`|)0|9AubI^vuCd3P!CvX^3N`mcQR{r0)?(xN6X z+d$Z3YW$_g|LOM5*Y=*LPikuF`*;7ew>r7m>-KkU5=eOEnpjLuZ*j1r4vTSqW;FT@ zq)1@lHYtK}$-OJBoPL{;Yg~KaqE@0hr}Q#!=QUNhS;y#8>3OX5pkq0@RMjMEkj2cJ z-lcvh1Q>Qp_$u1~!e z+43vCDV~)`72$wfr4M6PDon93;s0Rl9fM>Gx^>;QZ5yj?+qP}nwryjzZQC|h+qUh~ z``hQ7yW`$JHzFgWW{jL8D=I2tR=wjHZ=#Nh?bc*1G!1E_FO88dShUy#;AJ1I1+GhB z!YYwHbtz%sHKEHK>2vgq;oV8&vB_nQvR}-)1h&~~8-Ly;K%pPH{+SXhs>q z(|P^!H8OSVU{M}=M&@s4WVtJgEo6f7ZuCkNaZU%f7>=SI05##U?*0%?tJr2;E$D2T zzHY~-hZ?10a?7Ac02yXG3M z0L{MiuB<5q8XPk~(kJQMs(Rocgu0?V_Wd0O4kvZ&C56@ZP0g(Dl$Or+$Kj>5(@(q0 z$IoXG2XqLPfm4=Xn{?Z*Ol`ii7hD^z?-rV{kNPMsMs49%qWc7d4cW$CK|4GRt znfQa3&}oZ<>m}r?V?T{-zIE&sk9Q;#_e;n&%CX09w#-0Aw_5qzxflHQCj3q! zPz?c_-%HO9A6--yhf$lFUWbSdi7}z8eOY~NtOxc4g6=NPgf*zWfy7M`0rL)g#U?YW z)DP_>*A^e@J|}_;g;SCQbZ4!6<4#Sll!9v>Fz1WJMj*R!Lfb9=6acq!>A&VqtJ4Z7 zh`#09*7^M-mix?E&%!67B|w(T@;97UvYPFyaN1$h=0lEqt6VMT1rPPR-uMYgL1pDn zTq3fq&#Ojv${Y+8#6ldsNdFivhA#@+c(m5iAcS2i;>_V>C*FR_w&3j{4_(GO8|Z7S zyd;L2>J3PHTHLOh@n)TXXv%sH^#Kv(R`ZRHl57ccg9}B3w(p6W?iwOGqCRDIV1uGQ z3qv`mBQ?IbWluPabMGYrzFkuk- zEX9$$&RVfue0t&A6Gw5zZcOG$>3rI%X!Ph~cWDIG+*4Gs;Bm!OpW^8xFn6|k+;_d2 zodUH}<+v{x+djk+Ob{>N8p9b42FKjvGKztAVQmc6b)jft4Kidlv5Ab<*fniA1@^JO z!wk0LH$OxX>>!;#lXz=Be+DiNRP^+nSpQR{X<702$}24R`)EQi-kO{uM}|HnAf4x? zHb*SEDSF#ia#NK?GvHlgWxS!4e*};=L>3)uE`Z*cd;%S`V2Ke?eun-^-WhZYwE5Rq zvSB&3&+ci%vnZB2k0W`HaC|@70>Qx25zty--G(^l70q`iZj=3_UJqnz)2tOt%Y*5r z`lDGBn10jS)=FMfoF3XLZ}{&!mOsFqN3}pxgnA;$o`(a0^i($>%sF1xzTP?f!Jpmq z>AH}Vb~o+J>K{FO9iYLiULhO1d3EfaAe%hk)k|!BWMlC@{jSOYJzns1*yQ!)3tMcZ&?wh&lzq{vr5{NeC{Zl7WKDTyjdg6AHb!x-b}ol*c5Zo zyg%5J^2Ng%%M*pI(xon?11IL{|Gv&OIE_3v^5T4TJxxo7mwYAw4z%~B@=FW<%v?tv zo8-2d4D0pwc9y-s7{h3ByMXMFZOqP>2H-7Wx_jBLly_SRlH1&HW! zhK-S}|1tCu<&+6e4y&>-?-ZtBFh~a(n7ErLqq^nH;ZOAbIG5OX6hzl3LC4#OUk6AA zW^9PF4B59L%Ttl6-zz(S_RSI0akv$3_ctDv(Ze`;l!*ItnU7Zw`Q38F_jg$4JM)bj znWuZ-ppe#6%;LMsuZ>XB3dB@LkhnbcRAeoX-3W)1rF2HMlr@pJzu)NNUs_=j*7MP= zfD}T%m(!;(yPvE1y)SFy3n8_X?hrjTML!>{8eE?ql*pp3U5@a3?y^sY(-e(UHd(KnwfMg}PuyDH8CL<1k&v}hg z%BUqwrn8nsk(wMy29=2o6^6x3EdCg;OZIDJ@3m5+I$J0m`|W`aeXu5M}kJ%Sk_5 zV5xnt21I*5OK}itfU)?c^wBn_y`f23L@_dq1S}@u%Xx;E9m4Q9x)wxZqHP5MA+Uh$ zoJFU}!;V-7&CVz5(|n|g`ZEb+lT7XhhgFhSJkpd%&Y|mJ*3Tyojv%F1(e&~?6jV}d z5-N;e(s|LQe=&te)scGVHk;<+4vr*x99LAQO$KfJEwZ6H_IbOn+=~&JzlM5yKe#%Y zvD_6f`k)O&*ZQf}#ma?eDOYRnfGoI?^SyB{KV=lWN+!n z76{kwH|e}&c45*(UA)hcxtgEFQpZhUNufRAM4)6ekRl#GFxK7j)elG<|HM9Z4uWaK zAgMo)ATVb7s%pzryqlh3u1r(+ELQFqG1Sv&1LkLT(=4UGnc_(hqzXGDIMKlF*5VoX2N1Q9(wwVQ!#{`a zG@+@s%*iOiEy$P=KlsLUlh;ab5s*R<^lLSuL3Uit@DA`(g@SuQ?Ip#??R7lSp&CgL|NwYgSkbjyL{|_5LD;-uRCYT=$!|qNwd)Tu6#* z?(3-9!^`Y{NPIaM5X>{%)GUuf0!S+aKONNIvfs^Sm1`6oq{ zBuYc%!cH7ZG#QtImKMrLq(yEY^ugncjJk1Z5Mm{;cNb>V{-@Zd$sOuRjlsjeg=k*# zCb)@+y`z7<8{Nntji%*(`s?8Ws=^ETf8{`tr^^o+1zBQ*64u;#*Fq}c3hY@MsLvB$zp`IeI+ zXuxrDG%o4eSD)1m_)N(Y%kdqAp@Nn%>ob8Gp`Y3LDG}lw^8*qWT^U9w-DQC60kdjBLmN-2mvqu1tFl3nJ+3 z2lrV)=CQVX_LoOSqTJXgL)ksrPIm6=Kl^HecMEt+VJ^5V6&t{nOg7Ba_Kl!79UP4N zt+8CLP4^_rGN_#EUFkduQgi|d3r1I&>7)LQPz)A6RI2rUS94T_LqBe)A^auw3KbZL zDG&+3G1E{wFmfN-W~s4u_9rejou$3s+4-49DOOE1F3gQ+eA4`Z+&m^vEoO_$-|j`= z{iSJ=T8$->Onu|lDVMihRr{kQA>!~%xh+y>h?-$R#E0G*#AHnI0;;t+qOd9$M@8m0 zZ2%N!YTk;Q$Rnlv;QJK|AfrB;|E->fY82& zlRe1-Xkp`A&8MJ-30GI+KFlOlyyiW8?}kU935!!c(R|>_ADM(&;9<gbLX9Hy36v_HABmo^yAUNMmP0`~#|dgp-~b4u@kw zT$P<0&r0fUuxm^VM0dwg5% zDCzfg;cYomoY4b2H|(EIC}(MAhrcikM7KdWoeD>v{>zR>Fe}97_9_>Iyd2DycU%?f z@v|`1tJ3Fh=qd~_yhgE!wOx5rwDezfsuauN!6EI``1$zy!hf?$7ogoEfG03UV`XA313nxwma@y| zvZhCa5d*59X7`L?Xyf(AHK_Y-2@s~?+4S0uo5N%GLV1r2ubVVFj~Y7NvNWbaU$FOK z7S>NiY9yQP6ZkT{x?xN>ylF%FB)ndg z>h`3hcYla9(>VPhMCK-qKaMn2H}<#gWRmg^Efx%xDc_}DIfQi>e>a8(D!)SdRhdh7 zbi6@8ZZoE@%ubXfA&9fEo#r;3=71fY7B`?@Dzhr_ymTuTly!V18x$YD`zIMnJ(&7t zCo<8PE^MMLSPwR}D4$IvJ7Hw|1${e5JZRHMvNRCLWsBtakp;c4VArNtAkRo-jFYIw z>-7->ryA)7M^_%{Svb*R;Dr|p=bjOmF>JUpa58q5kRH3b+o6?m1J9av zv0uJgJc}njp)v1pDNcl=E3tx7q(7dIZG|>sQ)2))hZgeIwzoGigNNViS=crO*Wt#5 z=lsWq8xx^#_?N6;cu5wqJ9;2JXMf2i+Ykh<-lUFjH11UwU16} zM$20vgCrH(?QFR$`j8Pge|B&Fp_CSt7MSg(qVX1G!e2DEiSh+i2KG4_@OVfms8X|m zSdu^*QWC8kz~^5f-CpuL?O8HF=$p{NyUCoje9u&$yUZ77U`XezzA2)Txpbh@ILzX2 zX)E+BlVUa4zM(}ghE}PSOvb|!yHzP#p7=`xa?3Up;|0r0Q>+eQ9nU? z3~)8{46=CBKSxa4AGiWOO?u=_NaONYw;tCb$Ia}rX!&^d!qh^%l_E)|t|-9}UqN&= zD7EqtBI`|2ro4AbvOL^fEJS%UYn!?F$R8PFwoNUEM|viGKz{9nMh-8WKLhW8%&^Q5+mgkARZt}B z9bAHekOPtZAv7|_cB1t;^KL7+&o>Ew2kq{+_@qPQMLEi(_8h@j*Ttu>2GFB)%PgI( zBuVtnJD(s!K;$zeEE#&_F1_Zif6dT_A0+6aOXZUtFY~=Di~zIN#4pPP$Ai_9`tXmL zH5+p>ybuweDy9Z=f1qS<$Rd)J(<7~lzDuh*4O3dfBTLr!Nwsdb0mrhx2_>t+hOL9! zC?pA5HKE@G!z3hu$fF+=tO3h6#wk%pbFFuvFsiaOol za2bx}ROQ$FHWlHOhqk^?8gZJ9I@nCr3H<#J)8Bc0Fa{v`T@gRvX5&nDifZQ~Js822 z>&k+zx^!bn&^YID1D_OBY!!EYDlMswFj^I1>gzgHSi-H5jHG=;g;$STi9D){+S|vjr=3X_`tx&?s1UDdR~1yk8<++ zVRU+Pfdxhj20wyEJ?(AQi>B1B2x00E@bv=qR1+g;gq8tuHGoh9Qu#@~~?v)}8SBa1}287+b5cz2N`2e(b#;_Z-;a z`dz$gjqaqb0zD6Jbg_0%ZOYS`A!6q_&h=$ev3|i>A+{q32ac7K-dQTbPJezUubTGp z$`jH)qh>UM-hY#cf3{g88xhFRvQdks{j~Xdxj6ZKUB90n=n6O{pc1SP9vIpVhnrS@ zG$I&FdQdcG)Xp`1%F}=eixfDR^*}8DZ`|BLnqbc0<_dRv8}W;#{UmE=j{#$vwe_na zi@RXwJ!c1tw%t$mR`#}wl&y`472||e)0J7)aPFBtH`Cjg^np^WWEtpQPy_|rMy+#f zH)i`_f7bXpA-d(yxik1TaRj|50n@=tuX7AMoN~xnz?7@}y;vOn5He4G5E_l8mPmyR z;;@Y!&kOp0QRZdKpk?0Dz?d=O1=_|@0alLympQ3qg*>u&c=h5kDy~(h@utethE(_) zQF0G{f2${OxSgl}FJ!JtVM~A<%TW*TJs8+@&UvDk9}MARV#f6NhaJ0Bqi;+()4RgX z^WNPypvQ-&zUOXs(uuz>=LhK9X$-k92i$>6vZpck=+^OJvut=hvxvP9Q(HbhWk{1& zR-3x$A-IVO{F&c0tn8AdH0rp`T&-wQw9m#3Tcok!{GE||jJ!*3r|p#ef70d2pt9os zg_oPUi!;7`{mPu)Ppv<^Z+l9w*qfjF6%hF0IE=5lm~!K=(O|rqcV9#j?&HZ{fJlk3 zWZD%MT0ix>XP@LAWnNh5k6c8?%AP^R8`fHPsNPAC;|@^|)q1|AzXTrKeLD~C=J4I))15$ZL_*eFY7~r>q1q}=Y?;2eT#fxecCMNM3axGJ{J+bC-J=!N$EbIY zdgT*w-kKMM#{2xdFv&*lj-Lrb2&`b>$7uH=Ha1Zf>Z)T;iGOPBbE52TJpl3C_yclmn-P+BEdR!z!Q}^pxggn zh}=3i`LF19QfT8U*%9RbLyaS#xtNF6na7U*yExy&My>mQ1LMlw=mOTVY+1n86{Y*o z(#w%RfbaMPBHxyQOnA~=q000oY8~Yo(`1#vU@BE>?`c7za3hpO3yeNhlJa%$r}W~GfmCu9mMOC)6c{KQcH35o3NrqhkMNeQRWFrvAPAvu(H&s%cQ#D{prSU z`|nVgKT-`Mdl+a`7S+(XmD>jpl*E7t6r%X%5pICMD2^ffn0?{#FTXHubNus8pEptn z0~I32?g!vO4SdhHo`iv%P4qL{Q~KD07sCA`@`UW|(8MQ@!<|4P8eWcl@e6J%N6b%} zFR%;;lB4*BM^al|u*8X5)Jc(F6`w7u95ogC2lkN7!yR(hg>YU3TS|{ZLyqh4t3JwW z@5sabbqbM{n~|GAv;KIB49F8ETOuKuh87=`OGH+4A@(H6q|4Gapts68%?~3HliW?# zT*B{Jf~QbeSJp2j=gWOWP#DMbJiHX{zlWLQ5i5??L)^KGP76mgbi#kL_R7>E(xptY z0k2FCM%mk0LRP8aXT{G~b4rn9o4rF3s(3cv~_}LM*7uV~*q^sW&P2Y1n=W z;&miG01Whlw#MD3` zbQQhJwk6f|nskEFfo%cfs&4SXqZBo$ZsO$zP2sB6^Z#%-RYV$o0Pc`i%EtHDULNr7<}B zI5^$Kb>7et4o>N3DYe>cLJgfK3PEqW%~k5oDP~5~s-Cxiv8l~+`xXm%weo&~a}L{F zNAS+w^rT2+ASyPglHNRSJgm02bw%cbKp(L4-8#=3(j&c+jm$agfsc}mot4z+t4myEVL_`Y9uu7N!GEqKYpVMzs3sV%TAzXfPS;)x@Y-?Tq!z`ZGe`}+GV z5ilSekeN;>_v|)%jyhmo0fA$&vsG;~4KOddRndrA2+T|Db#vug-8a_pq2ZW?X?UH-!=>) znpgY#*zC^oU+LN;n;Kpc7(Y#>CpzMIotqq3#oYt?19~R1%Lh4bpyR3H+?5Mwa4c1T zbU*}CgD{e!7pst5%J@fvs4g2pyiOvwEU_d>E4;;(s?2V@z7J9W6fyZCtJ#?HC=Ld8 zJe~2(mB19jRtgZAzgSL^k00OaO~?i>B3SKmlvbejD0we3)P^}_h9^I`_(ZLW{{fbT z%0-hgU0jlpn+Z^uc%mS2 zh7A69OvFABn9~UOC;kBmcMg*R=g5DCy{&iUH4gqX+NQJjccxLe1I(${E#}K(f;)}= zU7VovyYe?b)XM9Ba(o^yE$TsX5}zHwjoIr=I(RZbyrwo>ew?KSibFn$hIu3jw)zMQ z$HiZfMIS{3TmBVIf2IgFe?iRM@-lA`3ob0pyE-x3jErN*7}eTby{a;`B#ZK1Y3pd! zzHOj@<)9E+{@gZNNmg4!TnKf*e-OxRT&H)NfPUc2tC)`7#!gg|No7baEt}3iIh;(; zOR!VVp|H!`*B9MeE#8z^>g&V` zP!R02U8KkcV2zhPklj$@|KLvjz6S3+YZ|;%Dsw_^_cP*p;ChkQS7^f1TY^l9uFqd;1bCza3IjF>0+T77phh{n0Vq{0(CNygFr$$ z0eDlmTO#Q}z&m}nvfdOK%8SJM%0?>Os-SUs;#9T!j2dUHB%T%Uw< zICtdASy0beTo+>23)EFe+ZxvTu>_F~t3Btl@Txb8P9vhHaU3pYQayu@Jp&H|!{0es znbH1BAmF4}zq2P_#p%q$Ji#A-!5jSnvfm0PMw$)?LJJ!As^%^M?qWkidkZr4BpG|^ z?DpJVpp|TtT2d=QFY_(~JD%+&Dh%}7NEt9=Uc%|yjsV^o%Al&BEmZ{89ZEj+0 zwhg8iT;Nq|R4elt= zlKu|dVUT$wL>>%XQQDvph1?XMGBtyvAy8pnxs8^1{-mLNt7E^^N#PIO9Vi{xHALsI zSn7`Pg}O3ztBWvRn;|g9gGM4#{Fs;O3U})~=BI|rq*t!d*rZNn!q(kK(Dr!l5d=kO zpbW(yxags100?fK_dw&p3S0l8rORdW7Es>_w?{|+LaHsl@Dawy zOTWld7Bal^BunZtuARzRTk}o1vqDb;=`xBK7kW{Cy1vRdzwbnw@P_?V6QFH{BA4_>8C^wy6xdQLHtB430)1}!`9BN8JX{gCbu_zgOp%P$hY z1HY{@4v{lSQcw4#E4+8MEEI9-%|c>i^Stvi$h0}UC}WqxvVD)N`5g*#To8W2bvyCgdihA0qOmJvd#>7hjMQEiBy6_LGqNcW4CL&`nb&m* z+7>>(QWXKb8J{X6ZVYtVG@^R-fu9J&39=+--s@rHk{UozKxeSCU@rh$H(o(t#3Dix zS#_m_<88B_c}cdNUqyob&%a;j^6O@&y8|y>g`aIV>!In^ zv}_-LD_0Ny>Nb5b+i0*x_ZH>DJFN+8yWS(YAgL!O-!BE5nUL=6ozsP-kb|#ioTsH% zD07og<~p9tO+1OKXdHLp5YED}KDSXQGyBU)Jjv^GmQco4iRvt#tW~7scg;pMud8^H zmKbdqbNlbDHP(cV|GBZc=;rsbaC!MR8Nf%3XYYT<|4(Y@_Wl3!@^_an%g3Yt>}y+D zd;UMRE_hu3@3+&MNv6g${4-o*(%s)KJbuT&Itr*w4OABE7K0Dm(9qZ%rz` z6sekPPq(ouiN82D?blrTFN{8H{zgYiu3ekQm87WxY9=wTjjcxpWv==?(pZIwfRxIF zXU76caJml{;ir#{XXS-GpNrb4S6PKY%%oBV=E-al_wF#FTnL&TyN2udGUeY*HOFnR zBm96E-cQOi&|kw2K}GvHT>o_yOUKDXzDy1TQ_%P*RK5^|@QNt{-)RNtJdtU0-Y|=N zEA3tFH4z{|Re9ZO?OBd6jqvtcv`O$bTR%?VIP(MXQ{oq@&wB!al@hEP_DXCf z3|B->kQy{3xZ+KU^8r~y%A8~{2@(lF#`=OQP7mBQCcZ|x;%1v55$o_=$+6h$Fiij6 zN9jXYhWaG?*AM8c3_ypAUa(o1v?n(qHZnt|396^TX3LA*^i!3#Bn>>}JPh!(_k(#O zic7T|gLHeSn-LF9(<}nUaH*YGlOiGW#Lkxwbt+@U9gXM9{i`M+4NQtpfCsKMcvhGo zu$GX3edvocq$D18pTUGVYi0JG`}C@4H&NDKkX+nHwstHz0jLnYmpY^Lq5Lfyoo5bp zY!dS%@=^@OT^<+AuNM|;yy$m_{>biN$j(B@iZq)xzeawpjK}%Cnz+BWCMDtP!u%YR z@@0Rr2ljq$obAZ^IsCj1o}G=4!|(L?_;~r;A@hA5fzKzqz1Hf<`s$$Th zNvNdfX>;imrW?E<-95$g;pyt$LOhS(wH0Vk4QJxq! zofl3ds@Wanm7JtVWno!)aH3%T$a8b)rpZ1n;mUV8TCGC*;3A@`61p{C>&>u^+{E$Q zf@m9V+vvW$lQ4dlvxXz zFb9M&TA^)g6gv)qE;GrGUb1VeBE$iK^FJ|{Y$qtL(g<`LmMuUi*9Q?0v2t34OYlcw zOKqrQ;fVC;gyWTfs)T-P)Q9KG(m>5XaXt@#2o*#(kUPr@7lVg`Qe=*v+Ygvgfd;oY z*@x6zn^batRz~Dg!}~g-lLbuh@pb)rnC7$f<@I`czNh4cpkWbWz^!ANJiMfMKb;h> zKzyYo0G2ttCwwPK2+H6xV~$l@sNk%W($HIR9Dr`Ig3?HzX}6rYjMMrKfrOik{VkL@ z`1fZS?5kwM9fAU~K}%urG;>`Yh|?I0LjSQ(cdgEP-$grt&Exr}gGNCzt#REJ8qeVb zYMpHj#Am)R)CzIne~0^AR*|8Ej$WFio2U=2Vt(e~a97I9lctUDN07eCZ*&?`S~GRGW=R4k`ii_K?k1nClZRQb2bmv!Bpe%waD&*?kmv%& zx|=-h-Gpo{Mea`_D(*vIDg8~CwiI2hNpZCfuk429YiHI9)M63#;^vq-l&w=Jy-_uy6XXP?X27JWK#e(#( zYjm3WRMT3+gM8zRIYAztyLj72)#j<~3DHxXUI0GC@r?NO^_;wy3CPb>eG)?P)0;8l zl+?L{p`Nf@t{fzn3sHt<&EwX9kccYeTL5Noqv8V3(2QzQ6*uZpe3D~z_cnBX zf%2`xE*GbTUZffYwteaD8WW|fK~;C3Hz*7@qs6)4nv;}6K`y&NulxO$#Du{_?((ylWK>c91K=`ppfkOUH;la;f=H) zRAdN4Y1A|ZD5r^ISQ=;k5E9Bs)0=qhDBE17QZXa~9=^!Nn@KxZi$w(R6eS0X2zWgRg2@rSjbExxuOPjRdTdnBb3tLmpg2uM zGMQi#i|A2Zf=aj0tLWImKYG$IvP-`artJT2VOpb1cQx;=Jj3)kJhbNsB<1x}=L9JD zb(jG_4V}1vjnf(qH%O;0<LhLdzjqK}n;Fn1BF=*|ESSEATOicbj4Uy7!iC|iq zNsX+(VqOA(`%l*ZvI->9fu87qf8KNSS?9%-bC8oc(w6MX;>Bm8s*8`^$8j~i#0Btf z0^v9`nG;XFZ$YFqLPZ4qZTED;4$LS|CyvGIx!(}Dpw4}7>F1Q#(TiDH!&MpDb3>cw z!j@@=IMTpt93A(2cE_81eKDW;#x-+hRW-#3I5w;hOlsMH^JB_dRtS+}(rx+b5ZPBW z)iV0UeO*lr{w?X811u`eKdf7eIcaVx-|2|5nGDzNY-hWefi;h+qUhgx2|grJf%X`% z=1O>^E$3N@IMo1D=f5iRv>viDgF8sJ{qg37Vpj}SUG~W6;z)ek>hE6n{fAcH(&cl> zZ=*XhBXWITtv3&Q{=?URJ~Uj)LIWE=Mm39I6S?)`DR{+fBze_XoSKt(2qQ(zI2K(L zcg!pB?+n9|d7~EZr8?e073Fxhd@pX466C0fX3^&cpEaWnrfJoF>Fmo?Hm};I)H^Y? zhR181FC8_6DOjFxW5VUBR&J5ikHWj+u`k=5y=-!OK9ny@Pn*Gp8i+e>PQsG;z2oeT zT?vRYCvQenIqQ{|u+Qj(d0^fGwiD_#9yNS72h8Bz1de7? z?doQ3;OqZ0cqED0o9y9%1H#a*$*gAl%VK}c8#z6fx(spSs8FQ7??^Q(F4?Abmy>54lSzh36Sz& z?YK;=w2DMBH#5NMco?YN^}%@^@>E`#7MdL9(u<6_@#RxJk=V47imJF)1<5{YEgGUf z({X~OZDlk9l7K4YFQ(lKUC@?}7A;?G!HS)+1mUJ;(-HU;vA|MDx~(xSIpM%5*r-V< zGEp+ubsr?MjOJfN9b)e8w(Rd=^FRQ06?yq=tS*V>4G(*HpV4I&_R#+%Nu2QC=VgtVoS`i9PxZ3T}N^K;{RDk_^w1S2DqUXe8 z`-Yr#jTF8m9G6gttBY5%ZT7+kF%BngCaEyv8RoCH`C6>s1UGWR$R~e+2*>suIk@!hVyx=GzF*0N&?{h zbY`OfN%xW29BFkQ6vj8X!MTt*b6VTn7>t|u20Y73CSYeU(86sp+T^*j5_Q2yYZzLr zow8tp(fvSrg2VJ$ur#}|9RGGdK$L=_GYr+R7nG;WU{lhsi`%hk4N~@Aeh*+O3tQKO@YYohf#Q%H7E5E;JX{ zxloqLjT*$)b=_69<0PeM^`W?WT;S*|?HeIhU7;O?4@=EV_3<V~_(=SO=tzZL+9| zGw5{RU$f#>!pn!VbOa1Cu<-(clqSx*iD)8KFmMu=7pK1BUIY|8KZh;XS66&>ofWE8 z=`i$ME1jLO0x>)-)I7Woo-fNM-PDqMaMQn%NE*U8!Sq!bXu^CO4z|0cj3>q}8Er)_5Xj5gWcR&e{u|ZgQ!z^Op4G)-B&dNO6-%o8txK1mpO*bSd@3k!wsw$|k+1+BxP$ zIfaHKN0*}^sG0DYm`|qUXN?yv3bUtlGFS!b?@fVok~gKsh)-Amv-Q9acIm86rJziS zC@YC}U$s>exKDw4aCp8EdDz@&EdPt#X?$tfQQE!l&DNGGer*Hb?oB-e%F?2-8xlJ* z5Nm*wdM2ql1)(4jF3A9t`&+>Z{8E=Ut^G7-8c%-hoK};w!fXCq?ef+9)zNkl2F&MK z*4|?L)Q3t;uHQt%omJ$ihwTh}sZ?93OOol6ea=|iGPeS5<=0AX`({XGV5)p5g2eK! zgZ5D5{W=K=nGq3V1X}ZqsUj?{$Oy$wcRFkKCRX}S?T6#8p76gRzS_2i=qdRsl@oex z-31bCTK2-{TQM3mD~IFMV%n+h7!UB4grp3(144YOQt8BkL|V~}typvjDKCxRPWrxJ zjukr-gj*zVnnC%p>S0SIodu|N4*bHGR=Es)mS$U_FzUu%6ud~>r_Ks5d@t+*l-8HV zz%YJ}yg@gQ_2O2W)QfFxNst4^J6nM8kJFfqm^#N@QM-~sS>|0SBSh6PJk4mE@rSS~bC?@^9$da`H{j2bO?!&Y?ICD0E9Rd4ry6hj6F^jIm0)S$(JG5k z*DCMv5(E@GQe$oDBssJ7V<1|6OQ3Ld%ox5^7km2D^nx*>OaeoRd^$wos+OIszY@i> zP_j#$bjIsKKP9+m3qZj(mHg27)XYpq{y+EY1pr&vNBe&CIYV3}3|GKC_trVOdH~U+ z4Kt3(_#7!?U)ST(Xsin3qG;pkXW%0s?OmbWSu_S5*-Pa5s8abC9@dzpyH&f)%ZTcc z`0I3R8En1>Z0ATZXX@Kb>|xxs#EU^lo8J+J?n(x!%37gn<0mV*Nl`6=Ke+DKx7@s# z_(w}8gCZ+c&3y3PUA#MFKc6$}Kc=TfBJL^8?sG2aTZUvo9T12$L+USkRW?j@HVFbC zu~rzx)%<3IQTkD8jQBKoF<+veckF|BpovdvFy1BiK*V6?kuUN?%hx+w{d#&m>w8?4 z?%z$@Yi^b{EJHD@Fg)0-8xAlnlPxB7e?vh$$|Y2ohJ*dQ-YB(s=-;j>!H;*6Om$WO zgjj4$>*SJ3c*l{t(jRm0fvDf|=oc1((R{m1M+6m5NyW!=_VPg~Otw)l2*^*&R`_;4 z4~|b-Z*+ZLOl?$oj$!JbC14>)19|!;Ku(i0T)N*nNJBX`)tR|N4e0SWETA6p%`Pr+ zrb|(JHrH7WGUvJ0%pG@70+*bpplwk|2d*>tUxVHPHI!Gaf@`y=tx=7vUbBQ4ndYZF z@vMDNUrkgVymad*9*&R=&(&OvDx1noox^idZQudi0huU_V`wd?nanr{NG^%$+h!=! zbsP>B@T^+~m&1L0UKwMS?K)Ciu3%(+I&`6{HEoPr&sn06&6jyqB@vjIoJ|r4`&o{o zmn}{7jrG_>90nUQ&#D`SCqwG-nbv{a@c*SYi#w{!Wf#z5u$ z)ow>A!LVI#M;`uE+;mY^E@$m>n0*LEtAn{!U4_B3|S>a1I6c4+Q_v2#F%r8CRl3&>A)N+lML?kn2%IR;pNbLBg1znu;91JWl zs#5y-Jp>6c_)4*Z&O9r$GF)~}ih_1$^urDHmcR>x6YUfK-f8C;rXbP03bs$-FPnOA zl&Kqh(+ujkv4+C5fakQ4P^!ReBg3F_C}KN`q!NK&+7h^ol@A#(3Aes1-=k!-O}0$b z!YM9h<+Kyk@nL={5n~W}0^Si%wo$kn_IYH}SVYefw z?{!a){{Z?}xXa)<3G6yRg}_bm+0?z~zRkN!U@6*vh1+L54~MgkMq2zN530;jWZ5$fN^k@?yt4{~!VKwb@qaLIUZg@$nN+i;ix7nYff zjj%04n5CVEElNV>zpCVy3tX=8`O0a_rF0=IFsQ@S<^d+Xios2(mOhNTf|CXfe8%Q4 z*66p3`CCM3W|4rxCucJWM6;(R0ncWjyem}Ba|K#}iI{hq% zoj4)Hv11_`;QXOqI-tYSBUdu8;?pHfGtg)m+;rr4xKYEWmzDbXZ3`2gsA6A4m-X3_ zN-k!viSveG981m^x3MRuj~M@pwReh=rR~syZ`NFXNqc7W3W0KswuX0UXm?B(m0?wwNG7bz4LwWzIC*8<+p&vy7lj}k3)RzGIW$_ z)bl&FefEQriN&FYtqgVl%fnm}4im017N@67^suZ1fddLd>ZC(jd<*xC(lMyIeGrMU z#)6P{>}@8c=I6{bJ>8wIJqn(#`W>}jqqj(9skD|9JQe9%4r`_Q*Bfrt#N|htyMuy> zZ8xdy_Wqz@UE&xehrQP-);4nPB$F-t(<2kNiI0KY^Ivpg*|at)$1l@Jt6U_n?a4+m z*0urFY+y#JLKq`tVa8~VFh!|LlNO}qBkFv2@M8Ora1bi915^aB7zQm8CV<3mImFirPa{^LZ(hvIG&@YUgmx| zC-l0$z}osv70S_5SPY@w}UAH5>li8CWcIFpiqVWTqwUWL2GR6@x{+380`D-QKr}VOj%FK zEJps$ULr56pBcAkE@<8G7cOoD^U9#{vAW(u-gtQO0Zl}|B2YXm+<3n{{W_Y{ZVC(A zm<D8hjGX0@Yf$ z`IPw`@>@*;&iC!a?V}9|2~Q{5*NBV{>zf^j&s)nvPuAD|*ImfM!cSMB+w0@~`F)?n z`+Wj7pX}yVrzi8Xm)bl0&*9^vO^=NB`7%a}i`z#r()V`QAVAcz&MXd(m)dpnn_HOu zq|jc$pIlEFWm5qve&4X-r?UFK!l}>skx1VVga}g)kG;l(gcIU4*bQ z&4JuGVW<|!ZUPKQ-c%$@vpeq~f8&;M{;1WLx&b(kugeUJ9Kx562yQxI&lbQDx4T&{ zJAw8Gk{i3=ndyc%IbPCO;qeY(gY%wJg1 z<)ySewYX(f+MXjm^;eMm+IexKByas$(c+`5kt3qaCcpn#LC2#kaYL4*B3VH0 zlJvgMftWGJ_7~ZN(iAx{LR?I&xj9Ill7YIf@~r z36+9-o2WMfkkJgbUt@b2QC59c%F(cyZ_9Nd(3HGTDfW_2=j<{9I?}RFDFlkEfJ9j0 zPcMhLVr4!BD20j6QG4Y1Ra5i>Bw!1f+*fD0o>a9;wC@JZZK#}oS|Vd4F#%JGDT8qc zdcpi6%GXP(5vye}7)*{yh>x5y2w|m5+jo|O0{c`GdK9_Cq-po`k;VHPqXS=XowYGfcu9l;AJNIpKW;|6mlCpw&X-eYYKLT5OgwB+r z=bWS_V|Nb+3q`bJ`~&+Nx=VlDF_dN}LyShMT~zMHpCu=P< zxD0p`3EPX=T7IK{E_#ZtL?MHdjP(401z%VrbBoSL&mV6&CL$E0Mj5|ralin*N`0;{HS->EG?h5S4H+j}N^294?w%C3Sxe6L59Di0yF zwnr?B885#M6qsSlcP*c;A+N|J^~Ivt+{BO_qXoL*ZyW|YSe_zC!|HT46-tQ?H>U{un}&^5a)+Xiha zs>w(ds^IN(EnAc*)uW(*$0jkEC~eK^l^yF$5O!M7)tkU|ntH0+7`e9W5dJFtd9X2g z6kev~N@54hRH**ewQ2!jp$AlQ-awYp|E+Rl#XX-|%q9_c?e}_&s%Q|G;A8vXM*z0n zHtVXOa1at_0q8gCgqVVup51+}grQ;`-Yy?VJtneZ8EuV%1O(73>g2G}v&On`@An}B zDNa;QV93@(svg@1Pl$IZQ+)8^xp&o$w8y}9c3?UP8CpY>A*>;W{*+d!w7;?-O>^iJ z{WGx_g&K(kXV-ifvjp)vX_BeJaFpC|dT8;VhW4r2>ZkiG5}-@=h22|fd_$?Z^l&l) znkh-2_XC`s&x6vPdoEd-*olUUh8&x2o)CSm0;y8#1teTi591HzRvX>r+L+L#_5k^- zA83jDT9#$7hlKo^fF$_V+r>(<+`q28h0M*My9=CPG6Ulrw(xB!^PMdxZB0V{LIBCaj6tv zc+n)H2g^UxH|^~S&qcu}EY8upa|H|Liy*6F)rHFo*bQ#-AWlSuqZmkDM$ySu;%-MA z=H`LiS+Ls>p?xHKvW=>F;={sOTp%gTR!4~jg@rjkOI}M;Lzp(2 zMZuum&;#-rbqvPq=MChwK=b&MP$+8TA*&{}6v ze%Qq88>X*=A3Sm6`sQbqQ1s+AsZ=@pc^5GIf$y}cp~`XVNLnTmRu3oA}%701HB0v_=$I0WU?;hqIb% z0$s(2xQ_(k5s@|nn`@(NRh+3m9;((bFEKPKd{Bt3R;;st79DXZ=aP5XIt-UzZsbI<6%QD`Xt->3G_L7 zy49A&14Fg8N{4Ap&Rp2-tsnaR5vzC?LHRtPz3~cT`J?d)j6`bg$)cd9BS+!GXhY(H zX!1rZB|LL=^R2`uW{kC}x^LG{dBfgs=ZEsz1n!PwIEB}ZM_e!?9&+Xt1M_mje80c; z6X^l3lJT9996t0kO2W2DgG zG$eD9FRQ%|sSULW?+jT>*)z6WSE_1lQO0RI^o#bmfzTw^hNjpbWlM?b#w{9;#-&Qw zdE^FG-OeMKpb+ghb5U-rnsSUQRwnEmA%4j5=I~~mQpxNbSBV&wpFb+EbRZ#u?^e{V zI}Ch)T7eif&Y;=vJl)CIc)EqgpdW8-F*u^5CYm|`>E6{iBq5@(lL7;M5xbwb%4vuz z4vr78rrOSTAR|BBmOxqLw8Ckg;(h#>u#djMhu(05nkmX{JQ+Ubmcm}kfKC3ww=v}d zp%@mX#Q6hne{u_7_m8($mwU9LA|q!f^Y6(}Myp3>?iwZKBr!NB_is)+;T!b4!k;Qh zRFFpI>z#}X_=ePc8uWr<2m1qyCf^cG>R>)bk^XF$8=nuc3U(G`if<0mvAGEN&zsO$&YCE{ zUBIF>{B#a;Vn&A7R&W>zT4=Nd(L-lz83ac-aE`qDLdTyQ*ZauC%4~XhxvVKsu@k}s zr!B=)`BVj|1LJq~z%(%p!$66$M|gG=@AOvbhv`J z1Od21(zBFxB$(V4GuuN94K<23<1BdnQEPq7o?1+}JW>9|5wSMkiR5?g;dTX1G(->j zcQT_jRV-vd*V2MlJGLhl_K-%G=bOy?)Z4U=TRSRE@%_I2MCfy?mXX1qjfZM1RRcKO zYqk=lQFQENU}Mb;)u&;x%z-PJYIrVSiiY1vZ6$k!fZ7EOXkEtrE6CN*kab)k&kal{Fp%Y7oss!#{_WLEqs?< z;CMxhRWC40CvE}0a-OYp8$^Z0BrW<3*0su2BBWx|B7K6;xRSc%}_VxEVH5?2Y zKMZ&7!&b$GpBm`nZ8w`tL`n$r!Q9UQb1*S%Qyc7;=b$sryY^B798?9F;_A>htDNF< zVms5bJ>%LD15-(kUUwtubD2K#n+hRUFBE4vf$HNP$pu{d!bbQtge>j1cnnVlJ}_xz zayHkoY3t`;A(M*&#^*m#i?QhyemdL#`{~cY zJs4hZv1va$x6Uir|K0zy@#jmBc0K>Q`EQkR@v`KdPo0I@tz~xPzSm5m<@qEyCE8CG zE}8`*!LcbSVRai!10sGcKZ#^>g1LEqgwg&%=ZcJtP!jXmV`W}}S|tVxqN?1#p%C{v z>P8yOv7#GaZ3#+rOk8coCB0CO18RupR%+&nOZpUMVwFZoX;N;@{KqK0W-r3&rfg+ayriGZi*m?VfiIa7_5YD5dbV z%IguIf44M~szYvhb?dt$vKE}WVnYnY8JX=*sESV_yDi0C8HrxNps8ZIqzXyJBv~^# zs`$V^YLv(rNnCDnibGkl<22h#v5t}+KC)3>%|Bx>P>2}HPdpAN`2T}smTkiNBt~vov9kX)= z&wP7Sz1qndZ2~T7<2rhl9N|>(eJfmm7m?tWjl~+qI@SrHlX>ql<1bE&<~EC3wO(|4 z!{Tzg?Q*l_a{nLJRD7CGWF>UCUw>_j`wB8&<`?4hmIz6#fk&xLmkxOh!No0d1m;;P zeP0ieQKoRh&HKgfD}@os*I)0&#K~t}qPAr=KG+Evjx`y#Evfk{UV-BL#GOcSdHF6N zfPh8R+YG*9G zaHRFGJdPFxZIRzAS=sz-x4337)dQ8HH1V=pEg&y<CJWn`fvFtOxFP> z=G5+ZPtHlZO|xV5TWKNUnjl$1o)uR@-YAzjHP~CN+4Guqqm#aKA2c_jnrUSpH20aE z2O68Pqk|}luF&6X`CLBG)Pmndj#CiLrqRZkrHOyU?|rmg_Cu|qEeo2AjikqXGG1i( z60WvV7qv}`(W^-uv4Y(AH-U978?Ym8Ybd`MGuw32kATt4?}h*_a|QCg zQVWV5Ob!O-2?(KZ1_;=4Bqt?x;eV^8IJzM#9{z-fjP}a(h{#F1+zl%~B9}$<!x;l|)0*0@f5yQ9;^-13;_=$JW=GqS@pVEQ2RQyU*Vb|Zj6y#1 zz5G);)^9(|Z}_N@+&y4l(Si?j&&}XXLl6%;ncypqt5+~on2w-IRVY6hfJCSYOK;4 zp_#@`L#^&k4x&fcE-e*S;X6cW-6~0hBrVL?O>a&h4Gc%8YR_+KGX{eY(B}SuScT&L z1F`lqLHr*O>-soMaf)rpS!v@B!%7asBpi=Xx4eJHkAiGd(9H?zUl9JPU(6x2SnAvvlsZ5x2Fe^jt`fh4B zIdoNdLwn+r6BS9PZbxx!&{<~A$K+S*QA(;_^PeP0SKJUzLI%lBuhqUk`1^KxVg)76 zC0_ZmL6fF{Os)&nP746T`<;Nz@$D1%L4|Cm+sRFblS*(7^6>tmVu(BV(J;d4;jX*rs3eZ< zuYw^Fs)u!OD0B!*R|E(SIRGDVRi%RkN1sE}Qb*Z=Mb;7~x|k9!$Mdr1!lT{#6jCjA zz+i>&1KxVyryfq+s3;N( zO|XKby%s(RLXkanKfW{%-|emY934q_opnA`-o8YgHiw#*5F5p3+g>fqv=;BS3m<|S^+pp}#wUpp3NSk+T^n7=5=9O%Ry8a)l@i+D#QBo+ww$#KC{V2yP24yA^cj2o5_~89TcVK+)wi%p*L-ULn0Aos1^vMiyTwSPbIhQJ~ zLn`uU&J7nXo@ZVh&yW2okk;%M(Ivj20LBZYNRBT@Ap5|t;N4ckm9!~+5kQzua#3=$ z+Vf-tiXS31M0)&8Or^GZs z+6n80mDzFm5ta-YT12RWUZU63WyDxVcRfAek9{V#e|?3kMW7q{0**Ja$X!|y2Na6KF@nza2edsv1_%t4aC=B&yC7OE=aU!B*)2cMiUIfRDCL6=xpa5 zNs>0c&6lphUCyTv-QbVQY250W6)N zYaz%9Wo@^edTDZiCx5A1WjN>q#V#N(H2{{W`6X>qezyvEe50KXQ1!x#L@@WqLty6F zE92SSGN{~G4@|<7AEyQ;#zo-~EIg23My=DnLP75ng^SdRF zT-x2|aAl9se{rQZ_>7=uMJ1sN=eDJm&lVO5%~9mHj2Am8?pZKWZg_8zXOT5j)rs)X za5T`KpHn;enw6cpSv421QCXM2GS6hx@2}X{Ot&)xtQ$*gG=#0*HK2sKh#BC>H~O+cJ?{;JxDkVjS<}gVpa$`j0w#_$CAN}iT1f0a z7^@=#Gv0Rd`$~yfMh{Z=^x*$9j1^YF&L#N*qxj?gsPwC8GNs;E%cSD9zv|enalJZr zIUXimst%UYf^ou%zWXUk58uw^`4n9XJIzWz_m`C$M}_c*J>ZYCptr@cFTNRPx#^o- zf{Zs#Zf~Q|{7ln<`E0w`J{rBuR7!$i38`*`%UB`04ajfxPVK`^37^;GI6cvmUBDd| zsT~#$ES8jT3ZnLxS^}7Qig5b8912MueXjJ2a`b^3b1U&&Z_` zH}ww#F#7u?nx#l&(8tduCa0pV+mc!F4r_Wxn9M)l_MEtqCMAb_;RaFGs`ik-c;_)b z3mw*YSGorZ?7 z!WsL&j7Ojwe1P5Hn}cRZuF_0aYMYI)!`H1lIAgykZ#L*|CQ5doZPZo%3XWd9tej|E z_<;pnwL@r|f45j$n5ta_P51v)+mv@6BE8O=O}(%sdcXuM11z$M(%X4abm9sMyq=JU zg`CH+H!>P7z(Ct_(|Q2^J5X#A@mBOEi)3IJRImVt@wL+$dvHRel)faK(`av+nugw% z8a@8$n23^TPdp-T)grxpP2jXKS?r^*6*qO%dA;`akg>6Gv66M16Z^N=>4>~+ctp($ zaf@W*ts!K>v5w<(gxx#dXY^Mg9Tuv?JujV!R2`!~2mJyzIs)^`Ep!3fvNZ^%{Y&%)zzzv|p*v82P-*`Ew4NA$E%W*3mz_F43@dZ;;SX zvwLZy7CEywiueiWLPp$5l<KA;1lq>%%yz+m1oH~ND*!<3 z2H#glY)38T8u(Bl#j3P4`)TE=ESsu@m${PoY%?;s4UFop}EKP7|md ztqNM&z|#fR^UkYg#uUQFp69-Q#w{_zOQ$D>h#HJ$#??4OGXmi_M8PQi3JX4=-%Bk)f!^Pr{W7^sJ< zXS)r|lfYz_jz8DzZb@83CSB~VRpInCm*z%Iry0NFdO3^hbi(ZYenI}P<^~+~(EntX z30ttjL@StF#i)TA&6%p4)qiH7RvVZ9 zpH;q10J(pvd`$g)^?=ovo*Z>)M)((zF%fKObN!_wvoS&O9H$&W1A{n&&<$_Uqydm* z#kFeYdwD;pWwX=!u~JiV&6K!PhqXuE(00?isvujeA=bnB5xxw=!WX_BhYkvJ%g zfntwBomdeuS5u(6AjM#^pL55k-1I?J`M_9emINl$qeC4v2imGR^RZ@Q*tOR#_y!$4 zHA-RK&+GO44H^(junH5=EkGNa-rxwWIfFu_~psc>`HYy4o9x*?b-N9b9B68cEQ( z2PSG~ZD5ZhWw(gh6jJs640%`Mb(P?gXCF+zI}D6kDD7+G$gAL1u1TB(jx^N})rRs; z%P6WMP;Z=}RBjF#s(ZgkQ|T}3j~$@qPf^bI*x4~0>$lXEmY#QYa?M@g{I@b(Cg_bB z3EBbSZL)%am-eX?!ll=o3^QwC`C7gZz)71|erGNJt~A>7LBQ`7*vK0UB4*O@ENxqc z(uPj-|5Z7t@0UqE&q>V%w1Ne={v)((#j{Vv7Z2Fi`6;OnY^d9tJPIg&H#YP|?KJJGb*Xp0!{A6C}(Bdo2% z+HKFGx1*a4#fIHILUlPPEP?qva+|)+0aM|oJY=*^eBf527Nagk{sG>byQTeINu%kXZ=i=KrGWIJv82Mtq-Ju1>^K4lAMEW7bJ(Kd zGdykMJ{}*imf}^&6vV)j4NuD7bfOOi6Y~cSr7ksrI}`b2YZT5i4Xv#rCrBJS&f0zc z6rwTMsPqvOCUF~pK!p4H8wIIZB)SwhKM6x6wfXXJSj?Xxy;3-~2hIw-%d(TCa|=1x zQT-4#3AXB44yItXDr$W@c&}-@-#X$^FX*FRlDxAdP-fOZtLvw0Fam03qKngO5ICG> z8L;iAuu`k0L*_$CarHX`6`VArG{7To*UdR7*66=#i0&chX2-SNUh8T{jaUuED)z84 zMCR1?n{fuTlN-YiCug;UuEaaFCN`&mCs8{#Er)8)h@L*^Z*MjV&hKp=5(Ixm8(hU% z4dAovu8Z}V>0UeAXkl;B72ipL82d4~i>Nx#>G3~+UlJa^Rvcv2+i;GlD?6qyih3ph zYh^N2S{hPPzjFP;r}!XCZgJW(raxnZ|FdsBsg9uPbKo}(QDsCBPw&`Ec9u16gUakc z1nO{?mndhmfncktM;ZxvZ~u5NbicmcP+5lDV{&5!T-@vD#@3cBerE@GYDhH(>YSmz z9U4Q|A8m-WJS(O;2O%#S+CT@K&8<`fa;?Wl)^?tkh%2*sMy=&jA*+>b(7y7yV#m01k5mDzM=rNmdcOO)}0b;(HF%Fh#aInr8Y^L|LVf3kEm zg3$c2gYsBxtt$l)fgbT933gn0x)h&3K3f``4S<$WeB1{&RUl$^@IV<^eU zFj&7zIk>MUo;Xdv54|ab$2^WGf}w@dN$=WflA|^&Y0?qemx_CNCzWtWRdcYc1B(VO z`MKUb$lDLXp?YT=cb5o;G{6r_;!k2>ssPXkw0c(xM@i3C9tlt7lkvvI!x=cQ@ zvMM3-*iF#x;{yc*_WlOg23REvUYS=zA?4d6Z`z)vq=FCSGWN6IW5$o!d}$Qgv+V$e z8_?K*=q%~9b~6@a;t!p8G;2Skdv2rFFw&2x@5ST>5YW@y;$!VE#0$V0}-i(v*wkh6hsIuv*Z{0Xws2n-pT64694aEjSp7u<29G5fIu0L6m7LJP>H5PhX6@as@sC4VkwAF5wPq!C4>d|Bg3@- z`Gb2l!eQ&>o|#x ze90sqD$!RR^E54X`{H=`w#cz3Y|E&#Rt5BYIJj9sW-J+UYFyw-KPf~E(O~4(J^Bhl z+MmSp^`hR~547m)Oy%fMqoZbjx&0XVz0qCfWCN;SPsc#qKba+%N0Bq$P(^zTqr%w9 zT%RANxN#WDk`-=rW1YOeO&+>Y-KCBt*!G86I?X?BV;ZM*VJ|I6z zvuv{WtuqN&ieM|WZyC{^Q%C= zTn1Ls+fBLud06E{9>Y({N{U4v{Uo;Q&>%aNGWsf4P`kD_`wooU9!fY?c#$$e0Z@qO z%7(A)5rt4mxQi&CY`MiVy4ipKlN#PTQglgsyTN_fwA*Wdru~^3yd()or|T*|Ry>{B z&+c^1LghJ@xWlm5-ky!>=%p9byR*-+!>soLno8R9 zQTC_V)Cz!L@RdG}s=T~oWLGxzlE(ZC!3Okm^P*qwdX^<7t3K%B&I=J`c-BE6I-_8t(MR*8 ziWgpOxs9529?GzrnKqMfD{%yYfTKz~F_qEqWN@3^&2ICnk6YC!qI!beB}^$%_3xOE zx(Pt5)W$t-&fwajfeFF0yX9;eG_B>}R!JeGj+M0pI~H8JWZC7Ro@Yo;0s^6OJeV?@aj6S7sE(g6+{@$j2e9_Cq<0p29KA;&?;q&si4Y zn-n6uN{0LyTJ1kerk+wsY;ar)1BsIzGn$rKP#Rjh-8n;ViUM?x?36Km(crmJ6?9&D z|5za~Dr|NictKbafj61wB-ok-X-?5Z zb}}hT&HoT38t?Wo<|N{MoS&Tgt&2?0fV{Hh+`Ze+lEd1ee&XBgc4vB2Y8|5(adL49 z)JT`XG4-`J3uMdK;xrKLoENX_v3uh4`2aa9{%Ib(&_u~XK46k3rTE3|iF+-Xb*399 zeqp6;Rt4r$YY`;tNgRXou*~_U7&zwkYPA{P|1?~^gk25ZTJueQd;o;$$BS26?>5AZ zE?D58nuA>dkIvyJOipNMog~q96KQD_fM^KW^LWH&b?nx?e5J>V{wQ@D)Tec#S>2y~ z2*59#G;ZaT^?aC0vRih>bp)CV*7ql&91(nI%22+vI1stgK_TWud3$K}@sC6v($eJt z)5|giaLuuQGN(g{ckl?*krF%8+ddbxv76DFsRLsH)x$9O3w7dTrZB~E*iQ*(fz z8B;y@-#X!m;ZuYcz&?Sv{pSaR{c5Ls6DY8a@nl`CTpt8A)z;qL4$nELHqF{ng#)KG z&aQ?xs3hu8{gkpn4d9ijDs=$5d;4pvD&u+!`Q0Aywcu=Ob%q0`D)JpQ#8^zQ?)T0f zDSPA3_9a;budiF}F~U;ypD@hr2Bn8bZjQdB!Hx!w-}3mY1NpJM-qzjl9wP>8p?}rs(8Y9(9=q z6b%FjV=vYVWpWhFy%mgD9ha&RU;MG+cZfCpiejhv`sF>W_$Z)S*0k!pEsO4eu{xRi z{4?JR!@F;7ThEH`T$2h?9$ArA!+(K8)E#TX#%~5|_D<~7Rb$BEqc8e}m9}hscUV41 zAmTJqmzd~I=kx0dMOHgklfwB1JNT5%o}%^oUm|<`ev*rk+>Trm4vEDJ2(NwyX$CaIP&--itE<&Izvqw!~@87s^px^!k zFd*e(`Vnm;6xDrlj;nqQc@qx?W(^A%4*CmSIYvICKnz>58@#lk zQO*Ji?pbz~f-5lG_K83ZYe4bd-5LV&2Q#r^=qiU$s3w;eH8)qwMIP7cJ1%kR(m3-i zn|u+leHBP&1i{yzpC8F$0xH0i#!<#r-cMwajg)%8f2*Ni#wt}}Y9y*zcU3aksD%$O zpMROW*y`vN?hH87-Y|J!H@#oW_x&0q-e}LBKht+%htJ_ z^HElJFsswNFU$x#VEHrADinmhQ1o`I04D}fVrwY%bRI##qt_0=^}r5CI;+d(rQIztw|*~l>uqX4 zBx3SoXCE1U2hj`l)OeUid^w7RUPA-F)#Nt#uVB<&xMy zDPqb1gg=$(b@5A$r_@8u8t>z&L``_$idEw3LyKXI5+DKuCvr@Ka6^P{H_2c3^g0?SQ0i zQh|U2j^su)z*h3vt5UyAC?R)2b;NQvv^w`DaJTxKmCbZ0SACDo6ns0y=qif>V?`J1 zblTNdCG@|yZHZMBP|643*M*fE2_;L5n_Y$tfcD!APaB#i{$AfByYWfqo)_FaAI+6T zJe}n#*P!bs=1#+@b;Qr`<6E?R(9_SA=^|I)y%2ma#amI1L|1TLQ;=MS8~C&!Mq{x9 zLwh;Kz~#7(F$QrLj*e~I?|^xKW$c}7ozr)?AF!a(Mg%yh8Z5sDC0E_LR<{)AmJUq5 z89Or}w^61n8&qcvHQ`v}2$LfZqT$EPj{Q=EJ5X26O(+ou4ePdZo(adEo@Ie6*&F9U zfZF7};#<`&RH3=0BYOFeaQ4ZFsA^Mmi)0{=f-(K^zQ?hIkuSUcvpBwXy&) z1+D-a3&GHQw&uQ%g~KGA@3c==2V2XJR|4OnKWIMknEE;7?l3;d0BWuD2t#uWngpdv z9dBSi>{@71RvsQ!oOxmGUOqoI9!G=HP?2F2J)HA!ATL9?*xRKU}D;d7{Tc#8?oaL3p;{ z551~UQ{)W`bMUu{sYp%J@uEdfQAja<9pBmpzjcG(Pz9#4=YZ1cgfK4;=m+ z4P^t$j5IcH)wd#bhxCUW8mzSkx0X9wDbf}6-*PrynZBb4)-3*YBrU)Aou&- zYwohxGsUFIp8;wTtj?+jh`)N+o|TjY?qKs3H7k|eyzGi>e}zSf2igPamJ#Xf~Dq)*z!%(yES4N!mY7bTDs6p zjlz=u(}2q|O0yhPUQpJdbAVH9_$+1s`xWQ#t-%i|T?cIN5}Ge-&Knx0k*5Z zCG8YgwAYex+R^!XBrO10VrDQJ!6elz?;-qy4l$P*%J!}{MC_hCs|F*jJbH47R6&n0 zbMPqWeDBm$H*1a$K>X4GbTvM?FBV)iB%oZhPNv6YYMS;5w`|fy>f$&gBrU904%PZS zpFb>nyLQ}7RKO4F^ocHc$$8=3WJ$x(L9eBpi`>--xZP`-keN-ne90 zB|$cBS99{)j%EoL)0FEdLV()vHl1QkAv)4;0gWe?lA9dk!Bo$kcBFbRlJ zBTbb*)TpXH)P@?fO5ifRkA8UNf*iJAT)C=8OG{VkOd*SvQbqP0S+iOwEX;7xToMaa+FE{v=L|45Ttr0xQvd2>Ofx*bVDaHprEr8tQ7*L zAxa1BL~?lr?(~F)EEQBiXFj0GJg01C8s+`gkEQR3n#Z*FvfA5`3_6e@iFcipbQG<` zx=vCWd@~L(u(b7FzW_T$_3|k2{bjB!VI#7&h$5np3?{7E~3i)KJWy_RI|40XY`bd)A|nm zju33j6dDdUbYoEsK=Rj}L65=1C_|5bjtjjfke{=Do<&mnIX##*YT&rBB&2?)7zv+s`{txVIpo~osY$g8os4=K@NlF*Un#Rp3ISwr-S6J5O? z`K_u0ZsdC7H#6KN&KtD2??)vSFgvbdPg@Z2fz>9B{^pjg`r@$4HLi*=G3q+TGp?D( zv(Qf-VTtG`9S^4sdPtq*lB@1_|IFUmymGH|{DfwYH-@b7Yr4 zY>9w`8M2S1_O)}XloC!os!guX^R;3&%N`*N_F?0eH zxO59x%iUo=n}MaExaCdqBG_H)lq=bysGT6U)Rchyr24pU{m)mA^+f zM>8-mT`;^v8qU-$pW_7$o)Y0rkK&%=5zxPdXK0{1x{(8P<1R)9KJR8`Z$krRI#8b| zuLSY)zP(`WAV+YKo*Fx~}oaJUZ*TE6G1*iW^ z;(caCWr7Y6N@`JY0F>cQ9uJL;#b>kO!ph}v-*Rd>#y{RsY+nNy%TRX{5h0np3^e5B z+cVC+kq5nwKlBz7ptl?S_s`i84D85BBPtuk256Hq<@LOK~mpt~2T* ziO}T4{iW@p7L{y}pnUZyX%np@Y^WcU^ux^^d)OVDgy944D&l9RHNz@MK-ECyw!vm2 z*gZ3^C+EucFUk|5+W&2c=R2XO=4!&t&Hi7ne9FfK zvu?V$xLIxSS(Am`E(`|rT0p!085QI$7q6`MJ8GJ*4LA8f%c@`l^oV~s{FfArSft;; zKPd|()snnK^{4WQ@<}b;JABFe_Z9i#`4`V$#u50XMYm>D+1k&`xAQV?|GyBk+n4sg zU)cYCNq&j^Hn}F(o&TEuM1Ff{E;l&dmxHdX=yVvg^aR=lE)n=@CQAlK_r?E?({0+G zc3cxF>}W4fVEd=!0it6iy+c>^S*9CWm<67EIed9ar09G+F14LYXxDCcdh#+dxoq)x zzF2WQos)sxP%(!kZpOnL`#KZz2Ripm%&B`*e$U8^PiK&}l~9eCAY==85_cuhapF1Q zk=n+kK2{K?L<`;Kt$L`k$4|ZlGJUa-A($9k{_=~h2bW9Fqjn|+{S2Lggv;D|k(nmE znl)S*7^4>kUJTt?>!sz`?P&~R!%sf;N36aD@B&4Sx@_VD@DKwCR0B!*YTcP_QtLb_5-0eF%1``hRyg9f!= zHTdv!DX}wwaO$FsIZA66COy{6$qGbu8{Wo7Jcc|VKRTowdqp7fQ!rowtkJRr)xF{u z31rr;U20ta-xs;HcN}*#&q$M}j{@|?VBwtX&;Ph9z37XtK8-Si|Bcp>TS8e$AAdY) zPwfb9Od?-uCm@ z0(3pwJ1@-?i20d(AP+XZ3Eq;e#Skvmz`Dv_43S8t9b__n+lZB=WxXD z;AFd}t8ku=QWX}~Mv&Vgbwtg!3kt%AX}G>@v|ZhR&_aR-+sFd*BbM6_PNZ{mFFI`*=#WGo`f~>wL2)~#E?m`bg1p=#{C0PVFt_a#&UJXlKnH+Z$Ed&vPQy9{ zY+5V^wyYY%Rdk5mD54yyI%Tv7HFBf*^PJVFh>Te$Wjhw+Y4!7VG)(7){4t#DuUJp% zXxwCXOuK+-4Jn~jqsf>SRAhz_L1&CjC9qCaEpb8|J$kiRsnRDK;}V1X4la~7tHQ*O zw8J`|GA&$6v607ZpF(g*pkwk`ESk#-@z%KzD*`))J2`Q2L84hvAh&FNCB|mc zvUQg5=vlf>-Iarf1G@}wh~dnA6OUM!+qltW%I}RnvE(UUm#|z=6P+;S<8{Sv@D!|Ign0f47lad7{7jufS1q zo}`4N?0o5&@y?qZwJc|}ZM(ISyZg=Q$tF-0NMck4R0AkU%x3@hcW^I&LQ#|i6iG?; z$a5xblPmxQT-@*XbEVe63qjr?!UtsSgwAAOuEQQU%Qk7U9cTXTfV)p-%dqDY|N8yG zo7DK%x_+Kh_`eXW$_iSYOjq%hus6M%oD$DU4Vu84LfGl|`%&oY_Xoc_c(Z5s?sxlk z7o&jH@1lTJc+#z!>6~!--kbdiq}c)(&b3sfD=G7EDE7@lYXx3=`Asbq`%N_D^ATjJ38G`~#D4I1kskiwZ*K|Fr?0!KX*0fjFa<9CHFIC`uFRC0 zHupS--Qzb}^!LHl^|+y3aSBFjYgZI?v4}ybE||5@F+>!Q_&0)~S4dH?#^?iWq@pu+ z;1;#vV!R)*2Y;_(0wIT8cYY7ul>o;9g=N4bX#qemuv=CIi8{K7)NRVCyrI|nxT4zd z;@e6Vm*+;RttqFg=IZUAAcsGllXqNS62-J^RpGmJ_W@97(sW&)HfY~f(_)TRnhTAx zn?~xcCsShQzjdkbSM*vl+=UCf3-)Wag(AMG{+!&mZFp`t&f@(WQ(iGFXUC>)F5cnp z1DxbSw&a?k&j}bqdOgNx-n=EpQ4jZ3Q(Cpecku8ICpdo0=eGPNz2F}I9HXwN-;y16 zWB#x+B0D!zZT}<8RPRj4N!?Vu;KnMyV#-wKXu$^t!tg#7rI8`!1>jNv^4T)Qg^4`T zMIVSwlH19YNK9GmBa(3)2D1d_W)lg}xvNOZ8Y6CqgOXhx=#st}!LpHF7EA9)HQ9Ny zlRncquX#mPwHU>F(~W0sF+VZ5US98zJp=*+`CT!OUmiM*KqhiBT6k^%GCv53nkg;6 z7u90h)aU2%=oPAU0g|aiVa2+ic(hh7oUZWE(v1Fcu75h-y4NXB-fjB;{5+7FC-^Td zBPZ{?plO)WTpK0~DSJu;N6ljIu%>^Pf*gN)?grZb3Yl?Uw;sQto31gUe{U7Ds$8}< z8J=9$4ejUT9W`m{AB=!(T6P%Dp#EI|41@>s&QShxm9PSI&e=y<4%5VTqXE3UgLV~1 z_<4FEom~O>Bn2ChGe%2dIh6bBPl_8h+1-|AU~l55p-s8`!~W$zMX>B;Dkb<;cm?62 zk{86?&uzZ-4`(NXU;2l$6aUWkuW|~n3tuI_^CP})uC)QG)LUZJ<43bl-Y=O7`6^-8 zBz)QI*u2d#d1PxN`^Yc3D2FjM=z{Y_=;XpUeOC)akIz($}#&Bf4#6S z7PfS2%spT4b~hi8)gMb)j}a(gVzOtyJGi!zZ+`+}5Pk?9&^moSy&8YcxlNCulcT64p}H> z2J55U<6b)por3VIjBSYFyO4m-op|w`hG3rbK3Y(ezAa7GaCUt7?&I;KE`K$b*JC5ct!86xwu4eQ07B?Vu$3BW=EDpP`=mY2uD8z*I&>Gy zAQy~ZD=E$GK4jCjs*pa6Vr-Ow5gEb>2hT`7UsHSNZT1(Xc0c}g9jd&fYR>B%Xg5>; zNY8tpV`$3E?Ezo11p$=4sZ`Jsb<`tqW(wG~gH|&&$<&pe=45h|BY2-PHhr9PIb+jR z$KpPQRw5zg)aqKTnPXW1b|QiwuRbTlVSKg+GXzssWaz9t7sbv_)>FfAKx3!2JG@N5 z)6cz@3Zcn#wFnw8<|URbVl&a=@|vm=`kyq^aC9>fuCr#1`cHQdo+qxL!gX-BTw>8M zmvAl|jFqe1w`@NLvl%C?t!i2@Wi6*_L9Ufdt@R%m%9B@2U2%4Oa4pp(7qhW_nK3H! z`T+Xn9sCOZld+1y+E*4B=6X1P_K)S)+vaM}$C_&*V?9lI38QE!HU7qKa*topvQ7k7 z`U2!gp0%1`j^tUNT$F5U(EK=h-$-?zx!jbH=tsR94sd>aI$uAVZ*a!m^10Uw{Yy2* zm5iIRJNU0OHSc)`zn)RT-6jd;%dp|fug+}S%$em|YL9IYbn>&2P*d*U`a$1E_U~QJ zP@vPb^!6xcg%l!PyMQX4?;9r04J|GwFG~E2691yaKSw40(X%r>KS4YKm&M%G#ULyG zlI8OANU|)XnDW_2+T;g|{F6(nc=$NEYmV!#*T@Di!X5MoyzmxuXUEtFXH;eob2L?7 zxYqWQ3vN2GshL59W7q3S3Q{;sGfhi0Ycp=ElLP*n3m)!k~!6 zqG9PEjj%JV>oo05v8=@15vlkkBUAlnCEKQJM1iQ4zRVIQ?PxO@$^NlwG_t~zLoY2% zvSj}`#E`DFLPL3|r+h~P>V02(D_DBo#Zh3J9ulxo^1qPC0J8y^(#P`(W zS6F~r(*?YSv&LXCyOYT-GAgNvHdHxoVn2V3Mui_~qp`=9H4dS3o&>NYGk2^sLo=ps zZ@-v+f$n`BwLeO~xjI9MS>Hg#`TB9gBu&Qk>#kUCtG_2yV4jB>gMUt@*pR%>k{#=b zl}s-?>{;Ya@^~7_BkCo?wUePbiep6?8AqGAm_APlH_wR)lmydoK;W_m!^|_Q?EC11 z4qi!mp=H$?oM`A!vHXGs`_61RLTh=@NcMrYfdVOi#6fI%`%Mt-K7Dl!5JgsIaISN9 z==EC9Df{>i*++NAuIq?>YA0;=6%SsVA(ZDI4%iIWbLO%^BNHUO#iSxuAcDQ*`xzjV z=(Yk=kJ=@&&^+HQsX5s@3YbT}AmT(Arlz!D`;ogc4c@ewvzjS;W^ODGC^V;{bg0CR z{0X};ObMFSrtj$dgp`V3F{R1gx6nHe5sUY?Q3r%GZQbPwvv^fRMBKA`TaQ(2cTdOO z-bs$T1T;ioVGG$R%;O22BMYgN)lFdmR1?K!TpK{F^qBLA!Z&#?^~`IfXoOshd&+)a zWm+4Jl2^R+#zrh7xF!wN5%3k&S{59f)!iv4&xc+aFkMP!tmPmX;hx*JciKUI# zQPU+U{OOFW66PNWCy4dwcyC_v9A5Gq9!b3>FL@4GR_LSW%6X?AvU^&N0mcEN=q+{# z30h6Amh?dR)Rv*V4vlLKoXovwS#sp%rvo_O01{g-OQm~3Fb$Y?LUZiSw1^nE^Oyz^ z5Q*AM-Fgy~U(kE;!(B~ES+q41rX!gys}`NmlkHr*O!$aY38=r; zPckm$Iy(*Wzu`YJr85w$@7QxUir+F_%vnvdheT3)Z;$HKMTf24Zy3T{Gga#WW6J5M zSKhO8m4hX{R$Mir>C^1+`mIeq?FGkKWfh76`=Q}dgnoD-)yyiBS9p$(xIOuyXH?3= zrVfaXXBp_5GSfRL{g_!6L(`r{Avo7pDl?~lNqc%pd&**!Us~Ezq-=PR)yrrak`xt( zAbk?l1@m^|poB!90p-)5q>8hcGDqCqolJHyYqv9xTQeR1Ghw2TqEbHM;Y( zM_FOHH0$3w&MAgQ!OGz_>uJBurZ%9Ls6?ZLYK(_ zfvuq73{cgkp;3w+b%{9kQzov+6;*ooF%{X8UvVu}{svL8_|52WtWci zJOn-LO_Hr?S~KiKYij(3^xe3^MH~$nvS-`8*~uQEp~f&J-je^l_n*K0$9VrwdwV~> z9{KYuS~^x8auA?(;Xk{A+$mw%3px=dUnp zIct@W3-FF_4H(If->=6utZt;DYC%fpqhH*M%KIoC3}LAySKEL9W+iXytpfnVEb&fX zLq9>+v(RUXx}Ui)-~}x%(J2s)A}99IYaq9?bf~NUsov$*;We^9~{O|08)v}Wl6>?OtVT#xIG4mP12KdBXBf;I{K)c6PLKAlOuCO!DLfvQ8;{P-fORnQzN-TasAl-k9Yt2+>Z-3#k5>}2Ao!j#vnSlD`o&KfQ5$H zm^)m{D|<_lM*2a!bRV85DCm9<;z*-cNibF-azWL-k!nTa`&{h%ni60x!Wzs~=&=r7 zXvOT20q(YvHPM1LdM*ukHEBbOjP6lJ(c;pM1Sub|^0Z|Z`>swirky@^gPAMY&gK9w zfWIm#su4LNEjw!E9L_tYNx2ZT=0)gltK=2eTmsCUn>%$IC^-#nL@@>7sG__kdzvwF z?5`GnbB-1+#!1q1gX7Eu*L}zV@7Yg${3l8`0|t})0-{{~3i}zvzj_a(f=qee*PDq_ zcKMP?V;`QL=-8KsJi#??g)b^8W^sdsz){lb`=(a+<)Y+QyliQO(l`0FkIqkU5qvf` z<7)u;yBJa1Sr!R^$4%kokY!h}(a2|1OJ$<0f=(eN&c{g)3$!!d@~ zR6z|Bpbm-j=2*moBf&Jl$0+RE9)w8{FmXhTl)z+D#R0GcLD&t~fYgg80f$Omji%Qr z?c|nD0upp1(_=lbBfk;S{3cu;UpfKGqa^f7@{+U~loUZ)0b$XMaFRRiiv_8v6dQYu7k`!Ey+R2ScB$Lz zQMf8|if*N*gehD_Q|4vARlsQXM~b4;Wq~O4gnRipY&Dy<6=?M7n?yxx7eo{*q|ipS1)MS^7fN0-(Nz!+tXu4=eXPDEBq`=p%viuz zBQ-67(!LAEWe%#cKSw&XW*Ikdpchsx47m@>zSBLRM-(`7dd2eddI!2qJG*D>J*c?6F)VNVCUYT~)fH<8 z#)pJvHx{CeXoebkJ}544xS_f;FK=FevtKY1tgp;+K%zKe(nUAtsvmu9VVK`$z+ zr9}i6*0rRz7#aQQj7>)a^3clu5NUIv-qU+<(v|Qy6L3K1oh7R24cSj2Uz$Be8eX~L z+%*hOorEGHND6;DD0gi=Hs!plFu!+X%-|nhmkJqK7tCJv4(EC@XtI-OZ@v}D{@pG5 z4tnal3xfga0IZ-oIqW=J*GI@yz#T$F2* z1Z<%X@MEURpe?(T3Lbk`#GnM@rCo`8$~$-Am%J;(vJe4-l#4R^p)BX(8n)yLZE%gk zG18X+Pp1!B@uU06(5S9~vFJSVG6s+cmIyfH>_{WRr&ckMVho8D*~{rd&d&OoxT+|0 zo&k4^RuxUvF<-JLq{yBb7&wl-kA-Y^L=7+UA=M-8DfYTW0>-OxT$J|4f*Jt+INcTn zW5Js~lvnlznA^gdnqtm0(GnUNX+x`ul_Bhm2Q5L4Q}dub$g1@!pH#@hz&G?1?m9S{_Y}wu4{!A3)C0r@&~ z@D6!us272D=FVZ@HC(NbZ(t*%y>qA3-zNXsk^VUFJwAH~H)7_BF~VzWZc<@L$)^G5 z&u3rUccw#Z{GT8PC>j-G2UR7nH8IyR_&>W?y8~?-^(P;okhoc_D6n)8@$Hel>({KR z?0@g$-g>f$cjhyJNLyTc&$uf!-Yb=C8W)GF_t$@CNPJUQOhZnlFL$;_Ib(t;Y9MvN z^?Lykyu7Gfk8N?P%b9}DceTs3vEKt=OTPo*gq*u{(51<5!-ba2P+sX_?9B_!&vyvzM-z%K z?z4v&$D;?^J7kQgnj|i>>VB}PNc+i@oUuY)F@>adeA7^&xfJgyuPmW?g6pDYq}5FI zb8e#0Y~Uu2K7 z*(=e8npXGQmKKm1^MqLqfes!~5yr~9AtG!&wCa8&br7Nl*#N2sSs&Shd|9v;Wb@D$ zn^I^#U==71b>Y~dFJSG8aSK^Dtzi+{bPQ{l=+3;qJ!txanhH8&kO+Alw>um`Y=#WG5})UNV+QP!!%y#qv7LcA#hYTbgdEpKK?$R%5hT*(Kqv_$=n2QCR~xVq->cIdoLESsZn=ibfjb${nMMSbsK-v^IMuobmO zB?*TF2rx{qF>Zc)>kTioHrnzA$~5WYmqAc z`4P%V{N%`CPQ-|OlE(h$7)t-(TD+4?f0711dD2tEmwpas=EB+>g3M4lfyiP2)#1*@ zCp(Ap<(ixbB9+_Xb#Bj``6b}wJ|@=XfvlR{ovBp4Bi+hE+#}_xgOYle=>v5z0kqMPpCu$5jtRfFE2BJoTmoQ*+tF$5d|bqcxqWwO`hQ^?V*4w#C#Bm_&@#mu-U_-#!^6X#(X*1H>`zOjeH3CnPV8FGDp3{YWH4XftB!xVaZnRxA-56KUk9pFRz1%D&q@e+T)c z!=*}9mDjve*ENqEHT2DC53I(K1TbFi}E3_V?X&!F}7> zF}i-?$?e3}3Z1UoCv4YuH}9zvozp6LA#+)kny?$As1-Q16)j=xDWPI9*?qMo6Z~ma z*7ECCSv4yZGa0ZYgZt3=iTc~y`}mfl3BUE#v@QspJP07> z9OKYEgsk*F4iG!xQzGQjs^c2skXP0FN%ABi3SVOrEGhut2L~GgmI?kH1M&79BR>he zW%|y!-Pj@ud~WT%yNYkg&dwu2kAI4^3tzZB$pkNt5z5$f$_h)4(0jlG?^~6;Y%wC> zF|PWURk=ClQJ}BOraKYZP=GOlH?k^%mE*4agQH*!Q+hC@%t>pj0TfJGBCQtWTFKm8 z|IF}?ykhE#v+Dy_jX$<8Gj=#P2VnXh{0jc_YOAj00dD1W(@t{S(&Kd1IMWQ0#U;z{ zdhDKC@-@2;uhK&bDZXXL&YiB9rs84ZfnSgd4HIo?tplBCMWQ6iwvDKvnwgvc&`=&1 zO=a0r%nVl_p+G8tYBTd@EzBe+j5!Q znyAlT{(N*y+^2n?NuP}*O?7t|Yn>3)+Rb-o1?v)9O0F-zg1FeS+rhV&jQZ6sWJ|7H z+pA3X#%JEVCC5Uyigkrb%Trnne&fy#=(ncdK5`80Nl0uEe{QHU;5pegsfCF=8;?E^ zo20dqsV~^>C%JJQ2J14!K*A28r}v?KDmf_G)qyVQn-MJA4Cv9_j!O>2jXDO&?a=!da$@Pu%sh)2Z@|Wt zU2e`;Be$mF;_yo$L2m2^Qgb8Kf>1*!^5deds5P=N*q3(R?~YH;j*kvMAHRD`{_JMs zu{|~+pX0{F5$uTbB!%^ioJ>pfL?2rz@Qy3&wMPXx5!@Vw!LhQxlEP)!+8JEXa>k8o z{(}?Rul;CAQ+M6{WVx%mH`+?%lPjo#-#T9j69v3h%wXz1&r>lz`u3o{k)~8c8>$SR z`|f12I{_tUce3kM<|-7CI6a}8d`Oq+KBZ3x`QC%iK%~T9!Zf;L* z_2|fBgW2z#Cw#iY*th_~&c#!`H4)?|Sn{rfj~)1x=*(nb-Fi+Hv%hSB$5wpGQoH)E z)IeOva_s?lowjih)^~7reE9C;@uV)dz{Qy;8_9*)dOBH{$lnJ4LDXB_q5DQ>U}PgJ zIa{oy<7MpdudwWGOvmXx-3NFC%zXRK05kVH-EC(ua?Sj!+RP~4`w#+-R4tZ0|D7oj0zBAZrmE61TcMuh`5^ zIp<=gJ0NP81dx`wE{S$v^8MiHM+oT$P(-}EpT7T01j8xyqeM3Bq7sQQSCl`RSn+|? zRYaHNMMeiM3MxwUqOQ2O9EE~ooy}m9tQ7;4?(^^ItlI8W1koPRw_c3;Zs_eebThh z_7R%WIVHy3^gA+Ck{0#3^JqY(q?Xl{oyAy#3LP*04u5MR^stS?fMXlqO^H ziCr<(UyraK!BD!t1~>}P9$O72C^k#Gb#xD`l~0$3#g>GQN>UeoGW{gYDL&I*Kd$_a zS!iW-;Dw*f9&gF@rNyP3 zPGQvU_bvm$F%#Sas-?imnbvfc!AzDrN#ZFuydlrHJeZJMmsKl0qPNFl2=u2~VR+4! z5Woop32vpy*r^;`I+Uk5%mv?%k(X|Lw-PUwSuf<)roVN>$ALQii*mb4hxXQKXnO$s z&>i~Ezxm7LDh|KF<5n%$x9h8!n)$|pH5Giyw55QBB^T#>hOVX8mQXaP6-%$9vxR{t z5C0;asl2D;KI}EXdfu?2HE@3*y@l)vX+KL0yPvlgsB*$IOH237d`)y(5AK|9FYJK( z=`V~~)G)4QMXR`3SmwuW%w(#ezupnxGC5&4rCJ5VYE2i!( zr;dYJvg>GDPspj%n%mJB-zO7vbVNF^lz`LqoYvSgMZAwh^%dPL3--4B~+N(2}M^{QdTiXUvv zJrT9`Kv|ofCE6?Ek*;{H>)YtsxK-?mL8y7kX9PQo)DGCy=lV$n4KfrjS#H%=P!a4A zT}yRYNm@p@b(&QCl96|;spP_;v5JzM8*13JtPtK3d zPrk&=@4Mm*@RD1NZWeRe9JXfuj%#SNI82$jHA)2Y#;hnvuM$DEl5l^NZoH#^KRhM0 zwHt4E0g`ZQg|hx_&Jm1xtsQY=Y-m6HXpgxY&jHNfr4rytD*=EFZA+2kf*VeaRF9Di z-J2|A9ktrX+Fh{|{|hLQ%L>-q8oWzMg3vZ>)vjc?3tq3e|Fe^RV!ls z5#RYU65EGt?7zwLDlGv%u=SKRH%%a^cneSAJMw{RgRnmMW57m0Tr;b&pISSKH-DvN zsotWd=~lY?Rkz(VEy^p^jK8(Gld+`}54|oX1G*6;u5sXZ1!M z^?PnXyV;tlt8^pqk#>@Nv6UirOL3c@eT~RAk1fw6QA2T-v#>o?1j%f<3d#ay^#iLJ zFs;30_?qs;R`Q_7ZhJN@`kPhl#I_}TR!-&`iut~1QFTr)Wg;yQdq*R=D0=|}c0&uc z)e4mrT`ZTI%Rz&qZ3sCS_%R|RVWMT&b((VygzAvF+IH>J@^IIm#X)@;GW4e{`_#7U zPs?MjfkHRVHs{hvu_l7yszLf3@DzS=6ta0Txhz64g>;ss+7p+wqLb?PufO(v$-+Q|*KPOitc8)5!y!ru*^mu8{7sEMJ6Yq3x9??>5ehq%zl14Q)Nse|)Jv z+I^R_b-o(p0e171u#$6NCDCr($5Oroj`Dms$)l@8UwJnVyqE*Ivwzq*UM+0|-kfNZ z0e5zsBnQZS%e>gX*~a;?lK%ybv-4T8e-g6@Ao+?`m8BFZy0%_ABkrt6F7>>ojmxqx z<+Zq`sw9V}C)L-XS46R%o^@5pe#sqo&`ZWfmUjNfS{S(xIkdhU=zWqb^)gRxyNuT;Z4nKY%v;aF6 z8}{J};B4bb^oab=^Y1@(J;?jt0(ph)%CPfXAtZ;g)h-M}%{-L+QHXm-rJ|1MP4cQM zYCfC8Hgf@FT=BvI3u0il@;;Z^0Yp^= z#x7{hpq>+#mpSE9hg!{Nf+=(%0~7$M=mo3Xgi>X{(s#a^mMP18HKDVaVp{Vn=5Q7G z-@E>}b4E^&KYEM_O|0r~n8=(nKn7d#>nyklA;(%8XwhnCaYv@daLiQE1)A_Og2ss~K%U=z5%uAzE`nTgejO-rk4F=jVbFxZO*LJt6t zq3opVU$h4E8b_-i8S>6oOfB3z*^dt=GbDIZ z%#Uvxg|&Qx##o}bwSF+8#tmfQh=FZW?}rvz(FmWJN)!6pT7-KAvb-| zog%Qbzwy2}tUvz^qT%~?pxy?g|3FIv&M){TEJkTe00r=;wT$n^ffVZvc@}sT2%lGEI!Tyj|YFOKKE#k#LKLSFPpB|O06EPy6q_O`wzTw&_YVRb|pQM3L zp8Wjq)t}4hIo}kA+*BxX0_y-?(Vvn>3<@XBH93LS6S&oDEdUagJvx$|-rLgvnB^)f zZqM&dSE}COZfPlQ8Q>$sZvg$Vsc6AU4^~I*u2I8giRA@%Y9UO`z`a+@*P+$5gy{y! zeMkY_sy`bjs#`CDu{x@8$rgj3_4NE|oqf&hUcSkL+4zxww*0}0@PU*|%aS85(COOx z&Dzwz88#j5396qpZ7hraKUT{GZuNgjL%Grua_F%WTkGJ~VW9>3$ot`hc;-puD+{ti9ed7U_?gDQNKA@%6qSoSMrF|^i z^;d%&;mdE=fy3GR=hFCEace%SJq(-^!K>{{QSbVWAnBkh1J^Kt$BcHT9gb&+t1CJ> z%|cTJrj%6n{g~E-!@k#4U9vK$r4w>$H^TeFUJVRjSQ+43G{ik@?Xq?!lU>)L>2^ok zSC8iW>fPUuw9)9+vpOLJ7346t1iqOI)YI%@yUm4d)(F|$5w2Oo14bgYK6xrHO`VZA zyDdjNq&RX-zV~KeT3W3^?4ybe_Y*&1UhVRqr_QroVHZO;jg|eZ<5cLNXX>FoB=V>U zxT~md>j}Bbe}*LJuD<&@3A>vyR5yG1<8HUAXZ1RD4iqf#2{a)AV4XPhM<;&q+S3ay ztJdT;*1c=cSf;~Ec=zxyLB2eEq#1ywhN5|$vDXwJ>qL--Mvn&Ag&Xr#4~rIUG*UFw zS3LyWa8p-(eW1fgN<2E;Vc%)`sGx^_>h+-y9V2-L0Eo{eTo8JwppLW1Zh_FZEIZGE zRK%(WU?~r`8C{RNNAz3|=8u~S#rayn!_$*BJaN$|8}Y@(<9)f_xD#}&#hJA4x);lA zC>$xWLsak0_$O{H|NT%iAO>5H2*yZPx;ZQ|?sZqE6Z)o%Sges>3A73S2h5uR4t?+b z?{*EyFt(xx>$giD+(wpfF>iqiA+d+29-iC=kw!goG1UkPN|b}gY>t{z!*QEBfQye#& zCG~tf#zQ^Dqma|w_xKK8$cv}=#Z&y^DITUYuB8TWO;)&!1+9l4?~=c{YfH9fe91eT zy{EZrTmEVtt!j67GTFtT@gxMYS~DI0^Wb5bC}hdXiEDMPv;k9X9UbuY8P#r21`ipB z$6*h4pDK9jA(Gy2(yUrTEIKm`z>Hm&Sx3~Nf3LHv0=N^Yc+Fj%n3WSuuVKdR;b4=Z zX-970NzH1h7Nan@{jxWk`^7$(Q$?#36Q){3hSd|=RbktPB`w~?14HxlwRC0(1OzP= zq`3nX9<5o?==`E~L?*7t6;*oosm-C1Uva%&*(Vi?@2BewFy4S9?T@r}by<(F&YcRj zej%}JP172ue0V0}`bw`ga9}*6*y3Jm$elMkYdqgjW0(?e$^YK_&)@!Iy#J@Yy`Nu? z|K%^g-TTi8{N*?Mf7<_t|Hp6l_xJXG{_f+SKc61|h4264XVKP|_^*HL{mhR4k~qlz z{-1umscJwj%vh>1lG0nKW_Y#k@obLQovmp__t({V9d^HT=YW5uug5pAUNXRc1X@9j&Tdg|7^b5|qnU9|HP=`R$Q)8l|d()i_|# z3-;eJQ(aqnE7)OC$X1xo@{)-3n0_}5{#j8rvPzP4!UTS%XpQGvR<3m8nz)~@40p%Go zo)q7lPvd`RBC8TaC>7+NUz3KZ0jI5!GhLbF0UDFI`TMUO&NI^!i zXq6*Ht8NFPEY)pY6T<`*X4_|qvk-Y53VUb#iizd>;S98IX>rLi%1wXJ{tU{6pfxXO zW$%-cSKMAFRxZt>N6veP?Xi3dKUA%^wb89+YTMgv=hzgF))=5w{HkEylCa}%Yk#hi zTjZ0~eXw1_CA`BRf0B-%C4@ zqSZy&mjP$Al$v}CpyTft*tlpkq$*!&hql6<&TeSkR7_7U{}~+Wj*lFui}K*?`0(Aw z<4Ij^LA9Ie&-O63%rHsH@(L0(qZ^^OHtzuo< zt>q~#Uk-hE9e0Mq#dZ#Wx{lpDljl8lC-|1^up9G-oe|l&nQHqVVWxU#LQd+Y;srO= z!k#@-Dny0wfu+R7jCSOC(HU)|qBG`6>{sfd+!o8_HJxppxL}gtm4h3MALW6C551JX zn#=34k>gggF*n=V$xF6?dV*J(aoYoCii3hR^8+`+mJh?t((}U@2@x&$+$x-B-@f}n z=lGH%qwd!Gd@ToDOTmp)TW{a)=RswrkX7YcNXscXV$D4K;F)f0W|Hav_u3Qqp$KWD ztgZ6!ypK`t5^c?#3BwBv6T0G@SQL_zDY_HNjKmFj#gyVD(M0a&)Stoln z*K$1e6cPAjup$C${yJ2VJdYlj#!0{QhQOvBdb*$$JNZ7_spp>Sd+g5LhYeGjYr_(Q zsor;O3@zra8<&roNkP5~*N6yNrlntO+w8GuiPkU(An@#)|87u z@4h^U-vp;`4|@62FeY(>THU;B{MM=rt=zd+Ctr=-dCM{+d-l5nXkNKL>1@y?12yX? zHf6|u5$0hRHuR}!Y@JhZCSTOfHRlR%Fv)1o<5~PkJ(eAV2L9$u?%-}a{75cc=F$oJG58qf=$Bses><`@O zp4bj8molo@&lh{fE$hh|!bySFUW%gIXDFFk7?JdHeKzmYwGXyN8m$gA5k zV#C#nAd2#!^>svMyD(dbHAUY^uF`y$%%OzxSa!WCuXf)Y8!jw5=n>~*z?qr$R;|Ne z>Rmm*aeNWOMiGK-YGL@v4_8NmOZK0W0)G&g%w;G)I4$Zdd*wG<`LC2%$JbDz9KAe* zSg?hhEe-mj@m8BKe^7>s5@f%oBvl1luLg0^2AQos#zZR_-tKz8Q$QM^wQD}_LOmA| z5_OxaSO}c-bP-l6Oegz)OcH&@7ce>u!;z72Y?3&Iz7tkrW2e9X_PagPUu_x&662tc z?x?Q?M0dPy>8=`BjiIPkL7Vn~WTJF}wE24`1Jr_q9(!aWl!Ba!dqsU@X8*DTU_w{I zIVK$B55S3$g1yI}_`0C?9GF#i`zlv_GzT8WV~OyBu7QEjdC(g>MF9wcoJ~7qU~ov` zsQ6Lv@o}(qTf;u&9({;GV0Msj4)Y)Z!7EOJio%3L29(A9X@Row$Qr32%Bh>m-W2Rw ziVLg>BVl1?M1wLblTC9~r-3iNd>eWM4*-t2@iYm#Wob81#I`!mTyEX;g_jK7=Yyvs znReaXeT3^mCYsPQ3L@+E;rMBxtWqCf=_>K4ph=F7_-Gey25UObE$QXc7E8u=DSWUB zHw=Oir|#7-{&`(*XTIst8H_u4N?ByRCBr%?FtYN!rm9ANEdkjTaH4WsIWSHBew~Od zF7u?N6u>&-^Z#KYjz!5AY7Y9EYpBVQRx~6c8AHC~-+C<6?;*#6^Mtn=xDZ>hnqIqW z!KOUpS)9x+sBq{orM-Vwimd`o0{&4Q>HtvSyxAc!-vuRP(ov+y@I@FPH)g)r*rkgQ zA^r1_XWe5NQ6f>SPY8kgRaNi*^bm1k`K>linkXHXko&0 zQ`;6C)!<14B7!k?DZl<4OW$;`wYw@18kjI!SZ5!qaWSEoi~~N4``Q#epY#a3xgr_Z z78Z`|4v=2sR9Lbc1^Gf!uKwX^QA$8ney7!^m)`=UQAU_}XjL_;KwR7%=u~P+2`USl zABS)934J{7_fW`QljL>Dgnw!$*n1E%Js=R{3UUoae^|B``=u2ors1q#=rxxiCJ@1>RCd8w_<^bhob z;(*mv6yn0YUHMu#m-UA0Xm8d5;YKO6pP}rb;x%$vSjCgBgFvsfn&rSlEXg;;0b&~d z3lCy>uz2%}hUq$L@;jX1`Hm0DoU7V`B*Qz-F(Q)?*in%Vn!Ws#tWmvj6;%`zEWDlb z=(Oxr@qtCZ%s2y(iPIf{uX>I)P({jriY>6THkFGJ#bN;Tm`;Y|b#WL$|I5^6rhDI9 zCW{zfA)x&+oWpXYC4-3>#v=E)V#-;6R)qP*r{X1K(d0~1y;^6?r|q~M)|R3tQ~xJw z;&fTF8R*9MIhE{VaSpF8X+81SIvXeFq1QMEw`8?D6`17!QB^WJNOJuqy6J8(#==r< zM{8wAG?LS5{a|6jOk*IxJ#bN`QN6pAG&55ajk~}Hdnt$$H(F~k=x-`L2w93~&J_k5 z68D%6ZLhEAq|`$p0pM&9bQW0HX6byHp)NBEOI=Xx(<+*vTj-w@9lE(8aT3#A<9Qv|41+N4=Ntx|mS< zJG1dNh&Fn;Si3b-nK94BZN_5G@leKyM~S4M;P?LFd9_a==<|lzp^W>cS?<%*J&`-; z)2#Ocb)w2e$xtIkp*^BOrfX2h2)SHZhI;Gc4QgmtOquzrv4~g_<1y*e~d0>juw7j%nlYH zL7FJ}JW35Zb9zJ-)7ehbNP8H`VsP}D6s%_Mn-(tF7@n_1TM!|BjNY$r+71>JKXT)S zB+}yH6a12Wm%c8(RwOMO&^t15M@LYgB1^6|lBJYBUj`k1#=Li#yMQc*ssWADPiS?9 zsD|RVm1`fDb0!@_1!%ecvX{7Q^-+IRy5xEGe%t34GM#jEM=wb~SO7p$W| zP7{jOsZYA>XSVmJyL#NL_N&mb7T{7%R!zASb9eP{xTO_!ekk1O$(U(Wf`|=bl(ikM zbTCtv~YdNn3>r8XYqbt(23`L&PYA&eI~A8CSd2S<9DmbZ+i~%3#{;bziq0 z6Is<6lduKP27U zS6q1nfk~HkrgC`2aZfzqbbpOI-soMiuwzb$j_df0Xn@ zHEN!ufp;qzKo9gPf#AA``%EUyk1@FV@bGwCRtP7Gdm@SNv%{rTp83JD7SP;(9`#q< z*sNA(kZS!;u6BaeI7+K2J4{8P8u~?PD>YJ zc&hgB2JY>^8NqEyiiyff5`cnrcX4j>DBh#dFx8QARQ}+J(-;-t-dt7Ubv;S?Ag6QZ zx8|x+pJzfkQZ)$~tLWudJn3$eklwN2@2Spm+yHKQuv5Z&D{4n}#a#5OtoF@SjR!Zi zi(|TDEQ;yi9~mOzIv_3l{+UNIF4w=w?s8V;h~W49iMG_^(MPoxah1bWq$=jxc# zi-8U>vG*@cx`)LU$FVib43@>$>sIcRrThi*@`1LaNxlvpi|k_RpI9%cRXz>-mhe}e z2N}X9%(8AUPMJ*M#H48LJ-_Cc-IGZ7WSK7BrZ@9sW>GHisk2|A*@d{~ug0#J|C8%F3EQMlZkuqqJ{yp!d3ei_}giX32e) zw#rBC4<4s$_JRoYDC- zXPS7XhoX#^Op^#P*3FM+j+alMI;5wM)EaV!D%v!f_Da6xEe3sj3Z~dZED0bb0fm5O zAboTV7L!!ahGaJFAp?=DrCjFO?vM^;%_TD529erhX?y|!zd1&jhpWMm35`SRAxfjK z3>H%)=_GMqP`d;XOc?t2V{&4=!0QiDAf#s4>|l4NE>=Css}(m5<0;jY|49K!(45*_ zd~|`&<;so(kIB>STkg>w>vg_)7b^iB^YyAXivy0jcoaxZc}?0GP9{*osz zP={ceBBKA?i2i}h2M?OAc+cgBX8EoIw38AaY#UabPLs{RvU?qnGiW;*eEfwJc8>K| znyc91uE99ZbJEmWS4ySu()Vgou${vyK-KyT9!jfe&DV&$y>x?cWdRMa*j~^gr4KI> zNg-J(NtR|9#X@!!Ee;z0W+$Fa^)PAOKm|$0fyL0rkb^B$Y0QqH?Z2$g%qO^{EdyyY z)4+CPA%AW^I9d5aDe1W9cViXb`oErH9lk9HE!#f#o%H^rtO%pt z%qcTg*{ruKp@CujdTGvw?sLJwygoRE^~!;$y=>ppTp~kopIT1hKCnd27oesOw2=M(6FNlDG6rMXC7V)X+`eWKsrgue_-=J-!-1E zMM0%QnA91!=UVL}+&7fVu@DZhsM()#l$|xAaowYJfN*N8d zexRDF-k5Rk2cS|Yd_Z2<7yhWMp!p|&vbbf<*^phGKf7=Y7*uN*&ollTIv!BvVw~o1 zLgEXyU7(Sy0VKF=n@QE0Xa8iA{BrFw*MWBWr>XO;&rFn<4REC2bxmgs>-i&@8e1-n z05JgBs7Bu{yNM0`8^vB|aHGHBth|A!#_0xkLn8qWDrCg>HE0-HhJ#dhL9z*fvckH# zGapIw4F~1W*_4e{#V*NR?8(*lN#L(&@xYTG4h|qWax`RE>RQ%zZ_?W>&WI$@iNJwtSL`+zC znPkcJry0cBE}AQiuN#cncyEZ!{p6YbKGBp5G#3C);AePCY=l~xaNW@_kLKVxvSsLS z0T|%bQzPl5!KT|(?%C1Fz9yTDQg>|RP8_JiYlIbj?h-OZUK&uqdWemb5$km8tsIUW zf4cYtV$?--L_5^Mr zAHlD-bPw+1QG1Wluo;PU6{gFtmDAn}cc0h|wCS*$TBzwT#Jxn%P-2v1AiDqH^0Q#p zgIIv%P+cu@8sssU3r)tPDWEWEJ~lg=*FavE7x`4HJL_F*XU|Y?qI+p}tJ7uFdqhXx z#I^P1-Vgz3#pSX@+{&ar}Kz4KScwN5x;{H0@?Tn+!BlI{( z3JI%O7!v=jVQT9Rt&ZGMwSc~Da-I1{SzoA5v;q}kjXk_F^%UaN_X0Ai4RM*U;D&@|!MW7+M-WoYm zp}zv=6QqiW2V*M-*>4|Bx|Q|bG%184+kdL<(xC-=v*Q5F3UM&>UIM*Cid`-Rrss|9 zfawiSPR%&haYdXp7N$nY#wY7|ZbQ@@z}#iFB_yQ3_W{?twFt_@AIuE%A`Al17I2dd zId6%Dwa8SaIz%H!|CMP1!%#1WoeybCO-{o)$A<(hF!eX3GdKMOMby(?|54K0^dlwjtqOeyhKkec z#|!W)jjNMtNv^ZUnfDa1C^Mbt@{c5^UR%E#f1Zp`g_IBBpHjP&na5Jp3O4F2`;5s0 ziJ40j6q;F_#I##A`b}xm2$b?7DQABSx%9Z9yVh#fpk+bu%7F~CO*H}R(93yt-W{C2P7zG>b+rTp>19UI#ieKkLC2(SgFckHP6_38frl-_L8F;1fbHs~ zfg&ky$_uS@wz{nA)+UB$8T%K*i_kFyJJP7X2zMtf`XLfbr*&5$2fu9+F1pe~hs_B) zIyhQ8fJY)MQJYhs56>$rCeJ-j5}{dK?2cj1tWnXN4{wn-=RmBlU5Lr*?3_9viAjfG zn6xbc{J2N=P2sjCl*Z5i>EIKgQAXcRIw{~DAXGla6c z$ZCPPdPjm|VKj%Bs?a~puPDM#F-WCvcS~ETE=y4=;)P?XF#+qwm)qeosPD*#Dyd*- zg&hW)!{d87LKdZ*lf3>b+-K2?k}|I!7b0+v#A+EAdHUFANo?wqf)2A=8}Mub*|^hj zMlQdJF&L;q*KN5b_a}&Hv=3Ug2_L0bK>0RSaJ{_EBwdHq)^A|8n^S*<&b%%CWTmS>Ed^Cn z^Y^v~FJcO7t8K7f>vQ;#M<3@4T6NrbrFvFkNTc>V*|0Q1XHKmRShJwc+ui8h;3qxt zf}SI8@D(hBbq(n7Yo^{riLB*$Cl&`$Rx4JL$dXd(h2~v`po4~yje%>PgHkBH33GU; zV;ctjjhH0r7Q_^ka*Igr$c08|rV#gLOjQ0lRa)5gI$t-RI-bTLg&KaCsO}ctf;HFc zo_rZ?`qFy-qW@JG6$P8@;!h?Y+%z)!-}%9DKFfk9WSgCy6ba#Kp#f%P@$WdH!@_^E z=INFzMsp=W98Un4+^XX#yP%UKBG!R8)DZvS+$yEopoek@4S@%hxR8gMox)8teK#5$@9Wt}l@oFX zS7liMFPK>y;RcwMgVDS3nnu;etp4wjgw4z61{V^JXF2 z8yycA;0&~(A;628H@HNemICs{>Euzh2N6GnI)Z@&Ufv!&vP}=Z7Z;mrS1)2uYftag zf~T$lDk!f+Xg2qXV{;zSQsIRp*xuM!lzo;ypALd3oD|4y_oH_8NOzvSEMV4h#C+^xB;N(P&ikZA5M zH3l$6g=zOH?lbg<=r^KM9tQ-G9-W=0YHl}l2A52DYu@q~8%-zkrm^DRq4pDj4!I8k zHJ$?|J%o6pY&+31ao zc<%W6VVmLlV0YclGh`4*;=mj<_4&lZbCK4H^yxaMp4boiNp7E(1R3eZq~|aN_q4B& zl9Fby=u|@NLx5|ozJ}(~psgmM?jv=Wu0y@(`H%Ci+^~T=%gKCL73scnUxnIH?1kPtZ4Ag>nhQH7$nwx=mI{?6>$`lyweh?!_8f_fnDf8xGBs4yX4GHs4baZF)dcEPc;_Stj`mZ;2W{VEBmI%RHHp*Sy;ohA z0G9R9C#B`u*8rO^@wvsCl|Gi_V9ejDU!`;Od{;qSylh zMt9+LVlAvbRL~=Z`YD+mIG-|~QUkm!8ci??QXsopWBbJ zLF9w?`{#%*A)e3c%k1b2>KP{`yf<8@({2lQL=7D&xlm8|VP8IuZd1OUC^P-&M3L0! znF^z*Z~kx|QlAHf(LG4}-TQsOV*XXXvLDifYdGptV=x;h*P<^e(=Lt&ve7_)$KhXanl2TGh=6qJk0k23V^@Q zT3g~qH$J`};^L1LF47Iw_lBQBGFQRY0S|@17gR~kIxsirkwBm# z4VuJheLL3YR&| z=lpqzmxJ+OYq(k}gYeeQrm~^jS(nG`EeC=43m%9``Z15ON(}26@kz;cx~Xz*Ja~LN z3>(5T^8YYZfA-)Sv;hk%<<%J?WGzK;9tZh`vFq_bFO^1}L`#iZ-Mz843gEmLCoC3L(9$WQn1st}qq#e5uYBl;@A&UD^X<=aLDa$C?V&<|9Eq?gr1W<4(&32L(jX4ql8G`lm`;%b(ob9!a#u z+&?aB=0dVfZd8Ogzf6zG0vN0}47 zt83jy0-{<4c?V)G!m&CLX1qLt4MT9B+TG7#%9lD#8`gLZN@wHO*!esCbxEaCkfxs) z1U}9=&@*c#|H=3|-_eK-8tIxXC)&d@@t&e_jlj_Y3a{9RaUiIMv^$gVRCb0vR`OJY zzUPLohuo(wsX1zaiQh7)^-)YT(1St4w$XDedzwSRw)vz?u3ol-F`mSviW#;9$Q*tM zpmP80D{HdKxf+zj{+$Dq5$lwc_7Itf;%YBzV5kSfIp7om}~j}Xq} zt$(^Z>jepQRe7pZSH3zvP!sIT`CCcO-n3Ew%aNfVbpY*D-%<_4*1}OrUjAV$3T=et zxJbJ65gop&nS*%q2Zpkii9r;x7Jjg!NC?-J<>YQ|8~C$&rf2MA)FS6B_ao!Ko> z_rD=(-W(^Q`U@AJln*AolyOi_K3o2HNJWPOd2Hyg6iP`W7Myc6v!_8T+=cWSLj>(` zZ6fina?qWQIal29R&Jgg6emeItz~5tH_SNs1nZ^3BMwO2u}}QN5%9 z=HATCIsjOQhtJhKJTz8f*!3^z8u-5qSjA@QT8AA=70PNX^R&B8`yB)M&xG*;oEdU3 z6jy>dNk#wZ@g$Vh;~%gu$$JDaQR&MZZEcB*YUQCn>tdw!*&wxVSKo?mvbcKR1TerO z*aLAZi(-;favjV{(JRcLomDiTt+ZEe9^VixzRcp@?5EIM(9amMY;kwvF4*+X!q=2* z71+8X3)iySyWC_O4{(DOauAO*Vlx`W&^9rm4M35gRB>hX=6$hkG?EB<)2&!?=YsyM z$`1`aiC7pX7fc66b`@X1E@t>3g@hRik&sj76Y|nQ_NoJ#6UmI z^CDXCWFU6DPf%bpJ|A|VM*;vd_%$t+M!PgKN-qqglw9+q=?EreL3e#KsL3BXi0}^^ zNfBm=4tzI(m?|GUTM|?y6!TW839^}`ismNXR9?mGn$r(HS%9CQ z-_Fj{#?8UGUUf7$Ex0K-RD<7Cm@b;%t?#X#jARt&O_3kc_x`D}#<8A!AL*%F= z$HjQEVq)z?%NO7y1Z;+}-&w0$4d!T|>-@zX_YJDVIrK{%+`I#+I!Y#ABz{O?V<;qcn9?L ze(8}0OP`^iH*&SsC$e z?7_dk0J8|1HRf^-eTDnCAR2}!p0jNF6UQV>T1DwPUL`3nNl5AXZM2kPNyxS0?91#p z9g>gvoi$xI8@UgxF++Lz+E3<&ueZrh0Jb@=e=dDW(RP%}W9S^@Vnk}j<*zd_CQ;Z= zX`&0Sn@`5IsOFgXuo3AEt{LQG%tOotH8aoNy>w8gNMXK0}u8y~OQpPx?fs`GjV5lEzThv&g4 z5F@~|tu~pO0AN%XzRUq%WSNQo`{YzHmE@ta&Ez{4-IYP`qDQtDcB?% z$|d&dLxV&RSW@y+cKqo1czvW`g2g)gGAmGY*(6YO)> zrAWUm;JTwCpN+#R`9_Zdt2)s*RasmD|Ja#G-D~&+asw&qf_7w$zWNqHvqi{Fv=~RdK4i4 zbE)UT@gyZVzp6n+KGoIGAxx9P~5o# zmvSoT>mP}MPWidziMqRTzVMN}5#I2@`@;q}?@6f#Ay3a>ez!?s9M+yQSAM0T^Jgaa z*j9*v?Il_7PB%H3xVl!fS5sVDiB*juKJ~Xw?BIuMz7j9(h>Q!-{Sz>6BawI(sjdLu zB4JOCy0dV2YcIH{uNx8t=|tbMD7V_Cnim<`(x;sg2} zm>AdG{N{>O@Wl2Pvt|EZ^a8T1lIDn{HLX>t@*ZUpE7%G$dZ;{^!fHuA}<-l3|7h|GeUTs~BeV&5o_fV*Xbb~pN>BYCA*jr)0xPZIm~eqV1w zL9+KxFR!m&0~Pu4B~gJq-nzT2aYi*CQd4KH-u#aQe_u-<)(`s69Xfq;*}nAZWkwFR z|1&FnCa_E{$(13grBk26Zj$;H0(tH>t5zr681>w|*o57fh*z<5xY$(p^~Ger_3p>Y z<@!L^xxkWN?QRkA|LE9nRhr&K0m7~L9% zi6WD`a~c!!p^+6Dttw1Rd$LIzEAqnT#qW0eOz8)=syqc8M6|yy_avNj*8~L z^|_bboDbv!#T+7E!A+@NC{(bm?|lGJTy!xy1OG&Fc<;^fG;S0;w%40>Um%R=gld^b zUd)ON9_Rgd;?B{-f-+p^2XL{A55w%fvTR-kHUD2s&V829=gpIUwqa6Eem)#?_?Oq` zK5E~5nZnVqW{BpxOjUJpmnnrM-o5^J!@=WAHU^c;ti6QVYAneEN5+YNXl$MXg1=ag z@7^xk_^)(>MChNQEXWGWMxilRW31Ecsa6&g(Ts8BP=6DB3+eY8k+wvdmx zxpD82eJ*d@Sy<@~j*kj=bxOvfBhr5Z(IvQKi`}8QyizIPG?RY?Y9*5|BB@)lt|OP5 za10v6g372IANGkIp}dc+*Z2=|?U`t6j`E6mOqz7@UqyCpJA=P%uzEC6cufXE zpg5YG(i{{ok~MrcZ5i6dcU05ICJ}42c|!eu{Q)PL#Qsc;SVK#!2M9x?+>Z&7jt%q~ zWkcGR8_l=ynwx2W=L>>xxoUeA_7ZeFC)_m>fs%pDf;u4CH64!gF5q$k8eU0TWPVqI zGfDaa#}W{*$G^=Woy>o%@!*G}Hz3S?opp(*`~~e==O`h{9#O6kf0;lD&gsQ(z8}{w zH}3yCAl(37)&{=Fd=mI;g_F&^*=!LL9)%p;>G`Yi>2lI`W2NW#>l1@o&PpaTK3Sla zZ=cV+%*AwAi6T`iF)_P6_Z(pbF_HTe%8P3znVagUb9-Q*XqKFUYlfU+9Z)f@iM-cd zm)qOCguq-_+C*s{UkgEqd}VVArDUfC$cJ)0Zx$4_0B?SQL7#~4Qz?w`%i)GhnRoFp z>ZEXi1WJ;6zBslR;@R)p;A0@cDSr-lPGfGTFN`$1L3fEkOJKl5&vD!!LC0F825$A)Qpd?(rOWJ%<S@uN28niL5u6F5Q~iNo zKom+jfv>WOvRn;o)L71$R@Aw-;DDU;_=7E(LEA*5YAaRS%A)tRCGX|e*-(w*6i&bj z_=)*GHb-F}yI7Sos#vEL-EJ!(6rCi(?e?Z)g~wqbn>e8PM!f|h>HcyR%MQR zH9?=*qXI10kXz}bgvz7|vQ=Fzh0=5>c{0`7hGfx#5MDUZ+;3?Kh#$|+V6aUU>C|OUqIOGl6h?xwuA?@I2CoT`ezg}GvkmkCPzLcr zQvM-RN;QlE@+%!+vhQ+dE14=5GCf} zv5~3;x?v7|gt#5dCmk0?@783YZsp1aEw1a_TNLIVVeh1OD!TsEJ%!#*~12E5bB zEeCxTyRC6`tgUGOi_zY*r30>CcL7vqT^6y^@*$Prz#O(-?WH#w1hcZM!p{J;n8TVu zd_R+-%y{#LloISLc2E<8J%0`>=uy=EM)Vq%j6#aoR*-kNE4}P(O&!@ei|JCUawHD&qz1?Fw9jqwo& zPK8>#?s7N&c95-MTrT0Wag7r94vK%pSZ8OMbS|{vbn!Qzlyz9MQJ`(UK!%yTZCNF` zCcv7(O<~SpK+VHGuFJJ3*_$7o@}KxaJLrVS_>d5%hnkJV^EXvtW6Y6@0alMEdxm%> z!wBwiS2*dI;5Se9s@T$qmo#~4M(Rm+^B-N~;}Y|V5hsr2P2A_PlV>nJoYptSIbv4& z1G&hIfRBxt&B5Qx=1sucQ!c|Oo#k*I)059N&VZfh6)xx$g+LxC@y#>+FGit_>7DBD zQR8{5&QTggkRbC?JMqmbqNOWw)n)U_1KQ57ULu(fOM%-6=W8%`P2xVQY&S0?Z{zufKcY@+CGC z>-$lmcOn0mvznTIxYSnO%;oXs*$eR{b6tQ8n-gM)`NQ_g5Xn{dhLK*E8XsX5=|NPF zni(9raz(h}t>u2Ps-{^6!)%q11r-#%(p`uW!!N;&FV?j>4%(S*D7v#!Fq+dn6_4Mb z0UA85wn~7O*1#0>)>@c@eS^=1R6&{|&y-$qCU>Y5de|SGR1yL*4EptGHjQXUiaj$` z+Ix+#dbB8l7DmvQUdW-SOsLT&y0=Y1a92Lst*Z|Q=)cG@&1XHKA~wz_+ZkaLvzqlG*}EUc5j|)e3IBmC=XR<}<<0(a$a7=Baoltgh*uF@U`A@QgXr{qxJ3qy z>+`KVlz^j80e!&zSPq}x{R$8N`aB$a-#@&z??ux8y7j(VH;CPel#M*RF z+Ic=uF}SaL&1p_siFzuaIU?&FOT?oRFY$ax+j(-j7w(NRh!sSb?DezvcX^+j6_SYi z{Cxf%E#1cVw7*D09(*nM&i4Jd;rh1na=$+x=*v5Ml&OltX>M8Ewjx?(qX2Cn>Mvx~eru^mOrDe`{RPof z(7fedP`!VrYSCNh(nyXzdt;hb!TL@J4f_7zcHrzXrh)2c(B z)TdYJaPI!yYJa6<*JD<)ZKB7ne=(;N(~n*ZsPR9qm!)Hzpz-A#?^o+@go@MeYA~jio-A zcD$zU(f&>eB`W1?qQsm$h-YO>a%4M&#f#4EaJnhYFU@uqo-j}c%|-aCdgKy0$t)+R zR#k8#TptNLRp#j^PPly7;-4ueNIyG0Pd_;>d@;@DAnjLQ#9 z88OgylSh=VXdX0+sf8P~kc|3Qj6gNT-RJI8T;9JOJZ^21q_j>NcWHI;#9&wPmSK&g zPquQJKY5lCMbM!=`ne?=F+Xo!)AtKp2%Y{NtYNxHFU?VYBJK@W=l8N$Ew>ARKx5U* z7w}m(OqAl`$>2yc+BOtdkx-D)1K5%ZUdPwxNd1Ot42fTL{t7M_P6217tgy{R%`WJYOddY`!u;PXYVxC zxV9;0Od8`~d3eHmzTs13*N+0nSBjV&`sc9-7&Q{R#_DXmDY^bl0~Uy%xC49g?TEYS z#%G@VePuJtig#qnuX$%2;w@8P3wy{|JvA|H%u%%*COA=RmRp_yr46FHW}e1f6?!A; z0sW9tEnXHQLv@oR0C%z0(_7wB+%KRx?B;M}G5l@V4tr5_lRRjQ#f1@_Uug?E%|Y0i zs)F+X(lPX(hQ+47G_^#8L1djp9#B*e>P6TLCo*t$<@Ms4euPytpi@QT;$NNNkJ5BA z%7=p`cIg7P3@@uuI1zU1h@qW9AB~I`l-eeHljZ|S3cDgJR6_IQO zX2WY4)TwDKgiO(K&wFa3!EVAl#?GwjS`xcp97G4D+SH6O({oV$f};*HUEF;I#REro zA#yR5f$oNBRE&%VH?OQoyc)4yj9R=QHY^M|5~IMhR-Z~9fqs5+x9|~!H4=s06>j8Pbx9{?WfK@K6?XWr8(XcxNCqW^5)Jm z1WO~9%4*)Eu#X?^H&@x(2>x=PgfCLc|1PluMO?!wwzwz%L{pB72YOq?q;UosKRBVN zjn{i!iR$CW&CnR9n3%*dggLhUV2jtz-+9nvawFockiiV$J;c>;s{}5V;8evR@vBj- z%hz(=ocDXh*q3TeRQi`)o9bV^eKI%4I zr)$6f*9&OK&Lm}_tq_9b`Bb4{igX$l>L88{E3rKec1Z;w(_zCPw-ZY=Qf)CfY_N?5=OiNYn6~Khtz8zxaB~L@Z~rTDtj`SV4Z4 zT$+*~GwlH`sY$ylH}qD*xd0>eE8>^pzO7<3dPNdCn~TYyMi-m3aT%=(gdia(e-Nn1 z#b~gZJ1BWmO~uiZlbpmA3zB1=VfiGl0Ns5n%xg$NR_ib>O9^9|usr9bp}$60Tb*%q zdZ1=XdPCarvy_$@X5TBIW9k`0o zaw@~7nXkHIT_m*`&x(9BGzn!nsKI0q=9I!{c&1tII)P5$m5`qwXDRJbZ}J-B>96DI ziaea1hs0`R+VjVgD|#8a4xy&WU7mb^r1cP*$x{uZ#r{)hxf zBjqPqO887k_YTF)cxhCRSV!?2+B}2>U)*A!-&=nzhK13`UU36WgQEle!6d2)$At1NlFYd+Fh)-fvt}Aapwe86s@9l%=a{<_Y{nTq z5-TI?Gca%TYi5**D*ZP`SHe%diA=6c&V8f1rD$6lO%jX>JOnLfp`eqrql=t=)_Um&NrcQ*%c@KD^wAHj8iDO5iAS z3jcGo6+yX#kt8NB>%y;61u}YPeUnp3Uc1~hTdeAqsCjmWnV@9R4{YvC<#}Q6L$OTs zEs6f1e@nbZygqeNOfrMJKj;pT2&oNF^dXI|4w3q}%v@`ya zv0N@!FLSK`mwQjI48p~`Bc4)zi59i&d8`yJ+a4CTVh9BYUDj3Gj-s@nOC00NXm;{7 za{*%AaM2fqBOhi&Mib-=3spRftZN!$?0SrT+Ehz9S zQxq3FUO$gMLF=~bThX;&{EqC6b*eY`@Bn@P+7xZxzo-Q;+g88CAWKUW%{pNnUwheN zt?kp1WM~%97WlMxsxS2{vrVnks+Q(KHih5FN`UPx?$E8Kp3KL^6t4q&J3#~pvQ;IR zP*)4y7Gv%py?iV2Ds!tKYtqS#ulwupaU3+4Z|Zsx?xJs3xK(v6cqm%^^J_l?3w9UFEo$%Sf>;T#0$cxqAU{G9K|&KV;UDDfnA~ z?z}i=hPV-%iH9hq%tGN50hC7X)?ZfKRiB}CKfl1dqYKEMcW!$3z-BHzTnz&?`urp2 z#wL`9Pk};iQS}#tE>E&o0Z6WpIH?RGXzvnvZK`tlZ;XvgozE$q8aJ-3E|=$sZLcA#Y$JHdTWX!9(OxXlUtPhGfKI=)DN3wj?v7H_y%L-^ga@Mm?o^P9G+Q9uG@U zTT}S#cynyAMJ33bD3dgr(*W>f@?%`ax)R&~StI)8@-Ab29eR7v%s(|E$%SUE;w4SC(0w(%iIsViJe9rT&$oZy+?SWvQ{olWTCOwD zT^sQ7&ikB7cu&q_Fu!J_;Xi>cTf$2QV>b6{t~Ta$iAE0dyr?&PZCVrs%S z8fAl>Hfg{r75-q*mQ4;LzS?oZWX$oH(iR5%;kH`J@j$C3IzG}-@slP)xs*;J8Ph~M z>1cvHRU9A%&rV>-%Ky`w&TxCDp}cnfFUL?{6EoX`Cl4D5-vhdU{DKp{cffq9U7BO| zE@P&ckt*MX0<8j<0c$oQBlg^ zsm_mTB8%;YoxgTkJD@@DTaYAFuzE<33!NYZc}Ec3no{ghW#i-wr>|-v+oq*@@hU1z zML^NT4hOV3F|*Kwz6nJfnSke|2QuRF$~R=q?PA7R;U^6mVnwKK(2q)3Wb|Xnir=eq z0cE%Y|3le32U*$#3*Iwr+tZk~jcJ>wt?6mow%ya_Y1_7K+qP|c`+axs?%f;TjW71U zsyrtvpA)B^h^owAW)@{nwnIhcOoLZDErruTm7;eM+j>|;bSWFMnPkv~S|_oluMe_k@`k1;mYjCMZ+nJxFo&oXweLx)E^;ajKlgAWJF3odDE@0GcS z^n&_FMu3K(4w_mPxY?ofH;X6Ee z9zN7pcG$abHuG@4OfB|pdc###h7df>FZOwlEa_;YdheH3hHNhOU7GJ)N}O56zrChB z5xhHSqkb}qbX10XR@WYz?^KpjcuK$hvrJs$<=Td`)g@QsW4p^^cGZ3Q>g&XLd)gDp zzVMgfL=5)?j4zx2T5OL}_eh+s> zDR|$_dd9uv+E#zP_e0Bf@+Xey`CqF4uFTkx|LpZ7$Vc zAO2m1RukD1@lgiH`rpMr_QE?Zw>KVb5cM9v#2rEI|BAl^m;cR{_`mjvP#Wcb(QkVd&*hwmh%t+Dvgc5oMyjbuovTv;7a;qr@o>MtH~ zwwT}_#J?or%FJYc+EK3AM^`i;X)57qPf^osGLYY@ztVBwB`84<%h2sVx{v7YX-ou} zCXC3m=c*RjCU48|dxY6zOK6~Dmr)Ryr6oeUViImeS~0yr+ZNnttAw$;&Dz7~V1KXS zN0Zo!isd3vK|bnVvy0Kr^Jp@2~27Hh%!E$#C)t?b92B zT9SZzWsWFpobZ)}fI|B5Ifn8sV<2u`8}Ttz`*!@wn2L6Hk0kK71plJ_ z`Njgv`+Qdk2EYZh3IsW3s|VlL8=yB#(&1gs+>h9C7HNJa&yNGMZgZXbnvxi>!Q zB5(|tm{$`&*-WLg<;Tba`z&m(|4uawmuM;~>`o7w$%&kE?jl6m_iT-wn2YsWzjMP@ zyf*V@(S**BP%ATS4#i9OSz-j$mZ-W3oji_`kz1;|Pbg|J%_QC->56-h22bzyATW|s z*2_4qnb65wWEtt}x=_~<&;Lp(Xb!6q+>nKke7nR-% zvf$$7YJVSf{Teo!nj!RnTXPuA6Li)?2(5mW!lgFbw~G`R2gkXrE5DvzPo8aZ0^2q{ z@G!Z&L%!CB!r<*sFX*>+pq!jthv$Idg=D*g6=;ugz-58kM>>)2Y(5Jf^y0sv`U5%r z-R=6|?dD)-HSD&uN}k9QuvL6&gVrrGhgAH7-GMeY3`j#!Dcs<&F+PrHdk;=DCTQGm zaqPkPgIskQj!ohXb6aMU=M++X4h7_JElaU(5gp_(bcP0h1`FiS>IBo5z$-WA{HH1c z!owg(2d(Rdue*kj09oQZGP$IOtDT8WI{dqoI&3#_${Rc@Ko~)c+k|=B0$_Y;bIqfw z6=Uk+Q|a1=1kPB+7hFxjSV|5R72x2XBibwXgNunn#gS>|h;oW-n@YGH+oiN}y1N^7 zr@#|$psLjb>_A!*EEYz%KoKb3@-vw_Eg4}d57^+b4@TbT+$04KaM99{AZN+7!w!oV zV}mm`vsQ~HLPY_EJseQXveFnNtdT1B47E998U9J7%w{1wk7_ zQ|7U6!{)n3x5jd5ZBW_ETG|xts_%g5b&_}bf4YaN^uT^4zW3C22KqeAVAjWwpFL%w zG$_Grk&(%ut_OhCjUkAw(PcEGXH~(&5BiXE5fU^m^MO7|J3nLf^P6XvG-qt*g3D?X z%cFzS=ks&Ku=k@x&7J;W=>+E<`1U^fE>{@$Rbl$|@$mC$vRC_K{}X`)J7WIvo|>`&@J_8DDhNnaT;pI6njGYfs0kY!qUwUPjjTEEWQ{8u=r-LtCMl@3OzF> za*oJzNoMjRbrD#7nnM80M!XUL>S8bwiZ$ffK?fb8e2rcUk2|D;xjuEZr@I$BSux6b zW2Z2;(l3Au`q@id(;OK6Dgae|aCh4dIp5y>R%s>vh*sk?U=-tPBZsOAcU|_)6M7k*K0L z2#8oX5gCm>&MqmXFL@E2>N|-4UC~jHi!B+YR#js9(6W!v1s7ZJ>;`{Awj928Y0bfS zuiSY92TuUTow2@D=6ZX*uUO2YJ8lpXPc;ZV`0yeH=XfdL3UJ9265H=O`D?zmbMPf!m3YgDQGqiuf# z#`Kq&%KrY<>Bwi&h?#^LX}>Sp*}(UwTOu7nzqPfzrS-0Q4hxYoO%m0?Ssc%9lzj?f&$x@Gtxs%mQ@aQM1#0vLDGh{ zP6OSiKC+1OP*L0+uF5RL`g=lrbX-+HKvrG$j1SN5u)_IM8f`d+oEDN z)%+-f92TMm_R`d}E43yeIp!f6SD+$iAD~L0-iURsdbwbull)9NO>IMj^s*w-S`yv+ z%ZCT8e05&DIVpC3k>xa4x<;Vbc-n|(+n}j@=5o)eHou;bq8?`exa7K;S3IynpN?aY zC{)sfMy#jfr)5)s1!u_Y1rleUxT zL@1igUArZVrnP5$TI8vVH@4#B-bi=Z|7s_2kBU>uiCROu35AB!OS1|jca2sHQEeA|`!7*>t zb`%1OzH2HiSEtMI_=6c^OCXQi0~KiEVL}U0N*q|zswE`0ep-A#=P)uA1+et9zy?5- zbGVUG+K+U>mg;(MJu7dLLencMB8LHVz0t=seR_TD_NHg3I8mgTGS*;}7$un7B_^x| z_9wPzJC?EoPk789C$n?6veiP^pEcj9pSr`=S55aaNYVky8Jn?v)Egy{=;ZS7btFNM zos(qLzV{{#<`z_}a0QK;)MI9HqS({rby+&_Ia3K`5@Vbrx?^~8GbmIRbe0`vb9QHM za?wEhkVphsgs8du_F4QrSQ~OAg1b}*TJVK9Gs5=LG$<6#fH);_RsDZ znrhm!a_^E4@q~C}8nK{GJOreeVa68!2eXqT(WAgzK=e??FQXgMN+jZy7P3bXQB{3b5KBJ3nd1}2V4dq8KZOs|2hUG6zm-axXK8SJ*^1-Keq(=v3j6D~6 z;;2(1w8W1&w)gvsq#rmo$V8CD>}*y=XfDMcJA@vs6`4}EeA&%FGs$0#Zcm!-3Owu0 z4wfir0UPFCEMAA1ieXtqOqb3)rz&;#`9a#iyLpdw<*#d~%0w27)>< zIoP_nzMS668q3*O%-!c(5wF9KU_}=)!ByxwIS$27{cR6-+=|~*36xwreieZ32YYyX z(chm!I`YsKtr>!%aA&X`0A#^L|Nf# zw>pi8-mSA)gqeJVG&8>Zhh7kQa{o28#RJp;Jd40$X_pJtpod1JF%N1{#L`g|Hp^_M z@T*NQG##{_Mxnt;ARz`=K5ge!Pl;9?nxFW9SE^o$KwFI-@StyxhvaWq!L#kL>;!@H z)5cD+8G(A*^L3Fn-2{EMr)yCA$a_f9n&|w3p`2U1yBO7DVHUcmXBowC{a*jZPjt5y z$ZzOsk{TjPE=7=_xA0(68IklFSV6y{;W5jReUOo%!7t<0-?j#R1{y9WQp`m~2!McUz#k0hYZh10Ma{@dH#*i7l_{IKT(kt5UHOi&;z~ z9z7m~!Y@5t|9|zIyvBJH4ny>rzEly_oTBDj<4nHvtgtMi=8j`H%R}n^myKG;_*Z6; z6EN>SKh*)!%(FI>aV#Hj7RF#~UQ4CKX;^F#wE2=S-Bkej9QTi)G33eWhv;%EQW=BDZ_(($Lr`NEI=Mj4ZmbNj9whAU z(=53D*Y3sx&?sOtM6W9|eMr(R%{u1BJ)R*;rQQ8qfF7vB_Ela_vYlGe@t?vMp%j#_ zf5Wt(2(l!6{93RYN}d#&(6oqh-~Hr+-L7=T(X5!iQZVqg4aNJdH%!x=*vJr>zGp)# zVv-vT_o z(q~;m?*OTXo}>Z6S*3kXjee<_nWuF23_ME#TV_p~>V$vOmw_`icJBeOtL#_EixR*t z;wrk|mJ*JVky$iY_Q^c_v_~>12vj(p5M@I^Wu-Q(mCwMB(gK@w*GUYvXnPXqVE44W8vdn+pMj;q|&`_DyA z2@UZF8za9km_NIm^}cP_jig*7;-Q8hdbQy+Tg6rCz+W`$ zClDzPi<0Z2QIYYgyQ#B-UZ9cN;eTJqIEE!GcGJ1|Zg*O~nzD{}!0EOG^g>-w?t6nE2a+edk$>xLUv|kh0aTR%{EYrU zbnL*pm2_kdN(8~6wkGF*;I%iJ}%z#K4u0_B6~;2ebY zV<^FTzZ~KBj9kMVboUv8JiQA29;Uh3kEq1oFOoF&)4TbniQ0{aNTFCF0z=kIoC z_9)SCuLtkXtOlRx`4u!U!FpJbMspH1Zt%xY={5Nu`3KbB6}WN*QKmY>E%macA?d}g zIzay!qAapN@;)ZeLFaBZ)II`bZ4Qce zG4)@I1ZmBuP}7quYVGcoI1AYM>8CfRd0x1lzJH*wd2r*fEtoSO3TlOOX-rLq(w4~2 zB?BVKlzw>l4gxMnDdPdwR+e{qh+eGqXNRJhjN2n&rl|O>LNI@aJ9Y)N%#7xSpdSw> zd~bg>sUAsPqAAbE2g#3ohc09GK2YL*Y($TgJrz}pis=sJL#9#B)phHygUyKK=YUUB zJ!iSk8z*i#6Q`c1GXjg^zm5j0MG_g{BrnXPaU@$ZFs;7QqhHIVXx@XK6s%m@Lt0CV zcohGkU;BsM1!})(YdOk#0JJ{k`wnRfFYF-a6Wdky`0|qu;vm6AJU*Kv5R=P6I*53C z0ojIwF0j#HS?xG>XCQWlNGpN++r;uDdbt;o_V}s-13-w@ih#OU-@@<&F1_c(kPMF zFBQlP{VxF}u&O+$qVsExq3v0~b;7*QVM3kIFxH-pZvKZ$DqqOF->>`(=U>kXZI57Wj6`F$@Td+HM7_TY4)hP@u0dp9FVER!t6quxlm1f}$k&E$6AkS^!hXKgtH z*pp!ZV4%yCa}ldCmEND-kDGXui2?^Bsm@qjW^Q63XSZHX`zM2-VSq%SX~!?TOE@F{ z6hc;B47*=z9gqlk0M=gYW$j}`M38eNnr=Jrd3R9NzGQs3?Xo03E02Hzly+67wAns| zThoMV;U7oRiliR>Zt-Tvu38e@!$VM-N8yGOHTgM?0(lW5I61Jo&B$5J5is~`RWhMu zG|+*?wxUS7H5ybi*td6T^0NA8{L(-1?=ga0P9vR4J!4yZe4_-cab>$_rvn%NKCu*p zyrvc1{sst2yls389Dv)6Ueh*lRF&03wso;6f?+cryRl|Z^TyV|sohi3Et5d8*ueog zFax@O3?K|yYHmCTRTKtNKBwxc{m!g8oD-P%_0wM%nrDka(4EJxlS6$~yWZV>rcRYc{Jr7$e0) z1Xv^$T_hA#(V>(W*2?|ld^ud~gpcav;C#8Vdur6~;C(rS`nZ1GwT|?z-SqJE@Nl@> zes#9C9ysir+TG->qUxOAe0RF@P5Yd&i{9O}?r^&n&aR~6zK6`p$09)!tCRy zOqz?wtgGKfQ%=vW(s!9zunaJv({Hl+JbDpp{3~XxlvHnQymlSM)5|4Gt$!WgyE*6P&jHSa?fQKlngI)UJnw`zD*!yZfHKL?zv1 z5Ug1iN@BdH18M^m)dtErV?g0Ho-g@Y6RNbqk9yv)z&{s$5u_rLTB_a-$f2_?5ZxJMx5>D1txsE* zy>YmbCO1(M2}p)DD)jGVdDo%qF^w5B8L|G@;di;V*T3Utj+MqCe$(t4MlP!OslL9k zlK%dNaISLikFpBsS3uIe@2kD{%o1y_HGXhU8$>{kg2In+k`=)Er~)NQ@x2En$M6UF z3&+N`&+`IN1gE9iW_JG-C{9qGmLMeYBa@vzu|u|E4&-(1e$bO& z8b74pvK$bj+dZph&si-7#z*vMml%T|{jv+BGZ8h~(;#679k!UP#7gKQDosCwfy?3IYq9d=HV;p_>RF=wBH?YJBH>vnsBdn#DaA>@KhPT z_FiIyVUn_~O6wo2(aSuTjY+^QT;%ba^0=(eAY$`QRXqvs{R9i23}a{mD`cY_FC1k1R;T8M-MF0Mts7UvJ;k5b=(-87HLiE zZ;yzr+Cdv`qQ_UO^2)?Irt%2Tv=oBHx8KI<6=FO*9}w&tG*7qI@QUbM_lf|wPVnvj+2*|a0o5LL#uRq+qRZ^k6Wu88%J^0@^h_i$7< z7xtnbtDa~Rz^F{oq#z-`#B{gnl86ZPxp?jcNV_KcFkh)2R+kPt;n|4W_a2mKp{%PU3zGbg`Oss29~1(51397PPhwYX|tp6MBJ}SB|qryjPErqL6!W1zdEqv zp=aJOA#n#OGStD)9JcIfL88NbideiQDuIgTe4mV7jYH=`x0cjG zf336J&}#cGAea2+_7!Bjds4kb!QIdJm{y{^;t*BpVHUi+LfMwGIjqr5&0%Fj%k$#- zqHZuiVW7ACqBT?XxAG4bp>5*IrAfbUX~mHcQtA%h<}bY)#e(K*+ZFto?=X4qd2SdB zd5MF+kVIYS^3nS)svI%}=NCrVr@`Xup)D<5Ko??^Y^4^GAIUv~I|-uHxe&?GJTLf-u|dOF0j ziG~p4lpk9PX0_IP7Y`Yj0cYU0^lWV_fA9M#AeCshqX3$z!P!{&LfEskoJSZdBGiat|_jD+d8!z+ry|qwFsn zxe_C*Fl)09Mg$0?6e^EDX3a*D*Y82(KwsC25mRP;w2YB)J&0ogCG|YQISk=_<)7VK zT^2s6yrdNI3ewwSP@o20zWW|N0N^noI`DGMY_tEXsd6jStB;vMGM(5Awrf4KW&Y>E z&3i;|gyf@Zua->s1-q9R{M7<})+O(SgT9;f*ln(AaaRZPDl{V2N4Hk;(5kK|jLVSO%$S@Ie&GgQL~{is_JnPi)dh zEgOG@;q8L=%u$(OX-iFS0k!;bUYT9CP=rdtyG z3b_upo0Yi{W#)I-vSAgxC4crM%Mk{W%c>p4wnQ)V>)i;rcb}~BrfXzVCa3IT47Pqo zh2h*PU7+sYeDvoz9M0VV_bpa*l1Gwa5((*T*xg@q63^r4VB==D6w>-wj75@qIQN_J zM_5^(@Z6JV=q2t1k?VsJCWJ{~%!@c0cBd^Y;B-yU3P0`dP?>T$Gy74s zD+EPf0km1CiXnEKsi)5V3=Z}J7*{~sG$I|+Heu(cv3Q4pVcaR^r0hC{N-%-(pnW-U zns`@{5&pzjo=Bc}n+*}ZAgnO47{|JLPcK=NO6c?)*!913b(pCwkDg0T(3AFKGoG-g z%sl1F%i~mW5~YNiDfLS&b?_49&%QYrB6}?>-{^a}{* z6SR49*5HQfcOcRb;7D+C@~Q+@*#|PM!K%ebt_rb!X&7Sp%=fN%FIH2Aa?ZlR`F3vO z6D$=ujb|LB(Aqo=k44vgYJ*De3a;697vH323H_VXEgxMf9Z9-|Mk9u!vrs`oR@Ha> zO`)_w*cQ8g7M+CGLl4`t{92td_M?DZjLdE+vm*VzSEhLc5})_Y zlh;MKwqP1n2~+NLu~1!QF0zQuM}CK(a$4DORRJPa7!GG`yk>Qm&B;^=6u&P1gJz-D z8bKa%aNbXJ+JCy`-o_8xm}R;{ejS*iQd2~NwGNG8(9}h$3@7BgXa@yWP^4&wkWdmP zcC0)=TkiiirF8&mC9w=WOOP~&{<7~d?y8MRW(${dIL;YI5 zNd~4HNGC%t>)rlwe6~0`>wK6?S6QK+iWDHsME)98uH9Sab&Zzm=#D+yqh7DDTL~hH|7X473#V1;QTgm z_lG@aY~!i=kf-SV%y!<0M66P4#g~s3vgNA~aM)Ck>gb?80qaf<_9xU|fwG!?72-t4 z8<>Ty3@)Q$kxmb_N9x#}6!Jzq52s(o`aC@aauQSWtyM5RW^?T#08Xw81V=NF39Vna zcF5nKqByS*+}A&_j~_&(DN^hE+}Ar&tv)@J6;>NU?H(HXpt1S>w&*3p1Eeu|l+0-( zl^FyT#QR8d!J@-j5WS@sPD$(K>aZ42g{kKhBRY46(obbAo?Qj*e0X>XY) zjH*m6wiX`2B}g1b$dZ)*YNgKnOlS)g)>t*kpJl>gmDp{u9muU)k@_igX?Z&&@F$b{ za8tu{J=^;LxGp^t*3i|g%S;0`4UX7?bi?bIdt-L zUF@gOUq{6+UijpAvtE29AiG|t9K|XF)X!3Bkeu%;%<3jGU^~h|p~ID9XquLh^fRkS zF&ohe4X15^kR@9wQp}b7i+L$B63~KR7!855j_`P8MNr2a(BH6#Amv;Z_lWoyB8qV# z@Ol75S?tt8hnJm}vcA=ZuDz&rI*6C5f)CD7siDwGr&3IGQYZvzmP~`;$fxyLC>r6Y zf>=A?)BL#)mVMD#7754!`F+D4dB6M4@D`Ho56A@*zz$#NB!T)Rg~IHV%}5lLOcJoa z{_v|33<+UyLXT$fZ1tViwF$IS)k;tNR<}dQl62A<%9pG$wN{War6d6`0g_m_ruJiW zv=#@GlG9;p9TWWERm8LHrPlhKisE4OYr-B>X_pd276Wvg5%P5QJGK5k$NVbal%L!G z2g)D;@SR)^>mA{~1n$MjIPIK?~>XamM9-nt^j03FT{SfQwWb;rUcg z4+YYC?{-(Cc6)B@%o@D+_=ozzqS5YB~uF9zu$P;%UI|@4-8f_7M~$CEmL`tJwv4b zyi<9vLZ+YA))wqY!!j9wnUN^Q;+UvG3M&0X>x4h)X&fvhPn;Y8Yi@pqTq00{0ZK!1 zVs5(9BM9iTv*Zw#ee`_5`GHTCq_q*krqo6KV3lcL^t%r*QsOuLD=`rJuY`au+Kr-7!ci!t<#{yLo(s#ZV;T5gP%hzbRBe-<^$zuGOrz8JE4 zQKPGq@Cp};UX{9an+mguc|@@EP6}!bscL9Cj3)rP_j)Sg)`0Gt@~^{)t_+X}^gJ{r zijWUfxl7eR(+hAcCvV(s-O_9^`R@cUoH<7=1cX&9j_j6^VcWE-4PLp(V%JE4zXnCV z9R|^9>n##Fjwvz|FgUOKTABx{);&y%h`%WjXe){qWeP(<^K767L;SH38%Bb||3;|y zuo4FfALLtrJ(9-4oQZ>>)>4r3!W-ezX6%-8d*9dteiMku~Wn#dDDUn zfW1{ADL=KEaJ_;)rQvpu9Uz*y_q+JIHvv^>*trmJ8z%@qtqc_t(~y6}!s0y_(A7%( zMG#U5)QZq*4_Dhomncz~0&ixD0OJw6>~|RZn-vQ;bp_0|Ul*|T6rx*z<=V~HCZ@4b z7JDIm{zJujbX}N*+r3^!A(j|35_3@QzFdD!*gWa{G2y~_N)|K6Cht$M9>>s( zAf}^O5&hf7wkNXOTt5spp)|@Olf3<-$9(EA-S)>^qt25#ZTmq|zUmdV6W2sSx9U5l z;vO?G>h;wK>|`~6cy5{wKZyXbYOgm=J#_1MX^1>5Smr*?!BZeiFWfx!ua#_so&)x! zy1R+ej&S)1EVcA@qjBn6Yx4`=W2B;jvTxkA-PQ6zrT)9BY$JT94HZrm_n(61It@B?a^g7?K%w}KCY zox|jtSeu4x#K2^$M1}MEI$#@7x=xFd&T-4})t=W;)|tLXSt+)VO(|Ri1aNb}aNy4- z3Hj`1;}dz;6x71De$6}i`qn}5_*Kv55j?QS?AjpclW!Q89xCoO7n_cmpHu5}mp>Lj z)PqsCN24a{=>n|3cY-}zDOH-MEGgcJo1mu_#T*@#%eIwT%TJ$NZ-Tj5)W8Qd zH?yOri${Qs=|067gAKJnUW)<=o2R{Nd2;hcBxNY28j}FEda5&CU!lH`cQ(R{k-Kp& ze7M%xpHD*2z}{V&(g2eE#yYf{tnos})ZbfFoOhl_ zh}X>V_OpOfo2j?>mklY;V3!aCdJf`pelTjzrdV$oJ8p`K@$b;U<2TbMktzQs#`MuS zoH83^VIsH)1D!V4bxtW~x2f{^rstZEb-J}EP=$xidruHBn49_4v@?h=GWzJM>7Fu_G!~0< zu6=u$mQuc8VL4Flq%B>Pp!Rpwe)JzY$^#N|(up)sWAem%e`G@|PQI_6%fA`n3tTt_ z4^uy=1|?TlqM3p{))c2vR+O*i=(Cie6%aHDSrl=_o7S@8q)L{ zo8uCxe_Rz%XdDT%{R%m+0Po2587JS#CC^!F?sm=4vS0&UOh?dnSwzbH?taMp;MLAd z*OF5f13?sdr@XupVU-8iDmNS>EvktRQiv9okwr1{fg%%2-1sFnX;kbap<^oVD3B7lwOx3KuDl5Av*6;WZ#0%OWXn!+BMvsMKzn+ z#3A*k<`+ec8+cAFAn*NnHDjYK^ZG_W=vf8okA--)gB*BpVbM$3xzv#!GP*p`-Cww0 zHbNLbo`mM;JXz&HZPP{oww@{ z$uD>cs%AT2OtrKT9S-BxCc6b&yX3Ro&Z|^i0rQkmzL_YrO&#oe8|uOp2>H~21dH3~ zMs@L#mM^x6ItYs%ZBbvUK8efF$3iRs?4^p_r`VTo1NVd0Z3QDS|AqJil+mLx5` zwqN#7@u`ReSW)W;Q_(LhBqH0&5jgOV2psv^)9~8JWWq$P>Sbw6To~6{)^bM9*ZZCE z=l${iM$AJM_wRp1?@o>(%Hd*Ke1#U7S}2D7ZU6?7i-b0}LK?CkVf;SynnD8+v1KL@ zsU%(|z}1g?e_AtWUwf-5%9T@??50hu@g2elLYs>MuW4w%R=$aA%i&vuwOzH>B#S3n znS`gCy{Wie%Om3zH7NY)-}lN5Qd&b@4;$B>7c5lgcyt})qts02OM+K8M9v_(4w4Bt z^Izvetn+5~OxS7DSRW8#{jVs*j%BFCNT}_3BPv1XD^JjwbDgtZ&P8 z3l_%Q*i0qv&-^Bm>DfaK@wjLOO5|mf_4u@LJw)azBnEAYb^o<=ET|W`FCHElV+V{O z?mu{VOQ}Y#eC;1JKICS^vP;)|d6LH8e=(Zc#R%ch*7hO?9TKjZD0`$1tJkat; z<857%1GC5;0EL}vhIgzhDj^R)7M+{6*QU{8ji;plRR`l5uB7w``hIZD`NcniwA@?m zbp=?WRN~!GHcD}p`VR-cCgc3ri$j8Loj*T`yoOrfwAN)>7vt}j7*0C{lDKn4`Q!q? zRJvlikjr@)RmGUkAJwoAKN`1Yp|r@!sj$cOuH;Cdqp;EDKm0aG0&(pmLi?iK{|pO5 zLrS@9ey1(=`S#Mdef{f;a@Rum{MtQYpSaz8USE}3udN?aE2E;KB6&SNFV4rG-Z0(X zR=+yBH@>uPM)F4AYO5jGdAyCQ$d9g@MV%BFEJIjVn~GJg7iq>P_R31m25&sn%Cmq; z0EnFTHRp51)=V|29_4--lX%V0shO+x|35PCNp3Qk#+C&>nvS%i!b?P8)KlnA#-7Yx z`dLm;SqvUEoYs(xCB73Y_HovMj@h#`LUFz~AfRB6SkxiK&ZZ}XbaM$Bx%;BAoI_>4 zXJwg=M04d65hy4PR;w@#m*ggAs+FS*7=GlOd}I0WLZ8I;{{?U-lwHZVHh@0ASn6>K zm$9tiIV)S$ko{YtMBmG=%WY$r6mTsa7azfVtN#jKrJc9bdc~$w-?Ec(_3r$siL$f0nUy?{ zvi-74MK-xrSy-`$Bwax~%p3Vcl?YsyQwW>f9;r#xGc*v5m+(&0-KQmtMj!FvrsN;W z{UvR0-mz@;B@F$y?A|k!7Ihx_WjSwM27=3dL#*)nZdsT`T9<{M3iz(hoo&3KyBRyY zn2M6B5sXhgU-u(i--LPLAr~_mBo?V&cS<^tD8^3Vjkz*h!GK$3d_$KxC z_~?MJ<7*5LJ8{eG#HEEu!M2qtWv(KSCYI)c@4G`_rU!gGYU#_;V{m~{Fdzpc$GK}N-{Coz!RAi<`E$}z$Q}E0#Wb{r& z09X2ncl#r)Ub%ANw6}^6^tY}43wRhiS)iOq$?;eldB|dn6kaJED`o|#?bU9ipXHp( zosdp*+_{%uj#^efF$HVr&Jkb0MII?6)ZG;6&wuq8;7z^!D)f zwOU)=G9xD3PTodvmh1O(HE}zWRt}MJ(Q9W-NMtAL&|$dfE%5~Q{DJoGe9GP=qa$(b z-2d54j1p-YV8vb{yf)l4oY*E9j-*&Su8gOwfpHBuG1K`Ih|zJ~IdNIcLWvprw_W2n zFn$`G;~y7sbKvV&Q2&#RG>C~~?2XLmq$_$Lp{$ubOlNeV8)J-;6l?=!IV92RAD64- z9t@WUXe?0D@36GKpFLOH}g-PILyNWiOF zihRdc9>(4lGOBp5TZG(c0C1!qF+V1AD=d*x^&rH`gL&+f*Ru+WP~lK4;YWhu5_)mV z9w>r1%<*kx)Z0Y=aldwBzyFeuSGF+%!t;AeRQ(kCmd7w&5}?u{B44$K8^FVFicURC z9igptNE2GrA`k^3|JHkHb5>PW$d%}Vg%1;y%*uz;c`Ng$!#KY}pklPG*24czX&6}s zh2pX|$C5&)swHTCBj2-ol%kxPbFkDsQ04Hg!V(G$FM|fT^ah&pbnZi;Vp#ueP|KGUz7zsdHXwW8c7k3n$y!to^gjxiZX&;&J z75Y2S&g#-LWy6moEq{Hy-JB=#UfOH}>z?W9{{ZKQtA77)!1=_tuEkS^4QH~U0M$?{ z3FL~f6(&E7<5{p5A8WN-E2LdcXE|qy8WTLr=f5i7?m?1~gqa>z(PpKF9cH_hzum?5 zh|Io4Uy~n3ah0vIa{M=}`90_VFKg~E1~2@O{=ZrC_tA&lkt06_qCRLH@)E&UX8|l_ zQf#mtAoTEdID{h)Nj+`P9b$--!_XqCX}F{_&8!Z_Mg0U-brLE?u+&8{np+N|l2}z? zao%HDAG82CC8Y{VrykS3Go18LIFqh5u>{|7Vw;=*|pAn(P~@mwI>@(@m9ICWRgV>k)iad{L5J-*~!mH5TA(%MP) z!C#Xvy&(91@$}_q)<^Ml^2p&YXr0Go`HMT3&-v18)vfv$Nv|WD`Cm3B@Rcqm(TY*u zkYSCm6FpE=NFaDA!zrFw0DS?0537Igg+_S-1Ml?vH4G>^shQnH3EvJ;JG|+#Up@bx zXOb8YU{{~~eA|8RP``Z^9meukZA(rR9&=5&fZcqKWJ*zKc5@S~{@!BqB(W`p312!_ z&2t~bLOMl_G%F>Ks5GCl`&SP+2Y@0tBRAXqDD`f;_d6dpSix88FWjN(_P+qR`etoS zFs80`<$!SpRPPvoVFZI)Jz$1!0hFrWgy4o|pvJ_E29RvkQfI8PTMHG0!s3L_Qy>nB zh*Rk@h@$fgGnymd3O0l`dQF+Wc>&hxf@^Yw$^E;DTp+4*Y$)fH^Uv^!1Cr+UnWylr z;wUge1XQkEJ1uEZ=tos9#IiG+Z9TB3cA`1PX4I?#g_<%Dfux*MQi zv?lYEI`i?im(V{NN`KD(231mrm+9B-=1lG9>h*E#_x5xeATd-@NNXp=*3t^{2ez!; z<-3<*xa7STN#`x%rZyCi)E}?2$(YP3LOmXP4&fa_ZeHK8?XshjgD-v-*ZQ~DZRymz zc7CX(E4Cl5enf5(^2P^b6OWul5V&vnuT0|av#SudyO~0M)Zrc>F(K`!2Jnul@#4QS zc(a$+7`+J%3h($6REkzeZ&CmGAEdowcw|l6up3V>NhWqOv2B|Z+qP{d6Hjd0wr$(C zZSJ1?e&26D-@EZ+?>}{{s_Lt%fAl)KyRP#(SAfT5a>4L?%X9rDEA-V8QyjLcjbJg+ zaj}*vxE}NACIN8-v4HLi& z{~IV*`agm4ezbflGjjsxR}?ZiZi(Gz}!AWu++%6kt*3gX2?G8-`ABQBMI zC4NB(p=TPNW&pX~8}I$jCGV-FfJ*((91w-%xp5c;Nl1jEb%4DsL4rmqlJ-L~ed$9DUx{DZ?sz-rfEOG~7PBTTb$!~t@=p1f`mA+3 zCs(~HMSQvl|CnrH=wPtnpa;52cmmax#X?iW}GlbwfsYGuVc$!PG88JA5PzE zrVe4DeF)1wCQb@*TH=^&;;Wt8mFIEqLO1abs?A5g7@hgIo*9Sc_=r7`E{sEAWG2uY zhwEB(q=TqnwsySYkM?=bLQdkx`Y|JkzazaB0ji5w4&922=J2nH2xa%kC5Izbk~A_@$oAoVaHp5~pGj0(GN*dhg1CeKs2TuU0J~hU`*mmWyU?!a~tN+ZUL!wL{mG{=M5bvkE;=NbtZAl-Uy#BKcq z^ff#mPU?#R%O&yV$P``RKSg;12H@LUc`_dLXf^(cd_1Z5;<%X85&Ng--7g7<18hjE z*aXsFr2e0%Op_MC693HcWH{dy0ruZqtLBEyCi4GGivZNS8X-fxhe2v5gd+OV zAyep<>vmgL)3hYaPg2^vX@{9a>lZzBxxKypx~o5HUUw*s%c8vazv9_g68}4%?U)zq zt3mXC!)MF4a@v^V8DYk#H-n%7Bz&DZ_HIU!sV1og`@A95Q42|L>k4uHpzV*c{guDS z=Jj`G&VTZreaoK<97MS?+$QTVdySjU(QOYUE6 zR}t;7BW<}Yo1w@U7W0wtdHr+RyBo0*MbVfpu=tu-D}AK<$7I;co`|3+^mH0=AG;J4 zbEdg~Y>?AjKcr5Lc7eKkrV64A1A_n>hf89eWyv`Q0M9OjFmB9Yt6mcs_-?IQbcC1) z);GpR7(%B;gK`q#=3Y@;MDLsp7A0CXfWOe1H*cPmD_!tZv-f9D&eEvwOu!R!ZpKx5 z-i>lc8?kGqp+?DX-QE-C5{k;ABnmCrIP51OiPMM~j%FK9`VxfED{_*h@5i5YU&Bdo zz3ZSFP+bj`1~1tX<0ty+?r?GfH~b_}A=G@$QFgF?MvC3$vaKdL-lUFT`qNCF5=tI`!P>Lt@x>lf^Z6rw^Z zokA;|LdsE8CFEj+eu1@ za?%|)IdPeWCgomf94{yn1rA2NjDKnSra--l%8YZ4N8|sAYajoEYbQDUH?D0)<$0du zsCt8gamZJsZdcGt%PZTK?w`2pNN^oSFLXyh$gYQo`!b!{watLNP$o@hph;k#^&>X* zw--V?DT(Zl%lq?f?pvG;i%CbPk=OHb8W^9;S=AThnCMlthwef!G&nni75P8_ul-tu zd^z6|7IYl=r^=#^21vDWX*SEBNJ;$QHiA+7@}RP+Gblrgm6U~sDlW>%JJ>bHBA-jO z)M>JeN=Zdt`MY(~M}@ZJ0*=``@Hzt9f^w3P9EdyAm(~pUBxKws>EI8ON8b8hM}(%b z|A81A*U`G{pMc)=6~uEP9a~EyHz`L+1_*Uqu=K^}q$d$grx}tUVy5JR{Q@w?z^Hz? zI!6G;cpho=U&ffD=RX+Z6wu6Kv<~P$;h?=-t|78QMFfy00OH)7OqLrDb{;|pr*Ezs zq~)b|7UawJyBg^Kz>K#+XhN02xtvXt^VyZqb};SeLAJi{yVb^TfoGH-dWWGxT2I;7 z$)zW<(+&}5%|@J`5m!iBXeksO4l!TNg>s%nx<&=aiEYXD1(qnmLdHYrRZv;nYhH0I z&ZOUfnaD=7ytg*(gllGO1$gUugp%#C5tkgWL=^R-p4HyMdG`0jZJCn6kL%r-35%Dn zwsH_JH4OU~xh>rnrfo3&7^0s)2x7?>D*7P`QX!dwH3kxLvi;4Y|C`4ca(1`53mEm%zVeeHO|X6ajco~N=f@l>R= zK_<9d12l(S@XDBQVrHRH>Xa&TB$UuL+5K!s#reSqaG_nDXr+5m_0517=OJ`9&rA zV4oNO^dNNoJ?)f+-3rHoqNL@wHV=eK06w} z4&p~`rE%8olqRtFZ)fBNNegq>BUfJvi20v^;$d}`Cv6!-OHPq_X{yh$7GaDxo&z%J z^g8ieZbOuS+ISpmfq2Bhf|>$;)+2>y>e(2KDj-oo_#Zo=mdYEpB}mF!3oLxZA+Gpd8(5sAPNU#J|wx+`)+IlX4#H<431Q*0)- zQ#_ijwRgCJ`e!qgq)H}^`74lAxD@1hyo$zMpE^Voud@ek?hyB;Y)=~Xu?+mOq!amY z?>+FOh7M3mo^$t#^h-KWJCc`eWJv%UH&1+p(n^$S)S8Eu$nhxZz(ol8>xqs?G%&Q3 z=}D^Z+&3O+Csjn(?|+cS!wbKcpCo>mLTAHB_aasAz!GYkJ9F_x5Xuodw5g1y+##pR z3-s7l(4LOCL}T$n%U>#3zBe1x?4+TdVz9t#b!DZC0SfK@u+{iXD=edtD}ufF)t+hx z2t1zRcJc(L`0elbH=|+IXm1Qtg@!|z!Q)o<`;M*7GZ)wzB;I;j=|#(+t&&ZLBRoYN zU<754_#0-l=kOLI%$HKPbVu*hm~!E#h(FJrto|j9dm;aSkjCpJ|3Mo2OaCQ}%Q*gD zNMqsw16y5|ZmBH9c<9&gGDbqG0!s**CtwQk!tgMRPKPotwBYNYbAofw(ikHS!A{0o zbEH*?uZe>;98yk-9VFJ@c&2|3#+`TLXDTuDe;pu>87-YT+T%<~5SCYt3B1 zF1RFrew}}KxnMOl@P>U|srVH*e-LHF#%I)jJnpaQv?;3DIU0<$CdO`V=97L)2NuVk z08#)VOzCL3`k*<7)ukh{f;SeDkUp+lSdI?~hSlZ0nwnhRH>fC${dTdnT6zV3?gK?g zkdFt+)loIv(^KiOvg^{mq!a&NlYZIogH~J%d`+`K-HV_K)KP`pR}eT@=Qb zjr#D3xOju!DN#~N+@HPi$hT(`Ge9Psrg1 zOuu~czd4_RzTIEkc=nl2#zSbO&uqMus}MV|@6<}s2!%u^9>pKmCJO7ZK1p){o-Tt6 zrn1q-i-SP#Z;Hlr!i{dD{$Zpj*2AcN?Nf)X!8f|9*Z`cznV0(Ks(3e1Pgf=4MgI*SOpoqB82 zQX&t5-vG$>eD7-kD^An4&+H}^#qmX`wZ1WL1g4apX4sy=h_M9S_4K#bj1tmipI0*V zlm0U?lNNEM!E>a{8IP2E$#K3#0P>*cr4_P<9y)34%zJnQN_ zH~`TQTUogCJE#ExCPLfM%_yLEQXJP{0P72N@+d7OYt?lPv z34dApvG88lIFV?JEq@3jZ~Osl7AY<1)3-1q2o6mKic{z%W9E>8a*x7-S5rWNDVR$_ z!?MKWQq0I8mwpy)^LqQj?H!6j{mQet=aj}g!=9dThSRFb3fNC|nvH?eo zAG;|Vsj=|yRz533VOO=i=*mb+q^&+BHlzlNVf*c!f>Sn*i8+SRZ*f9?~u9Kyz z6v>nq*Ct~zT!26zoAM*+?RYsRGwkT@rfhHZ@X^0yjr#bHmr*T?Ouw5ib7{;vrPkOQXbQ^t~hKm2}I9GpPRd(cv}dTeTf zd}k!&BXt5#!%H6Qq2$LRUizK_WB>VhcF9^z>ulzZ0&~7oF}!>^X{RvIz&TkSy{g0k zY*h!2;_p@k&bqCi&Oj1Uj!uH1r`+%GCno#~IQVt1tIk6N`Dm2eWzY+3&8~*ByUIXOk<2vE+r`LcR7Quxr$qUx#ixa#3^;=uoXqi?3K`H!L1f7;7D6Vb4S{dq@3OZTnzbNSM?GeECaH$?PoC11{T7<#sH*6G{Y z;Q3Q3ERshf21L|nVhHyjG)aE*cJ%ck%Alpbfx8(I6NKt5FHc9`L<&~RTcBgENqYgM za~+FC)E`c3T{L130`?$2c`2Q1K}WOS{w8JuY?4=b8!vV)*j}D-@Amk-VTgY<^V0BX zm>c^kq3i@4_*3VbZ@zk+Qb;&Yz({d;G_UuT@m`lI)DK&_*w(r&-8-}pg(kD@tx|*kU zI}YgHbB?)u&vtDb-WuA>uTPoh>QKdfnW>y;#i-~S|5anBt5{UwFwLl}-6e7EcjxE( z@!OH*Lsr(W?#cs^I(?R2iO=mtPB5j{>Chmi`=AZAQJ31{UyrdHH{~9D4Z>J_0eiB*O}|Ehy{^VEH7sgq5b{O01R0?lr%oRzI1js3dkq-m>aRzs`?~#OYr3q z?Q<}+DmG44!_f@lW=OMV3!jjXKDybM{&ys@pNKHU;8c7M6mWPt*BbvjSBGr2unYo0 zos`8EuK0@W5Je%=l5(yOeA82Tc6$VY%^InwJC_UgMsXl5aZE9*xtK)fi+iS7bx7bb2{$s2%R63q};E7eZqQ%~xXZL!I&sx!UMtdz8zn<z_P`#HV(&n_GX(8C& zPR&Rw{_{$;xejzP&f@zEEe1gB7VaYOwf0>`TY156n!|?B23lpj;wBRK!#CVoh^2XlrO=MMbZlG_*pWQ6EVT0p&b=9AHue;K)hw+CDo{m=7Iy ztV~7Q&lZFosEYf-x0G<4G>Ic@7-)=$jD%={5d!EirJ>RJhRF8s**+C}ywqa-dUv}< zLViE$G>5Qt>@SjYybh!*`s;&YN!IcaI#TR8L&j6Q=?J@zq?R>|oP>w5@i ziF}DfL%i18J1TzaEy z##1*9ep{TFJ0<+tUr^(DKi7-hmG}>hEML(sa`bbPXes#88eHNVtreKH3Dl;eR#M%y zqYHti6StS8;D8!YHW9#i;5}`O@lyvLZWtlbBg8ans4f5LiD|2-qZBylWWpByMLSBYuChv#KVMU4W4u@j+QlLTw;^8|w@Sp(4Maf-Zbkp# zANeECa_y(WOF`Z|@^IV-+o9de?uhNFDp|Bem(`DYp0B&7mRMY`oJTM<3fShoW#uD+ zm-ghT@9U6F`FU6@0}rjHt1Fhj6cPOPG=k2j*|b#AJ&_W|Nz3-_x5NljT%5D%96QtH z_<&XQUZWe4+gZsw^csDBs-V%j*pgCa@@{axWLyRka3(`ebDFZDv>w8?qViv20V(6d zmH^WAfo-War3k=1v)UKKpB~CCBVDys6(ybU(O^u-Q~|-sn{wK|MT8(S8fo(dqb|t} z(M+OXDH>y0F!H1AG;~n@=^5G|jvrBHS)1)%FB@mlkZ7umL#Y<51o4p0@RR@u*JLsQ zgD#ZHp~X1?2OYIf^|qYmoR<4ce-4AEh>p*oPYNZksg`BGk_Jq{aTLZnfxY)KxqBY% z4ZN+o84vY}L2vzV{K2~htgdcm+|3j_2=TpI{yh@SauTFnTxGn?*)tt=!9OyGoXzgT zU|X3gKN@=D3)1X_jU2gu&)`<8yW0JDHR--tBJ-OG+=jr^N&8OgAID4M+tLlCqWks< zxZpt{Uxbf0=cE+G@d8W_NDngXD~Tp)bwW0r%G2Ct;}m^k%JK(&qeys+q8Kcc>W&TC zaENL)iWT!}oU&Fjx}cVOQINJ-IcSEZ&h#ouR_ly7V?FlI5Q1qIN#Dp~%QLlP zr5}7NmD%>se24g~!#U#Dx6L!s)|lfSX}Jx*IkE3aJAJ4KnOc(O;waYowAl}e;b1fR zjeG~WzFgX*wV%FSSfDJ>jp&Xc;3-gwJq^KDNYiK>sBDDL2#lh&Zr{#Ks0e>L*xut$ z4gcfRDJqcw>vAh!2l1`~nVmCu4yDz?B z`>PZqr=UJn+&=hoA&&e6abEQ2um$|?-Zkd~WNf8tpcxlI)!NS?!nQ06FkUA-r#i4~ zWbH)?gRAJOq|Ulmj*;^Z`ujwv2>fM2f2J4p9;*>ls;Y2nQ5LP(6s2;_anGJ3n8H6# zNEv+e#ECzqEfEqSEs6c1&HOKky|yY8QTrj8ap_?vgtd~iF4V=TbX2xpDO>KW3NC7_%PAVw43tyA!ZktvV1S;JLmBMj=HDJLB}{Ne{PZ!Q@v2JA?y;7fzJ?3DI{bz4a#s z(i~PB9u|rI^IsXV-I+F8re?u-^(UF~j9QDsd4 z2;2j%7#+`;+N+`nhRsW1x5msQ z=(}hxwyFj_E~eq_x@UMr>6aNDlXTj}qD<(<^~+R{ZgqHH`>G4YU9t@=NhE?c4>7kAd##G|sv(ZW2`>H0Ey&7gh~~<*5!b803@I#l zz&+i_#|ZzqIen_p=s9Ez`lh2(=WnE}7^l(L-}=SK+haIXQDn1n>*tQS*5Qk2#QH>r zlNww)b!pn4i$Rpl1Yu|wZEyOCM(!P6+h4yE{H}C0u z!=S_Y=}<4PyhmOw>1&^0co+_&-=90zQ(Ku|d$hq{?^n+z*(17@n)fi=iLiY40*B{_ z%3xoPg=6Fpc^mLR%dY-TxI0-1<0@YTE*1RKVLnsw(jwp2T<@)keB7ra(jEvk` z<7|eB{=EI6j2n4$XLQW->)?#EXIXG^oFef`C3*A8=Q4WC>DYa!bpf>Mlf74uL`^Ab zJvuHnL7uvg7Zm$g6|3YJ+jrbn;9dP0&AX-{|G_4&szN*fDnz)=_*5j6`2C(x>N}t%_2;)LO@+R~y+8fKF#F=4!`_!>qPpJPJl~h{^6)aFzV5QR zIA85Rd|smVx^TbFzi!(0_5d@TUU#R5r|((s&qsJdTARCF-aM~<8DDjKKZk~VKCG$^ zcyy5tl}2uVv>#-k&FanPjE?Ob6`0*`KE}y0RZp5|Y${=Q45hHxJ_fgvH@}o?)o<3G z{)%mkRrj-0OI2Zu3 zJ)Z0Uc745|c3YhWG#haR+?ed_(7<}bqzLyyL8>i+yS~d6+}XKW(uw{6-h#SudXTkm zj%W@cX!+{2hS=Ds>ZH)1d~JPrDl=0iO^!-kvqNn1J~PQcZZErK<uh2+> z8Di;;vivZ0LLJFuNmIHs27_`eY$!-1kE0gN<+m793O+k?jLAFasB~)b4A_Q@QmKef6wh^hh<>mR0fSe3`rkZ@C!&Gr| z8-h;-xYcYJ~QnR(;lG9;}KymhY8>9B5E+t%|n*pHZH0kA3f0>_Iwang48x9>t{! zP7jsL4~Grb&U-c+Gk@=enQ-ce5)1U>o^^eqW|Z_M!%Glsi1@r?nhImlSTp=O4JmUy z*PUzM1M1>+@b&={jd|HT6P!;DI969nm0TI`xpzn&*G4bSKOH7&rQWNL2wf&>^9m>& zG2NgtD%8YW&*HTwTf8Mi2vLW6f27KSdhV}9whMQ8VQ738*t=8FZ&);cMA(-%%-lYd zb|mf{HJt%t-{$qj_3bs#&Er4z^DA@P6|?CPrY4~>#$A%>aoS%G%eXRtM2cpo^d$U0MAANH=Y}IPI&-VWxwKOOjK?l`dneX!2R2 z1{7=fVW~SB$yAL;rYn_hn;XznkZ$4CKr1OSP{(|sN<2uwahFfvsix|O9%1uZ8h%iF z=rd(;t=-IpEz&)+O`2O7ilbWp0yW(S3v0_oYJS8m6|8|))vEH_Satr(cGJnPd+``` zIDCnr#;vnEA~59I9NVpTm!C69?4(8VN88K};gpgAB(lWWrnimafDDOOhu5 zojK$Gm3EL{nQzOCo!w=qSVs>iSTI!L5E!UoT^57tm_s+6;`@I(w1d)Lg#Bhrq1KMHa-(KMp?a(bZW+OVNWoIXs6()1#d_SWAQG1TFzRKmpm>M*dHrtiM|_`J$?J@oARIBf*(Tmk82UPAiD_-bnS+fm-uA@Ow^ITe4Ia< z`wC^OuWKF{d3jEg&W@7!bT)Ui4?ZOot&7YFzmIAkY{rg$u$%tw=u{?mwlZv}i-%6X z1cgE=CayGA6jFnynP8>YyS+ap2(&Y#tOG7P)m*X;frKu62#rkZ_oFLC!6Gj7U{nVA zBLX`fzk!PMjL(P>ZbP4BUkgTht>?zDR!<`mIS=@+K0Feo%`bRt zkCJsFm^{Y0TdW~Zy*tQWd75~UX0jQ`rD1j`ZfAr^JUo^H^C*CHf>|!YI9zeTEUTn$ zYJSD!^qkS@F@@tz5~mi>^*7PYDiH+q&DJ(dPCZZU0JG~^UWL-TfM!zgwYOc=^54K< z*ngrjD)8=j6pmsH7~Ui^a7EVjIyuz~{iYuHC-39Mks8XE62%q8R7&z6|?VAvAxrb{;Yf$ zr&r&6=a-3tM3|ifS@KP zr#-&_j;{glsG|~CQX5pwG?y62Y^5dDQwmWSV|I&S(A-aWz7027dlVU0TF?pXaCdP+ z4W*-K2ix0~Ms+E9%G&i+mRtYLb!K4JV8FH5Y})pGbX7U&&bFNG^So~mBz&9w0UOha zW}EzL90C7>?{(J}e&JW8T~AL2hFXR-Z8U|_G*x|`l&sLeE*$0F0kK1GStvVhO<0`(* z>Ytn0K%26I9eJ89R^S)7OuA3mJ+0m?b^#N+6$kyW6Txo-82>8+=$4V0&%B>c+h_0) zs(LJHWztDqe3m(8uXB0XmmtmkTM~0Uc*VZsp2a_??P*E>&&z+L0-LW%y|LZkF?l^` zhgxt>1{`>=j}G{Pqx}|T2<~?lZ%>v1Td8T)@8O{%o#4H@(nD>K>2zcEIusF*!aeQX zacpV7L|HMC6VX~jCJ@DN1S&x`wX{mh6zc?ca!o@?oS&gi{hf$2GtT4nXHmB zRAt_DG9gylyG$K&?C9U@U#)R3QP8|WGQq@AB9$^KsJ~pC+-BH?EvwyTA zndw%(r4MbZ%{I(qIFs|-OO{{;Jii`of*Lbj;jil?MmRGKT<7uX77%71U$`Rd0Adh15CmWoZ_Rdzkd=q|tXeI2#s1A6=@Lzl zRo>ER6tT)9u6i|A z6%Z7dge2mtUEI1m$gB4e?{(Q%SQN@=bA8RB0lv-QaAoxyMLC}a@TiGAMG|)Bv*zlU znt<)mCI;G%jh-Iw_6%=WX48N4y#xJee@g7yLy{LdeI7{gQ7lPhJ`U}X94ggJhc*Z! zyiI>kX(#o8;O)kn3%bgJU0brKi(b{#gk{U_(r7dltzlx0m1M{C3^g>mf>ls1a4BMO zYXt2|)p}HD%kt_?K&x|hKS#A8lYIO$e~6eOS4P5>*#GF~6nk?!8)F_E2L{y(pvpYbA2@IB%hRpPOx-~ac zVI+``Ijg(Z3|@xdX=ZQZSO$KX>zLhZl)kkLuROVzY<~FO9IL#i(Rg0&_^`Igw!XUw zGcb`@uOa8F&vYb{o4Bzw5r_gyX-<29Gd5FrOUM|b605P{rE}6*<@i-yP+#S^rP5j@ zGW(ct37=mO<5lq1Is5M{VEm~@{wd+|P2%a3<>|kNzf|jObWT3~j+(0+@sPz`i-RM5 z0Kqt)=ad^GfETHJ2L4?H^Dyu4FrKRuAROLf1#k_x_-FZ#8+gNi_YX|@cmFR5r#HY9 zi{sx`^b4>O;neMaZwPoFHlK@t-j$Ur8!R9F#MeZnVNab=Rm6*n@EVpNrTvkL`fni`J}xT#2oH~;Pfkw^f{F2MNKJ1V(|}_VFbU;b>7i-QT#)R zR&-YCyASzPm&ts$TokpN0I@f9hh~|MXiE0(v(Q>6p+Agj$_;wdH@+qzsfPurMHzAjWZgvBl#Y6 zJ(efL8Hib}$^WovOZ}RH>pz{V?z#RfMosMAf?U6!(lt>ejL*n4y;dmk8fKT^d}5@A z#X-FssHr|KtZFi0yz1h@ z>`z89)cutES<(#)*ILDm6oD^)dcY=u4DsDa(30?exeU@1c-?)YgulL>jgQG)uM_17 zz$#vBR{zai)J>1pFiL!BCXl3!YSezEG|Fzck}q4yAeuVg7-W+UHgtXS-~vxE^1M^V zePqZ53^GmHOAiV~mT|BKGOe1=NWE&8(ijU>D(E4;g-d?V99YyD)HZK;3OYu9;&BZ#E5F$L^A9jGn$~GUzH_<1csL2vKi>b+$J>JK2McSlvSg9^C5(Mtca$486dpc9uQi)!nV%*F=ob07+k@rKLWOR z0HH-C$+45I`ubb}j~FO3ihP6$34#2}yx%f}<(pepL2`nHkYy4)C%`NotaG2~)3if! zv88GfC%3yy5lHlzW2IU8!~sfoyPPW*q-w`lITE9J_dX=Oo#VG7>ZbMq}&IP{u|$2 zx{)GI|IewS-)h6|ou<<48RDugj>EHS61 zkFEt-;KuU&;d(aHGO^O4VW>iJ8$`{ma9DASN}kh80u|E}_}$2)fY|u4qr(cO zM_d46aYoA1YwQ0Ki+kFm9L9c>9lF%(_6lt^``@QO|ANQ4pjX^tDMiLoVX$E;LXshP z4(VWBi@5il@A@~BHVJr;)bzPC${YJf_q#1|JXQSycDcEw6aM&lz>k4+dtji`&`J;1 zm_bK@vH=yvmAxmk05bSOc~m&CN>vt%ow|KbR(v1C^G{OFI+8m7Ni2+hKs}&C*ZBZE zB0}_(Ib}vgvgIwS!D6DZXct>husAc+UBQCAkKOl(+w=R=qB9@OFUQy0i2);Zkz}Ut z7nNce;{u+G^33_flKGjwgxFzs!ZcTLWZ|BRo8TgCWwQ09(n{nD39*dY)mk0-31(i{ z+w)%Q*Ba%Gno;f|?{R_?88{>pK~@J9`iJMEwInhRSn&?z$@bW3wlZ}=d*Wp|=4 zt@C&UPLlM z*DLAj+ax?8AP7|ZJRz#^&bOB``@eC5g8I^4K9lTt_g(aGwQ5+CGgvH#@LJOj(4|xk zZM@+AeCJllEkzp@W5Z;Ktn?^YcKsNAg0k(bE3_T)yf|?ccBgp1$$|4V?w}>W--2>w z^*=C_473!R?3XP4p6(~p8o43ec;PE3LdkIeE@Lz=i2Kd=R^-kfJnM+GB^G5APse|b zR<1B32)A#~FO1DXhDSv)d2p)18NraPuk7sYuwTa*O_cMCYJU8rE6mP{mOjCI&PWDFMh;*REccB zG77n^42AGfC2;2ldPusq2mzbd6^~TSe(H<>uhz}nRB~1P_~x*Bverf<9sWp4^x(>D zcG?)md%P|>mw=S{2{1>;>v9OX7Y#Y8Epf9>I4xplCx@>~ z>;|hQMAtVmDy#3`;ND42-DY6GhULG|hOHE-r5`AX&&-n=C(6Q;enOh7=+H)}3q)l- z(DZby&uy{C`zIURQ@E1s!)hU_n__6P=quC@2K%2!70psD>hqBUYyPOUaEQpAeJ=!x z3~wm;PSmUtW<+H|;L*zJ+_bQ7J4ahkU2d-8@W=57&`knUNX1Pkf*z_?q{FPgu<6@rM(+iF1NI|?)20ypgH z3oX=8H?z!sbz+$^i(-5iVue(WdmD#9E3TLT3=tY`T-ko^C|<%yNg*2pDLO?5z|0~H zKnwXhdLE(v!~4Gzyn~uEjQxl-v^`uXw4{8`7;9p+LqS6MMV0uge8Ua!X`04RD z6Ii}a&vIm7^>pAkHlscyEi)Af z&v3f8j<7t@l5J;~uT@(0Z_i`!>Z$m6H`?nh9BLh7+{w=|8Z{ZSQCCtXUZfUuxjT5R zY6BOm!x4n`OhHZjs$_niTyexduyX%LEuKe(HF5OX&plQ{JW9Wl3&{*nizA9~idt7m z#ZsLkPGZeA2rlk&U;yPoTaQl)y}nvC=Yyu{T5YN@0JGkQYUh}_vvPVsm||U2=1`f> zW6ZtMyT7!Sh)-5)rT`m)e&{yf|3&^!Lf%dR_ixQyZgsGz7P)(TH2AAmF<3BamMWNa z!Wj0GP;eB)44^0OL4g8CSWGjfZ;UKE}3WyCv5F2=G z3D?H)-V%fR3YYNYd0+1d<#lXdF5%;A6 z&}LiWDFqRVQd2YHe3Hm}1(qY#Da6n!-{={f5^zT^Yq60d7*Ezf0amnKByDJM2as|h z3VRtPvjWBm)M#!&@DYnQ2}oTYs-aa`g9*w)L@nt|Y)*gMO%dK;9@MCg#g2-LyC0EjBenSiTMgAbeP$h_m&$rl>XIh{i@SDa8nKu!ejU`0v4pF> ziYGW!%49%?phQ(TIGNbE9ze--7der0HZ`pkD}jebN>q9q!6BVfPPY=C1nPWy3oy#a z(;u5;r#oNQ3U9we7BD!ZBH?^^un~-F4eV(La`;(UTY&qB-Ej1F3(9lC(Qw7P&Buxt zW-JuxYJR4o2)7bAd}d-%a!bLo+&L&(r5rFM7HJrJ*Xq{Og&WD%)U?EsG}!W{oELr` ztElent=4jfc9vOFXIZv|5p-?*)ru#=1GM4?M+CRby8l6X`98NA2+() zTFmknbBy&3ACx8I5=HMXfLX^W1O*o^gSe=MNlDy#pR$QgwCXS7h>&RSpV#-D1mC{3 zT~HvUH$!RYCvG)p8SCnegQ zcnY4(Y(UiJzItDx<-)uV5aX=%r?}{;^Gxgl?rItEMOjx%ON;x0xi+j)!Jy*E@T+86 zvDv%HOiOD~+Nw+Qf0YoN@iG7JB?OVlb-ET;Gy>D(+#&ZS4J@@pFio)e zP&))#oDhy`mc+nI7+S}vWXgU#SzNgqbNdL>;ODO~t+2Vy@Q%O|waM+$Dg%uLAS5UB zruM93dQodD1s#~z(cey{cDdgoL(s9CU{ddKBUmef87cC&FoIpheQ57WwgWWC*dP-WZg&3d-m=Qe#m|Clz8AfnBD^W&xc*7{4^6)^|CDN$yK zCp7|4XF$jdtDivKl^j?ahgKbQy(58q!QiS;k1fr(V{=(_xGt9p(;oD_021AC%yMDFEC9RaNMZmi}~QXbV;!Wt|7&a z(UCYvD!`PbQTt3{1!=LXk;oLcmSQY##viB9t8=hFBBRn9nV zesW)7$*(4FotkmJB1;irt)10!h?;n(sR#?aivNSGa|#Zvi@I!VJGrrK+qP}nwr$%^ zZfx7Obz|Gg0*ebt2Lw0g_`2K-mrS^EF{Hf&@7{ZdSuM>pin16Rb_GT6;ah1Q(d=g($>8Q zTTl+9#IcF{Oila7VDW?-P`52$7ZiWI(kc0qxz;o``~;w}av!qzVXwwJeqi>q?93sx zE&;jWhqu>dhpQJa0{UyT+3}wLJy7hszZkRLCs`C;0Y*sDYoiF=!0wf_6N>Oj=-%7s?c zga28;dmPAU?38PyY?h)_(Qt^(k#ts1F`}$Hvr(98TL?7-G8t^aA>cwD!}>&rY__{< zp>wR+r_8GxA(1`{{nO@3(sBE1S%8a%^Zy17+KSA}za;#2{f(}7H_BD!H_FH3XM3Li z|C9l41%qAsit|h@l3}wCp6?-aV4tni)|^Tc|7FgtgtiLeCL;+gdppBj_rVB#n*zWC zJO^tPzy1|e?6rw>Td6x7m4isE3!;)}X=Ed2h6DMGzhfxkz?ekUP)Uw{<;cyP`+Y~T3i_SLv}!pK;fuV(?T?oiQCKkTlb>lPY7T|#-ZO+D#B2* zeih?QbMjupt|nReyaQ$VZ~K?zpZl&7NKJA|Tlse(T!iEDKO{eLq@hPQ8?V>wpWR6O zsamFz)5PYLszb8p;fLKHNIcer$j>v5rO7iyM%8#()nRfAg<^K?!?ITwMf4sVUY~Z4-FI(shpfk5rhnc#^KQtL~r29>l~F!2$G+9+{NQ4w;!t_ zZBbkEn`N@S(Gc*!1jT5Pi+!wHX75md)RQDsu-l4DZrXTq-uT}`*H2Ne_yZx1E|MA~ zc0T8~g-x^yHAc4wK&MeSYk1T2rxo5%h=`j%_su53U&UeZeRsG5%zc62lA%XOfCz-P ze{DGC95MECBBZg*mgEqASRE9-B({W)ZmR}!!Xb)l6H#-5QtG7$|IXkLwQE2l`Njh7 zPkf{`$lf>G6`JetqHr>ql6F z@!Ij|N$9g)HFr3ZmL+dEUMEnTf_a?oR;Y>C-M;KNU_A?kFO(v=uQ)D9|Q zuZaiRlw7BGC%-=N_5Xb0JSjv+HC*e>Hw3={kM9u=&MeY|EjERqxNB6lYs?*|=`qB$ zWGXc(%xru8i;bSWK+C6utHgy1x-NIMLShQckZ59a;)_=~=q&rSE_X}?{2}U9hS4-_E>m;8_C+`b2(1Of zzb!0W#;r@rGq3mW60L>rcTXXLN)2wztDh0H=P1BT9J(@U{F!^q5&q%^0_k;IVqw!P zKs?|t!4H!!z*IVJ7uUn*1DLHR}v(U}d2J z*runlq~ar$Tdcv*#_IL_Ye-2F%881Dbs|;Ua;3-}r8|oSW{xy-3szUff_f|u4p7bR zvL$hFU0~#YKWxdf`#&;q@_jkF=gfNqF21tju;lyTQ10B=s+%9x;3*9@6&bHc2sI?7@@PTUwHIXAGC&?fmFV3C= z>QT#D<6&jS!U8n#X27{Hlwj7ir0cTI${dMMH%BN#=b?PGr(pB36GN>rZ;MjS=g7BT zSygTyo7yV`b5a$CJ9Or1>CRY0sHD$PQ#?G)twkIz2Wbn^9giVD&~C8tV> za&3m?Pr%04XIL~c8`(AAjC8MJLoFD*eM|m8*%sW|vZ7h3IKgBf<#8sRMvaIX^B?}0 zIULnq|*E9H^yV)==Ogy}n8?Lvd@-N-#QL$-rk+QBm zRRzy-2A`?>oaEz+c)thvBDKifyTEC#;qDF{U@F+!&$<)NB9T>?>sYE(O!j-c=)N<# z?m_=;TwhH^`ImejzTW8)D~*oGit7%f2o8%Um5LO@0hCGdF%Z4q^yFB?UtFvN!bq{{7 z|M_kH_WGorrd$KpCe2jubS#oZszh42^J9pvG^k=iuLjB2n1BIyW3r%{k^x zz2PADEWT?9bOz)I_ms}%^sq7}yC7N?>W<3GU1;0hNzeaPzb+z10ZoXaHPOo%3B)4D z1AsRMM5Z~{j+v&A0IN5SL)=Y((LvjnY`q%XVNQ6c39hZ5zC%_0Ix8&HT>OKib}G49%an-6Vx~-a8hzknj6UZjbgpn9sw__jgSE zRy>85XrUHakF?{l8ad^>V&BXdwraN3dl~gIiQqH|!ps;dxFZz8k*1n*ar^)VdX~7& zz;gf4(=w<{5{(2N&K8s4Z=8g${K?zu0K^7Bc&G|5XC~NF7CEWO^@aEzCkbf%r%Di* z3KX8El+-OD$~V*BHjRrsg1=|6FLs9YLD-g4X6X9UF=Z56A1okc@LYjzXAGcQeJZp7gW=@V>@c=UqFj1Fjm1Jh5k>a z1)zsATcs)Zreau~%+EO9_aioQVO`-}7xKroc5}9P7LWH;}Eu;fB)6RX8o7EN)M%%XMEUzJ6BXnyKDblK8R(;=9Ga zOX4A)ZUeZGS*`u(8%-v5j(;H+6Y#**bwW{NTq;f3lPf2CDq0bS)};+Wj)mYYS|pDok11ZoxDAVr{{4HB`z!b3`t4~oCP)M5*bdCt zqm^UNLf>*9*|mJAjF^*HN9)}KHY1Gi}C843|r>J-pT5;c_#C2ZbDD=GY`E zZr&n{V>SGiqkQQ$Zg{`xfihVnMy(u9m8?{ku{3$P@Y zoRU+f4(Z&qW~l61X)I6~2z87f90Xj!2I&{W?WlJ)KTrSVToNWkv7<|Wjw8!oN&4Cp zI)`;m#zm=5d+7|7eoS<)>K%iTLliA6GZuocnF=TL*ga1K1bUQw>RFl5uO87%t}_xY z^|vCOvi(fG0EH!PFvOam?8}J$`!|?k##Y<$kG|C1p@{#gvPabQA{{sB^plm7sh33 zC54XRQNF%YCJp=s)w=xZ2{uYsHB_!c5Y@eZEzYr+vAR7fo*4>muF=?*mNBL;OpJ%2M;!p74dKI&AC(lr17i9M-I#`br(d{ISI+0jYGNtOD(^ z7WAujYni>9uIkcxKJMr;gJWASHz$Y~3IJOSKakFt~GuytMeI$ir4;} z{{np+=L$(4*%ArZieLQ(8;v#!fu-mT64Mw(GWqzMW7)0AMWtCS-2Sj#axyD|8Wk+;w)XT|6|TeD*Q zo7J-k|G3TYM8I`aH_z|j{Zo?gTlLk!3#uVr1P*GcP2a5=LaJ^`^3K9Wd%%@+vSFJY ztTBv}Vn${)mR$>m{dR~pq91Xj6HsXonvV?kD4O2uj7#^pDeZUV**HxZl2Xy-cfI5z zO{@y_#`;>-rX~~V9XR@-C};}D;^3%?R)3b@x2 z=*(sQG&6jb1pdUa!n6kh2(-YfQ9IsmB{vr&K{zCXq#h?BzJ%e(XRo&}n_uBNc{O7; zP4py&sd)0si&(H7cxm2pcez#~-AMzlkF$5Qr5GMQO=g7_AHk^I5MlZuHID>84ODaLMoa)lB&6M$pkoIIkFxiX+HMiYYlEjU1zoU z!OsmKgTUg$vu86)L8fT{shw(p7aMl1;#!ehUkO9&5@Z#Q_;gcg`QJw35Zi`)sHly; zLr0H|d6idsxZE*znR6|yuIgOmUFFf)Q%_H2pBSIhb7T;R1#S~PcSV_oc=vO~O~yKf z+O$Q)am{>|L{TyPQ)(^SGO+9QaXXa?yBG zLPKoOh&c)18KwfQ$|w@`?5IuE^S2tZa9OzF+H3 zR`Ma#Y7Ug(CJ$Rb{!qW6AMngC70RhMNi6d`3L@oV)7w0CSyKX05Fwi0V5uQdJN;=# zC<6Ah*NNw|-{OX)ddYRq?v$|@x9xolbrKJl^xABW&2E`x4u}2?>OE~#4_74TTub7< zDhb1Lk{Lsg`X-y;dhxAX!p`^}I)UbQ-$>H+WV7=WRlYRVJZEXkx>o-TDKDOsY*<|! zjG%mS*mm0uvZ5T#HM!2+!OLDAFw^D~ z1G6Dtt)2;Ra0`w>Lb)JZ85dVi90k?ZPj$!h>8~M_W!}BsCzIU%_jV^c2ywEF)6Y%I z$d?g#g^!p`!vPa{M!_7@pAx}EQ3JX^&kcYB5DW)PExQ9x98<@OF4+Rtp9UAWAF~ta$n7l;S_aTs*lbn zy`dNwtNTC(@-VjwO4?&oyv@2ird}N5DVI3H9%Kw?B7?_Q`g!v^;2;pT+SEnsT*837 zdEKiyDY?zvT0RxsZgX6`Y29QUo11MBo2fCjEk5EekipJG_C%4}Plx^&O7NG08JQ&r zgM=ZGnZx1as`7(3-u*#gm3-FyqHj`bLoN08F8tW;N|)$GDAtmlCa4$mcT>7LQ}Eq z-2qIw4Vto(V2!{%IT~w~HFaA8{g>|OD|8F%s=I#-dDVM>)-Y?HP-txVTVR(>dO9QO zNms}yZn|Fg#Vev= z{R*V3>*=QQ(qi&kBL;7nWCSwFmvE}{qukGL29}1a<=b7*mKs9&r_WoMr8bAA^*JJKLHJ&>QagZSOE2O?5P`d zt7(crv|K_}82$=(;L)=Q52^2+S~6U|5e6`NVuqd1!(!em^q5u)V+$O33%i<1OhLvC zJZ-`a`UPUyBzh!YrYBC<*(XAWo>l*s3}+&Om&s`}&|Ed}mLzCV)%*TSeXR*dqO6C3 z6_cN%ea68tqhb_WzLD(P$;=9_-1g$L5wY7A;E5y*(LEv`$nguJzw4*@fVP8H~RWQ%n-3e>b=) zvd^_Rb7@;`QW!~Zy<(r@zOvt!Ur*N-5_KFpCmey#{xJ-x+!ir~ZKOdOGht+hKa>cX z4t?YfjUx&mi3?I9dLVN~2)VY-;CrG%K~XLx4C#wPWSwD?$ySU5ZXOX{fXz=RYvl|~ zMNCVcn@%pe1P)sHTDDb@cG4d%lSb*2L+D<|$<(7PyV;++#fw(6vcG{?3|g;d!Lhs&R}(JbFgwA*nP{Qc=N>l{YhQv<+o+-sCw^9y zf}UHeG3uQxf%`q!)b;gBe_t{<1l0LuoOW8@o2>?ix9M@Z0|Yns@shQsZBq~zm!a{1GSjmrA+Jut9|A%&!(H>r%x@S*-srE+p2||u>~Rp z@JBU^RG{8-85{0dHb^7l?`@wAX*8|ScbZaiLNumU6RQupiWB5bVQnDnqSlzHF!;}y zwNamTM5EtEu=u;u5{D354&>_nDr6;2dpT%3CHDQmm;H`-B=wglMUcuw1SMEL>oHeJf&JJ+Py zK+9^xelm5FH^5HgbS|>EDO8a>#dgP78SCz@sLf4pATyNeTIBZaH>dx- z`-;z`A9VFW@T%9r|73rF(>_lAd_TS!{VG>3{&88X3%t zbTm3`vPAn9Ksnghksql}%amr>-b^*$l|EB>U)Sl-$hmJR=USGg@sVNF+VHA@XYk$Y zWwD)AKE)jbr%iudVY9(QcPvPmiYJD|bejeFro|tD>vF?}0VD%(bM(BI9?%I2E0rZN ze4S1x)-HT{#Fd+I2>I4qbDVo{18?eY#0Y3vb%yetjop7+4{3i%G_aQCdS866ounB3 zuDfv4jktgfRXIh~%FwqpX^e;z47M7(+eFAfFn0dCm8<#zF5oL%!)6ERd7`-sD_h52 z7?VVE?CGjSDSFF5*&x@?(>jy=E*RfZS4xYO)(Py+Hb`>Um_>)+fzu;qRzm8S*tge_ z2E0u>jTj|(`EK?%fqU>*yUV4;Q7TZ%Wx`S6e*focd0+)P5B_2`e0lb7l2%~G_jo2u z90z6BmNb$lCiJ==0nl<@krUWt=BB#js&p$<0tmHxwOW52PPumV&X{+X`}Z%2Sin^k zXnZ&#EX)A8Nw^^giuu%l2En*{6rGp(m>iwMH*X)l`~~gJdQVbads!ltgm{KljtZP@GWPYMsGN zXd!(QO28cIcJJPuIPeRD!hAa} zD+kQ>f?>8Vjk6lPL0LrGVn-5Q^hS#FDTq*k4e8YlyuyBXS?s_*;-dSSe{5$K8+R^% zzDEkquKo$x^i%^?&OBiiB2!RPYqHj*1lyO{WNbMD(WD^RYr+FEzITzK%XX(fGFeG1 zLB-S0cW7+N1J`=3!I^?Ip%9Aj5-I*S@A~_7f)@>e z_$(L}gQFkhk1JU)4CF%q0X+kLUh3Rx!IHryts_6XI{b2HI&WD( z8HyAyaZ%E#Gle5x{y#H1SM|_!hm$bq(1Xu3z5J5%7Q*FwR)(s25Z5uG&UhpeClWrI zhFw~#&H{@Sg7J!?=)BYPAus9Dxx z6y^e=1Jv$ev91PgGkA46*6RFi=Vk!j@4XWGkTth#%XRR&|M^L~SH!w0_`XSib}3N3 z%T04xXO1N0$eT`=^hYBlC~3KpvF38whGtQoE_fb3?Zr#nhUD?S>aHA}qpR_*=c zdi?#C02+WcbmFZ^e^PsiTcu1ZX4leikpj4HgD!0~wdXlFEQQW+%Dudr3fx_0qEd$_ ziqwPsqXi|ZO?pbM&6E~uP3#e1sfy}P?pUu)A<=1Qf^Og9SO%K#gs8;i{@k1O<-x7wK@Ht#g^~21>LUTzN*Mr^5PO8}C+s&m7 zlBSAx$HnWJAkW?|SxLtlUnxSehKeeses14$qOntH`-DK}?MJ5>;>&ftluP2(mW1mE65w4K&*`1WTaT2au8U`H}L7u5B(AV!J#a-bfe!cL0kwh}?S0SZ=o3;YEX#>BT zR|`G%@%G3Io@c!Tfm59xPb)8ZxkK0$d4ABjuwFBs_G;Kdpw%nE(MM|jd?} z5Wty``GQ{3#ra&YIr`0ghso*{5B75RqGF4eeMV!$){k;gWbVVDS&;GZ3BS42+w*$v zfJ*vlml(UlT^eI|3NHqW8b_e#6w?nlC76W7CZQ_8!p0&30^WbE^Q7d=Ohp`*jjqds zNun-{S<~!eZek+CUU=1fC!0$b*F$V{|GCz~9w3KZD_czsB+iV5ZfYg6L{p#Bz!v^8@s0s^??b zvZoW&F~|)O5l2HvW8-Axqj~z&js{iF2YQeIbv;;Ic3~TBqtJyngFYnE!^V}=?T7B# zm?;&2ervSTr6F*lB}LVG)Em&$zun#K-V@iuZ4Kiu5mWWI>>+y7)5P7O<}tq6p$Z#O zQ~dTwE5ne_u>E5=;}Pyhh0Nuo0^NUq$(SWhbGGz`>j~ugmeK7N%E6?!!|asqv~`ni zs>7o6-gZF?%oA)2EIM7Qe`0_E<8+CnHc9bMHQ^Ls3QduSHKsaM@QHN60Fw!g zcu;b&o*5Z45#bTw67NBh7EboXo={Ph_<Y)%tadHX5|%V~uUrB2BjI%?=>q`o)2?H7$6JztXl;O6R>8ynwMA#>T|$0+j-hK1RSF7q>)zDr}(it@t1E62l)v z*bn9qBT2{GHLfSFFI!`D=l3rbg0Ch^F)7U$JFXIzQk(AC5x!&r<%M(P-R>k2orW5e z`W{yr(Jho(ekh`&5~`gnTBG=O3aa9CqjDN&JP&&6Y3ayl712C%BGjBl(ds|~yu&U75|aFAmuCIAbda}8AT^H>tj!!u`M8_Q;l?$r~}4}DPTh*#Vui$Ta6bg9f*`6TB9%a=YRE{k+@ z1&0@No1cF?|6(c{SymYMO|_^uaW1k+@Nk4J}(+5ZKN z-_5Dr8Hy5@Hnku^wZj2#OrDsc;1G1P>9AQDw;RYhy1ErQy&Hl|j&?R_uTw1_?F|cT zSg&>+fvOBUqAXTmE^cJ`I@*iz@ZpO=KtXT{_j}ug&NV|eY$SEyKSDyiK*JQ7$~?3d zs$ZYGzQ1utMpx?aTaqdOG#smO8#1HI+Ibk=Bz=FXi)z|I{{7MI+FCAneZ#j=`dybr zYj-^cRvAh<*`(7G+fh(qv!oX-CVp!{IexcVvCZAlWmlBBNj!k;sOn&Uxz(hrka$@% z?{8MUSA1o66_<8GK^5U|ron!5m**?_>)pXZDSD?sM*bEBF#z*;e2X4`v5X9wBha}O zpw40X=Bi3tstMAT=o8(?&S$IJo_@c(4Z~((+iF?W(|tLLAByn#lZ}`x))K-}gkVK_ zk5bSr3d%+t0(RyPxZTi#A?|m&q341<|8)e@k}T>cvgwUO1wG$Y8>)QgYVGOKWe9k$ zXtL7|q|#xkXfcwL!R=2xcfql3ANybu|F{BEqDU)MbP*Cyl)U02(;W0{0UF%_)O}9Y{@nT@LTYF_@Bi#VdjRjZF&t@)uxB$A=Mq zk?Lx2#5)~xr)E;@v4A?{t~S9#V{LT+j>dkjj&>!NcOb&FpM=ic)|bi5%+a7$V4eV3 zrE0w$#wp9tx5aecFV}`ul(qkEzf`n8u3{)_zXVQz?WaRJ%-ba*5Vcku@uufy{b=l> zf5-U4Q~2HOm;+WZJ&RhI06_+-dq1}gdRz?09m6EB&=(D>pPR%>CAteB!7KEd?}*$K z3eN+hn3iPuEXirFA}#mpz%)J9Ig^U?2sPE!=4B6e=)d*SH>@HuzQuv?8xjJSuXFmfG= z)c(03TT^SiCzlH+7_rO{o>#3FVt-oBB&ZGd);u64O!>Rw6=@m!*V&^c#T3r9O)l2W zD~)cWMj5exyrF0c1k)Sm5zQL<6tJ#kLMEPv>jb5jNqp5kuN#C4Kep}8aBzve954^; zP>ioaSAfsni}%X%iv}X%l@3A3X{tIi=?$rdzhZsKQhhJ)BTGN(WU8C=j>F6E zBi%|Z+GC^63I;6fcM#8p|3r{UZHWRKdW>T?_R@+UqxauB4kBD>-CK++s3~chk;jUM zH|htiuQ=0_AONki;;bcFdlAP<9W2u#Gc7x`pk@|-cCMMyeozt>D%@;cYUy zKtR-qeC^ZUk5i2GUndDj!B6qJ}@)UEv?{9M7)?Gn%W#w)Ya6^bx7VSXO$Q(5cRL_R}& zVd}I5XMuJ{E62U z43LHzq2{00@G=-A$CAMy;q=a%x3XuCBPLWv!5LWxI z!-H94U=;@}t=HKg%(Bg3b8h~n@m%Ke;YMdz<|92R9CBt5ZJphU(|J|Qg4t9By;4?l zNkk(TLgINj+J0F4#Ej2~!RStgnun?aw zdzo*X5fT(64_-q^x_YtADG(}`CfiJH8I%NzNt+r~XTQ75q=Hckhdr(`XDL7S^rxB4 zYFFoQ+qmG1gLNBq9P`2ldrc_!pZ)05A$twpu)!C?%z4>f`SmbvU|HqO^dm4xbm|AF z)A~?#Sq>Fy{Q$Z(_ihvzL0T&ghaK`bMc5`M@QKNtVag;A@uz{s0lWR7@2wbhM zDvkPhGLIZvE;zN8d_oz3oow2-1mg1!y~)kk`V;6lYOj%GQc@qcJ;y2-f{%|M zf-rs#w@7wCKLr~>>oF0@>#V1`L`tF?_lu|^wO<=4>M|n0f9gK$?2qX)vgwE&a{5k3 z(`ry^b?7jv8GyuKRh9a!cL%79#wt%u`|h?KmzAN?_WExNn(+^zb>s*QEJ3F_^J?$b zQk=B8&&}MXJPO|pB3OHpYYsn+#Ov6PEj;x+mC7SrZgqJU3#l8OJxE$ zh$L#(#TwHx@i+I@pU0Ixjw(hW39ggMSa0K?r$QStmG(Pqh?(?e{dM9w?tW`2bGodB zAv71M#tb4NUm~plJQR`NFV8$7$_>m@ugG&ik0qT?t@CdUfR%!1s+|k5!&Q;}D^0F7 znS_)aF0HuF87N&(vD**S=$k1s3#v+vJ>n2);-D>ckUVeZ9s<{?zIl{5>%sfMWoKfJ zB5E5!sVx2)+yVL7#NAH(gHXR1t^0}VcC)iRYPk>nSuNwIs7ORfT9Q7PHa(FTE{Qf4 zvx=)WD_*-vJ&ftMjUOtq&4wM>jE&>hMW24<{MVf zhjOSHC47W*H=dsW;>HbM12VTDZ*e)hNxnBcF|^Byk6b|3^DM5Zupl<07WlL`6190T zCVFD3c4?4!xCcEFe>aI7(E{Nb>3oJ&!5r_C?`gR24vg1iL%E13bh_`(gMFh0J6c7d za18_t*TYVrIr?vTZ~-kF-Gw{n$;QSfY^t%YE1b05+;B-wv%Vg zWX|&Y+9`rDQ)$m#RsadP!dQ5=tTaPGF&TLCh@6ZAJu1LX)o<-|)c0^|X_qbpU)m%p z6R^(cYgcYTMtILLXRE?qgt|jSxeEJ2%4<9To=c3ia+A!^$*262B1dUlWUE-e z<<%AGWznE&vwU6jagfd?nF~*N(JUpcl3eAp<5<1j=!R0`{g5}@y2XH`hd0);q)!sD ze<%Nk_iNOw7iPAHo#$(R`IgW3^YRgo=ymjFricAw`0H+cef{hVUa$A#{QLP_%=38? zzIqC8O0PHjo4?j~t1h9jk^ITUQ}a_0&c=rCbZGu4y9a$NYA<&|eS>KK%Arvd*)*gnZzwdo zNp2$YS+7+nB90)-9(C5Gv*jP2#gT2UxhywL9!7gAkzJU??buNMXe@N!_y%4=aOs zFq=tt8RaTDRnU$A9e-1$jSlg1a~3lkltR+L3cBDrk98 zOLH3M&C^v~#LT+*m-iZ7-1U83Me#H?8?BOZ4dT?Rd^LLkxD=)>O!00oPd^-PU4z6i z<3s#g({_Cf82V$GeL|urb%_Yae}K4sXHMRUZMHUcOGx$#0ny=jt>#)K4ugz>i$f_$ zs&ZwM3F=?(GA6%(0*0FI@;LAkBH$nj(-18wA##bv?Fr1Z1L82y*$XJic{CuixldOB zVnO8U-t3c@vU+uP)ns-sSdhE2XZ$CIys5_Pn5S5>iau7JjlLq0N}xlXGG9BM0%t3V zvB?E&eh8ET=WFP`Z1m~vn53PYDIo>au3t0cuA|jfhFS*do?V=yYqpKYC7*uF4%MWB zfD2F_@u_uw$jt)4jlaC@0Wm38SXjuY|DooxgV<7DL(5;vZUT(hVJw%qj32 z12G9}NS-j$IGEZ#a-TWXRdY!l6;EQwO?f5TJ8niHj)3gX@Y zMdqzDw}QEE)cRjp4v6sg)6S3WL8T`{BiKQj>eC45tKZ1QYgI8JIc`{mynM~Z%n;Y} zf&!wsMT2txZF%JxkAkVn5s2*l3tDRx&OyV)AQi^GcyjgJ*_sMHF5Z5s9PQ<1$5>T> zwP;x59~4PvlsEutCWeXS_%ALRqE*AL*NcnaSy=R%ZnwH~6E!b0+hY_NDfIF1Y>eBG zr&8S$(Y&GxOV^pmhzTiKHy-}e1uE&lkJbj>Hl+Mh54 z14(Mp(^R2U5SbVde>0?`9}WZ{>Z&*tQguYIN_1$d({SwlKhB`{@PpX7f^FmR({S+z z*ulT(hv>}kmsx?vVxt!Bhbl86}P~c1za_oSS&*q<&qcGhtXv4#3Ce0%+M|NU$6Qb zmY#HV2&oaOtxy7?vWKj@P>)j?n-jC@EHA)*$);;@!jjZzq&`WMX{TC}j1)|D)B~h2jf(IvkcsxBDdfoN z9?>I#d1?u9aM+l7<7jRg!=CK5ZcYyIC?zJ!OMg4UMa8(Fwk?>9R--)i3__Dk z2`@E2#_%g4l$7JK%&hHOUG0*mvY(D=88*!&ldBbbqB{`hlN!!zF*(RLZBNwgPB*|| z<>eQlKLvlVJ# zbpQqQM(<1F+V?a>L8Sk>=+crDo~H1Ffgy)oI$pcpaOfY+{~=Y+1agJrhbjP0n{Bd% zaUeYQI!W%;hi0lzYlb!5+BB~W6C)vniDRn4Sz#@}+f4KRhIf^?O1bRPYCB#-$aogL$%SIH7FP2W=k-GZ+ zuc>nJAP{sjJ?@5pQF878C7SIT8pj8}#>Tj60ELTxWS{)FW82>PcKu%sOJH`psRbF8 zFTyPx_q9fpI9^a)vx$XrJkc3Lsx`~j@#J!ks!KiiWwk8pz|)vWh>@BY)mu)?nAZ3{ z$OtDa>HKjnTEN0x~MJRNClba z>Br{dqlYYnrr3~qCR2CWO7hAY&0-bMSq+SKF)Qvc;3AS$RCQOy(S3!L5GXyroyCpd z+VW>J-I6q651^$rt=T>#hYI&tRGGd_IWiUmVcUb^_eZYkg86>_KTTE(2kEtJFr1l7 z8{8WzRwss%8H$!My_lqp1M3SWc%C7#U{%ynnj|qX0|qbTA`<)R zT?!zwY2?+b8GJ|rOoux&f54uH1A@B@1_tr+YGO3uSf_`E|Ww|138Mj2T)knM{XQ|ItV273*$z`lqs)NL6-!-D{an1 zyb~{BsKO}pFsM`vn#0tj1x4Szmx~8@#!Vm#*US`ncC3%e`_oX*e%w1E-~hXh1m!M$ zFoxhq8=;PGgQ3hZtbL;{u_#Va!PXrUa5HEH(|=n+$NbI%fsHm36l;@OMj)a3j?7;J@f4SGadCV&ma3)1B`bq6nD%}e=#wB zCvVNCeYj3qx3IkwHyko3Ozq+P`B7bsQ={qaZDHfgjA>_5)n+5}&s??{n5B=qc6r4Z z0MD>kjpn&9o3L+~OB+NDoV<^;3}p^@1A9X1NG20U5okG8^B>w zhbgf0V9NI_YH{blUHg&njy>oN_K;*IBzv(-lRPL?KA`vo!({+}lF{m9z{OXlQbaB0 z!GNsCB!R!#BFrF6-!L83)@%pjn#*xv{gl?9XdDRlw`nqjet@w|sUMII$bkRF);9)c z(gtg9Y}>Z&WMd~An;SbDYASJtGhAM zC5+FlJmFAOcKsF2u3YUVty>qbb~pi-I2VloQZ1}{Bq9)!NX%YWzW1^%_}1Hh&h=6{ znuK%hz5YSBgSS6j94I?bkDA+AU~OXH7)HJ%FUpChKd~^u$7D{uHJ>VHSk9 zui_K|t*9cC57kdS8MBCGBg_Yr!DqlXyDcQEB0kj?t=)@p*0~x9S&dnY;{nu{DwD<2 zUPrZ`n;n`52^jntipW|HiE0h5pj9cngQ#=8Xpt2(GD;4+mqRv3yJ9DD8$oWPbw+LW z@8b(6^Qq1qnxua@_ug^anGP<9sI6=m0sYwja;$;ylgL=v*zBeVb9wSwYNPkG+xTqE zbU2TDW#>k&2FXvx^U<`(YKUf`p0jSmEid2zX*N=lSIqT8*y z?V22k(=`6_*;uI4XG_ut-?WUe1C)*#fxMuGjAijlq$XF5MS01M8pp(BdQUyYhJ-WI zigOv?7fc%Z67ngbf=nY*K}yo1OC1}WUGB|vV(yO=(?~r6&=mtGA0}(VVwf3n;nLU0 zlh^>-Kb#^rTeZZZI7_!MNNQg4%OSQ>sHWUJ=|H5Nvq7rCr=zrgQQ*jz8G!qbCoWbW zUv*o0`^7pKf3rCp3jdrkT+sH4OL)K#9Et!>sO-GODR&hW}|)B~Ai6 z6I(FOUhz}_Hnb1M29@sCci3!I>}9M#oqoFtvbYJmwOPEv-E8D`f2#q zIGQv)l5u>p%0x@~9S)K8Jx&d6DE%nt6jlDvb}{+)0J>XG6s6KU4NpE5&kL6$D^Y1_ z;WW%Lcz_lG16!z$fxb#3zf@KWl7X92?cGpJSNQ^Xq8T34Zj^bs*r7B`6;rdWfkDp) z%zzCGFVA|z#U8Wj#r;a5wrm1)0q9w%HW1#0fU~sg?lFRG#{0Yh$RI_$ghvUJs5RHn>y1>zcx)d)dcUXc=V)paU6c_&o zQ~XOSi2SK}^-ILA<)GR8%oJjU{?M(_YT+9s0ntrc)quQwqvIh=Vc0&r3#2G(dYqWO zR@vx2Y@7ueDT6%th~qZL%Z0IKv6cS{tdYlb(ZEk5NW~I0)M~a>SRU8Z%(;3pk;qf- z6?O$#FYjJ`Jo{GE+Y17Rm z2@snhN}R;%ny47KZPsKe)_PV-npm}uWPQ`fh6FAU33Inh%R?s7_Ev%OGcK#PTuv_C zdfVA&Ou$Y$82V~V3hYa?_m7*E)H_OdU`QkF-j2UmYcou91j??_>8$BiC8j#3tukMB z%hlD4MICbnWl;o#f$~e8?aG2osJzxAaD&cd$M!WPW8doFdO5P0aUilfvQrUF**{mA zmPoL_?YMz&B$=cJC~RM*~rQId0A(s-MI_@#KWmRcH7eicDO|x@J0@=}e zg~f0IN008F?AnX~2)))W>+bK^4qZW$?3n{sR6fJ2LJoC^9KCL40^ry==Onze%iUdo z!0$MkaO7ghI={biSGDqGEW$@lIV5_Cxb2iIbL*bO-QS*LXa4~ylwuaeapXT%*Jg;~ zaCw}!x!Z!=igZq$C_Sq#*GEjG8GJO>x%ouu-7}{+@cTuQ@&vDAdP_q7lbh&(L-e#0)aXN zZo##Kl&aBsQ@gFdX`t65Y_LXXxs`&S|K=(Djw=xV(DuTDa0$TKJ?mX)out4hxu!Kc z%}{z+b~9Y%V4a}{)qOGsn^CE-nde+A|6FZuaePv1*2u{jJ0EYsZ7%`Ergo)F^$yJ` zT<;66cp3y|xYcYn(G5*OB*4O%J`RmrIJ1HtotrrCFFj9DNo#c%;rhuk!^j`olr)7Tp5ufl!*B3i;nWNZAN5iD)qjLs9$WsGo-q-Pa z+qdz<5wCv1cuAdcjfIN;+bMpi3lRD)SC#32#VS^ELiTA_gKKVvn^9fN9ji8QtSO|h zPw?GHxUHYK52X{HsUv>Pgc!SuQV1&`8MTRq6ssyemKv{aak?IL|NK1P|GNdGR%~_W zLr%fWVr^z~FSwb@CVmR3XEf|!0Q0ofLuYayDUDqAH1DhW8Yv1#ysO(id+a8(YT*kF z&1he%Em_68v3eYz`74}mL8oG`>q8tMWmV&{)rV|B3`_dQ>Xhcuw)4uyrPa2@L#wGh zKNM)A)0)>~X9qU)^K!)bgZ-vYBrd`G>{jSNfNF&}49yG1jmMAa+~ zsVsfMNUMcs9hVxjN!^s-)!mA=08WRX4B1(MjTN{xL!CJhvAPo=$T>T-E46M+I8FS_ z=jY%Ks_C?A-_tD_m1cJ|Qy}4K2$IPm;j;RYd<434!D(8Dwawi-EglxHdx&F-1ZLpm zJ761+6>tw_R|y-?Xje1Fz-V^=Q09vL9_W`TSnKPwPfa@9=FLXcNb8= zlVc}P3eIb(D-RENdt3*RRJ{27`A?4#88bq`p}FrzJh-P=SV z*s1P#WBawVW#Jrm;lzMQCOlJQwJku$Y>i>oqTv?E812gjgzI$t5_OIj*bnwr@x=|= z69`Hs^)NyV$O$;wS4oaULuajkf-**2oRS?}=s|IiMVLoON1Mk-TeKH;YT!ZC#GEse zg*k4bscUbDBEn|MrsuGh*<2?;V|~6DN6=_JcAGY5F7z#-%ChYC-t%0N%~pPyvs%q@ zF71J5$#jxQ+RhQ}6}9d4OdT*U_xb510bZ{w1=gu`u_<|o0+Zlm%FZqzals|;u7I5L zJ7dogl9_F*&6aK~*abupHV^)KCTSbS2Pw1LXk6|BY8sy6U|dx1AYkC?79){71=NP1 zlqSJ>;xvddjV8%RT(be2L`P4?GqjFa=2x!CNZXj>@kPnOp}!*KoYey$60=@pWC|=L zD}5$B$sL#ad;SM{Ca})?w?Y{YD+?cI^+<;mOpN02jkpr4xKo4bca+7!6ymz5vw?z! z*mUtvN!xl4d-H(~$OL_)X`g+lfQS6dpn?mM?1L18tV!`lHmHYe^Rv+V_2_&T;eJrA zHE>@pmW9grJYwNEZ5}4xlZyVnr9Q@^Eb60-D+(NFau#p0WV+W9;gvQ~B!uw}e~fPt zoGYbF%dPlBR-`oQ;t4bl+3jextZ5FoK-Ombx1!|_QI@b)j7y-o5t3cp+guB|G)rtZC0ND3lUTq<>`uK6*YF^U7cF6X5;uJ=GFmT*<)VZz+$m>g)Tc`FYQKSAa`N5 zzh3st)6Ble%4BABdc1kN&>-b!tx3p<$rV85*11Une=r5}4<@oHI7EBFd+;RFtY)ryNEb)WIgH6!Xh ziG7^b@N~6l1D$0yGAB@y514FH6%g9ip#{X*aR_tOv%Ny#&$&xJk%qW#`g5(nQuyG8W6m4SAA0)_CWOZLqEJgs0ut>#MDY(JY2r-7W}1 zS1P4_laRFgbA+UHbCU!_^ar+qkE+CKB78&=rV@t;tEcGJkZuT>z3 z<+UQ;mJeHot~qka2vpxCF13I9QYYi~u|U0|XMlp5auxx5sfBt&hoJTGu3(nRJ>D!0 zFYo3(WgG=xAnE$``*Ni`Lr3uJ9G5Ds=!$6%yf=Bs+b48%Y&`1$i(t4G;fDrRiRa-X6rfSODb$YzECiG#-nVKB`bgk zPrTYkUm0X8QLj}Unhowpy6yW`)#iKWL-2aa6R+6vs9lGrCxE)tUfkBp!|7?(p7GQ3 z0pSunv%S_7%2em1=1kWN;zqXRgPgS`NpNItI@QfT{dZ@8e`EN267nvijc87Wooe@w zgfNdZ$Iihu@m#Tygd8)r2O{~Jg_cyd%`8+~_rpMs5b{gSzUMl#i&!J`IrZWCMN6v7u>^&|E{Yi0bYeSo*y!xP^v&-KU-He=NX z<8}sgj^nTv)Me=fZiV#1G)($DL(_drg`e(~n`rB?p}epQy7h}P<}SKgQhjad>WWI0 zeP|PedE;ljPU(fL^(Rll5p!=T58LUBsFQ=Xg(sk$BYh;;^RPfZz0M3xRu}ZKF}ifg zl!b@=SazQ}$!B*FMd$lp(TeRO;rTVKXk|~jy1V>OZ$X9Z@my;gfiBqcc=eOQ=9V8x zP$7l7g<}-K7R0r5t1A(7O>gH*T!x1KTeEQ5&n!ao^lc^nyp&e3lS$gdp(4%0wmK~e z8efxGO`Dn({Y`!QMD0cVWjFpUJ>j?tS*gkwKn8=ejk+Jh9dXsv!XE6A9}*XPX_V!H}81q_ll$5(Bn+YWJ- z--ms~XogbXc9|PF)Tz)-Pq(9_JR<;BXGu{fPrwE(j;1JD!-%8GOmG+zjIki4#5VYn znjep*bP-36$J7CzHA8bj1r6PG7zQCTemvjBQ}~y?Cod#r^_bZiqIW?)al;h)ua?;G ztG0EQgtEb5bnWy8w%AS#qh*WnvgH)Y9Nk$d8R)1yx~wT=BUoy~Y0h~Aq;tCvt%_MT z)y}}0R(NQeP?3&@*&SkzGDhSqN6u>d8=0>vilE6~x6?p0$bA+8dCE-^+l-DXtP(_P zPE5{_U&dj7uf=JA2D04&_5t%^wtI%q-Fi5|{hDhQ9Pa-TgQOZ0eOcGaD_$QJ%^=A< zj}u+8`8wbx?jHhVheJ){jTNP^eugTm{Ae%^fwG9Orkt|cX$J6+XeI~rF%@`v?~OOT zHzAX(I;8iKW6!`GYGsm0ICrZ0>n+fb4Yb|+FN2x7%du|IU@65uHcz@!iNHCA-#XKV zB{?h5YruYJsDb%N%v}NbMhVE`IgQYbP(D1@ox2(6;9P>{k`G~JqAzkT7o^E2O+xC} z$H7I3!^1P@X&aZBr>CG52$Dn8Um^pF$bz=5SeWx+)I(}Pipd$8h2gv(QS^3Zq4!BC z`aB1z`~>a^mXZ*6EsHv4S=2xygWDUfc*UKoApU?J;?D_2fqV~r!QOH7!YMbXGrhTt ziMGsxV<|TdA6peEM3PS^r z7(lU31`EVs88Z#VVCI*``78RXkt4H09r2LCg|m$5t(5{bYkq%ZcBz$DJ_g)n(!cx1 ztO}eGl{Fj4@ap558UJ4SwS1JG!poWWC)D_KFbbiJOg;*Baj+yj57j7;M9`apDY2s zOu~SFBtPPFr{%N>0`(jjap8J~_g3@$h~~XPr<%n>&=Po+<*Uq*=vWb?Mm^M3NR44@ z=1i7diCTMh`2-Q`rK>OW2`D;RV(F`6IR)axD|b2te3x8n0u!e@4C<=sF)TiJEHDa( z5l1a$5QNrjsa`dysEYX<4Km3JMNZo03O;?CO5hi+eU<#n3)JKO09D0p^O8;I`uiJxWHN<#qc|O5w z@Q5M%`OF!;thf-`6L-FQjR?OASLeO{s6bG`lVGDoS^FoegK`}bfz!uL^(77-dubi* z)jDm5G%NMI?+lNAZ8S- zEywg&^ZB!C*GHnloglYwOgF8e6YOYhPsZ%|9NxQBZ>;JlC9E|o;4cK9k_SBPH36e` z0i-uyoHNDWU)%V;Y*8NN_}#g-EW@y1lU?CeH{1@Z10HkJLsYlVghc=)hvdQ^gN=!tES+!PP{=u`%)S^7)P&j7erYMi4W#QGBh z-E7J=TUx56D}0ZS29=!MjbCW!tmFXd3~5a!bO5WZ67k-WDnB1?n?Mv7Wrea@Q3~h% zwD+s?w_@C;-fW7yBgZgg-P03tWl~t{xZig_m22f<=c;p%PgwXPT==vo1~E3vHLO-d z`Ea5YY!G^Z9aoZkZb+On&wRR42jD<$8@wGXaFksd1~D+n;(k{O@DH9#`tKL*%=Z+G zf&c_EueZ+9A_dvovkigdY17WP&a7_1Z&t(8=}4@5Y&}I?>-!LZu6sp8{jj3uU}B>4 zyx8@wJ(%Fk;b?Nge(oS>N7H=%ONU9>qc0IOFw)63+KGkw@Oo8Xw4(z=LFc&*K_`5j z$7Hyj(Z7)3->pB?DmR`O7Fu!8Gw=&_oOy7kFlvv?I0n(qZ~wc|veQqo#A&xeX@+Il z>w5kY8$0hpm}`CT5?kzgr7Y$w`cJ#l<5A>s-I?bm>o3q{2jQ*~Lj+3X%hRGC+rV`4 zRJGoCDrM9%W3jIv7eNi8*KcrP^}NC>45BHaUJp`2@9}E*%0+hecXjcMckBbF5W>wt zvm@zp-Bvu_OG5^(>a6*Ub<2<9(wM3_Xv5jWrp5z;9&cG!{ErgGtvo*)X9=|L8U^(k^w| z0HKT$%KE9|t{4TxA4RcnuLLOh7*~RAzil$FP_`u&lgFbD+aB!ZYndLFJDVw8y=V3l zR!yW(;;pk?c6s!9$A0`A{kJj-1%{K`R(|1JxrsX~LB(uIrqLIk(d?hHywZ$fu%6{` z__DH@6%D4fY{xw8V||H-zA2!7ds_fqKOgAd>Y9P#ryh7#2nD#$izRf%7C4n!bEQ=v zi<%Q;RAMq-@O;A_%`L9MO@Agos3_*ID8SO?ZoDq5Hgo*Likpoc7l>oAA)I2x`i@3Q zzgO<__?A*}6$D$}=Uv-1s8{V_w^EF0XbCJ!pG*XRf#H*ErRrr)GSogz$UQz>UL~wg z%LO%i(8+X19WR+>o_ocNr5ulzTRJ46AN6qxPDsxT8}IVX5q)=d+TjLc21LbUzAd2GpQa~*%b;z|Ro!A9Y_9<_ijl-ma@ zqYN8#b;@FOT&KN?Uw3OvgN;lIioWg~@3BNed>^CGpLd;&Ofx=?zTU2`uAZ-hD>pqI zz1^?zIUj$)=>9?dTVW~{imN30&g5QRM(((#SIYf=BAd=7!l6j>Ar;FcusZ8XLZ~LT z_Ahp}w~PYLlY4H7Vu?Um3Dk)59e>DlyXnv>`$E{2@aYrvHEP53gBGpVtTKyB-y3?e zsG<_f#nMTs&6my73r)QW@LSz@_+D7j2L!=!&wuVrQ+F11(b$nqK?#d1Mstp!JA_QMG@`Zii35*&^mK z2(-_IrkkKK-2kLzl$@HhIR02V)bqv=cCdK%4S5tL5c3?ErjQDM<~BB6Chn}fRgZEX zK>hL(D^E6NVU=~j(4S?mgbR*xR->QrSm@+4ug6Cn24qa7JPCH7dR426!L6Q+0 z#|FpK`K(M@Q4#B+2OOA4nAk>*=owxxgkp)>LMK;cDVx2+&n2o%xBc-@d)!M2^)SE- z`1GK+CD&Aup>IMdcJKbp2Ij2%B}$r8?7;CmR`o~+A%t~J!e}i5ESj1GIZ^WZGP=$dJ-3h^3TzdPp z?x@iOVXAlk(7%p3b#A9vC)2s#K>;o6>>t&0*7EXu4*DiyF@&GsAh2|?GsD~hzP>JwFV|K{r8=s7|K;u$VuoVRrg3uAFl^BD8(d3N zitm${+`|d$%hkXuQA@bD7WWZ4j2qBfx?Q%c(r@VQb;gk*?IO^H^^jF6`O0qSIvecg zSUi+-iIC>*7RAoLm^tD{$(*yXzNrHfn4XHrB;M#V?}ji-xXWIK35^~`-92}Bx)NuTTl&t?NjLCSowhAD3@sX%NS7-3{o5ZoWKU+;GklYjMiyW*=!7eP(dfbjS3v8CPBW29&y%P+1waU^kA3f?i&eF zyRCXX<&(p*Q%%GfNqCnwx9YuiKeqVf*Kj6|VvJ4CnLz+Tcblt?)*Q;sf@m_*e$~h` zR~mGldKzz1TM|o!&W0niQ=p1(b-Hk|O(S~a4T&fj8^+)HjkWj>+5L=r3lzFWcwWUg zy!+J`Q>~h}duqn@tJ5L3QoxLRI*@;$I=dNx0(#rB7`*>kNPvTS^B>Id*T{}52?zz18*_7aS>has)W|pq}nenye`sQ(3I}LwLYd?9K=SYmOZva4i{Y zk?^!MoFDqcJS%X~9Id9Ql_2%_6vy1GiaS7EVuLfjL^ECz-=1V`r?WUPeaMnF48D`D zy5(@@c=(Frb%MS7frc7fV}!6!kU3(zeJI!%|?&ui~j1Y(jIRpGPG| zyO}=?%-kcO-yL(5%fIm0L*3kVeoR%H%w4Ya4l5UFi z#KUi?$PKPQD6L0sEYzi^evo4*gp0wb?9{3e;$XVKP?&yK|0i!>H!h6hGX_3JLx3pV-_g;OePTRft z19m!4luEnP>)^(#8t~5`Xv(}zgDH%I)zw_fs|Y5G>qcvpR2LS4X<~{T={49MLJz(_ zG5yYF9!#?^cpYe2!=@Cx7~T$s=)=dZ-*!y;;rnKUb`c>!FK2JV9L=#wT>JU_a^i1= zfI+T=ugphZm^YeZB;5DRscJBjXr&CU>HzK4ivF9g5IwROIdU=&LBgULG#a$|@`13- z)J=i9(;9beD`E6Hjn$P%te&qhG`df!M-De?3H!JVvzW17w?o!)i3BOs7zI3-ZOX z$s(4F8>W;iI#2ZF8C!ne0-KB2!1)_ErvC*0QED+%v_Xs6DL!dBcdUL|!%%0T!fkK@ z$a^t_ParBowH#kZ@L%q2OR-5Y9{dV9UfJ1Z*n{8N5a_C_?Ea)V0B=pde|OApT_Egk zKHAt9DBo0X@KH>I1DT>0T#m>q%)c9e*W1(&-9v-`0aeNxpXYs>sA*~3&C(10&aj|p za08lJDWg>+tkU;^;BCB5c=fz(5^+oq(b##s zoSZ5(l0);&*tK*GT0$E$TsO0czy{fE^LmY?mG2SR_&IuR7dtn4d-xVJxww&?0PUD~ zUK&LA(~oD!sU@Asye$h0{_;E-s-_P8sj97gKJ# zya_j-1e)J~9iY135uhl;r4%B`39bAM-5e_+>9gC!AdknJns|~G0{K1N4_mA=_lmsI z@M=)?vA+dodwoA%dVN1jh2HP}`R?5B?WhUlB{A1j$x@CgvNJgiO?ca! zdl!pY^?TkoG^BJq6g9AfV$_uFZF7yylvZ^>iO{j;a?Jy5pgRH^04F2~!k`(020lhJP;p))!iiKd3^AC&H{dBY7312uAFDY1Y^gI zR*2EI@-U_gUCfp1=vuinx`(<3`wMtsF5naK-ZYm1R^i8@=H$Y4VQV)vz(DN{dnlQ$ zy2>Sn9_PtnA1(_wd3H<#@AqHR-*7qC0ObdA`0sKFWg|5o>Rm$A9A=DP zOfU-H46UD*>uIM!lgQ-}lgh*tA+SxZH6$yRa+iI&)0iVF%1oB+N^W_-HW#z^g7F`QVU{FyK}m$}W21)|vYH!1aYG)<}z zYRe{AkisI$RDxD%E1i|IJ{l zfzyQkcQZ7zedeR-iI$<@fj=5c`=Xh|=D%8=jJ2k4ag6O*LzD-#e1@aGshr=9*rPe&byIpA zc4-IrbY;PIx@#ieq{b32mRw!6*>~T7dZCU$6%TdWB%JJ-O+m*;Hz(gK)@p8SBhS6v z6rf|7t%SJPbt&3oB?xQAXep+eA<}5%@oUFQxu9&AmALMt9|_Zp#3^X~LS$=d$Y2{$ zw5=49(%*_|0#gyJoUJ@l-2LFSo&=M=VUSq9FsVTuC|--Kz%b5f)X+imgHj;(c;-oS zp!OZs&-FYArQZ!?vTa;N5AGngIZ{b`EaRk=eSIo|yZXp5tRBcBo}RKQR=fA{G|Vhga~x z=~#Vq=BPb^pW83|f|x}=S@9Xe?=+tc(q%XG=k?k8ptaoV$!t%)z^B=YI1MCR5kBt^W-1c!y6J zOyoNh%kR@&^2|#)9tzAn_dX)5N+7+)o_#3CxBVJN4WMwuCm_F^uMe|K0Wjx0lMmC)5a!4bNGT~`aZ{MEblI5#} zBCt6=xfF)#&TV{hdi0suo?Uz6@e(q>Il27E`d)-0xRl1Z`L?re(;fV85s1hOPB1hp zYtDw+$P!xV-xyn%A?dIJp~zPdNA1225*P$CN9mn*2To@!GT31mc^b!^cMBDGUh%ab zmDc0|fD;Z{MOR^P_oSVj@hpkRDK9Xuc=iMT;9ygms$)0rFKeAe(^v{B?_m?<*@gv3 zU5)_R3~bv9p%qD_88YIETBki4?b-`}T?CyDQ_;|dVP_2X&j)EQZ}1nHOeEa8UNq2E zL>xOei8U_?e|Poag|rh96MPN|qw5PjY3f4sUY^&FfxA$6ZznefIm)NCPeNOJtud5X zr_jvU;90Kd&cPPJQa4-cYijp~%uWiVFvBm?I_R*z>Ar%zXl&4(oNjSMkqQFz4OE#< zh7JE&GR9zaT^dPT1Z7>0S1D5Y+Fi(ULx+N?y>I9U&>I!$hzNB?M$^&~{|HiTv6;t) z7wlT_LC{Ui4$)i>)#7!Ab49Z|d;PUM3Z##04Sb#)VZr83h-RM3ud;kYp5jzrf+V4S z{`Z8E#8cNdMcJPoO@>)5xu7&uSXd@mI+|a3I=W0stS2VCE+N7j9^#COa%R=v(x966 z;tMAA{+tY3OV|*YxRBjeI~}i-@1$#?^d+RN0%`fwTh^yKhSK5Szx!&wY-C_v0yKmz z3;%MGKH*-}?jGN1i1{f{V`^>ZM`*5#+q{7V6!IaCJP;0=w5nDv{GPKu`aRingkm=u zzKq-FKXUu{`&l3M1+~vwfj?tl$n@6JgG4cAs~hGvhs1uB^y(i7MLJOk5sHzKGAvn- zA}!3de`&w`!hY9AfB+#5U27o{`guh|0#;{OA^tZSPSqQ#wIDTj7-g7<*o|+m;>FjJ zk}PyD?fN3o*_brBG>yjd_@jyp{B1;C)|_f03OFvg@{^u@p;jKElO=!H9M%yhnZ2Ra z3(vnfr6(ttMP6lq_yQ=foL0HE00IfKwu)9S+BQZ`MBkcYc+8p_E(i$X;1g9n4^I^v z7UF3(_mxAh8P2StP7Pz`4>8nKG1P6$FvpOGRxc*Dh`jpJb$8Fxjb}EnIamB7;i9cWPuEIm%lP(oOvF2Hs{SEQp)#F04;Gbx!n> z3^ZC2gUtAhwO$g41n%h+kS;cgnCDJfX^$^CNv?4+9D8u@`}tI*v{8&JK@{z)9`rE( z30S>2d~L*}1Cz zo>cvAIP2NnWnfd>%fLP4sd1kWZ0+vZ8{>lR>;!+KC)1L3$12-wcJ;4W#rd36-P{w6 zr6z-jRXh~P<^72bzNRbhSC|qLL96`RX#|kt*qr$Q5EJ`-s9KODN6ltKQD7ps0hYBE zv_GblZaoV=Q)R-ujqdL1P_8`_$+V9Y61dV(K#&i^zvoVf( zhO&ArJe5&%a*7!IJp#QAYK`|GC#aE$itV6>1VT037DJ}kt*r~px)vHg)qJ`7c|&k; zGDl5y2&X%Gg{X<`5Lth@ZGz177D}k3tW`MU`NCz4vaz`{DwGCm3E7;?lV5VRpPKfo z1X2*o+mC*lTtU;8;MW$Xhe8Ts88E4ZnDv%s9c)0w)S4Gq^d%wlUl>;N8v^3LD2^64 zkk|`}$~TnS0iMd|fmO(hkf-w$V}%dbHxhD0@O1}%Z79;6IY8kmx&XGlkAXhCCsa=0AZpJgnO^Y z`JEGT1)`6S-OAX{YqG2bN#Cn)?R#19wKI9LY4B=W(l__RmgoZ@xUeSat8r2k`^kwB zBlxAxg>hlKZ`D*(3D897k();?U{NQ_3v!rhMauvHmQgT3~Px~ocyyuo4umgAn z+?PkmN5#mue?9-r43yNpy$_%{2Q|u^u@SA%9U%H`TqJfdwt^bXH9N=1Mrs?wnB$aD zP@vi((wQuFTxY*Y3NG<*hnwf?C?U7!4!9*zN~^>{=~z?oXmQM98UZ<<1Q4lTzW~%` zbD_~nImv>ldp`Q4P={?PS;iO=m+;(+%uFmpmuO#wQjr9I9w?Jk5ZCjjJ2DOi707c0 z3(wMU%{a_JA6hx>LAcsdODxv!`fcZlBceGS&1qo+N3f$G=3_e*|#qVik*3cRFSD-3J_h4G{1gZ7| z9weREorgo^JYi>ARD&Oe9%51OwuOh2?$0H=#}-GnJ1#i;U_;FB@|t_3x%BtkU0FRh zH(KlS#V7@Hy9*02bh-PHlnv9IZb2H?rvSzSH$C`}w4zfli^!r$nx49h{7)zW0dDzb zMzNu}EaVu@`-w}v-r_-*L_J|Bu4 zZLdYES5P^>5WD2JTHJQ*z8tx}5VP?p|K1_&w6qegS-W)aLb^<}h8T&3A2m8`h}g> zJ&%3wHZJL!>ipB*uLXx{TW}g=oec=GF1Q84`(pyE39t^fBzf>=@%}g{24ELF2TC4n zaI8xnaJlL}%|GvId_(yi_qXPfq5O@`k<%?j_20lggIN_jawEO@Z9t;s0I>Ufuv`)2 z?u0z%cRJPizCtx=z(O~<;6gidt0H&Pc}53s1P~T<{&o*zM>!=fC!Ncm7tf2KA)8aT zAk)s>cJCI-5h*z8wnZg=>VGx%)E_^$N$T}snf5?doBXkkktPr0Dd7EauJ+8C?hya` z0aEF|D9KoJfm}Qd3LgoTRz^#oL8j^Jke2*hBR|-i=}g^rCFP3nppART9Ox7b1H4ng zue_74k;6O-1<30Hc4UN5rMe~pn*AxO{MR1rtJAx0@B45qHr!{3xdGACLJL+Q{m(Vm zhxZUe?NNxdtTF!rhGxZo{CcR;aD?$^c=&5}GSPx8#s81cye&DAd_uN|^SU_O)5Ou4f zO`W20AOjliW#*LOq!=XNo@?&r8$ELqxdHc2fwkY*8tWiK^wIpSZAkjQo!+Q3E+V#i zeWUDS>-UcDN@b!Cz3jwqt19>FslEC94rloL)zix-+VfLtSMS!)2blEwS8}Ja4#jD! zkbY=dny@P^&rxD&$}Cq{2Rd^kE?}9q9zroM88pi@lK}zVu~ES8 z_|KGw#BJySP`#U61X6J;MC2W>@<~-7pJkjobA8i*E`*D($G%d+hW~$f2ykhd)NzOk zU0JhS$uLjE$=RvkbV(>xqf=5G77Qya9b4{Mf~z*l?Od-~exCE2gENMp$xI*`rnZM5 zjBgN3^JJWy@zgk*IJdiaOuyPh2_r526fkNfbi*tyHhFj zP@#4M+{$aoegNmfm5Q$(RG|e+EmuG82{6%P*=+aa!-(Z7J9-5^uJ=EL>>0Q9vi<8N zb6}_LU71XWe;bdwkfInl(Q+>;!skdv9kqxYvm1- z9#6nAQ%Q)KqhNYCczB_S~jaqP&{``qwXk-r~CLPira34r55ct}rxE4f|7 z8-#Lr-&mI%&Bv}3aS*U6+)k&OE+YjsKW*E-hSocqk8A4j3|R`83jd*4rPm39Y7!xy zfr7I?yNl+#hVARW#W3WRh@(-+O>(Gz!~Kg^?ir6=r6SILW|pqoQ0&EH0?nhQ&O7cT z=`7MK%{Vxe1IO7;b;D@jAAX~A*cA-^E5P#;uM!!t{tMGols}0LwT`pjRV(Qjt5CtE zKyEF4EjX_0FY7%2DfE_uDUK|?Cek&SX6~c$zw_hI58o{xFM!jI?5w`$<3eL<&X%m= zz;S0>*34A7G-7Wv{m2HH#|LbPO*!Tpi+eEMqQmg<&K@OW0FdY(HT^Gp#Z23&RW_&n44PSi&{jE&{Uk|{8;8G#R)hwvJQ6){d z!)eIoLVvVIt0&1=)?Z~axEo77U$N9#$^rT=gDK4Ix5k!C3R~ms7zgx&pzU^r`Ft~! z%?n(46Q>W-wpMDuArAvt+f#`;>ia+0M6Ms-ZAIauENFJ zIPdv>yNNYfHkvx~DdDi$?pDH>qwxYlBWuU^1=5FtC^8cf>T%&bgEW7k$L&_*smn!` zGGw(vB(<_&ajMkF5aC_H0niGoRAa%-zYur@x7l#JYgn{c)X0Cr;G|aijkv83~&}Dv17A?7RKN zC*3~j_5ZrO*t{M1amtGV<%T?E&;}>a^JgMT^{ZY?2VYEny-(R|(ThHaj~HkisnyR*$~xO-x5k^yyFBb>hXVPd-VcGN`Q^^A2wi7r&80Zv#qq}lI>xzZl<5*5fXLjF{ z>_dGdV3hIWs6Wh?i?r3Po9!anKh{goCTTuvy@|CuLrlVKGAAQMLRtyYrm!&UP?!Ea zw3pxv-cpp@8ywABaE_)b&9qLtU~(A$8{ znh%f!;%Pj~+*IMP?_zjB0^ zn;X1y{^^U33}xjwG`2|jL{Kh>TKv2D$C=D+yHj?nbwmG*=LP#A5i{2J*LgU3IPrO1 z*Pxl40+#;7*b6xRh^;Lrn?UkBGYUTuNf=V=9g-0{R0_z(Cz#ef;?;Kk7GINBsH<1# zV|F}EH`m+Zt(hbuh1qh50}j5eht1Xxq=za&@cMB`!7V4vQGBdSFoHp3tgHT*namal=|Hsuk#z)pR>)R9Cwr$(?#I|irY}?5M6Wg|J+jcUs|9wCAe)jL(@0Zo9YxPxE ze`%~bk2+7Mq4yp440N5+$m-#<^ieVJaMdlyj~(Ph zH3jHnl1?WG z1GPBdNXEZvwCU9BS9I~W8l}6h@QOd{jzxKt^I=%>c~FdcjtBJQ!sw?$MVPU)GGJ_F zL((RLEmp+I)HmhuYW7Xlkkqo4*FmWuWpv@} zOss2I{iXLxEN9MOhbNYXLOfcg308LOKChPlIUMV$Nq(HyH1m0y6N|%Aop#)&blRGA z1qieMw*@Az9mW2~#F z57Sxy=Ju1OeJ4$OuGwfhmJm0zGQh|jqU$p<@62J%tQ(xPdDpBtVDBfA|@_@t+m;NRilBsMY03EEN;0p}-(dEVD)hA84=w78Cdy*UoO%gbX24 z+oZc0{!<)_DdftZ5K55}ZFO^-+|zi9GFb_Tw+<$4373nI1DH_tNGES0={~6;g>fTd z`Gnah_;s|N@l_&$m7yLi`im5guza70R4X(pu??#hRuP1hO9_~V+v~piD-pS<%UfQ- zmkWz9Kau4DQViiyC1#dSq9{5WZ|5^GEgZ?XC#i(E07`WYn3Qq9h*`y7BjjFlNgtqh ztkIIvZR+*WGu7i5R#i@qy+-fKPQf;3JFq<@+qG!<(ZE5TR(6^6ml&_~rr zT2fm6y88NhqRG$PXE!o?u+RYq^{|XccSyL=x}I)Tc}Hyrh`0f5HcfKit~$S1E++RS z=N_*hKQR~{N#gD}wLS2arHuL{M?8OwXceD>z2fwhHS;yy61S>mgq)J7<48cj+&3pj zo>-U>$li_qDreGOlj8|2Fo-b7k6K3+4_?2!>cGtPC1!bk`ed>UXc3k2`CH}HmP|Jf zOvRE^jUf0i+Fn|+LT_8P(Qcr%SEp?Ap_RAYVxDCJkDqKXj3wtF{w=1Yf}XWDi8Jes zOgo0i_)~VxiU18IX?r2k`6p6xMW)TLC+5%=xAuS)l9pVjrXc3J?@5VITb1CZ@%@Oc z+3o^Ov#RKcca51hO{_!W4=;5a-%kE9!ca1#`_-gPr#N&S(Q{imWRecaC|rE$CT5f_ z^zf9|^zcXpkZkFuUm%N3-e{>eD=jRc>_ZESVjnb(C4}V&b6$}oH@6PvY zy4k&$_1rzXM67_yz{`cz)*z(QGy0;utOj(6D%lYBaHEwxeD*nDBZNevuWju9?VTba zdKTfx)Z>?Iy8^h@m1a$7Wu<{IDe9FbsJKBJ56q2p*;YaeS%aisVIS3SK2pfj^YpUg zJPV2UN!YJg+Juh#;NWJRIkl}K#))ftR&NI^4# z>O7yL@cT9F!|!Nv9;81c$_BNMwh+$%r>Q9~aS|QTABIGu+51ECh9t!k=7J<@UV}6H z+_a?Jn6jRZURMJOodH{nuW|LVX7u%A;mIfd2vjf!UZE8gs3wT8$rPkv8Q3bKW^Z$l zEcVDL>4Eoh(c}4$kN41 zU!$|>vQ$ZSa{Undso1p)DsP!GBoEy})b?Z9$a)d1Ja|EBaR|~7wDtfkOVO4j&9)*Z zH)b}3-Pv7>*`qYs#KX`@jv_gLMt8J9(QEhTZf$F6)5B$?as+7K7_APV6Ge)TzRVi0 zWQ5FTcj+L}Q)j+ANQIC7jIav=gCkkGZ9E!-YHAC$g>&x(P#1u2GOtfYIyYf{CP(&%veGw?mw{YI_dHi~^ zy=otzy>2eBnP$;|vdLULKd!#+=H#iy`aMrB!-C4>kB$!qOLK5}%%effSDui*`&I$;T+D3cdP+o-091j17@$8K!0vrDCH_(1sIi zG;%r!B2L~657j74${!@onzRj1`pSg99S7SmM1r852ahlqhZM!?jb(~?CTSDGXjAvR zAH6o}Z?a)vG9hTcKoPIZavZ1u*`9Lgcpw!nFm@eK?P1knOFo22_6ZsvDJ*qPN^8Lt5eQ9ISY_)N!d~jsLx+lG$4S z3nk8^SzeT%00|AL5+a@sOR=;w189q9ASKpx2zUUXqhUazqyP74d-$G(xQ0^I;ib#5L6z zq?xIqc!_HY7()n-u0;vG)&(23ctgPNMe?S#^X@90xx~WM+{*-MW`FiL;?Bly$?7$P z<1XP_E;%{>Rl1gCQQ_Q29*Zd`#^JU21S)W}MLinu0dr3_59ZW)P?~%=F+J8%@Do%C zm8{Au1ebBn{dh`c=5lmIq~9sfMIQ;-Ap}$?&Lb88k#1fm%On{f(oH{<0j1x6u7`-B z1gcU&W*VV+m#j=jE`p}WBBNC)K^#YmRV=bnbgDocE_-CnnW%AOS;&*_ z7g?DiugR}#^Ys8aOM{yc(Ql%6esZv}HKtNr>~fPk-~2tk&fU@Zcs&0dkJi&aRw>^% zjtHIsm*^gP0&1wj{N`*aq*&6O)__+;F1iY=B|D*8Il-X<+(6-CfwZiH%!97#)sy^!j(Bf@gG{HrTKM?2^3^)F-phoF&pFn0~e2+oDi@mM>Sa4AR1G zdEU{X_CTHFWrwNXlDd)VaPNA|$xcSlZ@4o5ppxAunzFYK;;63BEmTN75V1t@r4U&o zflBVD{9y7Q;0lvar%pjAjC@*IY|ltE__G@s`k&XL#poJZ99`N7rxk}5pSddIiKF)` zHD3?!fM0ezymA9vSroj(R4l&mz^XhbTQHX=b)^HefzymikOsD{p zmiIJYoHlS{-&_vmlKKZJva!?--NgTE`bG{xQQY~$ZpP#N(a z4HSzzH=4Ttf=zTl6G}fYdJ#kiD%ZX=hX1^_W74^w<=SRt5<$$G4>F{4z>@P-u|0!v z;-@I9Ti4kLE&?lACqLza^OO&ovaQQm`0xEiw zxZvC_mjN;Or5{nV=%5J?`iKAAZi$*vcTyy#5zk?ev4%oh&;cXjXftL~O{3)b__>Z}H zHrf2zcurne*>6ohdv15-vZ~JfQFQUV%2tt&No9APg;JMwvRXViBC9^Q0V-Ro;Vef%61XEt7SpQ-yR2C~1CB6?{1~hzgli+GOac=W9;TvD6 z@nyklVH_;`w6*S}kQe;3VD5Gf3RE zW1Np|-4!@n4EV;n&o)tkaEMIl1_XNZXs^fcW%u_si8B zK8V{D=?zD1OSvo*qBNH;umIWgud(F4O-A%+bq~3m(d9fH?OJZFQ9_#R_kvY|Av#>K zimyssaU52UGsZ~CxW&i9^Azbp=Wq7Q?qI6p+p4T5l7RAP9TXW;a7?5yxK<{+Of>N{M$ z(VFoG{JhI5CYwIj35so}H?g$-3c_Vj+XPFV?#a3@P z4icLxV;qGrAYLSpZ_RQ2=Ym&%-f6u5Vp@1a8Vcrt^k;LYKSX(=Df3PlauK4bQaJx1f^`VeB z2YXLq7P&CbeFpeUwvOJNYcRq9>7H9d7#zD+Oa2KENM$91LLYJcdlw};<-wJ$T7kz> zCQ0odcY2mMon71LTG(x%Av}3C62X-d$5}d(0$C;NmRwKqhCQHP|Y-!yUFX&l(n$I9SQ;bR)=ccom-PA^ zU4%MJM=sznUPSuZWC8H$+Jpg5eMH=Rx4ryUL@tb$Km`kNw_ok4?QV_{Fz;?;adPsP zR>{rex>P2@=VroH`{1!F1`%oC?A(X0adG@+ynR@U;NhBYc7aM0#-)4db=p6Z9kT-2 z42M*uaji=_GKg$2=RUiH@`V}T?Q|jcYkG_1xuEHo4;XgM(wF!ZN&3j`YF>x0hVxMv zIs5vHQR1NuwEYgq-#khl;BOcBE@|#JeHpMmc0ra{Sjs@7wLRT;un-Lw<=^0;6*wb8 z#7G;W^~qT)@%)A3_s)sJVMe-F$lc_1Vs9PJs?m-QP|l!kwQOXAzovI-nG*Afj5;$t z`^j}kcYQw~S0P=(Iw1W?TYHT&=NufjuSb6nxm4NTA{CzfqV_PnrcN70g#oe>e!2jS z`eP!WxupPRR{po5%4SC4L44R!Tl_vs1t#K6=Q@Jyx(Y`a4uybB$gZcW-Tc(j z#wP|3(L9+yk&a``OFJxYhdlS3J)+aM^WCsS7UxE0nU+=y4&(^QbGk~w&$e0BJr5@E zK|MVmvA(oz6O|(bRcn#5V#fs~Yiyvq+6WIKWOs~5kGW~0YoAAq%1}qM0R`pA5gR)=dhCaLc_e}=_tn}eS`n#JSB{d{BZUrV5V#|>nNGmcG27mlwCsK;r zMcpZ4m5cMQnvxlyu|6ORNZTK^_;&DDhR@=MAG;6QsdU@U)!h#%if(O5JuR^Z#WFD7 zv6!J8!9x(5DivGEQeZ_Zdr2vx`6d{DY5FFNR@fKRSJE5|lT%;)sp|Z#)GF8jpRQxS z7FuLiOXLjy*jb)VLC&O2o~YIK)y$!IbaWI5xlNiU zR*uq5Jo_wCGn8sHGSaE0?ve>BKNJ;lk;r8XS8_tN(D3682o3-Sk*8?fOEZq(?suRI zu;Gk9E*em4GdX2N1`qnj#C1NtFT1?#NjOsZKm7cl`%x;dVo2R*;CT9W;X3U~Hqz}{ z40y&EMok)2(8wHh#g^k~*RT%>JAgP%JdA_n`yUydC<@NibYxl8RSef zWp=+W%KGJ3WC|TE2=#ppDpb*OBS@YNV>7TKNoc*Ma$O#xk^VfcR7DsVcp zar3p8a0p_0)Ma>fzNzWs1K(9JXp4@p&jFbJlYQ#0$a5J4VZ#&OzgoDDT-9it`q$gm zVjYxh>OvYq@%rnH_kfUFGrqD&;3>#XByI~2&w`aaGnRt1q3tRZq3BW7BJFupy8ybn z6)4o8hqHZ-!NArH5v!-J@6*bbSa*BI)^>dOZFzm&BBiREy?qX&hdm+IM=CM+WC)*8 zj|1GG!fOY8M)lhm7bw4xHd-UYeO8g_OX4;>C|uWP%{DgyF!MXkwm2+?Z!AK1qkkB9 z7!#I$5j*{&C}5<~6NW|t(}WpSV+ukte)2X@E5pT_mo&_wVv}(*rTo?0GuyCsjxIyJd11YQmB;<9$^ zb^-%>kr_c2U&fa}=63dDh6PdqqMqNg37PX4Mm;AB=AH?dvk;Dsz{sBUlO3};^)WiBs~ zrOO{eq-U}@s}Z2&N8l#NEq#19cr(^-Bea8H_m{jNT@Z!~^fwA;?jn8;QpwHnlh`5GcD5Vg%x{<+2yaWk zemwE0iWFFZ{>^@$g*;s+mO+6QAWGj6K|yf9!8`~Kmt3QBO%#m#GDJ;+6d7+sDL}ds z8PUJmhaGmyJ=lv#S4PU_|6;Y^NB$`!Z=;YZleCJEs_dDMCtVzwt<_Jfvywp>6B3KE zih)!pU>5v_K%PSuR&5d(`MN0NvQ-GoU}&jnbV=y1A3PB5A{!`xE**Og{)Jk|%$b&gn8I(8_#wARwTHi5i5(ps}OIpUq9%P!~T-$VoFEOJHsUN zeAk$pQJuYSFOeT(3ZlazJi>&&vh7Mj@o+W=vBN0nfTYZ1YKqqj3zWV+w>P#|M{;$C zM=*{UYQqd(VJVr3v4$+}YhlIx6N?EQqV-;9%1OT7#=mg+M_s`$)}AH5LeUJ&mdUw2 z`!Enu6{qNHP9BxvwOG5}ACZ_5H=biMrOf8+!wt7&8W%4`it=*I+++2aOD7*W)Su(! z$ToK?;!FSu%OayDYWh;P(o+ZX5>lpbJFZ2Wb-ts67rrlXIUd0F4R0sCq6EY|voUFaK{NLXhCf-ho#%1BNo~=F#VG-A9^=p(0aS}{@O6H?g zj#0~)@c0I>fSI4b`H{M4_{%si)vB-0&m0^F?oInQ@qHC40N^e_a`wt5atNmU1OJMh zqvh!yZD(%CBiMJJ%mT+Fk!}u;9jhgAP-h2Z>oou!#2Rou(SXSRXyLdUUAqK@AnIT*&4O<%b% zO{@tL23N7B>1wFu$=-yEvatYKi2m9jZ6Wye@sBuNW`XMK6)4NJg%NL4F)t5;svUSOvOH~jS&G4%Rw}r4vh?YoAfTvTFsJJoZj_n>mK|vN%J^T zWR;6aHzv=)DYw)q*BlU<#?*$Elg1Pf3s@g4%%v9^6?aqf4xpHp#e}1JS7bYtma3}n zMsz3_VOnR@e!-^019ByZ7^iVxWpiKauop}?IXiTAbXKz>koD>L>+Q~a@tVJxPBAE6 z_Si%qJiz*ttFXH+)4SK3V8Fcb{BZ%Ji$ezl4=-@WUMe(g1>m1?g^J>J#!%0i{jLOJ z5~4nwa#U-Iq=U&7?f$#1l%j^`iFI%0^Cby}m{L%(ef#zKV|w|CGk@~x%W=1<)Cjn) zQz){4qegduVQ$Ty2z!Ry#CYWXHC--*u583MbAV56==pcT%>FNsg4@6!4U2m|di#16 zg{{Y9wloC$Q){b@bq9zuQ){gYkTdYO5n<$gAm%$N*L|2g7Y*=3m?Q5VE~^`uGt0xZ z)DfjDo6qdSb~e!`)rcL)w#g0dz!WB(3}9nWjLGX-iKX*N-mF1LiP)Bh)Dw9b_)S8o zfpPWymZ^b*h~dIPYKksQ^K!=Jtk3;PtK7$~UagPuO~v?q$~| ztE|+95&U7J=mm6CiJOtDZu0-Pj)~Bm8i~=g?+O!10xbh}0)odB>ybzHbF5TLSaecV zh&Oh>9?U5YN^v!m>Ott~qghb5!XD0`(4O6{vZOQHaOp+Oel~WBpU*eEv~jMC5`5-b zv-_dzpvY6)4lYTY08^fo0{N~UP39SAL&~e-#QhQ~LYiKR>Vml95UAPoHw0#UtK9fm zGNgxe4V%9Z-1Q$$KhYErp$2uDdmwC1XJy}#{DPV23oHK5w`3ri(k9CWzJ z6{%=p1llC(2jdC#a3R_~Ign;`8w?xzp}F8LZ$4&dFuCKuJZxRzEY`=xE|UV| zqKc!xsU!tkY{9jl|4{J!201z*NkE<%ZB_K<*27fkriL_E*G6!4XBp@@^g}Ug3T9Wd z1IJUGf0qXkJXs8sf;eBg;v;Gyi; zINwFdqP?h9XcxunCa41IZPbeOpb(F3{-ELVbFlMjm3Yr5D}L`ZnY2ywAgQ7I(B{?J z$w=%A@1XfLnyR~=x^66m(Z?zoLmu}d8&ZQ{KwPegi9=EjgrQiuY?c?a{6UQUrY(ET zLY8mz*S`<^BmhI)KOinvDF0|u0)Uiai*NUw>+)oilt|L)3Di;SaNo6QHJD$&acMX> z?UF`q{|qIybGY1jhc(y)CT-w+1!zu!tHPCV~jN^`q ztZ?6g;yu0YO_qVZKlXb{HRNny7hL#bdpVYuORR=%Jp-0Hxo)>iNNeUQJ@lX#6b{_7 zfeYMWK+jH9C4{ChD}ULVW*{V}hEx?84LY1d11vunqk3eb;;NL^#2#2H{z`jT{cw~cew`S?=p)#)) zp@m#BYZ0;hZE0BRh^*9=@sC^9)=tqWsy-zAIG6uvT62Pz0*V*V)#0>CJj$ev5fMw# zf?APQbHS$H^NF_lG~Lco{#nc1>UnXTUZgn)_k2H{ie!U;p^OSmgEIqlT6>~FJgg|s zxZCu{pdm$p&26%o@ArTue(X46Vy&wluE#Oj%Adq6{MG4U>9o`0hwr|9$DeHUAPez>Qk5@}L;EY3mBXR=^sx{ks z1z3u5XLzrh!DvR}+Q$d(taMZI-o%DhkbG=^(g2yUt9%)uk{vNkZpVbdRR-4K?L+g652?aDFy$K-dyA`e}6yKtP<1HhNpM zBF1&-z23hIyL7#sJcETwI@OKXzSuL5zyOOTvJMb~1fMo(qOww<5!(oXnYVRg_afzy z1rrLY{YH%F=6>6IfS!FFNo`x1E>DjCYG?lYr_Co9>+raYgJ9d0!=5j$Q-y5WUz=r2 z`m;Yer@?xGN8WSBEQ-zYX0;Kv&c5ulzv>tIW+j{5dg$aTPWa+-a<0hP*pjeDw@=sa z;OPL*brZdICrj%b4ucNAYR40^mBv0*Ymc!^-5NIx>kEJ@y#!T~xdUZ0mg3}6TlL&x zS63(e*s`2#wEZuq;U$a;Pq3ekqazp*>8dGpG(KNfM~`ZJt?!1(H&ChI0&h=$*9~Hu zvCNgtd4CE5jU#cF2V8wY2k#v945arQ}sXzgG(k*GKa2wzdpfy zbX&jI?v>@xYX0is1F;yC?7|#E_bemX;oE1#MTpROPkCUxZPw!_=ZgP>T#RX8h*ywd zYD~$(?MwA6c8NVhP0TJwQhjojleOa6_R(86NAl@i-^8(DuxFoZg+jERtAVj7;;Gmp zsv+a0n6Ol!7&8T46y*YqbemLYx)V^q6cKL_)=7(L-9*}qo4NtpPTgsF(`JuP!ij*X zlNau<m!`xtq_4GyUHXxt<-ei6-O}`1}?gigq?|f_%`=gi`s4p-wWh z`<>K%=H)r*%psdvs1pVg&F9RNsrfGhGQ6Hm2QuJgJA=scet$02_?Ni&^)JfQt~Umm zs91irY9;le=uu0A6Ns}?y$dw7`~U-C54>9T2vSyomwnm!#dnlQ1?x(%^Y<_nGWjgQ z&fTWGvRq59m#%Mrig>XwJqZAv{xY9WvfYGue|e=JNG4YH8%4J`ID^c#f2+V{(0%3Pm4!%pV&r6x^&WFx+2W1^?-(Gj#ivAy5(JLxL^M+Bh; zmQi_N33rCc(y$WHBYT9j!Mg*IV;*$Tv9uU6KfbKlN8-yOKUV~t63Xtu~uc$ z97>?GO7B5ry-9sFoT!)^kdsr<3e4*c+80D0yN%-+RPMwENoe{ z%<7z7E+&YvFHh7Q@a@dYU>JZoa=lP^M&O2XDT<*S+*LnofV*^Hpo_!Kc@TD##h#3& zH34KxM|i7noIdq%;owmhT2XL>6YSXo7q9NTS(k11wh_;J6(h|eo9wcKcJWgh^HhA zf-U$T%K<|l>r22ue;gVySBu*UD{UyPQ_|WWMJSqvQK@-^!x(2OPs>MR*jiYN!`F2I z>KF0~Uj`Y|p{w6$0GK~`JW?TTc(y-pkv$!^?|rA$%Cggfx28O5Kvd#1v7(%~USZNg z=am^X^(2i$T_8q4S7-Zyxgjm1Yx2_c_}Y}*FpB=L=$YEaX?zvgzH#BA@@HyOh@xDB;*XIX_6^Jz1!_=5j5VJs7F&7{t%Frp$d!B}labTQm8cd#c8nd} zEPYpSftYfWC3KJV!xXzskH>P1Kw3%$)8>aI4i5df$K(mh%a8dq-mTTCNcK9wc(xp+ zR)lP$uuQQ7Jq`=vjSGw^qDKCevh$az`RSQz8i*`3(vEmr$y@$$C5O37aS;S^sG&ZC@Ows^sk?oONRu zbY_c5TSH*9Cz_z`!bE)wanhAl;4lW*&J@ngR|s|zT0yy~qGGPPSg~>#rCIm%`yG9{0881U^K1!xyQ?%6N{Yp|qROQU;-d;G%Y{RO~*lA1`wOa0MMZ5m8-S;BWoM=Ss zH*qDqvImB}Lt%JTjvWb_&sg>YPMQp@qCBd6T|bEr1H>bNX#fhK>~!Imu?IDFI1}dg zJ@TsXy|c&gjpGaMiPBoyFA$E+X#j<(Fh&*)<=c3?V3~NENc9dB?;d-sQyx|KUAKF7 zuHZzV1wns`AUX;O{5vMBx7pO4NsCsHObHfZkH?1^7b?dTDb`MXBTe-1Eu~`Ro?D}n zYxQ>ytE=+n7<#YPn)YHfamxx}wl2Pc;=xyTEW>PnR>GW2<>OV$c~O)PMElG2TKG32 zxRSG(Pmq5?zmHv;R}sT+k-|vjrC3>!0)VXDJf{e7_euSdK+wdJ9sDItmd~1m@XQT>YC$2XYORYngyTN#=Psv7K)~)O{wzcUwPYiq?tcQp1)WhAP^>|zb zrmKw)dry3#()~FmcDXOAX66E1sp~WBNkfNjSdg%fEDa~){#$pt?ypenJ2170#~o9+ zU28F5zUiY3ss<%ggEM ziJ!N(I}87J2>$mI3(dXu{er!OK+?T_v(4el%SH_`MCSqZ{o}Jahto)rug}4TZOsF2 zx9Df#&_iwtfHfu#_te`qWGViWd6Xx*{o@n(KU{tTFM!KO!o4=jsV{0OPGxf#Xxt1UjaRBpPpV5OoRn2#XVHjl=9`$4(aUusjVP_jw zZ54^RVtoUogZ8rN(eZccmdhp6G@QBM?G&7t`W?zBLxWD_L|9=0L%IenGB}HJwDx_j zuzoA^lD*8}t-9AdX*_3n9iBSanCh2N{fBoXrwV3+4A6nv3%b8OloXVVB+Di5;ZR3P z4s9Nwme5*$Q+A;3zt(hRo13oYqfXmP>EF!0v>nvQBlz;y5^XO9&tH=dnzDaXyTPD* zwXsDob3_gihLQaGNf%bu)4A!El_RJk8DdmfTT-r6=0H8_LUEo*X5v87xKhVUe3 z0V_Jq8Utbfk=#$`yZy07`=7(_ck__Jn=i|a0(Www!T7*!DxUT;WRtgR<%rLrs!?tE zKLW4GCZV$ft&{ceGBX3j=%%Tf-0kGnuMvF(E}2_Isef>5J2JVqzc&n>pJo zk>{x1$Oj@N0jPX{dMRK%L{Ipc<04*7hh5hoq_N-^q9r zBT>E{MKedSh_qbBBx^iYWEI@Fk1fD!VOjamkc)Xea-to?NR!;xeE}Wvo8p-g8K16s$onG?Q6D_74R=>7v0# zp#}Y&y`yZ3V%q57)urS`UjXC8Ou?z_4ANtk1@|on3ZX@ACJ=Ga>G;RYJk_Yur_Nd1 zn8Ga6mvZ|b48FUk4XcNwoZgN^O+0Qm-~ohrqx$GNOQuO7?>$Gx6nZ$leLI@elj}S} zF_!Duh*wzPPD$ZcAjKN;mZ4d8RGmi#2%kR)o$_O7Cj}>&aNsR5Ctlb3Knl!oT~G`c zP0>DA=0=2A@|z32Ap&<(m>Z~ipHqrxGqt3lXer(fKIq~45xcd$p8ydZVE`o*#*U|; z%o?$C40FTj8?b0xCEX!L6KGyOUcY35Sj#FIyWVv%d_IKEF??U*|4{I9|DoUy5&xy& z4Y(&EnX%%*&RD8AA<%%FFq;0b7~8V+;+b@dI;rttUoxRsH{F_n7+SpR=CTutqk>FT z__a+zHx3H(^Q0iuryTQmzz|VsItC|!LKYh-w?4{&GufOEUWv(2wuUcmXdPZ(5j)#M z{PODf#VhNtN~m^>LQFjlk!NL2itnmEy ze=E5_#v7%dc-wN_c(+vXKG--bjKE!+MXZ$;%loylKM&H9=OQQ#c|~Bt`6NSM+^r$a zdt`!i%z{zrrtYk_#Bn~#Y!_0@z!Vg>Ty4)`hj0Nre6VaAp@r~n{I4+YmC$ZdDDyJ_ z4{!ZHJp3=?mqj?(NslfA$F$)??SX}d6@ivkw>kyw-(KeGe1uJ&197(P=6wQ)X2`p9^FOcQ4?+8{9e=ljAi< zoSZW{kxAZ2!#2o(7PPxEhI5E@q1~Tu!ft^$v0nmc;Px5j5f)!fY__w8LG8wOQ3Ue> zj>&guKrg~CLAq0Q8*W9WW>BCG`P->+Vw3?rP=`j(1saD*m;VQL-8zXc;)^gQsMMUZ>kV@|VYK8g;FtJ%#%^YNGV$E|~eVZ%p7{V9TQ&ypsEls}$Ism4!R1fc0v2LOhB=HucY zEZ%ZQs2qT^)BZ!kTbip^kC3_k|4I1C|0Ur)hKXVTDgHykyAosl3(EKU0}^TD9s;sF z{Fj8s`Im$*_=kjVSKRM$&`}yp!?iJ|eIZT#0ohG+HK1JZT_)*+E!)lWmU}!KLJ>eG zDuljIFk(?E?3M6bNeDdym2>O;2WqPi`3Y1iJhr(>32XhO(ET(n);o?i^4^R-JnNqJ z3igAjDQdRb)Q`D1XY zQ@r{d=H|#A%htHI>z)r`rt1>Ty{;IaLNLBZ zj0p;995`wkI;_)GeY6i4+F*je!7{$p;_1;gyut6?$`G@Mfbfn`oh1R0zQgQLducb{ z9|<2?WaUCWq5Wm1i(wHZn%0;LnBwckbc5C zI$pAy7}l*y;%Gx_+KGEduCTlnThsb)QI~`o!!z0vJyUgbn6gR`rp`kSb~%Tz~0?KU*9`Ln1=80K^`w`qXF1^^{#i10?t7j%^;BCEDzug zU^^}F5L6=tL+MP4KeG>#$r1*w3#ArEZh}@Do4_0mlfT1;2iIk<3tTORZIW51{-r#Q5F^Y?TRDul!$?$PgN#Ks(d4IionrN$$Vc zJ2=Wnv6qv6yy$bX31~w-W5GiEcw$4Gf@np8;D3)V5v)e)L=tr9;|LnXp5lKL~$ZJ zOXtq%C}T^jFtZR`1_9(dvJ!K7Du8@X1CZ~rcpN=SAl;2ft|H1q1>pPsy_7yEA#z?_duC_S>6I&-Do6v zWu&~8wT`1IrldtnZWhBzAkUbu*z38SzTJ201^D$|z5}^OuC#NbO{)gqEMu^%(|oynV}Tp|K}vzn? z%|yqskI!NWtw)U#ywgtM@MM`cmODMT!O~MHpBtqbuFM}WjV7Yqbb1eN;SKQW>JweW;mC z`xdbIovauhekx-glU2}g^cTsgXZsNlzI>p4VBOpy&|Q(Bligz*@7X!E!B(0u*O=oa zly_R=U7ZWY>VFl7KkY;E&Zi3QQG1zECmz}^orIi4K8Y?_tfxtZzrQY$3JE+H-bv5+ zrP|BVZ;}fCJ)P^}mwG#+eE~!j60iR!>N(-(llWk)C7et8S(>s;A$21Unf^~``TLFZ z>6IMQ3_o z6NrTu>?+Vx+%|flcFJyB8uZWciRi}z?`(>j6L8c7TGTqISY$|l!mafY(F;RAz{$k8 z1i8lAG~7~A$st?qGWCkLD&JDhHtC9*|3(sPV~5n|6W-D&Y`Mx6D2b@|x~VK}LZ|8I zYNDKJo&3eCy@ff6K3p6?8XV%AOZI-DGqyuNe zji#j@SYcX5HH0)D`fxNZGYj-YL#g zfJAz4GMyJTAIKK*##WE=mVX$xE)tBdHRx)J+bJ{dHJPTpXZm@ij8y&T(+Be|l#x(7@coCh+O@z{2%yIw2k#*YkP4gg;;d zK*u3CwN-sVAd7H!g4~f#I!y?(kaIAKprgUe-FfRanD%U>&DlX+6OBX7O->=0Dp9+W z=!rwsxS!SQdLk7Z(5l`V+_0)DzUw9^i9aPV7RuGBk!H%2mec%2nB&l9QqZtntWXW8 zMaCn>_m&(O^$LJ{a79=dm=jTocmE<}2f$5FD6(CHvJ9gUKSL3f;jQ0DQV)f&80pkj zb*klUxK6FraBNoQFGTFk_y{xS63<6_Ravbkw(gl#Q%QJ4q*=>{qOEHr8CxObxp)8> zGqMNnyUO#?`~yU%nrBft)wFX$qN&`pZ@0PTu4Qq(W9^3+PW_5!^_ZTZUHVNT4bHP= z62?&M*#lb-`~;)pSpnjF3c(WYm*Nt@(ywwG!!$TGz`)l8gX&UK_Xj330gEdQYFZH0 z^$eBI#{jmLFVGx$Ybql3^z9sr5ccKLES)*oSqR&!&PIKH*Zu)S#UQVxf6rzzJKG^g zbGXb#GbU7?nmkC^VWirHR`-kBg7C{!6TA~UypAk7y1f2zMZW$(7$&hF^`B4)6~M(mmZngkkUwQ z!GuETWnOg$(Aeu0b72C1OPc7QmJpv9(Y_>`6;5n4%{MtmWUuS1C(e;7RzW}lpNSK> zigPTe5LE#6z`od+8Wfy#-rU9BL{o5r7#*fp?ZgPUxWWS6!gaujVU18pWs@55lEofR zbw%&i#h!i*3#R?S2%Dh+bu7QJD%oUQ6LWc}yWVSIDJQBwn9j7{#HZyB@xVBBUU$(% zcW8v?Zw=u;WhKDuZVz)dJ-(Vr1j^b}>SA7y?rp7dzsVNL7R@ds@ORyiqNOD+gZyC6 zGjsrg77D#cBcw-mPzB8E-A^13rLnJKBp{HUNnhMIAb#Kr>L`?32LRCjkG;DJszY5D zHI2Kw2AAMYaCdiicMq;1xLa^{cXxMpm*5`Udn9wNx%OV?bXQmPx$9j;6%>OpZU%7i z{qO&LnjXD{+NAnVoqo3}8wQx2O<EE|T zcdR{dU6ELQ*hop9B^$X()kqJ+-;=$1hDd)sU0(#tD%Djx`EXe>RNt)${b95$C3siPoC%zpY=h2LUS#HbxcwkLwRIu)1K4{uB> zE>%x>-@+Y>w7Y9vbMt852}{%-2nPbJTat}E_}<=wgn#$TTYv37`r!$eT*219#QO|q z)F~b96^9|#CK4Ngu>$`+!3cit!%q0`{<-K^#P;9n;NN zw7t2sS#_Y4A!f!veh&EoWftdYhP;Nhk~g558e`!G^s$c@7qwpDmIfY z$&hvUXt(&D=1@{UKU*&T*jg`6r%X5eejdN~p3P_1^%5)=hnSG~0C6_kQuq9f=MgLg zls53~G4h5%>@Zn-9^qvhcnu-HS_)xalFRh-)KU-xNBBgzCB*+@vTTt^?%(-4S$6FE z|Bx&LcDnsLS+3Xloh&=JzWqNW%e5f?B+J>2UAi2-ile4TU|56#OhTk)CbJdSF?hWd zj(zrOrM+|5n*sQMou)fu<6GK3tJp`P_#5 za>=WkwI~KGnYyn*bj8tYAzJ{Z$D1Sa45_tB)A?;KO*qPnlgHr`~QErgTRtMAk=FAXAsuJ@=M`JBna?*o6 zqF}*hLpCpwLU`00U9MpxF98cgA4FsQu_s+JMDdfL4*1JL-`7{+B8@_J6su5gN}+A+ zWoV~%LX{0+ri?f+=STaqn#9|vVn_5Ce2dK8La_)(kE$W`d(?nkb(nuQ%l%0I=bGh4 z2xR*vG1IB&vQ$&VIQPNnPs2sB0u&E2D#NSoQFi05NcZD8-)`nCNCw zdP$a~mSjdXP~a9Y>Q(Nqfm#I11pHnD=UAI6?X`sLeA3f^oSp*e`ZK&eb9?d{3TEmz zOI0ql#ECl)X$*`d!1-#7qcpxs6wB?Kn;uW|O>#m~U#!E}2X=o|Bn8X!?&zwi515#W zL)0}LpK9;BFncz(Xhgqj$4|0(D@NSC6bBR=kDJ-{RdT5x#v0>k_C9in7<{E*B>N3B zSlQ#Psw;rW7KKv)GpnA*#sJz>OtJBOJi=J%M--}>tb{k3UI=6pNS8`iQqF-vFD|-7nG5e@d=fRXjN}?mf-5 z2Bn#SWu$!z8&J7gverIwbcML&Q_pNJzG*`Ny9LhxC1O(Z2Qb%DKE!9c7HG2qWo|Ji zu9TyO>$9UENxZ1$YZ`G^-VvM^L7aEK=Es5D~*CncLcb^4<)pu(~GFzA-c9 zhUQNcSw7*j4O#1=+#{S1^y2}1i%$RpS8~8iZQTm4$m$&*RyAEALz4P$1Tp!N1}jOT;wc3{Lq0U-x)kfPQWw2DVuv?+t2B1z;~HFWB+ zg500EO@VtZ?J9AQ1~N{4SZ;W%Esjz_3t(ZxoPy$7Uqx^^>m3w~2Hpi+oO9t1(XJCHjG0eYg)e$U1}UWB+MF10d)S_8$Q{K~+nbg;wufEKR?h!q=(^F^U~3Y#V>iZS zJ==l&7gCTilG891)^nAj3!gcDXfHVcD}0llzlt%QWG>(!Jdm;Nl}$2;OkWR1g{>kL z5;7t6UD0PKN~{KbdM2=V=rjwI^1*a(uyUR`dhQ(beEDo{FL*d9BPyTmEc0D_i%YlE z?B^oG(}4euaE_1fN?7M(H`pbg-i_0as~Omvm_WdD?>9Y zUVYWXc4jmp#M`FOgBBR4xDG+u~`K5H|04!{n6Q<-*75Dt5{nD@HwlaDzIbri5MeRQy*gkVy>`I5Qm@PLhMa2R0ThtPGB5~?H8yX!Un<^Bvc=hpi*^&=s+_UnZ? zYa#hX-|dcI-w@1Jz$lv7UqQ1*rY-V;*(s3awV*VdcOAj_}u< z(oH5<{H{EFma5}ShL#j)vl1tKoo-N%NR%bwWFUibdfHGB1+e+*r^J+kqHNDE!KX&k z+sR;4S)j3(7MmA?U{#HkOHCEc;;w&U6f~v)C3k6_IliDO1xJkZa z_!1x4xoGd8Wk1d`2Ba?be%Oq)8E5hhm!N?Xu>kL9v-xc;KR(aOz#h**$vNsLM5Xg9 zry$~1Itoqm6uu8QbU9K7Q&^Rfjh8#22GF}hsSom#HzwMP z;#B&6yv}Z>y3MdyqF~$g3SO2*YUjrZ1GB0h8fHDP+@MY+r~A6+eEHP#KurF%Q zg`CeT9MS4lpd8L^mJ?h;W8T3(pkQ@tL&goEE*^n4hY~||97VOgy4=^;St&@naVY<@ zK!%?t#B=_;K$a#16v&_b0R?h2p+BHNM%n(mKn79%T_Ce?019N83ntC@zY1i->OTeY zgqDhDLFFpZe=U%6+5rW!8;90XQPknWzYFB_x5;Ob|FuB&qWY^qW`yr$cA`Pvqolw> zij}8c9WBCPG+WD@*kM5wqfDxJ^pYw-|MC2;wJ^`m<(MAFR;Fq*THH%rd9N3GnTUK= z`QT4`QNqQl&;XyA1_OhNjM+4Eomlb|Y;L2QHgp`k(tJ22XN5FO3OT?+KZ*~)GKd~Vh( z*lUbn&tF*l<4uoDXFCFm$-p&{%QMh~!yQ31y1|xzViWJ&d-~nt4uJqKn`G`+jsY&u zg~$HXCG5=IN!Au7mf-Zjr>32*5j=V*`G#4za5#qLB~yjzeC3d8aP)LwX;5|b`amRO zNiYNG*rT#M0@*6UkA19h)?HH-$?)ppNN#i6Ei;G~_mhcW*Nq9SDH3{>P-guqnOemz zhP=Zw)VrwoO|n%(nN?3=TF4tMs@ALEOmKFq$TtoM%KMbsuHe}@%%D*E`#W0{r=q%+ z+Lj4w$~qw96n5cdXY6X*uQIsW4M9K3SMDrbYZ@~}(mYuX6*ElH#?HNl7=IjkFMtDf zeQ9mOn9=3K=P_g_h1ZDewxY_de5A0j5V$df8eS!;t*x+Q19jr4M4BT}grHLTs7$Au zv7yh30#?cta|vR#V)dmSmEce{PEZ>qS{C-#YQQn6OQ;u|)7jscf6L;}Pw=_K>_%qW zhSoz;&tgH3oJGQg^vgvH-{9^;PB*R((cZJ9#kU2bV(#7d&%2Ar=c3i^wBE8rupxUH zcswh~e|}gb$!p_C2*Hr}{7{pMF^Mc7o{hxq4?^|&`T z$T48_rmr=h*7BHaENEZwvnh9A&;!JKU$JG`O)k3Ql?kr-Ke%a|%1-j_3$s=qmkPQ5OM1Pd?opt}{t4@#!sErG0hjlAox_VXY{(b-=tN$?v=05w15qSJaBBW^dN6LT_w|A;Vz><##w_{ zG%_A^$(Y!>KR_F}P68U7u#Fyt@@LzqXbQ$9`9_V~nhVblQ~d^GTk<{N#z@O&~yLfK@X=!!);6s)>Vv8N0i)#91PFP3m+tN@5rC zCtF#!qACDrl95X9(eJJ@zCoy|u(bK9uvu(qEhHE$X_}t=^wA@2AhBW%)iv|2t`5tQ zt-tSFzFF=9!!PwQtJ}E6(X2a^KNMjy$PtxD8P7ymAe1Fdx~AjpQclCNBENcF z=PiLJ(aMiiTFzEipJH>%6MOhen{ufJjN)VJ<8B)g7dylyPynxmMKgtiqi3)Pl|t_g z2sO;*QLTnPPjNpN5qXPcAYHODR$%Cu-C}ReXhi@TO=c1zR}LUKRQIDG=d!?uFb?tzJ^)bivrm~oCRWxrapxW7RFj-OeD z9OVZlgQk9CMqbR~b0D6)Mv64~$mGmYtL1v-ZMK2|Z6)0AlxyR;-AOpHXosPsja&t% zp6#vJsF9Pi`b=1tgs)mRq=J(c`%O&(X|iR}qW0hzq>L+JkE}SmnuY*xAl@V?5fmtU z#xL5_k9Qur71Bd?DJXO4-BN1|#2nG>Gm4{0zOU_E2s=$WuLR5%ifQgAk21pjY&3h@ zI1Y(1>$QqBni=IX6AS`|qB@S)(WfF=LfS=ua*fdW|K#@SHY7^|g=z6}AA51wksQItM_pVlwT8Lw!LPN)S! zV3BIsAVDbL*txuzf&sv}hs+8w?XZgb<%$n8J($QlTt~vHzxP$|ra$f4zgl!R zSBh6Wdp|ITPNYqL=|&vW7)n4ds4dl&Z=*^vVw;OT*U~*X_PRrd(}}5<#>pv^!D*XWMDnaOeaYZ7{>@7o^3{lGLM z4It23N^h+`KC#d}cWPK?>TPPjvAjDwxX=9~xwrku45V!ZNbZL<(9#zt+m$J64354*7PIY!JN8e9b9RrHD!rR9>AP{pRl%_HuRb zS$&v@HGRVKt{0A3^%;Al3xt2lu5k^*#l6(Hzm_ zr`?F7naao=mEUUN4->n@#WnOnyHCwq=eS*E+3)W{ubzmHmz=L3>)k+Ho$Ltf4X6(K zF>X>1*&tFKIFJ; z2fXtKMr500a2&+4A0(oi@jtKk0yySjdl8v9l|s>m>kYb`c0>6-OUbk3Y~jBSoe1gAy?GO z`x?bjt{*1-(0g8uHC}7KSnw4q%u!HG09`x!t&B;1a6p#??s;xTZc4;se>aQfqcx`0_v~joby8dK)p| zHk2k!uPDht0c;8kp3PT|p;1VV3s_3j!+{~f)}7SeEAmBp1^t2${jKZ|j_1RrF8Js! z9?s{dyDPgcFYnta#cdQ|04!6LMYnZ_<;# zE48tEdv;x6cvrGx*;Sourf;GYUR@AkEJ zgaa2)VbfFmmkL|T()M?SjsI7L4Y%>16?W-8jN!;E?ba5c!meAHFD(#?BR65zg4;45 z#po1DsRBe7m30k`v;pFL_3DzD@@Hkv&nG!wKkKiBNB}gE}*I{Is-w{UoCa%RDL_ z54f1jj;$(8m45~2Wq|Jf8MC3XbTt5V1=VG+6WKl?eN zr7Ynwc1Vc9Pq6+FqS2L!2(&w{nBLqfx)o$cUdNNM<5Na9SM12)oj0h!^b z$Y#TOhEvJ?XJ;7)igYfU_*h|jF`m=$8V0=xu_Fpy5I%-XW2q@4DhV2_V}8VK{xRha zqDyGmhVzo|m_5fw3!f^r8KUs5$2C5qPtB~$(&Wy!+%$O#SMk)7G{Yo8k=LpSz>o-@ zKi(4(F3~kn?6H-Lj!B%QQ(P^*ei^+O1&yn)NX`F}YYObwV z?>c$5Nu92;e>)rj%+GdlxwYHlrkj2T@qm`b&)-lqX2dU4BFkccqZq35HW=F~4fukv zf-(;2Y(_jaMZN4hJ1&&a+U0Y-Ku9juy3yO1jcPX8^zBNCR}2i>R*x1G_Eyjx!E&47 z&EB7^J1pU!teaHyf|%A3QX=+u)_tq_SJur7|0nC7)%=}x?*{+xvTnk$e`nqJn}4!y zCXyJHzq0Ot>dLf+F9F1?YfCI%+}63^{hvsMUfWm$>;6rgH~fn@*Seed{x@;{ucW(? zNda(HzdHQ`0C{YDmvFbybo5aq;Ulqw%WC!a$F9PpxtL8h{_2+c{(OkBM0amt<#c%l z_SsVNEPL zL~uT;RQ7@4R`C!ae2*Hg~zDrl+1>`Q}@3 zFvB9Q;cHlM2g(O=qFY%zE^!8+?lvTQ-#7met49OYqS}6h#%9>+1;k?5AofaF*(aEb z`%OY_2$T1<-)b2?{=>ZNO2{}K`Lmp{dOC7urDBYLQ8qUbNBMjQ~ubBE^=BWQh76ui>w{R;wuM4MIk^ox2`I=xQvb+Yg9q35}k`=vvO5t zN>-L+YG-LFR2uD0r2BbY%wu1?Q{4=u>0v)qA{?-7y!kh5Zn4BE8!h!b-t{N$zQ@l8 zaOYFAF968=>(_liqFn@1J47Z@0s@fE|1jl4ul9d^Nfc>@H+$qDp%cUc_%VvOoIejk z!Zm;TQ9mGX;UFPL9|kbdqou-Ha{$j!+hspl0DQ(Jdxr(Qpw~#x*TDce&K+zV(7#~w zBG&&7n|J(%&9kVd5{%g5DZtr3lzO;8w8ezF+}A-@@usgy^`j^W#o9-a;J3{`khrP0 z2c1A{xCy#NUG*b*fLU{mVk*!|*fO8-J&sBAEW&L%QecS>xux2KL45%lg-IoS{`{^# z9rvQDhtLeS7rAZOM@%7DShr#7@f;Yp6Ci=i6-ts26klDJhV>5e#d&hT9yFqwNi&6q z@QLAjyH>$`k4~Xn<#xRU5Jf#s!5ZsUVk-e=)u_uXB?}Xko4;Z6PrqSv9yG?^$v4My z_QA&yqZ1k6;1WXick-QF_b2%-zx+G-HiWm<;v(^j_+OIm0@OdrH}wW{>A}8n!quy= zp7RY~e1x2=BMmyTCSud&F~iK8A{*}h@%534@D%1B*BoS5i$`cEn43uR!0{>yydm)x z1aRZ~{&g^=7unSQ=N``Yudm#*$gAE9>sHec`&jhJdtS-smzTT>k^i{n1Y(S$l@lcY z%{4C)pyiEF7iK}AnM0^p#wb~?C7kUbGQTxDl`_J_jm`?y-5VdoSu+ zR84UYKGu%hU$D91U$D8S6jOV96mUhu+LM%TEIz};qx)%cH%B9Dsl*A858b2Xf=)Mb z3V;Qgvhz*7I`nxR3BuJ!JvKY1LY+C zWG%?{l{+hXWtlHMNrUHa*xXJO0Gs3f1Dm(^cSQAvEW$G)UakXR^8@R3i$F8TsD6B4 zBEC!c2#59|AVN>vJZiv4%--ePpxXtE1Ao@-KZk{?CS%Hf9TiG5iHUfCr_0STjU^3= zX&A9twLY;s)rt~>&-B3mCV&L(%P2zP7~g`H|C|-NNT@3{p;sbnNpQ|dr98KOoS*VDtH52+ZHGIou>9ry-t$c*5uZdSIBQ_Xjo?{5Nbq za$5m_%_jyUMU?))=Ko0peWpqw1%%zU$;=oPigW2#iaaDugB$7OH`DgH9I+ukN)siF zxh(ANUZZ%3kXHrk+{|A6TjA!X3FQ`hT%->+*Tzou_|o4Wo`T7_e4^sr#s3AHM`8Yf z&8ubua-+QHhyH=h>F)PI6T8o#@ihZke}Kt=PpOrIIv)e8N*O7B<6)%84gK|cKlj9H zY=hBTy#n$TS2KWDX0{Gngj2A{+Jcuwv1o?tzrg0-mPj7EKm+HL&6S4xHsTl7lOW!( z>%wGf^`Ed5{Jz;G48^UYL+C&Yri<>xOX`c5vIuY>GY7zw$8n5aqmXHb)G}BceO7Ij z@ZECoccfry=$>sozBA?5BNOZU$X84f&TMyo)AM~D)8lMaj&xvfagJFt1)71-2AZg+ zX=^(O4EPzFFi|U>#<{qZ5EyT9a%O&NRa_5ts8fv+SjS(}h1FKJ_@A&j>oZrYm@lMW zpI+M6X$T9vOR`j_61u?(wo|b`u(=Dm7%BiZAB2^@Ku2B`uIBpUu@I{!hxpEsxzdbu zK2VRq=QFDWyU}r|sH2IDcJLP5ad;&<|5N9tllMgFzrf~y#pr8Gf5qrtsrr0cGOUE& z^lqFWWK9lR9-S6fY&^N>(GdX`B!>31^4x`|< z>fB%p-UIwSUQ1xF5%rPtXOBWcclmOCIzom=!aRDWfp|e)U4>*f+xr&PNIa;-TW-Zt zmAS0*=^LrG7}CL?<0sGs+OMfXzO7ntOVXS}YviU-0ZPx->ag0P@dPSb&jIQ5;+ExX zOKb)ffIT_Te++SbPp$Gp7JdLR{_VOeb54IK^%H<62Rzgvru*jywk7`4HBb8OpAXds z0(5f)>WV{_Q^DQTTVF6e_->>GGL4xS}5Y3vTY!Zd8G%c%|A~@Q^lt^UFq7%WxCXD?#9Gh!=W77_KyQ~_9G5O zO2p4O7(Fdn9H{n$&3|y1K$|IX)|u{0L}6#f=I9L=B%=HL>W4P@6R0=3LHy&I^L_Xa zZ7Pz)iC&Q0>_V4lGP=Uy+S?6pm+ckPXXN+vJ^LoV!BB*WHw)H62|#1;I=QmVHxl?+(4iqx;o?F$jF|3~BWKd$-jIK2?;Pn@2G0*KQ`?QOAO2dUf1<$jLjXv^ul z)W-7II;r%?V4MC!Kza54~`~E#l#+g zRL81?yUD6UsEpB`Gmgtt2YL7mSC}) zz#NJh(NcC^rB!W%{|DjF7^^F=2}?dRo?A1>YmWDJss2Wf#n<2;zp_7<(Ct|_&!X@4 z_7Dl4)?T>#EC5_{tx=EPt~p}ChY)5;bv(c|e->v=`N{g*HCJuKQ7Z$u=641Ea?M+G zf4kj2B6z8FwB`!(?$&^^;#6Y2$sG0Ge)6Zz|rYZ1q zV$5qHZ(t8EQ@%)!B1TgIa|Pu4Ud=XO;wv`(_LVgp`1RP zo=DZy0$zA9N-CN7WLo?S+$$z>t$xtyI+Hw(RjsT1@|QT$f^19o!4%?$(s0aQt(zw2 zkcEefnMnoR=qVmhIFx8`4T;%|Uyt>br^lk(oE0 zLt+HXncHxdp#9cw0M@+B2+DqZ*@klTi)>G7^)3}zRKr}yI1*a~pz0@7$iakcq652@ z+m6IDSA7DU80n6 z*ms}>q8FkbV#<^`j|XP|l+ssAtDzfM?^uV`q?uQi1wmr%F`kkm}~v$?h9XKYW=bcMg7{ze2LU+K1b{{^)s{+S@RC(W`C6;MNilardLG?jmsEy4fr7fFwNV-o38Z# zGR*@mRZstz=381Aq4KH1P1Qz#?yRJG}+ZuN?Ak|9YyY;&d{jjJ_ zZqmcUKMMzsJ3NVT5T$G~q?#1YQ>&7jN&-f|P}IIEXkvjQipw2EG8Hh0;?;AH@4K`f zkx{qYQ9HDmhN-~F5c=M#AgW5i79q)W>(>3>{|6!eAB6mW5c2;)$o~f+{~v_>pGyb- zgOLCK0U`hJy?^g)iNqbl9Qt@Dg52!kx^vxC;!Y5TV#nBF9QURmwtG|!K;bOg42T!2Gr>NM7i-b z{AE~cOWGx7b&l4};)mwcwi$4HccT2tUQljpKXzyE3^ypMA{=;{VM#M8b#h;DoguG= zc;lkzV#a|>Q@SZjQrOWqJZK4H8xDDrW(|JfaaRHE$=+IU`)Eey`R|`;h#Zw8n2%K=Fn~DVvd2R!5nAdyP`gj(fQMpISahfp$%Lc; z^K-XZ{_-Sud~}>fmt%zQ?g~in^3D*9eFB>17;w5Gw-f2P_bfckCSISxNSv=+M)}&3 ztU#3fP+|`nHN$8WFq~>&s498Go!gSE8X<3F&WO9h3hkR|Fa;vH#_B z&i6RYcy}di-Qb?M?TqXsZ6p75KG5MYyDGmPismsD5T~QTcD6o=_nyKw zBRnXuIIKIdD;w^wyLZn$G}%p;2-+rE7Za-QK=D%sY-pC6g4ea;o=*UM_4pv!KZKu4 zm0eL$^y19IuK$giH;Ey7yHfY-!DRKT(ALPX42hW4-*zgkV?o@4!&4H1T3}OFY6-4e_>IQV-s4lh8IWvS7CynSJ`Ga)2;F>Bsv4wP&8Iq6;hZk zw>~JA8Q0BKWgtSTkN_)i0HitRb!=Ewjb7cZs_H~~MvazkzHl$6=Q?)Zi#~Ow49*g^ zT5U5?0WSF{yRNND%TpBTq4L&?t_>S&jVs~&;vTem}?vXtY@WD(BhS`}q9&lDC#|bVjYQd0l8|w^o z5{S~Wlv!;po^8+UBQ^AWr$aP}PmDap=^J7+JZS6@=1j+Cp=oQNMC7zJ?yNcPvZp)n zTBPfb4er@{eH;^rQ24dzGB!tX*{TFB$&XbVpc%(YaU+*snnQ+>Eu(-9bPZ2U6SsZ zBFsTv^s!Nwoq%4;U^X@8&w-YZ$I8&t> z>8!q7vPi>mPg#sSpJ{qh=CP$&y1yrR(^u$w)*)&K%WzqMN{CqDG1z4Dl2{(@rEI9d z2AJ=6ELJhoFuNbfbuAL^O7JScfd(POZJ@vW)X()ew7-&-PSFqJl&hLTbs7d!mWmad z&mtrkGvcFtzBU_~1AykF3$y3W?Ck}K zbWnC=TA@EjPHa>Ktf-~ngWv+B7C!eY<5m_r1N4?fRrmR;1$8Dnf17W)!(qKus{H~A zy~gU@iE9m;PdM*`_;8%f>I>1~fp+V4UMnnJdip?=KM-;>xKL|yP3?+0dV8`y%5XL_ z6d5==Srb|)^yHV~j`GRTtGVLl9j(VJMYFxQ`@9*ud$8>-ntD`~ zXnjHpcWo*9UC!*{&K6cDnaak~RGD!pf83J0wNRu=2RftQ198^V)kSFuJ&oDG9Q@ny zc~p{*KJC)mPSeNof%ns6_fylM(%!*lL3Fg%bZV6QKt*mthl2O`T}Hxz%P{-u=3~Qd zQ@i0Dypc;;l2~89sXh?E$5u zND5-r@YbkQqhTV8I?IL%hS^~(f;ED&;6Pv6>;!ZWzOpT-x>Mwqu-Ry(nWGlEwX@x$ zkzup{N-Dw5+qae32fHoC>o0~&C{D%cOK_8TznqEgdU1DqUCPTN$VB|u1FTzpb^y4( zYkOUvKF&U_+kW??1fA|TcQ0i(zE^u$zDN_V5Fd9pGl*xe1h@!}Uaw2Oo=In{Q=l57 z2BvoStVjcv11#FKaFJFk#R1@sM9NkgK<(rh2JYqrFoS!-RDvT3TTB9*MC@)VGcpFY z#Bx7eL+1vE<=w=JLtCSSMCzTM2b@Hr)i?7=6RcvxCnSy8aNw!tz0~=v;gfw8*xjvx zq+mj#Ufv;b?FPul>`fl;d#ln$-JJxh4(^et%h+rRe`X1dE zLbfkO#s!)skbZrAxj13j1dWODgl$$N`NgB6&9-`+HGQcDqkpPCgNsG`7|!VL$^3Lo z0TU8Z@C^dy>`z6AQz zX+x3O@DN!Z%9vjeZ*4@g3bp-};qqBnOLcw5bQCO_6kOKP*Nm3F%Md||H+J&|f8Pzl ztmj<*f;j!rUW{Yq(@#PvI-b#=I32cLJ&>VX@FT%NbK6vb(KCPbu|Viom5KAwsH#)p zaPrf7MnNzpi6a#TWhGmq`L}2e%U%dY0+(up3C@xoHDMV+-1IvMIMo4Ixb88htO3Hr zzHQLKY@`f%TdIw7B(1}C)v=ubi?;@{hBtXO@b7&Su+3hy{g8F*Kx200AqgfgKTU@m zPxiL6yNDJGGGcJ?A$=LNjk3OUtkV>4Ge}HN&Yi#1Z?v`AfpHcH<>~XW)(3FcK9m|j zwkZ`Rx5S?u{EXJ)h4iK>xA?LVGeR*BNyiF9yETyi^I;WNg6^B0lAgcwqT=!u{D6Ta zzCPARM+b{LhjP^}RVAmTD z^Lq^!Z?{~ycMeon(d5z%lI&X95c~yjD8!{CKp_xtkn+UK8@1Vj7KXmDqVPs5At)`C zT67*CyQJrFD&uZh0#W8LCurLp{?Wjq!VPeox^9w)cvC)x6?UHs5k4^()HZ%$ag&6k zUV}a#E;j0H2B8pgf~Mnz;EFf=u1u!6@zrXk`;&|9Fss*~GjU4!LcKoZcQD~&i=Rb6 z=on;Hy`)kCDF|~Bl75Nh;yMaFcz!PSdUwmidd!gCrI{lPzU(EL4#_KR@C6-x65<;{ zld4z#Lae8QlIgIi86dyj{o2v1kkP;TIn*P!APJAP^i{m+-}>>_@R7A*TbVMn8da$N zAlO3*vn)JVx7LQoXwdI3ILoKnHae$}_1TkkP`686N;xoy5}ay_$ZXe2E!qn`4LoSG zgh(*VR!fvlw6>&oT=g)c-psT)YEq620pk;eZk>a4hs)$%q*5qokf{GuF5)2{x%`FB z-@vP6^%sLE0u@`qz;1S1601w%BvJ37%K)scz04)%0CdTICY4^~Bs;^Zic|@JT zpa*#+_(fQ+*JFGL-uL+JvyI2?O8iY({F1@4c@Ym5B3FUhsO`o&_Pc+4>uqK`X!yETGUm;6c(oA@bg8kF;Z1ht zS5j^Tj2z@&8GbK>>J}n3)=kNwp~~h%7s=g<*Im__lqEl@fqTXlzF%r`X9S%sHaBB; z@C`r56wbKM-?4&7ddI1QtV0Fon7Q`v&jKlU@98^6L^J54k7i_84q6!-&o^h;O|lS@ z8V84JNq4PtgI$5_iT^^N`qA0>1-5A;>h9U&2FK;)5Ef5{nR%ZKn@d{FbU0Subs4lr zWLJ=Fzv2?suP#WDzgmkY26}yRvKFj(zhJf;;z8R@3%5A9%!%QGQ(9SI%c(ZoO`+on-Od_os^w-j8tIRlQ#f=HDGAIRb4)sntb zd|G64km3FBdEV9Q-$|{R48Z+pYCHxHq-RRo2$vdv6|jycbchz?Oj>ksMAhohC2Nw& z9bpEPiYcr5T&cFl&$jR2)JqiS5<+4{abQ(A1V3NSYSEq@pA>rXflPK+(r1|rs%8et zLz75jH58L!nl{`doo2Q@A@~-hAmo5oWfidR`U+tZHT0^K(z#ywgEeE@wzFf~wr$(yj%{iZHY_Wz87118-C2+2Blz_XpV~ajA8CK#5Ueic)UQ_jIrm&x|4N zl7z-o5IX&)t)k98{MrMIk0Qm7w3s2niH5oglBab&D$@fb8}8K{z7~vyCC%df?Hb@Q z@3!DfgfGjR+$uWy+!>B1kDiy???o4NJcXV#G;s+V?P{x;F#VYNzCrK^ft?P zh>}};figBuaza=KBO(;1)l^L&gXTbLuT$a_?PiaUS9sW}Mqt-B85*1yctf$1P;c~G zD#u;5{GbuJs)0ESAmDFZ7I4<3x;rPgr_!svNWDNjwF^7Eq`IN{$fPQ9XJ_$ud}K`cfH{`-f_`i%p$ zD34?Q5^gXjCNpKtk5W;(*U}v{*doP#=^x^eV14kYF1|^f(?uH+J)`}LKI8`Cc*S-m zK;YlDjkKk@`D_?mn|&2*c66X-igc$QY|o>{vHK(>wHYq2kCS}c_5P-TnXE3flkt45`~= zU^%Wm`Hr8PzSUZ`{<^m7yz-Nr7Xshe2BVti;wnYs?`C8$cV*8efxgcoTH$O_I z19lF}#KQtA`IC9r(hW+sZgjUJq`qsQn!DZ1R`+#H59pC~$a3L>5S+JhYKGJ zBPOAz-&moa{ldzw46(u2va+1?xNFR zsMlRxv!k8sY6eyZc02IQeffgO`ihG^Q`jkyv2QN3lX;qN$-6{;y$yRMii1o`73ZM4 z2pqz3J~>7w7IEupUkxYOM~PHX@c$s3Moa_9}S<{tW-I{5WSPv!V#p;|;fK zmL@!9r(w2a5r9ccp{mU<;Rk(i9#MJn8K|ScEfkdqd#h&H`h9WGnA&5`qXXP=aQ81I zLS+Do^+Qo4!DJa8#h^QyG9afQ~;05je)^3*WoJhOR^ zJun3LFQQ0pM>LkA_S)Dpe80Mf!OOmK&o`f0Di?^}J<<5y!^~!9UsQya?@+YXWj&a% z0{VO2=qR=5T!&~3k$xp?@OdzSO*Lu60btHUmO@?uFF#E2OI=c}X=4n|6T99*d@PY$ z?yM+FFk$rE^TQPSY48e|l(O}BlR`C5Vs@iW=>t=UnC)u7=T+vz&u5L@PB-V*bI*2G z_xr}s(OWE^?$627#{(PR_bb#!So^l*%H-#3K?m;p2>uQ_OO4;>-MH5h|M?p1Iz~RE zmQqkY_-w(g|K66Vato^c7=^$5bS+22x^jue#*L>Lj1^cN|1S?BsK8t9PLpzzmdE5sb{Ri81F!Zf@1=sG@TOE`N)Yt`d(KIz^ElP@`MEeq+ z(QaOODd2N)7RsfYTBF+WK9KJt_M7qDN&Qey)bDw9ymrVB^K8TL(QHLDvNaU0IiLaE z9opr9D4%K7UTej479r%Zz6OadahQ z?u^D*nZZ)PrenJU-4b^1y-x9~2BYlx%dS^I!6j#cQ`fqL~#+WQ0ZKmcfV(zl(`7GfVXt6%+ItiRO7-@b|}w zuNJZwv27ASLdv^=@z8DKLi(WQp9LY@*f)yzO?Q4D_VX752mL85AwbZjdh$;&{Ufx& zarc#%p}tRB%WtiHmCkC6w+C1)m|89}szg6R8S(yGn_g{lpR&>1FTXDnwbz8!2pH8mJ)P21}Dq`vk7b8(#iJS89-0izFu}N?dV@K-UiB z;&nNHV4d*?_)2q_4GXO}?B>-G4etTL7sE|uE8pP#r{`wTAyRz4Bb-?|U>>K0$4e9@=H?)^Mh z6L3X<6q!u}?jv?@SuWW?HXt^9*2hh%<${>O{@8?fxn6r5PP6-8y!+97>#qDyY5yYe z3Y)r=+Io<9zcNYD7{QEfxKc+DTsaI~Mg*1N+l+GgkSt_jmk1=4o1Uac54VLhqbb@W zr&XTQWk0&!3vp$p{R>^u)XX5$T`&1T&SUXLxCvFE;gJK09xyppd=1e1D?!nSDMTos zf2tS!2pNt@P*q+# zYY|5a6Ae@S4Q--}K3aGvr!rG`UbI(`AO~Qs(bGjtp2eRj;n5;gv^HwP2+I0;4H2kb zikIEdTiTB; z%V;wA*xgD*xnvDOT6ll-QSq}AklT;hQ$zdxTaL2AdwU)0y@6==&^b5!i5VRRUtcUL z2_q$}${(fM0hdU@|DapdmKKGTvFgrOUZ1HAwV$|klfmu!|a8XPrLVab4 zRbaaQ@Q^z!U_G_=uvy15kXhgY(lnTF0bmlFl0BTPuWb-7x_`@9+evDpw{Zit@1XS{ z06S7fe^E0|%#bXk&i9gg@u7PUg>s^rS+|MasJ^*yz^bmdZt0c0=|K=BZ;VfN^l7%_ zWwXN3wc!!IXyUkDzs@#yg0S)M!yd5B;&fF1o35}%VbyGT&oWqmt)NylrdCoC{imW1 zH)R5uMH6QwYcq}pThf%!A-xr9049YG#_65XpeJ#}_r>aIr3J%@UAi2ew2aoPXEJeb zO?5SbR^TRzYAM@?_3cNv`^Z;VtbK;a@g%{0=SFLY4V&2M@43__1!_>8peC#O>)>e% zw5?EefSm3N#BOHCAW7wmgX`dFMF6*1mJQVaGeJUFh2QM}o&;VE5zKb6s`p)Nqjl0e z*-UwE?OChsQPO()mexAJZ&bCzCh+UU~{b!IOPU;B~y&{&|u^b`AZs1(+dLig|>*l0nsH z5*o?QY!N+9=z5+-UDU`2U(?VD`|W@AUECz=28c9KE~2E8cSVuo;H{(!oe<+!D5 zZ+}vR8;Q)z#$y4Tvxb_rSAFy*^d}(zF_n)C0^~%oEt-T*o}~h~Gfb{oHmzmgLNG}z zkxsUo-6OuAf>yiOL!jxRrqh8G|gE+IlAmF+T;1$%jo+MS6Ct#;GKUU&vZZfHM- z7D(#R^6ZyyU_4mBL(t0D&9vX+X;I6>v|0b2flu862F8-Lt$AvJe4Mm!*8(SUQ(^qmaA{2S5d;G8j6Eoa;Kw%2YE>lE!57%b zZyZLBTtX~p7)1uI!|DYP&e&(_hVgfr$iZjrb7<21TX3|DV&m{`64&4tiw2^*MydBw z&g}}HpX53+N%<*yPIjvab5mqPHRktu*J-iuZ=Z?=u%aLln+=U6~ zPOB{QH&^;oJD1xf?*E0b!{Fxp&%5wMMAV``rtFj!kXx=h<;qJu@aF!-!~$XS9hZk= zta3tOx*OxqRo2I6Y{$}CxBJ^I720=p=|Rhi)m7E|fkH>-NZeX`|0@7lfeads2}Mmt zW(W~jsJ>hE_8-o|qN-RNV%>C1k~I}jo5n8Bhjpw#VJ)P4+rl0EuN;`M2O}?VP+c#A z0`(yf0!L=K$w{o7=TB$DZt`coi*oN+g(if0|D-=kF;b};M9YVzw|~P& zFd!RGYOVH*a#sC}*@*;hLB_6}PYA?*gSk+c9ESqNy81&8?G(kIkfzXo*O=s`mtNo9 zO}^zgL~!d8y>mvDJs@v;YHkLd`~AQ71{lLbqX}^tTOBP-jP~tn5R=Q6+P{Th?HDFp zP@$`XBVJ$L42Fki4Ne-8owan8Z7AXq_z$?URGeEDhTS|NFLCm1I~y1?Z*v`CSCsTe zPH(KFlWiSdp+VN9QPxpfXa2L2g3Nv6xF3krTY<=Dfp8PF_d#Q3{;Nx-f$*H@lt0(+ z(c8Pt*GH!N>%K*53yBwOAC!S!JF+L)4Qw#X$p2uXcB1rCvpNP*Z;aCB%ha{Wi%rx; zGJsk21S(eBYnx%{(gjH^COP?0EXHQ0m(53nOQA~nhA4jmFx`!@ zjWiV$R;6IDY&99;oQ$!<7xr<&sl^r#0h0YEqRU%bXNRj1CORCOPa5YP_0wx|Er+gR z65D@v4&2 zcaBu~Q*073pGrzC;jo2Bzf~o}nK_9uZ)W7-2$ zE2mR(;cFQ`ypYq)6J>7PCFu112m&w?BX}@qj1=|fxVZO`F^o6*qjz!zA^G{?fU@}c z-4BgL5_gp7;cRS02gK@jMNL#^%LcL&yF$6e;B!#@gnn3T4A4 zvoE$4+(z3>MP(^@BKPy5t}vZ;RaN|~vif^a>_BRYv60&mFnUK6f`ii_89^BWEaa9& zlV6Rdh*HV=E{t7vnv~sIbX7b0;+2-CwGc$`!TwU*<$zOJK*tIyDgzC09H7%krMx;e z1unCG_}I?nSu#Ui4olZvZtUP>Q@9>k-)*}+jS1oS{$k}g`~<5YJt2x({W63^k1CVa zcqIfaBX-i5mREHFbag322lWHnwE*L0Z0$DaJLRg4!##I{{4xJrHPwsR z6>8B>9r6!9N6*JApc6QU;42xOndSo0)10GkzYE^5?A*%P*?vE zUMi8K2jn+XmEX{-aEiW+eC8Cm&ZoaK4I!MeOXpLvdLS+ZWh=|jX!g}*&{oYk3b9p15#6L$lEV& z)&a)x3;DzTFu1`aw9S5f8e2-|{lS>5 z%ty8p&Bvj6=HSjraK8MPK#S7O@&Ht0Dr#eu#!z%c!HLnFnLEq+mV$EpUTVTVeO2pw zb$b7Pi{3T~Cj3$>NK+AEVmY||Fo9k1&8e+2Ta;H0SKuCPJ^`etGx8s=(<3z%ETQV~ zI5!1+e}0Vq^KqILI!CXo_#;CWINRB;FuGYCqAl46c8FU*T8D%D!{jj7Lv6cyYLEXO ze}t9#m67;P_=%?5Jf*70VE;!22;a$azBR#UpMAXB^`OHIormqeuP@!D-#BEa`nfLm zF6*I&(?92KlI2x*1g_hs1j>{O*&YYVe&?aqjbsxxPlQa`M0VR&iVynVraQEUE}2Df z7DUs{xVE|}#p*tj5{jhHXcd6m^J4YyF z86X~x`noxqk-ptQBmuZ1ypJ_6t5Cs>M>PrA453Y2zNY2rda)4Rk?(YMthNc-l8WI6 zZ+>@|2*L@q9&6#xjVS?md3$y6oE^fSw^R7(yv#VB2{J4w{Wg2vy_er4rTt;LG&LiVZNG2HL1WtIBeAu)6|8OKL(@o`&<-D5*_2l36idyievm`{)$kNup!OejFF z%#m?~j=74}({mGQtx@M8uBl1fB4qd(uDAbq#|rES%~1C4hsnL+WM)eZz%F~_eXGx1 z*!FkJX&=ozRg#3#P^r8%8Lb`rHv8g9q8*Wrwe1GMFM1HoA#lj*ftM{`1?8(T`z;BrO-p2FZV7lXi9ZJU}S~W|epq#R7S(Qixfd!@I zmhus0U=0g2BadA_T#MLVV1D)X5s zvMf(agj-)ylV}BKn-U}J+Y#~LcO5`$r^$IrLq{T4nQdzt+*Ks(T+kf(EaKZC+pXHY zrl(~rX6?d8??97ZWzZ`5O5=luV`^P^IR=G13?cBPd;x7;@FhBy>yA0&|7`M1Iy+iAO*&TSLUbh<~mC4?2G%83+tUlYjA zT)ErlxuPT6+(KTpi{?)e?8Xi56>aQi!stBJ<`tw_QSAnTS{M1(SACEqUPY0UN%|#2 zKQzTK`y%H+-w%&dE%GSZ*wH+C_PDOC)5@Ow-!D%a6U}GhFIx6cp;b~kA~+15 z-z9vXgl{j}fJESKqi)@j>6#Md82g!ly)UNNg4!54w2%VPX%A1M*X(eL}w+XP~Lv|SH|EmXU_CVX0_yF?AYA@Ou+~$B+C`m#5ymq@X=qJ@$ zBh+r1%C#I%o=FddLSMw{;56@3oM6xe&AHlhw<7()4oBKuC31k7HBHgvuGPo{PhzjQ z$+VKeSQ%h@)B)zMH^=|SL$l&4;9qvVoW5=4_Ey!8V^7YaXixmx!+m&g_{RU;HzB{X zRGy7EM}wQy;%bp+*Qgt08$+XZ)Y$7RPvJ+`>^)577)^1=DM@Q^v!q$5bO72BM zWod)+(8am+^G+f7Sd)7FrljFMG+{8{da;*Fb0 z%buhk1=~At+ic4LuOCkJgY+??x6xbDc__pwuN#vosa?p@lIENuB6WSA=i^rz!XIz^ zyb16NXkGK13+Cl4oUZ#Yzr4b8L5yOuM*R^9z_(pj)2FDZ^!kzu(Z|}~vu>POe)XPXJJ zLc0)@vw?mp^cuyaCbZ80bXn5UkO8JL@=&ohc5mKumjeQ++oSs&NpqYgienfrP~GjH z380GP&Z8o>UPFr^ma?*#fB0p~^}7hp=9GjEs*gc`71S7ws;v7s#qAub6G8)rCKhE6puf26=%jdrq1?g_)E}wTB#CJF6yLZG&Ir*C$f^0n0b*JoK z_?;K`6`}krfh&aH@q-eFo@w1Ta_;S2t+?Y#xU+s|Brf#f-&^7KT?9}_ijeiJCeM~c zid}G{5wu3W50bw5)X;-s@_Y)c+@0-WRPTL0O-!AtpeeB=u%>u3s+PdOTj+O`+E;&P z0GvN}eu%eUjh6mn;#?>Bf;q7-hbPh%@ZnJCJXh0cIRdM3Ej8{S0ricP9OS-&)jRX* zm_Q)lZ(qwuBW@bqT6Cjn4ZPMaq4j5RLw}!?L{QTook(Z%m)NZ#a1CN3^3A{00$@Vb z#h~qOSh~kmsv^NL;Slp*GjOW%4>#%aW4LVB93pJ0csb68)>Nj}vU?zU8wZ_^Yw*j} zjMB!jR-_0@PBMqavy^Pytui;Ba~q#$n|T$JhY88Vt>+}cg-(H|LzM|8HHN2smrUvP zp&-67%=i2o*_k@mv}^o5 z95QJJIxr&~+<8948IpGYc%LZTazxMyFj#4qwWojmwwxqHDLt(y-Lka-$WkK7et9x{C#O7Vj{!q zO9fT6PeOH6A4u4sC7&{og?qAvhPZno^q;H^tk0#jlC~(H4TLh{OZrYUDYs<`4eei% zyo?OwBvf!o#>%J@WnCb=H@K;@7TZeaD&;60UDj57x}`#id85kv$!5eCyyd>f#PvGM0*0EG9)a*|9P&A$hb%_t!~N+3Q0vp3m* z2nN&z#>vC#Mz!C4#{#@*I57x`=bHb;;t>F9SaPtVY~4*+MpL)%byaNzumVX@APn4& zQIr;?0UYX!J1Gm9#=OdG-hU)xxl#NY)e?oq_R>Ah8`Lho{&0U)WqCw-c?f`!i8S*J zX4m|-EI%L1AxyCzdu962J2N8*THvJf_5^52Cqv3V+J#zt14FV}(8yQ%(L+Unms~7^ z!SR}o#Mdc$+aRjHNR?vnjjwtq4Y;W(_f_0T*%IQC>dr*C?b{_Lv1amJCmzbuCyVPc zC0wz2ZQ0_ih4XoY$ga#m_BB7j5UV8uD(g%`aWM$0Uu4YAV{I0S#eXo`-NolG^!`FR z<~Bo^S-$1Uz10tvcTrrD)hsFu!@Y(k0%ZQatJd`?(E4keV83iY8y6(ons9lct5dlW zc4{L_N$)!pRquVkzvd#fUGs{Mgkj)wey=NlQZoz78<7N>Tr-WGX5pWlsf&n=E;6kj z9;oi$7`>DhMFltu6T7{oml~XD)can%p+7fWQK+&#BETjzAAz#MMyde>&KJ_~XcJt0 zN#r{fH+rxT_;v9=Y8b0lK%%mMQw0Ay3y(R*9fN8x#R?D7zu(L{0G~H{V16#y=|3-W zRbeHn#Xe5vF&x@BMx8%u{B+0e+f9cc*wlZR27tvOkVR=k4*pe@q?f@KO>X-V-;;mLrv#SO?~2}nYE2Z0R>LCA8*JpG#>`g!ePS3lha?GG1K z#=Lk|Y5NovPLUKQ77v+M3wEF^e6eXW>7amv)sQ@?JtVMT%M%Hr{Smw!(XtAH?*b;< zSRX^8N1K+S6l@^htLombTN8$0sU?VEY8b7LYeiQ(#jU_d#*1sIG^+umx zG*?@mVZin*rzqNw&ymO+mvS_s42#4!uhWA3I{W~}P7&KcT-r5BZnBn4=k(4Tr9i0D zmX&F0)l;zbjJM2LEvRZ`6~)B(Lb_mbQPkVlp^(iHqvQ=x`zXiEWfGFI zJe>e9{qp@4XJJD=Ozqv~Z@QGgdIoAOa}WD``>bUd`u&rl$;L8NX>?{hPkk1KrY6`P zWn{Cm#6GjiHfh|&3F9ar-1mp)FRr-mr7CF6oYP9ljL$u5wx#LJd(l523auU8i>y2{ zR`Gd)NqB{JhudbZR{8;u@MaX}quzkL#)=-h|BjYMLL*ne@ zfe#4hXRyfd&Lte4vTwfY>7U~ zVh|8RU7;a*h@sk2sECQl^AsZkwzigR;`VRKR5-ixVL(C4@f2p~k~T zq}8i;-Q}dJ&~Be0i9s-ZWCE`(ZA0IAx(3$j?mCC^bKt^*u*ZAyPo#N=c9f^I2gBK9 zdvR+?m&=a5LWm$$ES)*&>)8@$b<>QbE-;rS`$OMkD^)0qis|2+Jpr?9z=S`1ZS{WJ zx3K6(DWhP`%j5wh;I=6PXfKN6Lf5K2%%9V?I?Y|GjyQ3#el73D+dz2i=qAT<4KEF5 z^ViK2$`Wed+;F(dP=Z@D^!!IiQKslz@eKa*wavX>7GhWkymgX^C_>^JW7^7@8M~LA z@fEGY3F;+-BviIMZ$-tr2) zH)Wam`hSGEwS6tCEmjfpZIwj>*M;CTH8tgA$Sx4Q>+)MJ4uOodiX8JN>Z2JR30^+-1a9pRAR8)X z%nb{kW3&|@w=Ujo6cDu_kiv`dSL7DCjO3*XM5W@!##P~6RJfe_P0povo49f`c5#2` z;&I>pN#q8termKUm0I_`kqpVT#wSQy3DIu73SqmcC3XW+<+0TDVSLUo{MjpI%3iTPEcu^dxF^JdNw7MSkrHjIw`Mv#`|Xh z`r_rK6vhi}SPpK4JJg7O&s}^)KN9k{H*JVC?h1HgsEsG_4y3C;bmsG_@fGYS#&q0v`o)BYDdC;Wr*r zZ_Wq!4=Qr;BxRFbj`WMUkPK0`I|Jy^de7U=RDOK;!cn6s>+aS~QS|U>cOUxj(PjJa z(dOFl(G!RTAy?3)1|84~t;&~Sz>%1t_kgEngr2yEc7{KOLor`Z3Pea8{3TD5V_8s~ zCIhwq474hKBN;C1Z#YOaiZ+!`m0VQ6X@3{UhuB zTqyEh{Hx$$h5L-T?8mI&{PwNdpTpBbcS3E;*v^xVc{F~OX_^RD>Wxcl^Yb;wCagb= zsl6g+_~S;ffFBR=%i_S^cQhVhDWO*m0-QQZyAT6)&loJMC$)c!puzAQP%%nU&3w{K zYoflcpx^~%Q=hrF$+DoU!o9zW|XI|uWk5m1(8=x5oQs-BDrt>Z_st?nR-xaZ5 z59hKdZ3m3F6jJm12vvY2ixGH`2x2$K-jIUr-?bB=1qvOtdXg_L(E_Ay8P-`pEVi&V z!=;uuZgFWxXh8E(UttFg34&)*cYv6~@Kt1%*RtJioRVJ{x{*j}ydA*}xlP3V)A(Xp+wP@chxAOON?HJpGQ z*%GmquH*cD?FZ1n19f$-#l;G7w8H|Mr4+;ZMyo8$d`1kv5 zYp5dbK0{$p=xT}rA&vwSR_2&&nu7zEJxJ~(`CTUe%-)P;rF3-jML2c2DIjXfo7SX!FB)>-Ih{>8KKx_bmit9t)3wva2L}H6U+)$`xQ+t zy(uC%GlHP#YGVW_;2JJlB#j!#7$KhsV6W-0?3xhDx|r5*iRnd~H)wS&YUbX;$E_Nz zL5M}*NqeDZn(S;3#03wr*T?)rd>y?08wh=&#)DrpDV?V14s$T%nZqwRb~~wm4Vbg| zZsSywL~U6M7m`37(jtRxiIHSR!CIDId=WKiGDc1-^vyY*fJyM31soOGQD8z_z7z5i zAco)EX$`R-X)+SUO2wf*vv(thO(N#VSO>npAr*xu+ul894cDRL0EH{gv*=zjAg=Y) zuJU#2rGM%usQVCc7R;)SH*zxSSEnwI)i0vJLW>LAR(BKx@2;z*=(paNrJM2lM5l64 zyTwX<)4qO%8}6*hShTunxXR)lnMN?bT$@%)yDN~0Uqd%gbk2UOc~fV&|MjklLurFU zUOJJNGW^nMB*lZ{VG-#AC`*Q9O@vOGldTXwqQd51EB&$8SOK2-I~%}j#OKV@r?3kW z6Oe!3up|inl69odkB8+Mm6`3;KAmC!Hi72IWZ!%ag#EwhG9g1)^!VZ7P54S%jI-tf zA7_nnths|I_zKZgV5^Vv9$yg{6{VE(zjeUjro5(9RT>7q9~YaQ+(YJXs>+c*y0Cm^ zZZ$Lc%+-)p%*0~PeTiH{uv~cYZr^J19B`6f!o?M{>gWd{^OFRs751LT*j|(_a8)ry zmFh?YOp41pKfYxjv({XaW?aZ_qzIgCiRDx`?q^Vyqw%=!?9ovTKq_mRgTO6bB!|~A zsh+v$5M-D2ls69bC1Sg;?qfl9m1-7_AU)!sQrsu2qyVOY-`5We=r%P`DfgAeE!;C= z8Z1m<53mSs^OIr655tMC;mG+})E=$sFt4-Ho+uL|A9ap*yClD z3*(4smNLLCHXH}QA=RgyG)~n z4PC*6^+E_65_OXB#xU8GwOR#=Xl#_Ga%?1jdvqhDc0ob#bAY%%~(+00i{ z!dr*l4#DzivdJh3;5(%i~&r{EprY* z-3PzQ$y`E9+hDL;+*INFym)2y{X&FR1^Z^&JC_5f&5RbQJ3e3MS*>z>XzDU)_BSIq+bnL|)m#p_PojMH;UX##kaa=2_OcN4MIV*HyZL`si#MKY} z8dhdvwA80O6?(klpD%#nb)mwgWsJK&>*NX{5?!xhya*d5+F5OnVC#Wi@L?@xezy`_{C9h|By@zjr z90^d%3i1L}#|qE5_*(yxT|YJAKfjgW&mDKpsi!f{M6$n+%&M}E`=M?S-=-mzO85x7 z)~cT5KB3aq13BYUH~YBkZxfaV6K8Ifm2ek1c)>e)xK#GrJVq?r(3fyaG@%TfjQHX* zu*=Ybumlz0P-H|<6<`MDDIXnNeV?vBm(D&O8d49f*vMmPmJmJYHp1LsY>&Ks4OD>1 zbrSaxPE?Vy9fEx#$_UT{GSv*A26f8{?bn`N@z5=E9S=0o_Zy$B_%!>0*1t#VxficC zuDC5{Q1v4(lkI{ffMYnxRGM;6D1`02f!%iIwW;!9Rlq0@?U^}o{u;Wnb3TFzph7KY z@h;HzrM?!D6FA?DgvGVs!N9KD)2NIG6SbB@QHRC=ZE2<8;Qn=&Ws1=oORWU+sZkW1UMP_vH)8_GgG{ri~R z?2M%9aew{!csccEu6wQii{CCMKsT(Q%+QzhXng}WYUm%}dipsFiZ0hDEDWJBuLgbn zz@|LIozxqE-*MT8++`GRp!FC|5^&7FK80FniNSVKG? zMFH4;tYEL0kNyx5i4Ab+I=R`zYB=b-C!xlmX(Ls5{iyF1rqpm!O9}#Y#q*tMrlMhT zQ1?fA`t8FyhJ1pl-N<41#JN3n9WSkvcof9ryX-ft2 z0}-aQgZl3PU{&j!exp(1dh)>MV8olTg}OD?)hU_H8+xLYMfGj!R!i^G4ikD+Wk}Dh z(>gF+TlZPhEkG{{z6f5tYo)Ik0+_1U&Wef#tD%cZe zhvZ2g&0*Gm-f)o-)WVguzAg3e>INluEpe9kZ&wXZh{hd?qCV+Uj_3uMxh zSBR9WFZLZYv>KC_;-Zj>`?N3!C05nV%qjqvWFJU*pUGFPLu2pb4jZ3Jp0PQ8D{`kHh zv2!*!Nx`{VM+d9bj?;^aY;HM9edN+wmg?&B4hBG{oPK8EqJj7IkBXd1?(PA1H4E*K zF-hDZKH#J0|M>zosm0rf#01x85)wG8Ga1Tz>DT71^qU}EZ@4^G!wGujM$~(KgdcdN z@Na1S6FZc&oT8BcQC>T2PZui_T;fPY$Eh+{vIkTwse6{7Ab?RX z(rn=9#oQ)_T?v1?3s6=BD%4U!CX_OjKv!HbF9JV&U*u62QzBAkrEq4UUgoYf5|du^ zggNrGOmQypmut^oX?AP23eu|60<(ZX<|SSIvA*V?@Byn(o-LKSOE^*q16vxzThCTz zOD;4CN-3B4CCxdETu9O3EVPpj*X0MuWL&21BNE;8wEJB0-gH0VUGqtQ28Nz0E+S{T zV)nb!;5%=(iPnaqmP?}3vmIcksy`^ysEs_r=9K>wUij9iKOJhd!TD0>GF zCaN_Q_pTlSUs@Lv9kE;H#WZEAx-UpQ6|Ol&{fv*wX8ktnntz~eaAxUTbmeY8IYwoCgK`qWpBR~B17akI^;?h_? zV3Lh;#5%NSuRC7wvF8ncV=m9jBjaA1=ZwO$Vwgh=8>Y5^o6#0+Y!HuuP(Twf2bL-u zCzC2!-5+!ssV5{B7JM>AvbUJlP8@%Vd)2FWjm6G?Doq48dOCfr8QyOtC;FG4-;sPg ze7-;jpQUCOW$!lV#l5VzbDXWFvzcS-EGxGbqF;X#G2DHwt{@_m@wLt%Q!XG=s144^ z)vI}9##ttN#7=~aK`BOGGHTGLprf5spP-Z}t9>!4hPLk?U*(AQ&dMb`IC&d&-uD*@ z1$haFXCclY7HE3!*N&e2e7xS5-dgx^;oSNh(P^Lk6&iuNQv|9E`9Qr5%bexnCRSF$ zRlGi=iqMh2q$cxU@+hD4F1DM*^3vO7SPEa-XAaSoiA~vX;P)EJ;WobJjxnst^}vH* zNUTtsq-(XIkK*g!qnE7HmUTJ5>g|@NFAzg)A6&~C7ca0Hu7yadvsYQnpSDa3%DZIO zHgV+{Ej4R-lq;&vn7cOz(bsvdH30@lK4NNvB~r1gXOn*aiBRtQVjKC) z1*;=O+=COUou6)W{L}BxTn=fT*(X`dCuuCo=77Qu+tRQNvP(u%HTva><{Nh}c!Zc4 z@_?eK^(sXtN`tMa+LG$R@a1nMwRx98g=ayC7QPp9SBGO5Y53FZ^Vp$G;p-mBSmgp5 zx@p((mqJE2#V4zR62fElM9bl_S*Ds?#*G>oav55^lrHZc= z7nC6|d8vQygxJM8i;A=*1PMQlJu-fCDkR~4hPjgQB-cxn(3VkgzF*RGfY%@z%l{-x z^f7Y2UNpCZOX(grx+e1$bt1bd7nzTjsG)a&Sq)Wuil%(nm@UXc54FliTmLW|ixPh# z_9CUPN?4oNjp_u??>g*?tVJ{s)YCUQVuBc%USl91zCD~^DEoZh{CwZY?}$pO-oBj& z-Y(+nb&Bis`sH-LrOX0-!mXBKt^g(>$^EV!KfRu=M>TmmA7~QnX4!rO7|osI@Xf~Olxj*djTq4c6uEcw~ytVFtcElS=~(WtQp&J0O%g= zq9=l+P^#k0aem~#3{FZdk0t3ImmH}v8<#AMyBJwaVN|wFRT2P3!m?0(IF7QBmH8`??ButL$HjsNcoc%O zOIHJbQLa}&+?G%|Tz0C<#CB#Jl~VCQ_Z0yYh|bghJn_)eSsVyYjA(HZCO`me)B0q)(gf)Mx(ryE=gpUYF zsj;nSg+hZX$985g;>(Tw-AKse5;2Rzl2Ce|4_%iT*doe7Bi!6HHI%eWbpUXnCjlz_ z=baNGD!O2=Eq(w1^|;=runoCj7Qg#k{Qow39svs{Qku#hy3yeio+AI_^a6xK3;n3T zJHJfB)X3guZY)O=dhUb5^WqVh6x0Hun6(NrHI3up(XP-WbWbA#f2|e-0=oc|HLyu> zFrugPsQo5?yYZV??4L^E5Z4PBv@(oXdkfVxP;rWlPce7$*u^_Q66Sl-v`j_wioSqh z(2q-l#qpcBGN0g=qnmN$FoSP$`voA&NTfROZ&C+JpLX=saCV-zHhPja=|{Gzy5R71 zU+ia!D8X8JAv6ck=s~AB&spwSZ85*w3?jRQ{$!W<$W?pAhJR~tYj7Q+$qv9{js6wn z%C4$ZiSdk0(2=VfCb4JkJSw>IQ8{RG88H73uDt73pXNSbc?M~@mz8aHwD^T9{cBFq z|4JSOW5h{WT#$E&S65fp-FIJCOg`dZrXFXn%scBth_f$Z3kBxP z8(g%WOfnzQ>*_DpZwL^fwzld;)t36?GKU9VofvI|7lvEh9-%5)t(rOiI;D31g&H%Z z*!j*ibY*4RUX1!Ll}<<_z4K+#?iSb0>`HVz8`85z0Tkzc41S55<{iy%ooTA`^}*fC zla(I7mn9nV&XTMoatXfna(Xj+?KZ)6G}u{SE{;HPBRcf=wE23^kMH1x1X_NQVgeVg zuc=r4JbibD;-I_CY#eVHT4-cRNd1syOZ6s8Ld20xS#y<5(aF2sAKG{bbzWh#<}vtp zB6tnG`B6DALvRTVcs(xA<;J_tvhDO6JM;*(l{DK=3c%h^>kRFzxn-`#Vfg6rOMrAa zLGi_DuQIw@BVDaihEKl9&jDAyx?@qo7*!b6eyDynSDH#5$J@SSDRQdv3;?b;k!vNU2{9o2W z#V6e~4e0hkTZ(;w`C!_kKeD6qBKqNd=$4qcH8>y)zntKgu48bQ9QlA`dpzT61FW4Ts z0p4fITt7636#rGhd{CLi+HPa&Fk;k6`Yj%0c>ByGt@zjq(#0d(p5bZ&9@=zsrx~pW zzTvukHB#u_yY4SxT{_#*xUOmDgTXaiPe2@h$Z55!eEGxKtRYe(b1^!r4m*U{&rmYs}ipm#k*S-*U4$p#m3veZsLjG<*}A8Q4$&2_9A)O z%tYV*W$kcH3w@9ye&n-LYyiK5i^~HR-=F&fm8Z=h9iqwv(Y+=1P4jcu$R zf0H)GZ#TTuYl{x}8RQDu9X!LRadf|+iuc#M5%7Y6>ikLm!V$*rQH509Y0f$2lV-D% zYd^aej44;F+Up+bjGPbO;X*`My3c#OEiQBrO(=W|P4*xor=TvI6OH!2n0E|c0yKkn zWU)nHZDd0&@6$Et%!`6`D~x3+R;V`Ih_jSoE&ebD-CSY{9)o0HMGc$8_pF%3S45pk$SXFY8LX(4INaXah4@7bK#z(;s2 z%ISVSd~MMY*|M<84{xK3 z%T?v{8MQl5nFc)Xt`0nxFR0;6 z=RmayxHWvRG7%Yo;J3LLY4iY?QYI^SKc+^xLxWv4C;MXo*RCGULmX=*rn#5It4&1V zsKQfHhb*S&5p@F2B7j4h$cg^W;50O{yKCwgLQSBpSz!eGzg@Shp7*V?+W=at9{w5k zre>EfB6X3OlH-sRJa7;97c3MKKDOQS%k;fYRo^9u=kkZ)^O1CF6S-!PaP({kbx_Mg z;`z=Wsib)JDhrc6KLpe2TUEk^Q~JDDQ|I~Z5RM#XsRjc|F=B~7B`;2DV!f2gulFyi zKX$Zh*oj^+96e|;nt$I!T;f6zm(>I^S2hg}(6<&!6ADoDHy<bGR(x`Og7gEGgw-B}b`QTQ%JJGrNV?3;Hl17F*=@+kv9Yd2DG5_)%V;$H-% z|8xuU4ir?FtuE%tiI$+sI{Rx+=~f2r&T>;Pg8Xk-Im=lo;>o|yP&h1e0rYX)&_z`& z)10Vb@xGiOCjjQ)LYaI92ebJ9!#7~=fm=+qkUqy_DQ;VX)|cWJ-y2g9Kd*%fKFH5!e?LtTyFL#`KX=8i#8;AEIr0@IU(-rXdGG6FQGZG(C z-eHUaYIVD50{&Q77)~jBFe@C zDsO}Cu=vBzLePNvEF?CU_jy2YU(yB)_W4(7?0%qrD*eP}b^E6K*H7;kxCl#vnoPM^S~@rcr1#c*zC*Q zJX-Pb<~*$nhxLz)6D?4B0}^5<%OSoWK~+OLji4-s>AzKy`WGP^#zx;Y~|Mf0E$|x9;FBL&C79#GWDmv_hkm} zs@vu`Kf=85MDtVEj)_-Zn`nvIf&ifTEDO ziHR`qy4=mO+xrVJzj?@fmaL1)m->4;P`u+z&rGQD%mKzUxl+wik>gbm9uhL8qv|s= zhY`Nb#B%s@+z0H!m`}b_=V7O_lgUWVc6e7}k>l|$$HgW#Qxf;~!!*}o-ObQV5l%3Py^G_k1l>6Br{0fUJt0H^*@jP$;3o=9m=D+-_I_1GYO1BWRDX$KMv#i0C3g? zbMGBa-tsCt|L=_xag=&RRv@&1ztyxf$+UB^K#fmqjMf-Uyj8(ij5nJx=9Gs#Z4m>uUH-ONkW}wCo)O60 z&_oJTM~`9??MrwdduvKgf3d9mpSWgOY9NaoOQF*R+wr1_5ZbF5kK1UW*PQe_^zY_G z>9-iEf}be@ZJh(N!|^O@%D!VE7uK|L%)<^B421N@p<9>qT$#KyXr6P*fqu;|g*3mp zw=`_I$oQ@!^L2O(orGzfZ#m|P&%HMv>w){h)J;&zX%z%KwYbOigmNYpJJDj& z;YRvc+(GF*z;(WE$6&WXWIXBnCM;XLAVK5Ocbo;0p-1z%c=RKcU@3+#qRVR&Qco> zrX!%RD9B|pk_pC5XO>$3COSUO;@xG>8mQ|7Kk65GGbK$VRQww7L|)p~m}v8k%u*uI zAqABK3%LA@2ruV&w9c+8)8XCCYk=-?EOuVv2PFxI8lb7Lz-4WJ*rU+mjWvIDa! z6d&uzzY45o>9fB5VJ30gX4A%v*xY|97$Me}1Zm@7NlZkDwl@$fqnQ0%U#zF`v?||V zKuSwe4V+i5$BjN)LEP!fo;CI1PCaGX{IvGuJ-@RKTJOwAK1IIH=nL~mwnSBY_*JFI zwCr0BB6dzXx#Otj-N)#MA9I)9&^dX_NT4wjx&M&h4KthIf!dFE}$90h`nMEO9E+_os@q59~ z-Jhf8$m$hLC}UQ5(ncf0`C-1f(M)z*-Qk5e6r_CJHhD4*5Ob_Cgr<(eY~$hYUy*I6jbf${E*Dm@>Dh{}0w-4)d-b*jHk_B( z32vJ-k+xh)GJ>c~l0Zb@Of72G=-xC(UNJLtdGdQzAhh?t_r`oxQT3a$BYsr zQ<;uP$I;LT^a@66RjK)fe(Sj$Je-l( zO^GmrR23!?RMu~8jC#P&YFqtxgA>ooC|Fp%1kgGl6@>uwFLD2yyaMy3+^;2Qr+ECFQ3B>5FmaUsSVxu!*9xX{@I-|d zfJw~_4WjHULT@}YQ_-fyQqI#QHq7X2cPQ4d7e_4vc;d$gJf*~Pt4q@)3!!$?j2QJ7 zv-YF8uQQE%B;fV4Mw$DbMUtt|*^-L~t=X(VF{Te&irchWs!RdI2yfia(jrir3pE3>wW^#7j+&b&hLoDqvs+z7aJ2Dwkh{MoS47%nA?sn zF4;XB`3;x!M~g!knOct;QtW!Yd!a7lncKJ7w!<*P!|SaI-u5JcGPZ=2S0qs)U3qDV z2S3mq!?ND;&&#W~@U-oxU~qLZDkt%}qAgSL|UT`#4l z37SCkaIhYiVJ!LJ+wgk39k;pG6ZFBy2ae8VY2_+OMRk$|Ri+#e1%taE+*5+h~)lQBHgeFsCLbKw(M!2aTQvVzU0sxszFyM%zJ zLLjjBAAUc>Dnf#h!swo+|M0CCT5|o<#zL8aUua|kJ&28C9?;dRS=(dd=rQVduyV!3 z2*cai_HrAZJ|tM(88oTC911w2U0Z|7Bk*zk0sXSS6zoEUxTcL{-teYP(PyPs^|mc( zPI2;)JLT=rIKjkP?4pC;>WgF7XZ(jOgIOP0ZFP-NIqgtm<4IsW19dFkogu189~e9YNA5+!z>u!T0g<8AJ< zVD1NMr07pRoh#R;q%#T^LeElCN>yeiUsz*wIY3%cz=}k<(?~Nn48X-i$lT5NW8#X+ zTyxl|)J`&Fj;eK;#7LIyMSr3CIr7v-Yg3)pbueVoaD-fkD}0$JjyPt)n6Ly2& zYbK8UYFats9F;UbjMIR!q3T-D69G%e9pp~`uv;e=ij9>PhwU@>{iwrEzU&$Z;QytA zCIKs>bd<0P7Tk8|!qEm;6RhDrHNYZxv9Tv zgTvsicBrfq-9YB9`XDWG^fxI~CFkFI(j#_Tdv=J5hh)>ASNeB3cmS+C)l`FR?R1iH zyyqnI|0o|$Ml7aYL~1TK;g6b5tB(u>o&)iBapbdl`+qaT3?#c6eZ5dOdbf#SXT{0l ziJd$IjM_t9%O-oc*||A*dxgvYdt%?x`SNx5z~9!&BPhBJ$msTQ@gI_}fT4mPGjaY> zU{A*ENJtrNK4~*JVH$QhKspPHaC_? zMYj0*=Cb6c`6wr~5Uikvt;8eJN+$nVF`Or3T}yctlNEP$*h=k%%Uoz9$Wiz@Eh1u}SB_6X^NswEW7ostYGS+o9DBLsC~24>MvynP-$uzA`0# zV;y(hnzqnb7kP^SW#!uFj3l|pdaDbWOG?GyBp!=X(y5N25ryUMV;uPq?WIC>UxGE9A@wU3~4>&rUG)4oiE+JxIHs*tpsmW_M5gW=}Ws0 zSG*imrM)x|;8)1Fcu5jJ&x8MQK01RHT7JFr{*Umgrod@-ij)0ig7jsGzKqI^eIz*k z_rtx-*Tc&}PR!%wc>gTYpE*m=3bjThFA}VV*(x{n`-BSBKCOJ8w%%ZWVeb_6mXaoQ zNom>qku;Tnu#t6z5d{i(J`Q!8cHRTK_W>FDRTQWDa_I}k@4V6(79#vbt8oRFN-4Ol z=gu7o=!p&1*w735qhR&iQj{n7)hgN9UQpp0aSi6_&kSxlF^~1J^uRKGLFo$Q*quIx z`fhE5K{9XG@RI@~%bOe4>+|3$ziyneY}dr!Qqmc@qTki)kG54q(n{Z+rx8wdWaLDQ zlczY;n>fkT1-|#1{`3`%9iYgcLV;)@$e7N4u{C@T!`267fW@QYz`V>Czr4e=MbA?+uiTzqEX)10j zX?_(l&f7)(JH3Q}(@*p&|D@ z2`4kEBD*Vz-EnF0bZB{eb}||5_K?39CnE8a%L}nQ&diHSE2ldq4fv_uLfzfi# zNYYjT|CF8?&V;$fdEAFRZsSCwaEUiX1#!YGqZ8BR9$!!*UX&{(Zx5{6s4mcm z0bu(Zg3I8vT0bmr*t~^D`dWHxdtk+tsYvTJl?flLqv=}TGWa_g*$_r&2#)tt7JsXY-_L0jUmZb&d?CC>iYMBz9K!EhHoN2>&{MsrGqu^1OSzy;Z0l{(ig9j{4^D z{eIaWU8d|_)BSR^|EDgQcOmTR?Bw}Q&H8vlEx+UaT(2G$jJ?m+=CHTFdo%&{w}jCe zJ4i`3?DQfiV;0*Q+Wv>ELvq`pdQTy^T*oyHZkbldkzc)D4xi6SGRyW~oJibofL4tE zMJb<~p2(N|LaiZ_F}u*@CZXoEJMdD3+R{oC7Cp2u4T88$f|6JySdBh;LeBsEgYOxGP)b>H7tB4@lxp{n{w4 z`v6W0k($h=i72vd{+(XZ+3#(U9{PhqM4v)K-4pBT`zzg}%&DsVKKqIFOzC_HirU${YL z(Dv0DZ_|**N+78YmnuC!VVgA|mZzsc3{sbtG5>e|@(TPPewTF5i7oZ&S!NT6!vs~K zb$4Q`aq`}%m=THvGkFrA9~Mh?V_;~0P9LO|KLOjY14@k%=GpaT<%C+4fLw*9<#Zmn z#R7eVe%@fSpxWV!0= zD8Bq<;(*{^u{D4IlctPL6tbbaDQ~*r?e=l&EG&s|+BggqKl+f38+|_TkE|zF5u}~5 z4JF+Z`v&~)<&VV<5S!WLYTe_Li?iTKM9PzK&TOBY5r|4oLBSJ=OBoi~^@I>O_06dq zqZ`o0gHMs-kvCy!kUixhV(}-^&qEGdavkd;wikVEq?z?t|0EoezzP(-Ii|I|UvNU8 zge9L=Ojq=2p3Mea#i9Jx#hOEMz}E;p%|r;A$|l0hd0n$Cb*19~zzG=F-{uq%SXQnK z_S@Uz(zzjuNQ}XQLHcw=XSkOM2fFxQ`F{en zao8{YisL1wJaD58ysmsRRjF&qYt6cjB21l^;MpU1WkAmU-~tWOAQ!8PWlM@No0A=8 z-8*9)*0)M5Naa84NdE0UeX^jgv^J6LIaR2okj5V4X9h*|?Z_lDN>v9Up-97~V3DU_ z{-w>Vq&l-_Sx^a2(9}DLLuIdadKB7gwJ|Yr7S0Vs^^^hImGJG_yK?_v(w1useRL2qnn9(#4;vK266&AC;jnWAA zSVh`mZHZ?J=J%@5dAf|WiB@hNSS+-}3jovJj2Yy)&0@|?>|R8jCSaVSNjz36*3poC zweNtKn2f#uHFD!4$%jf(DN&_riYxv=_~{g@#f)Q|*bcm`xg~KCw=OLdcHN8}xSST)NCVoRbIQDm7?7Oat57;CNp!gg7?N8=ZkMovM zYQmw48pQ4eLA74(l1FZ3);=ettyzswRWiqH!nDE+7`&si`{Vee&-RD?+g1OdQFyhE zeNG8D6rTyxG8uw-qaKA`x&w3V;@dT~%&S*8I(+EOP97J%0}yBi^=OrG*T_)O>0$oW znq<}Yc`Hh3!$&oTDaG{=pNxcs{g#j2@(#E(h(71;GoisPV1g16j z8L_W|i8E_J#9pr2ElK>V+s2pdSM!AiuFPJRZ(|gf*z_95SMDhWE9rB%H2UQ<O_pJx8k(k% zTPz`Hy$v}i1NhB@wcG3pZ^AFBp>*i_&hd*jgSexCA?YEDYw1!!0hB5l<(*#P6KE(C ztv!Bg#OQ`CJ}^Orz76wWOa2S1Jr7%spNi+t5FW95G|9Z!gZvEW%T<RL^`igV&H zH5k~y#i9oBr5*)k*9-{ppui2boHKJ`av+ue&iD6NVP_d%E^OmxR;ejXs{QRg0(kG! zU|a7Od=B;ky_5+*sIh{WmkPL`rU-A~c4ci%6zhjQ$CvCX1+wg0S8g<#Nu@U}-Tr8>CkmCF z`ckc*2!Z7kQb~y@^~HU1=@WcBmHSw`5v>10-{k|^7X|cj$R7mW)3?+SJ-&it#@J06 z3~=H4OO}Z7eoPM{@C}d#J5GgAQ-jliHfe+m6&svJRUd>f{Z2HVHpM;g7z-6vKi~bb zy+!O4A2#oxyt$fSnq))gVOTzvGC?}s7i-AC*q|AZp+680u_G({qksI1+o3DPFX7Cw zA!6p+m{Gq5;Y&EiUYq14xH3dfw^LQUpf%tv;W#Qoum$OcDCb#82>vds?re9p4s28Y zVv_s852bx%WT2mdIc3 zRXe6D*R5)mj((A?G^%;r>aJPOZdIUD;=*wO$AcKm5Y0Fsh3;nIn5JLM*04#;*v1;> z+%|u~PqI6LVF#w)ER5b@g~(f)Ce@*-L|mx+l1Br3trkGwFKCHCT?rS;kdl-*pob4& z63O}%%jPy7d96ALU=N(EqTM0~XJCP`FM-2M@NYH@-QAD#?M&KcAk?0R&A}ELy31dB zKcICKf2?zDU&bJwa|Mq~EeZcjuY|0;q4f7;0&^XwSkrZ6T#A!ia7s*=474fQrGt-H z<})r{WxcE$;jl_?`-nIF5$ZEE^rk{rB3tU})!lRt*r|^z;K6%y-0b%r*l#AZmQU~W zCDBLWhxLrIO;ZL%hrb$i?0wZ4AByf|ISM!Dk#fBv?iu!*8l-hF8Zuv_45^zS-t9dw znX_Gq`3svnFbVU7kmG|`Ue9ebQW&VSJ%-xcnt$24jNAS;Fw!|OmlXadBRaY&ajChv z4oSRPCIo4nZ{DY>U6^|Iu@9DPeut8kn&`^5pvw#yUSgMeq*${^Dzh3S;6fNrC?Bv> zSV~vyJQ*v>J_L_ab@`YVbvkJ1>jz{SJN3s2-){QZsz)uRN$~wra2dduG9HOr`uLb$ zU*c6tR{XXTCt1jP-guK~Cw)&}oiG>|52kq!vwhn<8V|B{>&P?om6^=&8uCV)e3b!%Oul9W zp}dfoPFa{hnox93vOk?E*w+zB4_-;S+HN9pw_3e?=6$`XGa%lE-FJY}EK5Rs>_m!m z54!xcC0C_@zxPC|;;Uv@O6xSb%+b_npsWrWcqOovPT`x3RoaFEm<@ZNIZ!7D*#gIYRwy}6H2$Iyin|qx- z4xn}r@^G*`7@Mio@6~+of0;m5tUG_Dl=Z1pO}<&fr{xV|6NOdiGPYv{EzGsnA>`9U z(O^T^NCP4wXSbFw3C?@7Zx)hRvH=y>&G}b~cf6D+5xtWUf6mrgUrGx34UIObU4P_K zH8=Vl|AaIyl>%QoRQkfy+#yGnoXIe5nNTyAvv~;Uoq7&KX{{3#=sq7_35nPHzPAWFN3N5jnc86Z4sTHJe|T^YyWrDW@WS-@e}rGSd9(t+9K-_pkoIk-pbc4R zkd0$I==_(%Cq1sy=hR^Rxt+jp;h~M5?!j40j^NqdlXfmsdmB?F=g66?u$Pld9OP*2 zFodPjTO-4$i8ST>``Qt@qJ^xq2?fP}32~zL^l`*j|Ko9=ZYqx#3JSG!5Gy_-PUAwa z163!eNtFSRFpXOra0P8?Ys z%_uEkhFLbntiv?xs~4H%DHXy~VpUBNyepq>5zCYx0kz$MvX6%GznmDTZ(f3$%jm5G z%6w2!qEldL93Po!5nrPc3}A=lP<=}(De9>9creY;37l+2L2KC);? z1MlXNNP?fsC-bZg$rmezQwsn25>@kToJr%gp0f5Ix^06rMaE>Z;%I?mm3grOYqje? zctjrsd9dus-J;0l67bPiGLNu0v}qhqIeSFj_fyzh(GZO^WBXy-Ow)_<#H=HCxgoC4 zaw+I3)Iy=2I&T*oPmC1eEdyQG?I*3RD(+*58mV@)I7l{f_0OI7vb0RtPPST%Om6ft zBVquy!e@vVRS9B^)KWZ3YXS`O2nlN2Qi}qoJ*|xkT2<=mZpP0&2Wt7|tLrLOaJ^#- z%pQMGLE|8e9sF^lfjj9P`wsy$z!GnQ666lE~{1Pb9F+BEr)_WHwhf z6tk+ZxqPtXy?RI7EYt) zH&G@l`ouop$O^VLhB$>K1>pIafJR|dn$IJB>#Ff8#H9Tbgc#;?aWx&3TN*4=)0f>| zA*E|$64QgV3N?UE$X(~bNG+PA&CVL$tM|j7yUgZujs0gEs2bMhykCJkh;3&&xFVRj z7f7XVnZ6VB#W;^KPJ-;}#@BkeV2Uz(QQ*v5$(LTmQzt#SJ`u|nznZq{uT9wb6&dKX z^6K)oxTDAUYe*+f`Ru``M89QSlPaPPI#;?{oi>JF2tgdf|x98Pt!&66*qt=4xDUQcL_ITT z7DXg8HgdyhnUxffb9qeFiVFB(iRh5VO4!F^Z|cbMb|yMy#j#Q)LNK;>GWz&gaPM>h z3?f!$ncPQ>g;yULjEQ-NM9N3|iKWc=2gs|O{0t#G?%%Jyp~#ThYHXd>0(oXru4Y*z zL&|bUmr_wpOA$vuQ1N!fGGWPN_|kvlQdd7Nhx!;T)Gi9i1RdMjYdfn4dpLRU+#-@> zo!yxMGUdsTRZA~{fS+*52AOM9s{l@;wocR7im0Lz7opMs-+(&tM z$7X2U<>XvyoqEr(`A#hy{8Q*^kB^<_;cgt`V+&?G0S>>R{8E*+;lQs#E4wschWutx z=qNmiSe%$2p78H%4aVwKCu+SkvEuB8|+NK zaJs-y%bcF>{FfhFyGI8Lib@DgGurTv(1;ANB`;}D3B|ds%s!JM$nb29?Lz94WMp25K0j42e(xlL(vz2bWtOB~HZ-!%kOwR)r6X78wtt6Zb)fVVU z9h#*|%{lKo_fS|G z&qqW(k>U~0@|!~KaDhn?2~vt=Zi_r>ARO+(@w}I$rn)|K34DP&vJ6ugWUck{8%N`} zk}g1lWML*_``D81znbTPoWM>Hy3l`&f$NJ|RRKBAwUO(`9hYK>85@q?)TphcKxxll z$r94H))cyf20zh1laGG>FS-PQp$i0OQlES{i#k|dRc6E)4bFvqva)XSjjM6XW&GaXnK+=*^s0`9-M$9T~7j{m9^!%=90}!vq8*@!3G0z+|2j3Nb53o z>CZoyd{!P_LP=YOpA9k@o2Ocs(X^J%dOb3b$(Zd1ZC31dirE#`=F5`(5z+>H@TPJ$ z8GL!>jgSxxDkE9SjcaR3^yn-Ki)%rMioC#PcmApj50_0h&U!-AT}QjdkK@?4Zrx1F zfyn-Yk3W%J`lRj~o(Dx5rs7*nCieW6{_FW6pg+yi0fU1q(0JHesg@w}6o3$+H=N7-2Lbv=&_UFL^Y*y;f(KVZ^-u3G3ta%yRMVa!)I^Q z*r1FtvEoa{DlFArj~Y=UqQA91WWBzbl|NIbALjHo1HC~n?;#g@9;+JE7M=wkX8Fi@ zXLv8%0P9b0JkhCZ#c#rgh5iu@KM#7>ugKm5RaJUTIe{SB6b1#{vWG<#sA5gSc-x8` z(<6~-y#*L|V`W(=jwfF3AyLdIHy)o@nGH)3)kN|SReh;ea-qL;-S#K6<+`W-#ud+~*GgMk$m5KX8C z&QKPmLdqh=Z;iE#jV?q?@5nX?BrN}DW$xpd4tIlm>}?!I zu&teJ1);_QZnpf)PgxgXCB2%Nu~L+*!#OfPZOUNh0Y3QqBWO8 zR5Ja43M@8PlO*j_ZE+17w(w;X3;4N9VbM4g?6q?nYGIx=DB#Hn7BsAJUJW$r??JQF zbaG*N9*5RMpNE83HPr14l0&Aj8Kw9sQaHHU+X_bc-7zxHlamSSxtF3$#;3YDx0F`I z7rjt}KJnb^G1kvUaDG;VYSlFoTUC!mF#?*?ou5DVm*^6FJ=O!Zpf08r;tW#D^ zNq;d)c?`Yeks)EZuVaB=xIYd#@j=qbQsdF?q4@ubjE^ffX(fi#5~LCYu}Y-Z>=_cu zVV#tHDs;p~bPj8@UhaaXvQk75)95QlNyiJ4j%gH*)~REHka!TAxYEkrSi0W*(Suu+ zh_+hAyR2-%M)xOkzhCpIkXnl5wl{DXDEAJ7;GxbO*@6{;+Sf(9(;8 zH!_%*!slb6<)OW{UC|E|9!Cl1y@jKMMK(Y{QNpyt#a-V^Od>JFts>yWxLCYKcGk2^ z&)8KPbw%BpnGOK&iwpLH_mJSjy1fP++0u}`_XF?fXF`Na(M;`swYvPXKf$mU3xy=R zMk)!OA`oGiM^z=w;MQ0Fq#NdA%R-_la$POB!n7hHqLu}8Ot zz1>!p_w`Ep<{O!*bHO2sJF;Ko=4t2drlrtle(|QzFNyOo{m5yuU)}YxC*x{!`Ptpw zovk>*loRKqr)-to{$Y4>{Rf3-(tUEc$^Oqg{tZWo@Z!O5Wi@&Mg{T><&)27?v&G{Q zs>RYa!w&({(+r_f?Mb@4^3Mg$1jUl!Y;SIvA+o}_(rHuZ_ZP8`P_ z5xokg`jHm;%;$EP1!iM?dNyEwBsH17C)IH9Y0j5uQnq|hK!CGEFqi@;nxc>Z+70!_ z{8TE%XF=lV&+|Cj@gv4Maz;k&adc0n#;66=lvK*V={%lFJyyAP8KOcDRbOmJ7nViz zai!I>DfUqSiU#eIH`Xd@!%ghbJA8$DLZi7k;Z~N;J}(nnGElMK-VB=FyfN+q?GBJjyRo^dt+l<)%3PDO z;A8+$ANOALyhdAW<})+rzNTOH>yTU*cnQ&m5TQ;{DT&( z?O0VA8?_$Gp)2-tw8as-S&D++a{u&W$p9c~MwPtP-8)AXV(V5g!!i{B5DE4)=YOG|#mue6SGUzS!AiNq$)7Smq(LAt5 ze5fcgEai2$B+>sS4rH!jEe$3_PUxRb*x{4(=8tI*!sgFJ5h>T*JA?2t6`l%)P`RCvank3>vBbMV#1j2{s!omB^Ur)fhA~U?Kn7Rmf zf^kmvTn!eZ>KY20yTl4cieC>9xd`P|MEqj_5TwMTT=M zhUxWx6rC5&2+D|?C0f|e!@uNP@|We8u*pQkGO8ukOOXjdk!S_SG0@suP>~VeB_~GB zt|QDU{(ia}&RNWJ!_>7-ySL%kVPOT3O~O_m{2-fx+5>58iCFRg=0_nP>2AX508W?t ztDjMJEF(Zn|18wSBo_tL#GWeN572)jbcY5)+o`n5x{Q~!mzfw7QNt@d17<^S0djaxiWjELstdkFlA}#K?VxaJbL>UG{>z<$So3AR zH2zl$OBxMToduIDt(huMYJJ-s?(4^0@vb6jjKGi5A4CX*$EDWk5b`F#w=kp0(lI5o z)tEIqgA7W-bjx*Cw05!|B>HldmgR*n+#8VqOq5-=*5A;eCQ0sN2ZauD{)VeMIETt2 zuR3pRk&}t}mIBT`CXLM0OBNf84! zn7L)eKtAZdQG`>$1zEH8IWP#$Zc3NmqzzovZcaS&G?z;rrTr zC!nOBm;a&Xw45BWeWkD=>>BSne`#Q5O_I$okk0`hHnc5S6p=; z0qlm z(@WiFDWL6`{oV!#PY{mDc>DdtB-e_=ocs==&{l1LJ_N3s8=T->YI>yz1I;Hs-aPo< z-#0IWf`sYcuKZuiy5crIU-{4H)0DbCO_ve{mE=z<|HIll1zGZaVVh;!uIjS7Y}>YN z+g-NpsxI5MZQHidw8Y4Xxi zH3s!3h*?k!Q0}myI%bBVc5K*pC%k-YxTbF4W4O!cyv~)|)9m+_5Ce9~`1frVxAN9v zO^uo}(kxrv2ZeQ2txD?oq_WK0?K7!ImY?v4h3)5A{6n5XpYX@^VSZNr``^XUrSWa- z_vd0;PhQgBACHIT*Y)w#T5FCmkHsCUFV#Y))^G1Gjh64n)L5vGQ{Zuvzd%|Y&o&+X zfB%3K*^1@j6dPqaAjp?j<8&p?U~j7#23#4|A;c8d|JH#ueN2QxhPH-YkfnD~CwT1D zd!7_F!1=(Wc5GD+{EqL zj*I2NEzK=aXV4{~ByhbAr-TU`*R2ZKL?B(dJ!FIc$-6zf&mz3>`TaP+W9%|Rg7sk{ zwW{h3Shsm}uZ9(}K0etJGn1;jsh&w3B;(AR!*I1wxr_R3klH7)O9Wir`){RHXD;!R z3YHK;(!k~)V7MN0_{w6rbm2}*875d21}(DBGgm7R6IcamV!x{4 z&T=?&bE{sayoNbP6i(8E>G}ndLCDqaa+uRO`=O_8p6`b9XR6H`%W+Oa`^RyTpZ5Mj z>_ACxABDekW5i^o6>zuDU0t@ZFxq4Vu+H_XDV6;0WqzP%ou?LB4IpBF5#}T-WEq-Z znA4Kdk8@CP(yNoOWXQz6YiN*A*#C_|;u{h5wPu>`hV$HCCH8t*W_i9`%yf5+`I4>6evnh`hzmq+wk^O1ZY9%M4fv71)Eq7fx<0C_IdHTFuKO&$Ea(a&4J z)(2L8mJkjwQzZ|7j~P2Uys+|~oCz#y3FzyoAKH0WA%I@pB-qQlR`3FE#3Wsx5W;aD z$3Se+BiHqsuk5h>l<4(L=aeJWO>_q<#Wp3SoaSoiN;IXDvqiVwv!6R(GF!%XK!Y&P zc1@_S7tgq$(hH7BtJ&=8dsA@rJ1Hy+@Z4M8f0NUS9dRo=ar{C z`pwbOxsuRHhzq@qg|4HJ9>z$%(Ts{94A-pTz|%cy_AGcEjy@h(q`IPqxVrAu=31cK z_9}~C%s`wJv);p8^&<3Fy3=mI5uB`gB3!(Fc+qx|`po1$vFMii*=@H?5SI<_YxWe8 zv|LDVPWM`+4*uE0CWVYw%a#2aGPERV@L+SnM?;Abwpn5+uydNZ{fX=M<{Q{=&{`@x z+lyuz>=arK#209zKSl;q(i6<#r4A7Di8|1fmsr)1%=4cgH`Xjme(BD?E@fM#QZAO> zv`)DO6^Zku6LwL#^eOQpj&eVg^GZ@SSz$w;`NTGu1u zNzA)y39rRlH7oro3j1bIaxqg>0(GiaTtdpgG`NSRtR`3!GbCAQ>V&jxGDE$|5uQu% zI<6+ETftuqiUatfHC8aSH+5rR_&94%zwD9e43>xMIm+Xv0%tH^hBRPnJQoHKVB3%y!^<0<*YD$#%0g5ZWZRsk(xw3QWO%dU)yu(S-B4f-9!uq9yp4n=F4_rY+htX_#_wjV zVse#`Y77XnTseuMg6n+dMq`xCI&f05G?i-v?K+NEi?_zNG%AQFWV2sxp8yZ~5N1z? z)*>HP2^S@~W^`5aipAABUv~-9CYY{W<;VG3AY>tS*O8d6Jy@UOOnY*Ie@Y)Bcjt_N#mU4x>Xmkp$Q+6wJk%-)tk z9`Q(9*Ivr&8C68RUfI1mL0JNj_sm{5vh>kyH=4|{l?c;Ff4ndz=Wm7u20kGkgqBo< zfyk~g2_VZ@1UoLuMpdCCE7LB;Zw}P6w@mZDNFAumDsk1}*!qr6nYy{Vp4pHKvvBvz zY6`gq_TIxKl6&V(2gxM1VgQFwHu(Ptd#`pdF4?N97Yx&c_=M+p5bsg371hHIfW}Re zg3E9*)T*B&7WS?40VQUhv-er6wZ=tCt7su~&@lDt-dCdEfxQ*-6aBF>URT}M*DZV@ zvL2pmG4srLVg-h6`~e}3qPR}P$Ozk_{AIGv2Smm-YFH*G*nRSm9FGlt6rJh1LUlj| zmsE<(a3OCJq#klC-2SQtOHbk3ddge8)!~M9{snhOCP#v3!j61ck-eJVY(W;J(gP;o zT_7fssOM@pZTjVC$ksUaYqgV`P}`M`OSRKEnJ(^ntU!UxXRVD2z(p;;@tl%APKMc3h{lK{x|6L(g(DPB@@t(_Wu?B z{mB6F246jE;Nz{4*)qhphd@JgYe4A}{L^vz@$ zMg<^P-eR0e%A}2B8JF%I!)}QtIwh5YHc8J_oxh^F?})ZGeq9j7qoEh!T}}?nM>i~0 z>yc_kbR3(7xk)B3=%mi95fK6%_ntb&<13vM?&8_P{6nmcU5h&H4&pynL4^G3CO(F_ zC!?}POh>=G5M?ZeLZN+n9Cxf{g~Yl-2I^{da+J7C5A zC_j-DJ!Na&&v#!5wfC}G!iPj1I4{)XJKvJBj~(v`6#UXx^|SHi9LE<8(-IHgD_{#0 z`Dp6=0~X%W*hDC_~R38%|?bGj7`cRT+dKIk1SOIDuI5HE!9+JzQDoNs;=1FcAC zu4X}-A7)g(yUcTrY$15z?RK)6i;eja@N%&hJHpsG-ee$^_fi@{Zx3;`utGW0CmgGI ztA{}RSefzu=CA>58ilPFBtC8*Qt3*}#hO+R>r(P}cn3E519uZLy$#5xd`2AW`~hHE z53MX7q@MYfNJn++S=Q-;eFx=0(jF2ZkbMQ)fD#eR!?`e+chxXxsVj+CrkUG{0RpKp zoNK2iDrpb9MMKw)wK24fE|N+{7vpdIP~eTody((z{fQ6U0gx~FD%sH3{U{2*s4$$i z;+1>?y?`oWDNvR{^vlk>`sFJnBW3lf7mN#uQRO(sq^F@b4+^Lm#D6SGGqFVk)hSsm z0UiQnoB230ML~bx%Zf}wD`hSUfeX-+*U!G~WUBz}ftRFYEHhEc>aI%Ij-Be=@&%pP zX2d>;G*DdN^yh)K8nr3tgU417ZUg$%xF1b@r*6#o5TM`dLH9< z3tEhKBB(_X1hB4fl7JNS4ejH1%jL)Yvjho{#;-Oy=PH)oMg=Qy4$Ke;!|?Ycz3An* zBC51WA!ScI{Q3Bi!%0cF2!Lw~FdyFZ$6br_U-rj6yJOyifY|E0siDSk00p%(=z|AL zu+@tLW=Gl@@W+K>l!e$5IMae}3n*RA17^RguiS*S%X72tYO0-fH_2#{W>6qYWhsxy zyV5`Z*+3l4y2ZO(y*}xDV(Tiz@3xbFU(3d{D;%wE+l4Uyhw9d77V=HE13$mI+|_)3 zW7JxaM7EM(d$efgi&rA2kaS@-JfVrU$`UqJn+uf}fO9$~QEMuSPbP7y* zxmTp5dV`Qc%HX&>%eD)R6Y_jtJ{V@rFWdB!@4Nfjqp))g!Pt+hlbr9oyhi3owAZf* zZ+cW}vdZ)wA;nZ^c4Gqs;n;NJ_mhI286|zAp592%pEp-CA$9bMg{!(5I3Yhxqx-rh_q^J(NFwdSsH(Q^h^;>4FlloheUQ9%T&kB#s9y?z($j!? zKh4&cUV*M2B=0M3VUC#Ql5e>pgCH^=cGxVn(W5#7)ZW3Xo*XFNQI5UAl-5c7Cwc;u zxptL&6KvFb|HQD1DvdL=cBi2f6ECZ--jK$@L;lj68n7d^ewk$QVg0F6qWgbkPP3 zt&EfC<#HHZ79k5Kb1f`#iIo3M5*1GFIG#l+_FdnbK`OTUgj2luCB-XJ?*4T3;`JFu z`P{fmP_Q&RG8{_zEWTnEu4G?930U$<>B>G4%s2DJr$E+4Dm&0Zh4K?Z@5?N7Cku5{5ydX_vlktWlrSy=4K6Ww z;v^`wd{@r_9WFW2ZlHVr{x$g+7T+paLGK!@8;8LFbpra^l9auDgJ|7l!R1a>JHGi* z$ZB!FeCh)^Qw>EC`uuV{&>5MULl%Y}&PTG{ZQ5Ckh$1&luF5-To=y0x*&y{W8&>nS z%P~>7^rk&KO(aHySn=2jD)UR|Oys%m#!RYnaC`vQNiKXzkvj`iN12u-I^Q_!9p_h7 zM0_EICvEPI^&fnBA$%X`{~S%GI4bR2WGWy-;H0ACmF;>O(yVzDdXLI=Xj>5dwF~(> z8N$|xWi#hf^YVG%7<}ZVYQVX_(%)L4$OK$HXnBgQ1y5nU!;>{(NBj=&t}|B!w(*=( z1woxC7!denm7XvyVu~)$7wc;Gzu}u@IAl{0L?`Nd81bgd>IarC@PoxquT1!jBTLi#iCs#V1vaZM8bq}~o3yP!SyXVa zsbj7l*jR6u6f8*(%*$AqRB+a}RYir16Y>|v6f8~3SQ@4;8C7tg?F`FUP>eF+QpAK- zt;$#)ns5NC-`19|ixG1iNVs14Tf2va_&O@*Y zR#%UYp@@AO5oHiXkOy%e$GaIfiDL!mB?uV1SpHr~FT$3CZP?LzV;-TL4MUU9(Mgcc z=^@GH>Bh?f#Fk+X!i}&Fpv(Ba0Eq@jHei$gw?qT8*8%pp2N9@^#Xx|INZ z25fN;g3ry>yC2@AaP#a@?+Wx9!praiz6talzDXVpFNpb*#LbyPE0(YJ#CO3tkGJ?t z;6r>dz^n8^K)c}bY+ua19=}7u-NpCN735sn@$PJg-N_gie2YI*fG@Jj3;1_h`ZTl` zv+v`(>r{QlJ=5sJ?*bX$d(WpHMe_0NWbVQUP3^wD+soe{ISf;I6dCw>x$Q$9Q<-q6 z@FYjjT*wS^$zxLyBK>+kal*n2+0j>n6#?WJriS6lN<$g*#><;b>+5K!f#41}G7*_( zl)8~iq@3WTGFq*FGfC=kZQHySA}xA7;DIV5(WISFj1wYL;z=P>F^gY}1SEcG)&Pk% zlFF!YOsa*`mIDJ?sRM6_7Kdx?_tHX*4J2ADsijD2nXm=`da1~88<~gxlu#uA6uJS* z)LTP3RlS==Q3FN+iv=7GtUpDUG&@Xf@n&w@*-NC$;Z|VMdH@Wd(GviT-fJIGUY(;x z9B_^QpGG_VPot^-r_tvfHCq2^G&-VRqp<23t13XF2WwSN{a#&~02=LyU~U4?XdY33 zMyH(MKa{(Wzpg2zAj{tGDdKz#<8Bmnk_k`SPXJ}GM(k4gj;FyZEMUIeBq@0AeS~&TgQBd!vT5))fH(qBUVyv`Cwu*$ z{e>&>e_a2P`m)in30Lwnu?2{3NhFgo6GDFh4<{jHidX~&ncp?KP?I32s zb*Cq8^MLePZW2kr3}2UW99Cb*nM~!$i6V_uPm$$WLotV++vnl= zGksbq;f#D=Iw`<5aDk#yV3xs1!i^GMNA{ajxe?-rLQHzmVot&eO+pfEHkEYQ2FA{S z^`xY%==oD?PiFYPI_X5YIc~jj=sz#ma6P1!De(-8qG2FU?LRgqs<_20Kn@W9gCT_U zY8m|pL#SNGLg0Vonf1peLh5!=6Nw$q^Ty3L(M$}6sFtU9IoZ8srxZV9bI!8F4u1D!3`pE*`%$qIe*Xc2DXXx$ed)hgA5nfAN?}`IP z8x9vlIMamnp2JV9dGjd86Cb2ZXilI1d_L6dY6WY`%%)do_nyzkuCT<9Hbn+;l>UP~ z`^9TTg8ao-(A%^sN&FtQV3fNcL;P-k!GsLowTO*zr-V;%qqpFr{Lod{)D8^7mgcz> zN6W{gu^Z}3S#SfAv+OBWQ9PzoXeY1}+=!jy*Ej5L;wo$A*uQkrAS^O2z-K%LALF7D z*Gr(KLtDY!*N)oOUQbT`SSL4})kWGuDK}nD0g<@>n~P@)bf~A+7Ta(Wde&Q8`AI2Y z%tj1FGL6)^splL@mc}x|wUnd1bJUbtVc-!&nyVN)1s@hC)1cfCvV}pNoHR5n7*{`n z3!2L++O{Gll_K?b>mrg99jTN=xk|Q8$c^+)2zqZdZ?;2fyilaNPdN*T)6Ye2n)^hb zDk}b^MWF|{;)lAQ|BvT*)(|W_&XuIh77(qWX^KBxiYG=<>R&1NJSCT^57R>{G4`9b zt80?%EdK4hwtI%HnSbj&H0S}@7pKdy7z5AFwZ$a~!{G9}#ER(P^u+?42^_;H@=G!J ztbX86)Bgip$jyD9pY8T?|J?rYyuaCho98*7!=cjB+6jPkCR2i_@cve1pL1M z7amly;FwS1N|6SEKNN96metmt=+h&AjX6+u%N`v=jte#~J@l2RfR7B*SKn+*QWEK| zsfEqoPVPim8#8nZa$AtqHn+y?z}tLqB3zMl(}E`}VVjoBufZ{UE`ttQu3PV`G;SX3 zUoJfB3mZjn6O9C&ln!hF zDf|2EXr0n`>l^m3;gLGF-s-OmR=f$G{Q7e6?Y|hUQ)&3sVz(@KoV~u{+D$zjdncL? zRTB|doZVf*gL^S1(oI9f9A5J>L>)y1(xLQbmAB@)3o9K<0UI`@#ua-c5U6Cj&Lf^= zQBs+KnsgqI3J&w|Tzqfu|NfP(9{PQ~$&dQw z^!a_>A6+KtU(@?>adPr~pLZhe?&#(G$jJMCMX0{zd|$5~{vCUp{f*1X`S#v;_KQ82 z;(y4CKTtcwnrihQW|*z?87{UdK&wuK6ASaDmCG(#q20<>V3JbAud|cYqo%GG%~fT) z(TAO#_~mKkY(i#avlLLohxnNx>F+2SBr=HZCafH%kf)z}*@W4Tn=UiE7M9B1C;h!U zvWf7yZ@lwt1!1VIzQ!Ib2yP4mJ$+eLA-5pdkwNBfXl5*!oEC{Y-(vyfsYupX8D-y5r`{0KZErJA9Xi;+oIL}ltx^fVD}w!ffQABxwG-sZYM)BN-j0#HH_Snm$0-RMq zm(kq@aY3R9P_h8|WHwE~(Um)R$@yPsZ(d!5%HBWULZnw(Y6}Q9Q2)+uB!2}S`?f7g z=#2DpTyS)-k%?sQPwq;6m+?@_4RlUd@)>|)HR@OOVEy2P1uFQ1s;fvQ)BI^rkJYp6pmY| zi(h~sC@Gp^Q|BGlJwEXUjef9nj?yp+rI|=hAa|teI(Zg%GaFvNc?NbUikP4MUaE|j zIN2e+lIU_sIn-ZfU)jH6QvqIz(rX9G-BLr)n0OUczge9LzJHSP(D>2gvWfIGd<>&M> zKq^VHWN62liuWb%{j=phjHZ9(X5R!4$iB&KB|kBiBE;0c}p zQ&&fJ&hJr5)+e%beS+}<1_g1AOGTJNg-P9Z`Y*IxifW|@Za@BHxJz5)j)vEI!GCrp>@`Xs@n-qDOIMZ>Tb(@-(HGrXAG_sgAr_52eBVPB^C7S3_S9=3>4Zi zECwT)%nLTw(loB)L}vxsqH4t&@lO5_B=4+~{~al;oyl5d!sD|*#lq=YckPH> zE5?oj;g@f-o1P;m3Q5Yr>Yu^qLg)dlcqeSXN$$U!fob!Z`lX~RzDgRSA1bTz;``D= z6+TiEnXbjj&l&aJgwP0fE~TCI3G$T@hHMmPF&Foh^u3=4laj6H9CeJsn5Ai@|0|V5 zi%#ORbJve36|fM}+wgA>P%I0n4KCqeQHxfO7sb}P=RSaROTP2?RX%^5gFE+ny zGRCgb`uDpQTULL-uSyKM7beR<5Rb8*_ft7crf;e%H@apJEID-{Y0V> zSL&uzOpMQx>0FLTYkDX6elFv%8HQ;hGaW`rXCo)y!~s$IKT~}a*M&UqI|plhryg^| zgZIg`4N=(bX&_o085(Qi)G=D%Mny zDf!IJr^-+!znU3>8@A&#@F{b69wR^NP-Fmx$x&z8Z*F6jX1Ztt%d6;PQwO|YKL<2u zE3$a1=Cz+m2D;Tj40H(xCo}_|h@JA*^|I8}Jqah9P)2z#+}r`Mh~`5dSR)+*R8w zaKK`+slZ)m7@7-fj64_Bhw~YYq8cDPA=IGIgGLDEWP&-;M1@*9Vgu}mvUW2BPqLQ` z<*9~^2zBt)pHhf=kFykmS>LNdTu^OKhz6rQ3P?tJBQTxfsE&Eyl7-R2dh@&FufUQk zR9e%t_+-qWLYMFYH~cTiK7S|?Z85ENXObBD2k zjQsb6B}R+qFcvZe5$r{o96AFW06+nSsz3@wXaNg#?p4p1(UD|vYh5LeJzv%aup-e( z5|wG4CGg-#T|QJfEoft#*8+pguCQYX>r5E4V?L<0Bi>#x%M;jzZZLw`b)WPTo1m)^ zT=J%dm5r+6!&op}dGAz(uHu|@|BZ$W?CB@N%;%kYR47sD{OJUqfTF^D;v@;s3>{D8 z`Hx}tAM&)pLRq!OrmYR-0#atkpKNt;nco9P2Bq?DRsHy;Hsbta2^@2$jCpepg3)OO zxZjOYKylCIFOH4+7`N-(D~(Ato5+W6$Uf$2tX^=N?XmOG&c~yNT-+KYn*+{#F~`x% z+zRG+=I(Ijs}qNR4B5R$ZEsl38pG;D#j8$GIb^&4>HaFBZ>7_NH^DK45$u z^FxX2|AhKp0GmcA0Jq_JX;}1(X3Z)Dt=tZ($N{bR!vn=Vq;AI?zTpkj{JLk!liPrq zq(OFpW|~U0j}2WK)^>=e&85FLH;8@%VHdyZFx`9Dj<$6*=EUHK=J%- z4F3+^Q>DzAl{7$HJZ}yYmK7$kvU)fB4Ama3Q`61>!F!b_9)@sYWJUat$KK8E{ z>LrtrC!ebE&;6>2%zJYDpbHEeFJ{N)*xf#k?C?HX(a^%v#qC!MJln-3XCMuJ7WA&0 zP~#y)*=<1eI}g)xl(92BC5}C)?^ZuH^jtJVI6}*$^cr0!Xux-_ZJNAr_e(gy#o{Pb zqo*|7AXGCWRl)ij)6X#YOeG`p1rDp%Of|K-X}7805z`mV4$~w^1nwX^F1n%xEHV-I ztDp)?`8J00#%fOL5i>ZXbH*WPLuRgBi=dbW=u$HWe;&*K!ggdbSgsKrkW9!R?U3T> zDAY8|gw{{TC3eNrxFo|icTVS70VP5NmsHW06l17_;pHnnuJTabxDRo+E}WIaHDbYC zFSbqGvd!J!kfoEb^0>GuCnB+UB${x)l9IS|&i|>Vd`m3hIsRzT7jTmx;Nl3{6K@Kg==nw|<0{i0rt$+i~34npZEMZ+02OeG_&? zI#ENA5kYq0svu)2=YGl-GvvfqAUW(VS|KNLxOzijf@#P3VC?(0w{8JT)Lgi28mQYj zRlVL#LkQ)0_Bg=O+S|qdRr~mA_~93u6e7Pfi3HJ((BK)624<-8pKGJ9KUSBYf?MW3 zyn!m20kd{#*x%g~qeT%sfvT5&)Yi#HURHry4}NbKm{DEAV2YXeDMM z&@HSzS1Bj8dMH70mR$W5rbP~J1%Zv8=s#nAMGhOm?ZX@J;~wsinL1JDG!8QEH8Lld z?ks$+^M>YL&;^<)f+w$wDNJvGb%j@x7pYP{k5?`+gYDgXx(XSe6pc5qF%M{7Eth)G zOyKJF`M+DJZn2+Og1O!x{=>>Ux{myYPLE;$fh(#q7WWmCOiNYTDu+y58AlW{&sIWG z%G4=+I~(t6uC%PAuJ)9JZOVtydZ4-0CozAJ-}x7PO!z}P&gZwYhb#^XV@WIG#nSTB zCFQZd!YZ}{EHz(oCsvsAmG2Xuo5;#%g+VcH%qk6=!kr7qgpx~q>(PW<=f`eHSHx!GS(@&Gt|N`PNdSZuXV3~)sr+XqA&GaG+LS(*xmDJmw3c6K~yi)++ep3EdsmLvB#IkHRNUy1X=~oSD zfT_A3E1FQ<;2B5iX zBVeGU3k>42kwwyJ)h{%3@1MkSs>D8!H^kXVO?e_42*FIRYnMexeGx>B+hKC$2-+jU zXNkSlcy;epsL09YSLbsiuhBacf;vFm5mK8+0@KpXF8ajWy2g3fJR^=ogED=`r`aRu zW41C(`oRqyZtJD_90cs0-;};wh6Xg~&BBjDbYhS+6P?H~k;Yu{JRIiB1vurpw+VjP zbq6PG2kJ1pNv25J7Ax%EU3T(=-{&n#^jSjoS@38y5hli*X}$`k1^90CB2?crp`l z1`3skXCUv|6OFX_lFoxeju4cdM3{x1DG`-ikTSX&8j)^YlJ`M`TW3sINWu!@wKGY* z9ypuh)G2Mz_uAd+lWCx0e$UDc`@{$0*DaYWP4=5ja_Ss|O%^Y|XskM}9rQ$*yeFp< z5ZpiEN2C?Ok-rfUWJpg6?ipPTg}CIKGerpBDvL{EKsT=0i5E+->u zt>Nv3M~LiGsDvetsiem1#q2Bk+I(prHu>37Cq{`VS5=O~u`)Z+;5K=UKL8J;Xjdt>Dq6AhcY zvawFnsq!Bf^M+UoB6H~f;bI10v|qNZpQ9f77b-o-P{xRqA4#n)l*vJzPBEG)SP&C8 z8FHcG$W>H@gQQ((Xsng_DwkxNhsrQ%FCXSNQbv;;_A)sAL|y3SX@;IYYl|I0K=2Xt zzaIsHI2#q~`8;2(Xz1yahHk#w%k6;}2k89W_ksTV*FTS!yZin4q}Se;^Uv%40TbTa zDA{~VUTF_G9YT%TdcP>Sjtmh4P!SgG@oPU#^>Gsy2`Pgi6y~sH#^Xc95p!mO?rvN2 zjcxYP(x>z9T37N1GnMtEZnOtkRpxy+H_+9p`DJV2tg8>~GZeCEk3i-!No04pcbN3* zxWPepXhSB)CiNktEA(pL?3;Q=$X zn5Ck%q|;3H(dd~zfAGqj838T-?(ZGC*Zmo2j=f2Gmx-s(?eT@SaqbE=3qTo^H{8`^ zS$|T(>4i9@pHVS3UEzAbR%(8|J6U96G<&V6&D}q(j2q7S_VvT&wDu)JLTGXSwevjU zC(xjeVTs6*@Ov*&hfTUCI(A3S#wHld_;ArbRk zxAENIey_yLD@eW_+UlZ;)88{9qw3287xIPuEAX(1Tm9Rz0xZ0@gnkUc!z^sy6hUMn zt@75M?t6B_5WcsVm&eO{1hS;Y>Qva-*2*7l^yJmirht&<^kR*ZX2Nwiv)J`AKaWRB zzQc)Xbb7_7)mJk99hIC4OqnO_?@>p%c&V57_aw()#@f&6?;T0SWS+M^8glnPPCBEM zWYN-xgg_qwvwO-#hEN2`TEeeLj++0}rXBNDE2FHni%~>)Y7Upmg6C`mO{CNU5;P3j z=azUlG2BU)e~>cnuIk>Aben0sDlz53SWP zz1;pZ{21Y_%ev!u@G-(U!a)KbC95k_X6{K}kJ;-;rJ!@8@1}N9>kVLhyd2>3(5CDc z88qk!;fH68E-@Jq0s7r;Ga^7xM&f&H;D0pJqzMx6LC= zuh5t^4b`LFJl#w{1B9cbd2eHin5Ccu?iK%DDe4K?2rmMVXyH_z$zEM@5E0ui_Deg+ zKm2$MB084l;pe~Fd%Kq#^OVV?t&I;-sowv@x1C*OM#QOp1Il-hX1GYZvzr;FnQ6ilsj%4o^j=^q zEwaf#i2zRKtx=!m3YF4VdR+ew+bC<3r8j%VxUtut5&k>NjgJhaUH9_IDd&J$DOOoY zM>pm8qvG69d;YC#HgS-Poc!M;cbo&3av}a}AQ^NVy-}6qKN4h~i1=h@I$cbw?o}9P zV2Y{jJVFlPj8xIe++Gcoj*TJCKy`j$I@>x3d+|X@P0lHXkI6a*1Td(4e4}kR>jnMg zrwB6LBZ|`lJr={2x*VS`9T$SqR{I0pWm$_@#?Kt{7-(&xR90`|x> zwNkFnH@yRLGqC3n1Ec(^8b?Yd+l%At`FG2hlIlE=s4G$PF+^F4Xl3FnKV9V;{+s{% zUUJJwhhemsMTvW#-Hw^M*h_l}GEIcqTDmsr)8pX5bmxkp? z9vV<3l`XeN1i`r3nimo?QG9zy-DFglp5DL*)-&th6GL+!Xuf8!ZILORdzT=`0kvXy zibnO($`&*%c0Ht80a{NYNis>RaO3E|DoJWA;>cT#)9`G=^+RPPBdjIig&o-WKj5C0 zV_g?oE$TOyq@gBM{ZpAd3S6q4*6@PSlCOVJRQ6C#FPhQEYzC93%4Z56as@{jP)Gd1 z+#pjZOCLje^r3`VX-otqu}7&u<_D!+;wE*vUpO>@2B4Xf)h@_??MZ6JgbXQV_!LDu z6tA1E|6=`Yv056^GQ1dbe?Z9Qj~}AJsF;bW`F{H`zrD`FZLjv(|2b98Bd-x(2rV*8 zl(kxIc2(Aj;VFz1AH*4-k>%8*UxQCP0EKhu8Uf5`-JWI*6Pbb};6w$=(E%O!&gGKIvbD zr=&_|SrB^o5$5zu3CGFD#mnU&BDJC>&IM|8gm%h{IP+ArCA6N2{cq?Nuy$BwIP@~yp0#V z*GQ;3MxnqcN(6{@@*uf5%G>VB+D5AGem75Ry2_-%KJX8mTw1i5S$mC&53NY4B-Iv0 z($`5Ze7kAS(Q#u%=k>@_lxeGCu|?_@gl$-{Zl&GN{ju7TURlBB;=k|5r!GZt?iW+I zOAFz`54s|J?c4TxJmqpYWL4gQHFYcC%88{&3Y!Mq6An4wXBC|GNx^G3jsD<4(TGjT zwg&yA)mp?NSlz=t_jaZ2u0D%#3nlc%(4;$Hg04q0z?;vTL-t=;W-&L8lWL^CpA1tc zw@hW*^aieO@1W6$>x`&jQh9lD$9`RV0z1b^wtwUVdr{3%cSgi{AGEc$?2o*cssA4L zPt4wPGvGf)TJTiCgDvNfbcH7P!2c;(IZLqf?U?T=pa+te#BKQ8J~6GpBA$*QAo2FBUr1be)pqr zI$SYsmC-_(47r6YKz08CVjCCz$F>siW5G!tN*@4=cLGb|0 zFXAg`Hl@I4-QhQfqWi&13r19Z#NN`JO`T7KNdm&lD^uNrO522gk=p;3|7EzBt%>I1 z7Ap%*H*{7@OrzWxA~`wMji}FqtY{%Yj@5>_-KSDmb6 zoWb0zJrF*fz(P&h&0ysR>g4L?e7Sr9KE11(6Vj9G+FQLoNCcmmj3a~vwT12o9}t`4 zgQ24zq?faG$E!iXi9N_o69%5&YKX0_(k2jZbF2NU=s3cGR6uVsz^xMKL5rs8-5)$K z1S32(eo?@^EeB~2dvwQ?arB?SJ3#Y^cdsd#V#NF=%?8^^Xb^=93 z2!E&&lC_iR#j~zZWSRC*hO07Fp6bm}(NwfI#7ha!BYuV|P3ycRr>Ze^+MF--bmkjY z!^tmyvFU|Yn}-roW`Cegq!ZE$w_f^+rq@pc0?d>i6nq;nS`y^&E_i1 zbVY;MEY{yVHZ5E2=^{!y=q`ZHB2XHB5Q;6T5uz?8Ygm&pxhmXEnq&l=R7yv-*JBWIO*4*#ZW?gd zs5PUNir)a@^VGf&3g!+5uMvC}W!{z{A<6e<)L;1W3`MhhFGfZBsVhpikFs>tBK&T2 z$=4J1(q)0F6b-Iz@X+J!g|ry{MchymSYk~QC3ZiIUaELJ8<->%8$#Mp=TwfiRrM?J z^wKj|goyz3*JL#IMIhCSf;uh09By-qL^NUVX6GOp-IYgvka{~;@7UYplilWa-y|Wn zI_S{8o<%ABIE6pfgi) zbLw2OCm0B9w?9%z%<(;LsqKu_r#KbukGFOF%0HK4j2=7@8)-=tfpK)O$bKznfq8M) zCL)?XOOa6bb$d%^gxESQN!~2&6!I*WsseSw%dzF}?KT3)zZyWvSgV61TF_x&lPqCn z(oz7vvfSF{s9yLM#u%}T@~Y96;&qa-C@NVere*AlE^A0{(2d=$rbnN>f;`|{G%YLg zGO_DYu}QPUU6s<(PjyIUUMV@CziGn z2tu?0jdzdcK-Gx2Hxi6I9V$r5Imu!s-CC1rba#}0*&k!1R79w-h?~kmXPv(s3>P)bg$b?X{ z9^*vD`npbV;(A-Mt9a2qd!Z3$I|sp??|nB6S5pDsoDv&X9K0Aua?EQj?%TwA>|^3B zM^Mv|r01TSyjpUB)CYt=sGI6EL<5v2ie_~TM_G{*RZBKW0^}=c=n}ue=h|=Bq&JKa zvIuisEWhZ0RM4R;u0`HM&DNvITiGA9+volcg(xYpo~A2)xY0>F!jj=wzeu(_)ban- z#-FNKCi<-M8MpAc=)NqwN!Whe;3tJ@G`7tun!-6;*;oA>%<}s|L<13_ejv0ba#x8{ zAR3f5S53<{)6}ML_Au?W)YW_jQeUe2n5k-HflpYWl8`xjMkg=?(uDR#kig=8E*lM@Z69pf5 zZOwDsA9i$*)Ap@@7IHs;u!2S34GlkS!rE zB%r8TSg+jQkes}DpZlXvZ~1R#pJx-60SSAp#Cr$f-k^Wa+oQIikL0eay`5dZG;|Ss zR>k0kvvdrMv!8^kC2Y)9EVhSl<8m+rU=$U^Gzh^r+Gig{YLL~6!sQ$7^V|Q9EN1P! zQxdD@3LO4K;GRV09KSisR3MW^PRWOvd8F2Oa5M&#P%;pj$5?)ql%Sp92v6z6wDVA&}<{8$-bVPVZVsvh%) z&}J7`gFJp2S2R3$9&mAlWNoP7>OJkn2v)70AH_-gJg(^B5@~TF;lt05UjEvf$0QVp zSB-uUzCN}@^W8>{t{2*joG5kcXz*=)_+rrWosYUbs#J!(hzC`Vs;CB9)5(FK_K8U3 zCvAdr2rCLWo2QFa!wijHzqC@xf|mTlUIcIq@Mte!pr>zlvaxP9EyYz~$;I77ls^xx zSZHA~Qo-pP3?v}8bkKIc>yrc-@CBvQOLua-zn?J~aiz0)|M*^|AQ*ZBSn@x}?E6jN zUHO3C{c?F|eHfcmn2$lt za*+F7ZTg4261`~g9Ulu!5xOPZktI#i;xUVHj9oudp!A{>oqdZrQMm?W<<6B5qO2R8 zlt!)9FrZT=$w0%TtQFhguEf3xi{56w)j4SK zAPAPxjRDQP^X-ANZJm69rk)GY(%G?WBslK*c(JIYJUG)SHx=`~x$jzLx6d^+ddGIr za$mIBpwoV1eV0_tz5X%xI`n?l%1fvalSf6dl*dni)hKv>ph7q2Z|g zZ=L%)D!lu{ol7Sb@^W}YwOV?#4f8@rQi*^h+b@Q7TCcoora${Z7yAuWJ%UzMqP}iy zz16zYzy_QYUmDquA`hRb;dum~q&JCljEM@~tpe87YmC?xnai*>Wm`iwTMuhbSf!qN zQ#C8~a-_)i3>J0w9nU2%>_KCSJSsK2-=R|i_q+>C%(Y^^bJ#Oz?=KZts?efpQIkI5 z_IcnTGgr04x=22#r3>bwvxsS_3?YN>aapn?QLu7o=82AHRfKx{(t;*T$nz^Qdv?Wb zz@b90nmY~lr#vM;kE!fJ0gmFEs+iHu8)wz3SX?DQ=QR3xgs1x&8TrFyV z1saWrd~cp~ssV`jQeQV!M6`hUFuqZkP-$XGO4Ntm&bWe%DOKH`EfmSoe3Iz#5xL2- z>x7`f=2F@PLVb>$0NqgA2054K)GMg8=tW%!ypF(_#(|yH9L{I>WGevNTH&Jp=r?=) zL2$8Aa|Ccepn8WtQSOpoDYGGs;_808f_#knSO(@-m}q{g{1iu+eC|ZxNtUMHM z(cBaR&D7bULVE6p_1Q^jWJz1V?n!^<`^5EDgxS5SR`@YGWfYGUrWhHd%Hx)+l2d4~ zM3;8|fv|QL9w`@r>df5zK-s6HV2+v`{Ep_<->;9yMWbbD{F(4Kj?qQ_T^ z>KblT&MjVA%WX6u<1iBW?O)3-OIUEA2bhaj78FXzRvFus)&&-^pG?jB=Qy{67+uKnnCjWVz)m_MMq z{LGa0DA)RcNtlx=XFs}NH5>zWEH?2rMG9HU;4!+ujY`>225O$Ao#~&b2mEN*Bsu-o zSr7I>H`kb6&5|*CsnTL$$zvLFof7$F7Ly-IM#CV1JauhDKEXV8s69Jy(iJeyK?_&Q z@%+GDuEr%#9W9vy7Z4=Gp&P;t#UhMH|2u~Zbc#{A95@(I0wLL*;oD^Us zo{Cbb&z_vT3wMn;X)hwYuNPsh{C2mAB2Pv0cpA=shX6J(4a zB=fImiZ(y_o~Q|YPDvzn=v^KViKSs+U4p&Bo7$-LcN|ZAp#ThUoij&h@Q*_~+lrR; zGklh86}kYT>j-YVNZ1;|FqF*(5KOHX+y$1?)?g`dovMvao?;W$N6EQa)?V{AtmOs) z?}9e>EPk|9!jDT!cX1Wh1yZ`EXYU59u1M2?KG{w;l}X7jp=QmSouu8oJG?GZv#pf4 z1CGFn06-zBxO}8mU0nLULPZ%K)t5;%tdb)z^u%h#SB?8`-Zc#6=Qeq~OcV($Mzv1T z5%$GJ1sVrskt&92E%l!lo$7>7%V4Z=2Q}GC?{47X{J$%d4~0^CISzd%*r%#22-VF$ zS}D|NcdD=j&H~IDoG~Z6N>*>ebHb~iAbw~=*j&JraCTTHh?%3#d%dx4I4H;&CL`&_ zeXzd|Ui#o>z3x9f1%2QBc)flc{744Q0;~?}%6LEUzP|760iWse_Vi)uc#6q-ef1~5 z)qFiZ+{IfOh>Eh@+uNhl6-H<|#4JqF>}>brFqPu#_7F;w%uU6Jof1 zZC_)n%%(kyAx%(!Ewl4bCDr2>Tpk@t%=-IZr^o&x%HuatX5{2RNQ^c`2I zOz;i9Bv%6ttIFf|jc(diS>`m~#yZQ9UUMk3G~v-(a+*L;rPM}*dRKtlzs^_pav3wU zB{X3cMeIbKSub$?g_BuiX#*Fszci<}YNacQUw|0DXumzu=9JslpQ>Cb9;(F{ZT|ZA zNNGq2C{@ZO1{11Ht|N==GlTRqvn0G#EcRdS@1|oSCi*Qu?e{gwY@R<=K_a_Xw_7U) zY)1S856Xi;4SUxm`Shev!|tylDRiDgd6nGx-^f(Sd?H|})ch#6p*m!8XG~&5{B)YU z2uJsUvbL_=L5mK59&U$qWXWs;i*DUqV{!p-#GgD&Uk9!EA9#f^-JD7+O0yJ?dN7`jUNL#e(h6_P&>p2ovYl({Ukf1Zp*XaCRDW?mI}OJygkx|7ttvPyuMBS zgeoXhh}!@-loZ9xrmnkxFlwt~tl&29Uc74V-A3Gv<9F%dnUl^QMcs%6R>B+gGx-`!3Xx z^2S(jMjol5%*_Di_F{8TYqoc+lv(Xm08+oD5+BKl6Eb?(W=Y%B>%nlCK*!+jSE1%9 zHNKw##DNck6^tT0>m2T4-muP65)y)uQ8H07jXPT7&$nCfVkg5tGSTFS!%0Lt5A^PW z&p3UUiAAR4Ig1cp7v5v!3yA!vPM?3F8V9myAq0>YqO(fS-pIIUJq{_N zSjZgADl$7<2ChAzN8MpL?N$;oHjI=Oj>Z5-N#_zyX3M=J%*D7vu9BVXWXKi$b-neW z8Rp`VX#q+`Jk}N~oviUJ(WM6h_8YBm(6kezLfphpof#_pB(||AP0PFXCKn0rt)a)} z%wNzHt^!Q&bcwOG(Y+)~KPTel#sMIDNaP2Od~C+DEip*QFh9N27=rW8S^IVDqjJ@! zfGJD8$4+Y?Ye09uy;y<;8{f~5nWlT$h?H@iKdE5#>#W9@&W4OQ5lj3>Y<+>Qm5|%j z^e6oY6jX#lZDa*u3-1g@>X+fFGm^I2s-Vw;{h`5(Sq#dGe&mW-7+sGsL-QgoOA!d8 zZqnhQ5JWasQ9kdpS?^CpKzpwwrp8Y@Svld|`u>jn1WM+$cF|c&=xgPV1GM*n@=F19 zEH{_zQ~-9PY0gs4e6WCPR4OibVwTo+2g7EB7uI(tfZFu(T~}kpueq+8M4uB$0{^Vi zW4XapO}j-9z;{E367&=XHfQDHt*_P)<46=U-eU_EmZQHdcRNxOES>$q1s7YEAzLq;UAEn=n zoqR1t4m=zq$tM;@>3|jdO}8TIri2FjiqA=m=RK^tJWzr9#1{3rZc9}UwV4c3-RE_=XdkO+Hs54nlJb%i>l8(@#BrI3iRZwl z*=J;WdShR2JJ%zL0U(ld0obcm$&Zhi6b*iQk4^a`c+(R{kzbNX770sCepOto!7m2VXT(l5U)W#UvhBqhaFt)c;UW~`^ zvF6pht(tMkl~B++wZ@bM=IwLpKh_RyGT#OQEZS#wBo z5j?-^6n^?lwAUSWe!2&!Ha7IVT*3V6yxi$=-Hoy;uIF?6=7qEZ19yr#$4%F*D6OmZ zv2&Jr70WB-fQI>ml7O#T;dLv#dldeZjq#=rj-Y89v6baw)5fv%iGXDGoS=)SICCR0 zoiP@-)yCoOwyQzj_LU@6{hUy1+1;1P+1o9%e8LCgJp&S^C6v$zjRDK7OQUj{{QKK@e;k00ww*pT0n!2Z5_2~oX|m$%9pK=F-CUI@e)?4j6y~`n=7^&ZikxW zz2o6MWu?ueiJbi#6fl4er(_e05un^CamybXntfR~xqam{4#z@C1S5VL!Jc^GI>|0_ z>ZS}`vKMSWB} z02SZ_C!a~BhEt}-;`p%hDx|~kTNVp#4dkMyDAx|Snvm9OCH-(;h2)|^_w-U9ZF65_ za-ZoQZ^$5qH>lwem|)c(O5a)i52g1HS_cox@DBu1dXN6EJjU78kNEnRpuMhRdUT{8 zaRv}C-B9l*5k|Dl+sqz=jIEuK;H6I7P^y}G%y)v{5q%|3M(pb5{9im@7fL3-`VIU} z=H6S|6n9D!Pd%9x*!h^lc;qkjiZ}N`_xXN`_nIzdT*5+((i@3kt{8|=}GEL8^0Ket$PZa;!2m!G|@##2#du+c5~cX zUT`B&tWw=khL~Wb8+2R-^AH0Ifv-m%QBwCiOKTv4fYqP6&bA_cU-S5*{m<>ctQ!09 z4Q|$+-!RUd{=`1{Pdlk{qWxdZq&~;!)D;-DiUZ@sifNmbEjtSzWa~uBFmjlkbWRav zO|XDinJ&44uMZt|gX{1s=I^N?50y+S*_Sh(`bmQ)(A{UsrXb9p^*quf6Ry-7NhMR& z<35c##%X36SEf9`IE=#reC6qe^VPcdDByptqfys*kznNIQrmi}jH-_+{Jw6uToBEo zvJ$(p+)HW{-)r;=wD{g)QnG8%wwoG}J+3O>X-KFlMUmP1gX+Cf&VE1<*6qcr=~?pS zw>QGM8!#yz9i}lmyT7#vk|AP=j__1yS2yHY_%@}<_B=1b7QnczY0r2XP(zG%BZ=dss}FU z&sYgxu%VLbX@+pAgR;4GBk7h@;vqzq(ea7j++klxDfu(*LzZG3b=n`X9`!H>aj8kU9njsM zN3uwFObh54ZN=-L!-;`&K|Ml{+5X}>29qK)1R*s-c!)FWv!u6*PzuKEz(x&g8i z$f1J5>KkTYEGqz!GC=Y+k)n{C&#LM1)V@(RM-k7=2rQ^9v?={cyFCV-7@vc~X8n|l zp)7UW<}w?DGaoD_TsgUm50OK0|9s&2>>jm>bjXbJ{XR$4<0>5hy3*)t$xZ`kvRsz{ z4bQr0=D5%J91%Bpyz2mqoJWDDupFfHK#Ru)-eBo7D0b6@)9%N^gn0J zuSNLRp9P+F8R_xw)85Y_TFJ)+7{)&a3K@5YIXG;#nPzL1LcHX1*cJFZD!!rR-Y;z^ z6!SukB4$Zcthrm#iJ~cUZ`9xKws6Q%>J913RYb%jC~ZIIkmL!+*r7r^1K;1t{H3OK ztD)& zufmmbquXi)(V4f05PcaB-A{^0N1r@xYshAoeKq6m6YMQmq7T~W| zZTZY)aI9s!gt6u6`}OJR%jk)qHP58-nIRYPL6o$hWAWi3+7K{FI z2>rnTGUtI*L>;9hUN8hEa4Dgw!)4|8@vtf9@hbMW18M_9uym{Vz zwO^lxL}X;%Z|2LugFd&79NKJ`&o2|=H`==+bjqZTD*SwsFw%)9rRQxYx!seQ5(;J4 zZMI!DlY!8#wqpsmt6s2G-Utw`B(2!+SQgw#vGAs@zZ?6;zb*v|7fB+m>Wo?^BrZDu$^>6!2-_J%06!NRL8D6B? z)JkD!l`8=COn{6ToWH?l@oX7<_h*Cx7K4@?tr#sz*i~lB1aofq znV_DP_I6NM&Z%VoGg)jq8 zMDS%YqMgdLgb~RUWV;_fwagGExKyZOk`!?hm6JrKqYk8u6r@tJA7ZRNMywufwCzl+ zRavxzK$)-lhVk9li3!$@*Q@Sh+J67EZUt*~vo^_xChAH0EfsXzd5z&{l&pv)?9~77SMB<@)+B+Uo%p@K8dNet7X7b5^O>> z>n^C_ej_-g(KDb`24lSs6US-HxEMW5n(~`)1sMMFUJUz`UdR8j^+|ko7Pts$lO>Df zD;}>+u8trntDNHDI4CtTN?~A;4bGD8J&erk1@ee#2@3vpDsD9M$r=I?NW92M(s9x0 z=trBK+Un*EE^ddZ$T!0(ps>D8*|%rGPTL%-3GptcR2|%i+viOpbfgZ}BPWiUi7Qkl z)u?NzZcH=ie-EKgH(BOzY{CdxyT}NI3ZUCfH+lZeYjZ6VDloj4M;);II9$R(O7U%m z?E_>9!5%&fDclj_69?S7;2?>qJ4=zo>2)jH3T1(bdsCz$WakQL%vaSY@bG@WR1!wk zJ*CZBWFN#7NYx&G(3=C*I{UdgZb@T>W zVl;q9oV)Fz&tYtrp!l~8GKnG4R#N8KT6@J1?Q1f;wOYK%e06K#h95#+edpdH{Dlq~ zt9=71E}!XJ0t}!eQ^YbqLU^PEXx9R%D9V#pAHAB(Jw2=w`eqTVwb4j(qdnmTD(4vZ zjz-!0Yp$+d8`5jWtZ^q9%MZlh02*#GXyI~-+PFke-&&O}u&*9iB`8xWq3`6RJKH=( zH=+$W{~+tbc+==Xc-LPMTPxUnjIjeTDBWIU3OQnUhy)?DaRpgSNx3(v94FP}VjDIJ zT#lYTowHInJHLNhBTUNvxLG<~S^Ls%$j-!E5ZG!u{Ly3k95_hvRJ7cr7615Uhduc8 z^%+bAWD$&2<}(B7<@rT)j>9tBdmd!DaP$n%+YIHqQav!zM z;_#`t#)Y9$k8zgT@pp*@a*nxdbzrZ94T!8i`*t~hiW;b5`A8N46~|%6VtKN54{ZFP zbA90dYI6csx%gvDYd+562gnHb^>4x`h!F^4B;@pFxT_V(U!&?M7z8bCY2)5y=(+v2wIbvMc> zZ6(e&1{YbMHr$}TRiW+snR`-Vo3?X!&Rd{=XbC$0c@faH{iWS1O%2qjS?y?QfUmAx zG2cBYs%Nt?bi!cu!5V1+aQ6)J?we$GX{zG2Swvem#KNuWoEkWL)~Yw0lNtDkS2dYR z1bRQguuK1xKE-^fA!zA4o~W<^K*89M*JI-oHS3%zx_FK}2g~?MTT7XRfNjuwL_1KY z&nV8K$p#7XQWagcItt1xY#aMf$!M7sOSK}{QM&FBPLspGT+=QH`vGy|;}h>H~7`hY)nl{yJV(KMkOu_=rQqnXDJ! zB~nmh6ZWUx@g7zqRT?eO{Y*apG|DO8Fo+kd2R#ns>wS@TfCRIMH+pcC)aXGos zOvdu)`3-5p?W=P|3|XU0#TZ-(%1pyYoh19b8dfNSgbQ^A%!3vpEJDP~Ixq8PHMUT& z63W8(;_at*Uj@y7m3X-j4Cn7-;A={fV^w5@Zrg_z=pTrTW)JKjHn>Mh4JPI!6OH8> z*2qte>#qKx#`S(tQQkb5h$gY$?8ydV_&>;6Kr0x{sg(~c)CCIg*D^yV$!kP9sIL9dA? zO_$)(C`~wv%E*?Dfx_sN?Uy$P7>ezeYK(u&(_VRc^!bsPib6HkOjdrq8+4T#wvbt* z8;5kNP!A|RzL?YIl#YoN!#D#%vg13n4;HI$7F3L9jb3`p#uEM=-AC_l&pqfIHiLf) z(lgQ}eVEEpFJj^dgSgxnspKj@&k^UVu~zqf&V6#GZQfzAuqI{!B@u7llZgVUb6)ukS=@2q7OHtWub z29~6$N}Eww)i8>olVCjjBI5PDRQVCG&0eoKG{*WD!_UnQS<$-sY6l^`-PMzxPEd(E zdm+48ybo|FjiL_$Hi~j}XVz2-b!(JZACwy18ZbK`dbetI?6HeooOM2?)*s%upQw9E zQYXOF#w(bnS!O!+d5^}?z9zP$G6+DlaJ6x~96!I@W>rq`=t#A0%w6n%`7%8mi$hE9 z3D*%sO*_HybH}L`Tzaw)Z3f!6M{J|yw?m)VL!`0e+gPdVcSI1$QU;w-1;#)1Wg&rI z37>><{?>x#a(_3V?2+bwftk5{`M=eEiuxF0+r4(Nd|G^=zKs8P`?oE&_hF-}Giir1 zl*6IAlPyJ#X`+vm5R!{=Zyp?7cZow@WNbMQt~*N{UizfDW?SZO?ZPy zv4f2@>ZFx0p^kAP@+H@wX`;tpW;(CO*^QtvOwm*+PRYAk`?*0|C(u0t#Cpipv+750 zVUSWydo)o=nV5*VbXjQE7l1;%s`XJg_1Vzj-*QkWLR(*AKZBJ(oXqwe;-+?9DUEOL zj`#hi!;9use$8>-Q2XRSP9bL9rTv;#u&M5&F4U~Tw`f(oJmId~=1Pk(pj!H-j9nb_OA$SU z4YgAGs9>po%{-)TP%&vFrIN`r=Z7E*z=jbw(%)C0XjT^_wo%879pSV?690fcLuN85 zhACVU)RylBTP^Dl1kveSZt20ge>b%H#Wx6ed%m%=H(ts+ra=(2G`7j59%pjM)~j^U z3%yP^BOHyaU$oKw?zJS}`Au&?!O7!>4|xFa?HMr_P?m1c87AYYX#TNBPsW ztPH1Y0k|jw6Ih{9(%CDYYNP?&$XZM9i&k`)>}EY@j<#$3Zf&C;PnJ;cLswUu8vGh7 zS;`#kF2Vw*x&lG+#WIGW-;f^xeT6nRS8j`%QMjDpkJ|Jld8n=?r_hX-2-eVP z>_@elbo;Z!DIxI+aUGLDG-X2?_Hg`VbwiWHO_(8zVuvS@`=K>QW3>~IS1F=m*K{-! z=`#hBit`V4)s<-CT`BbM-}VCiIz@FPQT|66OXDY1oUDCMus_#AZq>dvK5@Ic)_X$v znYVopgB|jsJmn;(JF|GwaAFXS4Km~IgC`nMzoa1?gdI*p0SYNnoxr_Gt*^41$NeRM zED0*G6oTo{JJafxtxYVby4UJxvKS3#!G%(B3!xYa-8o@(!0o6JCBbF!;yw_v@QX6| z9E5Z->0r|AEn7ATI%b4D{DoIeLi5 zEz%A>ri?0OUTgV2?{a8r1*zyX21dp7@!@pALfpOji0Jxs1D@~&sUGGG&SBZ6b+-3| z0}CYzBuu3|a0PN6GMJ$(hlzL}pW7@S(z=EQx-A(DvmXbXq!`~|+BCBz>nn=B=& zhg>J%XopsL*Cui|FkPQy6&pF=A^~i1Va^p9s-A4dF^v(ik51q@xgQ%FbcZbjl5RmV z9p_~zQ)bAu&r=+dFW z!|1Wyxg5IpOl{Qye!JPp73JgVVDke!03;^#br4Ieb%6+ZD4S3tgSSY1K}GdT)!Fxo z(>mO((A4|7TavZ4R~%+wK-%a5k3>TETmBT z#ZR^1FfCa7!Le+cXUWN(43$p%-zF8eOq{bCFx7hN@Ai~xterA zVss)^KPk?1@*?P}dL;D3w*#D$;0<9~&USLP2QgHC z%lK~W7r%#yvnsg9K_se+cYZ(6hoV}W4MV>PUvrga_+^;3q6AlKr}r(Pqc3yAl_F0C zXFJ6^C}TLJ|H{Qtx;VBVcmyiQRv9JpwV9!XiXvH*wZGnb);0^Q*dOM``Q}0TPK~!N zUn#X}$y62X`xiUyi6R9Ho>6?rQ89bp0pBJvm7E`XfeHcjA&5kr04%gz6RSQlI%&LL zZ!1%fRUAkDGla!d^8#7%)neGd=xLk<7V$ioAU|BbFF3V>Vu2sN6^wk2qh$=BL^D z4wy3iAX2?ra1OKU!@(`YL`Dwy-0hP#TZUU+(Q=M!#xfP$>EmXe$EV_UL@9u%Wd)YB zfG+xQ>b0AIW!&>gajvC>+kPIxq~!k-Lj-;yLS~9Dk7_&+ZQX~;ytb|J$y||y` zv0d7OH?TZZ?6kQ9!OYWE1m`RD!ngsG7a7|FRfh>~ds?ks48Zr4B2V2;hj-G|`54l6 zG4r)|Lj<`9eFV06jb;E4J0>|nH57Tfhje?N`R^o0;Yfja@&!ndrAZ!jPJ3f?7((H> z9`a%-kKtn>pz&Q(MyaoI7`?0(AMdwuXKwlx(e1u^7vYUxtdI8cakqJ^6v8DgBvd+V zg7spViRC1lK7zCjfDacd__ z1$kVxhf*`Zt?SkmXOm*Ivy6fuaO}Kk?~ZSer&&DS?vLwDQ6|GZTB<5|yYwK@7HsyS&Y7tfk59sv2kN+I`rU@G$*?`u$UAp1G^;~dOW??UeBs#+?bHum&sm9|mn(^v^*QX7eq{$iZ_^({;{i z6X&zTJ6(LGuQ`Wd3Ub)b;EmF?C@)Ef?{F+=V9Ro&Xge|r1>GvWh&roBC=xcGU&)oY z7RHCMIs^i%E63@jn(v6mIx{=zUPDT&`z{bui4_WHtbK{sl=^9tw<1dJv+vcFe};}< z`*sxHeD(^wb>pgC{db4ciabpGWNED=y^=p7VBq({w=nifLvFBWJOs8w(Y;ktsJEes zQ_PUU^(tJO5)h|AYUN&@U96Zk$6K6H)5-XpCMi82#>2=Dh=-Put2Hu9;r50xwP0+L z$mmm}{%tT^f9+kPD3%UD#!RIy)Au8XE)F)27LmEc%Zi7dL}JN&LzLFiK6r=N;39Ju z=xhM*O%o0y+|{A6vOLVAgTImmH3%3h(Xpp6K#=o8+|E~0F&Hj6FGTI-6v-(tuW8zA zW@;dpr)>^`0uu<>J7<}rx$BD|J;d7twrTaX6=Zu>D$_1sgxz}5)c6NAB8W>a;u3V1 zM>r{fOno&iESFgZ5%#2;9%%&F@1>!C_a6qoGh;^^+8&#+rZ{&DHSX#a;xzjY%FqjH(tGrKJnW!60f4KSdhmCUFJ ziR!xOY+qeNj|mAy*E)n+U7rs8Tq{`?yzR*rjysdbiZp!*|F(b4Aujc-9*%k&ZGtrI zqnLD@4%O%*ej5?%CTivoL932{cp{35*V4)d_a-O04#9&U0jnMclN)z_I^#r+Dy9WC zC93h7P{CX0rWNyaP+J#HqJo1DY%?bYGek!|To2|9pje(Z(mrg`X!_u2H~IL31oW8y zTro{DMOwEW4E4U2kokVazp26R)BWoZMA7wgcECFU>?Fd})Qj}Z*T&5LPX-8!I#5T| zD-Rq>H*xrO)d_q!sdrBvMOZFc4{QSXuV|VlNFNb-iv+DF`!aMQ$a4k15%!dLtDNh< zR%Us`rz(3b-dV^(j6nJG2Wf z_)3uzfZ-;Dim{7jO;re`DdTTQ_w^Cg zD;FXHtLqDpPK(en>%PlVADn4Au_&1h;Iy8LkOeD^x`TB)(^8vUWrqd5YF3PuJ`ogD z?+&W-z>QWqDUpdVgG+P76eBSf&arh`rB3&T!Z}c@8e*8w#iqEK~a7_inb9LQPq+} z=@L^a=N;ee;<=E6bXDcdKs7At*^6!9uxu;~_BSI@zB=fPZk;_<3CWlDkGi~Hb)6LF zkI_nHYP5v%M*or4{+NbfFa2O5WHeB8tFV~Z1x(?pVWaW*0>N< z=WuV)kS;QIb(VlTXy@jo`yM%ZI+u*6kuuPQIq2fTiS#Eu{y^g3djIn)Z*WP~4}juk zAB`r^jQ6nf{!Nfq~_vuI@#^So26j}|&m7%6MxAE6%+^2roN-#Ez>QxM4!?J|uW%b*7i-3KUu@?e3|KM9}z82ad02&!(Z(O8OgAU;a zj>+^BSjCLS{?cF#X}{yQXKdm+jYOfmd=T7F8^tuQL7J}Oxb36F|35m~)gyK9o2wlw*{2 z1+JP?&SI#dD(}7eDA(=AhFC-N7J_3!LLJhNp=<$l0t_*yIFZ~v6j~M`uiIz`^Bef#hrJaKhJ!OTG_LylUFKBO2Z}1Ze$v*;J zAVgGc8*bpO-_A`rZkmfO+mT8#(v#%q2vMCP>RwI{1$%KR_=k9SE`>`d`9BEVBRlfH z8x+3#snJ;Y%TYm6FQDbdPz6f1Wuf6@Yep8)6GDk9c`b`75RzxL%=*un6tDIi1U^Y5)vB{Vh>PyT5(X@IH43^~8CR~SB7_$%ls4o> zOgrA3@ToGOOVVjN>X;Kb)b^9^3QPIZjN|M=X+^mrM&%2JqL7=cqazRtHC1w7_`SS^ zI|-D>60dzeJfm$ZXtV|GbBw>ZKBms7Imu@-NxchoBZXJlN5$$eJAd;AezN0oq|6b3 z&YTMW2rI~^&bMWk%4Q-_s$MFanpk1Yz}2fPD#S0nn3BJ7q}s^yLl<5- zNNOJ9dRHXDEOZgzM7B{7jd=`9O|<0;d6>K}iF$e>E~>Hce&3~{qrix4tl)mU7V*Aa z%f?yHmUILD6FI^zM=$f-yzWJk36=xJSr6&}m-nR%pM#2&d}^+c@+b+MQ^gfq%-qrK z5MYb|><~wVYr`^HHxGj_P0`$CQc*4bh0d!acz#j}jdq_r6dSo1o;X2JTlGj@L3b{4U(btbrj zeiOcr<%a-uo{RuY6!OXLsxFvA!3cj7fi7&WPZg$+qP+Z+QUiguaw#A_!AcqqRV2r zfX?yZ7{!75`JEZTwU7B%$)7rA=gU!Ej)rQ$NCiNy=kK*N5 zJ~nLR3_<9MJk%+wgNIaeD%PjcUV_sR*a1-_ z!>I=Zw~_2%#{Cp-?QZ7gd@3ph#}wl50tQmOqmI|zd`t{`QwQmdMk~gv_Sb8Ks)v0$ z^?zqIak*la1bbn$s13t92N>k3n5`djVpx!{8k#SBHUA}*Ycvj~l2C#oyJWQbB{WW# z8$(7L-rVIeTQj~n;zYqP`#)0M&-oVYNDC;{=K}sqs{hS=A6F~rrfXJDtOI_dR{ZZ? zYPrni#+IFd$AJOE@NC?0S#c*_M&0DT{iph2=fq4+kH1ho3+5#FKTzFP6l3)dRENJy z`jh!yhE6B^{nZo=6!%o|eAfi;zcb$<{$#$d1e~w+{{Xby?Am4El*beSGvCAiWWJ02 zADQnw7Gov|`eBi3IH zx<$8O6@^r3gH+~(RC;m!`Vd^@If8F~2|BmoUv%v{q9gTAHX#4toAWx4HLPX(PhZjt z2Xg1s9jUVMr~6TmbuLNLoU*@Z2_(<6T&-djsBpk7gU1$;YboEw_Uro4v2C{E75@ez zFnl7~koHxa4XT~Q>6MTJUtiPXU%5wLQ>UNR15($Lz1bJ`-feYE#$CSwvRfh2He^aQ z2LBa(L=u^IxPRfn9NB1gq71S`&phwGjx#8Rp4!Tub;12N^Bw$e=KJ#hWWEzvCcXLc z`HVJ>j>=Vm@034=l3Kcpp^U<`a8Q*j)R#Vm>A@-*o3AkGy{0CqH;Qzb`R`V~$v3Ol z(jYMeJ6xbETR|a9aJ&g52rNAU{s}s=AYEyj_yLT3zYWmFE&EU8`?(dZaj`@5 zVsuwXhu0r4D%bZ?3;rLv?lHKMzH1kFoQZ9FV%xTD+s?!jPcpG>+t_g?wr$(i$=vtz zya!dMYFB^R-Tke#{_Fa!sN#z#YqkJ3ebMKra^x8gx-0kKIO;5;p;ddpy#C7MqnE{G z1RGJmMh!SP*#a+bnB^Z|BuxsXEQP)bD5?kusQvKS607naNuog^WGG3A!Ywnn*mhG( z398FKl4Ib)G{9ucW)_)3d!|L?GE@{ZR)i%b3M1YhuZTtyz`McUH35zDOojZ-G;L?$lic zLdG_v`+!I`mg?>9`aaui{^@gV3wZC!8D=8AL<<8VHfR$2E54H{pMQz(*>qJzUQG%^VX!6c z7BHUu1unSaTtwzZkpy`%R;5@bIyewq+4bo>q!;(;P;Z;oLbEEGePvSU5^T&NdX3I@ z8oJ?=bp}Y_%rW9qOx)jHso}WLgDey^=fo8I(0PbBm0`r7t>VarkH3mgrF8;r{t4JC zRfF~g#H|=*YB`$Y51#BTwMh-zeomFrMe>`S&(ZvE$MRn77D!>)bHGu^$i+^gmXs}BF&_Jcy;{Rc(isG@s-Y`qfN@Q(?*ar|FvWYJpWg+Gb0ST6}$I;!1u5A`7iLDG|^vS5Vqyt z{`qZOz`y%Dt6LSp#h);ZxQrp_99k1kZAc0V0Ql`C&rPH{YE@bN8+GN#>__p@(1 z50=HcQe+M7hZkC4yeyA?A zt@`6FeaG%e{294Z>O=cqeEsg8@QZ85iS_>nU-yvc`Z^5X6dc|LRi23OYV%8fzxlf< zcG^)*_-B`+QyzFM-G2_ht{LX9QN&%5z+d`q1^Mz{%ssl5^1o+4TgmE<&_PlH~I3YLrUerT1u2PGSF_t0G<;{8$P*w~R5kU8p?tB9 zL){PCqe-^vSR@rwKW};`JGFl2-vxTh(^{?lUdKvRz=|W&2)U-e$HFxH1JrX!w$I0r zY~0-3sl&gb|LUgU)2ic|o^qR!%Hk5k%6^RVkcnQZ4|@{fg3;v;=B-+$}y6Q1c|F5q4pKIZu2y|SFq{Wss zt0cr7$qaScG1e1wl+1x@(2HokKA**e_%7H`9Opook&qD&!gWZzr+i0D-(qkM)*A4c zWPL`S8~TKiu<*{Yan3QZXiFGnepL^D;(qQv@<+>mIC+1uU)O#gPp`XvGF$u~=X>-N z{I3MQeolY?(dX~%O4WNClK1@(On9dKxVybZINKQ%XLEA8Nv_tz;}}~~<@19vubkik zu1%cvx=NqXN$Mr^g4PmE-1)uC?&V+0`wMSYQizk1PiJa7;9@&~Yu@rSylrYAxvC`m z%X^kC=in~bDXNV?IUFbU>5AW-mr8F(Evm%=%UeenlAXW(sP@pBp)tw7 z3ko@J;=EGc`|6-i+6o9wNY01$2lhL{@8ueLdik4ze{bWJtdB8jpT>$E5#|vBG#)I?->TT!a8@o>L0hvXgHGdiD_insNCO@O zsQI$%3krWlBtUSli1XBJ=Y0iBAFR8TX5z4d*PVr>3_Z*cpcHKs%~1zQSj;qE;CuSN;JY5JU%B0Tnrci8lwJwdg{%@^_9}kQ4Z704ix*1+ zSzxVC5>ONYQ@GIa&&9N2n%omt2`9g#)Iu7JQP0KQq$CT_ZcKp13V@moKDG%VMgp@l zI&RywBcJwlE&Pnw6XWpACI6Qh7Y-jtkgKoP_1O) zKY)r;XmtJ;eD9S>hyDWJwI^oRe^FFF=4dKW*)1F@VY=Br+=uJ218m##eI-QDB}AGP zdyOea|AOyI{vH+-%k+m2!+-t--`ODddYr?s-pq&9S&VzZHd8>>sNQCjMMhxfn;Y8( z27+a*&(G%Q(cZ$>(|x|Csm7;J{j+)plQ{+3Fyu2hK7o%3iHk} zp{YQZGbJKo(dSx(Bxh9_7C_}i!()TGnUI>0N)kV|aPRTN1HJowPd;G%-Nb9rBx_pq zJ&CbC_-C|u4PQ%6cZS9a#wH(tHHjVd*SYO`Glj#?QgYSo^JnA_2ate=|0j!ZBaa=@ zWRTNIu8O->AT;q z*5~Z$hDl|6$Q|=&OtY5fyLJ%Qeq(pT89C(=Ac)YTmgA2`+L2l6LzH?}q!Sskmy1~` zoeh_#xexxH?D^}xPYV9^-YfMl7QHC8{OJjf61LyUjP=*yIU;CBwDe4)(%b?Z%=YZ` z$d2EI#5keM&HZo&f)NcaefQ9Yd{BQ6nElJWYg&Ozv=l*_K=M5HePeI^EG0HSw7s-PjN--9odY+SKhNu^+0<9EECMa~@$PqyfgA z6?3q?K*Cm34tbR;5Q^m^7DE0iukf4GiSDzy;>Id^9B&qQ96fz_7pHRlxxHP-&&rwZpHb$bef2`u?GFibeG*RhT_LqU%- z=y&Je_)RIzXoa~xoFA=ptQo?5Tnk;9@> zKHPZWLz{o~o)(_UN^H2T;<+8DM~^xq%mS~&`1K)qtg_XD3^~+JzyDrYB1Acs1ix{=wcq zpUb%h|C2`OANn_qPyu)OH;qvG0{dO= zC@{7DZyG^J_HP=25@_;o8o{*z7PuS!EHWGkSc;znE1(gqcX|-xllNf2SK z=EhD!mu@bi<#*6`l~i=3NB}5?yL|aGOU?bh4*kCyn;bHQ1_6z6D$CmngJ2Xmz$1F3 z5__#7Ut_8QRe{=z?&f-=>qyeKhXYe+K~m)r-8emt_7OJ*OR9re5WIG!zm<|bu^E|B z&C(7)=_fYvrgre^H5L*U+y;Pw8!hunOZ6#<%y`@(=rEV#MrQ$z7HB_|SN4I4kUCUFRWj{wTq0K|EFlD!AQL~+Bd@URlmxH0 zmJy~tE9~HF8m-}(xCUj=Y00#Qfnon%WirjEn27dztS6WJTje`R{s@|NPG}ffD+eVsPXkt70{^}G3Ym$VLC1NC?(K-&(Y3n zHD>D6vvdQ3z*(~VRLv@CGy<4uOC{*XVUwxsM_t~v}Sw4GZ5T89C`BeA+Pi`X7uSNtO*9(^2B2NRjNnEAR&L$qT z&T*-AD}0gWBi05tfpj~Ln-^PJDPoAtur!|v*&PFVVj_x8PRz4haX+8f&Jd}_0Fv!^ zj<8yKib6%A2jvbY{>$-MIsLz56QwmmVc-4}n{ed&pV&m-<$q%ngIr!;u?Z_}(#{?9 ze`6DFv4>*ED*uU1fO%sE>EKpet2X#IqW=NDp*9*n7~&FMr-pN_Vx5}|^xyASoz(;? zZ*?Va3*c^ggY0+)>8(p=7Aql4Hdi<2*Y*1%eCC%2BP18oS2}o*iMTZD`nCzHavx(_ zhBP4sMT0ra%4F#a&_DuX+m&xa4PLmDrtev$3nU(+&^pWLS_gntz-gBtsAK5CcSr1> zYCJG|2EQncxdE?#E0o~MxH(Lr9N+|dndOwuW5N7T6hE5_qV)eT3t&n9$Ib`>&EqLo zh{u_wA561Cm1W*Q1+`vAy|t%MRh#<{oWa|Byng|Z47W~QbGx(L$dIp16tgJfU9}HT zWeJ#?8-I4Wj4jo}O&3og7p||iHaE?{dlZE<&j^If%o7hhzY(&*D4r-W!wc?lJ6?4A zfpyYIPd`-X@qgwgVx<0eexj3!SaT_s%NkMMb1z*qeZ#Likxm*FUZB-gowA_GI#$h@u{2VJ$=EpU68UfA4f-k8 z2RHZs%}z)|wa{xAy-G2OiE3^=RV39XnTA#X{adcY69h#kdfCCUBUg%(@6@;|@!{xQ zd9L3oh%qMmMN^?e19x(>~+AY^*-=h4SRu*s}98j4_2qE2{-w|B)HNbQ0@{O=1 z^1xf4?qmTdE5Tembc#p^x00kbpRa(3r9NBdILY%k9m9@zGPHmn2=MG%S(%LFVKkT^ zvnz@Iq-hbN1K(If$VY$SC0==HnJkxPU^_^7*H1#1JJ zCkYf77^90B!>i7>+EeYXw0?I1jM1J*^KbQBQ3HwhonmkxKbGg5){M9$9->n~=Ewzh zMdyo=dsOBL!w)1S_=A!xjbzy;F_d3V;|KM|j|K^tzWf_Yz%srhCbZhb1lvC~21+bq z-kbXhCYV9Jf1bVjIODJ<(MNq=s{fp^Ni4vMtBW6#S#bAdF4S-mOV?Ov%qVzB9$NcabcHS1#6QW{*_QRv;*o70Y7p3oY7>fy3-qGxPJ?9g(MKF|9jf9&y0nWRBArlrL;1vxm1vzJykvN5+| zvm)3V5o-G$1pD5yIs0q@HR#(Zsd+O6?F=S{6F&P&1Rq+uBwa5Yy8=3o_Qfguiyepw z(d86-t#6~20A^yv2?$$AJFJ3Pqggo+Q)y=fI!3^^%yhfPLf(a%=Y2`_g-Z~DTtR^b zlw&g5aLRGJjS>5Ooi6l4Dam=CfdH-L){=g>1`C@hiv6|1_ve+91!Ym%Uj9UTtjN3# zKvWvmGWW()D43yJWv%E)MZMj6XLvU6`;N59U+aBexdd;X#AJx@xZ*@ieCsI2@Xig$77ZOTzXr15go*KIA@Wc}BMe-yPS(w!Im^HY$kpVs1F6??{s2_)yK5gW?8Gk&Xq_tFm9R;TW6MZ1u zzvMWJORtOWo@nSEsVGSr>dVIkvJ9xK&?!y9HDo&>%61vWV3`$H8T8vPeo_1nDIi~3 zD6v9RDVU05Qn+Ssexy%Vs>eHn)%30zu7F_?)94_8C!NPDt$l%=rabYW`7F{}X4R=h zf2^ut!COl}G%R)x7jRp}uMu|KhaxpUKr5rMjI`0>2A_@h^rWnOy49{W*EWi8g?Dq* zaS?E-?oH6Xop{T^4)?3oz6FYYoziZtv12zamZYX%eTW{)#?}Af#HU+n{-m%ydR_kz zR(9ql)47j@V)w~~1PeoDPju!SL0+FGUV)d+tYCahr=bjduruc?*&roGln#fT$^9|k zqkYcLj!X@m!glHg`|#U(P=VImL5m7vRXVNS*J}}XqF|UsTUW4eM}2`|?~P>9aY0U7 zxNaUoTroZ`1~vUIcJ*Qx{Ac5XO}aWZdK$yyAc5`Njw)a5U+kUJpSn&;_%HUp1((X) zgX+3rl`Vca^%r|j?B*AeNc!~EQS7cJqXA<1O;W06SAGkypTP?87(sI&Bt1Z9$ytBP zw)T~V32B-Bu)0*$IQ4u{fcXG-H4L=Z<>&TFV`^)q1*BxHHm3$5i{I(R+G391 zQ4V@OfjFTWD&##d->Fkhuhnb^2g-?^?gM(Xk+$Me0*a^)*z1veYncP~=Nl~3*u-`i z--Xo0#CIFxeLxo1?K^_O^tyl1YYQtrb{`BHTYgxVnz5TNOwU(6?hTHgT4G08!7!*z znn0(iCYc+Fz$zO2&DEz6rx@!jU7ErY)wfc9W`n^WYsJn5ORp!^9Vm!=nKsN|)qLXO zdYWT)_*+JkTpD7zED_a0`Dx@q!O`E)#Jj*YdB!Q5i)>I|3(ue*ZBKl;ey=Qb;R1)8 zGYExH330_`sh)b%4R2LNn?;<-tQi-&Q@&)un{FU+Y|yPOeslTL-Up3%`8VEc8d~>a zbH3kZMi{#Kv|x?}ev0q%oA9>i)CYaL;|5P~!(-xXwqsx@zVpH&&VaZ%LzHt#?S0b+ zW~E?~$k2}MADUuGq2m%eN_I)@mM$|ptUsrnF;p!bMQ>Ms2E@jYz+d-cYG64G^>-Yyo73y4H zU|RNrel3nY%baH|GBVehZbrY~@y?0pTk;pFaNJT|X-8dIrmQNpkL~-2BU5iri`O1y z_5QI@o8x**c$#)xomf#W2=Svfr?)u#(Yv>BLIw#S3saV2Jz?HRraN76coFpR0~D6I zoA0!(3|fuJ&;q#cICoH*vpt0U^Xb(O)+>UK$0eKlzBYE&ekoLZ!a}sTeM?Od$4Q5P z*_Wj72i8q_NJg>M~v%r>nOk_e~A*!3cs+6|>eK6$+bHr&K`&lPNfPV>oTtHs1U& z`L}TC8i_|u-8S^EBq^=VAr~V>rmt3`l1z%xGW3yPBCnNsj^NLm1D1G9`AN%~7@cij z$z-))%jKnd>=*(8?mtC8V-!)qPaATRLn=}%qF(>}SQtoxF}8?V0g5>nA19Ds-rL(N8ArKWY zR@ptHn%H*VY>_@nCE_JFm>LaOHai6IgUOOY2$F)Z)D*w4mpMeCQH~A%_IB%!P`2@?){7txD z60-g018GlrHgr?sz?mNR31PVEm-g&}d9hfhh6S@HT(ZCfTBAl2ecLUEDl?m^5){YT zWg5^fD0iKgvkHpa_-3`&1=c{`0ld6w*a_A_~XU#KcZzi$-w6Y`UK znerpv7&2W-zZ1J*jHt3_HHPL#boA_<{4^)Fl1UDdG%ZXBzb*#UZ*-N=tv)tD3ANI0 zqjN)(@0EBbR89qMKz_3vu_bD;0mJ-{bn*wZ{*tZ5Kg-#OHkJELD>FZ!-|Gs2nxf~I ze!>MlV$ZHMC)WFx`RAo_Tcp$!7XOUwSQ?XWUG`*hJ58?SvHQmSO^hH5=PONsN3m;omAHO>CRi$9(T#oF?TxDG>QS5r_kzgBThSxYPMXzfoI!z5(` z%6PO=CVRftbEM1{3%(=2uq3PEJ+WqC6-=eNY>tjKwykXkiz&~!ac0)_lQwy9t&yv_2oWElq_L&!!fth^kRl!`%301JvkcD|@RB`R)TB z)Nw_1&9yenOgg?#_WwFm}>hQ2I{h0a&tilOA9{J`9xLGUH9X(8>eFh_JD@ib` zr_&$sf{`50lV$^L*@mB0Ls+@Z2^cX@aO911nI@p#Ei%x$$u)7d!c?<`@$ZZuI8j$t zqK!`sQP&U9v&%|TapoT`a@QO{ZIyxUxR<7NqqL(dOO@$oDVO3w_yQsg61AC|6C2ZG zsM1##?tCo7zlCk_$mh{l&6%b5<**xjCQ9rzHG$<)l-*fDG}5%75RRK~A}fn+_i07v zR&+3hZgo(wSH608t*mIDCRwSIf(~U;$bpLiiSRN5>x1XduMr1)xk_Ui7uv!c{0m+a zuJhK1yvw0!=?wkyubW2A59;)S06Dm3L{~WG3jUKTlU@t0%hF zIfXRE8FViktv4&=+OflQTEAXPoXcXOXtNOc7Qwg=Ufz~U7$>wVBq!K-0?@#4TBVP4 zkVDA^BuLl1{je>npY$wp>rDxYpgyNdh8K=45LbX2W(lRW<}rave-gn+fdRD9tiHRS zu^U+$Q(4E7*tbfD(zF!-d~bJFQ6(hfal}onDn$gB=Pg9kXTU`jT3g*^dY>0 zlM^W;ziJMV!?TJYK^4Tf;ST;lB)QRQ7s1^@5(B2#6GEhw4o!TUI6?w8fRJ+GogXvc z8POV`!x<`6P?bqs6Zu*ftZJHcjlB154mbux-M4F|mhYmR z72vweqkOj()Td9S)>PLMU~N27!{u9Ei7&I7R>ELNo7NSG+1^q%i3CZ~<`zGl77Kz` zjR^Y&P5xbCK4FW|Nuh>wJPC4r-loLb6&_Ph4ScetNA(B8Dea>F(9#yJusRj?(h6@? z^x9JtNlJ9FTNdB-ApIxD16J6T1|a8Eu6E}mfDf;%R*Q5IO#u_^G)+QEL+>+e?V$8+ zTTCvYX;f#-9bX4Z)OdX7!mex_Jm-HR`M8WKt} z9YV(&dP;|nO^kC83)wur4$a#7lT6V#k`dA9W8Ff3?WEJpO}%o+by;`LKm%IEG`Tu7 zMa_US;RYQVweoip*Zo(`H|vb=X+f7wL0H1-H%yzZVb6Yh5mo4srSm`+6+-sto@K0` zw_lI5RiTqR*xivJtVh3Ga7Cs#U{~G1y#h7#EMd~Um>5QLFJ~wc#%(mb26gleHG5$I z4c4TN1~A^u5NKoF-+A0LH$e_I!B;zKk~oY%%c}yl?mR!- z%v9>Kp2w9Qf2WUF1+nPhn66jn#^zmu9ziEi$x{eomg`$u9@^`J`TRH3_syXcPU;gO3_JovKXSi2h( zEFa$yRf)HeZZiG3i8btkn{ArXex<5d&|m)C+ELu9(W0*VLkHbIu6l7PoRS&Bv*otT z`H~gGOI(YX@uhuWW+AvxZlN6< zI;f0$X&Wz{s2uhlxa`l4f6lT28u-ecK?Wnb_CFWx;e9CA)-~262X`IE_lYpbF?DLYcJc2-s_NfVmn+=)krPQ9%rpsOCUQ@O} zddQl+dy%}$bhQl8;OO2^of=;(T+@;KCgyHUJ$
R*G+s2GaHY0kX>c-19TwxU3Y z_Gl2QQLHJ!xM1^77Le#QV@?9jKrctHFbPLrCu>Eoo~&#wKI<2nuPac6Woy=D!gSz_ zJ~0l2mM%a0+-mgrdXw#EJ<3;z7lLPSaA8>9%DBHQY-8Cx{3KX_@u@erSDJz&v+mcc4~s!bH990kj>j|J0)Cb~@N>NsnzfHmv|ZG@@?DH-{xkMH7-Xn7Dk4n>n4 z(9+IBnSw-RqESUbP>lW@qwW(Qwe`tZ<^J!^XduFURt-W^NK0%*s5Z$(Bv5%TI4fx$%gp;x=kE8ru$mC5 zC3o7^!RGROB;zjKs3ce9jJ-D7v?J9EO+pUl6etYS)R?244iqQ6S-@t+QdSo;TlqpY zeYldGsW|wz+T2OAUVXvd)(`hW1~$UOb&{-4>-KsrP;tG5HENrw26DV6-%lOOjdTR4 zA1{?#Re#lDeqHVOa(DJbokOmKbg&#Ce9jeKKK-F7Ybd;&IXNcP26H&S82PZ-`rw~_ zU5HL@jc8+_SUrH`QDh(J%_%}JuIOp_R01$^PQAmuViEI>jA}2WApn{L!eOmv z^AZ0XfpzgG3UDWwtgIVph$t)8V#u)q_ub2fge_d&WZ7W!=tpioX1K7Xbn#iuK)qQJznrFA zyh2^P+Bk$Ao;Ec?Z|Ud&BV%}YbmRO1KR5&T#OlZTfp6h18zJaYA6G^e#Mm8ioM$b5 zes^bFLub4c%{smrm_o3{vZCSb2p;v)AslEobuxX)Etn2I3oB&$;C48D#4mR!wh`fh{e4s3Bc7=u8{+GBsrWXDNj}nvFB4wzA z`VmRvy@Wu#?LZZT{1f1rOnb|V<_7rq4x&UpmxNb?B6GTTL!wKzJt@mePVua^1Ah1E z+u!I|Li~>~yl@igy2xNojWX`08!IVe7FUZ7D?6^#otZ8+LJk7N}6*2O=V~Qx|5Csf93pY&lSX zPQ(-wafl)yHhN=`?PIBt_WpV@V1Zcsjf=zMaOetEXaJ3jJ;@opATK;gZ9i-_uY;L? zN81uU{7I{-QM?Owirx-tihpk(6ynl@$=NII+za>DtdSomoWs*0S#p)9 z<(7~yg)bYbt#Fx^P9)Ze3OV8etTo>@DAGYI+d0~OLS+;tsYC88W&(c<$8x%Ky z`fj!cxe`w@Nj@k&T*2%`l7sM_rLU2 zK9cJ$yCz@PhTE`xH_FU?5z7uL;cJ-tW>a?Gwpzv5-g&=H>T71gU9)TsGB88UJ1-kMUzMH{3C8saoUu|%HXSDLRmX%=qseGAHyY6 zl@l7U=v%vzhVvGgoKjAF4I~8Sm%c(cs^%ecVylbBNk{!_=n*P56OD>hwyBv-nq{{I zA<=6!WY4echo8^-o1MOnFXw(8Tt3&25u>-+U-LI6Pn!>`eP52jn+{#^2CGt^k15>M z@8tPA#;Nq)?`9JpF#=o~v{8Ytf@L*?7@;lnJ;ttFV9L^8`oStBzcc?bG+wn9!tt88 zOh}|9YnX1JPK_3-C`wnVNs^oDi78mA`nA7z^7RNP-46Wn`X?5){=gG00+8rB$PLpnT~<0770yzy}7t4km` z+_vxdpC%;46>`I_$Sh7{EF@@zrnppwc&go5BT4jzZ|gOduPo%+r}sr2+rg-}O>$wn zm%zFyii+wjROKHurg-Pz7TTZL{P*cN>-sLE1*uCeSR^nZ6xqEzY&*yJCl4hEC4=$y z@WA4qY&1w`CdS`$uC*3|FM(9H<1{L}UAdjEX621=N5rnn4EE^RZ8&?Q>PwgrIWML%{v?-zCR(sraCf~`HH7>L)a>RJHshJPg1!l5`J8EhKl5LG713eOS5 zxwd_;fUAo(DV}V!v80n{dghoc?0JpLetkz(TsW3L9rg`6_a()eJ8wtoYy1*6hga{3c)E4h!Fl()x-qNE@PrB&y=nlZzAmP6 zdSZ5ylr%rUXs-_RI%k)tm12Y=61F0^R{#hhQUaZqtY7nA7XhwI3>UU)DwcWqwfz8aIi~xM5g!FRg;-5O<8bs^p}SM>uIP;O1QqbXFxpQ$4-he(4jM;r%HVB-nSx)bn-zyyXf^93Iw)YE-uPS1SG#pxWa=P zg+%?_g((IR@?QmnK`n0HWhm3og0}SaAhe(CLF&JZoVTckaNNd}@qI@PNvJi4o^RTe zejm^mOo;*GY!t5UoWGL0?#k-+^0@93JDCMXkk@6=*Jy%z$4<#e9&v~F*;<$R%S*_6 zA3|K}!{QMd&|Ot+OcxqUxQsX#X|wkE4r@23H-lVsOxpH89q%tU{oHq%J@e5fK)pVf zrSuXS*8~fMa*Ixy!BTSe9Tf@sHn4v_^Cv*-)mp8_Cc139`CQV+6#VH|tHyFWN7J|C z+epga#5nk|7W{Z~F+*CI8BBX%hD!}}#WK&_NmCA4<%thJKM>bt)6a;0;%2?SWvZsR z=#Za> z7|g-X6%DQ>u`t-X1Ag5O+V|HzV9~U`STS@g@^H9OIg!0#Xg`#(!#)&*E|IWij2z81 zBt3GFz-3n3JvAM>zXgGnygFyK*~QBFb8jg_*7xWIP4_2EN6Of*F$2Op?pWG#E76OS z$b0IQhrk@jNS;NZOV=$l(fk6JO_LBKKw`g%QmVn#q^$9{h zNE@izY?pg#rjshCjK111uwvzD-I4+_J7x|^tKc!nva@p^4);lZe@6SqF5>ajLKcRj$ zev+Z>OUVzF$~#$g9q!Cm8X_ktR02v%3CHF=Mwipr_=(-8K9{UGdK!N?^d&*&2kqd` z7>+<)f`IJpBaMnmCT((PZ?xgb0yT=H7GUX^)?S^m0GRf}UdSV-$^K|~@lLBA)v z`CmztYi}d0TsY0aDjx{*C&fmzeHT);3-au?fMa08xXRR4Gj9`N`6Vpqb;_hlnaU)L zSKcH((}R3$cN5$a_hYgD8CU_yY43CT&MS)QUP^)MwR>OmlneBjlg!86r|;rnFw3w_ zjRvUx;vk$zh{OSIiBXLo<14dJxkZ_ImkusYHq!a$Gy0bE%4ZeceoW9`^(!({G*{`&LNKQc#NS zERH;O(dPbj8)`k9Km*l6%X^|2+<9W7<*43CGp9bWSRV`ZNuZ};z`I*rC!-ZpZq9|C zcF|LOw9J6y_oX-AM>6W~7M?vtLf<d(s~vCpu^XR^A|9b7OdXn0e}H-flO`!(VqU zr|qv>Z<(%N?ES14dNX|&A^>J`?egz$m_9Z?hT3WV+15`qimB<(6jz9OeiiJgi@N(G zas!zT)77I)5^U2${U=IsKC4e_E0I!*Vm4k+6K*_r7;W+$1SmyuYN@3mhUeY$(=uah z-t9HV6DD0{=Wh1fk*3M;^+}UYli?~CH-xqZe`qlMiO&+N>fyIMM0wJ1x#l^_-bH!EE=K_yt+JZyKsZoCOI>nbIDOOQKta_2uzluSQL|69tk?JqM`| zTEOAruLG?lV5($f@ePSK=Nge0alNxa)+;;_EKfZA6;ct121gc4vE)YKLXXQ9rk`u4 zert_ckIT2Axe42s7C5=B4mg*~1kq_JYSl!Adx#|7reDWAuff z#>0|)XOmhl19e&N7OSurPVUU{mMt?Zn5I>EyjGp`*1{y3J-k-u{xX3G_}r3?HI!HJ zxLuSW<(O-^Tlkv=M^I^g6!n(WsZl?f54vEL$?0!((({ovewW4^A6aOw8@-cu4M=#_ zwmSa&SI>iHuzJEJBjbTN&r?D>@d4~=mLa~g*4sp{bT-7L{(R)TYjMUlj{82pJ6Rg` z)f?(H)7ongDqB${5(sWoCZM@Cf7Ax@Mm5KHw|%}YXFUcsqomgvgrA+9ho`wFvm-&R zN)c(|G_N4#1_y@m73J|<2xQ_CR^nr}D**)Z&(bON) zJFBuhx|a2^c2>=#E|EJ8b4Gy@4OlEe0Q(t?@GiihC33W1kL@jw5Zld z*BO*56-5=G-;vX~F3u9oRKW0lmKkK+RfmOpFjUs#n0XuM1Lp{+;xHA0qosB%<=_mW z>v2-F#d0uCAa{6*yuvGBwCt(f<6^8VsTyhOu8?PL+?I@%VrOjC$EO>NDL?VH*_wNVwh}c!8AL+g-Jx``h?Qxh%Zpj`y>r z3nyxU7fosb?9O8~0!{fD=vbi_UV-U#$wYvV0Roj-HcJEOZlU^uSho(yW4QB`z;_^! z?zNcBWTo28<`CZ}5Oy|3@3TN$ZNE;_Vw?9&R`j}@GVCtA()8<2nd&$- zZST*&4MT2PH}93!Q2Rwrg+3{X3T-oU$8~U$oTkC3&aj#xc8Y<-4`xax6-+N8U(<9m zGF$r3TfknuQVO-9Le2>tKF$H6oInctA22eRWKr5Tlx$N^Mi~AuD80Y(TT85*KFzRfbTI%UW9f-78evz#wzp(XYoEc0*k#0DU zm%8Nooo3{j&_?o!pZFz9pKj`kZmoSiZ-U+A-RhCv)VsC1gWBa{+I0X?3JhDg+cno3 z7SD5G#W0>|HYr2J6IWw%yQkbla21-hm9=_tlU2rLz#(3o0G)=p(X%1X8SF1NcbEk**8VY4@z@4cgxg%-O9%Ei~8Bup5Gie_#%QD^=_|*~m8RxG#Z|yO4 z)L+YLBpBv;h-xyHRd6UFdw7Lm`9&@+S~Gr)$8nOraK_#jyu3{5e7*4XN!!u{~PA}(aFXgZ8_T!%q<%zB9o!ab+AKXf`q6+%iDAv7e z>50!y{bSPVllrZHEZfYuE+*_U*5dn9WT@D6_TE$K@p6y3rbu)D;NZhpxq>P!1Ou=A zXyJOG6WkZyAtU~Ck3h2b+i;7^??RBR7xsF~4!VuLuObo+9)?K#p(CEV92i$# zz{ej!Nc>>HO&Yn}9-`?!4vU9s=W&kFo=8bp#W4p>NT~zsl=;2O$6iSzy90Ss zgru5Gng%iE{c$mRr07KkMhU2hPvNmC=B#ZVfgWSF`1j8$!&x=#iI;HhL? zM9%R>;TAv%P04eJ!v@H{_3ATAkDBa&jf50F{yFb#UEh7fa+VxE&@*r=@^&VF0QBm@ zfC%*LYSRt7i0BEf%e0?K124rdYcWhzvOb-=%;_jg zL$NPYI=;A3;|h-jZ=|p<${pMvJA2A0-?U%HFy4V73%-6ZJRg;ST~1HSmEU+oYp_3V zjS?&3{ucP=Q-R9A-Si}>9uUu6YS)qyecl&2?LR{?<@&bgenoA_+R->k>Tqj#m3SVV zNb-76<|gL_Y@)5Ns)tOKCLNhw8#=R{%tB8hNXjqxBl+N`1jFMD50uUq>JnM&&{VbC z({bQ*C)CS&KTPDWe>7VPSf1-(E>b@((mBbFV63vx*@xSH6I0ITuk5}DmGRgB|LL>R zjLxK30EQZzgRnm=t{x`TF2?wwylocNjkP|+h)hMZQHhO+d8rNa_^nI z!5hA{{#AoBtU=XTdw*YTf>Khw$yOItJ^sAZyFD~YwW9FkNyeqzQ4@twH*07GFkMwC zEu8c%Dg3B>(Sp-C$~QV3;pc4Q>sPFO+%{^Ak{gFTmR;R~?%UdgBW5ZSudi^Gvwuqb z{o&#Vy3t^f|5m98-IVJ6vpL`e>8h1Dv_m-s7P&Wg_v#yquc#4YEeQi zR%yV;aF=N@ajM`McwsRFj7)e))y+0GAb>{E5WCW35j8Pt>4ESkdNOdR2bkFxDo6Kxxh}8 zoYR0r^#Vu2{Mi)myu=bHgzAXk>6gIXh`aTA)@=cm(Es9{1}xQX5W`K{Se&FT+;)cs zEoT|1VX7K2g-m?H0HXRlJChXu9rYI)(TDC9PR=dag$Ge5fe4Ma=32};ig<;9+X)@) zp?z8B&xC0;ARcIAr;4R0;_*BoNPRckqVQCsaw=GD1B%%{GY~SrTUL{qBEN`>!~I#^ zcYIGzM0^{_)hIn}9gj{?g8JP`%5tTzu<$(iNZ?CqG-(SgMlWagnXo?cnh9DyI#b8RBmM_Vc|i3Sp7v_P}6Jp(WRt4OBV zx9bwWvEhqLLsxo=DJ}40fQX@eMrTmkr61vFgBF((EZJKYK9w)Q* zI@X#B4q^9QswDLI2Um^9ccMpaIU{*SMtHX+vVFd1U*XdsUD7KIhIrFC(H8`H#(-~a zk?WPy_X|M=5i_Cd3t0(q_#^}-I=AVoY?P@7!kN&aHUQ^UYba~R13A2=sr0PrdS{+m z$U00wZh{GVIia$1r|B6|@R( z^-*CNn)w$V7#B&J()c=DEn@jk7FG?X*Z6*=7>dE`eo9FmKXlE>vG%4i+dx{E9^7d%Pvd=Uz4DL0jw<88hvuz^ z`F$q5*4p~R)6(_dC3oHe;k@_qN}caNEB_#MMEp;l=wG%1*O8RWDONwi$&O{Ka3gJ+ z@{;;zt9ddqyTpKGQf4neNY+?#(GphwRd1pR)lMc$1XDW_%Egl#?-K zbt(F@+X_wnF{ry&Sd)*7m_sE^VJG=4Go0h;75JI0kYn%(*UMHwsm4;=d;Bhqu}xr6 z_fL>q-Ct{dfi$mjCe-FBZ*x< zC+2)IhRXB$WcN9t-u2(BtBNI!7DEXs#8q0WB&`@nEs;X{GAZaXTgHbQCx`y~y$4d*bpxDycGx`11 z)fJCJiG@C?N;3%hqCq6fY|tFob7Pk3H^h@)?<%*~Y4cFpR@;vGHY~1)V}{6PqiPBq zQz=hoHH@Z(BcaR7%&-}UjLH_oroiU8Gp!==0UL?%ETE}|S={bPnad3>k*ZlnnW*>+ zYXop9+)k#?SjM~%;9>t7U1D%@vQY!bH94B`UwOo@3i7hVqPCrK+>Oh^3P=dHb(s*5 zQ*Ki^-u`>qv(`@rxz6(FoS4NnpdfgoF@#8`zjt}?`9c=SV#%Xvm0Ow*?0$3dT&6^x zr>a_;956tkq)h_SclzNet&u~TOq5X`Ew1a1lEVV351aNE2)0F=0&LZv95@`w)$KSB zAVCPL-U34`%*lFlSW`;_t~NRF;d{tcs6GlqDB1HhMQOr@Rj` zVg6RHLPaT7YnDp)tT*cq)+P_mV85*(XO@1zPHJrL2w_X{Kt=xL0c$m4xSzbKP=s*3 zfI9X{x1vyFi;On0((q6d<0;v8#nHuP7!+GARN9bZPiv{TG)x7St=wue*A=?Jlp+u5 zf3~|OaF>hrpK;5~kDh{@ubYV<1XWa&pAYZ*o&U@3VFB#x_Whs-?|c9IW#IO9ZZ2l0 z^X(0=^veHo%fKJr-r3jV!}%7M^=&vI5FPXVWIHB#r6#PETBl@4>VGm@{UF{mg%?i2 z%9Tjq{tM^xW`VWX(|Q23q3B&{;{)QtBwhs}P1@AUwl~?Qn;t$=-IlMaJn(Iox7hmI!ahv{_}9Qny>5v=vQTF$asvXqmd|43phXV48PEQhKvtWw06q zpzwA@Ko^;Ua!6$NNM^fI_5jtEM}i4*x|Nce)#VlRAy56$Ox zY?T;PgU~!Zjv;*`!veMZfvY_Z2CGu(GVUy1PPYinEk+XHg)Di;5u#XIZ+!gLa6>-F za)?UY2KsAu^=l@C!7F(^ajNzgXb-5(^k^k0>?-Oc&^Qu`K4U2^)lpTKwLFPW3Em&V z7*wr?R#1Vmxo=9$38a5zX4y(hb!%t~hPto6q@={a1t>drFqP#e@#!9hV&V>cs#iLma4>xt?y-u*Z7BLwj zYkrxZh_r3{!H=$W?;qq>{ngSlnnNJHn~HJDv!H3}U2>maS_T z?=qoHgV>>|j;n^P;6}nLT65HK{30@U?mCOXEnwz0Zuj3nFosTGn7a)T{|P(S?j3*0 zzi9L^tmLYusuj{=&<#(u6-b&5o`v&ppS@rG-E?9uBgWw`m+3M%gvbKXoGb;Rfd{!u z_W8F`buB|{TKe-%Fri$V;<7HPI30hbh|lIWh#wRbZSj)#-)D#lh(LlNRA)-KmlTXA zdrh}5@KJHL&=WPO$Hh^d-8xe`Cr%oP(|dY7E?BXqdH9a*HdHZ>fUftq!QYoa=?`6f zvl@0xPRNG{KU~xt0_uAxJ`3*j?d*#j-?|LFozIIGhG?My-S0*VNx`69epM0k0dT*M z7T?c%iQ^Tt3G&vdw;XXDQNbTNiIYeb`P{=T8W%KMy6*dFjL2@12s%T40IbwqoUQ;* zjZ6yCE1};9Utkp;xJ`6W3`&Ymg$H;7DX{ARn@eq1yl+o2#A(;rtmBuYY+i!|*#U`2 ztzr+x{0E;FpKdPXcs=QZ%3bCnb|RgC1uzsCR3DWn+5#C>8{0#$^k~yk#fJE-7>&Qw zkC9cIx!8Y0JO+($5lClsw&_=?VX9J9fr{f9MZll5>-={N!=;8ZLcY}kCfuf%I;2dC z*M|6Bjpn-LoqW&s84@ujJ62L|D8jh^OPv>!w~$^@q&CuAdI>oqz3sN>w#lTS9SDZP z=f1QQ01{@9hPqzL1!|FNRDKAD0_P(-Q~p(qzH}ZZ7;hyolnIWjtuT*Ed7q(IS>i-= zdm?SRoOuKR?_;r~o8yi8O=v5~9DqF0RPkUIu;_LJv+9HkCFf%|uep9M*tU zVsU`t*MD6$?(WOZXNPSBI$M@3V_g@n4K4pDeV(6&Mp9#xosGsOm9Aq-1@{0=3~L_h zwF^nOq3Q}WB+p2HN+(&PD&~~I{byTT76>d|2#0Z!tHJD`bKQqCXPJyg>nTP;y8$+td8Dw!c<_SZ^br_+53$Bw_%P$*6sYE_NGsrueD7O8Pn zvR(QO@-bfXo!pBqj`{d2ult2mUrxnNF*3O{N=sPnnp_R_;V!08Q8YP)&-kZlUU1+= zaCTSuI2qyEE^AYN%+i*j2Q*LXgc|q&lwRRjqmCrQTFkyZOHphjz6R)QR6{W@2_2P$@t?Wqu!?Nq5?Zma z<#P>1w*F9DD5`i>Ab+?Mj$!}P=f6uOXVS468s-}r8_2yn(#OOt3IvN(H}~gU6PPga z1S8iAaGTM&u9}((33RFC8iIzZ)@JgxJcagzL}InHTH&G|0BOV0LR;eriYpK!WdZF? zJ(UPpn3Y!b;L!K-IVc%bxV;1?N7G>Whh$JzL<%z{$;CYMhh)|`yA`yV@;|`hO^|?Q zL^V+}yv8IK`_<{KLj%hDqR&myswr(|%jf{kf;1?`=`*kF)K1&ll++-D$ea53aXEdl zX;a{$HB$la)5-T2D<41H?&VX=3uM}R4VoiG~Q$#jXQlK z_hCF5==XOa1wq;1O;A3JP|@+Gm{|_9$O5ggatHg9JGWL|zd%*z36?_#&s2ENMMv_~ zdQ={ zMl`6UFJlA4u}AHrE!wI zXl$a$tiF|^&1O48k5a9RC1JhQLa3+LZ4vG$LM4-ITYEcpZ3!ei$8s63kMx5bQX5j zf?Ythgb5~SFfUCFSr+{Tqd)pJp(DICfO4v#w(710E?cxpm(uh!G|(jQD2;P6q<-_- z@4KqnDKK9#4oV@R6UVl=*dCFdDPXb*Iw}M0bkKI~zDH6mJpQoK;`E1DVCUKT^o5Zi!rnsAudK8d{~#q?9nV&*L} z4;+@Lgr;L>n}Mh32L9$wP9McT|1g+Nv^p|I?XBo*fHx)Bxrg%32D6yC@X-tRE4fV$ zLh}jF;D`s{=&=ERed>mlNg>{bwFoTT|>jU%O6NQ4J@T6ZFZj=Nk* zM8+-R%raS%ofzFe5UxRP@LoFC;BFSOS>Tl)r=n~V3D#+BOTi9_&wtA(##$qY+p~;I zv}XalZRpP?*v`LJL{gzQe`}y@`NdlIn98YSm!U;mE*qnWFC|{T-;6JhnE8J9ye?0S zMVU0|I@>E%KWY@O=uOM6__y};*(dh-jf<0&o`4rjEds&>Kl|(_mI7(x6ZFzeyow6$ zN6b?5Kb$XyGD=ksL7A{)1D)DwPkQQY?BWv#!m!cpwjbaqa6HeS@u3X}_A1<)gV+_O zOx_uR>KpRsRxqAt@yw(1{0Ff(yI(>=?Nj?VIr4*8=xx#B4mW!K;!Dv^ylu_hjqo=> z@AnRAR9#u&gzzuj-Y&eNS#S>Vy^O#_tsmf`ZOYHX!`W(1RQ9qvECR*>f?F=>leW~{d& z$q4Dl!MUF^r%)&_mB;)2l^R+vIuG*SC8Xndg3mV#$}qjQG~0J=pML{FVbHia7PRpx zJ$!ui$92*L`)jU+EtrpIpkqq|K9s&*oaJm|=Bk}4>KN!#Waku(KX7T$zl z&)s^P>sjth@Yh8gWt)N9ePdp*z`sa?1`wiwO`D|OC(NoTc3b1OSVYwMpiEY-*it$w zJJ#1rV0xQ&EliyIp5i`Uw{5)1!0wXF*wLS10%qIx0Uk@P^6VmtI$d}&N z&seqQl;KnP0s_vjYvdkzg794w$~Vklm7a|lE~MVpX(g{}E2^KNa>Evt_4UEP6CiOFa*T=5M{`7J5g4cL^nwHJIj?cigd<(XKy-O@bJk!VE ztIEBU-ohx-*k6kUz-#NNd8W0dPjEL4`X+CB zg7o9D#Dl_;8-`afDh*a=!wKHbETJE!qfcgFsdDWX*)J>-~QN8r~w_i@^cTv%H^-GVxly(*->l zVID!)SHU$GP+L()=6oYJAmzkLcZ{vtTKQe zCxjc-XFAM1#F?ATS%4)oyp`6PK-2+%qvg}V=Q=B4guoM;=<}HbeG(3niv7_q@# zS<5U( zm9wvR@-!ThG{jrDx7zV0f#FAoH)GjaEC>@uqrMsB0VZh% zq})>q}ULNrLx6?B|mF;TrHY6pZvfPWFY%=JFBUq6gHC z$WyOJb)3(P_4V zP`(%lJ}*BHNpac7t`?sIG2%TqPv~uec%MVeCA*NIVL)OiM~IncSUoEUu`*Njt#p61 zu{If(S*rlH{c))wTF1Mm@Ugy^(w6h@#Vtuo5Pyj&4IcmFBT=ghcRWshs}Bs$9*Z2} zopRmL&U|zfW;t+^T>s+kt};2{$TWc9RQNb)2P8>hac-LNfBCp@hUHB$FAsjqw05th zaNAh#)60Im4RPAn_)tpWvL2Z{aVVv>nu;WTZSfdN*K18bPT3NQbe-4VMZL(k_r)<7 zOJ8XXZ2iSp)70UmI1yYKk7^qOHLWVm;+)JCYV`4uy}k4#o6>P8$RGE6w%#GdCVw(0 ze)L}jkJ6dsF_K^xNV#A%OPGRA7bM{FqT~oG~8m;YT_c`KNa_{J&sM}s-6ScE9gyG1>vNf5Rk}* z=7%zuuBjz;l7+Tfe;>haM}w6!Z82|`gii_|X4va(?RA>zvn-B`F%OR^3XlF<_#T=O5js;M7P_z+ zx6|P1c1Pd~+~2Ho^Y=V0s)Nj(b{V;Z&oTLL$}Y<4E|Tf3hvSV;UeNl2Y}sA6bWLCK ziTDvnFbpLI!Jfq8Q^qJfifi%o8!LGG6OhKzCVh+IXL7;MK$`7_oBDuH)F^Jyzfn4J zLzfgZvEbl2Oaq2BClGLvoG=S5Mt1AV;%X;j8TShfswWA*cLEmmiEBtPOs>mn^bLG3 zK48qC`jCG9cULw1u8$AOWPqFtB9Ym5RWsl%K9D$mzrRc!`pV)m*^@0?d)iH8jMy2R z<`+Y81aQoLLS052l}fAv4+G68Fey6(M!QospZrYz=$e@B8rlz2vqor3YS4^V6RV36 zfIQthA6XTxVxFH?<*^-r>v^z1+eu!ENFS3Z@*M_|pieCQQa0=l0wRCPqz!Mi0P^Fn zThD?Co#o`oRjSt2Dte(~SlyKA+BK-oNT#I!M^Qa4h74FIcSc4Vr4bu}s$Cu>rPu8t z^WR(as6pcFXcaS%QC7Z^#(fieA%lk%)F$PcplRqS6!zSkG23yv8%D{}!~RVmNK|YZT;e#i3a@c7>LO zg<_fCVU|SuR~#pU8|(0Xx90$zw$;emkrq$p0~SHzNEwmqF2j-o!*_s%7L2;c7)M7j z_z1k%CtsZ^&x5K8T_&ONGL%cvB>`_&YGV9JVr62mK?KU08dZ;|> zY4!g{eleiJ^p#g7eako(bFP2mggWC0W77m!VdOD=R8||7|OM=!Irpj(u9N)<8UoGfJI4_Q6vDH7ZP`JSnnO0LCEfOlm;A~@C ztX0?6YFI?R`Ap2R%QEj~M+nZc8Vazt|J;AuUy;+l>uyEidC(~Utof^yG^uCW!&Pm* zQyE$oxR0*6;g*935C$veKs}3I5B~^C3l*mV=GF*{H?X81+r8)?)mnxJkpeEOQLHy# zZ}R*_Ti&6)e`3(&5$cJdS1-Czjp$qV7R8(ihH$gSgwXY?qG-32dv0?46iy0r_tr_J z!-)!gdR=sX4J<~QSb-PWg6YUS`3J_`*>)-48Z5nfCo| z=s^_|e3|)t;rmvvBo!K17Pejet}_28>eMzfxlK<|(`X zycmvjecizU{5sCIjqTTK2m!GT%r?)*giRIluf7OtTWPtWH+B7(AFtxSXJ9HHbOytD^JD|E_2N{p^5aw})7mTo755&Z`~C9nt*4v!^ZY|I7c(SN{}lnl zv#($s>RK{izpe*FqE3N}YV>I0VzhkBJDD^z?Tb~FGzXu7S{f-_F3e~Vj+OkLKA|3tsq&^hNGw!v|& zx>L%~z_^wS{b9^lnVxvlG?f#w7Ry_~BA!`VY>e6e`76vt0*Vyugi5}af>&lgoCpMQ z$lGIn>8H9i2_r^vG6*`4PS#|oOq+N8!(aU8v(D9r;CVN8wL*|F`~g~Yotec@7-v9e zbSxdv#Nn_Qt>^$=&}G*C!u||Ogya3Dwb{D$_9sPbWvamQat&t!l)&@JMu6H~R?36h zkG~^=*|gsS=+%v%X`A8h3+5@e>3|11l0S-F!uYE2e1or^TS`QycW%GOIUab+=>`oPhqh!}alweVr8{O;~p90Mfn>L{jt2Aw=-Ba%c>{KJ!+# z+flus@Y8GauH9fb#xG4c)=?mW7+p^}yqLAlro6fqwyY3MMQ=#|Zk*d*R9Qx*hW-f{ zI1=MCY4Uj|wk!=hyeuz&^NZ<7gyaS7k3w2A%0@a7(FR6%N6A0UN4X*JxU{^WiG4= zWaePVbcLxcH39_)EkL+V68lT9+|YhOE*p%6g?NpDB8Bl(l6g-)=0Sl*zlL8`me#$L zHxj(F6P4NimJ<>&EUN^#yV4H;<&&rnvnagt*J*6BYg~ z1X<_L?735uPgi4}V2tB%HKWoWYpScJ2+XuJPu<6wJdRdC(Kdwr&jemi$`q)!NG&W{ z1R|$xxtD2^SE7?gvolTRXfr;<^1JE9t%V1|CA`JF8n{t@2R-xxlyW-9=BZVD_BHv{ zHBCc+$fb;-%kZB5)OxcQf1G2;IMhh5YJ5v=j7$uyK@=DA)uPw9X;LqP!1ZCHe&^EG zqo=3I$AI?=SD*@xnLGH+(2(Z7;O~+yU!^TW+_#8k^vs(g*raK+zs850$!5FROEq)& zs!~--U$ZVl7gfV=SsMATw(0Z$6ijB~vg9ErQJ9N2Y94Sm>sJ;g8l1YO$Jw=_~t^ zKOxmQ*KnF~^~8{^BsdQ8AmC=9LA7;UCvuY#$wg!!Fl$7W#~}*(=Qf`vmBKS{l*xk`FkU$Fuo^Uhb2ncH4>(kY1YQyIl48i&h*8*{e7r_rlc>J0+Dg zC03j`t#F|O+yj6OYz5HbYw!Zk{D`c_s~-@-hHJK6w$$YUP#l_{;G;8f26}9`#vb}^ zElnU;HPq+YdW6}pvZh3v#^RAn0%FBjI=FfWF2nDk#7m|fh!vNejoA2^Wd}^MMfjc* z>Q3N@kL0+(Vp#xz8QZ+zoD#BF0fMN)rrv`ww5LxEY$;KjEaT;Tc*N5jI3Kd4dxxI% zq~f%mGv6Wi+)V$AD_xG;p=Tn6O>Ocr4Z>PGnQbAmRyh5rzZ#nI0TXbPE zOGhqovYUaAn>Kq{eM*PcJ5}D#Ioa8q>JXsl-_X?bP6LNor4HXqI4pjRERPtkb7@Qo zk}wx1Dm}TUEe8&`Cn{~#@}vTzI2f}JzbofF*bj0(jx)rp z!h8hc87#vd6**O!$Y-Oi<9>gKu)Scqk>hKelXX|mDlS6%_Y}?vBs9qsONc|fzRo_+ zb#T1UcF=H(zd-kP8ok*QT$!)n7u6<*w%8_ zHlCZJ-}G0H;O%DKS%NX1#HNRpYkH6)3lp+v#AEor!uo(BMPc`YDvZ8x8_G)&=kzI-iBM_u+d zj7KC|)i=*@IKH8<02CgV3QO*-{gHu4R_+{QT&YVzg@72$RujB|ElAwOw&7cCQj9vB zoc)$~D`%z@JObp}jQFpm6%)3v`sY+Bqu(v>FX+>vaJ6Zhwe?|Wu;P-0ZGvvq >o zV5W{Qa`TEjY9|xgtuzpF+}P|vx@G(pqBl~UzCxhf8aMGB=T0=PY52Xg-2DO+>dG;j zUd2?fEqXAIqdQfp^%33 zWRn6`Q0SB%3A4SVP1$@1!c^hR4;?oEq%T+J#fxK*d+Vnm=Q`MTe#kxO4Oq11P2G`P zE-sH!qz!q9sC7+R1l>yQwR7G5^l)M9mHYq6#|ary*y#@|M8$dK2muZg@}h0sboeu~ zW_2jIuq)Z@UM!s*UPV72-&=#le^vt_H7eiKy8A`@KILDQX|)InP>%ow7rFZ0L54ULQ%Ow`<|1);l* z{`~X>w)S%cb>-$!$@5l9=T*u3Io6L271WL0Xvo#!Rf-Y8_B z0+6K+Fzf0&0>-v797z+7l>F#uRihG<8lXV2A}GUH^DwAJ+pvc~y9uub%W zx9q1vbw=Aq!wkLwbb>9Qwi}Ku+@8;xe=p>>q8Gw7$4#FjJ|?!6wfdquP0~B{T23EiPHI0K{oS{irTlzqJNkbWoftw6ZTx1zP@AqrMVh^Nz~+Ek%vx|UafhKJJP$wYHAG5jsuL@Q@m7uhLG2f3LGj5 zbbCoq7ymXQ;f0xnV-Pr^t6Z~j{`V~VzJ_5$WLZRES{3mQt8*a)M{Em?21|vx6wKo( zAwh4k09ny`)c?AGA$zsp&P=#(Uz4%pveerX&@&t~DM9!o;eaYbu`h1{4r#n zym4Ko^NhoWNg;=9nUAY@xM#Ybka7xEWIU*?ZX*kI`iR~UH}ug}Txh+wjN`i(d2-3- z6PNMA-qZ8fV?YBU-V_>fh{eHws#*2Vs7`Z>bXVhToODU)Zhv%nQ`iq8O{#u@wkYlg zxIEIQ=w#)-hr?sZU*w#8Zu!Np|5tbn&HJB@5ReAgtK8OW32xP~7?6NP;(Fh#@O#2c zt|xaz^#tx0AENQHb$`svgwdv1W+~ChS|2CS6+bINLdCPtfrx%FA)?bdlgMa(g9#ean=+5hQl#^?LwC5fI%;3FmYU%Wc!ARt#8Z37?$``RF{w|8RE|Jl zpU*~^nOeUE#)hiXb=#tTvF+x1mnj*P))0AHcl=t1DoFIlDSPO)Ey%%0>XkwVpYYl} z2CqQh1I-x)CMAD-wu9ma+My&$N)-OB7?#4}T{cbMLIbj!!~!*IfMb#Vy^b{$;6%}V zYinWh$Me9SFdjftA-o;3bE7lEI;0)e6bHpY?=8zF1-pH#vQ1hwq78@ct+O?y!=_%ch_xtmS zh-~{wr6)dreW1v#__c8>sn8t@;7u?{v@PGeo3ss$$um)-2`+h$7!W~nj1=e7Ci4uP{0&y0<*HWE?iO&oapa`Qq6(%J4O<6qorkq zDbSVqGZ%O0Yyo&m*H2Ck#vla_})m|tO4JN+d5%}(1A6+2{SBUwoT+es>GEZ$+x(pCjhGKk)ML&C?gx_=c6GNva&p$WTtCH!bG z=ofhp7QsOJqipo0CxvS`jth_Z2OQ0Gs9Vntd?rRtBTO#Z_%kI()Y^3zuG5fQZ5(69 zEN*?UCYtE3B+vpC&{nRl{@?B-NBT~ySo&)pkps9d(TVrDJD85zEnv!MOEja~NJeAYeicHzVRGYcWVfg3WV_WDYmjBo*R5G zB>yR`>WO0Ln@lb3em7djjCuBqS>Yxh>gi>vjr_0fV+B2dQ>FTH7}`U)!8vX?Hu7(| zk^Op?F1ZQhRlZ0FiMk$I-@y9=ZXMFiC$z7Jn=upxqUQg@E5lp%O2GYkCQx5EA! zLueC`f%W7e>DUgaWd^%Se?kSWoriy8CM0^)`nnBt^+xpD-FOCW>hkpic=|1MGcW?W1kEUpSGx)5H;6pQTWeJQ)F=F_e(#eY3(&CjtG6M_ zEcOq20It0Xi;9C+;e{0&^9EMr$K*H<0T=92Jw1B|$?S$qJ^R|7_m&TvM+ysWJN1N! z+6Gp7kO9tj1&7&^JZ6NT=eC(}*N+qoL(?Hj%rsP^MA~l=> z$Vjy(;JYw6hO6@{7RGQuD9s8HvlFhn(BYNao0P!|<;9dt-@(A3b@rI1iN1OBAiG%V z$wWk4OTAo|){DY^?4#)5GpXF4d1qE}g;TycJBB=tdHwfH7_?nG%v8E@2MG?B8tGt> zq>kW$hDprX)gGG7stBu>W}ZipVI?jKzHRWxM#>tTq%0WoH1g2_1sR9`5*sg# z?kqmEI!?t!*Y4ppK#>KS!2y*p7=pNjRLgDI=Uc6QK!7mZ$6NT`+xhGYYlc!aS|e2r ztl!oJExIo(yDr~bm{psg&d9P-tuomZjj(91sApl$)a|mDLB{%C2~X`aE;g1FKTby6 zM1XN2`{JdaD)>!kvL%a5oG~oVvnb#UM8rUAWY-YBY9O(RU!5vi2D7r zqiz*5&y;+SI?4)rkEF;69|qIGNcfkY%Dj2k{}W>JNL10irr1F&mfo$olLLo*4SPgF z`fHYGZarG4J3syC5Dm>K+CHSt8WS^UZ@NA$R4p2@PTS!*WVtZGaG^OcRAq-q_oy#;91Nab z1~plh>(rm1JF*{K=wx71osVIUTlGm8;X|Hm?)_Nx@N9Y89l_uv52vqzQ$7x-);LG- znysy;K83vLZc&@}pZi#U#6uOtuKmMjSNmf@4b#*#D@-)E&uCUE&K*5VnNM_PTMg}P z2QsL^pD@Qen50s-=~dtK&{i36ClE+X*5+8HxqTCr6oTymORV6ue;{yr&mX)xSJ9B) z(dAqOP@l2FH|2rEtOs>dT3AO-6`xN}ed@CU8ei2sQ5oxgAXQ+bIEiEI$-o~qHMPIT zvEtR&?7>>=frIeyT}zdALGx-6fkqb;bctf%-0#b;z<2i+5YYJyh%qtqF%vJNc^ge{ zW>3rIonZ^tIH9`2EL88C3t^+UTGao`OYBNu`og)gEK=46(be?u37HLBNR;@ng=1ok zR?h$BM$dM$*1+)fF_ez8qq&ijg_!EB14f_k`_gdSLTY;BY0a7pV{YV@6)yU6P(O|`xR1|h|I*>_p0afiHMD*?8oyfja>Kz z%~^1vNqolQ_S&MfIqRJoT1+MO*i$jCF61R2Y27)*8;F_2q{sKO?HB-TpkaRV0 zxzujvnJ{M-DKUSX6RV4wZw6suMMC~$eFFRU*R`+LvRYfL$-U+H@1@fhx_{Jt0Nq*5 zj0s;6g=OWxui?o$GKld=I+o(Bld3pr2rE;vTz+{Dtb%4N76PP)2CtlwdI`O?+NRDV z+j+uoxHFGP6IE}WzS$98T+Gi{2wS=NrUP=L_gil#9gWj{n`Z8|n;}*?0fL1~hHOlB z?3!D+EYfAgI1CrepdTygcZuVR((8HUB3?SX*GRd2ndY*s=P#BI=cGtll5BKMvFD^g zWJVO>QzJ^`FvUc3MS&<9wf4j>$Q#9I+zY$(5rdGKhYonLWNM~cU2HugU7W?2kz*{p;K(mU>0wVavVgGnD&?YF1+ zm$_v6pibYz?c)R+SRWfg{(_zaAN?t2MA?RGPNRdRRjS%<(po!5c9%>QVdF}P=^W~* zU?y}{x_;ezA(%z1N>5J^R0?!=gXDV_C&HU#uYt{Pz+!%9HxuHaWou@1NunjBYmf%H z3fq;C9B=iZU!)=pvt(O23%32(t=idRR}_`dn$jE9EO6E|krTDZu1eYey#}5SPF8p= zVF;_Gny)+AR_f#BgCJhUk43a4GyV>7%S)Mn0HYCB2eJ5>{~Lfve;uk;YC@g646bSN z5)}B}@5ZKPsb=WQa{-nlm?+zn4A_74A!{B{X@c6?!a5| zd2)}%Nh}>rs%BG^>%l7Lp>!m=%>Gs|!AqQ*Fm`*q2^=;Wp@8FTdu-MITcr(D>NE)T zXmh%f2B8wEYWe!NDuVPvY=DE>cK7Cj$vJz1%*4Wo^r_b{@ z)^Gu>0*AuLPj4^Z6MZ3?`e1)Wi-WM;!V9mNDgvXgaq+3AxSavV}r|Nqnwfy4IU~%&N`rq zP-vUb>rq~>4`3LZvaAl}RjvKTNa(()v|%Lt?iIp!(l=--WnR!b2?EBSuqe4Q|FWPw zND}ooB;3deTAlG)ptk=^;gLnL2oSZMO1JXS!3A<`;fx=neGxaYYP5j^D zxJL)gss-mz9;xFC!-n%`bd`|tARI7N7!|^0Ja?19$^ba+Auu(KsZi-&tsA5^)b)>@ zS6;>8ZXb$2+R#xuGNYc>F@P2i+vUaSk?yKfsulr@27_p_0lG|pB+$|%PLz9JRDiK5 zpT}vqyEG?~*JcO|-;=Wyf^jX3&7m3nEBcVTM&k46`Aro*D<5u6^m!lBf?oP3mo)eRwh>WZs-99>gvEf~T8SXkq%4GDszf1FA*rzIY<;RcoMUn6 zsu@iFfg{d6*y=G;wUrC8qFc=IHC}L&Xv|_|X5Rot;?`@sMZ_Qk~W02Kt-8 zonAuxm1hyoliBC~?Qw!sBs#CP$}t0&p!ppTanSA!omNMHNNT#GL>Z?dwg;soC#;qb zK<;#pW?g)z4v$t?y(V7Z&tLTSMF;no^#8E-j?uM6&9>-HvSZt}y<*$gv2EM7ZSL5% zZQIF?ZR_QHr=9oieXZSp_s>|PYK}4HZ0ko~Yxb(@CFgP<0x1ug6AA(F6)6u#YICO^ zZemNHqz=$EbNHS3I*2@_AUjS}gcTzO%YS@SN^UqlQQi?E)W7aj5qL%f~H-~EoU%tFwQb2Y5$ z#E$bLpMA3T!5uJI$GpsG=071|(|?8763C&~0JEEd@4s*H64mp>Vi3 z#4yR&rnON9GHJZRcnWmHG*}p#_JELxXD}}$?GYvc&v;5m#`7QNFUN2@O0cwv|I>Fq zWQ%fy#3q+LEq?-r$0wcE-6^XoK!iec88=#tU@pcD0J!Id*^?oa_X&be?$b7#H5|dS zw5*08zNYT#jd4uUhC%>u#C0K@#&h!Gu{(lIXZbUm?oPTduN<>GqA9I88pvFhPr zu5R-5n1kUuhHVxDsP)5pK75#8F28c(aZdsJ( zpT=A*n=JojJv})2c~}JWXSn)Opg_(I^vGc0MrnIMHxx+oD;*pyMhJg({BG-4iE>_? zs_;3i-IR_GqVhZ+aXUh1mJ%A8put`{uFhVQl=?UC=wGGA*KsSTkk7Ddlxmh+|a%9Ojftd-l=ll+H@DF^Ll&Q!iE+2!nJ=6)D_4 zPU74ar>QCU)8uhTBd!${=3Jtw;Lp-!(_9*}<@j+UgX-`%D;k%@#J3pKcwt7|pIkrN zuW3|CHauindHvsvbO%b4)g>xKf6Czl!-;x(AubU56>BezF3<+zrpmKhY=SO@j;HV6bA^P0BKTl8DM zNhFo_?QTEuTaz)AH@z%IiH802Ol2WIpEdE0rC7OWji8rUP^Rd~GEG}jE>d}=ogk?~s6u_6r9#UZ_(1vCcr=?%sVX2U+7{b}|pIQit`=#m~Okox?2G5=JLznVnK@JwwdEP!ob;q^Nn`U z4>5#}gK|{bSHO`u@Q;GACc=qRS#k-c{ig^uHaPqaheGwB7epFf`4b&nfpgOJv)qW| zt>^o?b?LMM&6uB^Tk{bXA{1z^cCH)_BJ>2AjQ|HMXu!|TZF%KqIC{lLfXi(j{c&x- zBwC{V^Rrt$6Z}~Sw0A@4|KqKeecUF(IqMzc6@JPPU{C?FxF0n6O5@NQRw_R)RP6CkNlox1|}qa z`O$C|ZRj+?ah7y6!4UlJXOi9;*_K)$=brV=Xw$8PzsQ{ZC-8gK6%XS7X9j3xp1v{TUIg6OR{4iUhRQO2}#vH)YHvX z7OjHEtL;nzdzeQ!7nc`BIWuSg$BSyj1PuGc*nb5V`kAiv+VQwhloj-)TnFFu6(HMz zcJhr^!T6{gj9|vC4tM>kDr$|xyP@h-nabCYI^j7@rFNTuJPTuDyRbWL62PO-3 zbB^tD1df95$^Xg_TnS$Yz{}LkWwf7LQ50hJA*%ahU*!t{zSjKD39)-_SK^|-5A5;k zZ)4RZg#g{?xzb5o3)s_T)Tg`KNh?UB`Gbj?DJh|V5{qo4+*v^oGj{!_iGM1LZZ{j=p zK7C<+a^wPrq)|4m1VMWK3^Dw&{b+}#e4t~&ewc&pJ3a!MUIkk5NTFIJVvT{%ehXS! ztwY{32~B={J1z9tZS&t2p@)>bA&BWI84S@uU}J(1O>IOj#+wu$36Pmd2sm-%0;6Yt z&B6?MlHUx0{6DoQegwT|hOI-Ys`uJ{6G#5%O(BI`MExuv8axY=I)N-QWR=6O*(gyzi_K0v)_&}trkDb`q<_UTM0!C1qScIcx;aq~^ zVqyh{jC<%tq*umOPxYkNlfc9cP(dL%8c&1K)DJiUlDV`PnAj75bHeqhY8Ixu()6kR&xjhOrlEi z6PjatG%IkK=_;)o;5O4@uabTghFNh1D?^5*{U^oX-Ma9yQYqGJyZx|nXSL(3U76OY zE{$-%w==JGyPwrwU+k`nswI(k=QcBvzASi!-<($UElzkwz7cg}c&J8++33cJpLx#& zp9)@aUG!6ALO=PBI^0|GzD{1x4-!Ck*xf#y`M{VqdFUP>*0ESY50daqn?gPQo~6X= z2=$jURaa*%taXk4*gdO?FgdqL;{Ok|8oY)9(KU2evAM0WUi0nUZnE&AX z`#uu^PCzar0ghl+7SJ97rzuGP-a^QNpWVxc{T0L>f>ikYCC3`P1;w|k#K*`Icv>UYU}!KW0_;0xkps5Rhhs5Pu2N%9iz>F=-t)ze z7+*$t#O>q$I{BO}{{I&9M*qLGZpLN+fl zQ#Y)JZ^}Wr@|ion=4!6rukIk+dh!)&+^$N7`->sZF__kz1j=g~wxv}GWfw`}#wQvm z=?qxyXgtp!FxruTx1Z?{kJE|9!^Oq&Y?Il3O}^zEdF=gc(+Q)L`H9A&W=_PCW&ntefa#ZNh~A3zSI^{sYPgq5lKQDD(Id{{zZQ{|(B(^K3w)aHBt<{9*`R_LAHwT z`z3b?-htz#FbS!3TIDDv-yDjvZ`YVECG#X1d+~u6N|<%b4{MEMu|KmTAfY1qpvn(8 zg=}vPe`~hV{Mz+;GygPQZ?xGbQbx9;O<(0NiR&OiUFqmrsO!<05CLkY`kx)`;9wM~6dND(FMiU3 zg~`@Z=D6Ny%&(<6_!4~IHttV5pQvf7P4uM90)_I7z|k?6QivPU#DGk;5egbospnZ( z<1%|jq@>Z5I1RdJ$wsV5rKI(dJf5u5gRy`+jzZ)+*`%Z7?l-8R)VLH#t24N@-@!bP zx<4UgBN(_KWJGyL+7BldQ|YSK;%8f0^pQT*XkFZ(-3utnsF&r*#-HAWun8syd9N3c z9BfDe>sk_0EFe;G7}+l8>_kXMO2wF^GKMS$D%yI0uRRxdrFa**2NEs&*BW@>G8vP9 zwCZNsr7h&-O|VS*|42~7Ip>tN%8`eo?HOdtiuXgNIqQm_*b&`j-8qY|oMI}vJny(n z(_+jYRik20)oAqwoT2WB18HaF|5P&zO;0V;C7-(&NnJ+DjTX(u=#K(%-sxNu<%=Rpdh74CsAoCBc6=JKEAn=C51-I zyp~7U@o+9?EZwEy-chlL??hhC=Yt}%+!3qH++U{UAqHCImQHP;;nps2-5GdS0YJ^JZ%t^GM=&5)@DPwb8Dvc^3tRp{F zOk>E|-rZXihTG8;3guWtsiWFIV-9QQjU4J3{s71_fco-ei zTCEIS>fFG{m8LS+b?3D`t9~>GT`K{2-czM1KafxjN`kqh^I*K*(UeY`jGZS@6b&7)bf z@DixErwlX}6p2EoK=+~2^b7S0J>yJK22JoZvf^w*u<(4RjF*;JqZXgP+|L9}&IabW z6Qf~{n&O9SE_s=$nUZb$>w&nks2$HW)+pK=c}`1UQZPyv2is!#dX(pyX>(Lk-ayI< zA8AW?NMeZ)v6?2Dg{uvO_M_l3I)BxblS=UQFJ0v7sjJh6o9lbU(-qcz>_bKg6n1AF zDXhHa*O=P0)@hNtu{`3^h1ipRVj+dtRbL{G1EK1V*d`4DxI{zUyx=@XRJs1;Ncr^z zxMl#)M`QHhvOJ^fS_rzGlRc<%qBrJcB;aXiPyN^IIplXM=PX8oMnvVLKs3=rPL-`5 zahaDyicAMC$90HdE2ol#W?ZVcg)L{`d6((4dV@48lc%= zU(1pTmoO8>1LT0TfYMN6JpFJQBAX~pHQ2Eb!ZijG><}X%zOfL?iqH`00b9NW1VY6V z>6lvw_aVSz#40ph@S~C<;`zE4XFSe7T@_ovirMZ zw-XwegcS62%aJQO@fC>4Mix1^TSfjxO+aTjRfsksk_x6zlTfWousF1nJiePIt8D&V zQvcnK;sFoAgTZEwTWSflYl)_6i7#7;D|(6Lv&oKDrdbQwa?x4@*TS%j^M1%+J3%}h zf6I|Cq5kky!Q&TYjiV6_X!TkYC|zb91}RGKZ7qllO{)Cwmpm}mYpK`R=sRcUJ)N2% z($Ec;R_p%^o^96IOjy4xMbwgPTgR;vvJgVo5gAQthL~e+x;9`WgomzerKm!xR!> zc7Wc&MMziTFf+)FH5Fr-7`woF2!%Q3(FqaOjMH==VM?iVZ0e!6Rz)yPi2DcHws&;z zyKUu~xH=Vy9w4O?#I{XNCm55?o*V%jL=z(EwX z-(hSz&(rvjc*l*gq+9b#L~ng&0t6Z*8*wRHmO!eoE;dfVg66463SCKHFv^az$LB%y zp=!84o>m97)R{Exqr$hmK<7U+6)W1>G!9~)sruMC#vx@G$04O0$1#;ho6DGDtubl> zy$b`&#@XK0<#A8(Ctw<@BNRReNb(0xVvdM+fr1#u^m?U|2o}fKCzv1lqL$XlTk2nW z8m?6BmyEF>k5v(ouFy2l(+twKqhjV9scN=#VxQ%|9k;IE8WP8HzG{C`fAm#sc~sfK zOMtU19Y4phoNN!=mZ6|RC-BW{o>aK&Ah2x!vg{4{|5CVh+xc$TG7t9w3`Fh#{k2op z?+~jr8IJGb<0N<1HQXHnLGn6KB*kcgi3J0R1N-2oGeU-9CVKQi;f7%qI%0L5qbe_8 zp{HRCxZt=<&wey*xsW3UY($a>v}Lwj2EGJ3v`dsr`kkIrHTRyy+QGRBVKw$EYF%=rj47#h z^}jMsgR53{17g2H6$#kzU3*UJdi2TN3VAN`kWYIup^te^;G7=W*w|FB1y;^D1IBj5 z4=a2a)?ppUjJ8v^bs&nK1J_~B3pS2~apyv~acP&;%Nj?=`gvJIftoO0H-~d}9QiArd=KUajCm7HtPOFjs$iijXS8+Nyw?_l`lTk{dPi27_yVMkKPhJ;4JXdq;e zWHvCc^#pYpDHKu1O<=&z_UX_CMb+XGN5*zg@!(x+ib+S3#Ra9lq8R-MqdRwne#kpDZRdnTc zxyBo|W)-#Mv3}DG4hrW0_T2hfM$b+(DaeAne=vSHrq)UeBI&BXZG0g(>ktBmQ6n$S zw@`u|!j>1(#LLnAvYB>s%#A}k1j`BedWP%Mq%wX{@-yjI#wVOBZQ3&hbL@}T$fYs( zoFYyPDlK)`3i38*1Y#(YNM?s=pthT1ipYUrvA1rLx=2+wr_`vd$jwNO_HVdzQ*m@_ zL1AjIVO?VGrv=2Fxgo(rKqJVE4lm?v0!br6vZx1PdVm~VP;PPPO*b1$!KO7Fb5esG zS@mYc135maJG1<_s~f>5*N&U-{5Y@~OlP!zE;RS8{Kw$RuK6*Zi;sUSMFH=k0I*5* z+_F3%$K^~>Oa@bsBO2*uAV;+NJ@83&-!t#tte^9N90BZD-kew?8nT#IN){es4k&k{to}7lPMTv+0ZAq{q3G${Tg(4Gq-?kJ$`QqANm)- zwys)mv-yrH_J$lJ z8Jn~at{eo=MVBiWC;TeZzcOZyc^i=+%{10Gjp$ta$d6W=!T-LoNM+1vqcNzNd&aMs z##*5`+-srOmx|wmQg~N0e7ky+g?$#-Xyf14X;S3!Z#Rp%hjti4N~>&B@sGV&B#hkX z)c#Gmbr3|Sp(Pe+Q(0>H4z(R&@pgK3i6EFZSLje;uj>`4dawqZun~(Ek4n)S4_un% z&#@5cz$qc@V}KI$Im3dykeEdUJO2WHwT(w^Vl{l6$jy_}35E07|D!}nD;?;B?T8ao z;teJ03mt_3@YqccPwQNNWyQZ*25)~rOm$-rd|EhJ<*!=CWCF;}g^sv&aNK(n>$ms? zL^oy@%+=)bcO%gTB*w%R=-|4rwh}v_hpkZ1A43*eMytTa4z?&%!xmbwSGPb1S)!jT zzuhufw)KMB0yH znr6&h;2JqO<3d?P?O>+D`$Ym|w@nJw$;MhdT!k& zhUWHTUPLz$85erT5chplaoGN{I9HPrxf|p>A2s=YNXh-x@^kt`R zaLJiGnl?x@vOUAwwUtkhbeGRT8tHtPf9C(a+INvj8nqbbc?x?fq9fzz96|gcS1}cD(U9fy{2HCH@FiU;9&e!$NsD z3}jWXFyjU8b=$-^rOJw?Wyl4ip3w&F#!t9fIe9s*;)w=|GE19CYI&$IQbJ8LTULre|5s0xGO& zGo)`gWPddg^?Cz^W73ZK-PPA3VZED?N}iI!V6K`jP)kKLSRIhJ%DBGR@vRVgftKx4 zcN#*8zD%n6_dS*OwQu1+nf@06vQE;i>tpyB)Oip8B3$#(x@!pM7-3S|kwY`Q5M#v{ z!fgz>q?e_s&^RiQ8U^NpWpjb)FbImMwyW$=teRG(os?1dF9tspmn||r&j>1G`A}Ua zlj{e2 z+1K%}d5g+u>+mQ_STkBq`pE%ccpbSLU+|#fG8P*RWsCUA75%PtmM{Y=2EY{7pD_VL z+~HwX)v`?c#wms&D_EH#(j^8{$T5Lwe-tHh=!6ASiJ&B7eREvJ^3!5NA6k;Ju60(Z znbo0{SP&jd?adGJKjpuj?~WdSEF|uS1OwQ#2c<0{2gM|^mJvJmthBUV4KdN&`7>LfOw>|rTVn;3S&>iHy z0vHqdLSc|1^zgnZ|Hgdft_zSV@> zc)c*7_OTeK9CBeq+od@cO^g&QL}qCZ9A>-D0bF&-shuI8vn#EMa&wB(lO~0#P)nrw zCpF7Dx8E%>SR7-O#R*0V8|iQKE@5nCX+JhbR%gAmvZwv7OsL=K>A=Dq6BQPB@J5T+ zG9QM6vze}jbLkY$#|*aL$naIaY1s>=v+6- zHXnbAp&V2_Ri1Yiz)XlFEnL2Tm;uj03$U*O!;6>oD)Sc$g5f60%}DeYI|vUO?Io@^ zj!gXV;GU=Ziy3Ls-SraRnIW0&BCcr%neHNDx(Wa6yVUsy5<80A7V)!W7xDGuk?AJP z>cSX~n`lEB@5yo(5%LYoO}N=`)$QKoKYN;6#6mOL{KX1CseU#dgTVZZyc`EVqeqx; zDHGw(q?h>Tke?g>Obh-n|4e!3(7gZt+#r>A?ti)V^MK|y|9$AsgV0Bc2?rQ_2T;e? z8KEk0+#@DY7#q;(co9(Egqr|#tgFv}t9Ic|fp`JMq#9^5#$AmLXb z_;^H%X`rl)V>Pn&$TeY2;@`5gK6t@~jX6mAh2T2PYQ5EXAD^}3tTvS=oOyhSt zG^@-wn6G&jNnFdp%J+=rc3YF>P2u0kpEnev)CE{wUQWDtYy6Bc800L}Xt^#!{Y2x} zDp^M~x2C%+6T$`{*w)XqN*>g8?$O-ztl1_Egy<=^0ykF1vSGlN@xV5THq~c=K;^Mm z)veGpO}}qQ5WuL|5q34IC&@aHPMg+Ds}KV&Qq-)>VX7OcrOk0UnhTkxG?JDZQHip- z9eBgjKx3_k+mXr`%}G{;G%gjeBik7awhSP`&(@dorN)^W{E2Ia&O03wyP?Ez`#Z0r zcO$|z9>vE5DRxT{&6K+pfcCfIr7uB|;cmOq5J#cAun^S(v`#_E{!;S_EB%s=&nk{i z#z)-SPwE3HSa!^jO*cP(W>D7udS63i)Z5e}utA9efC+A(N=OcFxC*%l#=S6^a`M|{ zcqsZ+`~-8@8?bX%qNmT$lq};uXkTjzd_TTcZht7E7#|Owuh?8)--lSpul=2vZ;y|P z@1y6Dkssav&&z@L%!>Qmwc8rLTc)0-ollPk-0dBNlaKH{eHH*qeE9G)-N2*jwpPoMpK<#WUK^hQDa=%z>El@_JjgIM=|F(IXDXO#&G5vA zUz8+2;L72@k7bL#tL9nXM8`A}7kim~BOzRD@I0x%o? zm758kJQZ?38OB*XTJNt9CcCC-?qHXv2@V<$G+_>2kfs;r&|R#+=j;5zyW4nP#ZbW72`1m*;d<#Q1EoGv7kqu z=l}OEtS4IE&@$FU`4v%dYkfqq`>qKj%(-4!QkhVwdHqFu3vx}O1QAq%GF>MdvU*SF z`mVUk27asEvP(@EB_+VFDt=NhNj(CmgjB4ku57y}E!}eIil}pf;k$t@0UTN*>%ceF zN`iT}{WhFBD4CV7F`GN5_^cUlqo)sv9i84$~%u|ZQmQ`o3x)p??nnJs+RrfrX4j#p+ zGixpB1iABYkePzsu`szxPR*g`OM=a02_EeXe0_WF}DJ7vq<`q!{fp19G9g?FNk$RZ z8dUO+PH(shkL?{4K-cnD#GcBLu%AO^BJf@{BbW z@^({`-8Pa<7un_x=}AZnz=jMtRf{fpl5#?q-a;n@@+|u)m?%Aq!h>CMq>lPrnBT6l0)|FXsG#ax*Rrs3HKieidITF3g*reE`N%2mzezyr8hCCNHR@oi9bJ0mn<`!|f6;^=T!voTxWyvo*+T=62Fvd@RjQfgbcGJq|KNO&7y4 z6w@sg@^28D(>N2F3+o_0JxRIksGofd3y(w*#Rcw3x_a7jlb-lb+k{$+tFSTt$ryVJD$Io?MPIYg?SCeiL$3K~CH^nfu0KsOV zk3A~<6l1kr$edA@YL7eoTNsKm0siknd#tw-tgPa$V2Ai`4-$ir9#S;Xx!rzCby}(< z$i7T*WvW%pr{!--)Y6g50xab^cTzI@s-;;_+EN`gjiRh_P4P?k0I8`84X{bDclR#F zKc|rh3=kHOMYX$xliug%uHehLa}QiORlHF7J=L7zk97;aPX3QB6!7~sC_^~=#r(L- zC(ft>8z##~k+XXB2PJ1+D8_TCvCm)saCP{L459Zls%r!DVhx-VOkTF@YFV1(e+z&R z;-5s!N{3VN593cPy$a{OJ}R9I1;P-(S_FLB6)FVU6w)OcfAslbK}$hFLKgNWrq&I- z?+^6e+f%kG1bAn~MN&1N9Cf(kF$`WLsG~3{mjX+ns;oFN4n$>cN%~Tk#H9xtshe#0 z{IR}Yq$zPEP*NR{fCkn|p1*`d`**9Phxp97quWVbGZycjYuI^gnq(|a6NpjOB)mJh zF9WQL-Si7_p6U~EFQAl3{pIf@(ub!qXtQH-6Z z#6{VJz(SJ6SvB1^_<#-$T`hSxir0jjyKVY|2lRn%ZUG9(Z6pPjI|IIZER|%J8E)7d z@)x;Bj(xH}S!AB<5bC@hy8%7K=r5^J&M*mI%N)b)0qZh(8W3Kx zffn}{bSclEE62VS8qrsTrx#@X(0phT={7Co@m30-X_KOMgcg_kG}Sb`Xrr&z5;Wq8e06V zlXu&$jGosdcqb=2<3cy^bFJ!gXih@1(Q)LZ3g80*bZ25&V)kAI91(CoS*|pjn>h2HgIQ zFVHYUOXHr3fe?iINDVdnTW`|Nh1`;7+>NkY@fxf*WVi{U(q5D%9b9aWF5^^8q55-t zjre-wa1x?-8a!a7KbrAwuk2bley8bb1psm~vnxz223|Dlqt^fFkLk}#Be)i-2e@*O zMP}m+Yuz+2JtdAD#7m~&lEjbQBadlHI9v++t*8kz-38B7BH}XMH&aZj4J|FEj_1__ z+-?;Jt#hSas(4~+wTep>L^E#+DO!l*>c>aUXgDM_;{X}HpE<0!$VOYNn;y?9u}Bf$ z`O0T<2DG&wKPW%iLlnG8Qao290|3qb24d0&;v~iVP*mnNS4JY62GaV|R@U14<&|mC z0OLKu>tOSQhS&Za4#7GpoZFi^>Di<_Nu-2Lg>>u+KQidSzY4PR^I5b3liU?`^Gevq zh(_|O({fX?8^l&M6MtIyOyT&A7WY2z5=23C8(gjUw&TRFu z+0ugA@mN8Tg1!b-#MkKUoCcLN0wP5H<~-TXmoOVIo}KO zlZ_e6-VI$R!yAR=H1eNU8mJKt-x9kdQjW@lWaSk#k2a8^15!t`(8=Cmwzjd>Hn`Yr zVjb^0p>Z@#oA_r0#Ig{nd`Ps~92=3e`Ds%B-aeYYUHj$`ZTeho+nspcV0~W&Id795 z`-E6{t9}3dGSBsI#u(8vy8GMBdHF05-zO6AzPoSuZWe^UUDU-&t14*GU*VIt_q>L4 zjQ*25!=P>{iHF5-QJ?I-c@(|XdxEw;2C2=$J#9PIClIT$*yJ5NBG`6yISV;%rpWqt z#QPRjC*BlV+y=ML#mq_$C{8Tlx2VACw=^>&&sfgR!{1qY{ET7#l_d}6R?o!&!RE)HyiED;ZBiPa0dj zqwaGPgo$rHiMecd!?+Zvx$rw1*QtNkV23=$bZ)E}kT~bX*O4Q@CW7{%AA&91i=_V# zZWHfeK*OMDorF4stZ-SRqTvfsKGd)jRl0&mvz%G5+n$M>CE@H$jYERUG6qlqkK=uC zw}z>u8ol7+;09>a*o+GcALG`AxU?A8`(7Gm*jc#PQ@l-Op2u7;o8%*Kon0CRk-#>G zeX?3#9KUs}q*HFw_n!A8t^!J+bmm?F^Evy3TW*O11rm&wK$U z5fYcz<@$$hf9%16%XsBFWKS&l=UBsleY}ytR_A@B=Yb?5aN|NdV)93R5=T86-Qswx zf0yfgr7u%L+4u>Jv@8kyux!Ms>wzaP@t+1s%N*l!W&`zmmWm{8&^c#awiy;>Q)WYj z4wD5gs0O@s9pcpXa{hum+fgFW%It0e>;fJf?_A_VcorqehPF6Zt{E5aoD|ct_kCx~>)D3*eMDSgbuMF>brY50Adq6koY#mV6WhF|6VtGRWt1%?VUj#1{uzvFZjX<2 z^?CyZRln^H=m737;^tct;Phj2$D%L;M`n?3)as(yyXaMm|DE8}D^ZrTsWYYEB{OoU z?o$@gh$KVyib3YrW*qwe@tfnN9#FY5H8hmdINl-FkB?)Yn{*naJFjFr1#x)!%C+8A zhiRYcB42ys-;;NS`DVQoe7ne9^ZNm@KlM1-0sk+pi5jPCuF#nAmktxwig$ID`5+9M z2&$k;a{2+Pp{aZS%AR3|P4QY<;7=+e19s=XUt%7%KZK_5e-WChjl2Iln@N+wm(|wF zM<(b95Gi|RP%<i2Rj>P#wOxsR0*4>-0!G#^9X3#UTOu1Qj5ZmcJu|-3{2Fopy+cZ1AT&z6s zE>eqRz3Q&5FCV28H>dWsH%Hcg}iy8D(6V5d$%E2&tCZ z2}}LM`(lb91t^m4lKa|MGF$S2*z>eT3U_`qZ;bflkcm-@vGR1!$Meefttpc><^WU| z9xcn%$1Dj5FPmjZxx;<+lZ*~{4+`c)dUN945q!%+uTYT3B}wq1gkfS7Gvm&KyQw{D zy+7fYm~7FgTK)QA+XD~VsNm*W>&U?Vavl|>u!0G@z`=P$akWyd>^m|ZqvZ+tBvCui ziVcV=@d+qOj9DLPaXA?lkbZfMd5Vm=gC}t}s^cRrU#jBhlc&kzW^H7`j={oWhO5#f z?&h3o3){A(5aUPY>+tn5#Z6zChjgo{^X~{`hItzu5;)p5<$JfUy~mz!_zJABxC7tL zVM%S)`f;plA;Y5smI(m7EnDpE3J_16Cug%qQ)>0n)YQ&gje&*$09!pLfok3{%w;ma zBSi4U^3C2pg^1eG4O}Y4DnrI<%JZCq_bRI{8ai(`*}s~ykOGwTRjc_kbG9T`<_+fa9t!*T%wB zC2}_04>s{Ub-BP1)3Cf<*@u5TSZ*QyNf-!i_@f>BRCc9!ah@O)&)K#Q48-V0+0J3M zV^$>}yJZL)IuB|64UI~9BRH|5_c^xhR~Zb=kdTgvz_hjRWhlL>KJpE1uvNV(*qDov zoIY}g8J_nCQ?!s)8(h@D#5jAShu3k#9#`yh5i4)2xZ9RS*6VWXOTka39M`m~G!Td2 zj<2;47}6bJpRbR0bRWX`@J;{H5*N=R8f(`3Sdjk_i$hyyrvUorpR_!)l4D#IFs3@4 zNnzQAmQR(yZBJf=oq$7L*bZ|`(26as$~K*7OXiA>u*~C?(hXN});Q~PQiBt%89@o< z;Iv+scp{d13!RWRY^B6p)o+@rogK-sy5Tx&p+yp0LISIu8~$6bQ5$iJQ!d4{0*k~cR~U+J{QPWQ)CQ15lM zTnlO>VuID#I;qHRll>=0GApGy7sEEQLK&nl*9o-dU@7*S_>bATXXRN}iOffxbS#hf za`k+tsUkb);NSLzyIwtBBg4ffQm zA()ZBJhNRVrnVPiI=qspo7p=cLckik@cRhR_rjgP260m?+f$p&cNw? z_y#%M`u91PQV|WCgoiaC0fL1Hvb(DUpQ>o_g|X`PN2GcYtb7L9xc>r$I^G*%`Vz2t zRIJ{oIyi@99=FX0s-#NLXjN7;4#jgo+>X;<}s=B%au>Z$SUeXdzlCt{B@5~wILg{ zR(JBAXeWd&N7t11uPT9mwi-DmJ+x@R3ezLTZDgIYZ_K!{K+fKPlF2qMnkVQnQ9KPi z<{JQyD5uohpo8eH`ZK+I)<#ml6tw&$DXQabn|E@uW5Fo zh<5`~s>K0k&RwX#!x*<{*p9^O)8a-x#F#13a^n2h?y?>K^h6?Ta`o_Qzs zH)gHBfyL)gp{}NU0`(zJ(2OHtp6B(IT2cpm7z-!ER@L@QS`@AP0ERpl~rzvrj(v| z&ANRZ+}<@KX2JSajgRgC<;EY17tdQySVARglJH&jvY}IaUp*Zke*TcHX#J@JBNuEh z24Rt5)1UD)`nsijw9=>>=REc*J(FWi5la5E|2ygK)!Uuzr}|r_{f~}qeuD`QfE4Ra zP@GEj92G`XHIvqa%*k4`TG*m`1NOI@ktJ=qy(8X+cI=f)C!m$UCuU4^zrqN}Je-B& zK0$Jv6=Ui!FQx)5Dz99jrnNvttsrztXG*Si;KGCf+*CV5j74(S7L7kDJjH*wEQD05 z(f0$U%{ib?sXVK~>G~NmD{(#>i zD;OJ!njJ@2lTUdIu+PU%{aqn^4@5N#Z77lNylUo>Ktqw6DfIEb)DMge6Yv)3@}Q5a z;sqIc!shFOP~`{b#vDbdTb|SEBd5g$)?oE;@~gAe=ra7{nn(58k22@h1)nmN8*Ph+ zU@TdZ1iY{oKXS%|1#u*sgJC1Xhp8K{Yk+aIc*?8m(K4fZAZ31e(#a$jpQ&W4-MeQg z%8WGD5`vPwgh%7VX!GqcI-5FZU2Y}ra2yRVR((a>GU6JxY+a_#34=|xi$8%si9nvY z;g%@QP{Ug2dI{16S99H{1`7&gCuB-K{bcU%@6>d6w02@TUFKPWYjd5f`#96y9MU~n zp@O>8)E;PP^ZtVDYW=(9*!|&;1zs4JH!IJEC-*syP2N+!aEcdpn~O!*JW;+ROVvk* z{cz5BKcAJ^4gyMQ+N0IU)zjJb^7ijze>#7nFB`n$7p<)U=x73Gqn zpQ?a?yMMbL(aK9y?{oRfl_?_^akUO>-Bzh^v}t+q_;nsRwj)9LS3>_^O#t|>7ax|{ z6~%)&jg$4^3$OAfYHjb6uyf5PW3yvDfMxA2b`3541C`OVAs4tN_Yt_p2doOd{<6fx zk>%tnmEag{C4l_Y@{I=-ZAO*m;1XQrEIk#9vz6DBFO0TfNv;;K{}K|(}dYVs$Au?vb;ttuwr zM~cBkt-@v$R5jzDGpQE$$VEiP0b(0LRgNG5dolv0FkKQEq`0AyF|`9ZXzqy2b;!e~ z!T-bAJ4Q+Ntn0qz>MpywY}>ZoWp~-OZQHhO+qTUvblE!D|F!nn>z;kb8ROnBF=ot+ z84({cb7aQzd!NV5lA)E$?uf$RuHGO5TY)UpiuVQQQ)kCS$~96zmF!?$Pd#%Ek>&i@ zRjEBqjZwT{B_H=pDmcpsw+kpw3zioPbHke3lQ;h@s76H7USp8;-8tz=d+i&dc{?jz zrAn1XevyrrS2rPtX}W?8*)es@`cAn&DLvs=rFZHah2dGrK+pT-`_-4*^BrGCkB}z3 zkoy$hH#FQje62#;F#ElhX&!DFBzgNwZ@AJfj+F$j{Ys~ume{dNBQTmukoyKuioW_tKuN5H;rY({)|K9_v%-VP250D10@ zdmmFvFEba<%dJL^~8Jg?uUeY}r_0AcNb!n@fl7t&w<;{L@03TRZV{@<{F7V#^1 z)H3YH9W{_#=&xm2)k}n_!cl`C)gS2lh{iU@vuVmdg84236`zR?xv&6 z?~hEdh=f57nBMA>fml6OY=wcnDH%kb?Z8*B-?SQ*4(?)o2$GPEWheoRf9U_o_=9xJ zNZP_IMVGmn{hC1!YsSJcs3K5l_%xLgwXs4in`ejZR(~kgF88|c&0n7kl1)Mit17QL z#i{Km%0NogUeE+HC5;7<)4kI`I&8jf;MLYUF#Lz|=V^iJHu@isfBPaT;-$Wd$@3Ci z-#RKG&8&deuGJEI(6}PhgpX9*lkaSVoyWd*cJ*eKwRYC$R_3;LX7yHvmxERQR=Qe> zG1VVwEViX6ETnUP;dY>{^sgX*|830jTS@11TpdcQF%w#=<*sZn^>d`vU_h<0iDH;T^y`KI{= zqoe$LG8nc5p1~>RXY*w^aTkK5js>L?v7&qIhwBj}n?vd>bwFY*cSxeV6*VvC0)EEU z$jz0dO=p-@(Md+2Bx3+#MsL}?+!Xk^TWi8%3F(Ts-r~MuVsQf!Ui+?$oRK3Ia zZ*2ROy&9U?|HAGac&dhIG&(SF>r^$_zG0a(1F(Beq5s70ITZoeeJaN)oc)f_{HZ(^ zTt=f(-6)^94QLzq{+zIYu-?WsVw843{_acvhgHwi4@jwjFlqfjJEIH({shhi&$ry6 z6uR~pgmEdo!gxVbdq-D$*LQp>7gO$em_q; z*!DocNG3QRYz;hfs-6Y1+LDg@-Qe3nwLhk&rj|TPJs!bBh$gejo%SN+F~N4deqRRt zbjM44WD_*o5SNB7fCHQXyKAU@guA`(n+tx@O5VLH{&NOwkDeMO5Q_VQ04`FNHR2w2 z7Nt;d2$DIg^Xw*+_3*c8-Lzzph%DM;9s^XV2+9fTBUp^|q=+s?5;2MjDi?=`QX@Qy zzjn+hq59n<1^Iy+GdeMxDmiyZj5;b2C2k@)voX4@!#wb6nj?%dDcqW2$(9kQAc&q^ zAv(7^MV^*z@@(V>w1NVfe}^DhV>42@ZsboI>!e)s*wlYx_rNrNv3rA{i&7+Nf0L^* zrz&(3cY=%j{LnWBo%(?OLVQz`H{d#^dEjAZ8yUIEX-!AR6+(Gmv_R6a{bYZKL&)d-rlF2<4wnA+0Re4T~(u@z9{SBW#(HX4pIx0 zQQsBZMQZ7;4x1_~*(o#O8KOwVV3Pd#g?)hWn`kzd(KkPUexS)s7G}FGnz67Ol|yt) zqnz+wX_eObx!MqliNH@9m`y)v@RTYv4$9LlhH^T>sb1r!#o zWAXd0jTT7!ZQ`aNqPdd?lwORf1?W2cxvrg2Ahaf(l~dSAfqDpKB1dN&$_QA&ofAOD z9e#`j6um&It|y=NQ=uw?U8TmmKyp?3FXg(7t3d5Cdl0}XM6jO{u))CRUJi? zJ}fjK)n7Z~#z;ofr3i|uHaofwvlrtRa%K;L_Jar`*WWafxd15&gUF!i1#kOucu2ay zr8ZctDF8V9w(0SIz`>SY5OH(~&)~=*_(B z`VAoWgOz^j{Da+_P@48;J`~FX#`aQqy1yuUa;0PBa17R`ws_T@NV zn0xN0H}S)}|5-^zQOq!?{p@x`+Z5hBp%sKBE0EKutv8!6(<_VZd#Jn$iwMRM))88e zOS)A$(-n|Vp4WCB?}v$7 zTE;*{B&Xg4lwTz!`ZP#_)$qzYffR5DzhZLQR~Q=x z>}CyofMLcaX28YgcTTJURc-eY>Tx{dz~r&YZ{q-8Z!c8POCd;&vKW3K)ynI zm#w4MdYDc03a+6Iyz@BQfQ3r}TiAfP#c$>UB4e%ZeUNQ5y4PS>kDHYR4A7#m0WHy!DL}i5`@A50SQA)p$e1uFX&t{T=4^>Q-V_XHf)HDi_tl$t9<#1Z z(1{e9W?zUj{;Nct{~Z1GKHcgGG|^yp%L;I&X+M65b#E=05#e&cP8fU4x|dx=1QZ5O zfypdM5M20}TL(f=2Tqu}%G|dSbtAe6Pcqxu54P^0jzp2mWLboXXQ!TYxn1IY z%_oqj$|{&&DHzID&zy{I;!TfNW|1|tHLMb^WwhViVkH(T;3k#~-arAwDFnYlE3O1o zpyo6MXf@!Gpn2)CzhG~?{5kn-I|Opn4I>|pEo`u{Xs-}tPN_|fX$rz@ADO$vuzLW) zMOWc>tp}?@HxacJ)}>~RB!+#U=E>q>Cn44b^)w?YtO+Z-mmtf?3~=sTaf>A_mcw$M z2tg>5l+kzx2evj)-9gsuKoDG{?gE-K)}dqL@W?2?N^2()5h*aHZRycKnFdt#*-3Ve z<}#i(BlTlkWJZpaTq~6g4d%36hfE-I5#KiuBqoh`;3FeF66VJ_$;5f7AB?p##j43D zCTpW|wi%OGd*0PT&rk{@!!9Ezk=JQ8AOV-c)B=*poS9W0>I6gaMXX_0DLZOB1Xu5% zymM4Mw9lynnfVwO)npMjjYOJ}1aj(vEI&?EP-xDkNDBPC9I-rtvF!u^;mJvvzfp}X z(2GdRS`S9_MyB+M$Y&Q0dC^8B!_bh@= z@Vg^_lE45~u^snck;sxL2Y1x{zr(-(BA!Br zzKQR!m9<#9+LCXj3k11})~NWiA3n>RgkS zL5xO;jc%>>t%$BVAN^(oeCe5*=|Cfv0=n|ES=JZA%u=MKK0%cW4krzL?W7AQJE1eR z$!<_{XN&!iP4Kr=Nq@T0tIT)%@&pl0li;YQzKi~0uT-c`DNFSg|FRSpQK3F3ZOb7! ztJEfZOby&=B{SP2bi%AyhjeQHf3fR42XF;IfbTw!<){z@K8rkMGn{`z?8!mme<5~! z+xA-{$p3`cQ5`n_17f!-tJnEYh#e~4l+z57%|sQF>sS8l>}9K!OxL>nJ1ffk6T*1P zg1#qo1?<#l>I$xE34wwm*LMz@Usm=5>U+HaQr)*8;h%W``vA=CXS+LB zh5@+DF^%Az1MEjC$MnC}{b$%JGaC~E9AJxpr&C1O@dfI9`jP;GS$OID^Eb&vMh0{` z-dL39LErh9Z&a-Vw~sMVW?>W#Db~MmWu-FP@D5l#(r7)X`z&u|l^hC1iX2ki&Cg8b z4Pih8XjYUH>=xP2Xe7ezh3!X!v@Sb=6hMMY1>tmDsBY$<_DWV<3q|6pDg`tbe3UBs z-Vx;jq)JwNQ>9z_iBkl99BDTr%3LgQ%VObIHg=Ctr6Bqd-V@8T=eHJD7KvEGV^neC z@5H>r%Vd>PLzJ%S^2nJJ_!m&2V)3M-+bY`P_cm#z z$vBhDTQtmQ5P_*<)|t-!AU0JcpXFS(Fq2MSBC0ZDf;x}8lNh9SwPadO5|2V8?8BWm zk6AyR3qm|c3*c2+S^loAkemjK?FZpJ)Hs*-e0m=+5ct%EsP!UO;ZLsq4IG;Vqza0b zlDvZ}zN||ZkbQy91mc#bN(3SmOV!o8Scif}=?l-MaKvm?`5yWWPaOvLM^pBl%(%3s z95{_m#Qc?s*kG?oKwg7<@5+x^3)Mz!M4KJhuHct(wP{D+UFHX)cU5^+Nn2W0USzDS z5Sr3u{1?qz9kH+Hj~akO0WQwRz56>J-`D3U9`gH056;)av+CE>RZI*3V1L~Rd16yN z;cGb1@!2=>w&;0#I3;8MdP#%l0N5g^0{1&-Pe-oo@9b|^SlIlmJG#dus$oVa5d%*N z?Id>^$h+yDgA=R|+ht;PDeFqewC4=5BN=UF<&<+y%W_)xW^W#-yWCslY&h@hbD~I) z!_mS(=T#y2UK_)vg+0|6_qa zVt5n)fjN~mUqc>mn+5xW?7JBD6l7k1z4+7Am6KC)1xA8TJUKN=*JOF)^!Y>?u_qOA zH8Z2myOYShFe7B2yNyKjc4-|dagm|B6KQL(N7sam zf)H!XhG1)LUInaE4^e*Yq^cZJ7M*Zfk#V2lm>r~L`z<1-xeB68 zl|n)WKw5ILRP!ZC{HD%gs^!5AO9l1#(2r9Lxa8&HEau`xpLo%ADyJ9sXDR1BI$!p$ ze+Gt(zAcKF)V` znjN9gLhcIzlG}>0s4|cE!X;J0)QHoQNO6-6MO+1WQLo)gw@pZ?#oI>$O%pc_7XFi` zncvZR-A)ZVOY{IO!ziFA-UV?$2rl<=YNHu>3>j`~kY~moKVpK0)cL)ex3-o##V^3i z)I@MIqY2^TzfkLecY^iU;UW#x)=RkVhXn6gDKBpC&JiAyle~2kG-}6*N$Dx`qQjQJ z9w8cia@Uj(UmUQ`;sQqh?E(#6Mr-VN!T#qxf$uZvp44r|9MArvH|spKmo@OpL*8g@ zcHdvJ57L;dpZ_;&?S4*IX}JN@IiY4C2+e7P0o{SQAA??x!e-d3E0vyIsDSbacaMt7 z@XePMu63+8B=}DyO}~~xwFSag0L?r2<}>r9zyBLE%Xkq-3y-3ImYY20?z--!!^76Y z(ATexPJ)t_1S)&@Fz-pkKf1_Xfm+ldR`)^H1)&vHP!;!rI@fiFQ}_@zq5op7%Wo|B zYS!w0f-v~>vlOT|n+Lx()^33kV(GBh%wtR~PPD=qfDi_uj6urVZ;kMlLy%F}=4lgy z@E|b^>>y>tSs)NJRidG4K%C1ImWz-7H*C!*(Bg&xXXmB&FW6c(>ff;SISiE=b^rT& z=XPq@y{ih0zXmJf;B;`C4bSDD-hI$??~#TjF@pOQ<+oT~=hqn8+HJTaz4xU2RD(^F z(gDmUSweP?$#qXeOzGrzrLL5G>et(wZJo{m=u*d`eXR*)f*&iq;Y$+fdmgI(JBxyd zOKpLkxm1EoB;`ODKGbr@3AV>8ih1bp^)}3<0Ve}-qEQ-m?V{l~kf7#+B1n4rh% zCm*Sc){jHgWl#7^tD_Upz*;GbHjt?Y4yp1^5%Nx~SUrmA4=(j{;(jo*t4^c*)a-DgA*Vj;;y=7_1 z%RG%??>)w^Dy9S!Q!FiNYk$HAcd={f%6U>eC*Ry|`zQPxxOV+%izq?_e@t{G{~&{2 z1(68F@#*W<$kCsS%4ED6Fke&JpCY(TDIlFpo4WEy}%Zj-7=24Y~I$DG~|UpcMdc`=>~N9Gbb*llKnl&TLXxG+6& zv|g@%%TwJC;ZtvcFFTkXi%|u|d?Oa+F&jQ9$H>l8M+?JprRF`D(G|(Pr8k}{Hig4_ zVya6@!K4_AJgBueH_4`QC!Lxw7i~BPiRZ0}dBUrf+nCSQJ7rG?@{u!5XDXr(0bK|} zXvwo0z%#?x=5M56RVDi)9y7&^gFM_&{s&noLaU+`;^kRT;$gUztoE>=pO`Nbsc@=a zdnqF9#&$$&UiNi>=d;)k`AwYZwW$s6veZ&#)ZOpnKCO9M07UX(3urS809YUX_Pv0) zA4@m+3s`?){ROPy8vg;Tqn7!UWPFKVX`tH!+e7F8qE#;6Tq0kw#W~_a_gxR!>tcVj zq{Q@}D9fq3bRgMm{phb79lQ|PQSh2kJyBzx^q41>*7Z%kWAv2Vs1xIp^0amO&t&R? zW>+=)RA%Hs9JAzI?dUOmvJeLJPOGMmLk73Pcq2#&!`pY^Ta|b%kJ!^Qs(p-Sft3I_ z4`bn41FfYl`W5jAmuVCv>)C#<(3kb}kUI}?%**r1?uM;XL4O7iw!>O?lmK>UQ^FC;XLQ;v^u5jt z26W)Xz#Q$R|A6r=vffHs&l18W7GrvoNM3Xqp_l2eNR~3xOIo0=p1F_K&%PbKXPXr? zJBCd!SXd+hF(xVo;jqg6JqT7)W{tOUFBBi!vLWrDgTaETE~i^@wz_H`=4|8LU|wbo zv0{rH60MINev14!kh+58N)fqWdHpYLbCm(IYd1RO{ zm#x?S_rXhaB_nfd9?&R}yg~u~g->x>YWAU$Su9zZ_+}T6?Br_X44Iod_xCA4ymKsV zXMM88by;nALR=Z8BAfP}aA&RKQnltldXN|1*8N*XMn=U`Rq~FW#+10?&HDhKH!|)IeVOpX3NUm*bcq|-o;vuTJwQ`2;+w%ILlcqPEm&Ad_KiSSuijcb8W#va zofU3mEyVOAc{YY*+~LMDj>j=j6|N)E-gl@+j5q69m92Wc;^e1>U6kx5!fQ`dzP&(u zltd-!o`mGWR5#6lcxWCEKDpywz+?!Winwz3@&cFpoi3Cx!Tespf#}%`0qJ_3h%E_i zQIc=?(T`bV3_Qk<0S1{+>98*;?+dG0VJH^%{u{#G9x{@^T^D8++<<(~57t3>g$LO2 zax;zqDrPi%L|<)3sH1K(5s`9q==ag@n3bKZ4_CL>SI_w`Sopc_00e+Pu=T*kN-dx| zBDR4&H?Hsk9JlEVwFySwCe}wd<0yDxpM>i4_Pg`>)iEn|>F_Gk#8~1~p*}(zVzQKy z3_Zq^Ve|aK@02y?oY-&*I1a>)c0`>CK+tONcl*Jh9@G&UIvHe(-=On`#BMj6@Ly$+ z^7xsB4++q5ZBAa?qQ~BLD(GA67{k-A5M#fj1=FUW)g*Lrl@58!jud-9Nh_+2oHsT` z+#Q^Q4jcu|DRg@@8Ru}-a z(&~PDkyV;Py6|KwW8|etHWYxiINWq-09ZZT=GzVwBAl$V?F&T}$Lhm@<bbSXifY1y&*)T`L%V_B~K*m9W|4lqKO^{n-Toz&{qmn-@bf7oR%x|iFT z4#mX@sV1t-`=*fcnSUhdM{)oBe2uA6+s&F0kz zW#vHMunc|0UA0y{ToK&MSZ=^j#BbJ(vv7ON>xo8}s>@bLK^mU?cHOT%_ToY6fbTc! ztiS*(KIfm=^TPNz{3wJhR}~(h-Gzv;F{N(PpJywUjjGUbCYkHmoGVCwMYj);IjA@c z7Kb`8*trUqB<#RI_akq3){>#_9O*MNLtpeMj5T}?X-2ay-aa#RR$8J;;%|hdUacuN zB{}Dc+RO0_*qLCu`^4)(j3?s_`tn)X{N;q$Xs32V%Zg%ANrsoaCPX4m@Unpg5|?1xZ3WO zW@V-YS5Ri>YdXh83NdRG&r*H$18iG6u#N_{EThS@Fh>N@I#gc0(sU+=V;3wPu8kOj z#ty}nE)v)mzj;+gNUabEd;`S>%gu=BtEld_r)D;t+Cxfec~xeEbZl*{tfzhK;^lGjH`-OTzgQR7BjaC)wBGo<+$=qSiD+q_s6lGu;K zwlgs-t3q&dJe+X;2O%`=4M7MEWYG!Zxg(fQ1wdIKMmn@j9#lLhnrIUi#VlD36{mq2 zc^Wf9(Xs=Yd3PRup%!}y&M*2KYx4X11@e*CRhv}SFrcJxRw28B)r4Fw(4<5r_JdY4 zhD#=K2EhA?ab^3KqQ=OH%#XvP~J|q*KU{;A+mV^>b4rmn^CB4CI?| zH;k`KlW_%*;P2`aG#`K4&J0_T>vnz39{%6>SXzN>jR)e_Pu>y{`jQPBSTB;(4_J29 zfP8K*w8irB2#rm_wE7yZ@)WEf+;?p0O zP3Mny?-wt4pOKY*;v%2#*pk~1-_lsflb3&yo5>GapkA818ws)<;qYiSc*gUFXJq*J zPY*;H4_O9=lJyz6e}m}DLl&YCXYaT{-5krOU6XPiX;`iG4Ed8~=LuA@JC_t_FIpTl zX)W<5*gTwfIh}Syp1f@IsLSI*ep^sNqGJ545=f}%)XQX|*Roao2kYAVX+o-BM7N|g z{ncy>#<)&yc)Ey&2y(w+rFvp_h>Cu=FBhIt{RIUGa!%p?pW^M8#}gmI6XR75 zG@Bz2e~bmX>qtmTRG6AUc4ZxwWV2m8LXKO%&_H-kfTL4{GZkzw| zX?xXnpo~%@xv6u=RgA4*FHb1$h>xx<6bLlXm&6oT6bq;(zjzM4wr{4EKj}-uZ}m#o zjUAQ~gX>}6Y)=$kr&6EeNDb}a^j~sn+>=AU?f&SbdeVG!@({lWk}l*cE_q#WE~aV5 zYAulxOsW*o58?$|0`Sof;}CR3y1a5kR___nOz87Ut@w1LblFd8SDML7l6PE>%MVS~ zLHo4EstOjoG`aax;`Y%r-79S9^#Qsm3QfSw3CqrbHMLnhmLa^K%uSN)ZNQD=6S>NS zcsrado_BZJc2O(SLEDaOC*53ho{lxGb~7*87NXI;X0`SN$P9iP(FKRJryulQOJ@dK zOB|IBQN@M(+b_}SO!l3faRtznw}f#_V3;Tq!qSBi7-mdd=zG>d<7i6A1QKK9hO-(J zdoP-hKU_oCZf^7O%^O@s{B`Uv)vA*eWkrS?v*M2{>A*9D(wW<;DlAL~d&dE8mwlrw zmCeWojSc}W6#)0PNsC^-3QZV)D=VLj70l#x=C_d+HuD?$obCuS)ath&Dh!#RkjGzd zyUl4#eJDuqQdrcg_p$1m}Adad6KNMUpiF$PYY9s|I zWAT_xOgEPobd&mst|VhmgGR?>+Vq*`08dhf@VobA5D-t_)y|)iN*Nhw{)OrAb;+`^ zl6@>V+1xIRXT($HNR!pQk)A{I4DY>$_AlNp;g3Nj06Mb=wpyKa@@CJq}*10UyQF@MgDDan12 ztAPn<@Rj?@i=A0gcOC7H>bpOv_V#^h7a75|r1%*| zZMY}t=QZ4)nC3AI*47GwxmHDFcwj8aaisI3$o&68OQPD+Pl~=dA`0>uSbr~L~iVaxTI8f z7(7D(v!J%CICNT73ENfy-G`EO>#zBcTFXUBpEv2H~C zqe-9TsEN+nktHf-+)y8hR>5u3sf4L0XQ`r774ds>#+Fo69K?Y;+#W3-N$LdqW0W({ zJ5h8wq6Q;-di7a;t8tqC@f^nU(75N717+jp{j9~%N4=>AW@ETEmgP}7EmY(<-G^Tv zoKpq1zl9Dy&d4QKNvXP8B42N~`BtF#mFuqs5Hzq)ZdGXMx`Dz2JtN1IJ8}Ay;E_yx z9PfY)HG2lU0WxPlYcd0hVaNAtnA@s0)n8+gXg0l*?&nuQCOPT!1m`axq4|=LFjx{$ z#<@$yRHET`1%VqWT<6tWqD>xexEn)+Nca%*m=zCq6FSYVg+w`FbZ-_?rL*F=_EfX6 zcmld~HHcCv#w|}uN9l(3m8E<51t~XS=vCFXioroq$o2${dA+{eDt<=QGTKCme1wb( z{uH96`g04(DtHsA_2_MC@g5{1jzzY_VzDmLTXxPq`;+~3TeClG+Iameyl%0cXAU|{ z;z4Y1&_yIFP*hjxcm;dGZGKAPYM1ud3vT$G-}w-sTT`?us)#e#G;!CCzYP@C>=qZ$ z3f1DiY6@LQmt)78u#*zH90xTw&@^IE;{I_z%5ux|)?_QjLF;iRw;`&kpBCKER4UV^ zCCXDLRmKIR*zS??Kj>oc3KIA(o1%WBR8jq=QcB4ML|%s@1<0x`$H*&cbXST~A#HHi zC(-t&gdui6+B?KnX~q?pC$f29`4OkOB??K@(P9xu?SeO?#tY;_XpvOcLM|Z*30>#VtQ%2rhit`N+dNFW24ddJbo zGw>9`4@#pSN7Lp;%8~lt)h67mm03hksE)KFk?X`JJMpUXNse2KvTX!TcwzZ%9;F*l zNpS+$bZZintG(!?l_QGHsl%H}D29}#S~MbE%a2*?VaU4ieU&A$^ADLU;gfO)aHR`X&sa>f4bW#yhPia1CDK*)d8&;I)HRtMEgFj2D zZ_s=^w$yqK-Eu}$qwteKcL(mgO6yosglhpZy>Aaq6{YEd@~*n^J^M)ycU!9PaC$Wk zB8V?!c1m@SP}ZyaXxp%&%UtRJP6owHtR687*I(?hbZt4JYj$)gAvd zn$;77TdW+_Pb-#`v9>p84ao`bV|s>}5gk{hEZ?<%Zj-14(}=AI2Cn>u5FU86M90Sn zK*?^=e)Kw>FP2%bkB)D^qx%Ig?$`2kv-@;DPf{4G8V)Iq3!ejtCpBdbk*wpQwa}&x zi}_*Wy?ky;I(C$oirLH5>{gCiWt4y$A3R-0wPnN)OKNrut1&b1klIMtgR*tvwJtpm z)s8s}^x)jF$FD=F{&+07a*FU6CMR@rCbUvkUZ8U}xYB1*)vUVg9YAg_DK_+rhAxb% z!d<4qX7!dv>Y}#Q9jH<_O&NcbMsqmedy8v{GPnGD3+{K#jU(ZoGnP2UQAuB1MA z_g8`6U;?he{U4Jxu{v|{gk(>KPPqLqn7GusGsQKT^x<*Eb(#6$Gs0l;y;D?G_rkzv zF;`9AF&p~jfV6a}8pr$mb}w8%He|4>X3wgX-0t>1v*Rv!d{j)Rufi7ZA-)6L`_nma z=YkdzhMST%NY~Y$oPzMITczlu%0lpm@6hm&g%V>hQ)PL`Z^>d?KJC(jZ=t>CMmVF&m~Ooatu!WA3&JI(zSxr?+x)U zXFd0{EJPYNSE=)1t0`D?!B{l_Nfsk<8m+SMdWG&%rcDz34NjxOnseZrDV!e3N$EojdOA$)XKfAUx^b@U5u>>+%nZLASxjH?#c>O>9O_uMP z`6O-*n7X<>?e)nIbE}*|pFtu5-)FRS&~V%d8h&VnEb)o4uM$_B#bLQB)GL%5J(~bs z;aU)(W`Q=6$wufvfA!Njqcwk8-4C*s83KFQq!5xzUC}=KdDyuo$DTgmzOi$g+DdRF zK6(ohp#b@jDR0JcmEZ4GlZZ`UwI-L(5d5t3WIAW+ZB_lckRf^?9v3B=OqzJ;0A0QO zKH(~;d!|j|M7L0;*XX{eBI^$8cAX#NQ<6VSPtuJKFt;DL^1;RXx;#C7J307zU%86$ zPVLq4c6o0($ob&&_VM&|y#HXU>1l7{{6gpZeDRP^X?d>6iRm^u`EuXI9~tp^vx*UM z0mDZ(re69{Gmrig^rQ0X24>`Z`%#I#{oEX-aX?Nx=2r8Sq)c9|>zTRhqqSMZAzNrG z5$>}e1W0JLMZ9_#Kb=DT(X4dui=Ve2hsO2KNvqL>SFIkNpfUZgh`*Fi2U`A%Q?szbqg zaOHmYv4AFyJCjE5IgA`LWgZ!>EZu7WIg41qe=p8!8lRpc`2z3$5{%TkLghWy)wm_K z@lTT_t~nC`eAcnMH>&_s}~MEsjMXitu1Vu$ft3@n7`4N_*+vt%e- z`W=`kAtaun-Q=Zu>j*+8yL^`X*+pm4EBBjQ^@>(NlP0Nb_`K&p zP|3)@lBogdcuVQiFp7cBIFHq317IE_&&PnzH4stcAtjK zp;~WK?qiqJxyoJ1SYzy36=@TFr|k#$Ax$bOfHgFUsdj5w6?#5r-+Zpqj^TZgxg&bs zU30Edd5ktPb9Z4&zm7KYD|#A}6Uo#7@+oWy!-xA=XpXO4!yWk10!2&>zKyk?r)Sw1k zZZP&zu!FxKgrk;g4!LJ3XlqKLIWS<(;ojIowyHA89pJ2JiX&0q&Gq{66Ebc_`{Fxy z8m~W4UIj`;ENX|-pV+x-1~BfJ?4?oXNs5K#TT?vo9xz!9PBnXADJPlluPETKu`6;A zqm*QzCxOT*g29F^yqzd92OgRJyfV}Uf>7CW{$CkKh1G?&s7AR~MoJ9~hlAe@{o8}k zOs=}(kQU=+_rI@fxWs`bZ9V`odtSA%hRna=U;oq#lMDD|?h930h`s&?$cuuJpYbk1 znI{j%o*}jpTo~eF87ZcJ^YcK%0z_&AB9&`NlCsSGG@2(OCInzrUwZAzo)@?V77oa! zXI$F7BwyGh@Ij7WDCR=m7v?oSXW^hcj_j;;_ymyh@9!}Q-I`EgtNvP zePx5)o(51qKw@D%phV|bDhs!S7S%tBqA0e)4B@FtphMip>; z+2MVe@9tXr=~8Qg6!S5HD+!sq^KGzjYRW zV0+N`obPSVJ`Z7F%5qb53U~+^bOaBVlwCeP+ZU}i#VyIs``Dg65r45;C3my2jTEhu zkr3qk(hjuExj5X>t*WhV2#R5bE`A8JWfu*!nQj=Jy_Sx*Gt z?j$rL`P@g1_BrqWWW=^)9pfjXj;$Nk>jgzq7i=2>fkQ@j6pPeZRs(G6*{(_?$NPwI zV$6XvCe#H|)CzbGKm-va@#J2~0!gT4%9l;Ax`4HihfJHr74IK8Os)>H-ZC3T=8FK= z@GL5T_AQgCeum?XoNN6glZSI*`y|urg7V5?`tJaqIIn5-!5pvYHQp1;iu=DK+?E`2 z9Gc<5FDX$PXE-*D$V~xVrf9$v8z-4XRaQ9;$@Y6s93~wGSAd27^`wpI#r*8*N#^DJ z>~qo)fUcV>(buNn*rF6&MJ?>Po;>m%Q-^$lMrV--ReNht{*K`=ax(=ln`(KT1MFwZx@V6FGu_Rq>N;jyXr9G zl63om{&%q|c54gB3fnq^ZDBzO>8?G%u|E8F5cRFLVSLZAe~ObCUQP2w;7$;>6;7lU zO6d>emYz`u2-qj*(v7Po}nv&l6lcQe^m z3nNUb$~p|MOB)pp$v=t+wZrZ0$e`H?!Ur^#Ts9b4wjeHtL8{!AzF!jG=5x%ZlyPO( zjfI;fu^;9_sP{~81t!cRl0D46IyL?fjc6aHaiuA@D(Tn~fYK9a(3KtNdd3t!T=O?- zv0)&hvI7=NuhWH@Y*SxckRY8vp1{BUDh|rbPE^7wuJqbHd}MEMaA3$ue7QMP-@ibg z-a9qd)5zUGIIv~ER5X5R0fX18fXpTtJX~8I@@!YnF$G|)`(qgOc+=4`hvor*`Vxct z-@My;+wav(dUyoT%_ZBLTyGDcX*GkRhq(vGm8vNn>6?gdp^+=URHi)`wKFW29OXRO zCYqAU(z92%goDo!)~Y@{?%`9GUcX^f$#G{m#m-7zQ7?~->f(@kI03!hUoCAD88qoM zn_o#UksY`EPRM%on^YnciAgNjRxjhZGHv(UCgSH7wsrOGy{7Lss7Nb`!GiG)kgOOqp7I8{;MW{esO3Legi6rQy zyIFP+D-`U(sM#0e+8KLq{c#gIR5yTi)6_|3Z=8xRruD38w|P6yT$ha7hs?DdapL_f z#}f*&M_iSo6*^gUe!#%^6w%OrIy^@2yv!#4nExC26GR0m6%v)hZd5Ag73Vb0Vv~hb zDBPB-FxeIaj;0t#p@XAiT|(FjU+i3x&t#mVbTu>C8c*;nYy9D2)rP_+BQgqPh(u;1+`fE zqTshc0#fG>HxQRSxRoo1gZ>q27uTRCtp>!%7l5pZi&=T_(K|y)a^)H=;OXQ-^H|4QlHRXmBHSrHYrE+s8r0LQhUlco*ymvu2}S4tQSgo1X{WDGpKMqW_mt zhj42F{lZPMlo65@^2F@}>&!XoiZ3w?az~_h8Acj~dbJw$AtUiB%^VF7>$HEF8p-ET z#+^l}el<$Ixlk8SB*Yz|DNr?WPOM_xARggJ_%aDpH7uyroVp$_&GqgJ*ryA>X*QKb zagaiUWfR=FdL{DEnXacsqc}yXA7|ugdc~$ZwjyyEdn292e1&)MQy3*Jf6T_p62Qv1{Bj71fHZ(qWfLM(dXiGjI6ZP?+wy- z+`iLTnHZr8zR3Fb?!ws#J+Ur_bh}J1isQsyOsj8heP{|95f$4Z&+rqTmukuTqVW`R zfmZw*i}Xe^KV0e)Of(-&*Q2`gy-?YCe5}pn4;OZamZi-3?U2(UPvE#F*ndpT*p><*^;_ zfj7HIX38FDO2mybk-|u+P?3a^!BA8SaIU{!qIW;XE>oBvl}0mL`ImG3f=na=^Si@(3*MZ3M>%D!bBHpv5AG48R1_Nw2|8fp)mSV zmv+LVxMGJw%xedsog+l`BZzK>Q3)4z!Vt$y9@udsjB!uwgjfOmfa^;)*GG0jzy(kA z=mFFXp!Lv4{a1HnXaZM^czt~HAe5W-u9pvRxA1Q>B5&F0-Y|j)n4EhXXKQ5=uw1() zpy#>SR|{B565<6{%y4O_r-qT>Uo!!^F@JZlCY}yhV4KukIKvchL*}3MpNRl_18k$N zE)DSdKcjEY`w``(?|d4l0J7{_f&DSfLcK~FY9~E@nXa%@l*vsctvoa?0`9I=1htng zNMgYRr}}Hzo7bdqPw@m!a&7aTAxKmt40p)+Q(Ux)a-X`s!h{p7U_zT|>!-h=G7ljM zRgs2um+zvJudJKrK-Qq9Rv=kTlXykBTE5`M?pxr(?q}G^SkVbpSn8(ZCUMGqrxWQW znIhe~w*rNZb#-5fs%3g=@UBF&vQivsM93_BDk)Lx2MMM>v2dAPpG4e9S(iy_P>le)Rl!beZ&1?DCB5X|m>$=X^4@u;qS=<}yK* zQ%TnBfJ5|Wnlp`Lh-K0w8P+ygT*G18rIspV3uTo4v-kjxh|Xe=_A*UtY^j+ej=Pwf z9-VE>=DzY~+gLS}n0D|>HJv{d>DeNY5p-D^P&B{Li>P>{iyWgEa6GdmIOokKn!p7%U})JITA zKxx2+4Kf;+t8F76x(~QZP1zyvu*$s4?nqHlWsm?ic4MPINHmL#ro^5H@kA-vm;I*q zm*vxpGvv&2nYIP|3b-!sy{$65&E`;LTABr+lr_vmR`H7ex%v$-Cn>0}3X0`kP(yalzEy1+0)7ojYgFf*v~YERIU>r$qx08e2V1_9XnUP8`-Pw z{*9_5eNWisKSwYEFRolQ=5xc8PjsnuqTBbONltCY-@5@V`tqXqMIb(Nkdq{A9(Ed= z3q?F)fM2KzQL4p+3U7_c&BB{c)1*Q-%cJX{%67}Id2l_agq$`H((Zm;7Ihh2Cc|ty ztK)MEAp!URE5wi{r2r9SUwcyPNC2bnYf;8?#f(6*R%w8ag|KMM)DcA`z!L`*C##$` zV$?9Sm(KVRQx^fvUlum3Jo1z|ajXP}SGqAz_th}gN5XwD6QXpd=h%TkCDR-5p?}!x zO0>kB{_Q?M)p;g7tW8csvUJ6j7hYWC&7rTVSn;n3V7a2MYyUONZip%mbomw7_fr{h z>oS$(Na*Fm7JjPDo8Q2xx5=zSk;T&|t!wjrHN4O^rWfRMYKQ| zVGR1Jq#h=M(#zhLr0>iHKrXxwqZB4L^*plKa|u3K7wp#A6F-RLaNZpT#IABSRH5UX z{#A|GxeJ(DYnwI$Y(JsN_b9nhht|R(hPG}(tC-f@f^pQqA7v`h;!8{{l#9%eG32Of z5}+u*M7c9>AagEOK!sClRAZssZy3Ck!S_YJ{1)ZxA!!Jh#&RE-)Ntm@`Mdp+@Vkm@ zS%IZ?w`s)GvSu9dvO7f2Wl=7k#td3{)L^JLXj-DmtMl{lKC+NnLrbGrk6om1rrf-^ zO*ME;-6VKEA#@H2LI%((ba>C>zdt&1<*h$=q|#U}qfEY(oWQ74tKHe3O#IsV>#uL! zJ5l@T!U46lH-S^`a!Es;=t?|^emka&%pC2Y0fCrcch{sPg`xKnhU(CPe=5`t%yPtc&YfjUKoY^uhp$AbX#9-d+p85kW z{nkxJR2e>~#LmlT(}O74;NU4V0q5LD6M8`M*m8e_(m`u47Ty`4_tO0rsZi<7K%ycw zBrtOPtq>KC_YS>>tpupt)qOShwM-UCZ(tV)WWNL~Y%`9bHME;l_ z-4}Om^9vm!4SF=ly7@^yEm}zgs2@x4;Xc@eD2N+Y>QPP5UdW_m97qc7vmt85 z4q<#G=%Vw3gIDG>oG1_jRO1E6RxO{+eiqZurrzZfl@WBq#kDM%>G&xvP$4Rm6uc)K zKJnTXxfU7tm{u2_h7;z|@codG0gG~X@u$2t=B{Kac?;gKXK7A!lVMZsa32f!>_V z1qNYsJCjja(<}n#`p2y}iJRpa+r@=T*Ky!NH1cLkG^j-a%omcZUNA;$*bz$qqe8Pp z!X%?AL!MBZ7J4?{yxrQ<@BL^^>SyTF?u?7v)pUBa=>fN+@6baH&ZN-uZ<49FXiMWC zgsqupeM&-k#&eg~+px8ulbMp_Fbzt-&5R{F!5j-%Rdez+QvywCi*_VqU~ zGe=$rpcdxPtO^9klxK&?exnyOOlkB{R7foU!fNkjstBT6GZZd8js3gA7_-ZGu_e7s zJG|7kg&=?ZIMKU*i?B~sl|j(aq7+H{8|~TEp0;=UL9{_tYw_lfxUFf-=Y^vh=-4Oz zxXRy2&uveXA{{53!Bg&MQak$y;^Rq7c%{R7EzNA^WQ-|P@vI2J;=b>2Z-H=_%NP{k z*FxPj`%9k9WQi$bKYb_;QP-S_8_HZv z@O2*(O!VTbc@qkO?}4FdU%VPne;|^St460b2tVj2k3!Ze>roVw#_j`gcOk~5LIc~Yk%;ul|W33(IJ=oFX>(Y2pcanc!IQ|N=JyLv1LSD98 zZGL8?W#(3NY4~j?w;Qa{42`JP)W~*J01nLpUeLd?=edtHg`mS_&=ht`XI_q`%1;09 zi-!vFk=jEWkG-si)tHl1E1AF$c`Iu09F5Ve;Q$ddX9y=CF)lu-dlr$OGenC8DaG)6 zbX#FyBg9tcf>3|+-H(=>Y4CEH^wSk&9!e|HJ?90Up~lU9k--p;(MMwvHp!}85Y*9})-9XFA&pO^#qhSpju?iQnfpyLDYv-Yb zfPq46Je;rwq8$ea)DCAg=e`S-SX?d=EDH7JKSCt%@I<^4aTkyv?Jh@mQlAD`5=Iqf zf2|>qy0e#@h##1ZkaoB^in;>3Y@(a+3m+OA*V!6esVZP6u=U~}W(e3MT(Kb?|Cho( z7IW3qbQUD72;mKPw-`>Z`i$KhrF(ilx0`{Y^9&?9dQSe*FZ^%VU{ZVpg@(qz%5cso z@(jnyD$DS3KP%+Oe$ZuT*22RF&~A2tykCR;nc;7P2QY;QrLCmAQ3G%aH7-&FbjNz*#^)=fKm{)+trz{3N3dZ(^(sdr@?`*I{>KQhUVwb^>k zq@Jp5)U}Kya}Ti-VPC}v)=R2dNaSd4gOdaJ9>ufc5||$Zy-Yx|Cp_!m?n?zzkFMfj3MbFQ;8g1GYg&&Ro4Rx3^pI_B-gQvk|Pm|2Ch0=#Hs&blr` z`SfhlufhKRmT!Rl8tl=H!Yd#bXpLdFMDOB&2+TyPqU8$|qIMB0$rP3<9$Yuo%B(GJ zvAINxGS#zJ0s~XUae8vA*ijfwW;nlD7IuQYJG|>6R;)ltg53j--zc1`v}i*5Vx86h*bT_DGB}MK zC~~(D9V-P@g+J;U?!mFfAMx7AELX z_$nz6=O6EPrv^fAj`h9N54l{`-*zDK-Y|s)s-RlIA{8DQ9FQQ<7zb%JBJ^r)lZ1=X zNPM?R!0aT&Y%6%@{4SMc9-kXKhCa3Lx%K~H+fqyuHz6W|Klkdxwx73 z>E!bE<~jQb2{mIE37bZ>YFWen58lX3X<*a97E`EOQ0dA~;lG5vk2PzE`+J*!;^aQX zKz^TfD5nbsmTQlYhYh3ISAK7uto4D3du)3`w3|Jt@~*(6Bj(Za{rXFAkn{g9f@7uN zEDYM0GDcG;on(0)ykJu7e!mU(TM<6L)hUA%jll8Z-7Tte4DEb)Y<}KVWvDMwyp+Kc zkP>n-Fk$@|>3*J!C}(#G>c&33xlE>lK~8uWr=K!&L*ZhdNPNru1zm|X+DZZ!;5 zm+AkF-{^?@zxa)Xi75(HF_9d3?IewWE#YL3{tR*mDDPSXqh_5zH(+izEDZ>g-0L5iolbvHE|o z8+`v+?DI^?U=jr~Esx6yDU01y>n5jDP@*E*`%yH$%*Zcvo{)LcahS;a zu)V;V-`{t=uTiTKSi8qw$cjYG?3!yD{|?Lmy@pc$G>pIp=R-DlffITZlNO>wvxu^R zl4jG5Aej#T9f%vPiLT_3mcL<>{*~DOKd6oWB=!v3tb8|8|MF-I^l06B0l?jCKl1T@ zzD*AJ!PRcrnZ;{-0*hl#p7$b$m`xr^2+?lf!e_gs)iX#>%lTKsQ|w11i^D5DeAFXWXSv)x%Ao0_;_$jKzFbeFQ7_mx7+;v?=?ZA%YU3;=Ho4%d)>!t z%56iOW$fn5Y`hikI4`#@NN#;KyToslyGCIR>Dvl4;g=)riWu3fdWfJ-8hmOptyj=J zRAXf}HY#V!h{p&;q#<*{#Q)?OQh=!X7f{7PxnaxhLi}Ur$x@DrVf|zeqDR9+=tx%K z@O*@u{d1Bt>ST^WS?dwS)Of3y7!cmZ&8U}rr+u~23inM0rQ;AN%Ch7XPbXDaoOfzsZ$?79o)XQ0t&c4RE?UoCo@6vQ9ykFVivgMPV z!~HXm6VLpoY&0iFu&5dbzX8nMY$-DVRk+i$<14C1>eRopz|=BDcD}K_?TvI*dDBc~ zQY1_fk&8PtcjPYzeNupK0~XSW465^(PCaM#kO&f|(@oUD*EDD>jb8#I1GSNhqywu` zVnz2d1$&Huc-z~}mSJqAmxP3dG9ZHB+{bi^|C(Jqot4AiN?9%pH1Fgzw2eGl#-tuT z6z0fA0hr2b-9X`cbo%gkX;+?7qcuek5+3*FBLGK>DZ$Wnf*@i^i&;9c{D{~Y1qJ$| zwq}lT*IR{YsPT!xg9n7Ls{|Hi7!I4Qt$0jRwAdZyLF+=o@Yqd2=K!}s`7F83-(+MWGN5#78ynhZtjVxV zMTT7(EfGv2y`dY~3eJ4qXt}C74*{660^J4lKXtu~mY~5J7^!liT|Bz-(Eon~f-8() zd&Q$4rLxF?(euRfwtMoQUIYhi`wX^IDc=7(0+H7HzYqvM?q39=Pw@YXK!maV{~!>7 zm;VO>(S81ZA`t2T{}%!QuzdfEK-5S5-w=qTXGlPm$Rn}{z%Z9#rnjp(*Am-2hQn3- z$d0TfOo;(il^x9MvQ6Us7fV`@Q%Ij57hYp)3k`r2Ta`r>327b_G+R5MEAGx%z7qYv zop4?6#Ht%kH`z@(hnPMtK>I&nvqQ|m7jocu!!F@r3y1~__y97v6>;8{6U?%NPM@#H z4iNh}Zih(HZkcy8^*@18>&iPXq(;*Q9>;~a{iYC582 zrD%*&V3^Ke`9uW-C(mGp4%~27xa|#LpGxIhjf`Gy7@#2FXH%9YqorFR%lZ(0>)DJ9 z#{Z}=4sTTya8%a&H$&dI>!pQE*s4q%zQ#B>m{ZL^nGVE!U(9=*ZDNh=2>bc>}G^x3)x;HGBkW~cg4lwEI|1WkPxT~OuHAhh0ec)e_iV+|{Wl22oMr^}1$Jv06D)XV=XXMWx=6_*cn z!fR48c^%$D%~vZ*3poiRhtXK3Fx=xnC~uksm`004;8sEd9dIQ<$upY1NyD>^kIY12 z0;?`RI(pax(ZIrhQh;O*Iwl4u0u#cti$Ru z0Q_3DOK#cEs>;w)t;S+eozANho3$t3n$4|s4?Y$)jTNkOvbjC-e80}lzvWuQK{LXi6LjDX|so_SOq*)^cqe zZB2KruR4Fkf|SS#lN_^#O2g^xi}#s)3qUMw1BI&(@-2pf#`5Y>_Y|KNGz_#d23`u& z>r4&i*3SKs6xyu3E$|6)kQncp>iWIT_I{``kqsTF0e09J51GBU z0=CjElibElBr>qHQ;t_iWpSCyA~qkL{smE05ZYlPail7G2Z$(2 z(hs<~KxPPWNH2*P$LqFgI6YTAcY395MF#OrSsrQ4#mcciT$vJh z;FcqBhAn!cn>m9x;(?185^yG6S!+RZ>ToM zLW2)yFS>lK?|{N#a#PaT^L~Vv7hIdB(vmCo4|+Uv4JB;RL(7%ru{rKWF!P}gP&Ip| zHJcxVe1^gX=5bpcJQcU@btN~P4a8#0>ikY^whHpATnmp(FA`ueL>W+TekzJ*KBQUJ zfZO>$v$;0V?L5G^Ke5yCJ1!=U42zV2l)Ny#pKxcSv3A|+->_LP0)!-{4uU#T`cq|p z>+s7MN>4omxU@>FcSRgTPO*Z*77c( zl^iRGMaz!O9k{!JI(6SUgC1CL0s6=DX*o5y10;^`)I2(e>c=J~E^u|TzP~<~RN~G! z1TGyWgnT+U-Yfu|kC;6wX3G>fx?9kr2&mL@tif7sR8^0^id>0I&@L|$nq{(_ie|O zmI*}Ru2Z6M=3MT5r=d(bb%~S@L6za`2>Vij3i34|ume6VA^tA-Fc37x8C>Iu5t3R( z6b6#IH;QPO5sG62k0`UfZJrP`#|}*wtC1+jWP?bWIAKb0(4J!Q7;@~LTP@!Dr3(OY ze0Ls&0xENqwQWk&Rt(r_7Trmp7E+Yz!Nu{mUV9)q&v<`etbI__WGm}bi}t$or`g&w zC@||!Fn2!4onaSAwfh~BJCR_-v+hd@HQ?yOI112!%+))kZQMlRAOOte3auzk!HCRO zK3zHFXw9n)DUC@hB^y?N($bvFQ7G~(_MaTGUK@=lOMTmRnsz7S)JBgkZY000Z;#nf z2&-24BjP{uCAK9v9}TGoBIymC47i6kYTA-xpgW(@;E#IWs@99RPgGqMKA92LN0bXI zRe~$h7Fvw|#b75}%`}+^y!K#OMBxmD{|%2#H+S2)xZVR*{mm|RQv^q7(r?$T*F^t$ z&0tN6Drt(rz8&QJusy4%)i#ZIfM@m3UV)WzxQ?FZX{p-n;Ek>ddG#ZPh7s9hPvSZ~ zDwRDR8+0ef?K-=1yjU0sSA|#)Riog^IQ4FwR1rE10*T$KWZ5^yHH>t3!+G*mrQ*Vg zVzobR)^vE8LD{5@WP7;{Ym|n;T}EJ9?)uk0@JinTS1YhKG&b$888^uX>NCl;Yc~@R zr1)bijB}EKHQjGoocw}(9`#vq`mSeov;qdEQt6E-x~gjX95rR*`0pE;&$ROcr}Zfs zvIIqj$LviCot$7E$oMH;2H=7r^V=zJ!s`r_#w@WkNbL*p!Bh@%U?ZR%hQeww#t9L! zmIqKY4C*OcDlTb)X!>MKTYc+m$#|e7F8VNV9bJrc&7E#80TR;?4Y8!>@`bml$qs$i z%EVPt7hNSK*Mx2GF_u~qqv8d{S-okXhkq#-@TR~mh6=MGPBx{< z+$4ngX^ljn$*y8}kr@ESH_kt+s01xoL37kEmJhh-V!;HbBIZ!x*+TxcvxUO*C`U9Y z+;A#N0Qg7b96N*CuH6)CJ2`hIZDInxFh09CG?IX?ygEX?aTjU z1bx7^fKRReW}6I_I32v#axjX@pt5b8zhU!+tS{IQlIx z5WX3-db!S@Q?`qDy*x_M#!=u&)X*+-^27V0-F@!^yy~dfO}(SVt}L5AN~4g))|wIe zzYtw(c|b|O2{-M2KLstF3F%E9&`m{Ts=wswU3X{9fJdz_&j125qhoer*+7y>$X`g8 zSZ;kS5GJEvRK1&w&2cp$2D5E8``|#GM@_OXpjx9!G`-ts4 z)7lO)iQ%N-XdF`WZIKF0NtPnTW=2+}#X{Nj^9+9OFF`YzEAZ!d^XgDYnP9?E67?x| z%lSQwF0IqO`^=a}7s(!RN01~{m);G1`QB?_qR$`9QH2IzT)G z7x8G)`q=i2u!Rs zx5C71l@(MT>3_u>b#a67Cy~rqo3!=Bru)h zL0jHps%P{~a(s?AQK!M%t^>vQW^m1EcO8=jKRktnX@1MSYZC)zMg#uG`9s}}Y)E(9 zHT5Hsk&Y}?o#V1+MZ_cFtJsttZI8PAOi34ZOM0WPeh69h22-wUZ5Nu*j$~_$de15b zTnS+LtN6U;>5p(JF33PsS%1DulIQ?}Yg0^MA`^bk@`U+n{IER_#xmX3lltIvv$`uJzCh4d zo+oHl9^;xAqvjUoP!SxXJC<|}+{_wi?p{?jja3S(cz*rG}^ z`Lo1+RG}^vy_lVw56+r@BY!MAw3J39z@G$EkM1pqW;i zV?f<%bzCpV+L8_#KPU3XWR&!6Ns0+tChKNDKLPHu0R z1WYi9TT*IJ3-cpVRK&xVhP>nO&ppd*;*u!Zm=U$WPH|eIP7j%H5E;e6EDaK!^|rWM zzOg|Jd78RPno?!W5O`xQ*ZJl%#&47m&RM!Qiy2>V*&G+DR?)V(P4JdIt^U?SUa4hP z)J?MR2|h03ifgYbH%qn$1&HWv*xAiUO3xqzX9{X)sZwn;)819Atdwk__=SOyxp;~^ zpHyW-y;!x!-LrV47r=5)XvAv#mp0!xT7^6Orp3fyt!3zjp7Nx+WqdFY{(^e zRf&5Hbrv1qBf34XsgMpmj)W{Xl<9t09G6wIOdeuvz)=>~(Y-WDhF9a(Jqg!$L04}motUs@8*$`;aEA~cZJ2>+QzsGQF;jG1kB6AjxIU9+0~BFtGOlTdn}RKh1FbTW zr`lV<_+v%}>NPU%@A4#H`#?yR-!~DgrR<$a zV_n&X$MGan6~B&tzV02McWrb&+kD8OS$*l?H>e3Vq}wj1=HUE%k)=)9!2LBI9A@ z_eaQ8rM7RErHRjLU)SHS0eDvRp$c4|)_WY8xcn3``C8jm5i;Z-zv+-r*bhy)Eu~}< z{A2-j*}s92b2*Oz6vvMpk~f~dP9!aUYIJj2lfUzUOA}xlUgMM8j#JGlOGU`&L13nv zC?BqV*4p2(mQM~NYuwMSM~?iFDw(kFe+{6NAHKkA@}l>3#=MaCWw(qXnuH?YfXJ6z zl&MHn(P+maLy-U)-BP*iyUs=fS4JqT>U!rbb5%E5Tc4OxiH{6E;<6js$x;qW(ostt ze_x`F`1ftq zWVEZ8?We?C)bIW?zjrEuuM?M?FPiu@=Fj>ZHLAqEsR?f%)r_mIa*nDDg!>wMssxF66)5`b0!mqr(tpk#Nxg$fb>&Y`iK zwU^1zd1i<3v!n0Io^s{c!a8~scbNOn%}pt!nYNL%*mPU1H+Vm|>d@%7MfTdLmH3y^#PqV zr0Xz36_n8TmzPIyF%=1Zt|^3rsXTU!J=!;2#6M)lbYW|v-tM2iA&DC0fMC%-`;;NQB1BkVgBl|fX zW$vsl^Ex2vcK9pqP4oc+669zS`_9LQyvjS-;-M!9o|yQELWk&dc)VP#!D`8|WfhrH)SGFllRB2@&HwmW$2yW(QERAifj8O&Xkeik= zRMmzOR$@EYGB(6lLAw9Z=pf>;L6X8AX$5jC09GqZ=OPuQ6W^F* ze^v+gD1)UWmF4o}#s{t0Df*ty(3$GXQ?GUK6m-P;N7f!tyT5){E3-%qa?oW3?fT)1 zhbM)c0o=o%T?cCOP{wq|QlAi%mStMWY{5Uv&~wEWq%HZP)a!+&qx`< z6#wMqKP9vq8Dt_Mz^n83daoGXE$?yA74`0WYPJBx1Q`B?Tbsj0xP;5JU3udQ?dulH z*&=w)^TlT9`|M~UG<5I--Rp)kAa`~o&Gfl4T@g|QT`oRl9>dhpXGADzbeNc@c@OLq ze5F!vEhTwE^(x#O7TT?LO@AXn0kGK9&xCT^0g`vmOIa2=m;2edPhB2a^x7pMC;f4lI$oUv?c`a+GV8KH6I?hh^qakZxT z`;b?U?7l$MXc5yhFZIiy3iOUbL@%B-r+S!SWQKGUVewb2B z;~A}SALdF$Ry+2!V^_|$rTp642KXDqgqbBa5xP=1QWVy(N-&}@3A zsl_xzW0q7`>{x(p?nS0XAakvME5(n3#MbT1>U~Aw&0EJiRdUGOk?j|}p4@z3H)OFu zf3y_was8t3MZamKQy)j` z+lZ2zaB~d;T$s{Kl;&MmS}uJ$Cb5dGhREo4?5fcXJb+jMuN4fgO{iC~8(Qd3)}LOd z66>wLH+Yt8gNs8hFj`oTe!R$Oi|+1zjq`1PC@t7;+|*N7W5>}gweHrU%1%W9i3)pN zx%;)jdpr+9%GnpSIB;uJC69wM=L3?oR)wXrhddcO2!piSeSP%Qb2IMPb5JMSG7Sw+ z#*g}^Rmf@?sO&-m%%j`lYTDe+2&CeXe9Yh0I5kv2vGhOB9`1UI^ahViV*81mh(SaW z&`N<+0Aof;Uqz?4_TusTi^oh!bFwlNt0Wpn7%t0i6}vP=YAx`%^mXI65^-tF zw9|!RI$+MC9n{haj7g4$`T|Pry-5iBIv`G*cLg{Wtq9yBkW-(T5QM!vrd;c{a^rU~BSfJ_}4qnac#$D$)*X`N%Lt`3}R z=GoLqdHGNMU|%a<#lM}cbZ$G)N4AB?0r^2 z0kMEpr6*28C%mV@%t zi*g9>S9lIrARG3O`?23U?Sdl)2F?u7Fg!;I|c(^CRTrvjj2pUAtZB1rM1e^A7X3`g}DyP zFp<3wrdsPgkcKKlG>sj7&wQ=pruWruKo3(cQARf&7~=Gtq2F!mN$p^u26XZmf2Dry zs~t9^<+x*;WJA-{*{KPY3^y+r9IdV$EJr<(CKZMeYIYC%D#%br**sJ9PN3@`249q? zTLy*_>ToaGcT!_PgawWvn@*RtwAMcT85su&j38xVHi@1G)KvqE;lx!XQ6o~_DndXn z$gD3Sf-GePLvmbvVNQetzr@-U4-XCm!DY716ujEf2p2nk#t$ptZPYYqi@u2{c@dM1 zY1CSXDUhmrwgF3vmJk0VJ}mROmz@r-*O-59H&`o)Yi#?e{ZB8$%V%%h|kW18U1_32HmQy7gXgVWqLB zNhScG*5_H~plWc1zjtXPh{dVn0cs0EMQYci_JR;ZfX6ol`=(8f@}X^cD=W-2QZf|l zT}!-=MeRwtuW2bUpje~-Ur8rgY+M>&ibH)nwM%9GoS*Q2)T;%lD?JX$MK&C&IN@)| zMU<~uG#j@C|LNmKYAsIfSm+!aNrZwe$2_yfQM+{>L?uDl0id;k_h;-vvg-nDW9x)fsSOKarfXpO(@zuH~DH;@7rXeo;)y!PG`$-puE@Ma)|3n!G6OyLYPd<*nOqd)j3p_OGlc z>Wh9uA^uZphv-oZH@_L&^m)4xgBBg=S`XM6O;reLk6LX!eN*u<9EekrBJpGi?NIAf zqpsM(-Pxf6>U}z98y`v)yuyyEM+y+UpZBD6?t*p{(9`tWj*bBQ#$^3lH0>WxT3lp3 zh1+m6v*ut_*>GQ|?&+48;mb?%BPBJkDvnkc_T~m%TTgox{NRDsdL%}C+Wgi~`w<6> zWldRzst|MgsuO+H-Wk|`J+<)Zvq%IhP}?mB;~k~MKs0c2N1cZRG$oV?`3#4t(PfGk zNr01`hhH;jZ+-3>^5ly!R2N|$4N=~UoCg$t4iiu#k=eHh@`U!L32)2DtwIyD&@@qm z?(%2EO;T_0LZ!||MhUMqSknB>nB^lHt*gC}kByib>SsxMd}OMQo{E%tH7rudA+~Tt zfs3<~#=Ng2|MZW~6&T0&5@g)sc2)N6MyW^9Pc=$Ixg9(#cT`=u@x~0{*jEB(L^*o| z*4m>3ps>ey!5xO=4iB05nT)DCVq!3TsWLI=qP;33;sDMwQn$cxgzi)kU$PwqD&iJx5L1M+Nxw&n9{;2Cqeh#9U7Jw~cYz}7auVmt z-rlBd^n<~kQ)FQP2+0bOOblhjZ*`zA)k^6r9Kn8SQBj+NK}51J~UQZ*cRh?=a4? zO0XfW|F&W_MP zA0;EC?#J`h(DEcK<-mJ7mP-`iBrN)HljP?kbqwj=?wD`aP0F7shM`FS7jee8mF~Ws zbd&U5JxaRfTdh&&d_F$JGP>)!SIOeb$0H1Ssz zplk}~JL2GcY(9p!-#I09T6l_x5{3I1bf0Zw$-S)6XX+;G7ZW#l18-K{!)0&NvpTtX zz2Dxw6pnk;M?dP~4>TI+SvLfkZ$JayjBbT~DJ*YkW6qE+Wa*hLTV_dnsrK(f9!ha> za}Mlrq~sWNZDcpWXalQ7xTI_^hLCMNOx=@Q9`^?8)XIXMa|~NeSVxTXwT_j&_6PQm zhpB#@7|muPzcKE_aURjZx6n%|23AgVwV~ zV4+b$pOFaTS|hp>-Wr6yr2nyX&N4RUhj6w~$y&eed5&^8DXld1CYxJq79A;0M6?+& z?g8jwt0bamFpY5yo=IAA7)hyPb^a&oJdn@kxy9p2`ip;))_!pLeW+8eNpo!2(MOMI z{0`Fol=BJpo!FQS*tjA@_LXPXafM51?b7Fb)%h$w+B{s*IjO{yis?IoBNv)8-zgR0 z>J^YczqAo=BvSlN)jGPSveCI0rA-?8F?qxuQs_c)uLmu@hqT=Bb`E@4ShHQ=Q07Z9 z+*)j(%!+oVn(g&{26~fYB=L5j)uJ083amzc-* zppQq!S{#uwNw>6%XsQr~EDdvyy{#mrg_XV~^Bl6gJ-d#j;VEdjkV7iJw)=*?uhZs1 zSir;icsGA8u{eQ@C}c7mxDtHKFq6aU z8?y!VeZc!2C?-rcranCJr28wwJcvXik9<8jyHJ%Y)y0VJYPO?x8J%|W5c;mUU`>^F zEV-8LGsh82t50wNxqi#}z6~v;Zv_&o!85bX23q$@)iU0R4G1hL8j!h8K4c^pymf@K- z+hdjLoD&WS`K1mb)oJj+dDvOZuPfYk?~E_^6|7;Y3O}8Y|6}TZ+lY4>B#3+Jn#1sro8EcO^*EQe)<2X)D-$_sXJk zuSqQfUVc0A7{nbaBflG1pOG^Ew!Inb+V#sk^ZQ_l@;hz0dPPQj ziFzW?D?N@<5@JUZf2>16&QdauHJL`=aymQ27+Z!Ubd;iqU|sI0HSNo*zRM|p`omkA zGJ30~Y7D;X(Kz9W-`-S)OwJW7v2kO4U^SGLMNIH{18&*IL(%4JXJ@*7F;%>LH;x8I zOOVxZm;XJN-&eSgWQ>s)lKHlbJn;3z08Gk4_>}E=P35pFvyn8JXN#t>E$h@t(9GrQ ziE~6I@t!8s`;*5OY3yo{NTOezO$)+vj>0bkYwci)^Dc5GTc(N1B2e)ew~=~5AE_J3 zGaXMCySoU(uQ(*XZDy4lR4koylrwnV=mecCBa-;;3}3OTC(0&PO*71A-c`TStYRYUlreJ|S;2=jCT7|**CIV~8-&f=X46*IP=m3XN z{{}|!88us5H%{x06IHvz>Cx5Xo7MB?dvnzKDE7O$2dL;*d6rgqxnh_Bo-jP; zA+qdB#L0T(W>||wRCf-x4v>!6u>JFe3WpO-Qij3O{+9O7qcAN#RmCaWAB_Y4qm{Y4aC@#HK*|K=%Upr_~aXZz<~&)O+s|i zDo1CJTu4giJgZJ_v}ua&=zmp40vyJPSJ6ga&x!|8O^6nHy`Yw3$iGUDXJzf3r?br# zOS&(dgj8HYt*mE}IyG(VD8Q9wUDkSv`D-|ylh)l$b8VyXiX5OK;Opnv@q5nbzzQ0X zi&;fo)eMEUf}#GLlnc7(AB-qaNO;KysnD|(^?sHc!YNb!L;52qXHkGHWQt1K2Pm~rQ3%+cY zkPm9Jcwuo0Drq|Ee>Zfn+q}P!M~y@spntro{=Lz{Zo!Yjg*W1qg;9J63)3RxKx;N( z6ocZ;98gj^8yqi9pZ|pdSQt;(R>>_;RWhEYBoBFC-GG;`Af>Wn-lQ?PxpIeC=l6|b zU^j2u(Bi0Wchyz#Sm|bPuH#?#52T{Tz?uU-^KaydYyLAoOc7nR2tYZ9DQr!Q)9Oys zXFoFN&Y{_VgM+#PXy0Z>+NXe3b$)2X%*nvPl6+C3oVbTqkU>JdMd-C*HcXW$c|IN>TlIC=y|_s%J$=*Dgj7s%r&lY!9c+-3Jhr2#9C!_r>Fl z?k6Yr$NS^s2wmvU_X1jpFJ2#?kL&Z(TFrGV-So7qobTuJ6&sxE3g29VuCI~tx7qjm z)1#Qf*Fs2iW|sHY25MlnhI3LG0h&-lvW@`8UP1H3LRO{c%6vq1Y{flirq|^(;EQ-p zjBW1U>P6Gc2jn+sKb($yE)rE|?`4fZ)(kDiFw}yw)N`>zt$m&5o?wG zy>143pK=>1*krlBfQ@&k!DR28O{-C1p}aCEqoMKQ71fz?U);+aB;np_B6)4^5? zeXJ&%m_tp4Av4u~D)>yMeGW;Jq*OlA4u?ebR-8?y22B+IfZCLt z$4~w)KT=?E*KwrktCF~+x#$pT$X_Wu5b1f;S!mRYTk#NT_IdFHELi#G#$ulTIObu8 z0%IqJOZ;Q#0YtMlWJ`k3GZqFEL)Gu*7=0Lf)6VG$vG$>(fg!Q>YTIkTd@5tkRuUDr zVDGFycNba1&SB(r#HhmGMn{8BXDZniP6L^Xgnr5enTa=Qjbp0D5AcJ^lt&Su%L9?c zb9G-OM;OSVp&Cj>1u7e%tx)zuE$;yu0=MQ zo|L53i4v9nMy&577F4K+Pz>FA$vD6)R+0ekbOl@WcDU_ED$UG7>Pa@3xn_}`Vx`@4 zKsW*jjVY`>C{*Y4Wtb&`Q}M(!2zof7@oz7)Hc2fFbZYOA@zD}XLpjxFFEFa1j7H1M z){1H@26N<_UMQ)^Np=CzRJ0*1HEH4li;12yt85R+pS?+efJ);AH?*4ASqXwO%;FlU z$=;bsM9vKm)Y|&qhNiB+I_k=;+`9BJhs?o^Zf|4g%B{#t#jp9L0#qeDI> z;Cc7j#>Cmukh%&bOCHcxTCr9AO8tNk{WBb4ulaQg?jNwQn9aPRzT0pa zx-_kv*s;A(en@Rg&QBqkpzk~00G|c~tnXLlQf8^wOwPY`h%!Qcwb|dqo{@!*0AeDh zwfhrsIWoZ@iaPW>F4#16W|jnyY`M4}mt9PC8$4UtT0-cg0deIJ67gVIfntNyB_(hv z%>7qD#moVc*-uDdg+X}tV#M?MRFpt#KbnQ$g`Ow+$Laf}KuBC9(7?BMoP%@MZ~As& zDfvc@en(j)Wl8xNbV>|wHE0{N_B8XC)B_<)uh;k%~&7U3Kopl0=9wSBlW`Bw5jG8RC209SdVA+ zPwO8)Yhtle)WugT1eQq6KvrIjNDKpF0!4#5`tS>74#ptG1c@~NWMrn`)MSr1@wgzS zGU2q|?p`zMS0a6s3@J<6P|5*y%bchJqmZ>EH&N5117w%8mTf?IHT{t)?rgJzRLWH# z!zy-9WHsS|8*Y1kT>dunDVM2-IHX{u%G(Ma2bRtusF@!#2TDDKKx~)_Z_X0{ItiZNx@re0)k;azxM^Q0P@MVh!^Bkxj z`S@Zo=K#MZ*4#cg#NtwXzzt8M6Bo4PN%5F>A4V)2$)rj}#prjWt(^LZc6Ug6j7{7I zos%Xw9I17{4^@YXV>4%Zo5-fnjy+*rR$KRVFoqSBq&n8jh@=ra1&OfUR62lh&E?XY zDHP5D$9d8~;(2RbjU1tqJ~m*Se*RG1A)#!;22w6?YFai#OYutkmzJW)CoVWDuN}nB z#jWcqQA1~f42<0dWoi<#ymwT~hVq^Y4Tk0F=WGKZ3+Hp~5uy&6zkOBjD>Ix}VCWVX z-qAGgt#xO4(;|5?-hg|)BMfDr!MhrwAkwYQh;%AnvzCF-ujMd`%fSMjY{Btlwl4bS z;}Bf8EFdnGwt83M>tW?R?RsONTB&8-kj*&|jJajF#K~dw+|!MsSXK&r{Ul2=ce`3Q z;uXkJVb?*?*2@)hUlx4WDioc|?6xXAuI8>wm^EZ9G|@zrhz%~6cLiraov`M8amA12 z#wJ#3f^+YW`qIl+8IK$K)Q4wm9aT9;hq zCq%VW*3Z4JH71CYirRXtRi-T6^>Q9JBbV`?2{JQL`D~NdR{_<%u*wcgeR6P9wyWvf zFi02KjWMyIS&N4T3`(|vKwTSb%>7PwS{dxMCHFA0`;Di3TYI0WYNSO%N-o zqYOhWQl??@WSDS*kc3g0T}u2Su>54mu3m<^on*#m;rOaKQ=Y%tcF0u(^v3H@p;5TR zTY<&i9pD5Qb*l$Ski$B<3=+~E+-9>+K>%(uefkRYj08$U3kzY>`k{uW*42UEjjJAY zl>C(`83AW<{dH9C?V4O5g$_f^bf*Gw_`UMJ^)-OeUPr4k_ecaOJYaqzKy_k8QC?@8_N0cJTu(+fLW2t+V^8qp{uN&BXlFOJE2DL z&qw!%P)vK)A5PI3MO*`*bEQNs0JX&EU$(96~!%4Y%Juk4_v| z?u*oG9roo-nA0MvlL2=@+2k`ZYB_7`;&h<@?w7xe$O5msehZY#UlgBo%D1JQ`5oc* zf8vbJtV?fJZkm#pPAzy5+K0LGvGP#dEr0&Z zPebUawUeHT*cZGhyD7T;2y05`ePfG%_Jbk*PF~dJp2ar{_n;(lX{I&RNv@%TJx~-g zxCoJ()~jjz3k+A-zAnwX7|^#E{RsP~;A;dPHl)WGVlUaCwZWAcP?H3QXUqj@Tm6>g z3UUrg5(H-_eSq!T`Ux8*Z*F5M+X9;Ae9w9fnP7a!ajCO}8Y$x>-m*$xGtm;J7vp~@ z;JrHW9&d{V(u(>jiS@0nA(PESQfO2|03*S5N-xhtDrc=zU_g~lwrU67Rtvz3a%q{tup_54c9 zm;7o4;Rwe_q2jpFFl0(0Ey#-lQv!ik){&BjwT;4#MSkVUY>kynwBKhrME;pr!r27t z)*!n$4(u|K%5`#eI1H;ChEewLKyeJ99AvnEx)Xz91 z0JB3+4Xl`X)k2}T)knozxP_X@G!6VVZCYGdR7?wnWX?uzgc;#Z_MS_syUo*zXWxi| zrv~OY{q9T`M|^B7@BR?DpwqC8OAVxcS9_4%U!a-(86jVpPqQJ>Y;6k%@ee{}L7_3~a>PO#de;|OE8x*qp6e_oq-Q_ju zvApW~;6r)8A^!9{{c2E5=8kk)m6028$~hPz?bV{w(@!KebS7vt%bGa5i(!3NK(r>A z+9QIsJ${f8FCYd=KezTp*W3>T^>}kCZj0P_1cu;r+Of(=p6&4c+TeqEnehv#Wly!S z8YxVJ^=<2gZ%mwbNBhO0&z9##=ax7vEx}shZGfiXy*FSkE|o1EtiDVP72c8HZpLph z%S(KkA2gIyNDe_XaYfcC@hc{*KDcyq9((}dvqbh03_NL9_o)_@_N(JQd`XlDkVR|K5;4k+F?{H1*ahip==n(mt{ki zYE0FEBz+AKbVVp2VW}5VtFY@*J`q!x2O01%Y?yozpWd7gO?tZ#Q2XfZ;{9f`@1A0p z41#Dy#@Jc7En6`%2`af5$d3WI3uIhT_ODQCVzJ$xpl^I4+sj;fk}CzJJG5-Q=OJc4 zW8KFtmz_X?npw;}P_do!@Zz@40iz+ZPTOi91FqI+fJWzo!1Ys%I@RguzUPc{p7a?V&+2i3S{d$S#5oJS?>9wFM zzYbd{0U0U}x8SRJ<|&Jt71k%-H+GFn$vBS^!*#ILJWLyZGTy8s1&_9LhvWe_k`+o- zjLR82xK>(x{G|NkW-ZeZ+Tz9w-R3t(EfZhIQSzzBB%#ol=e4A!fYk?Jm&(ffTriwH z^tF%ON1-6yPuV|B+m#Nmq1tVQeVf6ptU0F58^JNm#lut<~c#t2m;6Me? znx-<$Ck?kz)|$`Js#9c6M)~-~cULjy%ckRIV*GW02KHvs&hVI5%IC**i$4NAO z`ZxgO^vM?g!Yd}dcMGy{fGf|Zo)r)O@?Z7^#_?&Ku<+0r)2#lIP-W~2+gVFkubI6- zhcpTvN+RdM9US+P7Jk@oDc1h8OaJqvOGw55J3s+0+he$9fHyMw^j9~q0^pdfFOB$A z+4t^KLSAHA^fc=Og`U$-pJ~sYG-&Lgeo7)A+Lxu_{YlF^%=j`J*D*VOxbWG8?(33BvZdsnn-Q0*YXC9>eJL{143X2b*uW5=1P`-bWu5BOC>DZk^+ypn#VPY zx$bf>cj6!;2$}{*k(Jtet#iKz+~osouGBy=)E0VPT|0?dDFw$Z0C9Bl1^diJ~NHXN%k zcmlu7)mMZ>wSKt85o!aIiaI<3O1fVKj*@b7GyO9$z<1QQ zV#xPgYx9?eE6=4(H<8FTGXcu?Uc~sw-*yl+&YFoSTiTjPb+ z8V1vQwuK<&+QFppks0#i?n~N+lcnw8+D|B*d@`*t&UJwo{z$dc^fARa$}P7xN9!!> zxxas$7mq5shE85O6|}Z#K~W*Sd_PueyB%F|Ky7QUE>|XN)6yFnbHwVTa!x4Z~x4IAhKpvX7Yu2$@vGy%^AT>X*Ye;ql|oCk&m& zL8D0epp6GrsO#!&^()64V(-N-;b;{LKNRz1X8dtck43irs=V+SYf_-+-CTiMKe05R zJvnvQdYSa!;$vS*Ro~Nabk9+Sof7F>q5tuz#2d=2&q7C0?ji!+DWlMD*A76HFWT8n zprl0wrwW?tN=U*$6^92#F>Cj&uD;p+tEUToVUQu$J}Y`5q5* zIkm+e;#g(dN<}Fx<1;x6Q*b`-YS@~@IBlgGem&8TU#z^A&|Yl$RcBeXho5U3H^34{UDi-Am5JK4`&cen6qBk}Yf|RY zSc~RNmUgo;DIzR<^DOy-^Y8uoTMNw)bptd$x7TWXh1%YVc|JAb#?-JZ=)+gNsvfb{cDLCDUb zQH)bGT&PN5HQyR=jpmhExs`EQ_=lvjr<{EyGp2`0)KO7KOJ9fxIPVqc@!Col582z`n{gUyW^r7N%BMuT%<0LQ^Buhdd3=!tXMb_p! zW?o9Lb2Mp43V0X`wgLt9yqN0bv-46Au;5C+oQ6Ihy#=R+MT(?Le%zMO6yBbR!^Be7 zoAZ~%Bva=s-gM(qo!!^@kX)IU6^onPA}X!KXK@x{QOnirjrK;zcGs;&Io?7xzQAt{ zOs~y;oIor$fq+AG6l|H8Mb|pS3`D=9G!yqUfNU97i?v8AB;9TqAnM>&_=4xajdg<- zb)Bml^pqcCHh0MSuJG-q*HB?@ahzzt0qrRvDKW+2gM1XYyyBs+6t_q+z1(f|zY>?a6 z?l+g!Iad0~ji$rCC^Q==TCEAe=W|Q0z#aWHB*zl;SUQ%O%)ml@4}6@V0EzIaTK$y9 zNQfNq1cc{oibw!Z(}R9(-@!g(gh%mcnjPg6`WUbDB}YP$lHC*_S*RrH;sq1f`g)>H zp)PNrXoNvSLf&tu=ID>9Qi|c$uEr#BY81w)Ktdt1GEE~m$&3Yayu9%bqB{RoFy{aw zgUy2FBtr#j=XOv>gb(U>!cHimjj5r5alv-y5=1MgnaYqyco6iIEK8 zC3Tj9@rm3~%d0Ohsm+ftJfN7Ew_vS;(*rQ4)oqep7I@esobg?J7KMVbyA}mWJ>nWg z)kx9gg_w8F#}$G@nKa>g@lQUN_Pzh+rtbER7iAk8-B9;#pKA=bxJhd(`O;;?xG-LL z^Z16OXUsW!h>(>izMBDxt?(8rJ&jnS+uOfLt{|zd)(FyEn9HQ|{;BCMikJC0?Tuy5 z9P^dx?4~f$7xtM@$^FHXSRA`=bnxB3<*P{wB6a%)y5Y1*nTZbQ6cR&!chlusBl#;eAC|tOF03HjX$@NZBK6@ z?`%Zj>olWC1n*KC9sb|_DSo^3!3an%zEYWKL+x8E`EA_{1}1|J=c~S#2VqMu&3_Sl z;$irJ!>kTM^<65c{L>aao2o|k26;aqI8`L3m=BD@*RL9NF2BQsdcIg!K3;>re$% zy4WYKw&lOgNk+Ftk4G;S4wKuqw1Rk8bmgX2lpd^qp4Yfoa;RkWm7}_LIRn4*sWQF^wI}}$N~iBIU$I(=%OQ@W>LLv4Zf*Y@ObzAK?DW#s zf0AhmDl`*)*U^lPjAl)_WDJg{%&TFvdwcKDJ36wQq8bure2C2Lj7sr1&PLMgnNLTFpWpJxnW5O! zn4X-KJ*7o1Huy_$_gAOG@{e9*L42{10Jbz63pIYF2+h=J}>dyPex zH-mOz6&1lL=_cj2wg%A}3QgFZZiZ5*4zCNIe{mKoH7EiVUz@y~b|YPurfq5Tq!UdQ zawJ5M&VpuCPE}av9|Vk5oExNMyoy3@D+qV^h;!4wSo7cs%P&cm!(hyhI+l)7`rsd+ z3D>u_By7N3ZD}AYJ!YJv??s2}^z78V`McDXvP%;hDuY_881|$*)FZ)TF_zhgEJ6r8URR4H`f1j3qyhra0haBEXnLxABB)bs+xS1@szt+GTIm0(? z;S3tppOIY{Lxvim)t|gIU?F5@x)zariX>l-M*uic+# zp)?la$dY7Hhmb~M9W2?TY~5%1(!?)X$uXqJdH627iWODgB3K7Q`(+H?# z=I7wa`_?Dzhm7=AhfF|9Vn`*hm_-EU5d{Bf21%O*B<1tsBRotb2}6Tr)`X}n^lP3~)hTmC8B;+_mw zU&Ym8UK}!E5tATkxHu6+Pw`KwqPf~Q8rEu#hW@Buh|Sj zcOjXK%!~lHQnNR1QmSPe+<6j2(59%X?uay;=E?6-ZOBem`07hGPfI0w*65KH6LDB7 zUP}{GG>y@#7-BBX*ZPOggxP}(D+d7Z#`cBN7s_r@F=%=df-*AYIJxqFA<)|Kc7OEb z_Yjy_8nt)6l^JEtrq2i@_X(6v$2ck*(&M<54G%L4BRzgbR`n!$K%2ifrX@RA%K2F2 zF=i=+6-r!C>{3tsnXZ4x?W~Uv&&4l&cJrHYnt92&@NbbbAEDm|Sgv1?_0!^1VIJz==!KtsL@v7H+b;~^S0 zG#6=Te-k`y8{Ze-opzavLi!%6g>V`N8kV1y5GS)3C>^v~Hh|7NQQ1siU(-fLBK#z2Lb7+Jae7jF^)R910<=tCE zu5SGimDU=UJvSt1#EzJ~hJ!iHgBGlVrh>$61*X5p>uNP_5m8rdzgUSnk|sKxHYmtN zrPk9?6DH6rAaG9Jb{GrW#i`j$z_>j=-gv&bh;ovVpW?J^5)6EhibM*;$^y%U2jz?= z3=kAh)NM6neK6n?RpsGVD6s>a)i=k0{D{I}I`V#`+g?c6jl!%- zMCG*9MRXJ2oBXo%06cSg$}qf^1sq;@eUH?5O4dm-dR{c#n|E=K?1cmjNjYH63qtyh zv1W5i3fK+o>E-gYzNGFU5i&Ql9n3=S2<{lBsU2}c9ZE5C%MF>je(S@SV9qOs)2(%x zg(5Eo{v(g}egK5b$;LWnW_V@mT@I;h$(f$Iyw`f2#p}Zl5|q_&iKm&)B#Wh=bh2u*lBT3uV+G0P`G)0n%@-R; z{Ifs7{C$X5EXv4*i0uwbA(z#+kt@?0Sk!gI-T3$j0Z?C)!ji)JDh|}R66Z4E2<;1r z2i)yqSJ=L)#ty+?PY0S4IACs!m|3giWBse(#}Eo~8;c00?&16tbR2hBD&ssVlk zVyoOO{?tv*MNRmqr>giLVwdcbwz9JdZ1GSXK&paKuwy5wVnwc28;<6~MZrU^(|ffV z_o0m31sG?9l#cLgcoQ`PV_+Hwk^be$PnOF%5c-_4Ef7YYbQJ4K>7(sm?E{VRC{R5r zm}4Gqi2qdQo1~1ifWT;l&Lfn;vQbMg10x{6%L(E4vqn)WQqkuEMiNYm7_(g0yh>ks z1KaSWw2V#L3ixAzemlxh+s}#`DylP5?LORYYrD8=7DnYaUT;zidaj6suE-ZV(7682 zm;HI!#+6vQ$P49BsOS)IlBl&A+7OtlbR~!7g4=%^FBN^UB*Xt}-kWZjuM(+^UuEXs zGqg8srSi=q7R4ENxb^24XQg*&ONPwnD7PzyXF`91LY2LJY@Gea$ZLXan1dR@#@R(1 zY+Qw9iEof0ZgDc%MO1BNyE!@3b8DL%Ix@_;_Uwz7k@$u4Jz#FkzX)e9;G17qTjU7tw>Ql}EsfZOY3x^Tw@HBc)3kl*YAv(IU;M%C zEZpJYA|hN(6X$=zd_mm+fiu&?$vbNFFI+2?#%c<7Wy5o=JwDAv1v;i^00&vrp36eX zvcl1E7|ssZIgN|qOj^__^|_XYy1^aCf}4glhV>TJkq2_QFYcXv{ap=D?Bc-85_$Fd zz$`0Wz?C&y6{^%3r34Cc+Bu&fZB$3ar-!N059(qqRR?W2w-h*4RG`y{UppUe@RDi@P z=16@wM$nF}$XQZPC5SSKx+BwWZn0VhrfSC#8}!PT8{cZaeDO zXKZRZ7x`4>cG=B&U0C=3sK2{zVIu+ZFSt&z$8weY$126X-24`|H1Ucr z$7{WkozXlq)Ubl$Rr}}1`UgU0Dq6Ods<3Q17^*Xr5a;Wz=&l#!}(kZldnBV zw86ofmuNjh`1Aooi`O7$2L0*$0Ylg4=`AW+#~ZsjcA)y{`|&MG%Ash+bn{DP<%G#c zm^PX>$Ns>xX2w+5o7b>m<@p|Cqae{*)QF~H&W(&y()^MMlg%DHl0#-xB;hup?9aEj zy4dz!vde*YZ4-B0dP{4EEbPcp)I<_u3O2M_30AC9UQ=DKVuC8(YK80>D0&_gvA7_V zg~UXidk@Z17w>4Z^rs-6ZGy}v%)!J+WG|>i@?H@^^TN=hA-WPAbO#wF#BT|c2?gR| zPgb(I=^nx5mdItiJxU7{kaDc$2@aZa4oJN(tku1?JYdOVHgRUrA0!S940U5TApR+2 z;X7x+6ZV2K)l}_>R0QU$nD}_b@1#+S7JO~wfCD2@BYMRHW%tG>nUgD}$jHh}MH4T^ z*&HQ;HV5~HXMulzKH#B+e>%6i;8+V0p{zM6ab0Xp0%iO8i1#%h&jC$NV|DWb!Un@q zQtFr?oCTl289B*v**Z7xdOmTd+$auDDcQ$$`#i5)K&2{}5rY{6iUf7N!yU#YmttDD z?2y0qUc^3J5S+(lN!gF7`XBRl{tMb_% zAfL^mAihEOBzbkp!z2%`6V%fzUYvgEJ{&73{cGi=+dS1~vd#{?`8!>U-(344p`^wB z3xYPT87yNIx;Rci&vSroBrx8&t{-)7{44qNfP|gip*C7MH|n}m-W60q7blV@RER} zuH6BJm~q5bc^>Yyp)Wi`KN6%W2Qpo%1i?V*;7*F5pIqd94m`uSBLiFij>@ITLTdKH ztZMr<&E!TzJt+OqX@8PJgFa#IOXD30m?`5OCjNV_09z@yj+`TQ5IhIgC%k?%pOg^h zO&xXJ@}VV$SsRl35X6q9_|iCm!6LWdAiLEKeJEiBY9HIm3~I&XJwdJQVqMs+!c)B6 zYHw_xZ^}NchweeJdTWXh6Dyvq8MT0V?sT9;K5YJ#;`QsrpUcUU-#`K6n-q;`QcKRT z31HEuyN`MxByxCs6$AIuFn6aNK6~r9h^%N?Dd66Um*kCy^B{P^$CjCy7Ri z(pB)=yp!;A&|t*V-g*6*q~@1NG^P% zvEAz?sXHe+Eo9DT9jiF(bK)y{(*@?tk`oS-m*5_l-FttYaJn5m0ThO6Tva z5k~zDQN*;PP7)DmKmB5j$a4_zRhdhF_NzonceyPI91N~YowQ?pvG8k2yUTy+iqJ~U?C8cfe3=3DJepdF==gWi*8`Cj$^zpLjerB zts|qhz`-@5pwA$=+g)F<(puz@b~Kdj@DaFa59~- z%hWAvuo=Hrb)U+V zV#$kT%}mRWTi1B~nOkNR8jb^!=_vrRv1*9MkfK9~D*8?Pavx*nT;#>~t!XCoQR|Xb z^+Okzsw>1Y(vH8j8-t9sDh2>jlU2`VFJ`-~jIdY6QcEB6>Bf3F-ncNnM^zccJ(j#Db=w=@yvHsb^P0|2kJt~{!KedGs<-)* z(wfaXv^c$y8TcH=jGK! z6l0q3m34&*u3sGsIAqonOInVF&4rS-QDmmR#pmF{Airk8XI-T|yZP!Ly5uum1+2ES zE6QQyRMbL!PlArL@+fu~Io_Vh$Fy|tt-+fNBt{A(T6sais#uvvZV1?1K!y~g=_itg z0cq+K(up*!`;0bmbL;8gJeEG{|M2vdvY!E2fx~~%(l1smvfbh)%o<}yJ2&ozKk;Vh zJFbE8SgpTa`#|trZpT2bR0hMW`1itWe2MD~@Z7U`bwzz{@gfY3#FR;UN*`@AB-?z9 z-BvUHLrg97szfmqi#iq|{e|4ZPuUPaO8e|d#V)K=LXqoP>lzg56p87SCo#?ll6ea2 zLzQh7(z(&akMjD=OpIgJiD z{;>_1G0wZ0#v^2xy6`H}=kll6wgaM1-w_hpI=)$7CkTnd_>Zhbl$Xe`lWp+DAk4GZ zT{LlMN=5C~c|nuAP{iZ3-3H~Sjg=%Yt$O>_sz&Az5q-c^0~^GjESn2zyNXZkam z^iW9a;cJ0Km?=Q1cX$)csEg|ecmSS6iJ5H0>boVf#dg`XAr1AtdRY-Y9QdU04H<#8 zB!H<`r&XDjov6 zM^T3lxis!*Rq#<~n)VL&%Jzr+Co%R3P0K3uLo?Rj(PTJhmg~jtAafn&9+qy__o0r+ z$1N1?p9!bVJiiS(XSJw@m7U#AiebJMBiJ9bW$-_p)K0}GY<5|nC1-P9s4)lLUEfE0 zd)MCHlKqNX1yvVq@8a(`O!TF#x7o0Kjp18NfBO~I^=@=ROkp$6mpZw>6}q_-;IMjr zy*2k^hcHuTp7moE1#gbGNN^>qR@`QWjoYDjZg_F=KEkft-n9X~NDOtyrm{HBFQFy7 z6t7aWF!BNd^zxXeTOPQ$=lRc zIkqirVnSe{Vtfp_H9W7gK1)XY5*>*PU2QbvOpBtl?cG0ypTTPub23%BwKh2-`Tz>H zq{gUnu)o(}h$w%2dwsXvE9gc=HeM3tCocI57=+zBT1VM#Mh`^s>Z@m`6a1!l)|fj@ zzH0)L`p6eX&v1%h%>b|N}Wuk1V5Qq1DyEnq*L}s7B|VXos+^JO;iz2X}f+8pFNWv1I#ao z?sD=V6KpnN8O($S%@eQj1;yG_&*4zFF9|>|4=I@95fCo-Z_)qPa=LDejh~+|w^nQ( z^sCiHTr}gMIsnZd7r|VVzICuTp3eWGf)Dp^Gs}xwaB}3^tV??|l#^3CuGZt4G>bt& zo@FbH5QNvTE8bws(N>#z`(eG^xG0u8(r7|4apH930^H~*?G9w!fRDaMFf>#^gDq0B zzR`;`*IJXqx)cm_XMmHk)*G>Lc|02K%05@}FI5$N;=sM%w+f<#r|MGo*x^TiIjDs> z>yyxFa&GL3)8A7n%wYGhoN&}{E+EDr{u&s70a_}E$6K}Sc#mO~nkVLk$Kw0T%QfZB zVxg3%05NQ09f9e2XIXFG>9;IPJbs&M6dhmeDRrv5P2Q)p)?VNPbvDs&Q8{nQEf~`} zHZd^i$!`EmlN*DS(*re?p6wD16)h`XO%tvPrYDqo*xI^Y#k+Z;3WO#CMqV;%)R9tV zY!k10h4B1NEB)X`9EoHu!|^N46(sS5E*tYg?DFqC5>fh@1PgH;06R^g`pUU5rEG$A z84B+_g8atLvrXbav|ZFu9p%2`T{#IzO9gvHlVL3U3ntVcMQKWrd>Ue&ygq3b32eY{ zRBUKU#WWoPE7`f;puxN4+3J@Cw|@la1uWriyNLg4Pwmro?htZPSng(EWb%HU+FOVb(*Sw1n0-Lp23Je*sm>d8m zWmF2)Lo0YcXV&Nu%O(XkN%hDx0>^&k@CGaY(M-(*A)cV{T@HCmYhKqWnE~UsW&W2^ z>m}D`>L^Y-A#xtrKo39Pe%Y_ZM6(%OY~>y=(X&rIkv%*th z;fuPPIxJIsi{5;<0j6H<;Qw|GwdOnRR>Az(wLFzIf$-Xz1a6?2h&eSESxV*zXn zh1)vIPW!e|rMnIyki;wXZt$Pna`ZYP3E~OqI2-@ZVU(4FH$rk$<6^)?tehhWiYjTA()G-3>W-V2rUnaF8yF|R zikptSuZh8^Ar9UjhH;wirkX^EE2&2v0_#BKYF()_mNZY4qMd}QdQ}MX$f}}GVmgIx z((+JsPFP`=X?bQ!1A*47B+YuXg|Uu#+Eos>RxNSnY1XZkOq-jyN~)x=oYjk3Fr~U4 z_f)+%u|C@N)EBk4KD3)yZLV!rCON$++!S0dUU|_^UPxhYqj?5thG*AXHrIPL=bmkD zOt%ASFq$ewaCD>FlXozhXgrr)7&vvnW;J#IwpmP2?9vqn8r)YD<;>e=I^g^Iw9$_k zMuJg$ehQq0<3xWVA*F*CNy}iugL5>870sOv@ zk)KpdsxdwTV^{+nYgcfiV;Oz;cw#3j3G9djW&$IQYO84|6-SOBoh~yo^ z3O7V{FVvKe4Nm!;f+faUCqUXC(%vQ2!M%Bh%`^JXF1yJBqwLHWO@ zoXkpH4)m??kK|&{@SintFJ0MJzkK)M44VYRlyM}gjPNbUr>prkeJnw)vVlKI)a3ez zPBz5*jLJp!Gt;k!QQ~2-2(g-woPo_5e5i#!Y6{>fv}A#5Jp5*~WT_HPm%B>(JU0j6 z8vAauU*NJoC%!WlnIr%K#+JaQZ9MpxskOlf4?^gKm51>tt% zSk!u>;Bj{z;wb8$MtPG$j%8}T5}GIbf>xlt%1;~xXJz-^12eun;t=p<;~~jU$CT3} zhSSpZqALhO6N6H?B4gg9QFCMhLluvs3DPlU?1;7?>8t)s)7?z(L%^6Wjc%M$umHHe zi8}>HgFf+BjCskO{AYY9M)CJqdd{;Y`_W!+SkgHV8 zWyqiVv#+FeJlRD{ldN^eC-Zuv6|&-z@$0gBKD6>n0Cq$N6I)KtV(9N@js}GHizq)3 z#KByiap}4I{mOWgBgTV}jMdD~_Rj^w{xPeC&DA0&(JdfHP{yD~E(Gxu6m zd(~c%PrTu!n4DHCI9fg!Ldv~dq0~~Bm$5Odv{)XCE#`Nc#rbY7eG;q&vsTW_;}W{u zK-}&`V23s{?a);cxu3Z2zNG12jW#3zjy>t`qbbI_!CO*6v4u;6uYvTZd^WFRvs}hfIW%>lm=Y?MX*C79RnZ7QJZKiMM?9!@lQMWzK(a-C@ zHMtQs^vuo8)5u&^Nh>1K7JTulEA|VTow7zToLbW=D4C_z_+EZSgUTJU^Y#;eL&I1l zpEpSf8m7XVUY$K=tRf?o*S)eBmY28lw92|RSC_Zl^tLxTo`-)5ob7ioRGS$XJ^!k( zF)(Vat=rUCyPSew0$dr9V!Xaia|FC%_~-5=9N5bMO>aoh#}7bK*#w`z>nwrZA|k+p zPR!T@FKW_$BuAj*B<@NP?D=EfW;qPd`=6^-(m$cQ?Vd-T(p!I`ozzzQ26aTyW&T;MsbT@bBQC$6yZEf?*QYG`& zSCYqOxAWSb6?+34OhEHpyA#=H0nnYD=+=7{23$k7ZS?tQp52Hc`Dok${dFiW^{brZ zPK%=}t@cdnoc~6*8=aq<-3&bbt*{Z}TfN<5ZIb!kQQN!;d;Z_rt$?P?CA`5u)4#o1 z{~urWTiMP^f!F``e*R=v0nDjw*6|1IAkeDaiQ|Wy8hO%XdX^LZ-=WcIFa8V3t6++; zoW5>XDVC^~H0n(n-*k9}FC!K@iIuj{8tX5#CKYJRAI&{}L<|VrUvpP1_e{sUBrm>9 zB9*SlL^!-(ki$nfm1L^bg-qj|nH$mEG&lUN0rZu*n`zq0FhXe)7>sm^D5$#PT)F-) zy!-)&E^a*Plq8yQzZ0zUwB3wdTl?31;%AEl{q`Vn^hO@}j^RKQgHaph$>~=TD54KDNFVvQ(UTf9{7ZT- zB5@s-`6TT9q=VCrZ<1xFHMYrWFz@|x+oN~zm6KKp(jsqjjKiOa)nyFHHAwGTJJ^LF zz;lS@paNSJVUv#Y?0};twm9XqbqcO|f(_jXcT79e00 zI}Yek43DS6z4V!O31!*KhQroz-qXhddtRy;A&5VltX`IKP&T9MS?DwQJX{{Y2D#R^ zXwZ2#~f+wf?1)UR%jNK9{~@N~8EbamrAv>xZ|ejqr^ zDJZnm!S%{-%8)2Y&igi@Fbl46n=%Bokfo`oz zCQ9mh2I3G}5$ZP&=!s@Zj`^e$5d&1}9!0amd0!NwtB|J_-z8E$@3)5vyZYak0`h)u zTpw`ue!nacdgdI*8VJY3ZARiy zS<}s&yE83Kv*#{lCSQea9CrVm799A){B3zxg501=DY{(l_@J0t8ojlkUFafEV0nG3*y^bM5?`MzE?z56H^f2<@F|+G3T;;1Z!4D))Cq5SVqu zX96!9xnoA=B?v*T<(jd?{u%qA`Ecq*VR{{WksY5KgnGxX-t&Mzw!K{+)9Jv@Ac9mc zq4PsZ=QKX2eiR<-FS}o4Dw6aoM2^S6aps8KreIRms*sZ^;0ULV7>AgaUBUunG5eQ; zDw0B>#mV^}rc>fcB(pq0K~S8>V@1@UAJ3oz5Dl!Wad)P$#37bS5k&*SO6+*)K~mxr zP5HhPKI=s>9nav(K7QBI+g!iV$&@ux6&Co*mbr<8 zQd|rSj!nkUldDHE^H9@T_Y24a0^1t(_i)8y-MH_Ej6JmVaXnZJ+;4QdtvDB{(j8`$ zg7yuZ%DWa7_fEy1&&9`AQp1C-=jN~j*u(_T^0EjmY9`bH>PIz}bNe%ST_dOqSK3S|8vdmZ?$Z9H zxDVMVN5rqT^htQ$VAwH;3ka=dl?E$~I6^IM>59jQT4Gv`2z=v($NRGe8<9A`A?B_X z93JAwq>U*k6HJ3DA_K*36e1@enmORx@t$T>m#A=VbOg+?DldvRv?Rg>(flZ`Vv*4d zv~d_R^W6iHvJ)3MWifjOF{a>fr!`Z6BIWa@_0`o*pg0ATjd195^XBjSJP@fnVo$-C z&H?u0S>eDP>08b}EoiT%H=tRtWKF{5=V~C{l28?R$}2}EtH0ie3N0UI{ANN1!{1VBo?(<> zA)W(UtO7PQ9ri2_AEN@jE&#R+}jzEYKXRwAJSP6SE%ZP-hDusX+iFkUDM zII>_t9DOAzYr)E@Eu^yw?q_=Z#(l%%UMh?~=`H6E;%L#V%jEq{nh|{YmQD(Ml$Rq8 z7uG_BAV#pKr+CJ(2S`5ET&zHHJ?EUdq*N5gViHpbkOWbp#8%lY8=-NShD=fh_Hlwu z-CXgYtZ~TGJkZTw{C)qsF`02ZVkL&zc<5rRLG2|ho|T~iJo=p!kFdfRXpqH0-%U5J6k#nhFhL8 zywR-q0hFk027NNRn9TIrqx)UfSY)u8j538(zF=#O`7{a^)C9o=#S8As8f;7@a1Q4GAnv zK0lVur%xx<2*l__N3ZI|g(82JQ_=#Z?g$d`#TpbDTxq1A>M_EVNNy-Gt7b0#bFecw zkeQN<2hrr5h2Nx9)UY$O^7yV1U^2{Lyih7BuN}tabDb(q2$*%wKmSfqK5MWR{ZiA; zxnM^ZbrV3eljgIHUaI(}y^<1m5=I1Ye1EJ<^7l6!A>B|+#6%;+q=vd)%cFKIz%gUk za>`r_+{Q%*eOCq;mBa%z)8it_n5#mx_3lG~I<+}UJTZ|QO3M0drROv{N(DYPdq2;6 zWA;O@1HIt3b+`0C4!*)<2+y&axqjNI1Q1@PUugg73DbWJPFaXQ(5&Xk^H>QLyiZ5K zJA1piK1CbdLKRu--DD>>6PHA`lioQuU>Jm2WG=|4bI;a@RXAr3LoQ%SzX?rvU<$n4 z*K6x!R3MY$l-W#|dM<%K`E}1sw_m^vZzUq5V7xeOugvsE_+{|A5JCFBlJu%*jbi)B zNJ+|R_V*}8PipAEjdX6+P%t|_{k80gm&umjvm#j_C=5=wQ_XAd+8(4il^1D9WX0K~ zAnQ-K>9M;l6CtDn^3SRGQ69!<2HfqkRq7jXB`}j`S;lEXb#-Pj7vtL_sXv80MgZ~u z8@_Cx6Gu9>T;*V!k6-*{zxDa}2p^Xs9Ys9mDogRoOT#ME*uJEQ#=7^1S47TBGBH8U z#Rs)xt00jbVQX!HOgq~;Z;e3oqntCNiL}UBFqMo&CtSkwQ$ucf2EY|zm@R>d$w_{QT-6*XV#ssizZV= zxWgGX%PUW{FuVt6WBIAg>=B=2>S5$uPo5Nrqc+y4e zwo@*5*D;kl>nbYmX)>7aN)3Os;v-RTxjS;fj;9AV z@dRqQ?a-?>&ut@n&k3#Dl@WP()-CNed&>=-87qT}Z&nfiitE*__d+J!F*8#=!>3pV zijeY4&JJGR1bX>gI8ETBN)LoJB_vBB$GrCcTq?YaMFYN4Ihl8Vxh9?@=B zXD3_@($p5b@OqZTJ-Ih_=19{*^R#zqIl6Kjyro#z6?e`6T$)$Wbdk!lw8xZrgnR#5 zm_2(sQ&+9s)u%U$9u0;BuHtQA?eZOcY($DR zt`D&gAMoc~VPUntj$BNgIE3bbOgRy|wTNyKXr}EIrEPRXB(j&>r;?rgmLH>H_MhB> zhkD7aMe(%K5%v-9TY09JVyNn;Uu|S@!q%r~UdWATzXT&C2_p@O2}rN{V^se)jTe)2-j@`szlwIM?IEUTuh{Wmql*VK zm*4 zRjQOn3Kv!5LQG5|(IcXNU{Wqk(?x^-IF-zT2tpIA&=Y1LRY?oYp;bbZy2kift{i-B z21}p6=jJ6!6%Z#)#N4A1CrM|Z>R0gLY$~v1ObIsz9E%d`S4f8JvhRwQe%sRCi*r1d z)EA|}qoj1f-xcGs_8tDx{Xe;pwk@%%oarG$ws1!Ag@mXU9`Z=yWQDy+81_zQn*y7} zwyg;>RUsu?D_OnyD4M-_s8QC3iGpOuWS>~uUU?(Ju$s24S6J(eftLUs5bv=Ys$ipf z&?3?%;O_Jx_Cie%1z~K^(S1W#qY#s;}!j(ouuhJ^xXt((*ZoAjl;0Nz1Dk zFJ^NHJd1u)rUD?74kBlsW)^5}sNn4AJJRI7NLi~OXW|ktbWJjrqyun(q_UZjG7La@ z(44RP801Jk%_^{ft?T_UFIRdXp(j?tQ3finmCyyX zlZ|U*Yl&T17$Aj&y%{&{iz~(XFlKxfb8&Kp;D*u2cb^4baXp>QLn%kAGB?{5VZ;1a zmV>;7P0*k~Am8GFqAvHl&U4Ypvj0czNVLpP7tD4f|7Bvi*1Qu1C)Sdc0OHw5sL)38 zS0D4AE!x8dh>wa-dW(wd(?tk4TMIM4V!sB8JJ^Cu%bgvU9T1c|$Rjcy?l7!Zz1^gf zoCU=*Y=2=Thk2-H5Jl~`&b+61jmJT(|BcD=1Z z*gDd|qD--?QD&*w3kNe|kspK%eoj31WiMG&-T~Ax%X0VI)Y3G5>#QuizOVV&WGz2Sb2Z+&MTr~b1Hf{5W&#)ui@jLpTrRMHMa&V^yKrLUe2Jc1L(yy?OJ|6rI-I_05}1f`KlNHYx(9`*-z7lBfz47 zEnPJp|35eTqSgNFQle+U3ju$XRKI0{Hp+)r67F9OO8+&8& zO2<#N)Zdxr6h}!!FvaM8xAI6MKre=VHqu-v&?hsiBGj7WcNS<|WTCib^p?c&=r4#jZgs=y zz={s@znPV%aDS@Ox(%cpKrlPB8M79LE|;Qc4%C7s}W*9w0LDHs~E_g_l{Mbl~#$%UrdgV#LL1@#7p z=E{!}Wcyrdkwxkj(8_>1S&UD0pn*UT)fAFUHM$%)h=b>3duqo2gvSNSqzXF$~m=c12NAk}0pHVtWwSs#YG zvg@?;AzbK`=&`` zW?0amYA8@NVQfV_7`W!xyyv0AsXC-i<6q=Zn(^Vv$V66h-TeuAJIEYkz9IOFNTEB+ zG{-JaPP#q^#_~7WW`oUxM@$RAgCH+0o!E-NAXX@jqfI~TlL(cVg^FB<3G>p3qa16P3*QQG;za<@-WjgZk@R+)tlGRh|39o)KW-ryLei;NcsZT z1*dsOpt@}3+=Tj8i}t`R%v7Zh%7P02DgqYFNw7e_6$CsF3uKUw>eg|#`0Rp=oEKA} zH_^oqcl~p}9-3(i3gc_u!hU?M4~s^-nQV z^lxO(aP%w7v^AK<=o~v0n9Kx=Hv8D@coHC}OPWtXV7sRfP|QtV)9$AdiOr&{aa|Xn zR)<~$et5oxU8xmFg_bUF3oXH1HE~g24o85B=PB9uK!6D$Lj6EmY ze;oL8feN{MIz7j|_ile50F;2HFOMk+Yr^ZDKpyFDokaMe_&@meoxx%aB!SJT09w)H zF;dgu9e_VtP*PB0y<=yX(0aK=~a!Gy|%u>^3j z2pjHuv%*OJ&~!gX=Z{^_zV0f+)-F_*_Fm-N;~iG)F}Z<3;i_c00~g_*j&L}tM|Bib zLkTV|ThmVNd4~2X@XK=orN~gW8ai-B11hnv;s^p-)}=6Mv0ac~$OH!sX)uG}G+JF2 z3AcFcw2z05HsDk3IgR-F6LLr?tqX}B_MiB>5oM#5S^V}IRS8V5_z4u zEdI{p+}0@_n8MCoV!7Ql&@Qqx^ufeNcHZ+Hv`?h$4|YS<^6l{kE%D;cCtu5nze(b) z^kfO=*B{i>Jbwy+(_l&stI#y3;hF_wk4U7Zao~S%@v37KsMsIlKbUg0(X{2j0hBYX zGt+>1wqs(L!5rW4CNZu#HEQ`PUs~Lc4XQ~RuJnNsF++*qjf{ls!E{ezB19f!{fU)D zk}ib(NIbMI2Gn1Uk72;vt?mQTQDDnj>$Jz~>N_93GllSSK1Rkfpt^@)gIUJA--D{7 zXxr*SzC5OuZOQcOt$>i}2a=y4h29*Kc}3TyIbeXsLkOXzpk)voB(BKc(cNz8EVQ?5 zy$N;Zs8fQ_Q8BITAHp5!kuk-S z5ReWeQ^u$GNRFnwM`>48NN^7=oT5pNKA#iA^}^q;TPd$Z(Deyw-@q~(6aJ}e*L6c| zVZP}-NEtvRcI8+3u7rkyRwOsCj(?AxHvRk=|(sZO1 z^BQ0!u@6fA&hQiRX%K*dMEW9iNfh%2**!HlPw21%p3aAhlUkWdhrlWch0PmYiJf-c zr`zwjZiAjN5rFP!kF!V67Goi>eQC}Bb=>l?bx~xWGgLnotQ2_U0s`MZ$oMrvZ!DD< zd@>x%$b<;NSmwQj_O1NQY!8Op2&bc&?maqh`zg@>6B@+f6`{GCZDivG-xidw&wLXo zGk^&A*L@9y-|G)Uc13HQSyZEdEc&_(_;|u}Bj?7ugZZ_03P=7epshb;bK;@Q`RLRZ zcqpSjCr_PWj4oh}AyUHD@f`KF@6|F*Gj~<)@GDji?~j_w$=N5 zKbH!6Dr+o+7F~EQN86m0G$qBbR$WGH%y)M29ShM zS!Jir{}zeJ#k-*Cv*k0k^LK`7h7l)KQjIpt_jl3Fv zigwFDqIw>;vg{P4VzNpDTbQ{0rKN%NlRJR;^{EolZW1=zG+p1rR>c}r5#e-8t@ zTf*wK>f2E&sRe4PBUoO!dOoUQ_)j$S=~584&i2OGJv1=>W~m=;_R5@EAEccim^;Qx z1k~5X+|j40{k5-0UI9OT$-Z04Id;kW!eiN2TB(gzU+j8?CjcSjrM63TvXO!`RXNNd zEha9P8)@xf`Yow@-R%@dA2*fwV@ybGXQpDlN4F2?6LE}Ar> zn$t-}qtr8dvWu`{Ao$Ep`(z z&r+S`8{jzolly9#QeWv5t)iHjW&#I)?IJ*gmmNZFJGtdW<6vwD03)$M?EXzm?Sn9B z>0h;!u3PbyDeOxpl}|KtnizcLQ+q>KAzO>8joz?4t*ZwkvibX*s{fO12oEKB>Z{mf z;G&k0VLe?VWB13)8FgM)`LZ<$y<0wFGj(ABoH#@r(6)` zY|LL0NV%bE4t zUqM-ahOU7wPQ8)7N?j7SHjMq8_|c@GHug@JFsDCeNe_M>;L3;vuTWg_u*$l|B~6Qa zda*{r%5(iTsczAsb9fHyu=83l08|X$$#SRrE3mt}6}`JWCP4HqS3aPoY=>|0NZK)* z^4b_O>6(o9;1ZpCN})es6HT#>vTgLckz8sk#JjxHx4gD)C-fPtw8V{MEq>d(q>^8U z;ob_M+cm$1uw50HETts#M>=PaiH%-20*oqp$7Yk@k_l8RZm>#TkqM&^j2t&T*f!MR zkzE9QbV{_at#8Ain3@#jJ=>1vg~z{E4+OPl2^6c@I$XUd3feqte#m19?vP-r=g|ow*GguIY5&nU|$Pt zN@L$*CCC?u@8^0bNTsK@9zo?RH+TaEo6>nn^Aw+7^ zp638S%#fQA$ug^Z{q!_~g$myd2JhkRF6Ytg-89N;dtKikb}=~Ii-i$ zoN=!p=(j`3R)}d&@h%p}V$#mnYz5`!0RQgl_?`K-OM*N+ivPlf=H#ApkzT)6nLe>t zBf{9Ehoi6~CW7`UJA(JX@IOf;MTQ21A<>FC|;LK;nefK*I!fet$sq)~LDr#*8Df zs65!q4@SPksf8TvpJd~4RTl9EBKDLEx~9vmo*u_Y^8$(gRhiN)zz={4TkkJLS0`LdJL^}=+(X|4Fg3T<_Vc(q;O z!&|T0Cs$Z=y=d68;X;XHylo{0TTVkcI7PVog@G>blO&+bj|Q}DTB$wCj~)N4Os_faW8afc?Dz1to8U|X<0LFp$3r$ zB~oQ510{fdco|Me8y^t-PJ{v!+dbQQ$awrN%+5t2lX2|h06xMvMVkiNx3kL<=zV)P zm^Uq7hMk*4*Wh%)D~PU3-$6&xlEd0NlT9~g6cly9*n1>!piq*)JDhZq#u)8SL1f{# zS(&iPq)ny7Tnxooq_V56yox;jm|2@R*C3MYpz}~N55;|~W~NJ$HQ$(ZS(i*x9Ge+&g><_1|N8jmU=W7xR_JT5(wFGHnet+tkq5 zXMx>v@-*$Y#r}C^{)cjD3bkWqTjt8@DAh3hl!;w#in!Tn;^yq*b~*RkoStSMl@-8n?N0L zw#_-4OtJeqU{g;R9}9AO3h-6zNGvh0w}V2Zl0hdl`N|dqHBfsR*cS&VKk=9ERpkR; zhz0{xk5iJ2B_3d7yN7~7MRi=1<`%apNCoOUAW=-MvmIpe)&vI4Qlv)Dw>DfxXh!&Hz~9@+l^yZ0>Rx=PVb(WLO`;h|=m)U5kx95&IY@`rPa)!RDwRRr z>k5)|TNI_EWiMP)VhwBv+@A^`_te0&mynnGYOlc5p0u}>q~?g$^l3)PD}Yup9(?Rz zEF+d>8spUTddtz{uV*smmm_uBHM~C7;lVuk>QB*J?5swp)W;7?7#3M@b zjCtw-jI$GQ+LzT&z8Qn&U>=mz)rLHU3oy*SCe`ZZLH=B^%)>!!6&U`iL^A5$HKJ$I zyPc=*hUv+!?qmUU{G$%h1;(`W`>MII{zxmw4W7Y%0m#1m{K-OuGPsH(U~aZfs<2ye zl~u>OAv<>bB+*c|-Tnv^CXRt%GBd`A7bK9PlY^rKy7$fDv42AN4~>snw0e(9THQ0X zs!5+S2eFcx=c9C$n(oSbGqU+Z8?tzlg$&vJ6%{$c7Vw+kwR^%C3<^XRj|0mOT$|#W z`Fa_K_*b4kdis#2X>v^Szkg5N1_`FqWEdp?H(YMe0%s(_Ah+ZNHR=*LjLEEU=pa$r zr6NW2aDDtaghA?)XlcpKR-ht^u{E0#a#3)?kI&59(f2Ue)_q*NS56N7`N}^oP4_M_ zJDPpG4P=naL1(_Q$+>7d+|EMX;ar>z(P~)Tqj$iaiW50qRgMa|fSCtP)bkW#i;)rR zoJig{uH;%t*Yn+G;-^P5=i=sx zm%1^osx{v&#DZ*k@jpk`)1H=np7yPE&P-*-c3j`DPA`5a}!i`nJY{Rb49_ zBgGdh=Sm`8HrPN?p6MrA9u40cihcQYB0ZF0ngH7|!!L8tH)hZSwvV4+u+KUDQ^X~6 zFHkjTEV8rowJ~#SDs|w0OznEH`AY2NGA5STjY@x%I%hjC^b9!#&meobsNUv}trV*M z>6U_*s*^l6MKm6K2v)Z5n^B=B1Y!%8AV2tMp>?~jS&MC&?qqRNwz)s|(!$Q!Od2QG zT##&zqb$4-npE=)9z9Z?E)$rRE)VyR2~lMrcrVFM0hKZoE`|KF>R)7SZzYWY@hI>& z*RB;Z9M;uw9LtmSw5wq9t#wT10Ww#vlQPki{2KLZ?j>MXll;eaFX~$&WpI9ZdNamg9t(Am_bEfv&tlOrAE}_(MtC_BF($?Fqe{vHRu~o8U0=1<~|z zxkU!1oEznKlxCx;h=y;bcZhh*v2tHqbdI*k!@gjRYynyN^ntz$myzE(#EK=OFKb3+ z&N$XtjB&zZnIn)=OcY$mocFPj+4d8V?^7@K!iK%&aSl?v9^Aqwnf%EI#f82FaJ<*g zs_X&RXuR;SCjecUdLXsZ7`c=Ymi?LV*<^7{I47e?$9-0@I0#RUDn_4HZ3dRKhDdh2u``|iph@Qf z^hIS}b-T{_1FXudth%!wS)p3$Kd2Hhbq(uvhvTiG%i<#R%aXa`JSzxFB|5hqc`;OJ zW-{f+MqFTs)0_@h5ZF%D$Xyas;$Qe^>icAo={RVEEwo@E-)LD3<_)2_Km>0J{6Te@ z<${vbc@_-dlvnMkmGw1c8}cHYW8{sLIB@I1Qpx2mooesXbU-45E!Qhdn^iHG6mY~y z9JXTCSo6*T1e}}tW36dr4Igrm75(Dat1hCp*DW=txIuk$!nM*xu;bL$Y1A8!ttj?z z`QuL?4zx6d2x&8}&O}&k9>TUEa_2%?a<)+cxO?(i%W=ogzIR}*R1mX}6$(LEn2Bd{ zmnrkNpzK>^g~Ln-!?~a$vjwEJ0<(-soUf6PF1ux`YERVky#YBsUA#hZ;J6tnvWl8J6DEbY2!K;tB+H92*H>h|FE|X-vTKH{fcVT!myz_uPN(yrMyuI`19B+O% z)V=2w2mjZ+?fr(pL!!ikj{m*RAX#TBm;?B%j`S~^gI`WYg>o7Pgb;R!2jzby)LCk+ zO|5AN8C1d6EqShpe{sBpNBJinp{Lsmow>r_l0-0ekjbDv3;c|Udm0*Wl}X{r`X3!5(v!Mmb{EfzvesGQ4gAh!;CKdxwcx zYp&N9pk^pFN1O4tAm*lg_h=`#dz=RSZ(cpST6#;64A;d3_#+J5YWe3(u`o39+xSe% zsk|e*O=CbpG3%>v@5ABM8i;S0m$3+HZt7@20M8fbN3ZV=&=qm`X7(YV@+1NQ2Sbc@ zRW(&4XktGZxRzYi(f-hpaEGKM4|knT!xm)YY9w98K>?mr>U6;76hcR?+1TGzt`&um6M4bH1wX9@dgPV|IG&Ywi@L&7jtI%k2Yc@f@{dOI2dU zGnx9Qv%dYTxVK`zRv+~(X36mMYG`Bi9P15s&TUIw2z&lfKmU=jG^VOEa2lWU&@Wj%VZEGQy`VNzxAo_oujJk6i+#8(EI|wu~_p%9my1lS) zeHR;%NJ#AZqVg?eJ2|bGy|-}e6pMA0A3k6pe)iw{Pj0jJR+Z`|iXttC0C;rVb_X@B zQP7AkqK9Fwk1X(cb7))Le(E-r{(Ge#x7vHrCDm66_)++J|MBSEsCUnM!Cm$=240AV zUps53S0?90AN*%ItXaeCWC~C^tTCJRpi0jAVew1(SOW!Ic$T}y;6yRO8+c3|7UZrIllUVIoa zu6An`)~97uib$gVf{{o6?}R{7MU?tbqvK~bZJ>5I9j=J^nX`kD_~rXL$G&W?7c37{ zfJI*ZNE)qrb(qN2fk6XFS^6l7C>FWRMjOFNr2IA$3bFpBUFDs9>D86+vM)8;bA;;F zc3JDJNV8q}N{fmWPgFU8J#SGl&?!&qRHT3vDtYKB=H)d??#(I*rs6IpS8G8kBBVum zm2_kd3!AH8LKm!b)(~xKHP#H`VT(m zsg<+t0ABBy3}*HpuP}WzIhDHV*h6XJrf;k=S&-E%DVSo`(JfuK_BFmi%L4lQ1f-V; zjtuI`)O~1edokxxAh!6UK;?maFj6uk$p{r;V#rzPyoj1(H$SlFbN%I4WjOQ2ag~SL z`>D!ZzwhD^YFikJBPK%UT|vH)gT=NOR&S8Z&+0nGYU@|AQ*M)0_Zf)lX5=pg55`ji^c91ZhrpdrzV(Bjd7zmQ~ z>JI|~2uI$D07{>kSGrRZEZlEQymF>~y1Q?#2MXUZ38J`f#ZPFz+X5c3?5*FV;t}OM z=T}lFbwj6StVbEf{iraz;$|t$F&Pp`tZQcRPfM=nnp~HZ0duQp8-W>+DP@g*iZL{v z)r9RbW0Clg=QyV~TpFi0doO#I)2#*;a9||N<>W+G`hr+~iMX6AF{cTZ?Y7n-{Z)4P zVZe#FRDmkia{e#XU-2l#bl`3tH=F%H$jP|*JTnMrw6jp#j;a$c=(l(bo)JHYTAmVW zTS|+`_KTU9-RdH#B$2;(dP^seP4B6^ng>)9=fSyhP_xS0fpwT?Lchs_L|$Rczfm^r zf^qyLfDbvVy~D6yQjQ}qcCN#%>0tj0dm=XzBmAtKH!@d5@fGUw9UJCKwcZuJwo6&3)z4LAn<7X>xP)4mN4O6@pL)js}v>8ao-9Snvn;>hh) zUs{Rs(7n8no1%P+JAKLO;}r2CgNV<_=|{pjL(icb4>F|c8s#SIjiPO{I$vc!Ot}ex z7o6zmDH7^3k@p)T=1gSEnDsJ0sEn-%{t~Pplop|nzTh^A-QAdX&+%vd7Z}ToWf04o zLt;V0-c(ShP^x4dsR$4gBZW%NrnM`GPa|?8x&<3 zQ>unA<%Y$mbJS}~(*@7Ms=q>O+cXbT@cm{p6BUbancsRM~1j%48 zD#++?r{l9xj&!19ioFH7UM71M;>=_d24b?6B#yQhvI%16I6*MvpIU$WpGy0=iEbRj zGfn4{1&R9^6sKPn+20)KFDsf4};t-iz_P;Y<(vqITxc3MPQmF#NDh+7HG|3`mY-bLN; z;i)d+-T);9!4co&1|QbZ|Dxv6R|sRg zmnlF&C5FpE8c8;YwNfo&&Z9-voPkDLBYjqtS{(n;P`wRvrr}KPD6A+1cJyXOmoEK^ zCxXgcDBFP-juo`2+y|!kH)I6d0^wBc1|M+a)r&P!rMQ{!1WLekcYlHX zf9Q9ooB#7^G~V_;e)esf^zJ`@0P3xYuJPX%wvI6XE5Ni|S=x*0D!w#|7jH!SLfZ2K z%tuE1_R~iOkjvqP6Xsbc+-jE-jzP3(2R6w9z04oScV){>p; zM8Q8|iRIvA!r>xxm}OzL+SU=5G~YG#_%rcMdog6RMyj>}M=~?w{S0kd5QICU=1~sZl!fOyTU! zuJ_fH^xgI_kf?!#Cz{wuI1SUcy25Lrm)sdCXKyp(6Qyl@X<=S_NAcyZtZ0-Rg7YZz zFN?(6ai`c~;(gAe2;Z>Kh^vH#bxMUeafX7ot~z<}sC@W}p}^dETD}Y)Z- zFMvp7Hos0igk;h!@+YO(eh{{C5bj(-`)^uo#3k|5;Wy)!b|-~j-|JXUrOEs&DEEs{ z*QjI1eKVo96ennYDM0g7Gu;D{4f zms!Cs6=`>P#Hbw;J{XVGa!+k{9_+Ipa3+s}b9}3qU4c`1E5RK5eYlbLI{kNAL&UEZ zoG&u5HG>b1VSA(zA1m$ZPu7wS* z@LliOPA6vpgf|Rv7b+*wfo}z@yrM@iOGOie|1uU4gtJa^Wx&M$VC^oW+Isth-{Z8n zySo&3cXuf6?i6=-cXxMpcXxL!6o=wa^o0KBp1Eh{dDc4Xtn)6(-r32!Bv(Gy_XoSc zIFUSztX#n}?G>mO_Fr#E8P+@_J)f?`lQf*An#AnBWBL1dr}^$N`vj3*QXn2vG?KzAPKUWAvT->euFxu3 zq~fKD;;JIAg@ihq(S<7R1O~@m*H$9a&yYFk@6ugcRe&wnahxok*$)f8Unb zu{t_D)EIqpdVC-t_PgzSifC-5ZLG@Mp&IFJJ4yJ-FJfCE)!#zn0ykK;PnqLKuBtx) zTd`uMkDA?wE`WcRdxZw9$AHU;9pjMSic>u>yCOEWfo)A+3+6CvK>Nx^R_|%g+4eSl z6Q>tSXcaD^MC^}Gpu?3`a;5@@>O+-tMQcbx8+Ui$+#Yz3xd77l>k~SE`@NGB1?i5` z3*qU7g}x1={F=g6vY&!W>yYVSrM3c{O+4cg9kEBF&Z_A6vxMoc#2y;y28U0lL>3`) zo>%5F@6Wxty;r(c4%te{*ni$!bXYDKTl4Ws-yHaOW^S{tD&&>12@$N3OE^=(2TVo6 z>!AJTIY~5ufa8FJ`)`+khZ~y#_wXJzj$a@C+f@N9dg0*jiPdrmFI&eGf1lm5ptSUF zFDhPh@%N+}78zLti4U1!0E=eCDsUGep%cl+|7He(IvdUQwZO5dN~y8hzYMHQBPxlU zDMeelD9Zfcxo1=9R48@e_^7(Ir|mm$BS|X-qh!8+>q<5m)Mb@tz`s;9my7@c*e384 z{8RIpyD_I@MA1b`{%Er3S~-%?+q$BQ+x+Na9%?GAGK<4Qd4?-{^=&ewEhXUYTX@pGLJWahQ=uB#D5&{yy zKQl+})0B7qcfqP^yuD5@OPH?0*evK?2k%DR+@Z5gb^aHiEXa}IR0HS~I$b7TOov`? zp`2j=7l9Y*@W{ku`>7T;!0bUPGQc z)~p~Ol*?T4S>EZ4}vt{5Y!RszB_!u`f3B>HCu=$ z6}_PM2oVbSs8?*ntJ#8xE%1WP%t8EN`7MjmcACjl_&7k9qi= z1rJMB+kL$?2q-%8XJgb}W8^N18YR&x_ML9X$MZwW-keKF2wxA@`&-Hn_zxEUkEfV| z-_hOF)e0<#chyOxlF!J3*iI{;+Ov{0Wo@+0Gz>O1I_|(%DWA7ft-}K# z>4-($!o%{xb>;f%tr=}n#EXU2RFyY{Fjv{6`ek~!g-X>|@`w6`G6j?}n1ul97O8!G z*Df$_L72JDQE}N+eS?5sn89cO^m7?Q|x! zAI0kAC0zX@Ug_95MDvZT(o3Z@bO9Od(pqlcI<=S*RKo|-Jjp#u3nQkmki!~G!0{gx zG-Ury-;?7|I?!=y>O##tZr{#x`n6h#^ujg>trW7`SMTGXV^XH$t-uJFXw>1tu;iit zbFmhES4{&qK7u0S@}s2?Iuo(j|3@UDD5iHAbO^Q?lc4}QGcl7rEpnz`@9{dg%ME`| zEwI0-qUM{%_-TZpFX%2uy8gKQGi+DX0*LUmI4Fe_U;Ec^b;hCK6z@fd8|zoZY|rb2 zyK5S+Q$Ae5LeCcxu4`8Kv_IVKAFhAayCI}g8)TpGXQ~sLs?Le&ifTpox&ULEef#AS zr8+N8-*p7WmXOp+AuBvRq;^F)CBDLDRK#^u&QaG>eUKtytLE|zGCmXpMu%-dYp6^> z2WHpASiIn6Ua+LfbRgS)A_*=Q9J_O>KE+Y^r3 zzOj!y$JIy{o5^aGkJ$bSLOz^6Laeri=U$|Meg-Axl8I63gT9qz7q<4?tc@0iu1NEBXRUJKMouk*`q}0@ z>y1uFAiI$=%_VUk1h--pGejT@u)X{+QGD9EuRWt1I($}BKju^vddOu=S5(XHi*@5N z>7K1W-c&bkiye+6!cy3}S6*d>=t(2{} zJ=KfZ6J~z9g|fLr_a(%18kL_R?{W!J>T9WYAm8cm2K5jjOMF3kw){~aDRpc&(;bq{ zl{J67Tts4jo1qJiR;(qw3v3HDEa|tjY1DkEwjICtO2jzQKhOn4w6Ah0N|!8T<##rA z@Ee5e+c3=PnRz5g>CZdliDcExik9wD>lN}_v9r+14%Oa)AboM6Rkc`bfwHaRPN%N< z9^^7YNoS&+uR3TiwW!v3d2I{usiMOm)dp2*|WLH6Ke4MIZ@?u=WY{}YO zbZceaz0kR5eX?y?bR)3zmN&n4Y`+tX>QvO8C002V04b?{iH}XJJ@}I~`bMJLN7;QK z9g$Hza~1op|5U6 zWm04s1YI!g^3LYu@7x;_j6sf%dw?>oMnK;^V0tq2ql!x5;k9Z zFrEa^uI14;uz~TFdtCj*z4Xx_(Q6=dYH2Pb%5H!7`Wyhb_a3&XOT^IH6TaCGPj7*x zmrVR-Xt%S%Pq!N^74|Ci)QRX@krl+{0WcB?Q&3>#Yb(pcJMbnv8;}bJRb$-P^A=Q6 z!`CoFUWY#?WME8&&CKlXPva{A%j(+q+5t}a-94BU3tLrY!8__Azpa*jQ6-voO`%bL z4DGvB?(P(DycC?wt*M2ffrZzn!N{z0;=VhxR&}8APPuM9RG^1NoLnG!+)dOoF;!JN zmw`^&hsgzHN9VJ5(55#;0#xsgS0``TBaB5%6$St9?=K}BNl2apMvOY?ILw9Q4me}PrYpPuE$(do?^Cqb1WQY7e^YL+b;%sK@Mnp2YBfT^w+s9&E3+zsFqM%KS*nt60*IPRNQyE zABjUTc$%-v+PQyyaVbJ*eBfVnt)Ao`x@HOTf236#@pzhW}8nQP0k zuUpj(0mM@1`JJnpB$_8YsNU;1bd#|AN&T?E9VMz_9!WtUC6~QY0`M~q`$g0mrWBWP zGN30)5k{cf>vEbdS=LBqFGOU1ej1-1<-!zPK6td6LRaIGxljSa4v(&|WqYf{iADEvzjwMm{g4w6+?ke z4UY2yL`P)qR7^H&i3(OsBS}iNu&*8Yf^@cmB%GZXIdHR2KPL@3aU*OBhkO`BPzBOh zY*}H4Q#N6P5x!H`?qhr4c3n@dkdjlJXE(x+A)%O_aV&{VmDW7O$5PR4sW4j|qq0^# zpRDpUoS$0-<*pVB)!b+?K~HROP;2HY`H(37#KdHSiTxoq4H0)InGGzZVM3+%v=dkc zpEsA8;m0~@%Vzo_D_`^_REmNCR?NJqQkG{ZU*W}huK%G)a#S&)lb+Cwh$1bYGU)v4 zC>KVmLEn z%fO#drqx1;h^z}8AU~Xxwp;1t=4n{z=`+Ax-$phh5+g%G?%7Er_Vp%0FmqTee=Q!h zrHHIE3$|WFog~)GF9nJ?c$;Plf4aJ*Pgh58cejj=MoFqJ!xZC$1~CuZ@6nu*R5C%f z+^TPv(P~y8)4geFShJ;>X^fn?xqSuiNyb;_K-gM3Lph+qk&$w~JUj86F70W8Xc>In z)NC8Y>C#|3XesLA*xg=EZZAh2f+HjxQ}-tVCt4|3^(dz1{gur@9TENbM-2gk3>=`@ zsVsTc%Q0PF{KM9DXq)v=Xni0_ae}4%Ls<9WGcycH+?THE%P6OhHZAL_{zQ%y$; zFFH51gS{vkvpL#TMU}zD#x!oW!q@gul6{E=B8)O?0^i;0cv6-$H?Y#DKJ~@jK@QsR z!)rzg34*JvwAQTeLz~!TSHmItG9mjEz6RE~86s z0qX#$Ix$%4aY57Uz0fc;VR5Q^>KD+=@dcO#?obRTwzpRHeR_5p*q&JorA;c29!fk{ zT*%CQ16Qf!s7v*OMtl(5qgFGDG`|E9=lf1`uGKq$sV?v)MGLc?BSu38w8WmvLoLb`&Dr3Do%hVp*`yB($7o1XswcG{vkoz-~&bdDi3~U^o32*g3r^dulIU@BKTl!?6AYcH1jx zR=(D!PX7h$?(V18cm6l9^L+XX?4k_8s~=Tf$_p6 zsaK~<8~vqs)GDUaY~7*@tXlP=%7^fk)hd^=spHz`A8DU))riQqvcf__(eo=$sAFs@!BfyV|^auwkl5vd1{Ig$T;MBT|j5^*pUHDT&GZ zMlNTJpf-h%vN*-i3TYjX{CjUA<#@KyO^+%E_rlP6<~MX*%soKa(FX!YkgFemkT`S_ z$YVsCs2TZ!cd<(Ok`)OC(vr@c<>-3t4Gp-Fday3DFMS5XmF z-qwX(v`Xc`NYD{(!4YQyud!kTlpngICJZv18Ppljt^Q?S2@Ko}Uj(AaHO1)6sYkY21Rlm|E@40r14eP0MGoRmv^}a_Cz;~27WwEVf5BpVhUtvu4dX6htKg4|FTxVF=LesXXZE>%g@uBZp)Jome<8x&3OPc_jSgA9sW=4`y$7jvIvhoBXR zv}uBwlAm5+WCFH#bZ30Oe=a%G<@4rzc^@cHzb7*d{VrzVOK)%hCpyeuB3*elwL&x3 zzqg*hCxbCONL*rE%y;*bflF)-Gnh`*c}f_;b3!Tz&3HJp7a(e?C(6y zx6;#^E1677@{6ysqzo}sq@IUcrab61;5s|6;lt|=Z`}h{(X=-LlyZh+1S?pv*OJ^} ztJ=6M;i*L-n3x=m;|Tkj#;|zGllKe?NdI2C+_&GU`xozPT>+Y{z~C2S5VGdqW0qQ}1D94tvV z@Y5E`{;`7(_qP=k}^5Y?tlpp(V9WX|*+AvEf zulVB*at;s*5{8wqy$U(W#op+$Hot4dNhMu%^hEaVHCSQQAu>&o4HPj&Kh4tKb4MjF zLDhX4Ozf4O{j(o+9i<)Xw0)HeiSY4$GR<7pZlyki+=15tG!a_jCa$KCVJRdZv#A2! zz}{c7w1u^ZIDMavSuV3NJRL;03@?FMRd4R{+~`#+mqMd2@nK!4n(HqkKD_pg0=-b!d^a<--Qkytx5=G#s@L8e0$ zsc_w&>PzA1l-%Y%B2U{OUMn6R4j@}t*wX;G*%0@>S-PQ{=^yqmVfqm1 zQ3-AMc%h-WU;?O_Q`c|%=H3Jkb@oefktd`Rn_XrVFLYwZ0eGQpguuoQUNQ#-=s>5E z{6<(Zb(w6t8M-_e$#d+KLEiQaq~ATHl&V6%kcdyC$8b647pQJ5mCr>m!=Q}fS_YAK ze-GJCF%qFqkJ7ohxsEOSwK02GBR0T>kk0^xgVrz|Pb527$^5v)3xukATq@EAB!3D| z!ujTkhL#59=A`wt&SdJNR4BiAhF|^a#4;uCayq8oBCju?zYw&^Sh&++k#ZP&is9R zb&6xf#}`dU(qEazJzrDX`@PUIGTK%$7af|PfGv>+A;4%3M&!x~Zx(j|abAXW*$!FB zEuxcejzKo-gBZ*;#7MozcPkMMw-Xr=E$Bz8Sw9DePx{x12Ml$FLQN_j8xV{H+$bB0 zN^Unj%a$Gdh>;bNvl*NWA*tyeidWdlAf0=DKfvk!4t;0syq(&)O1%qYt$7ooa}(~l zXX|k)=6{Ue1yrb{%x$f;9N@-lPb_c-Nz&7D9iGC~r#h)ht1o~1$zHS3vCF=Vy(3wRWqx8r}oJrUkN3GmDzuTu{gYmU#cMs93$ zyKg{B=P-Ic2lg#)1*zo7iYjN{zv3x zxC%1;CcGhGtkpF@#9LS50WSZ20Q^%Zj3Y%!n^VGCG4xlIv+pcOk7{z z3qEvq^*hPqz3e_<$Z%j}`FVp~hBdWKi|;5(OY6?_#E<*#4g-(wce((^0&6@}-YTV> zF0D@QHdSkbQ?kZiL<8xosH`c(<#4{AwvZcZP5^9f>IZH*nW%&UoRvetXNl%5rki40o{IJgW*j1I zdUEAe?l7WhuhM73)tw3`poYAjt}cD>2G`Rh1-{dCUXApZDHDZXgB}x*ss}4z{s5gKw8L|4;q(+ z*ZOSxgoxRH!9vOdP<2vBPdhk1Q-MZx5Ny?2sLStHo+zT7XzT^%6&{5n`C=O!ZCgSTL5tv8v*_u?v(r6n z)WL=0wLd1VFq~tI!k82Yv-~Jx=>*N0WmSrevr5-2kQOlUv7*9jlkLHCF|IxItlAhK=ehcyR3TKjLu(rnr*{#_81;hTqs#9Z#d<5UwqLXePOE zIk}*hUn-Tg$rb**6z%fDE4P+vYR;0}P?z??Ms3b|UY9J|4B}M(!dUc3^J`B+{vhD5 zDg;wZh8BS8CyI-SwJ66ItQh_dR5h-RIQMh=Gih7D)nM?g=3VAwC-GgzZTW{JHTsz9 zZSMIRF@h=^%LvoFo4U;c?C7#A)hytPu7$$e2;XPS*7%g~ixE^OrqtJ8se5UJ6{?#l zAV(l49<=H9p4Ek3{(YBFsJ-~$GmJzeF#VC?Y5AF~#+Y-(pDfYg!WB!1AZK{qv3oDunH*AAB%p9eAf<7P*3^mg` zpqM1QpT8_IM7fYhZ2;*sm&>M$Ds%Z2S zBa2tge0o*7vOfbIr=5=6SIwF6G~p;JN*ZM;D5RwPioW(pnJFxy#<{7z(Bewh57hDqvtX#*Z$Sg9&$CA4NnKWR)hkS>3nQ``ZHej>%`uaRk@UG*Xft)Jou}4J_ zAI)c+cEkLM6Zbj=FEh;cRn#_9JH3klJhbc>?IQJoq;a{rKz%Dbr5kn@igfQrqohdU z!(>(>LR1EE5}l@d8ObS3qX2A9cGi1Tkm{TWt40ctrv?4@JT2M($kQ_Y6opWxx;y!v zIRJVNZIYqTjJ>YX`*Dm!krR~@7_nuarqBMO8{%Bbf)cV=?o+iTtvRrr9h446iE&%C zjW^27%;J*A8CFN65hfy(T-UPFvP-0~Z=NLEsLeR1QYO3sWrJ*t5}NcUFqCRQqx7C9 z25fbyonfjYG?qmK+=R(r=PdjEIi=_Igs#AO;E|svm{yin&b#Y<4<-CQvboJ^qZmkF zooTJ5-wU=Yn43uD3{J9%ZLjZ7r;xVpJI~fn*588&(?3>*3n#x&YGlN(Ja1Ic_cx-* z`w&dm9BLgU?Ju-X>Ia$$7IQPk+nipg(r$XCmc#?*g99qHEB{n#tA>P(vF3x5bp}@l zJ}b2XPiHxr+KEpkg4BV;6u*Enp5r@edUPL2R}X!-Y0pjFcRkauQp9 zebq?F-_}>44JccHFN8Y|NuxNS#nE}dALYPJ1j91VX_R=sd+CP_!l~gfgpUbY^)Iq1 zi!lesVTziDQpkg1E(7ak==vWW<*&a4^A@31UyFIkLw318M=mN+_sjGJ=UDCVQak)zPQu2YgE2;N65{!l{X13WX zpqqONA3nja1Yg5{3(G?7q{f&kF3H%*+@_{9-Wx@AFlXk0AhHi^huC<%JFOT7);Gp2 zN^|m=k=?KJ0kmh8T1)@hMA?40zpT*nQa5JWwKr)45SPUd`5T#ixUtO#1Z;zRM%}^y z1Tj(=@bs>i7sL;{nCaD~mku%0^J6rCMI2Fn{(C-vHI}ymg0_IG2UvFgy?Tuf&s$b5 zQ$FvdFN|Ryd6sephEQx885(I{NfQ*32~fdkKbx|?s_LHay5(*h#tp|3f}OJ2fgPYv z`~9;|yR|BW^>REI*nu6=UeLc{ojr!8e`XyB*qf}%7bAb~@I~o2v2fSpm16{joRo1v zR*e(6_wupJw84?hq5mky%zSlS5I$vrTiV$f$$e8Xbj(h|wiQE6SeVzzr)P}sKLT*Z zao+#pjE0-~6|K zT|%_xB&(Bdi`!$(6hT+Sx7zaaC*b7x|5min!F=Wa-VzdzP{Tc4nx2xN(0^;_H8=So z&B)6!UH4ts%xk_|Xh8T;e9bDcM63W^!AnPNBzy=FvgoOI#o~-sHuP_v7C6T@`|enz zWJaYD5zW|FgikVVwKTYu0+QAIk^;~WZ9;UfdoQ)N*<8@ebq@BiD2izsWuPsEL!W&P2vhugFWj$cf(=+pY~VQx07qF$-KC z;2Fb4?yZcgo&1m1ASP~yusIVIMC_bJ89O_pL9-gS#-w~saB2si7EVjEIGQ6p*FjiL zifN6Ek~sL`Q?D!FZqZ$Ed)IPyrVm^%%~f{*Rjv|1o>n|Oix*@#&xRrSeD1^FzkQ;T%IrV*oin$R@ zTHdYUBp&RIu87B#?=sLZS(|swx42hL|8X*vS&7>Jh2d!Lvo3+K*$jO$Y9cS8(WGdD>q@c?niwhwQhG^B7z6-x{Wvbwe%lrL{&o`!G$J zQopg{CcOBQw8xMT|7TtT#YStY!{KjWA{}-UX&Jh`@*3=Q?_f4Gx`sv`vNqPz+l|}5%=#!aT2{ovLzjfNG**QRI zLNxb3N)vi6W2G8`gL*nti8ox%C=LqJlL!T06)r_S#{gh;0> z1*p^ZM9F?EsU0fNIW4FuW`H6oZbST0gu~Li5tyc}@p-eo$vMJLzu4!HJ=E7{4W=Je zhz$d*;C?BHpZlPp+59=%;lrW_Bsaj!Y9ESFcrzr07j$6}BmHCKZ76t4&bIE<{p`!<3Pi z9gNHcsW6JR?-c%up8ri+Qa>EkW_dxAR{k<%lZMhS=`6RJSnI1U*b^@cguHh z3LRyThVE_9wdxd$IqIvz7NNTtrV!30|!@z&W`5Cj67DJp@r26 zocULp7Uyr8Hk}G(nv#uFFcl_YY<|z%xI{4*LF+oultI(HZVo^vdCosp;SkU;1mSpd zZ{OBjbHxBdOjkCNz)LjHFPW719&2W#tsM^mwOL8N z0%=sCa$8v-c7;E!8>-4f4j8XZPkuY zUHsKvL6BfptvN@%1wTs>(TE?(Z+{5{c1NSCIpmhxtUkMk>G9;CDk1zLLR+lt^PMPLsYu+yDVadoV+hiajtd2$H}@WlayaR%_?o^` z2SkT7*i6TWgj~9>2$ypKQOl~pVla$hcg#yr+ye5xBZ;_f`ZxQLUdDnusUT^-h15pe&QKRaD5cO)nKLTk#*S%bwKQ{>cKP+2;IK&aikP4J`aeT? zL}|PJMrn)bCnQhyIOrNssHl@*s_mG*dN3@>C9e6n685rnb0R=iGCbvd(42sn2A1t& z8~R!V>aH6P3hKLQmT?%VT{X^IL~qUa){J(Y5@`U_-KoujPmM-qPJo%_T07zPu)F2z zuDbUzmFc=x^0X}l1|DotI^hyHfQ$|ar}_X~NNNfU)qQ)NGQ zfkF#;zlYKOu>ugKH4ipj+7MX8x@5*ndf4q35Xn{SDSTG{8>O|dDd^_)hYlgV@hXby zc^x4}a4aF;*RAnsf4@IC-ybq|KtWI)TxIujPkCz1(-Fac$Ft)A8KEl#YDnB+(UBk~ zzt2YAly2%1u_Lml9rl+q7j5q*q~7A-dklMXBBhfFUUUCW8|#O16YIz{buoZ-I7@)1 z8qDl&U4);gL!ZfwFubJD?!o z0r#l^VUSOdddqIHGqc_2^^}m6D*_9L?*Z(O$%S2>5`Ay)c%f43l%B4)?Zk8at+4Ua ze70;vS`Hr}gsPL&J(M(c+-~zy7dwO&svEDjZ~c^PyB?x;MDGt^EC{hS1fy2(Ov3Q3AK*FXHvL;_WCSb{rDt?-eBW}z50$*Hw~O*8D{+1i&?0Jj?A{%>v- zB@E^n^o;F4+-hrLuNUKI z>&$i=&_G&1i=q3;C$6Q-do_J0zls1^PD4&uw`e)T+t?E4apRqf&gE3*Xu6@lMpC6D zw7gx^Eis$^;l%V_MUbtMK-?#g5=!>6_qw2!(quCnp7{R_TMdS#Dqq|-!c0*($@aoe zx92RHtCEs9IArSyAD3+l1yb5WiA(~iXrrWwHFT3wyp0$oVT(x$I!@GV!|qapyM(GH zSK_$I!N-I>gOV-nU@OQXP; zx{f5Me9cqEK_rvXe@tSf!`dAfbUxnE&CF9hmXCS+qSFglFd^{(tu`Q4V{nQ)ET!!4 z64t^{;N*`aRwr{{6B|!^j9$}_Q(zyh`H9t%__!lQ^gwvo^ILv1gsQV!=N_EZg_s#<%WeMaPB6%Mb~o?p@+K~$_U zKSF){wn@O8DRR*E8T=aXNy8m*MXnxO*+hO7@ZOA9#)USEGkz6lWH-$`O3%^+)Xcta z&6M?oX17CapT@@es!;{ZZqwb!LP5-&+Qc$@D(3~u@Ap){Ot3EU zO(OoPa}}BFa2N=}DMsd+?jC-hVmGy_HYmx>MJui91DQL8;7VmjenXRadH7Ke`xuy2 z8`R1l>MKU33(M6+3Qz-ZxpkLpAh3OVuLUo^zfo~~MiPF@@HU_C%O1ootymB%BPnvT z;4dq|pQ$sSTWp!c=rz+x3PU@?O5Asx!~Va?RgFyJDkAp`D-9QF&!}LhjT)UIrs5W{ z^EtYS>B+|S^&&X>2~u8Ech2yi=j~>)x1CiK-jCPMh1*P}Jk2=Jem%sQhuJh_`!tV_{JJv zMy%+9(T3*4*D8E7q3}CN)7aB8n+a0bSr^;o+^;xwBv6jh@eOsB^%Q7~+z;S3W9a1X zo@$04PDo0=M17MBg;bYX4y-5J#rCpasA_xLyq8GoCwK^%EbPJkO-D>G*eZ~KIhm9x zN*dbQ;J<*r0NDeDMed{cneBpX>BpU|ppZN8Yw88hD+0ZB6R~T<74=JIkP8d9YHl)~ z=Xsu1TPBTYz&sx}*Ne~mmZ%prw#JNXG#Y6sl)@(8bBk7QuL-$XbUd0DwEn%G$P|m~ z@TSc(x=w9=S+#BKMIkfHgx<~Vwn(qlBk!K>5CjJw4IizA_;u%(A52~x*+5CUZ=qWp zjs7HL@iTK&0iQDgvF@AVv_vxSQ4P{+(gR};IF+>m*)WIgzvl`P?FRC(>4EZxce%80 zkXH33S)Z_EzAm@_Fh^le7?i00ChNxDQzF@8P!8sCouq&bi_4l!U=(u%W@^!WS6)GQ z8Cz=fK!=NL;sfr6nB0FP!Et+zfmbxiBxjvP3$No7-)ubjvT$oUkP()@g?w{q;n~gv z$8A<=iPWj~-onYkjBXL6?g~N8Kj=?(P-Z^2j+5*R38C(9nJ%2(ZcDSBn?m)H=V)arPP zTlNa_Ij`o)92%|Siu2qY!&hOxYa|^*!2c4*a2c5_5o7>^ z_eM#Km^>?5Fsay0$gqaMm*1(B3FB6OTWR*~{PDOzO0_M*8!c59Y9!K@{v4P$TwpTF z=2jN2=qHh?ClL(&t+YEmm~CjpFO63!+@ZF_lcYbA-dl~vn(!bMzhyJ7X(%ct%Zeq! z6#^J#hDfg)iLU|e9Q3$sf)$W(-#B|#n_8iaZ7L1ZV5>5|1diBP(5S&s%{`7lHLNOW zivETe6DxgaawITGb?EpkPOQEs`2B5Z=$q51-%(Et6=k)X{l+}JqXQ@F&UPB~ z>!IUj8>A<->8zh0|Ni-usF+-)#C5IGaqddWVag3*#!!gz1f#5}pwDEHR zOE>RESQ$KIOJ!r{jTHu-Apd6!6=0LHF?3+*-z~}t8CT}v`92TQ%gEBR z>RoMGsql09~;EvoV=zEZ{%a_qZz+sB{*^zoVgu z&5?qU1K~18v)jI;L{jB;JbH486}U#5GZUWP!YufTcCNMhQFqSOxm)Mrr@uYDGSoTt z&Ls^!D7^^p5a^VXvo_0M8SA0R>v7X*svLV-XM9eVviNo@p6Vp2uKIKeS;#Yr%22=lo)bvXPS+EqOU&h5|d)St0c%U~>MH0uhmawTSR7s}t zQKPak4a(*9!1UgSK^pG-K1`_Il0n}o#whg1qc;;#{djqFi5r-boyWg{W0%R#@2-25 zRF@@6S`tl|P`IBL|>16#U4L>{p?XPiLj-p zYlKwiEf-EH8h%)!tLQN7exNn2<1actUxTKq-p1rK4YYSQLJ;moTzjTjikl%Wfg~bZ zi(C|DJyI3ugT=VRsd?f_wf=|uiBZLXy=(+##6;=2M37LN-4#$J6(sDB)I{Ol#!BU* z;+4t|{EOE_3UzvyOZX)*r+w^7?HCeSG9pxazub-PA0 zT>~R;VS5G58IwEL`a*l^Fkb6ja7fz0@@=EL9VthBH+q5?+GL^Bfel(L&m?=+=~4%v z-Fv`3wY#|iT3OWp;_iH60PY@$E>Z#V$=#<%uEM*lC#@PosEd;2s&95SUya1el1SDZ z^=_=XmVAI}{(N$Ghtdw!{{Q0ckEMUPJ9VnU*OR$~Nb;$1^}=R9ps-d+k42+H#MN;~ zQw1UA!&Zv)5LkiAgbubA{Vq5iASPaf9oB#+EiU8S=U+#=*BA!$+r@2?Q1@_SEzBmT^zfPl1lO zxh%v$x4;A}%wIuA!B0q?R<} zBXkGNLEw17Tx5hX?Q{EDyVZ=}yBGG9)+nJHFP z3>?ym+mIYs)?*iv@&#I_z7z)6R?@xXW+Z7$%U)E=lcIPjqZ|w;i3~*))1Altjmq6h zm4q^=5Zd1OYbprj;gI8GQDFki>{xh#!vJd&FCxBSAf_GS@@yF3owN}WPK&a+-f1Nv zt%jf$9n=RY8WB$ zD)CEI>ZE&Bo<7S|`Fgr6M^>wBjt`v%r5PrhD74JYFlve#Kk7&;P)Jv*@=|DWj36;Y z+xr7Ifdsm#crRM~X!^Z?!%EyMq+kxdftjQddxCA*#EASlVV6jWR;keRQ~;%3zwSV) zsQSDu(gQkCS=d1hY<11yZHR(uXopzx?D!f`%bGGpd zW72}OZ3kyMw=zoSAUr=w?rrJ(jv?{a0pQ9Ne**m5IT9aot!Hm|JpUl!rnACJY}MWcg_Yt6w#z{Fawn0%cr{2p2hj+*)F*;PoVzI)A| z-@GGG{BCLwo?O)zh_}A!?c7uHtGN-iAzZpg_^t0IfZX@LoF0~sWT%ufS57=is93D= zx$|B!|Au%f?=)v<*d9(Zp`OlCm`kF*U~UDsyuZL&Y1@>yGLgw*kL?DM`EuPvXVfrD zl9<4pCKDZRI7JMWWSS!uS#x4+;Yfp}8(OTIcR^}KzwT_BeVxdy!@+TxsJqSz3dU;u z@Qaa~Q9gf(1aj0!cD~*(-I0Ms|61VU41+YaC>R5cKe3p@tEe*if)jyIKlJ05aeyEm zP_U+w9VDS~)D$=v3j?W{AW}9mzaMHQChn>+v^z=FSMRhkZq~qF854Xrm#{**Jqn=5 z+{f_na;`2}GPP4xEW2RaA=^B$SG5lr`GKW*_%VM$#%+e=`03T!-4l5ihgZnAo*2Ae z!7JJ=PBM(K8997C)wvwg!%3EbV?^ z4qD@Hsb+5sQY1tCpcjWk=0y<3*A%DAS9anQUv-`9Jq<#v*0}BMHZa;wG{x)*_-?Xb zx_jf!UF1TcT0XiW;k_O++sw4;>`8_Up+-kLo$m8w;JN-f0Qenm)DTlci%uJk9Ot0_ zb$^{j@#w3~hNyV?YIRhZC%LP-)%~vp4ih-6HG49tw9?8G#>qN@WCS5g`1)Rni8YVQ zaGLJ!{|90~oxjEU$+9Riq787IK{Ik!?Zj+GB?Eu{k6(6k+CSf9o3sB|i}5$%^DQR0 zz!RELt*GpxJr^^(;#|G;_>5h&TnM_L8JRsAk7>5uT5^wFljkzQSrZg(!;$g?oXymYPIuAAqZ31G6;M?_M*{JK%)R9CW7Df z#B1_45i-Y1{j>+R-L!;^SkR6PAqPbA8?=a z4F7-j{{Fd*+u9$;fA?R3Q9C24B`G^;dwc!vw;9EfQ{C8(-j*5%NtSi_E_8!?PmkiF!6o|r6- z<)B1fut1;j3o3GSMvz9-ER$0Kppy1m5H7t2NtF?AiD-)TC7+|Upg6lrgdHPNw3j!m zo-vwlU-65=97)3&Nx~MXsMf3&FUkKL{rLM|hR0tV9sTfp_@6)h{^-XM{PMfwFOGk4 zfBgRV_~__|uit+4;qv?^e*DW1qA9QOuV0RSVCO$YHgbIY#c!V_SXEQOI?|ZUH+FYP z&8nIyI1K2XRI;fv=C_|vrDO&`WtbM>03;}=NKnF;O!*BHuBz*+g02Tv2<$Ds@@=D~ zPkA9!2lW^sSJ0Z9!d`+$30EYU#ylrEGbj2N7sJ_GYS(k zDl(9`)yXxh)QRF#w3__s&wmCJid*cH_vfc)Z_h_%z73{Z)r^&JhGR0((vJM)LfcNx zuz#78?7A0IbGQ{A_(I-7b}W}hq*`yw1)=2~yJ7Xf_4&p$aT(b{=zv`Hek3|Zg#BkDe_n7O#qgv%QwcRh(sb?#% zR`WHk*LK2g+%RjG4~-?8gHfYAx{`@Gmaf+FWWNN^iJJqkfEX_Vz+CgGLm7Z|M|)cE zX&f2HV`;<+Db?a^wn=Z@GJs*VL5EJ{r%eu!0D+kevzlw3(c&N?-pRd~6^#+c!*^>& z^sU4((1KjE`K_#TcRz*UPwTuOW}}pL!-a4~!-CK#T9>RQW9DKV79O6D$f=m)RS}`R zd7EWWEClXXlx;fHXZf}p3)NBZ^s8OSMriT}*WH}Xhb(K0tON(TMw}_R{vI_iZ-pBg zxq%C&)K`Xpc1NQWu5jBIU@NsL=ggd349z!JlFZvD*Dd*}^IC%ZgD7OCE3Uice8Gfo zo;ZEC&H!!m;!(>{-5RMjp06_tkj8B8H2b-s1;o*Os2g^$3y}oN1E2iy&(A?B@uL|| z*RtHG44;KEQTh9jK-`)-)xkz8t=D;_)gJu9uDyifGkJWQ9}LN-Mwq+`-^M8^9B=Gc z?c$1n`}$Wg9~amNI0}K4%`5g`AsU^zIGYPv^31LJDK0a>t8_5=;w=qkcxBmWvoxbE z^Tr7CF0jvlgu>WP_K{|~n1dy?^U#1W4*41wz*ek}1eCSiF_dIFt$ko+1y~h81(db0&1_%g7b>EVS@tiwj!Glv>=L z{gJbGSATzV`sMkXt)q!_82!5)F4_CVjuxFT3#Yrus`LnUEl#5M^diOkF z5puF($cE9bJnbG$I;tPEU0h_<6J_%mk((@t|uyB54q0_Z!S<3d{~)&NGfqgl_9 z%qK=SlwxWjnBuDZ`eZ9_RU0fV9u)GRmMvo6DTU7OTyE1x%`#c%s1urnMD=gfq4+EK zjHzfsSShvGa^8T9ISq7X5c}{xi)wzs*xq4nLF!yB3sMzSFbl!y4J5sm?cagVJw?YS zMaNqd&+R)q-ebe0YMmQP2!}rZD?;$GZjDpG@EwqE#B0R98=$9SL@oqTvb4B-AGI0~ zNMfZ#HDkpz+USiYW+Mh{SVhI^lIw^pji8_q4r0r(!uLT^pZvz|)390}cNS}D3>9{+)*EfV=LtnR{XR%8chtH;H3>kfviZqt^IWkrp==Wq}$^98!(!GhaG0I?%nzza~yG@gsEOHQBh6TG} zh40!hmzs|oZG&MmS(e=b1sD~wRxoXmQ;B90)= z_*1)(46%y{?}aX$E$Y;MOeAJaxKhE3I5uslaiq2OorK5>s>1-A%;$(3&JJ#Jh@zKj zB&>lfI%GTzWwDSMEgNNU)ZuU zkXJPeLMl5QNq{GaZTxK)Jjtj1nL&qQNW4N6WJgGmXu{~)u>cu2@-l$TW{YKHW^SVnfIPmoKnvPgmocXN)^sxC6|xwr z?dvf|8_fQufv_y$%NQf@3-l7mTgWu{>f+3wOw-EitC*Zj^Ya91e#u>*0(# zgU(KQwXc*ppUfBb%W(5{2p}*~F}L%36p}+))xwl?pcRSByEwKhy73`HV>Y8VTtWdu zI8TO>@7f)Ae@CA!8$N`=uR3bv9VTIopqOLmW=+P_ zbn2&cv_NoCP+=FAib3lXlCa5yWdOm!su?R;O^fbOY`LYEMlGq$fk4r!q*V+~q!}V8 z8aa=`5XXx>NU1#dM!c6&zvT+#Fo>Hb2WXH#7_Bg}x$jb-n|FTjQuK76IlGGkWZ0zP z?%rh12ot)VSCR|8T)QHnXz#bS)-f2Gos^SOR`o5Nd)Kwi^IFi9(Xem7gW5=os&pKP z=q1#D4PP7tm3=z_$BP|kdd(Cl7%Y#9ymjGPRHuRIf5LYGQ1Ckm=XMt~d=aEu*wQL3 z#exoKNSi{)n-s%ZN^prpE&GMT!d|-eZwzO0n6iKlM`-I=(ULHsn%eFek8z0SlA+Do z4)@!OGxD4q0qUc#_T{5f7xC!;tJgF!N)Ts=tR3XDkjUEYyv&{GM6in0ko%GUYL9I! z$BsiD_s9vm;@IErgky)sKyyC2ONb<1GEJclDRS}_{`-la|J(5MF1pOFX#DCNdU4?# zR++VSp=B*Rp6jgv6m>CkzL;H2MJ~&w@tAFAapi|w!bm*@F^b3?=VXIZ5Mx?cI46(w z6eQZl*$pdt7(?HHpN1<>B_M1bq*X7lF(udgOE&!0vX}Ayy+of0^^-LXJ@UEs zG;)8iHu!d~V9kEqb8VsokEOX9d8Ip7kO3a^ToX*kUHr0L-C4GmCl}(LU5EjY3B?M- zp_A{iIuapu$E*s&An=On*?`onpqk&fU~&h)eAt17afH_TP%$yDr{8gvuMwYl_ss|Q z97OQC{Ql^q*-y(Kxwx*|b4Nbg#2dK_`i}EZF8U{XcDCrOTy8CCg+CdOpe2zt;X-LfbLaa* ze{#vnF}iHjG6q+2!HGq!R>%-M(NRPVkczlavYF17hY}SjdEa0mqeAIUM0?IEd%GdW z+IWi|g}4-5nm5e|lV*lVt4`A!>fydE*#$E@v+moOyj>BCp`3*idD@bl(|QRrT4V@g z*w4xMmeo^ceT!EBG{4%nXrD`PEgqc~)r=mKZ+*omBu|ge5XJZ)S{N;2mH|fkURPi8 zAm8%R_SyQGUBwUE?Wg8CwV>y_pwcUB8e(+`_4lvOJkDL`nEo0}UV?AHhn8yv$J zZHq^Dl~z2qrHq`PsGyC=W}g}&gAorMaqOt103fA|6exSe{ak zxHC0hdDwLc#pvP&N&1vv1NPQysNaFXcz`0?v@bQl>zH} z?4_-oMs~QyjksU$aeodA5c1nL;7v=d6U>z4^!PEgq#)yUd5LV*wFbD+T=kG zgoDAH{DyVL8+#GL@#4PNdH|1SQxr@7(TpW*JH+?|pcFA&{dII5+l$uf0wvSK=4|nJ zgiteIyhAijT#L}nt|dH~LUM7jKAm`LB*V_G2z4cU^Bk7HBaZ$TYNrn4ml2 zf|l>*(RLz($iBnIcIVTKd{D9ahG*dRMuW%7lO1=V|ek|;=U!LLIpVf3_5m}bw66v6D(y|>&KDv{B@sDx^6N} z8m-LdF++3LN5tT2Hswm!wwQwHt{tMaZTUy8*_{0C>+}Eq`NvVKmKTGUj6vvhpgh&k3U{I3lH2rW@#cIKoHW}hf#x+ zg4+qRno-qexFn!GL4b4&ds#0rtgNiKRTeV?PY+RCOd*XG1*H}dam@5Bv!LuNOSDF% zGPXE{Hl42n>t%iB7XJ8#vs584fu$a*Okg-c`~I4qL!8G-<^yll?m9V_9Go2# zCoIIva&mcbMqaF&?BB3XD)e-M#P`f1vhHVpCGtOm6x!5g0(xfShIr+Jv z1uWFjHOIC9bFr~=1imv1+hUB)$9u#@Ue*ByTV>!y}c36x|&8T3XMj&bf z5NK92iV2Gr71WiqTDRP23wH5PM!gekit$R;Yl*41cuim_4IgWxw5$j(OO|u0SurO! zoJQ0My9UfAWFALC5(zGoMsRC~o;8}}@>U#=VhSsBs{n&q%Zot=7!4MSRIrC#0MjTb z6E(C>BUZp$wP}dxX;C9AOLVo}T3keMNGi6rrY8+a(Jhx_ju_aEOq(tEN++_;__&yp z87+K`IU26|`ZDY~T-ZX2X(&5kZAZY>V^lO1j>d=$bJ;{|rJ*+Z`E$33(N%=jxZh(P z?^+fNcQWO0uSAwRRAFlmfQIj2fMM%C_@RLQrZoi|I>Sov8vq@e6LXtdym$+o_$(%r zN7uY682QUJn-8!q6pCSgC5?B4`+^);YgN|dFYdR$ZW9*_d)^f?*f}OpcWXf5s@dFX zEpe(Nat?5fC~YIGj1kMiha-(O-#gbqTipk2bwNzSJ86O27!-w@u)6&O_{2MT{*h&k zW?Kgyzh?7Yt@hWKeFs&RQ?$NJE7~b(Az=`Yk;WAy za(uKUHf5dawAMUp3R)AxF{ZMfZ++X_lMp4E_TT>xiybZ2x3zMS(LFgM zBK*z?`0e7OXu3dc9{O1(v3$?uxw3qt<3aFFKX)g1TKp%trAW=DaN6U)roCqY(q(@4S1FlEUYV==?r0d;YC8TT0%u4Y;K*X z4YmX|a+tl&zT`1~p%1X8-O{;b40OzMrh;rRH`KL770rcl=RD0dc>3;aox?3%g}7^J zy`Tvz8-w&~*VqgAusXDqR`AE-&ZOhm3f zga0KJ<+U1-Qy1nCerG*rK{E+mSIY)KDhRWAadNUGXAa0M)D$x}zc(@+T(kMXAR(Zf zbeP}4g*a&A8afQj3t)gIdjQ`ZtQJkvM9xIYPkg*X@+Ga(D8Ajwuh-W9=udlZJH0sR zS6d~j74Q`1J{91}nORL=-mn_A{HVTAALdYG(?+AU39GQ1(2?0ir%M)6UN@Da28f)A zr7tcrSy~Mg=K=OAOSVJnw8azW2?yQ49sN1d${j9_x4AHLe|y!xU9NkVBC10deiqKs zf{xSDyJse{9`ig0GXoCBM-ne=yg7FrjegXJd{$)kESd?#p^hzo34ShX63C@_(NAG< z>TV-b3ej4lxM8TGDGFb>4)tfWX9L5ruBxeFLApyEDOFYBY>k;-S)blasJoaK;!@^a zW7Iu~(B#Pd($-LmN;XIkzWPAgM9T;p$_FL=cy-NgcgH57`M|o0@iWt!Aqom5&wMnFDV$Y zk$wYDQZ|%SOY#E6pUNl7CzW{b@Fj2GSLCyk&rV*%9qXsSNVbVCa>z@yKHFOG>6oyX%++qr~t?RKU|FCvr6 z7MJI<70c6E8JGzhYld&GPho2 zrU_KDhARVO^p}CZhOVvl(sJx}H3qTnC-3_sR^Ec}0!0qGZ14cW4ZN;uQK78-ZmX)e z!<7UgPspjr+kn-ctH6G4XaQElDMUxq{njH0ZSACx7+w`$xztVz&y{Klh! zC{PPpgLhAt5-Sr3CoWo_qqKTq60u%PRzy^{?rki@V~z*pM~5fJZV_DhDKTI{SfgbL zs=LL{6OLIscBygv-!F1&Z#eEqo{=g~9|7pI!NNM(&;Ph9x#+W(4$)LsV7` zRHZiL;S407AAL@S~Fq>MI#mL|EPy~%hqppwknro6xgRsFNW4G%p?_;0*UQDi`S%})2)XG zo8|rh;bgl%S7AT*o+<>^#vr%F(-GC%E=~{Vyo2*2K4BFu-j3uijKW*`H=uA^6A z1gl|<0yZrc9a~n6;VRn2ZUj*ds7@IzLW$f+{yb+j3L<0HN!gAK^0e}K+Zv{GL;e_c z_E*GHIv6+EHPg;vT0=@`)o3!N1r?b&h#)h@rV?1As+QOxwjMoNM5^@3#yG?vzk>^< z&8RT(BW*EG3nd4JMY>Lw%)Z#2{T=<%_O544Ta1RlNh~LE83)U<5JEe3S$W(f*k}p? zKMUk~&;=)P`Qiry+yz6m7FGr)=Sm?buAs;PLTa&d-%58BMr3v)0EKPy5v(?>ZJXUJ zSsQo7be0w6?tKn{$+U1O#pXV4>lA`P0u7VTVxzh2CEgkrVz0oC;!cj7Uyx{46v!=G zTZz%xv}}zfJi3>zQFmqIVZ$!N8+;OJud)K8<$cDAiqtG` zGL{gbq9PSQLQX@|59B=h>)@b6D+jisNV-(SC%}O~nc)+{Zdg4hl~f9NA>4O}@BvXf zzA_o8>#zn^vyJNPz^cCoVD6L2ci7|ce|>)PBG&)4sh#H%{`1bNB&XHIWEEfWYtuQ& zG5)MrrwPmy+)ke#2d=NrPd+<&ab(u+vtzT0fy3&vz+vUzw4-J!#hrfi;&=pMwg856 zE9=~*lsPyQGc#XWftOCW82}UT+$GM?0H$DYz;R7}SIV0GE@<-cTx78UqC;oIUi(}i zhp&Bh=@NbTyt|TC!|SynaPiYjo#b7SDJ8A$c@L|{Z#C)fjm_(^P21!Y^wyR(DQY|q zL8->fXz1u7@<{w!!O+X3$XKP-iPEyBQ+8qpwdG{I@301cCu0I3hi!F!2i+9`#{r3D zz$9q~KrpacMg$2ex`^0iO3A#Tx9Ys0O7rZ?LT1-jTGl%gPFMBSSwBHezrP}9TwRlz zDcRJStJd8GK%r6HwRPH{d{;@c8ER?PHOf9#QnfXiA}jxGN`=3nw~FB^jLj;Tuh|Zg z_$KN@bltZ8vEkT@vu{jz!7!a2i@I5VhdmFllXKaSTZ%R(U<~p37$11`lAH%M+*fsJ z#1dD*!x@}l`!Sx|@SFIAd;B>^Sy8;Kf0FPglI;1+9zuAY7ZaJ)?`cj=}Wu{D2%G z5E$Iw1#|f2zS0OpB1fZzr#c|hoscM*()=4y%y&(Ft{;b9Q8(5>GLd*$vGx&%*2;yI z72X<}QJ>E7kH=egGUdhDt~bE-1EG0>|J*Qgapo?Xiq(oM&4ea-M~L7kSsa~~^j}ht z^DnP#L&v{DW}H^7$1Ui#bByS}nwl9=E?JuNZ!XG)=5umJb)5JIEg+kgErv6Qe`A1w zaA4jE!e6QomVwMUeJk^RlGtI;fV=OYT*WK=JU)@mjDWn8f(^)fMss2~l>PPlnrk*X z+?8ZtPU5@1MY-I|Zu0K}SoS=Y65J@9fG{rPnAr2V%UOT>{-XD!zkPq<=4^hI6PPZ1 z6+P!YPB&HC08#2CG2-#wG?dp1szSO**cJ(2R6Dk>bBy+^Z6t5`H5Yk5rUtdWJqB&y zs0J+Wg!J>WDKwy%0*RZm0a4A4*om!)=`qU=PuOfhrfM)u@lkIaN~ZWp?PzqXlTBO zT4YA?LijEu;46E*xJpCN zPkbFMNJ`%or-zHA|GK%i-Lm052p^Mo@ih#0stUK88~3=I)3*6z#;VyQJyFiQ_WG*p zkb1?c8F_u>IbC#I4{$?FRW#F7=*l`&q~H^#dASRhMJWZ>vR1q1b*>*ni|-wUkhj67 zlw`6fSmsjZU>V(On^=r&+=&`p{~2sX);e^8TYmEX{PgVY`KZi)Gn2PNEr*R_L#}s& zQrH7R=s~cRDynDw6bg76Qlcp$;K6K64AOR!NyAw_x zk$AqL^w3%CPfG0G|8earyu?b*n`~&?Q+G?RI-jF!O3%y&U$Z#@l)j0qp(N^{L*h(l zuxclbV)ZCdR(hD7NfC}fa0Yc+a?X#rS?2p(R&Pl&_# zXg8)0rmRSiSvw|*gM*~E`s09xMs2%!iGZgcIwci+k?CUYC18w8%p1gJqS^H=t#hb< zQc=y(%tW|K>NTi8-9mU?*n|pO!QF6)dBtqNxv(%+Hh15!{S?e*?6fqZX~t@!ITdqq zTg%v7|DLWqdBf@(&Tda`Wqr-XbZ90sL}6Z?K)t+^-@rc!qv$PtC4pgXhVy5B48Ptr zSA#m%ToD<{X|=myYMM)h&zME-@GF{^k>E<7z&(+l37eM5S9bv z>mCBPATrYq(!&EHKY#xGIih|3$AA0>ku{+tXBjV-AB3m=_~J!;*Z+O`_6?+>Razk? zJC49K)agVG$bVgZ^R5L8?|utLM!#I~{fC}|=YRUc9}?c=HDpWTL*R zWmC0D?pV6`Jm zspGad6W=4_d z%;)Xy1a|IRq>-r6& zBpF+;yI`rM{vJ?(c^qO4{#;D3AbFQ1ThnPQSVWh$1O6IMq1Y_jqk)l-<8UcJId6O%1URZG=re;@*nW zhw|LZ9-HBI#%wfbpn}Agn5>Dp5J6vZ^$ZY7bXSh4y;h0L6i+uxDo&1GdCVi15OE z0928hO}Wy5Sm`k55ruE`Sn`=S3eoVf826<8K1guTyPH14nZ&n&%vO z`DG8z*Mr11i&2&w^7_R<{MJ@egZ7Xic zR5oKJO>PoN?X}seOY0pra=&H>bIs~f^=MO02A%YtgPRmA>5crV0Zm^fyVsW%`Lg32 zXN3_cdaQ?nL*eV;`A{<>Oy1x=?s0k2UC$_#`$g^H9Zyowx22}HxAZ=vEc%8$4ZL%% zD^w+}VpV*bq}{k36luHJpW*JKyA^~yUv%t?}st{UBMM@y>Rfppcl{6QS_ zXtJZ*yf{d1p`u!|TD&CxbM)iye;FQsadh;<^WlH~^!uY9NASz4|S;fVP5Lf6eUh@he|$nxB&H)hEXJR~WUNG)jmCIO975Msn?U^RYE6D_PTePI7Cb zpWlnhyO#=vFw~sayMO>@Id96H0{}xW;Y!~^JwaQu&_#;c$E+K0OtWh=3iz!^@qKg( z$Q>NmxVe&1;eBrqb18FnsQj}ygkJ713|_Ix6K#EUS(CRQY3{A^RRX#6;^2z^i@h;Y zc66XZiU`(E@EYzjp207G3*6Nb%sm$LdjfUr+W_dcL^K4Za7E4??@0|&F=TCeM=b}O zO@+cBPo6=Ua4;bOy(QDKX}qK#dD)btQD)>)blYkNj#cExc!o+Y;H8c^Eh#Vf%CBf^ z6UG2AYwJ-BsOEMwR%MhEh#d@y9J><0y%~kzMVC{!x~G1@8(~uh9_x92^fL7pBLigreIBO>QBK6W`Jz+8k;m0&qNSuLCC4B!RusjQ$1kt5=wqgM7|tz(Mhb3sd<`TDlC zyy1#VfVs1M$1VdUm%fZBh9JBuC@;y8VvL--qlMpGp@xfolK9x*FfzebAF{xE<`d`t z3Bt{Q!9-6$)F`gtp4PE859{^9 zMYyGNC_-b?XF36)uH+?lGm&*9VR})eQN&t$ED1#Dwd6KIn_Us ztW<73E9jovqvQOyL1Ln6L77IEN6&<*f03hQ?2oz)QdX-n@Vktrjsd6=Y~!^5_BcweLb)uzY|jb&L19^Isw9? zB=kn|oHPoA6hRsRZqWpHlHKj`oRnN?dd)`U%E*@}CU{v<8`%-cbhDf^Lh~X?d-UUi zr3z$EK;~*PBL-D98#;*>pGvS^;SJ+gsa@-VKPoedW~C~JE^J0q;$dG(U^MF^Mb^r) zKon|3-hpUZ%oEnh$cJcIvmSZDGNu%*=a~M*wMzzU6q_^!Nc8cGL`7o;L~52vp>*A3 zu*;N;YkAE?n?c+&Z?VnxG4hs>B%4t&WgcIRlr#rQ``Rm)DX7YBAL+!XWn9CCj*VF8 zV;`cMd+*cg${+wj@M4#lj3v`Fr<%UpQ(od*I`?rL7>^1Y6U3=nDa)ywvf?48-Q{<$ zlKKPv67~Yn%e6TJHs}Xv1T#im)}a}S>lv9o%SqlOd#TzBWb|yu?j|GF?`!iuw9%JY zH)(FA9x{R#n9#gt0S4l;v^WRY|3=`s?NS1g%9LrODheLB-(>VR*wAi33T@uGrWy-p4h&H5Y9_GgQ;7ULgvbQd`@zz5Q*3 zr;i=BU_gu;?w`Ihzw1cXR%i2eJcKMba)!N;B`XJe-hKRd!_GljSZzB>Zu!Edj5r0? zD33KbCN0Vi9Nk(L1*>66tub#Q6dy|DV~9n1M36HtdktK$TdxFAg@uJ&I3SqGUDn2` z{?l2;_s$4?D(1MDX8|#xX_rjR(2RC2=(sRaT0n4qTnl20mcdi+*<{co4lVCC5!V;W zJ)IXvZ3>T70ee*5S)iIulid*Vh2BG?;gus!9Yg=nNk}4spzxQyV%IifQ`Wi){X0X( z6#j6sl#j?7GjrHmoa<4q&MqdM{#FG0x2Nc9sHty*6C8$U7WELw}a3;`s( zCISXI)6#(Oi4jbs7(!r0@^D(0v(Nj1;Fy3HqnL%#GCynUP+%0(cZ*?s7F3|9-}TUf|o?d&df zm&r{#kRN-_qr8?? zW&LpF@#b@ez_CR>=r%bS#Y6xAh^EHu{^$rRWeTdjY_r!)7k%gBJBv^w`3BY7lBQ{AX;+0qOmV;(T7!PCJW1!2s* z>vM%|hF0D6r1nDeAX`B7Ae$q5kWUNtf@~l9Vp{^u8_WX5rpA^Xx&+oX88?$<)o2#5 zP0O(QUEPW2w*yUoQc^*uZ0LtFY;#vndf(i=d_%+oAw3=Dr^Ed4hxxsw#>L}oWuE(v z^Cs%Qm3?gAToGAvt(i;a@W*pNxbC1GS>^*iFU-YYAFG0ATnAMu@=nBr|QTS89nSs^4?{|)drANB1WJLH8V-g--hQOn~j1#M}; zdmoH&EL}K{Yc@Yr78q6vHRBar&)`3olZ{9ff4+ls64yC$mWoI~m#9E&r_ z)H|u+lLy^3ob+SZGwaso;ADo>2}BkHs1Ca~F4)=cFIVJ35Lw$L-sJSmm>U9i?tEfx z9LTEP?VifIv!y#3h#dr2obem5q(iupOO1n+`I>4r?GZ;6$Oo*R0&;RT>rb52dhN|r7HfSz*v{QKkO1Yvm!dH{ z_1&r$0>BTh`fldMWHEp$fUpYlC1q^^M3yoapl*CyK-HQ6(*wpuTkl_05+nInPTWe0 zGpZm`b7CVbs-~`_vPHZX4@a+jQJdWBPyrBs!=D^;-W&i2WX^^lyXXQAfI5X-J(=Wj zxe#(xt39S74hg|h4lyz=aDE$AQHODuT6V_^NG^yXnp~a6(orMY*gJQttPUdvsM4`) zge5r#a=P6GGBb8?IDQzGaD#MXJ~8vQ;5B`7RD+;`39D;cTYSFyq?24S+#%?7fcvs~ zsqAFZF9(dAr{t31Nm80>1|i-xn-oF$PDVySy90 z(+*p%G=!^$^|10s1v0PZ&v%DHks~>o1T}4sNBg}4-zJw^+t#HARlXeTYX@n)&0X7) zobny8U^yYLdBN0NX-taYCGt&73zC{}krj;%z(Fo*_}HFzWMeU;ZN|xg*89@K`M!Yk zwaA&$6QnGdY=1$e=j!&*6H?6=L!g%3)MHP2e3B(&NKdk^8TmCI*NnWHQ6X57#_XM^ znQBe58Qy=C7+zg4Bah6a(xjx>j0+YG6AB0qt{6}YJGTpH(24qB7JJ8NpxBE$pp%v_ z0Ta}cKqx_v5)Vsl`C4I)iNXe;`VZm&JR9HB`Nt7CU?24#4hH1l<3yQ%gr2B_5xFRR zZ~+&A^uz26M#2hY!_ls4KNOAVwSR|2Bg~exH9=p$BAV7yrcImCOC*|d%<98ORWiJ0 zH2+2v^Ia3R>!-VpVHNrbEfgb1=E0NOnrn7EBJbT>kc|nV?nI`DmlzFi8?i;`%{Bg= zl{K0KpE@q?SjvJ$f@30$eE&&u!uaIWmnNAMU0`dyl*L z0SY=1a(l~jwQItMPP%*NY$Ik*2y6<^nb3y$)ruU!8}SfY=Nyrfp&zJET8_DBk-%FT zGCH?mWzMUnm`3ze07Q~{MieibLQ}zHql)OP>0c#zq={Aq`-m)MS7kcd9=Z}4mCKe$XUfi4)xeyx5ow7ZD-qP>xD;`6Whpjx@n!TLs#9rBTlqN ztLTBuWRWYvKI)p93#T+!i*I{EsF;rqpY2Eme^`+<|8(IH5y35=7 zc*)R&-@0sC>x53R_$Ve6huY|2Ssg7&MV0av1i0ob<4_%htoS+(;5*?HBILrT!yIBC zSJinWx)bk(FR=&~6#(#qjST?H1b;_IyqRNQCxKbUubf?t9iqUe=HA<*_>vqP>0kD=!(;tDVb++}g5NiRO(Dyn0E_g=%WnCMqpZ9b5SYgx|; z)r2BH&YFT6B^#Z6ar=FCe)<0V)#-=xvzO#6+Zzw{u@U(Y7A6j0N312uk7r=-v`9sP*U%w*IHE9ZFN7HAF6gnJMnz- z8nWPb&R2X#9~5^c9(s<{@)kSJOqM0qxSd58wQFVB_RBg(S5g1a}Fkx~8#Ihec+weF?sgR65yM-_MxHLv{EqZqIK`bGb=iYdR zHLebt7%kWg-PQWo`Y0=3n^mpPx&^vV(d4ITt>k#|u9|KgEJJ8ExWiJ&vb-3}P*Hx3Q{^4{=c=Jt{&=f-T%0(*=CO(w>{ zAU4I_YK4W&d;WAr$eu=uiD=mw%+Gv768)iK>|@TAfV#{i7yh*(8P&xT zUk40fIYdtQL$29l)Xfuka8iNE`Z^Oa(XIgju>q3xDWpc8jrSJ`dZZ6&**&4V+l;gz zyVZ?)5Hz#+ECb?!pF1HPnIxiKRVB?v1H9&KJ>vPiES!=$_4HkcP_eNbP^xn~2ZKsd zyatgA0y+xPEWg4iOWMNb{ry9UT3irM3oMFjLs(Vrr8HrvX6g6zIa)6=@aBJu>Ofrh zMm2xsvlJ_bP1$Q) zpRyH6L0AaeHY~V+SWi%`?dD41yLPQX*VK@y24^x_U1ZHnn4&;#)4Xx;N;0_4TYs_r z>6MShQ*IUWF@94pjvt)5tpkMEPMR)*c78P0&iK@> znMNQ6@h3kRGqytULl4F2A3y=Vjxv?zUGMZH62Ccor{zI`5T`iTJZVjnGnO?*OTZSL z9kc9fqMmWwJWxlw8n3#be`NXz74yYJ1cXV%uOmTQEFHYnIUM;jJ&sWY#ldDyO<1Oy zX~}9HS(S=8%8qq!SIap3M#_;jiq`BJP4#yO$}=>2)A0#IH8x00l+; zjAW9m0jam0+6J}XAS72wylGuzeY3P9Lknp=*>9QMC*9g0q|TZ_31YZyCTxYJsR6&T z&ZBxj?}x!<48&zd3Q(J;4e6j|H&vGW#1oLmne|p$AY!h*PUNze?T6e(Y&xZIU@eB& z+N_U38)WhnWMTyvX9}P~j}7cPBlPl(hIMCpnuO}>{g#5?v|FR|*nrw5H;zQL&;gx9 z81%Cl<9nYoB=Fl1JoRI31}KSHBYn-FP}5O#$Jn%X^_-%O=2Dc8@@AYGSMDM9Ez* zM)@s^lU zqNDxn+`x2*)y6yKM9Rk>#Z-aJfTA@4sqC(f_m7MP+L@hRxQ$68$LqFL^5|g|cWK0` ze_JHyVZq2a{M%jK5|h(wqGmOzrLyehtfi$p^w8-;DwW;;RDoU^;|~+2km)7ECq7Rd zgKQfVCYOYv>m4}=knE#ZIibct=b-KMRhbMqr{vV-haBo!v60!Z` zOUOU?O7G(;q&0b{VO_uEtrkn{Lw$M-GuCac%*`j46i=7ej-@QfS%m1cmc&wkb^4KN zyqIJOQ-uGL;!W4bdKv_(C^>x>Mp(L+S2PtC@GLD6DsLk(E_I$hAMQ_<%fOv7ZAXWi z1~-Np-{EsXnbY>LZE(=WH+3gD@60o9t)Sf(T)E0ip49U%*JkKKQnmr9AXYAWQynSp z{*4JuxYQ`1BFTbd0ynTnIZurr+ySMb|(os8I;Y@Uv>e3Si~nKd20 zm4Pk{(5Kq?7p6#TF8N6T(#Mp!Ht&R_$|}*D#&+n}gXG}hUI+b`lS_)cslYREtx21| zKCMb&dNkEC7#{;bKTq%=r!B(E35&R^>9?pc%qL(}YN7vKene`AfpqlYWM+=w>67>S zfEOP==#c8*=sxgv5l^=>?B=~kcBh+w378ud`gspR&vrm&(eHC*iS=e3^Ki@7zw^_9 zBGLM=uhaLylPMq{P*NGL|A%f=$Qp~r*pqxw(%lVt77VaO{`*7z{mDVsKQ z^z||JDU%?`-w>@B)bZUbqM_2^aioAS(W#55zuH%U2+fxl=h!?oV@WXbLaddhI*k~Y zGPys}2_88Bbeb_gQhR+_Ed|vj#*R^r9j*WaNXxq%tAy`bYL*10Opkae96?R8#|Rtx zMIA_meKLKp(mVWQ{KD13j~JC}D6d0(xnT;;OmG{bS;0|lnZvn%5g`Hcdd@suAe0}n z(y`5ZiDm+d2)x3tZE}M>#us@4NW#@BE z!P2|et?!p8#iAml%dphpZ?WD?Xo2izc4vS9424}x0`%OcbM)A;l92UG(ACj><{RV(j%9YTiXK{Y2oIa#+fhJJmsz#YBA!mV`I)>_;TF&Q615m7 zT%($yPz68iZupmAZgKuJ2TbMJU)nAR~Oin^QWx#4QwtzAnFNm^o z{Y|b16wyVX7R72@Ai8%C=CdNVu$1iS(}x^ca5nx;3grp3Gke(%Q&z&8+rSod*a&sL zKH%UhwOz}Pw*N08HS-%+t;}^HP4ggQ_PXEw2vzi@>M*rd#5PbQ7=vsCPR-vS2}M72 zO{a7KEF!ZjW5omjq2P2PUg%mV0i>HSQK5RVbyZ!?n%bI^yQJ$C3E2U7H&mXDJc6r? zPFF5SmnK?(zD^WS;|6`uVc7^$VQrUc5xe$AJXQI!A}F(Agdeu38U9uCeOBfk8l;z* z>pb3ASWPZ8f#&bA9D`p9wSGM}_Wm@i$+j%T-&eI~deU;AROZHNk3WnGq;?hmFE|yN zh3;cY4b0Ej1+Ks!_B8|BlNTJUlpjDhn8swNvFHMJ=t7DTUdbbRv-=B60mBmW*kiV{ zBY@L*;SY89?8EM#gWv54G*&iMEHW-oY@M6PFl}w(Ei234V5w#s*VaeJnZZ#hWwxT) z;P!BWNsjL=HxMf|w7}%vAR4eUGl8!I6=^NUV!?0w)i5xb1rN7`=?1)pXmH97OuH^Y z{pwETf~%IG>fHL)VeNEimfe*{%F;TT4KsQuiS6i1#SvV#5k=^NszJL%YkqHOx6dpU z=&&|F22Gc9Gf<4ix)w73JRu{YA>DOdWA8m@v281Ju=KJ&w$PW4R3$ep!sr98um{F3 zpHf>uZXwa@f#KzIGVG(A-v=U_w(;2g<=IVFT*S5wDVc%dvgz~+dXP*4t8Pk6g-j`} zspRw~=QTB~cV(Zg`833|2@47h*9yZ~Pyr^IlNF@_Ptd(ki_pWDVZX5d zh%}@j6ojHr8-e(ct>vUW@t_s$K_U%o+HRyXqsh!V=!E=8I-C8h`pG)lk z65}Wt??Q3drOvsud_6zXJA-PLq|%o6o@*?d_(+lW>u>$nC>T*h`%eeWR&C&h$I({gVk*pQ8?BYJ&5`#J*r6B8Xxr2^rjRPXC*5FTB%gq^pX(dGH8Pl&L{M`jNw&QhyjL-FHO%`%>D>}b@YXDEBql#0b2IYjW z7bt`82pE9nPOztGg%;P7L#5u#$wGgt0~7YfDK&@@{-t9SEd!nyICk;YV!QAPmNr(J zOU%1Vl>4#2YOVakJ)-%X{M@qQ8rY__s-T;FR^LwY66dOe! z-_RcI6_euc!DNCYz7zArvID-Q!65K#;>Sf~^Y9if^-9;=Ujq2^=y#cVK*6g^7 zmA$@aPBrs25Uiy>@s&Df-C#h!KmLa3<94*Sf%$9AM>!zDss|sP#`@xvPqsZ-hep~g zit(UP)T30c^(t@f4v5{U=gLnlHD5?K%93WccI~bTgYQ3kXeEe$%v{PpyDYa%<^E4) zU~+Q5I;zQlqm;Y-m~+7ZIE-*H%1`tXIkW%rb+vU(PDP6+)PEdi!MeH`*4>w6-Qb@C zrc(aesY^w1JuDGEF!C61Hagf`Zn>9()Jx0FkM&FCtQr(t6?%!lDHP`&Sj=l0KU376 zfsHS{nzimuhem=HJ8XmY9r6k>BLH~w7e;W<=8vdQ~C0tX@n0IEj8 zV`{VYKhHF)85{LRSnnyBb!+PB8}&ysEOF&<@S+8RPM9&BL=|%hdpVx06GEhp$uU;H zlrwX=C6+kAl$X**%lVG6S#CH>yH~ksMew*sj|__?u$6q}0f5QXU%H9V)y0cO*LWaj zTtLd^@xL_BfHK&{Rw65_UrJ9e3m1ht`Ndl=t4;Uiqe%`Mm@|dtc{-%_B=FEV;1flU z)mH%O&AxxP+q(z@BO6d>B2-HoCYUkCezEZXwU`N32(tI=Su`}tQXKC{v6$<% z24^&G%nz>_t?) zJ~Eo(%kap1p|jM(?KV;P~2qMtmH+jEsni@_KzeSHP#7@ZKHh`uvf0 zkMjL;cm(GBA+L`+IH2|EsIuatd6ZigBrZycow~JJxLU)l@$AV%-NI3|bY}Zpi~_ci z=)v9SHR%KR8J79VcI2C!g5@$^?N!M%YA!1=d+HLY*ci{0%{z~wQ^I*UCYSO4IC&l2ArCG?4CMFVb>y_L`AZ!5dO!rK-qioFG;TK`pk0nG{ci-HM~S$U zD_lk(tI9Rzt%jB8c$Au~i-sB%EOAKXnG|W{ctdqEONp7A^psr`A_PDGIN9pl084ZzIkXCt;rsf?P^g%?U3$91BQM6&eW8kEF}2)nZFxU~GbBh? ziJ12JdjjS>ty#yl^wp*#M^Z@ZaFLOqMO_D`qNYN9KZcrqFdCfU%Aug{%Ig%qt&7p&S;cW=it_%sr;z+sH zb2u^tAQ&~mMA%D>W~%sM3R_F`b5e47u3aUo&~Yt1b*|&KscKw!D*PYuCM#3~TaTJY z&-gjN9KcA_W3Hw9kc@-D`(U5NUX7btu4Kx+(EO)r)OPvG?I(r|p5~zf6b>^>p9K06 z@ziJ6CLY!~lKZi9j@9N}#BBZ60PjXQF45e8+5X$ZHd>)hzJlh~{ZsnHF_%cfyyEBP zp=ysz9ZAmk{0sZ-F5vA2EsA>*zvvwLS$lc&FqH0n7VBX4Ebl^NWjf~eZ$|V*YCQI` zk+-prhlPfglLCeve=}Bjyx-(Us$UlZ32yS_B zA_w{Z5b0=+O>We+J!}4S*Zyfum5b(@;{2s`oF%d&en@jqyACq;cT@o8ET@E(a9!%ACIbG6e1c*ZO zumYWY(9U!#lQ$pom1(BW)EEa!lD8&5O;5CR_{gP>8Og6A&N~R`)L9(kxfMcZpck$W zbUp*P{1u=ek2oB_nQ8k_h`>-uY3e64&tzuneq#G+!d?~C|8~FWR+0)_><8t4tZEUl zSP#uQ;eRF{stjtQgM8cdb{%&GSsuZ6`N!fh9CVD)4r%QmzrV1c*rOtotUhcwy{f!HvF(7B)vA$Ff-fiID0 zAXqODd|+5FZ;RWjBMR#!>jFhUf2a{e7H>H9x4?cXR0L_Zo>j!4UL9f`6XE{QSut=m z=B-q#YMrN=Pdm3-(O9rDsjz3SNkcxg^owa zvGzq>({(@TBh2=jsh$rcTDVAln8QD7w+Po-FwIyND@mLpf#7aDIC{AAclVpPvK*Bs zV@AQ6))#6$rhga^*&tMQZdu-^bW0mxG1t3mzE+hX6*^ez;Cxv;PLoW%rykUaJ5nX81t+b113a!;=5aCaohW2NGWi({a%j?T8wZ@V~qCJYUa7 zU75(K9de@zMK>~sUir(PUSy`Pkxx_onFdzcCW^3U>^xR#+6993;&jt)V6dPM$tuaY%?h#?2n^2y*V@1>8DuR>Syc{~x|1}(t;D~jK|S?V-{Mqdh9nmQoMG~Bz+XHGcP!Y} z$Fb?p#1XFy0PTCCKx_7Xol6$X+iQXOeTi8*wZr90CG$jStSXTr7_(sXfX5AW0()0y z*fn(phTs02K$OZE-t>xf6IO|yiPDrORB@Xq#_w}2F6_=0N9rNA`cz#}fu89z zCpw~fvh6QSRM^q3xEG*(5WrX2e~_$AM@pRl9pGfVj}1LX!fjrAI-`e(i$U7ZJ&Vyz zCDimz%k!GwA~Cf ze<;^B2+l}AKR7{d9i!&WW&8OIh8`anz6eQP3!fS>mX^?q$bikkN=^x2xeTpV3 zD3y*dN#quQeB1AGGvx5v(to@i3zAY}yhrE(_P@wXf*t39+HZtJHs`I{j(+XSQ>`8T zztQmGOoZ@WK9X(EkxjzfJAQ>cUTzKLq9cjCUT%iRnQ3?N z#ZaVPC+%l~!4eJ3Vd$7BhZf7HTH8>kM?HtDE`RX2%p&wt-Pnu>|40yMXY&B7K{ZFIWO^@3;a@ibOS;`%rbm2| zxG(*so_J|~eaFbK15|piG)g~KQe5UiaHy~PdCTWJi!}9rg zQB<>Sr#V?hk)DnUP1GlRXt0jlczO^#)_BEFWtbpn77?U^F z+&?N-)V)zqDX98ra7QRh4GKQ12!~)Rn9I~jO9Fl-TCR{Br{v`@;Y_HtUJ*f#sT#HB z+bSj(!g9GEA*iAagZ*8`n>cY;Dr9D4{bt`<=jv6+*gN!x{_QUiR5f6%P%TJL$(Rj= z(jfqn87Uq2lwG+i%n%L=-RBk^e-2To)$hV-cBVPE zDH|v)%vcC4S_HaVp9Ac$ucE79Wj~dN>Zk1|(imKU<>mZEWs=>uJg}nB#05_d>WODD z7ROy26~LG^SrmvpV-j-{D6Ly{Sg+TM9SmMT4yC%1H(lis8? zAnQq-)$^9S=aYrnu9R{>~_QhQ~295Hr-fxy108!3u&C0c>SE8-9oio9WE@JAF1Y zOs*Z1T~9#P~hP)mxr%Px^iMfz# zMS#(EY-QFX1qkxv<$r_n6P?bNXLa1W$ArL;F%8_&reS+}M87{7f9Y#-^JxQ<4Vv{Z zx7E}kpLN5YSw#<8V8Mn|w8q0*Gj$Vd8-?3|4T7z;v3)siK@H54^Sjr*&9#AdeYgNd zQ$Z0YMf*|piu2bQibz?olS z%g~G5_^u9juL!gtS_FM&rtXThTD+{EA*jvQ6`W9AX^j@c9e;`|7eNdJDJ9(zIh0%{ zl;yqNcm1&Sp+mzL70wT&Rd;`tY}NVb2O~s*w~c5!+P4@RZJ6XFzu|RW=L= zB*XoX6$QV6vyf%%d|M!;>K$Bf*Z1$68=0MKyuj9x%~SeEJ^LVj$Bk{G{b6esz}N1K zPz=?zQ!oknx)}92beUzxnwtk&7|o%5^?-o`9aVv`A<$Bl!=VObH0f~7<8J$r553gZ zx9H`I#O0(!TU}u&4Ssuyzl5N5O#0ukgp%W`Sa`J-DZG=Z0bIoiEEt?&LS1>C01P8i zEEnVqnT|)Z@8$g`bSE|#dK<5;3XM(v>!L;;E^+L;=Y()s!?&veIiMun<~KSmcp#2*XVjib zq_R=-alLm3m+y^=h$x?%raB9}5@IPZ7{5ngPl!4@Y3T_DEmz;(3>5(wTW_X1w9Wcs zIPgSTPby{H=pcz+-JK3C&D%}vdH-jD`ayGdT^MWh!G0I`^ehGKxd^sl7m^*%+`e_ zLOIAZ2ujGAZ{RiysWvZNgyi-K8#dVB?`y+1fn`|pQ?iO~!&|!1zw1M_zT9jw;>rN> z{UpotZo0Hpy7sWcA5hAFGY_hCjdKmu!R3QS%!8&3g<$Vb6Kbov$^F|5Nbe>@TuNX9 zOx{DqD=sm*tTCvg135kT!Y)_$g~qeP#ri`*N>vk8(UFDcDXxH~M8P{%?B&#)m-w*C z48CWTOyi!Dza`;dTfBV*JorDic&NVvrl)~D-EN+rn;zWn$C%p=d>8_^-EF|~aAP6@ z$FLN<-5eESj-mLLXJ~2X0ujct$DyL1hWNLAT?;I&Z<>}*4%QUs@s3Kf$Nr38xp1k) zKi1Oxr}|NYKsVowXv{F$0e5##{3OydMy1S3yq&E5L#z- zzHuL}J?C8z$MxjXx2wVMW&rw9$9?FKWZ3qN4?@8X0x_5H#UJS9**=Iivc4Q+HAR$! z(wO_jx87n{f+!Q-8f3ATHLB*V=XOZuWWu&!IhnZ$fhFAC(qwx$?gjdRMxq+(d4Kj| zk7afgL065-hg(c7LQJ2D%-%4_-nq3(G5Z0l`fz<-SZT5+trDJ1hQAG?SV*_#52=<< zx>e(xHod|u7V$40M%WHr_82_5WWSRGYCDZeZ`faZnOCAovw(QOv9!S1D!) z4F{5;pa}rxcH|jt&K0Zg!cQLZ zo*$j@8$>o_vu65|lO2f%?S;p$?G~QzrmMGEm1o==o`(B9%$Kx?k-<^}u=|(~7%!vk zsH79OTXXr!d#Euj_UR{C_HC-{h74%yWdhf3GXS|#qv$}&n^6+RgQ?#znMkV#EhTuex~8#`^_jfN!m%&T_5BNl{otGf=AwBCPHVemErYY8R(&^g zi^&cRo;&g9uyyf;QR#*EE^s0ZG)hIOz z2gA=N@c)#L_3obX3}rxdZid+z>#W5TⓈDJY^x>R+K8{70XC#^2I$YtA*$`0Q3vfot?gw<2Uv82TJB2O8Jm`$CRViAgv_nX(#&r3-dk)gV^K|LnHU6M{xrcHT=Upa z9U|&*dWTw_yI{9=@b-ht;(n{IUw#PW5vS{cr5+$M?Q9R%pt_c+hglP`2e9r=8uPy5 z{L?*7yV4^$=nEQTGVotO+!y4M3%<FBcl9q=>N||-3AXsTm_A?K$LUdU zzl~Nf!8HvSuB3-Lb_tMGkZo(^Y>aO5Tbz0J1V6mb-CgW9o8vFOzEFuTD&phi8;qWg z4QNmEg*%8XV-?e$tAo3@ppH&! zM@2^X870!QmtfYbVQ;owSaA-02?phKAYC{Q^n9{cf|HEnYB@JGn69~?7UQtH#v`?% zr2dTRe*9_P9)hCuwAK%sNl3sqar`8^4E;uwrLPbOp81Q4a^)?u+F(Cnsjr9bg8Cb< zM7c`miJq@Qt_}QSdINQnkuUp%_oFlF+I@QvdXcpVOy)el1pM->@dV)0TEwRGM-wYl zv;x=?0$lO=f!4XIDF3rXepF@m9oWTUa#?J-*`|S=2C-k?5IHGOBSI6%SE?4{MU3!| z*}(fr&+V3A8+6RthmBuZSLw?XfUVC^i~R<7_CE&}dU34Hu6LJz)VooCO)sykXWufi zE{?lB8DP?FWnm}}h1A%0h8|g0Bs8?sysyl4vok)1$-Zbj#?^}>0)7jMKtlU{SsshY zNBNL)-EURiph`}bLMtLBTvZR;zu_6b*y=r$u;}Bp;o?Dt8-+xbG1WGbwJ5z4g{+IF zkf4~Qba^qQxM0kS2t$>suQfG>#$1Q(`()1Zd*2$8{+!gcT`k2aA|}|JTZ-1F6oroYh7&g0;DbpK&6W_`3#z^Rbztr^f2N z#X(h*HKjOsek%CrM;)ziaikc+o;hbuxVdU%g_56lpgfc&eWG;BwJu>u{E8ZBfgi1~ z{=D-efK5F7Vy<+<5tex<9!w0nW90%$^hduY0f9PX&QDfG9%xlScYG=r8m|S(L#EgP z+!90Lhsd!J^=ZaF4@0UBQF5Ru%R|sO79^hXlPsSF$rr1R*1lfV(>zs|?7C~4&*x?o@b%=A6tX(IN#P1X2rJv3JenK+YRfz|5W z-7D#fjomCMf^<82NXfcg@|XR-^yrLQNqV06`YUz~hLJlSftz!2k zwq+0U1p^S$MpfXpE|e*60d`kV+l+z&V#BCPRjEjbd=_TU^V1-`#$QNAbpFwghBz$- zJhL#=&V1R%=Ca7CWtIlbS6_zu6H1(Cr}A=bA#?8tdD9K;ezh3Eda5bxNYr{qi^j9~~*6RFy0&>+g_QVR9I{K zKmArJb4Vl!0IzWkKxXFyS=)fiPE}vSL&}R#qk^?%g{p?wjyB7OjHADuEMTL3J##M6 zD9%6q*(B0`{__sa@szczpT8G@u1?#0(=`pm8mReQUe5W zpknt}1Y0>vD$pti13xe;pzmWPPU?=3KZ1(#3*WIuX!u?ZQ^f(P$Vn7?bn4P++5bge zP7oMRmXeYAS=>VJUF(u-Q{rU5xSO&tZR>g?rm@;8@>~m&a{W;D{7w;Y;RHAvO(uzC zS(X*4`_RqOjExAI|98)~sp*)xF8%^Z+;y^jc@$M{&Nl7*I~i-Zl>=KkJ@D&!7?5gj zLZu6qVtXDD%1Gbs(8LrQIL1WUMOhe+d2Uk#`UFRaWSHgY)9`s7?geVgDMSYg6Y8|L>1+JeN(Mo+$|%m(jPVJ(uzp7ZEl^pD^q@UN1*JFOyE zi5dUb9FcKBJjLU|yS(>h$QO3x+TP4FQaM1CmB5pvz1;VhxEhQ@zB^CaA5G62&Gc?aThc((d+1&XoRaY*-xKf6V2I`!~ze zA-hO#w&u(c1b)MxBS&u(tcWxqp4T*t-B%{>0FVGhyOTlo@4{6F7|na(OT^=xLE!HG zN~9)`nU_M|WYv8a-w3>mWjz7iy=6!)o)nKg2qxJVfY=*#d&cp%-WdKvAPq@}mNeW& z$oBG=7jERDH3?zfxarG7!^c}gdrpC2%q#b5c?!G!q(MWctH=&yO+v!(&DZ3ikqo<3 zh0Em3n44FCJns}qesjW{ZJJ!#wBkyvrJ*s^Uy-ORs^^LsZ_hrwEV;kKxcnb7&lrx3 zL57B!eXcOwu%8zOTVZX?>q?lf!z!N`8>fPStxU^TWnXj_jbAl^Zp_UJ{)p@>Vy061 z>$e<(e=wH6N!#uM+gK4pcZg7Jr+N{H9~KVx)C!_tjStKw%B-I!XeP12ReX_JIikzI zibZI=0qw=Rr~zVTiZaE8iS)vdSlVJZ^H#WQIcKi#B#=WIW-wQQa(v$62?37-D)fXy!r-$akdU^A zEE1yHPgx4-`V}b-C&`m<_$t``I@r!El1kDL{V00fG#1IADec5%`uvH3+slH=y3ONK z88A)+=q9b4#h~rKw;-3Ut*!p#k+FwsKRU7yqDGP`4eq-|NCU)T*0P9er8bss13LLk z$Xsz$!CDchTHj%X^%|u1zi_2N@mg#jeK}U$44Jd26gYh@&L}P0I~SnFch-`{Zzmx1 zdr?9SWGRRj_GI;d^QTK~%J=J>>GI7kG-)d0S@I=(&+W~M5z>|WkcxB%T}KGs6m1Xw z5~tZk)0TvqC=g`QGz!ur(L}~q!=WJ)!^Uj;xeFn?(Q;EZfFPZA1BZ{>ug4m zQZ3dvT&#(X);5+pW}p{*1iW7OV3h^*QM$|BsknWl5?k@43GlR)&R)fm2Fc%@td4;_ zYi+Iopo69yFe(V@P@a;%XTJB%C~c_=`A3X#2o1nT04h3F6Mp)Kq2@pb~$$*xV z!yRyg_W-ID%j#rrU zdlc`|2we3o49(_i`|~lR9@Tz@4BX%Q(Lpi!#CLAQ{C_hxdF!uOtD2(^ThB{z=}6!;_J<|+VOg2C&mXY?*8-J)V1~P}YZYYIQ4Ms;KT$ zCvvT7!Fk^fGE;QO4^HH;Qbh%|i#1%17e=rUoL|Hn(a&XjA>s2MD$S#L+YAD|nCKE1j8R}k$JPy*(t*N)y z#N1oV!+NlghR^X57ogfgmA*o#IH^uKt53?_MfJ=xFa0wXQ2t)ps&WA((PS;Mq6}gc z$ZFbAjldTsC6XtRoEOb&U&+l}U*S6Qe4pEi)??Jv3*eQ@;gDSYl1RmehH9qmsOAM5 z^d}lzKqS1+A27MI^xAWlmqia%owLxh>YF13KB&GP5n+2c!Uqp$Qn|{`w$T)} z#F?VfJV4;CRr2pz{CglZ@X`1TvAW{2%b~CY3|eaLt(s$~>y8kdy(S^j8E00>8(S#n zKTTyOa+cIG<6ebOT0X8IbN`^+*{nP8IMk`eqP5bwI8lF2*mR}<(x`6fQZ51&2gx6c zbV_5g6bYuD&HD|b&`p+#yfS|on(%*+2#Sf{l!XZ)=0xum$;~3|p0&<;4l8A>K9T0@ z+#s4CPYu)*AS8}oTU=$07ynAq2PGpr*pzmIZrxjLHk8``3V{v?566&z;O7zuwis$8 z`mv@2XCxuU@W&m3_1Sx~fv3KEH6FKfEHt0ybjn`|P}}9w6oC34Q?WQt*|*RP-9VLa z3q_DN8lSy3ad=z=re=~26CGa7%#V*ebUf{~Le3Uh@)O=)&QqA7EWib6meY&4Xl#rp zdV~hI^3BrzHZsb<1g(k)#TG%H-=)lue-2!L^zHbPZGu_R^$w4rXvu0gS3+`Hsl4I& z-~7uAm~4+Jd;raa3W`^Tx*gz4|dPzD(l5yo^xKSW$#KoEcu zdnlRiu!XqE#N}wykVZFY25`fij21i^H~~Q>6SypVUD$ZqDWWfl6+NmWC^A#OYm}0{ zNHGvAZH`7yZzHJ=CKv?RpqU=d%74Reqy!NGBGU$%ZOJ&}3WugDQ0XM9Ji08aB`7cQ z!)_s7Cr}`>-wIOJEWft@ohbs`m_MEciZUI>w`ewhh*3}rt)t0(qGF;~m5?xM&H`V_ zP0$Q9<1|ZZ771>Z@D#q29EcnR9p&K(;q^k;Wy0sl>wynyVw&_vYI@5#2>9-e7~jy1 z)^r8CX#3_H#OJP*xX}?c*oSG7=Uk7|;?oGEz;3&m?(ViIJ}b=G5o2uFvE>I>Sibe= zCO+1r(|YGnN&mZxX&XoklDWjaXd>R~{h{IN9n2qVDUj~Z!vT`}=Au+EB1+4y0Pn7Yx-jY`SN7&jXVOJc?l-(^jt|*4@iVStCmn~Xb}2?tO6-r@UU-(= z*6z&=JOdw7tDe?l>hR-#cg%DT@E>vC3@U?2xJYdYzjrUb1dkH2+m(ep$8C+^(@~ZT zn6ss%F9EoC=?|-1dIZa@v8H?`ZOAvx%I=k8Gqu!cpc80VJ_+va2l||61@|vltb@F2 z1k*^;rN;jPvBII*>Z}eWxNz&^2^HfB1uytMl^u?2JMTECw~oC;&<0~)1k5E~9YQ;11ttG4HT6IXtJwGDRtog*#xU$#wj6@=Q zent;Q%G#O7J1}t5T2d0)3keW>rDC?aYWVrp9BbW`p$DIfO503S@RumnOJ@WRKAc27 zcE-K`@XT7MZ%a5uyIL3Jsj_p-l<}CSW=mQI!^gvF9%PDdj4PH*!MV6ax}xL&ommuc zk+A~Pl*i2HYE-6Xri{J-WVi^m>2H5yYh?0Y;vnVpQu;-`TO5t%uF$V>!+Pg6$$f=_ zAdcJ!_4$+uH|fcoJ+uLtLt(}yW2PB?2fmzO_ZxsjXI99{+<|fxRoRqG1UK5>>-~*i zQ9&P8s~Y;#`znrjYDcHq)g!gD4!l3@f}j$qJxi#*Vu;;$3q2bC=L`*I@=}LtnmjSN zHTUcYaH?1B_gpZJvJwN^u~AO+paRvgJX{x@+H8mIZ(*vo6X?4D&tyc>(Kma*KV;>c z-#}-|2{VvOg0`|I!`6isfh0I9sgagO#XZMxxt9kG^y3o%ps7o+ZB(Letl>czY7(xt2k)qXQY6si&u|&EQaY)m0E2!+N`#&M6GgK zbr{9!tR2Nf+ioE}u}x7D-lr1MGqq^_D04ubuQW1aAPCkNTpEGuo9a#YLKhclcbFm4hCG4@so>2yY?Ewgj&Sw zxC#A>%M-&>GCnTlWC>wpM^;{`4~N=_S|2pv5cvN*0m@KS^pV_q2HL)U8O_}KwpgSB z=Do+%+?|OV=nBTd(l+La8Zsf+W=bs_$r-X;*$0Ntf}IlOMQ&MqLHG}~LJ6X7d^bqW zvx%0+5GJOqxaHfbr0R$Y8^o3wL)+sW61A+1)ro%z<+L8RtVD<#l1&*C4>H-?lptJ9 zrW@<5rW?|f0D0v=%?vXqr+Ni5>VOq5YrqLuiheNzfz7Wn3%&fR-L`7uVq(3$d(5!W zq;4%8k5f_=mmauUHVVSOZ4mIA;<#rxoC%qzO%k{-3Hd5EYFKcQEw-sgOG;|0aEec# zGD>8k1>zj&kopxd+%N%`fj?09#C%n0r#wwShF@GK1c-)hM0;c$6&A)**M~uDtiJCH zA5IKq3C9`IieIG_guaPwrzEV*my0M)F7;S9i}?K$z;g$@JmZ^}mU;a8S9MP>!mQ1c zurvv6a?95D=`-a1W|`r*L-q`w62EU3paJ zWlEw(Mmth%+jbHwB2M|l4DfABG5arX=<_{N!utmvE{|G?0!O{lPtEG(Y^0pA}cfJ|J?qDQGQ@GSz7N-YQM6;Cht~k#MBGJ?>bhTRq*JzxZR0SKCCfvP!;#htfj|VX;DQBww$5=Jg1HLVxz^X$3Crll);vcAUt>iCteES8=_^@!O ztlfIqyOE}uXzbRP(%kQQEnF$Ez3=|;=8DnN_PwxeuH9JtFf(p9)ys}_EYnOhj>ue^ z`zL4@UcQQXVuXm+>xoalX*>R<2?aFXbT}DO=l*RV=3uXK@$kyz{M&PA zp3lN<_k|nwvX2-2R?cO?t76fjVs6TCrqRjWHP+)1%8Q9``xNOb_soa4vWz|V6at} z&=VW!ZsSV-Ed*EM*2(ht#{H?Z1OtqfxSxG#FE&RD`k)J7B@Hd#$t&QQI`?*O3QVdV1qj`W&GY_aFfhe? z>IPjQ+Z$1=kH;rtIDAjWCtmF;LI5Y7YfXizJU{K#H)iYypW+ITpG$F~Aq2D`TSP>v z_8OM`JddWniyKQJrBnGpNP^?tWv_2e@+`+apex^p?{{wpZo2`kpnXbh$n<=U%q6pc z`4y(6=eXz-9Z7xY7p~2~Z7pDtFJ@wP7O}pdrQ2wDOlj$=xvK1q{g^nKjE7yK)9zJh z)2k0x7nkp@#`Am!=_OQNYN_nrT=$_szvBFZh$)INnxb*Ey3~c94kr>GFiQV`A=o26 z>Sm=%yk*(h{w`Q{=0~d;l5n}OvoB^;j;f-r3>9ACVM}_7tcC@u6xCc}>d~Qf*Mp7z zu4-%8o0}Ee&5ZZGQrC&Un~rUfI`O`ow#+RnF$x?b6Inn$Npt8&B6u_9Bw78u=@7pz z7R%PZMlD1Wy)=Xc+eMBRxa-KE#iz&-QV||yS;+@kh$*mWN;NeNlovUqHb<%Hz_!_T zAiM+jQbSV>8|3{DAAy|~yjTabV!DQOES8j`XYV4^ThrGqie4)e8iP;}b7mIvLaO`o zzzQ?b1Y0PW`@t`jO7LPk5xvL3P8QF0kH%s!*t+%puYS9tc_3M~_V}Hj#E>Oe0(KCX za?Yna$J0R}Em5i14{xpm(Nlas$9HwbcxB~;XZDPhHGuyE^f?`zv}-naEfc89F;c|n z;=z?{8>;cTBzGs~lo4CROx{%o-;Ax|MA$J46vAz+zC?5I3ypCJTI6PzAS}`wi zp}8(D!-+N~2BaPI#H$l7iy#eZde9+7N9?_L5LLSk$~}V5O}kzgX3$1z1J?tD@Lrjg zp7zPfQn3hXPb;9SvDcIL+TP%M?KWd;H?)`P>Qb%3&tO~Ecjgku4v$&7rv9@ z4yx)2L4smhrE8&^*{}w2t_dh3-WXVQp|kt=L`R}}PhAVUlrI?1b7tgJ0X?L$tl(Ct zY%ykpM`|K|Ok2qi5ZfZfhY%V)nJn>XlJ`wFWdD`mMk}mb7{jMMLSUJ507sj0k-d=d ziN-#VqVlS9OTUD*x?Rg0wje+rT#hvS_BG5Pp-==&rkM1vXYTt^Zaal}4EK+w* zBRLBz&vy<8E(u|w(MC=ok*%5BKmY14O!4xhWD^bDAbMK$%X_H(Y}L>M>I3%4xD zPzKx;k8VLA5B*XXOp>q;e7My3=E`(h=`&)_RU`39WVMC;0B&1$sN{89S$D@UiVVs!BC9KD7fw)#Pj);pfB^pn^0evDnQ_CErjRpS z_|3v}i)9zIq2|w?vwCXvXYD8zbE`1^kPiSimQ6j6&GpmvfNfmEmb0 zrKxH-&c3VRHDJkEX8Cjp4PIBR?l!=cd$6lhkM^}YJjeEHcD=k9+R@8|i_tw$--#K2 z5NqXasq^s~aRp1TyHKa^hEU;v*Iy`eU&KiG5Q6#`kSp5F=@1l^(-#7H$Xf25~4ka?XyW5{J`AfRYk-w;=;1}%bztz0J zA$q@$sE+m!cejqRrRYEQwW!*2OkEBWia8S?An=}}XdG5IE}1HZBFrLmZDj4wlu8{!c-4eZA0 ztFrtLp6AtTJWah(v$A~&A)fD!tDya!+C@n$tsmdqKq-d(_BY>vO7=hh={Mh;jlX=v ziLW&t_E7o1txR6PX>@wnEhfM^flsmp7H(&E+$svkSGp$moWA;;zWPBDtt;88B$-~Pw< z7vEid4Eo~|ix(t3_&i1>7?HBd9 zYa2eLU3mQ6cd(=M3r5z&O))?!`47k_kvTQkj5mc0*dL(@`CL6cfvjjq1ZRye1pc@@oX`q)1>*^DS zmZB^Z23O9{ z7(s2Dw|~YFNT^9q)>YQXDm8j611gTO>B=$*7|D(@x5S0^BhVkhX zI9(qE<(J9-Y&bP$g56xb^Z9j#IrYy9l~7NBYI4h_(Z0uwT^VhGD#DdaLvI3Y9RyE} zAdxO@gqmn>%)Xlmp8ND9r;3%O$j6MmaocPnGvM3DbhW&8$~s_TB~z04SR|GP#TaTUSfejWLaB`xf?CYr&WhGXC&jO%S1BJaiG#NgU+yL%=}WLm zFA)6u3G%#sm3UlVt8?~VTlargKvfG%@KOu&UR!u_v^l_T9eW1oRxTk@21)UqP8g9; zo4|?^lj>gscq9(pR?EG`!|!i4kWx;ui>r0g4C$&hL3^y8ss`%-{sNdKWx+F%dw2%! zMu%H5O`zIeIsZg81p`Jl8@jFr9}qh($J8+Gb$?riZV$I>uD5mGi^ZV#{Th2QbZip% zfp#6xy$mJjOW(UAYQPE_PC2%RmAAb(i~5Kv%$%1_l>L{}RA9INH!HcU%$Qwx6pr{e zx@Sej463|@K)PRQ`M?WGI(AofL?&_VSk_t3+2c$bRC_{%_{)2-czK?9k6$@NT>;nX zWn*k?3Av!eL6F)@`0eGYn!AniI(Rn>{$RVf>*h}-QhM)t4fQK)jVb4D;Zb4eI=FgU zOsovBO?v7+Hp_wmvUwbgAo!cHXemKESQvfxJWjloe%VS6ZhY*H(jB60K!-(9?r# zt7fvEkeMv938$U{J+`lpGtt<4^@NBK(Tu(Q(hC{NwZa5obVm*|!490Vbe*&_)_1GJ z5WV5=csbBZ6TWM4Yw|l?Tc6O4zb6bA@UdC#U0-`P91pnWY_}6IAfX*FbIt_MW=<6w zp>y0Rie4kUbp1sQO)sB5sVg@OzhAFWfQ0vG_^2yGaYdvbswU+3MY357_JM~lBoh_% z4DQuSgbVIgA#(Rw*iSE#4R6c6Xy6Lm+NYq6TCLOpoayQHAoP}NJojd|2p9n{9HoIm zJ^B^RUKsjRGeI;JMg#~(QB}Hf{Sd}531d3v)xF4rS~O-iuEXAz@M3@g)X7Q@;uy1Y zxTc(p$0vkiyvo(_V(*57)}DIDOM~VPr=NgvQv-2~osnxQ{W6&pV!oS6cc3|UlPQ<_ zNo#w2=o|06Cj@Dq20sWiY4JEG1t<{#cp-+_td1O^wEXuuB9_+NX8_A`EyGP5Fc$O} zGvwaQ4j#(1ZUr4Av}yS-HPmVS9_$N!+WxY!`zDNeJ*w7ih13f;{NZj^MgnQz-lL-g zw;#VVx~OdmZOEpA;xh>v>rGqf2hnno(P;*n10FL&%iBWe4z(%|u_!0%{5#pr(k;k4 zTaSm>%~GK^cG*JP!Ei*9@+e>&ZG+FjAvt_?+W~XXru=#4Md)^Lp`gec~ST{hiH zAUuiGoKyvD`&-;f)80D$q6xT24^fIH)M`pwXfzhm5Qf}+nn2F9=-zE_hERSFC|2hf zo-_c`Xh}sQO?scf!j9ce1V50%MgV$KITMDg#fo1Pm|wkg6GtU{uiyVG}g+(?|g_VVG|mq!Z? zn)t5lG_*IZJBCw(;JRN731Mht!IHJlN&E+#yCV(DcCjI0C=VU%vAfUu`1YY~?Ly~4 z`)S*H237uS7kah}eN}d$NNMoLP|N5u+-ej<#Ca=G1J67Li|a7;flNf+KGab0p!4Bm zJU%%Y+|F?@{nmCe`YjVG(>YMYL%XYijdZGSbLWo@W^>PH4yL3*g`yJ$`gd|@6TeS) ze!Ht0I#8T9s%l_xpkWOB7NmzZ3E735ONxSaa!;5+i4TmOVgcii@m$PxwKxw2 zx*uCL&>LrTs}xMJVAh~#SfFAAC0z@*6pviig|8Lpw<^Eada{g;0{{r55a$FI)* z*Oh4l6?2Rj62eBcg|Pjq8ZRW#9A3>(Z!E8 z7<1{zoLl}PNnSn2j3sLUguwqCFm~jCF&y{jXtBY&={Z{LIa=&FTI|^>_-qy2=O%oP z7JH5s`$DcF&(UJf(PCd~v=}agH8EquZgWk**w+*$~3c;ObUC?-Mf1R&!ox>VC~ztouIY}A#-zRcUf3g=h!3Er`+5ZqYy?Kf;LRc%vq`=jgEZ+`va5Erj4(EyTnOnl9P z{|5Gr8TWzb+OY2cTI^E5U11nI@E|@)@rlpNLYVRWZ^1A| z`|VYAm%sXOb#eLbYCO-s5RSSQIW-Q^Lf7n(Bi>;Gf89q}zGhd-)D?Zdt{NwU_QIou zBmepcoOV0&&Q^E%t|Caz`E+j!0SXIN1K2V6s2>t++fDEfGyOQSdQ=-Rl6Ke8xEBkE z6?~E4gLoxu_9_$Q>=k{rr$MCBTK7t;!TJl&okH^AyUQOx3D9v+~XE()St(G&JYl(X={XC9n{V4B3M{Q>XHjFnm!NgJhGN)6(l*Sctn7kJVy zU81c}-Q{1x;y>zjb}!__I}p+zVto5a>9b1Mp1hi2y!2EcST!OaGOz7SC@V8Gom2R0 zEXLXO1g}|nk4sNKh^msgFf2)q$7U8;eCBCMb&aKj1EdE63^nX-=lnmO+y7_||7~6U zpUcTV=t%6fHgD(H?zdUcvgP@_(?T-XxY0Tk;mM#w5!&3p-xZC};^TDYGk=9$<&d0Qb7_pYko zlw0x0(=|bQ-w!SHoacEZjPWNMt;5cKT-;ohVt`cB3A zRZ3@WLOF~In&>J5v92qTGcvC{T_W_$FP8}Yfa-Q+{`T2Hjf(9hE}V%WP^hP5Wh8j< zqewEMmXfAP078BJoPDpS($m?w1%~6!yA|$dZTMLm{>rssV~Q)q?~3lAlD5(mK4EVQ zK4k*_@hX?b{Xe_C{T|%9wW7;(w{zD`IGtc2VIJk??&1uOig!JURbgj-oJn=lI8yf| z1DjSbsdCT%8(~w@8z2gK$T`Fa$vfeimEA|}V!B#xYO=C3;Jc*NNlm}$H8aNRr}@Tf z3jP4NrICQ&xG_4DypKgB8~5 zAG;3%Wys2IN?h)9SjFV$4!&%kiA(d1D7)!=7bny^;@(%|48anjj0{vNcQ z&2Mz|xBq6_ndP54bEJ2*gM;uHxyC{8l)9`742TT_J;*&i#7Bzt?{GVgQYiPey5p?O z;i^w`vN*t96@UN7dLzyg5MxIO}u#=!wJ4h@@eSg?!(Lu4ES z7~^1Y7zuv+fzTI;kQV7lXG!oC`vR>PkbjT{P%%77;}~ERp9O;A^8rsB3~u5WAQK0M zmKX*oF(jKQEt6^Z200*5#E$R~djo~o2NXoIC`(JD>IZvBNV59;c~fag@&i-jo22(1 zjc_o7$`D7zK5ZFIiEHgQ9xBh1Je-(_nZar&KMxv2+<8GxT>Sm^1*}w?G~YSC{lo2& z_dGoIIygDUH+#=NuhabR!yCTOkoeE*U(KKWt84tLrUhv1?Rg-WKz-DA#NdPCJ8aTE zvZkyO0H?qUGhU}xdXEihs$QSu){1%AgYW(9UVV12e&z1fV>n%A^_;7blY!uWZAINI z9mZqE5;)9YaKXF^xzV+fztp0!b_B&|k1i=FZ~sGt`q`3f%=Aja8}%&=mlt*Y!vqzy z%xiWgP))p9QWINuZITeMdHd~=u}4y<55n9@nxpE@&&+u6kN7i~rR;TC?{R?Q~F z98BA1bDqtl5>HUoepa*70x$}hEv_$z2VvlM{wy&e0K+|J-*MC9ih0KspQ5ej^!Dbv zcV}*~fIaH^a*fYE97$Ng``r`Smxf@_QM>Zx0fs^Cql>u5`)C5!;1Jv_gCAiyg-1v; zUWz1~r|e;#-yb@}qG|PZGPbLR5zNJlfo0C60#ex5?AsSZ%yz5BI;S+=i^Uolx}jy8 z?0xek7UT7u9yPp2@h$enx&d!LT{|8t4*^jxj~H8M)6kRu9{>RV|7h6}xH=m@076$s AUjP6A literal 0 HcmV?d00001 diff --git a/pkg/cmd/printcrds/printcrds.go b/pkg/cmd/printcrds/printcrds.go new file mode 100644 index 0000000..06f9a5e --- /dev/null +++ b/pkg/cmd/printcrds/printcrds.go @@ -0,0 +1,59 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package printcrds + +import ( + "bytes" + "compress/gzip" + "io" + + "github.com/spf13/cobra" + "k8s.io/kubectl/pkg/util/templates" + + _ "embed" +) + +var ( + crdsExample = templates.Examples(` + # Install or update the kjobctl CRDs + kjobctl printcrds | kubectl apply --server-side -f - + + # Remove the kjobctl CRDs + kjobctl printcrds | kubectl delete --ignore-not-found=true -f - + `) +) + +//go:embed embed/manifest.gz +var crds []byte + +func NewCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "printcrds", + Short: "Print the kjobctl CRDs", + Example: crdsExample, + SilenceUsage: false, + RunE: func(cmd *cobra.Command, args []string) error { + reader, err := gzip.NewReader(bytes.NewReader(crds)) + if err != nil { + return err + } + _, err = io.Copy(cmd.OutOrStdout(), reader) + return err + }, + } + return cmd +} diff --git a/pkg/cmd/printcrds/printcrds_test.go b/pkg/cmd/printcrds/printcrds_test.go new file mode 100644 index 0000000..5824391 --- /dev/null +++ b/pkg/cmd/printcrds/printcrds_test.go @@ -0,0 +1,51 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package printcrds + +import ( + "bytes" + "io" + "os/exec" + "testing" + + "github.com/google/go-cmp/cmp" +) + +func TestCRDs(t *testing.T) { + kustcmd := exec.Command("../../../bin/kustomize", "build", "../../../config/crd/") + wantOut, err := kustcmd.CombinedOutput() + if err != nil { + t.Fatal(err) + } + + wr := bytes.NewBuffer(nil) + cmd := NewCmd() + cmd.SetOut(wr) + err = cmd.Execute() + if err != nil { + t.Fatal(err) + } + + gotOut, err := io.ReadAll(wr) + if err != nil { + t.Fatal(err) + } + + if diff := cmp.Diff(string(wantOut), string(gotOut)); diff != "" { + t.Errorf("Unexpected output (-want/+got)\n%s", diff) + } +} diff --git a/pkg/cmd/testing/fake.go b/pkg/cmd/testing/fake.go new file mode 100644 index 0000000..2be6e8b --- /dev/null +++ b/pkg/cmd/testing/fake.go @@ -0,0 +1,148 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package testing + +import ( + rayversioned "github.com/ray-project/kuberay/ray-operator/pkg/client/clientset/versioned" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/cli-runtime/pkg/genericclioptions" + "k8s.io/cli-runtime/pkg/resource" + "k8s.io/client-go/dynamic" + k8s "k8s.io/client-go/kubernetes" + k8sfake "k8s.io/client-go/kubernetes/fake" + "k8s.io/client-go/rest" + "k8s.io/client-go/restmapper" + "k8s.io/client-go/tools/clientcmd" + kueueversioned "sigs.k8s.io/kueue/client-go/clientset/versioned" + + kjobctlversioned "sigs.k8s.io/kueue/cmd/experimental/kjobctl/client-go/clientset/versioned" + kjobctlfake "sigs.k8s.io/kueue/cmd/experimental/kjobctl/client-go/clientset/versioned/fake" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/cmd/util" +) + +type TestClientGetter struct { + util.ClientGetter + + k8sClientset k8s.Interface + kueueClientset kueueversioned.Interface + kjobctlClientset kjobctlversioned.Interface + rayClientset rayversioned.Interface + dynamicClient dynamic.Interface + restClient resource.RESTClient + restConfig *rest.Config + + configFlags *genericclioptions.TestConfigFlags +} + +func NewTestClientGetter() *TestClientGetter { + clientConfig := &clientcmd.DeferredLoadingClientConfig{} + restConfig := &rest.Config{} + + configFlags := genericclioptions.NewTestConfigFlags(). + WithClientConfig(clientConfig). + WithNamespace(metav1.NamespaceDefault) + return &TestClientGetter{ + ClientGetter: util.NewClientGetter(configFlags), + kjobctlClientset: kjobctlfake.NewSimpleClientset(), + k8sClientset: k8sfake.NewSimpleClientset(), + restConfig: restConfig, + configFlags: configFlags, + } +} + +func (cg *TestClientGetter) WithNamespace(ns string) *TestClientGetter { + cg.configFlags.WithNamespace(ns) + return cg +} + +func (cg *TestClientGetter) WithRESTMapper(mapper meta.RESTMapper) *TestClientGetter { + cg.configFlags.WithRESTMapper(mapper) + return cg +} + +func (cg *TestClientGetter) WithRESTConfig(config *rest.Config) *TestClientGetter { + cg.restConfig = config + return cg +} + +func (cg *TestClientGetter) ToRESTConfig() (*rest.Config, error) { + return cg.restConfig, nil +} + +func (cg *TestClientGetter) WithK8sClientset(clientset k8s.Interface) *TestClientGetter { + cg.k8sClientset = clientset + return cg +} + +func (cg *TestClientGetter) WithKueueClientset(clientset kueueversioned.Interface) *TestClientGetter { + cg.kueueClientset = clientset + return cg +} + +func (cg *TestClientGetter) WithRayClientset(clientset rayversioned.Interface) *TestClientGetter { + cg.rayClientset = clientset + return cg +} + +func (cg *TestClientGetter) WithKjobctlClientset(clientset kjobctlversioned.Interface) *TestClientGetter { + cg.kjobctlClientset = clientset + return cg +} + +func (cg *TestClientGetter) WithDynamicClient(dynamicClient dynamic.Interface) *TestClientGetter { + cg.dynamicClient = dynamicClient + return cg +} + +func (cg *TestClientGetter) WithRESTClient(restClient resource.RESTClient) *TestClientGetter { + cg.restClient = restClient + return cg +} + +func (cg *TestClientGetter) K8sClientset() (k8s.Interface, error) { + return cg.k8sClientset, nil +} + +func (cg *TestClientGetter) KueueClientset() (kueueversioned.Interface, error) { + return cg.kueueClientset, nil +} + +func (cg *TestClientGetter) KjobctlClientset() (kjobctlversioned.Interface, error) { + return cg.kjobctlClientset, nil +} + +func (cg *TestClientGetter) RayClientset() (rayversioned.Interface, error) { + return cg.rayClientset, nil +} + +func (cg *TestClientGetter) DynamicClient() (dynamic.Interface, error) { + return cg.dynamicClient, nil +} + +func (cg *TestClientGetter) NewResourceBuilder() *resource.Builder { + return resource.NewFakeBuilder( + func(version schema.GroupVersion) (resource.RESTClient, error) { + return cg.restClient, nil + }, + cg.ToRESTMapper, + func() (restmapper.CategoryExpander, error) { + return resource.FakeCategoryExpander, nil + }, + ) +} diff --git a/pkg/cmd/util/client_getter.go b/pkg/cmd/util/client_getter.go new file mode 100644 index 0000000..e97d653 --- /dev/null +++ b/pkg/cmd/util/client_getter.go @@ -0,0 +1,129 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import ( + rayversioned "github.com/ray-project/kuberay/ray-operator/pkg/client/clientset/versioned" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/cli-runtime/pkg/genericclioptions" + "k8s.io/cli-runtime/pkg/resource" + "k8s.io/client-go/dynamic" + k8s "k8s.io/client-go/kubernetes" + kueueversioned "sigs.k8s.io/kueue/client-go/clientset/versioned" + + kjobctlversioned "sigs.k8s.io/kueue/cmd/experimental/kjobctl/client-go/clientset/versioned" + + _ "k8s.io/client-go/plugin/pkg/client/auth" +) + +type ClientGetter interface { + genericclioptions.RESTClientGetter + + K8sClientset() (k8s.Interface, error) + KueueClientset() (kueueversioned.Interface, error) + KjobctlClientset() (kjobctlversioned.Interface, error) + RayClientset() (rayversioned.Interface, error) + DynamicClient() (dynamic.Interface, error) + NewResourceBuilder() *resource.Builder +} + +type clientGetterImpl struct { + genericclioptions.RESTClientGetter +} + +var _ ClientGetter = (*clientGetterImpl)(nil) + +func NewClientGetter(clientGetter genericclioptions.RESTClientGetter) ClientGetter { + return &clientGetterImpl{ + RESTClientGetter: clientGetter, + } +} + +func (cg *clientGetterImpl) K8sClientset() (k8s.Interface, error) { + config, err := cg.ToRESTConfig() + if err != nil { + return nil, err + } + + config.ContentType = runtime.ContentTypeProtobuf + clientset, err := k8s.NewForConfig(config) + if err != nil { + return nil, err + } + + return clientset, nil +} + +func (cg *clientGetterImpl) KueueClientset() (kueueversioned.Interface, error) { + config, err := cg.ToRESTConfig() + if err != nil { + return nil, err + } + + clientset, err := kueueversioned.NewForConfig(config) + if err != nil { + return nil, err + } + + return clientset, nil +} + +func (cg *clientGetterImpl) KjobctlClientset() (kjobctlversioned.Interface, error) { + config, err := cg.ToRESTConfig() + if err != nil { + return nil, err + } + + clientset, err := kjobctlversioned.NewForConfig(config) + if err != nil { + return nil, err + } + + return clientset, nil +} + +func (cg *clientGetterImpl) RayClientset() (rayversioned.Interface, error) { + config, err := cg.ToRESTConfig() + if err != nil { + return nil, err + } + + clientset, err := rayversioned.NewForConfig(config) + if err != nil { + return nil, err + } + + return clientset, nil +} + +func (cg *clientGetterImpl) DynamicClient() (dynamic.Interface, error) { + config, err := cg.ToRESTConfig() + if err != nil { + return nil, err + } + + dynamicClient, err := dynamic.NewForConfig(config) + if err != nil { + return nil, err + } + + return dynamicClient, nil +} + +func (cg *clientGetterImpl) NewResourceBuilder() *resource.Builder { + return resource.NewBuilder(cg.RESTClientGetter) +} diff --git a/pkg/cmd/util/dry_run.go b/pkg/cmd/util/dry_run.go new file mode 100644 index 0000000..5546705 --- /dev/null +++ b/pkg/cmd/util/dry_run.go @@ -0,0 +1,85 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import ( + "fmt" + + "github.com/spf13/cobra" + "k8s.io/cli-runtime/pkg/genericclioptions" +) + +type DryRunStrategy int + +const ( + // DryRunNone indicates the client will make all mutating calls + DryRunNone DryRunStrategy = iota + + // DryRunClient or client-side dry-run, indicates the client will prevent + // making mutating calls such as CREATE, PATCH, and DELETE + DryRunClient + + // DryRunServer or server-side dry-run, indicates the client will send + // mutating calls to the APIServer with the dry-run parameter to prevent + // persisting changes. + // + // Note that clients sending server-side dry-run calls should verify that + // the APIServer and the resource supports server-side dry-run, and otherwise + // clients should fail early. + // + // If a client sends a server-side dry-run call to an APIServer that doesn't + // support server-side dry-run, then the APIServer will persist changes inadvertently. + DryRunServer +) + +// AddDryRunFlag adds dry-run flag to a command. +func AddDryRunFlag(cmd *cobra.Command) { + cmd.PersistentFlags().String( + "dry-run", + "none", + `Must be "none", "server", or "client". If client strategy, only print the object that would be sent, without sending it. If server strategy, submit server-side request without persisting the resource.`, + ) +} + +func GetDryRunStrategy(cmd *cobra.Command) (DryRunStrategy, error) { + var dryRunFlag = FlagString(cmd, "dry-run") + switch dryRunFlag { + case "client": + return DryRunClient, nil + case "server": + return DryRunServer, nil + case "none": + return DryRunNone, nil + default: + return DryRunNone, fmt.Errorf(`Invalid dry-run value (%v). Must be "none", "server", or "client".`, dryRunFlag) + } +} + +// PrintFlagsWithDryRunStrategy sets a success message at print time for the dry run strategy +func PrintFlagsWithDryRunStrategy(printFlags *genericclioptions.PrintFlags, dryRunStrategy DryRunStrategy) error { + switch dryRunStrategy { + case DryRunClient: + if err := printFlags.Complete("%s (client dry run)"); err != nil { + return err + } + case DryRunServer: + if err := printFlags.Complete("%s (server dry run)"); err != nil { + return err + } + } + return nil +} diff --git a/pkg/cmd/util/helpers.go b/pkg/cmd/util/helpers.go new file mode 100644 index 0000000..4dc6d3d --- /dev/null +++ b/pkg/cmd/util/helpers.go @@ -0,0 +1,55 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import ( + "github.com/spf13/cobra" + "k8s.io/klog/v2" +) + +func AddFieldSelectorFlagVar(cmd *cobra.Command, p *string) { + cmd.Flags().StringVar(p, "field-selector", "", + "Selector (field query) to filter on, supports '=', '==', and '!='.(e.g. --field-selector key1=value1,key2=value2). The server only supports a limited number of field queries per type.") +} + +func AddLabelSelectorFlagVar(cmd *cobra.Command, p *string) { + cmd.Flags().StringVarP(p, "selector", "l", "", + "Selector (label query) to filter on, supports '=', '==', and '!='.(e.g. -l key1=value1,key2=value2). Matching objects must satisfy all of the specified label constraints.") +} + +func AddAllNamespacesFlagVar(cmd *cobra.Command, p *bool) { + cmd.Flags().BoolVarP(p, "all-namespaces", "A", false, + "If present, list the requested object(s) across all namespaces. Namespace in current context is ignored even if specified with --namespace.") +} + +func AddProfileFlagVar(cmd *cobra.Command, p *string) { + cmd.Flags().StringVarP(p, "profile", "p", "", + "Filter by profile name which is associated with the resource.") +} + +func AddLocalQueueFlagVar(cmd *cobra.Command, p *string) { + cmd.Flags().StringVarP(p, "localqueue", "q", "", + "Filter by localqueue which is associated with the resource.") +} + +func FlagString(cmd *cobra.Command, flag string) string { + s, err := cmd.Flags().GetString(flag) + if err != nil { + klog.Fatalf("error accessing flag %s for command %s: %v", flag, cmd.Name(), err) + } + return s +} diff --git a/pkg/constants/constants.go b/pkg/constants/constants.go new file mode 100644 index 0000000..778cb95 --- /dev/null +++ b/pkg/constants/constants.go @@ -0,0 +1,35 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package constants + +const ( + ProfileLabel = "kjobctl.x-k8s.io/profile" + ModeLabel = "kjobctl.x-k8s.io/mode" + ScriptAnnotation = "kjobctl.x-k8s.io/script" +) + +const ( + SystemEnvVarNameUser = "USER" +) + +const ( + EnvVarNameUserID = "USER_ID" + EnvVarTaskName = "TASK_NAME" + EnvVarTaskID = "TASK_ID" + EnvVarNameProfile = "PROFILE" + EnvVarNameTimestamp = "TIMESTAMP" +) diff --git a/pkg/parser/array_indexes.go b/pkg/parser/array_indexes.go new file mode 100644 index 0000000..24e8435 --- /dev/null +++ b/pkg/parser/array_indexes.go @@ -0,0 +1,170 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package parser + +import ( + "errors" + "regexp" + "slices" + "strconv" + "strings" + + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/utils/ptr" +) + +var ( + invalidArrayFlagFormatErr = errors.New("invalid array flag format") +) + +type ArrayIndexes struct { + Indexes []int32 + Step *int32 + Parallelism *int32 +} + +func (ai ArrayIndexes) Count() int { + return len(ai.Indexes) +} + +func (ai ArrayIndexes) Min() int32 { + return slices.Min(ai.Indexes) +} + +func (ai ArrayIndexes) Max() int32 { + return slices.Max(ai.Indexes) +} + +func GenerateArrayIndexes(count int32) ArrayIndexes { + ai := ArrayIndexes{} + for i := int32(0); i < count; i++ { + ai.Indexes = append(ai.Indexes, i) + } + return ai +} + +// ParseArrayIndexes parse array flag to ArrayIndexes. +// Include syntax like: +// - 1-5 - which results in indexes: 1, 2, 3, 4, 5 +// - 1,4,5 - which results exactly in the mentioned indexes 1, 4, 5 +// - 3-9:3 - with step indicator, which results in 3,6,9 +// - 1-5%2 - which results in indexes: 1, 2, 3, 4, 5 but only 2 of them are processed at the same time. +func ParseArrayIndexes(str string) (ArrayIndexes, error) { + arrayIndexes := ArrayIndexes{} + + var ( + indexes []int32 + step *int32 + parallelism *int32 + err error + ) + + if regexp.MustCompile(`^[0-9]\d*(,[1-9]\d*)*$`).MatchString(str) { + indexes, err = parseCommaSeparatedIndexes(str) + } else if matches := regexp.MustCompile(`(^[0-9]\d*-[1-9]\d*)(([:%])([1-9]\d*))?$`).FindStringSubmatch(str); matches != nil { + var num int64 + + if matches[4] != "" { + num, err = strconv.ParseInt(matches[4], 10, 32) + if err != nil { + return arrayIndexes, invalidArrayFlagFormatErr + } + } + + switch matches[3] { + case ":": + step = ptr.To(int32(num)) + case "%": + parallelism = ptr.To(int32(num)) + } + + indexes, err = parseRangeIndexes(matches[1], step) + } else { + return arrayIndexes, invalidArrayFlagFormatErr + } + + if err != nil { + return arrayIndexes, err + } + + slices.Sort(indexes) + + arrayIndexes.Indexes = indexes + arrayIndexes.Step = step + arrayIndexes.Parallelism = parallelism + + return arrayIndexes, nil +} + +func parseRangeIndexes(str string, step *int32) ([]int32, error) { + if step == nil { + step = ptr.To[int32](1) + } + + if *step <= 0 { + return nil, invalidArrayFlagFormatErr + } + + parts := strings.Split(str, "-") + if len(parts) != 2 { + return nil, invalidArrayFlagFormatErr + } + + from, err := strconv.ParseInt(parts[0], 10, 32) + if err != nil { + return nil, invalidArrayFlagFormatErr + } + + to, err := strconv.ParseInt(parts[1], 10, 32) + if err != nil { + return nil, invalidArrayFlagFormatErr + } + + if from > to { + return nil, invalidArrayFlagFormatErr + } + + var indexes []int32 + + for i := int32(from); i <= int32(to); i += *step { + indexes = append(indexes, i) + } + + return indexes, nil +} + +func parseCommaSeparatedIndexes(str string) ([]int32, error) { + strIndexes := strings.Split(str, ",") + maxValue := int32(-1) + set := sets.New[int32]() + for _, strIndex := range strIndexes { + num, err := strconv.ParseInt(strIndex, 10, 32) + if err != nil { + return nil, invalidArrayFlagFormatErr + } + index := int32(num) + if index < maxValue { + return nil, invalidArrayFlagFormatErr + } + if set.Has(index) { + return nil, invalidArrayFlagFormatErr + } + set.Insert(index) + maxValue = index + } + return set.UnsortedList(), nil +} diff --git a/pkg/parser/array_indexes_test.go b/pkg/parser/array_indexes_test.go new file mode 100644 index 0000000..94dfadc --- /dev/null +++ b/pkg/parser/array_indexes_test.go @@ -0,0 +1,115 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package parser + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + "k8s.io/utils/ptr" +) + +func TestParseSlurmArrayIndexes(t *testing.T) { + testCases := map[string]struct { + array string + wantArrayIndexes ArrayIndexes + wantErr string + }{ + "should parse 0": { + array: "0", + wantArrayIndexes: ArrayIndexes{ + Indexes: []int32{0}, + }, + }, + "should parse 1,4,5": { + array: "1,4,5", + wantArrayIndexes: ArrayIndexes{ + Indexes: []int32{1, 4, 5}, + }, + }, + "should parse 1,4,5, because duplicate indexes": { + array: "1,1,4,5", + wantErr: invalidArrayFlagFormatErr.Error(), + }, + "shouldn't parse 4,1,5, because invalid sequence": { + array: "4,1,5", + wantErr: invalidArrayFlagFormatErr.Error(), + }, + "should parse 1-5": { + array: "1-5", + wantArrayIndexes: ArrayIndexes{ + Indexes: []int32{1, 2, 3, 4, 5}, + }, + }, + "shouldn't parse 5-1, because from > to": { + array: "5-1", + wantErr: invalidArrayFlagFormatErr.Error(), + }, + "should parse 3-9:3": { + array: "3-9:3", + wantArrayIndexes: ArrayIndexes{ + Indexes: []int32{3, 6, 9}, + Step: ptr.To[int32](3), + }, + }, + "should parse 1-5%2": { + array: "1-5%2", + wantArrayIndexes: ArrayIndexes{ + Indexes: []int32{1, 2, 3, 4, 5}, + Parallelism: ptr.To[int32](2), + }, + }, + "shouldn't parse 1-5?2": { + array: "1-5?2", + wantErr: invalidArrayFlagFormatErr.Error(), + }, + "shouldn't parse 1-5:2147483648, because value out of range": { + array: "1-5:2147483648", + wantErr: invalidArrayFlagFormatErr.Error(), + }, + "should parse 0-5": { + array: "0-5", + wantArrayIndexes: ArrayIndexes{ + Indexes: []int32{0, 1, 2, 3, 4, 5}, + }, + }, + "should parse 0,2,4": { + array: "0,2,4", + wantArrayIndexes: ArrayIndexes{ + Indexes: []int32{0, 2, 4}, + }, + }, + } + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + gotArrayIndexes, gotErr := ParseArrayIndexes(tc.array) + + var gotErrStr string + if gotErr != nil { + gotErrStr = gotErr.Error() + } + if diff := cmp.Diff(tc.wantErr, gotErrStr); diff != "" { + t.Errorf("Unexpected error (-want/+got)\n%s", diff) + return + } + + if diff := cmp.Diff(tc.wantArrayIndexes, gotArrayIndexes); diff != "" { + t.Errorf("Unexpected array indexes (-want/+got)\n%s", diff) + } + }) + } +} diff --git a/pkg/parser/slurm.go b/pkg/parser/slurm.go new file mode 100644 index 0000000..ebdb2f9 --- /dev/null +++ b/pkg/parser/slurm.go @@ -0,0 +1,185 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package parser + +import ( + "bufio" + "errors" + "fmt" + "regexp" + "strconv" + "strings" + + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/utils/ptr" + + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/apis/v1alpha1" +) + +const slurmDirective = "#SBATCH" + +var longFlagFormat = regexp.MustCompile(`(\w+[\w+-]*)=(\S+)`) +var shortFlagFormat = regexp.MustCompile(`-(\w)\s+(\S+)`) + +type ParsedSlurmFlags struct { + Array string + CpusPerTask *resource.Quantity + GpusPerTask map[string]*resource.Quantity + MemPerNode *resource.Quantity + MemPerTask *resource.Quantity + MemPerCPU *resource.Quantity + MemPerGPU *resource.Quantity + Nodes *int32 + NTasks *int32 + Output string + Error string + Input string + JobName string + Partition string + TimeLimit string +} + +func SlurmFlags(script string, ignoreUnknown bool) (ParsedSlurmFlags, error) { + var flags ParsedSlurmFlags + + scanner := bufio.NewScanner(strings.NewReader(script)) + + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + if len(line) == 0 { + continue + } + + if !strings.HasPrefix(line, slurmDirective) { + if strings.HasPrefix(line, "#") { + continue + } + break + } + + key, val := extractKeyValue(line) + if len(key) == 0 || len(val) == 0 { + continue + } + + switch key { + case "a", string(v1alpha1.ArrayFlag): + flags.Array = val + case string(v1alpha1.CpusPerTaskFlag): + cpusPerTask, err := resource.ParseQuantity(val) + if err != nil { + return flags, fmt.Errorf("cannot parse '%s': %w", val, err) + } + flags.CpusPerTask = ptr.To(cpusPerTask) + case "e", string(v1alpha1.ErrorFlag): + flags.Error = val + case string(v1alpha1.GpusPerTaskFlag): + gpusPerTask, err := GpusFlag(val) + if err != nil { + return flags, fmt.Errorf("cannot parse '%s': %w", val, err) + } + flags.GpusPerTask = gpusPerTask + case "i", string(v1alpha1.InputFlag): + flags.Input = val + case "J", string(v1alpha1.JobNameFlag): + flags.JobName = val + case string(v1alpha1.MemPerNodeFlag): + memPerNode, err := resource.ParseQuantity(val) + if err != nil { + return flags, fmt.Errorf("cannot parse '%s': %w", val, err) + } + flags.MemPerNode = ptr.To(memPerNode) + case string(v1alpha1.MemPerCPUFlag): + memPerCPU, err := resource.ParseQuantity(val) + if err != nil { + return flags, fmt.Errorf("cannot parse '%s': %w", val, err) + } + flags.MemPerCPU = ptr.To(memPerCPU) + case string(v1alpha1.MemPerGPUFlag): + memPerGPU, err := resource.ParseQuantity(val) + if err != nil { + return flags, fmt.Errorf("cannot parse '%s': %w", val, err) + } + flags.MemPerGPU = ptr.To(memPerGPU) + case string(v1alpha1.MemPerTaskFlag): + memPerTask, err := resource.ParseQuantity(val) + if err != nil { + return flags, fmt.Errorf("cannot parse '%s': %w", val, err) + } + flags.MemPerTask = ptr.To(memPerTask) + case "N", string(v1alpha1.NodesFlag): + intVal, err := strconv.ParseInt(val, 10, 32) + if err != nil { + return flags, err + } + flags.Nodes = ptr.To(int32(intVal)) + case "n", string(v1alpha1.NTasksFlag): + intVal, err := strconv.ParseInt(val, 10, 32) + if err != nil { + return flags, err + } + flags.NTasks = ptr.To(int32(intVal)) + case "o", string(v1alpha1.OutputFlag): + flags.Output = val + case "p", string(v1alpha1.PartitionFlag): + flags.Partition = val + case "t", string(v1alpha1.TimeFlag): + flags.TimeLimit = val + default: + if !ignoreUnknown { + return ParsedSlurmFlags{}, fmt.Errorf("unknown flag: %s", key) + } + } + } + + return flags, nil +} + +func extractKeyValue(s string) (string, string) { + matches := longFlagFormat.FindStringSubmatch(s) + + if len(matches) != 3 { + matches = shortFlagFormat.FindStringSubmatch(s) + if len(matches) != 3 { + return "", "" + } + } + + return matches[1], matches[2] +} + +func GpusFlag(val string) (map[string]*resource.Quantity, error) { + gpus := make(map[string]*resource.Quantity) + + items := strings.Split(val, ",") + for _, v := range items { + gpu := strings.Split(v, ":") + if len(gpu) != 2 { + return nil, errors.New("invalid GPU format. It must be :") + } + + name := gpu[0] + quantity, err := resource.ParseQuantity(gpu[1]) + if err != nil { + return nil, err + } + + gpus[name] = &quantity + } + + return gpus, nil +} diff --git a/pkg/parser/slurm_test.go b/pkg/parser/slurm_test.go new file mode 100644 index 0000000..abb1871 --- /dev/null +++ b/pkg/parser/slurm_test.go @@ -0,0 +1,141 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package parser + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/utils/ptr" +) + +func TestParseSlurmOptions(t *testing.T) { + testCases := map[string]struct { + script string + ignoreUnknown bool + want ParsedSlurmFlags + wantErr string + }{ + "should parse simple script": { + script: `#!/bin/bash +#SBATCH --job-name=single_Cpu +#SBATCH --ntasks=1 +#SBATCH --cpus-per-task=1 + +sleep 30 +echo "hello"`, + want: ParsedSlurmFlags{ + JobName: "single_Cpu", + NTasks: ptr.To[int32](1), + CpusPerTask: ptr.To(resource.MustParse("1")), + }, + }, + "should parse script with short flags": { + script: `#SBATCH -J serial_Test_Job +#SBATCH -n 1 +#SBATCH -o output.%j +#SBATCH -e error.%j + +./myexec +exit 0`, + want: ParsedSlurmFlags{ + JobName: "serial_Test_Job", + NTasks: ptr.To[int32](1), + Output: "output.%j", + Error: "error.%j", + }, + }, + "should parse script with comments": { + script: `#!/bin/bash +# Job name +#SBATCH --job-name=job-array +# Defines a job array from task ID 1 to 20 +#SBATCH --array=1-20 +# Number of tasks (in this case, one task per array element) +#SBATCH -n 1 +# Partition or queue name +#SBATCH --partition=shared +#SBATCH # This is an empty line to separate Slurm directives from the job commands + +echo "Start Job $SLURM_ARRAY_TASK_ID on $HOSTNAME" # Display job start information + +sleep 10`, + want: ParsedSlurmFlags{ + JobName: "job-array", + Array: "1-20", + NTasks: ptr.To[int32](1), + Partition: "shared", + }, + }, + "should parse script and ignore unknown flags": { + script: `#!/bin/bash +#SBATCH --job-name=my_job_name +#SBATCH --output=output.txt +#SBATCH --error=error.txt +#SBATCH --partition=partition_name +#SBATCH --nodes=1 +#SBATCH --ntasks=1 +#SBATCH --cpus-per-task=1 +#SBATCH --time=1:00:00 +#SBATCH --mail-type=END +#SBATCH --mail-user=your@email.com +#SBATCH --unknown=unknown + +python my_script.py`, + ignoreUnknown: true, + want: ParsedSlurmFlags{ + JobName: "my_job_name", + Output: "output.txt", + Error: "error.txt", + Partition: "partition_name", + Nodes: ptr.To[int32](1), + NTasks: ptr.To[int32](1), + CpusPerTask: ptr.To(resource.MustParse("1")), + TimeLimit: "1:00:00", + }, + }, + "should fail due to unknown flags": { + script: `#!/bin/bash +#SBATCH --job-name=my_job_name +#SBATCH --nodes=1 +#SBATCH --cpus-per-task=1 +#SBATCH --unknown=unknown + +python my_script.py`, + wantErr: "unknown flag: unknown", + }, + } + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + got, gotErr := SlurmFlags(tc.script, tc.ignoreUnknown) + + var gotErrStr string + if gotErr != nil { + gotErrStr = gotErr.Error() + } + if diff := cmp.Diff(tc.wantErr, gotErrStr); diff != "" { + t.Errorf("Unexpected error (-want/+got)\n%s", diff) + return + } + + if diff := cmp.Diff(tc.want, got); diff != "" { + t.Errorf("Unexpected options (-want/+got)\n%s", diff) + } + }) + } +} diff --git a/pkg/parser/time_limit.go b/pkg/parser/time_limit.go new file mode 100644 index 0000000..4125a52 --- /dev/null +++ b/pkg/parser/time_limit.go @@ -0,0 +1,148 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package parser + +import ( + "errors" + "fmt" + "strings" +) + +var ( + invalidTimeFormatErr = errors.New("invalid time format") +) + +// isValidTimeSpec validates that time format follows is supported. +// Inspired by https://github.com/SchedMD/slurm/blob/9322d01b1aadde28181254499dd0d80638a3df89/src/common/parse_time.c#L87-L145. +func isValidTimeSpec(val string) bool { + var digit, dash, colon int + + var alreadyDigit bool + for _, ch := range val { + switch { + case ch >= '0' && ch <= '9': + if !alreadyDigit { + digit++ + alreadyDigit = true + } + case ch == '-': + alreadyDigit = false + dash++ + if colon > 0 { + return false + } + case ch == ':': + alreadyDigit = false + colon++ + default: + return false + } + } + + if digit == 0 { + return false + } + + if dash > 1 || colon > 2 { + return false + } + + if dash > 0 { + if colon == 0 && digit != 2 { + return false + } + + if colon == 1 && digit < 3 { + return false + } + + if colon == 2 && digit < 4 { + return false + } + } else { + if colon == 1 && digit < 2 { + return false + } + + if colon == 2 && digit < 3 { + return false + } + } + + return true +} + +// TimeLimitToSeconds converts a string to an equivalent time value. +// If val is empty, it returns nil. If format is invalid, it returns +// invalidTimeFormatErr. +// +// Possible formats: +// - "minutes" +// - "minutes:seconds" +// - "hours:minutes:seconds" +// - "days-hours" +// - "days-hours:minutes" +// - "days-hours:minutes:seconds" +// - "INFINITE" +// - "UNLIMITED" +// - "-1" +// +// For more details, see https://slurm.schedmd.com/sbatch.html#OPT_time. +// +// Inspired by https://github.com/SchedMD/slurm/blob/9322d01b1aadde28181254499dd0d80638a3df89/src/common/parse_time.c#L731-L784 +func TimeLimitToSeconds(val string) (*int32, error) { + val = strings.TrimSpace(val) + + if val == "" || strings.EqualFold(val, "-1") || strings.EqualFold(val, "INFINITE") || strings.EqualFold(val, "UNLIMITED") { + return nil, nil + } + + if !isValidTimeSpec(val) { + return nil, invalidTimeFormatErr + } + + var d, h, m, s int32 + + if strings.Contains(val, "-") { + // days-[hours[:minutes[:seconds]]] + // nolint:errcheck + fmt.Sscanf(val, "%d-%d:%d:%d", &d, &h, &m, &s) + d *= 86400 + h *= 3600 + m *= 60 + } else { + // nolint:errcheck + if n, _ := fmt.Sscanf(val, "%d:%d:%d", &h, &m, &s); n == 3 { + // hours:minutes:seconds + h *= 3600 + m *= 60 + } else { + // minutes[:seconds] + // h is minutes here and m is seconds due to sscanf parsing left to right + s = m + m = h * 60 + h = 0 + } + } + + res := d + h + m + s + if res == int32(0) { + return nil, nil + } + + return &res, nil +} diff --git a/pkg/parser/time_limit_test.go b/pkg/parser/time_limit_test.go new file mode 100644 index 0000000..1e666be --- /dev/null +++ b/pkg/parser/time_limit_test.go @@ -0,0 +1,121 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package parser + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "k8s.io/utils/ptr" +) + +func TestParseTimeLimit(t *testing.T) { + testCases := map[string]struct { + val string + want *int32 + wantErr error + }{ + "empty value": {}, + "-1": { + val: "-1", + }, + "INFINITE": { + val: "INFINITE", + }, + "UNLIMITED": { + val: "UNLIMITED", + }, + "infinite": { + val: "infinite", + }, + "unlimited": { + val: "unlimited", + }, + "incomplete formats 12-": { + val: "12-", + wantErr: invalidTimeFormatErr, + }, + "incomplete formats 12:": { + val: "12:", + wantErr: invalidTimeFormatErr, + }, + "incomplete formats 12-05:": { + val: "12-05:", + wantErr: invalidTimeFormatErr, + }, + "single digit minimal values 0": { + val: "0", + }, + "single digit minimal values 0:0:0": { + val: "0:0:0", + }, + "single digit minimal values 0-0:0:0": { + val: "0-0:0:0", + }, + "single digit minimal values 1-0:0:0": { + val: "1-0:0:0", + want: ptr.To[int32](24 * 60 * 60), + }, + "not supported chars": { + val: "12-0m-23", + wantErr: invalidTimeFormatErr, + }, + "more than one dashes": { + val: "12-05-23", + wantErr: invalidTimeFormatErr, + }, + "more than two colons": { + val: "2:12:05:23", + wantErr: invalidTimeFormatErr, + }, + "minutes": { + val: "05", + want: ptr.To[int32](5 * 60), + }, + "minutes:seconds": { + val: "05:23", + want: ptr.To[int32](5*60 + 23), + }, + "hours:minutes:seconds": { + val: "12:05:23", + want: ptr.To[int32](12*60*60 + 5*60 + 23), + }, + "days-hours:minutes:seconds": { + val: "2-12:05:23", + want: ptr.To[int32](2*24*60*60 + 12*60*60 + 5*60 + 23), + }, + "days-hours": { + val: "02-12", + want: ptr.To[int32](2*24*60*60 + 12*60*60), + }, + } + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + got, gotErr := TimeLimitToSeconds(tc.val) + + if diff := cmp.Diff(tc.wantErr, gotErr, cmpopts.EquateErrors()); diff != "" { + t.Errorf("Unexpected error (-want/+got)\n%s", diff) + return + } + + if diff := cmp.Diff(tc.want, got); diff != "" { + t.Errorf("Unexpected resutl (-want/+got)\n%s", diff) + } + }) + } +} diff --git a/pkg/testing/wrappers/application_profile_wrappers.go b/pkg/testing/wrappers/application_profile_wrappers.go new file mode 100644 index 0000000..de6cb88 --- /dev/null +++ b/pkg/testing/wrappers/application_profile_wrappers.go @@ -0,0 +1,59 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package wrappers + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/apis/v1alpha1" +) + +// ApplicationProfileWrapper wraps an ApplicationProfile. +type ApplicationProfileWrapper struct{ v1alpha1.ApplicationProfile } + +// MakeApplicationProfile creates a wrapper for an ApplicationProfile +func MakeApplicationProfile(name, ns string) *ApplicationProfileWrapper { + return &ApplicationProfileWrapper{ + ApplicationProfile: v1alpha1.ApplicationProfile{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: ns, + }, + Spec: v1alpha1.ApplicationProfileSpec{ + SupportedModes: nil, + VolumeBundles: nil, + }, + }, + } +} + +// Obj returns the inner ApplicationProfile. +func (ap *ApplicationProfileWrapper) Obj() *v1alpha1.ApplicationProfile { + return &ap.ApplicationProfile +} + +// WithSupportedMode adds the SupportedMode +func (ap *ApplicationProfileWrapper) WithSupportedMode(supportedMode v1alpha1.SupportedMode) *ApplicationProfileWrapper { + ap.Spec.SupportedModes = append(ap.Spec.SupportedModes, supportedMode) + return ap +} + +// WithVolumeBundleReferences adds WithVolumeBundleReferences to VolumeBundles. +func (ap *ApplicationProfileWrapper) WithVolumeBundleReferences(volumeBundleReferences ...v1alpha1.VolumeBundleReference) *ApplicationProfileWrapper { + ap.Spec.VolumeBundles = append(ap.Spec.VolumeBundles, volumeBundleReferences...) + return ap +} diff --git a/pkg/testing/wrappers/configmap_wrappers.go b/pkg/testing/wrappers/configmap_wrappers.go new file mode 100644 index 0000000..e73e4e2 --- /dev/null +++ b/pkg/testing/wrappers/configmap_wrappers.go @@ -0,0 +1,83 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package wrappers + +import ( + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + kueueconstants "sigs.k8s.io/kueue/pkg/controller/constants" + + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/apis/v1alpha1" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/constants" +) + +// ConfigMapWrapper wraps a ConfigMap. +type ConfigMapWrapper struct{ corev1.ConfigMap } + +// MakeConfigMap creates a wrapper for a ConfigMap +func MakeConfigMap(name, ns string) *ConfigMapWrapper { + return &ConfigMapWrapper{ + corev1.ConfigMap{ + TypeMeta: metav1.TypeMeta{Kind: "ConfigMap", APIVersion: "v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: ns, + }, + }, + } +} + +// Obj returns the inner ConfigMap. +func (w *ConfigMapWrapper) Obj() *corev1.ConfigMap { + return &w.ConfigMap +} + +// WithOwnerReference adds the owner reference. +func (w *ConfigMapWrapper) WithOwnerReference(ref metav1.OwnerReference) *ConfigMapWrapper { + w.OwnerReferences = append(w.OwnerReferences, ref) + return w +} + +// Profile sets the profile label. +func (w *ConfigMapWrapper) Profile(v string) *ConfigMapWrapper { + return w.Label(constants.ProfileLabel, v) +} + +// Mode sets the profile label. +func (w *ConfigMapWrapper) Mode(v v1alpha1.ApplicationProfileMode) *ConfigMapWrapper { + return w.Label(constants.ModeLabel, string(v)) +} + +// LocalQueue sets the localqueue label. +func (w *ConfigMapWrapper) LocalQueue(v string) *ConfigMapWrapper { + return w.Label(kueueconstants.QueueLabel, v) +} + +// Label sets the label key and value. +func (w *ConfigMapWrapper) Label(key, value string) *ConfigMapWrapper { + if w.Labels == nil { + w.Labels = make(map[string]string) + } + w.ObjectMeta.Labels[key] = value + return w +} + +// Data sets data. +func (w *ConfigMapWrapper) Data(data map[string]string) *ConfigMapWrapper { + w.ConfigMap.Data = data + return w +} diff --git a/pkg/testing/wrappers/container_wrappers.go b/pkg/testing/wrappers/container_wrappers.go new file mode 100644 index 0000000..4d434a5 --- /dev/null +++ b/pkg/testing/wrappers/container_wrappers.go @@ -0,0 +1,82 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package wrappers + +import ( + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" +) + +// ContainerWrapper wraps a Container. +type ContainerWrapper struct{ corev1.Container } + +// MakeContainer creates a wrapper for a Container +func MakeContainer(name, image string) *ContainerWrapper { + return &ContainerWrapper{ + Container: corev1.Container{Name: name, Image: image}, + } +} + +// Obj returns the inner Container. +func (c *ContainerWrapper) Obj() *corev1.Container { + return &c.Container +} + +// Command set command. +func (c *ContainerWrapper) Command(command ...string) *ContainerWrapper { + c.Container.Command = command + return c +} + +// WithRequest add Request to the ResourceList. +func (c *ContainerWrapper) WithRequest(resourceName corev1.ResourceName, quantity resource.Quantity) *ContainerWrapper { + if c.Resources.Requests == nil { + c.Resources.Requests = corev1.ResourceList{} + } + c.Resources.Requests[resourceName] = quantity + return c +} + +// WithResources set Resources. +func (c *ContainerWrapper) WithResources(resourceRequirements corev1.ResourceRequirements) *ContainerWrapper { + c.Container.Resources = resourceRequirements + return c +} + +// WithEnvVar add EnvVar to Env. +func (c *ContainerWrapper) WithEnvVar(envVar corev1.EnvVar) *ContainerWrapper { + c.Container.Env = append(c.Container.Env, envVar) + return c +} + +// WithVolumeMount add VolumeMount to VolumeMounts. +func (c *ContainerWrapper) WithVolumeMount(volumeMount corev1.VolumeMount) *ContainerWrapper { + c.Container.VolumeMounts = append(c.Container.VolumeMounts, volumeMount) + return c +} + +// TTY set tty=true. +func (c *ContainerWrapper) TTY() *ContainerWrapper { + c.Container.TTY = true + return c +} + +// Stdin set stdin=true. +func (c *ContainerWrapper) Stdin() *ContainerWrapper { + c.Container.Stdin = true + return c +} diff --git a/pkg/testing/wrappers/job_template_wrappers.go b/pkg/testing/wrappers/job_template_wrappers.go new file mode 100644 index 0000000..cc72f98 --- /dev/null +++ b/pkg/testing/wrappers/job_template_wrappers.go @@ -0,0 +1,167 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package wrappers + +import ( + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" + + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/apis/v1alpha1" +) + +// JobTemplateWrapper wraps a JobTemplate. +type JobTemplateWrapper struct{ v1alpha1.JobTemplate } + +// MakeJobTemplate creates a wrapper for a JobTemplate +func MakeJobTemplate(name, ns string) *JobTemplateWrapper { + return &JobTemplateWrapper{ + JobTemplate: v1alpha1.JobTemplate{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: ns, + }, + }, + } +} + +// Obj returns the inner JobTemplate. +func (j *JobTemplateWrapper) Obj() *v1alpha1.JobTemplate { + return &j.JobTemplate +} + +// Label sets the label key and value. +func (j *JobTemplateWrapper) Label(key, value string) *JobTemplateWrapper { + if j.Template.ObjectMeta.Labels == nil { + j.Template.ObjectMeta.Labels = make(map[string]string) + } + j.Template.ObjectMeta.Labels[key] = value + return j +} + +// Annotation sets the label key and value. +func (j *JobTemplateWrapper) Annotation(key, value string) *JobTemplateWrapper { + if j.Template.ObjectMeta.Annotations == nil { + j.Template.ObjectMeta.Annotations = make(map[string]string) + } + j.Template.ObjectMeta.Annotations[key] = value + return j +} + +// Completions updates the completions on the job template. +func (j *JobTemplateWrapper) Completions(completion int32) *JobTemplateWrapper { + j.Template.Spec.Completions = ptr.To(completion) + return j +} + +// Parallelism updates the parallelism on the job template. +func (j *JobTemplateWrapper) Parallelism(parallelism int32) *JobTemplateWrapper { + j.Template.Spec.Parallelism = ptr.To(parallelism) + return j +} + +// RestartPolicy updates the restartPolicy on the pod template. +func (j *JobTemplateWrapper) RestartPolicy(restartPolicy corev1.RestartPolicy) *JobTemplateWrapper { + j.Template.Spec.Template.Spec.RestartPolicy = restartPolicy + return j +} + +// BackoffLimitPerIndex updates the backoffLimitPerIndex on the job template. +func (j *JobTemplateWrapper) BackoffLimitPerIndex(backoffLimitPerIndex int32) *JobTemplateWrapper { + j.Template.Spec.BackoffLimitPerIndex = ptr.To[int32](backoffLimitPerIndex) + return j +} + +// WithInitContainer add init container to the pod template. +func (j *JobTemplateWrapper) WithInitContainer(container corev1.Container) *JobTemplateWrapper { + j.Template.Spec.Template.Spec.InitContainers = append(j.Template.Spec.Template.Spec.InitContainers, container) + return j +} + +// WithContainer add container to the pod template. +func (j *JobTemplateWrapper) WithContainer(container corev1.Container) *JobTemplateWrapper { + j.Template.Spec.Template.Spec.Containers = append(j.Template.Spec.Template.Spec.Containers, container) + return j +} + +// Clone clone JobTemplateWrapper. +func (j *JobTemplateWrapper) Clone() *JobTemplateWrapper { + return &JobTemplateWrapper{ + JobTemplate: *j.JobTemplate.DeepCopy(), + } +} + +// WithVolume add volume to the job template. +func (j *JobTemplateWrapper) WithVolume(name, localObjectReferenceName string) *JobTemplateWrapper { + j.Template.Spec.Template.Spec.Volumes = append(j.Template.Spec.Template.Spec.Volumes, corev1.Volume{ + Name: name, + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: localObjectReferenceName, + }, + }, + }, + }) + return j +} + +// WithEnvVar add volume to the job template. +func (j *JobTemplateWrapper) WithEnvVar(envVar corev1.EnvVar) *JobTemplateWrapper { + for index := range j.Template.Spec.Template.Spec.InitContainers { + j.Template.Spec.Template.Spec.InitContainers[index].Env = + append(j.Template.Spec.Template.Spec.InitContainers[index].Env, envVar) + } + for index := range j.Template.Spec.Template.Spec.Containers { + j.Template.Spec.Template.Spec.Containers[index].Env = + append(j.Template.Spec.Template.Spec.Containers[index].Env, envVar) + } + return j +} + +// WithVolumeMount add volume mount to pod templates. +func (j *JobTemplateWrapper) WithVolumeMount(volumeMount corev1.VolumeMount) *JobTemplateWrapper { + for index := range j.Template.Spec.Template.Spec.InitContainers { + j.Template.Spec.Template.Spec.InitContainers[index].VolumeMounts = + append(j.Template.Spec.Template.Spec.InitContainers[index].VolumeMounts, volumeMount) + } + for index := range j.Template.Spec.Template.Spec.Containers { + j.Template.Spec.Template.Spec.Containers[index].VolumeMounts = + append(j.Template.Spec.Template.Spec.Containers[index].VolumeMounts, volumeMount) + } + return j +} + +// Command set command to primary pod templates. +func (j *JobTemplateWrapper) Command(command []string) *JobTemplateWrapper { + if len(j.Template.Spec.Template.Spec.Containers) > 0 { + j.Template.Spec.Template.Spec.Containers[0].Command = command + } + return j +} + +// WithRequest set command to primary pod templates. +func (j *JobTemplateWrapper) WithRequest(key corev1.ResourceName, value resource.Quantity) *JobTemplateWrapper { + if len(j.Template.Spec.Template.Spec.Containers) > 0 { + if j.Template.Spec.Template.Spec.Containers[0].Resources.Requests == nil { + j.Template.Spec.Template.Spec.Containers[0].Resources.Requests = make(corev1.ResourceList) + } + j.Template.Spec.Template.Spec.Containers[0].Resources.Requests[key] = value + } + return j +} diff --git a/pkg/testing/wrappers/job_wrappers.go b/pkg/testing/wrappers/job_wrappers.go new file mode 100644 index 0000000..15b6fca --- /dev/null +++ b/pkg/testing/wrappers/job_wrappers.go @@ -0,0 +1,197 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package wrappers + +import ( + "strconv" + "time" + + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" + kueueconstants "sigs.k8s.io/kueue/pkg/controller/constants" + + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/apis/v1alpha1" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/constants" +) + +// JobWrapper wraps a Job. +type JobWrapper struct{ batchv1.Job } + +// MakeJob creates a wrapper for a Job +func MakeJob(name, ns string) *JobWrapper { + return &JobWrapper{ + batchv1.Job{ + TypeMeta: metav1.TypeMeta{Kind: "Job", APIVersion: "batch/v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: ns, + }, + }, + } +} + +// Obj returns the inner Job. +func (j *JobWrapper) Obj() *batchv1.Job { + return &j.Job +} + +// GenerateName updates generateName. +func (j *JobWrapper) GenerateName(v string) *JobWrapper { + j.ObjectMeta.GenerateName = v + return j +} + +// Completions updates job completions. +func (j *JobWrapper) Completions(v int32) *JobWrapper { + j.Job.Spec.Completions = ptr.To(v) + return j +} + +// CompletionMode updates job completions. +func (j *JobWrapper) CompletionMode(completionMode batchv1.CompletionMode) *JobWrapper { + j.Job.Spec.CompletionMode = &completionMode + return j +} + +// Subdomain updates pod template subdomain. +func (j *JobWrapper) Subdomain(subdomain string) *JobWrapper { + j.Job.Spec.Template.Spec.Subdomain = subdomain + return j +} + +// Parallelism updates job parallelism. +func (j *JobWrapper) Parallelism(v int32) *JobWrapper { + j.Job.Spec.Parallelism = ptr.To(v) + return j +} + +// Profile sets the profile label. +func (j *JobWrapper) Profile(v string) *JobWrapper { + return j.Label(constants.ProfileLabel, v) +} + +// Priority sets the workload priority class label. +func (j *JobWrapper) Priority(v string) *JobWrapper { + return j.Label(kueueconstants.WorkloadPriorityClassLabel, v) +} + +// MaxExecTimeSecondsLabel sets the max exec time seconds label. +func (j *JobWrapper) MaxExecTimeSecondsLabel(v string) *JobWrapper { + return j.Label(kueueconstants.MaxExecTimeSecondsLabel, v) +} + +// Mode sets the profile label. +func (j *JobWrapper) Mode(v v1alpha1.ApplicationProfileMode) *JobWrapper { + return j.Label(constants.ModeLabel, string(v)) +} + +// LocalQueue sets the localqueue label. +func (j *JobWrapper) LocalQueue(v string) *JobWrapper { + return j.Label(kueueconstants.QueueLabel, v) +} + +// Label sets the label key and value. +func (j *JobWrapper) Label(key, value string) *JobWrapper { + if j.Labels == nil { + j.Labels = make(map[string]string) + } + j.ObjectMeta.Labels[key] = value + return j +} + +// Annotation sets the label key and value. +func (j *JobWrapper) Annotation(key, value string) *JobWrapper { + if j.Annotations == nil { + j.Annotations = make(map[string]string) + } + j.ObjectMeta.Annotations[key] = value + return j +} + +// WithContainer add container on the pod template. +func (j *JobWrapper) WithContainer(container corev1.Container) *JobWrapper { + j.Job.Spec.Template.Spec.Containers = append(j.Job.Spec.Template.Spec.Containers, container) + return j +} + +// WithInitContainer add init container on the pod template. +func (j *JobWrapper) WithInitContainer(initContainer corev1.Container) *JobWrapper { + j.Job.Spec.Template.Spec.InitContainers = append(j.Job.Spec.Template.Spec.InitContainers, initContainer) + return j +} + +// WithVolume add volume. +func (j *JobWrapper) WithVolume(volume corev1.Volume) *JobWrapper { + j.Job.Spec.Template.Spec.Volumes = append(j.Job.Spec.Template.Spec.Volumes, volume) + return j +} + +// WithEnvVar add env var to the container template. +func (j *JobWrapper) WithEnvVar(envVar corev1.EnvVar) *JobWrapper { + for index := range j.Job.Spec.Template.Spec.Containers { + j.Job.Spec.Template.Spec.Containers[index].Env = + append(j.Job.Spec.Template.Spec.Containers[index].Env, envVar) + } + return j +} + +// WithEnvVarIndexValue add env var to the container template with index value. +func (j *JobWrapper) WithEnvVarIndexValue(name string) *JobWrapper { + for index := range j.Job.Spec.Template.Spec.Containers { + j.Job.Spec.Template.Spec.Containers[index].Env = append(j.Job.Spec.Template.Spec.Containers[index].Env, + corev1.EnvVar{Name: name, Value: strconv.Itoa(index)}) + } + return j +} + +// RestartPolicy updates the restartPolicy on the pod template. +func (j *JobWrapper) RestartPolicy(restartPolicy corev1.RestartPolicy) *JobWrapper { + j.Job.Spec.Template.Spec.RestartPolicy = restartPolicy + return j +} + +// CreationTimestamp sets the .metadata.creationTimestamp +func (j *JobWrapper) CreationTimestamp(t time.Time) *JobWrapper { + j.ObjectMeta.CreationTimestamp = metav1.NewTime(t) + return j +} + +// StartTime sets the .status.startTime +func (j *JobWrapper) StartTime(t time.Time) *JobWrapper { + j.Status.StartTime = &metav1.Time{Time: t} + return j +} + +// CompletionTime sets the .status.completionTime +func (j *JobWrapper) CompletionTime(t time.Time) *JobWrapper { + j.Status.CompletionTime = &metav1.Time{Time: t} + return j +} + +// Succeeded sets the .status.succeeded +func (j *JobWrapper) Succeeded(value int32) *JobWrapper { + j.Status.Succeeded = value + return j +} + +// Spec set job spec. +func (j *JobWrapper) Spec(spec batchv1.JobSpec) *JobWrapper { + j.Job.Spec = spec + return j +} diff --git a/pkg/testing/wrappers/pod_template_wrappers.go b/pkg/testing/wrappers/pod_template_wrappers.go new file mode 100644 index 0000000..8be6683 --- /dev/null +++ b/pkg/testing/wrappers/pod_template_wrappers.go @@ -0,0 +1,153 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package wrappers + +import ( + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// PodTemplateWrapper wraps a PodTemplate. +type PodTemplateWrapper struct{ corev1.PodTemplate } + +// MakePodTemplate creates a wrapper for a PodTemplate. +func MakePodTemplate(name, ns string) *PodTemplateWrapper { + return &PodTemplateWrapper{ + PodTemplate: corev1.PodTemplate{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns}, + }, + } +} + +// Obj returns the inner PodTemplate. +func (p *PodTemplateWrapper) Obj() *corev1.PodTemplate { + return &p.PodTemplate +} + +// Clone clone PodTemplateWrapper. +func (p *PodTemplateWrapper) Clone() *PodTemplateWrapper { + return &PodTemplateWrapper{ + PodTemplate: *p.PodTemplate.DeepCopy(), + } +} + +// Label sets the label key and value. +func (p *PodTemplateWrapper) Label(key, value string) *PodTemplateWrapper { + if p.Template.ObjectMeta.Labels == nil { + p.Template.ObjectMeta.Labels = make(map[string]string) + } + p.Template.ObjectMeta.Labels[key] = value + return p +} + +// Annotation sets the label key and value. +func (p *PodTemplateWrapper) Annotation(key, value string) *PodTemplateWrapper { + if p.Template.ObjectMeta.Annotations == nil { + p.Template.ObjectMeta.Annotations = make(map[string]string) + } + p.Template.ObjectMeta.Annotations[key] = value + return p +} + +// WithInitContainer add container to the pod template. +func (p *PodTemplateWrapper) WithInitContainer(container corev1.Container) *PodTemplateWrapper { + p.PodTemplate.Template.Spec.InitContainers = append(p.PodTemplate.Template.Spec.InitContainers, container) + return p +} + +// WithContainer add container to the pod template. +func (p *PodTemplateWrapper) WithContainer(container corev1.Container) *PodTemplateWrapper { + p.PodTemplate.Template.Spec.Containers = append(p.PodTemplate.Template.Spec.Containers, container) + return p +} + +// WithVolume add volume to the pod template. +func (p *PodTemplateWrapper) WithVolume(name, localObjectReferenceName string) *PodTemplateWrapper { + p.Template.Spec.Volumes = append(p.Template.Spec.Volumes, corev1.Volume{ + Name: name, + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: localObjectReferenceName, + }, + }, + }, + }) + return p +} + +// WithEnvVar add volume to the pod template. +func (p *PodTemplateWrapper) WithEnvVar(envVar corev1.EnvVar) *PodTemplateWrapper { + for index := range p.Template.Spec.InitContainers { + p.Template.Spec.InitContainers[index].Env = + append(p.Template.Spec.InitContainers[index].Env, envVar) + } + for index := range p.Template.Spec.Containers { + p.Template.Spec.Containers[index].Env = + append(p.Template.Spec.Containers[index].Env, envVar) + } + return p +} + +// WithVolumeMount add volume mount to pod templates. +func (p *PodTemplateWrapper) WithVolumeMount(volumeMount corev1.VolumeMount) *PodTemplateWrapper { + for index := range p.Template.Spec.InitContainers { + p.Template.Spec.InitContainers[index].VolumeMounts = + append(p.Template.Spec.InitContainers[index].VolumeMounts, volumeMount) + } + for index := range p.Template.Spec.Containers { + p.Template.Spec.Containers[index].VolumeMounts = + append(p.Template.Spec.Containers[index].VolumeMounts, volumeMount) + } + return p +} + +// Command set command to primary pod templates. +func (p *PodTemplateWrapper) Command(command []string) *PodTemplateWrapper { + if len(p.Template.Spec.Containers) > 0 { + p.Template.Spec.Containers[0].Command = command + } + return p +} + +// WithRequest set command to primary pod templates. +func (p *PodTemplateWrapper) WithRequest(key corev1.ResourceName, value resource.Quantity) *PodTemplateWrapper { + if len(p.Template.Spec.Containers) > 0 { + if p.Template.Spec.Containers[0].Resources.Requests == nil { + p.Template.Spec.Containers[0].Resources.Requests = make(corev1.ResourceList) + } + p.Template.Spec.Containers[0].Resources.Requests[key] = value + } + return p +} + +// TTY set tty=true on primary pod templates. +func (p *PodTemplateWrapper) TTY() *PodTemplateWrapper { + if len(p.Template.Spec.Containers) > 0 { + p.Template.Spec.Containers[0].TTY = true + } + return p +} + +// Stdin set stdin=true on primary pod templates. +func (p *PodTemplateWrapper) Stdin() *PodTemplateWrapper { + if len(p.Template.Spec.Containers) > 0 { + p.Template.Spec.Containers[0].Stdin = true + } + return p +} diff --git a/pkg/testing/wrappers/pod_wrappers.go b/pkg/testing/wrappers/pod_wrappers.go new file mode 100644 index 0000000..f05c636 --- /dev/null +++ b/pkg/testing/wrappers/pod_wrappers.go @@ -0,0 +1,121 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package wrappers + +import ( + "time" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + kueueconstants "sigs.k8s.io/kueue/pkg/controller/constants" + + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/apis/v1alpha1" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/constants" +) + +// PodWrapper wraps a Pod. +type PodWrapper struct{ corev1.Pod } + +// MakePod creates a wrapper for a Pod +func MakePod(name, ns string) *PodWrapper { + return &PodWrapper{ + corev1.Pod{ + TypeMeta: metav1.TypeMeta{Kind: "Pod", APIVersion: "v1"}, + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns}, + }, + } +} + +// Obj returns the inner Pod. +func (j *PodWrapper) Obj() *corev1.Pod { + return &j.Pod +} + +// GenerateName updates generateName. +func (j *PodWrapper) GenerateName(v string) *PodWrapper { + j.ObjectMeta.GenerateName = v + return j +} + +// Profile sets the profile label. +func (j *PodWrapper) Profile(v string) *PodWrapper { + return j.Label(constants.ProfileLabel, v) +} + +// Mode sets the profile label. +func (j *PodWrapper) Mode(v v1alpha1.ApplicationProfileMode) *PodWrapper { + return j.Label(constants.ModeLabel, string(v)) +} + +// LocalQueue sets the localqueue label. +func (j *PodWrapper) LocalQueue(v string) *PodWrapper { + return j.Label(kueueconstants.QueueLabel, v) +} + +// Label sets the label key and value. +func (j *PodWrapper) Label(key, value string) *PodWrapper { + if j.Labels == nil { + j.Labels = make(map[string]string) + } + j.ObjectMeta.Labels[key] = value + return j +} + +// Annotation sets the label key and value. +func (j *PodWrapper) Annotation(key, value string) *PodWrapper { + if j.Annotations == nil { + j.Annotations = make(map[string]string) + } + j.ObjectMeta.Annotations[key] = value + return j +} + +// WithContainer add container on the pod template. +func (j *PodWrapper) WithContainer(container corev1.Container) *PodWrapper { + j.Pod.Spec.Containers = append(j.Pod.Spec.Containers, container) + return j +} + +// RestartPolicy updates the restartPolicy on the pod template. +func (j *PodWrapper) RestartPolicy(restartPolicy corev1.RestartPolicy) *PodWrapper { + j.Pod.Spec.RestartPolicy = restartPolicy + return j +} + +// CreationTimestamp sets the .metadata.creationTimestamp. +func (j *PodWrapper) CreationTimestamp(t time.Time) *PodWrapper { + j.ObjectMeta.CreationTimestamp = metav1.NewTime(t) + return j +} + +// StartTime sets the .status.startTime. +func (j *PodWrapper) StartTime(t time.Time) *PodWrapper { + j.Status.StartTime = &metav1.Time{Time: t} + return j +} + +// Spec set pod spec. +func (j *PodWrapper) Spec(spec corev1.PodSpec) *PodWrapper { + j.Pod.Spec = spec + return j +} + +// Phase set pod status phase. +func (j *PodWrapper) Phase(phase corev1.PodPhase) *PodWrapper { + j.Status.Phase = phase + return j +} diff --git a/pkg/testing/wrappers/ray_cluster_spec_wrapper.go b/pkg/testing/wrappers/ray_cluster_spec_wrapper.go new file mode 100644 index 0000000..1d24779 --- /dev/null +++ b/pkg/testing/wrappers/ray_cluster_spec_wrapper.go @@ -0,0 +1,192 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package wrappers + +import ( + rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/utils/ptr" +) + +// RayClusterSpecWrapper wraps a RayClusterSpec. +type RayClusterSpecWrapper struct{ rayv1.RayClusterSpec } + +// MakeRayClusterSpec creates a wrapper for a RayClusterSpec +func MakeRayClusterSpec() *RayClusterSpecWrapper { + return &RayClusterSpecWrapper{} +} + +// FromRayClusterSpec creates a wrapper for a RayClusterSpec. +func FromRayClusterSpec(spec rayv1.RayClusterSpec) *RayClusterSpecWrapper { + return &RayClusterSpecWrapper{ + RayClusterSpec: spec, + } +} + +// Obj returns the inner RayClusterSpec. +func (w *RayClusterSpecWrapper) Obj() *rayv1.RayClusterSpec { + return &w.RayClusterSpec +} + +// Clone RayClusterSpecWrapper. +func (w *RayClusterSpecWrapper) Clone() *RayClusterSpecWrapper { + return &RayClusterSpecWrapper{ + RayClusterSpec: *w.RayClusterSpec.DeepCopy(), + } +} + +// HeadGroupSpec add worker group to the ray cluster spec. +func (w *RayClusterSpecWrapper) HeadGroupSpec(spec rayv1.HeadGroupSpec) *RayClusterSpecWrapper { + w.RayClusterSpec.HeadGroupSpec = spec + return w +} + +// WithWorkerGroupSpec add worker group to the ray cluster spec. +func (w *RayClusterSpecWrapper) WithWorkerGroupSpec(spec rayv1.WorkerGroupSpec) *RayClusterSpecWrapper { + w.RayClusterSpec.WorkerGroupSpecs = append(w.RayClusterSpec.WorkerGroupSpecs, spec) + + return w +} + +// WithVolume add volume to the ray cluster spec. +func (w *RayClusterSpecWrapper) WithVolume(name, localObjectReferenceName string) *RayClusterSpecWrapper { + headGroupSpec := &w.RayClusterSpec.HeadGroupSpec + headGroupSpec.Template.Spec.Volumes = + append(headGroupSpec.Template.Spec.Volumes, corev1.Volume{ + Name: name, + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: localObjectReferenceName, + }, + }, + }, + }) + + for index := range w.RayClusterSpec.WorkerGroupSpecs { + workerGroupSpec := &w.RayClusterSpec.WorkerGroupSpecs[index] + workerGroupSpec.Template.Spec.Volumes = + append(workerGroupSpec.Template.Spec.Volumes, corev1.Volume{ + Name: name, + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: localObjectReferenceName, + }, + }, + }, + }) + } + + return w +} + +// WithEnvVar add volume to the ray cluster spec. +func (w *RayClusterSpecWrapper) WithEnvVar(envVar corev1.EnvVar) *RayClusterSpecWrapper { + headGroupSpec := &w.RayClusterSpec.HeadGroupSpec + for j := range headGroupSpec.Template.Spec.InitContainers { + container := &headGroupSpec.Template.Spec.InitContainers[j] + container.Env = append(container.Env, envVar) + } + for j := range headGroupSpec.Template.Spec.Containers { + container := &headGroupSpec.Template.Spec.Containers[j] + container.Env = append(container.Env, envVar) + } + + for i := range w.RayClusterSpec.WorkerGroupSpecs { + workerGroupSpec := &w.RayClusterSpec.WorkerGroupSpecs[i] + for j := range workerGroupSpec.Template.Spec.InitContainers { + container := &workerGroupSpec.Template.Spec.InitContainers[j] + container.Env = append(container.Env, envVar) + } + for j := range workerGroupSpec.Template.Spec.Containers { + container := &workerGroupSpec.Template.Spec.Containers[j] + container.Env = append(container.Env, envVar) + } + } + + return w +} + +// WithVolumeMount add volume mount to pod templates. +func (w *RayClusterSpecWrapper) WithVolumeMount(volumeMount corev1.VolumeMount) *RayClusterSpecWrapper { + headGroupSpec := &w.RayClusterSpec.HeadGroupSpec + for j := range headGroupSpec.Template.Spec.InitContainers { + container := &headGroupSpec.Template.Spec.InitContainers[j] + container.VolumeMounts = append(container.VolumeMounts, volumeMount) + } + for j := range headGroupSpec.Template.Spec.Containers { + container := &headGroupSpec.Template.Spec.Containers[j] + container.VolumeMounts = append(container.VolumeMounts, volumeMount) + } + + for i := range w.RayClusterSpec.WorkerGroupSpecs { + workerGroupSpec := &w.RayClusterSpec.WorkerGroupSpecs[i] + for j := range workerGroupSpec.Template.Spec.InitContainers { + container := &workerGroupSpec.Template.Spec.InitContainers[j] + container.VolumeMounts = append(container.VolumeMounts, volumeMount) + } + for j := range workerGroupSpec.Template.Spec.Containers { + container := &workerGroupSpec.Template.Spec.Containers[j] + container.VolumeMounts = append(container.VolumeMounts, volumeMount) + } + } + + return w +} + +// Replicas set Replicas on WorkerGroupSpec. +func (w *RayClusterSpecWrapper) Replicas(groupName string, replicas int32) *RayClusterSpecWrapper { + for i := range w.RayClusterSpec.WorkerGroupSpecs { + if w.RayClusterSpec.WorkerGroupSpecs[i].GroupName == groupName { + w.RayClusterSpec.WorkerGroupSpecs[i].Replicas = &replicas + break + } + } + + return w +} + +// MinReplicas set MinReplicas on WorkerGroupSpec. +func (w *RayClusterSpecWrapper) MinReplicas(groupName string, minReplicas int32) *RayClusterSpecWrapper { + for i := range w.RayClusterSpec.WorkerGroupSpecs { + if w.RayClusterSpec.WorkerGroupSpecs[i].GroupName == groupName { + w.RayClusterSpec.WorkerGroupSpecs[i].MinReplicas = &minReplicas + break + } + } + + return w +} + +// MaxReplicas set MaxReplicas on WorkerGroupSpec. +func (w *RayClusterSpecWrapper) MaxReplicas(groupName string, maxReplicas int32) *RayClusterSpecWrapper { + for i := range w.RayClusterSpec.WorkerGroupSpecs { + if w.RayClusterSpec.WorkerGroupSpecs[i].GroupName == groupName { + w.RayClusterSpec.WorkerGroupSpecs[i].MaxReplicas = &maxReplicas + break + } + } + + return w +} + +// Suspend set suspend. +func (w *RayClusterSpecWrapper) Suspend(suspend bool) *RayClusterSpecWrapper { + w.RayClusterSpec.Suspend = ptr.To(suspend) + return w +} diff --git a/pkg/testing/wrappers/ray_cluster_template_wrappers.go b/pkg/testing/wrappers/ray_cluster_template_wrappers.go new file mode 100644 index 0000000..bc3459d --- /dev/null +++ b/pkg/testing/wrappers/ray_cluster_template_wrappers.go @@ -0,0 +1,75 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package wrappers + +import ( + rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/apis/v1alpha1" +) + +// RayClusterTemplateWrapper wraps a RayClusterTemplate. +type RayClusterTemplateWrapper struct{ v1alpha1.RayClusterTemplate } + +// MakeRayClusterTemplate creates a wrapper for a RayClusterTemplate +func MakeRayClusterTemplate(name, ns string) *RayClusterTemplateWrapper { + return &RayClusterTemplateWrapper{ + RayClusterTemplate: v1alpha1.RayClusterTemplate{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: ns, + }, + }, + } +} + +// Obj returns the inner RayClusterTemplate. +func (w *RayClusterTemplateWrapper) Obj() *v1alpha1.RayClusterTemplate { + return &w.RayClusterTemplate +} + +// Clone RayClusterTemplateWrapper. +func (w *RayClusterTemplateWrapper) Clone() *RayClusterTemplateWrapper { + return &RayClusterTemplateWrapper{ + RayClusterTemplate: *w.RayClusterTemplate.DeepCopy(), + } +} + +// Label sets the label key and value. +func (w *RayClusterTemplateWrapper) Label(key, value string) *RayClusterTemplateWrapper { + if w.Template.ObjectMeta.Labels == nil { + w.Template.ObjectMeta.Labels = make(map[string]string) + } + w.Template.ObjectMeta.Labels[key] = value + return w +} + +// Annotation sets the label key and value. +func (w *RayClusterTemplateWrapper) Annotation(key, value string) *RayClusterTemplateWrapper { + if w.Template.ObjectMeta.Annotations == nil { + w.Template.ObjectMeta.Annotations = make(map[string]string) + } + w.Template.ObjectMeta.Annotations[key] = value + return w +} + +// Spec set entrypoint. +func (w *RayClusterTemplateWrapper) Spec(spec rayv1.RayClusterSpec) *RayClusterTemplateWrapper { + w.Template.Spec = spec + return w +} diff --git a/pkg/testing/wrappers/ray_cluster_wrappers.go b/pkg/testing/wrappers/ray_cluster_wrappers.go new file mode 100644 index 0000000..ad0f45e --- /dev/null +++ b/pkg/testing/wrappers/ray_cluster_wrappers.go @@ -0,0 +1,178 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package wrappers + +import ( + "time" + + rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + kueueconstants "sigs.k8s.io/kueue/pkg/controller/constants" + + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/apis/v1alpha1" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/constants" +) + +// RayClusterWrapper wraps a RayCluster. +type RayClusterWrapper struct{ rayv1.RayCluster } + +// MakeRayCluster creates a wrapper for a RayCluster +func MakeRayCluster(name, ns string) *RayClusterWrapper { + return &RayClusterWrapper{ + rayv1.RayCluster{ + TypeMeta: metav1.TypeMeta{Kind: "RayCluster", APIVersion: "ray.io/v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: ns, + }, + }, + } +} + +// Obj returns the inner RayCluster. +func (j *RayClusterWrapper) Obj() *rayv1.RayCluster { + return &j.RayCluster +} + +// GenerateName updates generateName. +func (j *RayClusterWrapper) GenerateName(v string) *RayClusterWrapper { + j.ObjectMeta.GenerateName = v + return j +} + +// CreationTimestamp sets the .metadata.creationTimestamp +func (j *RayClusterWrapper) CreationTimestamp(t time.Time) *RayClusterWrapper { + j.RayCluster.ObjectMeta.CreationTimestamp = metav1.NewTime(t) + return j +} + +// Profile sets the profile label. +func (j *RayClusterWrapper) Profile(v string) *RayClusterWrapper { + return j.Label(constants.ProfileLabel, v) +} + +// Mode sets the profile label. +func (j *RayClusterWrapper) Mode(v v1alpha1.ApplicationProfileMode) *RayClusterWrapper { + return j.Label(constants.ModeLabel, string(v)) +} + +// LocalQueue sets the localqueue label. +func (j *RayClusterWrapper) LocalQueue(v string) *RayClusterWrapper { + return j.Label(kueueconstants.QueueLabel, v) +} + +// Label sets the label key and value. +func (j *RayClusterWrapper) Label(key, value string) *RayClusterWrapper { + if j.Labels == nil { + j.Labels = make(map[string]string) + } + j.ObjectMeta.Labels[key] = value + return j +} + +// Annotation sets the label key and value. +func (j *RayClusterWrapper) Annotation(key, value string) *RayClusterWrapper { + if j.Annotations == nil { + j.Annotations = make(map[string]string) + } + j.ObjectMeta.Annotations[key] = value + return j +} + +// WithWorkerGroupSpec add worker group to the ray cluster template. +func (j *RayClusterWrapper) WithWorkerGroupSpec(spec rayv1.WorkerGroupSpec) *RayClusterWrapper { + j.RayCluster.Spec.WorkerGroupSpecs = append(j.RayCluster.Spec.WorkerGroupSpecs, spec) + return j +} + +// Spec set job spec. +func (j *RayClusterWrapper) Spec(spec rayv1.RayClusterSpec) *RayClusterWrapper { + j.RayCluster.Spec = spec + return j +} + +// DesiredWorkerReplicas set DesiredWorkerReplicas. +func (j *RayClusterWrapper) DesiredWorkerReplicas(desiredWorkerReplicas int32) *RayClusterWrapper { + j.RayCluster.Status.DesiredWorkerReplicas = desiredWorkerReplicas + return j +} + +// ReadyWorkerReplicas set ReadyWorkerReplicas. +func (j *RayClusterWrapper) ReadyWorkerReplicas(readyWorkerReplicas int32) *RayClusterWrapper { + j.RayCluster.Status.ReadyWorkerReplicas = readyWorkerReplicas + return j +} + +// AvailableWorkerReplicas set AvailableWorkerReplicas. +func (j *RayClusterWrapper) AvailableWorkerReplicas(availableWorkerReplicas int32) *RayClusterWrapper { + j.RayCluster.Status.AvailableWorkerReplicas = availableWorkerReplicas + return j +} + +// DesiredCPU set DesiredCPU. +func (j *RayClusterWrapper) DesiredCPU(desiredCPU resource.Quantity) *RayClusterWrapper { + j.RayCluster.Status.DesiredCPU = desiredCPU + return j +} + +// DesiredMemory set DesiredMemory. +func (j *RayClusterWrapper) DesiredMemory(desiredMemory resource.Quantity) *RayClusterWrapper { + j.RayCluster.Status.DesiredMemory = desiredMemory + return j +} + +// DesiredGPU set DesiredGPU. +func (j *RayClusterWrapper) DesiredGPU(desiredGPU resource.Quantity) *RayClusterWrapper { + j.RayCluster.Status.DesiredGPU = desiredGPU + return j +} + +// DesiredTPU set DesiredTPU. +func (j *RayClusterWrapper) DesiredTPU(desiredTPU resource.Quantity) *RayClusterWrapper { + j.RayCluster.Status.DesiredTPU = desiredTPU + return j +} + +// MinWorkerReplicas set MinWorkerReplicas. +func (j *RayClusterWrapper) MinWorkerReplicas(minWorkerReplicas int32) *RayClusterWrapper { + j.RayCluster.Status.MinWorkerReplicas = minWorkerReplicas + return j +} + +// MaxWorkerReplicas set MaxWorkerReplicas. +func (j *RayClusterWrapper) MaxWorkerReplicas(maxWorkerReplicas int32) *RayClusterWrapper { + j.RayCluster.Status.MaxWorkerReplicas = maxWorkerReplicas + return j +} + +// State set State. +func (j *RayClusterWrapper) State(state rayv1.ClusterState) *RayClusterWrapper { + j.RayCluster.Status.State = state + return j +} + +// Reason set Reason. +func (j *RayClusterWrapper) Reason(reason string) *RayClusterWrapper { + j.RayCluster.Status.Reason = reason + return j +} + +// Priority sets the workload priority class label. +func (j *RayClusterWrapper) Priority(v string) *RayClusterWrapper { + return j.Label(kueueconstants.WorkloadPriorityClassLabel, v) +} diff --git a/pkg/testing/wrappers/ray_job_template_wrappers.go b/pkg/testing/wrappers/ray_job_template_wrappers.go new file mode 100644 index 0000000..cacd4e0 --- /dev/null +++ b/pkg/testing/wrappers/ray_job_template_wrappers.go @@ -0,0 +1,81 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package wrappers + +import ( + rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/apis/v1alpha1" +) + +// RayJobTemplateWrapper wraps a RayJobTemplate. +type RayJobTemplateWrapper struct{ v1alpha1.RayJobTemplate } + +// MakeRayJobTemplate creates a wrapper for a RayJobTemplate +func MakeRayJobTemplate(name, ns string) *RayJobTemplateWrapper { + return &RayJobTemplateWrapper{ + RayJobTemplate: v1alpha1.RayJobTemplate{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: ns, + }, + }, + } +} + +// Obj returns the inner RayJobTemplate. +func (w *RayJobTemplateWrapper) Obj() *v1alpha1.RayJobTemplate { + return &w.RayJobTemplate +} + +// Clone RayJobTemplateWrapper. +func (w *RayJobTemplateWrapper) Clone() *RayJobTemplateWrapper { + return &RayJobTemplateWrapper{ + RayJobTemplate: *w.RayJobTemplate.DeepCopy(), + } +} + +// Label sets the label key and value. +func (w *RayJobTemplateWrapper) Label(key, value string) *RayJobTemplateWrapper { + if w.Template.ObjectMeta.Labels == nil { + w.Template.ObjectMeta.Labels = make(map[string]string) + } + w.Template.ObjectMeta.Labels[key] = value + return w +} + +// Annotation sets the label key and value. +func (w *RayJobTemplateWrapper) Annotation(key, value string) *RayJobTemplateWrapper { + if w.Template.ObjectMeta.Annotations == nil { + w.Template.ObjectMeta.Annotations = make(map[string]string) + } + w.Template.ObjectMeta.Annotations[key] = value + return w +} + +// Entrypoint set entrypoint. +func (w *RayJobTemplateWrapper) Entrypoint(entrypoint string) *RayJobTemplateWrapper { + w.Template.Spec.Entrypoint = entrypoint + return w +} + +// WithRayClusterSpec set entrypoint. +func (w *RayJobTemplateWrapper) WithRayClusterSpec(spec *rayv1.RayClusterSpec) *RayJobTemplateWrapper { + w.Template.Spec.RayClusterSpec = spec + return w +} diff --git a/pkg/testing/wrappers/ray_job_wrappers.go b/pkg/testing/wrappers/ray_job_wrappers.go new file mode 100644 index 0000000..b8557d6 --- /dev/null +++ b/pkg/testing/wrappers/ray_job_wrappers.go @@ -0,0 +1,181 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package wrappers + +import ( + "time" + + rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1" + rayutil "github.com/ray-project/kuberay/ray-operator/controllers/ray/utils" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" + kueueconstants "sigs.k8s.io/kueue/pkg/controller/constants" + + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/apis/v1alpha1" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/constants" +) + +// RayJobWrapper wraps a RayJob. +type RayJobWrapper struct{ rayv1.RayJob } + +// MakeRayJob creates a wrapper for a RayJob +func MakeRayJob(name, ns string) *RayJobWrapper { + return &RayJobWrapper{ + rayv1.RayJob{ + TypeMeta: metav1.TypeMeta{Kind: "RayJob", APIVersion: "ray.io/v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: ns, + }, + }, + } +} + +// Obj returns the inner RayJob. +func (j *RayJobWrapper) Obj() *rayv1.RayJob { + return &j.RayJob +} + +// GenerateName updates generateName. +func (j *RayJobWrapper) GenerateName(v string) *RayJobWrapper { + j.ObjectMeta.GenerateName = v + return j +} + +// CreationTimestamp sets the .metadata.creationTimestamp +func (j *RayJobWrapper) CreationTimestamp(t time.Time) *RayJobWrapper { + j.RayJob.ObjectMeta.CreationTimestamp = metav1.NewTime(t) + return j +} + +// Profile sets the profile label. +func (j *RayJobWrapper) Profile(v string) *RayJobWrapper { + return j.Label(constants.ProfileLabel, v) +} + +// Mode sets the profile label. +func (j *RayJobWrapper) Mode(v v1alpha1.ApplicationProfileMode) *RayJobWrapper { + return j.Label(constants.ModeLabel, string(v)) +} + +// LocalQueue sets the localqueue label. +func (j *RayJobWrapper) LocalQueue(v string) *RayJobWrapper { + return j.Label(kueueconstants.QueueLabel, v) +} + +// Label sets the label key and value. +func (j *RayJobWrapper) Label(key, value string) *RayJobWrapper { + if j.Labels == nil { + j.Labels = make(map[string]string) + } + j.ObjectMeta.Labels[key] = value + return j +} + +// Annotation sets the label key and value. +func (j *RayJobWrapper) Annotation(key, value string) *RayJobWrapper { + if j.Annotations == nil { + j.Annotations = make(map[string]string) + } + j.ObjectMeta.Annotations[key] = value + return j +} + +// WithWorkerGroupSpec add worker group to the ray cluster template. +func (j *RayJobWrapper) WithWorkerGroupSpec(spec rayv1.WorkerGroupSpec) *RayJobWrapper { + if j.RayJob.Spec.RayClusterSpec == nil { + j.RayJob.Spec.RayClusterSpec = &rayv1.RayClusterSpec{} + } + + j.RayJob.Spec.RayClusterSpec.WorkerGroupSpecs = append(j.RayJob.Spec.RayClusterSpec.WorkerGroupSpecs, spec) + + return j +} + +// Spec set job spec. +func (j *RayJobWrapper) Spec(spec rayv1.RayJobSpec) *RayJobWrapper { + j.RayJob.Spec = spec + return j +} + +// Entrypoint set entrypoint. +func (j *RayJobWrapper) Entrypoint(entrypoint string) *RayJobWrapper { + j.RayJob.Spec.Entrypoint = entrypoint + return j +} + +// JobStatus set jobStatus. +func (j *RayJobWrapper) JobStatus(jobStatus rayv1.JobStatus) *RayJobWrapper { + j.RayJob.Status.JobStatus = jobStatus + return j +} + +// JobDeploymentStatus set jobDeploymentStatus. +func (j *RayJobWrapper) JobDeploymentStatus(jobDeploymentStatus rayv1.JobDeploymentStatus) *RayJobWrapper { + j.RayJob.Status.JobDeploymentStatus = jobDeploymentStatus + return j +} + +// Reason set reason. +func (j *RayJobWrapper) Reason(reason rayv1.JobFailedReason) *RayJobWrapper { + j.RayJob.Status.Reason = reason + return j +} + +// Message set message. +func (j *RayJobWrapper) Message(message string) *RayJobWrapper { + j.RayJob.Status.Message = message + return j +} + +// StartTime set startTime. +func (j *RayJobWrapper) StartTime(startTime time.Time) *RayJobWrapper { + j.RayJob.Status.StartTime = ptr.To(metav1.NewTime(startTime)) + return j +} + +// EndTime set endTime. +func (j *RayJobWrapper) EndTime(endTime time.Time) *RayJobWrapper { + j.RayJob.Status.EndTime = ptr.To(metav1.NewTime(endTime)) + return j +} + +// Suspend set suspend. +func (j *RayJobWrapper) Suspend(suspend bool) *RayJobWrapper { + j.RayJob.Spec.Suspend = suspend + return j +} + +// WithRayClusterLabelSelector sets the ClusterSelector. +func (j *RayJobWrapper) WithRayClusterLabelSelector(v string) *RayJobWrapper { + if j.RayJob.Spec.ClusterSelector == nil { + j.RayJob.Spec.ClusterSelector = map[string]string{} + } + j.RayJob.Spec.ClusterSelector[rayutil.RayClusterLabelKey] = v + return j +} + +// RayClusterName set rayClusterName. +func (j *RayJobWrapper) RayClusterName(rayClusterName string) *RayJobWrapper { + j.RayJob.Status.RayClusterName = rayClusterName + return j +} + +// Priority sets the workload priority class label. +func (j *RayJobWrapper) Priority(v string) *RayJobWrapper { + return j.Label(kueueconstants.WorkloadPriorityClassLabel, v) +} diff --git a/pkg/testing/wrappers/service_wrappers.go b/pkg/testing/wrappers/service_wrappers.go new file mode 100644 index 0000000..8a2abba --- /dev/null +++ b/pkg/testing/wrappers/service_wrappers.go @@ -0,0 +1,92 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package wrappers + +import ( + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + kueueconstants "sigs.k8s.io/kueue/pkg/controller/constants" + + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/apis/v1alpha1" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/constants" +) + +// ServiceWrapper wraps a Service. +type ServiceWrapper struct{ corev1.Service } + +// MakeService creates a wrapper for a Service +func MakeService(name, ns string) *ServiceWrapper { + return &ServiceWrapper{ + corev1.Service{ + TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: "v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: ns, + }, + }, + } +} + +// Obj returns the inner Service. +func (w *ServiceWrapper) Obj() *corev1.Service { + return &w.Service +} + +// Profile sets the profile label. +func (w *ServiceWrapper) Profile(v string) *ServiceWrapper { + return w.Label(constants.ProfileLabel, v) +} + +// Mode sets the mode label. +func (w *ServiceWrapper) Mode(v v1alpha1.ApplicationProfileMode) *ServiceWrapper { + return w.Label(constants.ModeLabel, string(v)) +} + +// LocalQueue sets the localqueue label. +func (w *ServiceWrapper) LocalQueue(v string) *ServiceWrapper { + return w.Label(kueueconstants.QueueLabel, v) +} + +// Label sets the label key and value. +func (w *ServiceWrapper) Label(key, value string) *ServiceWrapper { + if w.Labels == nil { + w.Labels = make(map[string]string) + } + w.ObjectMeta.Labels[key] = value + return w +} + +// ClusterIP sets clusterIP. +func (w *ServiceWrapper) ClusterIP(clusterIP string) *ServiceWrapper { + w.Service.Spec.ClusterIP = clusterIP + return w +} + +// Selector sets the selector key and value. +func (w *ServiceWrapper) Selector(key, value string) *ServiceWrapper { + if w.Service.Spec.Selector == nil { + w.Service.Spec.Selector = make(map[string]string) + } + w.Service.Spec.Selector[key] = value + return w +} + +// WithOwnerReference adds the owner reference. +func (w *ServiceWrapper) WithOwnerReference(ref metav1.OwnerReference) *ServiceWrapper { + w.OwnerReferences = append(w.OwnerReferences, ref) + return w +} diff --git a/pkg/testing/wrappers/supported_mode_wrappers.go b/pkg/testing/wrappers/supported_mode_wrappers.go new file mode 100644 index 0000000..2788ef6 --- /dev/null +++ b/pkg/testing/wrappers/supported_mode_wrappers.go @@ -0,0 +1,39 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package wrappers + +import ( + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/apis/v1alpha1" +) + +// SupportedModeWrapper wraps a SupportedMode. +type SupportedModeWrapper struct{ v1alpha1.SupportedMode } + +// MakeSupportedMode creates a wrapper for a SupportedMode +func MakeSupportedMode(name v1alpha1.ApplicationProfileMode, template v1alpha1.TemplateReference) *SupportedModeWrapper { + return &SupportedModeWrapper{ + SupportedMode: v1alpha1.SupportedMode{ + Name: name, + Template: template, + }, + } +} + +// Obj returns the inner SupportedMode. +func (m *SupportedModeWrapper) Obj() *v1alpha1.SupportedMode { + return &m.SupportedMode +} diff --git a/pkg/testing/wrappers/volume_bundle_wrappers.go b/pkg/testing/wrappers/volume_bundle_wrappers.go new file mode 100644 index 0000000..d220c93 --- /dev/null +++ b/pkg/testing/wrappers/volume_bundle_wrappers.go @@ -0,0 +1,71 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package wrappers + +import ( + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/apis/v1alpha1" +) + +// VolumeBundleWrapper wraps a VolumeBundle. +type VolumeBundleWrapper struct{ v1alpha1.VolumeBundle } + +// MakeVolumeBundle creates a wrapper for a VolumeBundle +func MakeVolumeBundle(name, namespace string) *VolumeBundleWrapper { + return &VolumeBundleWrapper{ + VolumeBundle: v1alpha1.VolumeBundle{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + }, + } +} + +// Obj returns the inner VolumeBundle. +func (vb *VolumeBundleWrapper) Obj() *v1alpha1.VolumeBundle { + return &vb.VolumeBundle +} + +// WithVolume add volume on the volumes. +func (vb *VolumeBundleWrapper) WithVolume(name, localObjectReferenceName string) *VolumeBundleWrapper { + vb.VolumeBundle.Spec.Volumes = append(vb.VolumeBundle.Spec.Volumes, corev1.Volume{ + Name: name, + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: localObjectReferenceName, + }, + }, + }, + }) + return vb +} + +// WithVolumeMount add volume mount on the volumes. +func (vb *VolumeBundleWrapper) WithVolumeMount(volumeMount corev1.VolumeMount) *VolumeBundleWrapper { + vb.VolumeBundle.Spec.ContainerVolumeMounts = append(vb.VolumeBundle.Spec.ContainerVolumeMounts, volumeMount) + return vb +} + +// WithEnvVar add EnvVar to EnvVars. +func (vb *VolumeBundleWrapper) WithEnvVar(envVar corev1.EnvVar) *VolumeBundleWrapper { + vb.Spec.EnvVars = append(vb.Spec.EnvVars, envVar) + return vb +} diff --git a/pkg/testing/wrappers/worker_group_spec_wrapper.go b/pkg/testing/wrappers/worker_group_spec_wrapper.go new file mode 100644 index 0000000..23f4c3d --- /dev/null +++ b/pkg/testing/wrappers/worker_group_spec_wrapper.go @@ -0,0 +1,94 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package wrappers + +import ( + rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1" + corev1 "k8s.io/api/core/v1" +) + +// WorkerGroupSpecWrapper wraps a WorkerGroupSpec. +type WorkerGroupSpecWrapper struct{ rayv1.WorkerGroupSpec } + +// MakeWorkerGroupSpec creates a wrapper for a WorkerGroupSpec +func MakeWorkerGroupSpec(groupName string) *WorkerGroupSpecWrapper { + return &WorkerGroupSpecWrapper{WorkerGroupSpec: rayv1.WorkerGroupSpec{GroupName: groupName}} +} + +// Obj returns the inner WorkerGroupSpec. +func (w *WorkerGroupSpecWrapper) Obj() *rayv1.WorkerGroupSpec { + return &w.WorkerGroupSpec +} + +// Clone WorkerGroupSpecWrapper. +func (w *WorkerGroupSpecWrapper) Clone() *WorkerGroupSpecWrapper { + return &WorkerGroupSpecWrapper{ + WorkerGroupSpec: *w.WorkerGroupSpec.DeepCopy(), + } +} + +// WithInitContainer add init container to the pod template. +func (w *WorkerGroupSpecWrapper) WithInitContainer(container corev1.Container) *WorkerGroupSpecWrapper { + w.Template.Spec.InitContainers = append(w.Template.Spec.InitContainers, container) + return w +} + +// WithContainer add container to the pod template. +func (w *WorkerGroupSpecWrapper) WithContainer(container corev1.Container) *WorkerGroupSpecWrapper { + w.Template.Spec.Containers = append(w.Template.Spec.Containers, container) + return w +} + +// WithVolume add volume to the WorkerGroupSpec. +func (w *WorkerGroupSpecWrapper) WithVolume(name, localObjectReferenceName string) *WorkerGroupSpecWrapper { + w.Template.Spec.Volumes = append(w.Template.Spec.Volumes, corev1.Volume{ + Name: name, + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: localObjectReferenceName, + }, + }, + }, + }) + + return w +} + +// Replicas set replicas on WorkerGroupSpec. +func (w *WorkerGroupSpecWrapper) Replicas(replicas int32) *WorkerGroupSpecWrapper { + w.WorkerGroupSpec.Replicas = &replicas + return w +} + +// MinReplicas set minReplicas on WorkerGroupSpec. +func (w *WorkerGroupSpecWrapper) MinReplicas(minReplicas int32) *WorkerGroupSpecWrapper { + w.WorkerGroupSpec.MinReplicas = &minReplicas + return w +} + +// MaxReplicas set maxReplicas on WorkerGroupSpec. +func (w *WorkerGroupSpecWrapper) MaxReplicas(maxReplicas int32) *WorkerGroupSpecWrapper { + w.WorkerGroupSpec.MaxReplicas = &maxReplicas + return w +} + +// RayStartParams set rayStartParams on WorkerGroupSpec. +func (w *WorkerGroupSpecWrapper) RayStartParams(rayStartParams map[string]string) *WorkerGroupSpecWrapper { + w.WorkerGroupSpec.RayStartParams = rayStartParams + return w +} diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go new file mode 100644 index 0000000..a11fc53 --- /dev/null +++ b/test/e2e/e2e_test.go @@ -0,0 +1,53 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "os/exec" + + "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/test/util" +) + +var _ = ginkgo.Describe("kjobctl", ginkgo.Ordered, func() { + var ns *corev1.Namespace + + ginkgo.BeforeEach(func() { + ns = &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "e2e-", + }, + } + gomega.Expect(k8sClient.Create(ctx, ns)).To(gomega.Succeed()) + }) + ginkgo.AfterEach(func() { + gomega.Expect(util.DeleteNamespace(ctx, k8sClient, ns)).To(gomega.Succeed()) + }) + + ginkgo.It("Should print kjobctl information", func() { + cmd := exec.Command(kjobctlPath, "--help") + out, err := util.Run(cmd) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + info := util.GetNonEmptyLines(string(out)) + gomega.Expect(info[0]).To(gomega.Equal("ML/AI/Batch Jobs Made Easy")) + }) +}) diff --git a/test/e2e/slurm_test.go b/test/e2e/slurm_test.go new file mode 100644 index 0000000..eaec679 --- /dev/null +++ b/test/e2e/slurm_test.go @@ -0,0 +1,369 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "bytes" + "fmt" + "os" + "os/exec" + "regexp" + "strconv" + + "github.com/google/go-cmp/cmp/cmpopts" + "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/kueue/pkg/util/maps" + + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/apis/v1alpha1" + kjobctlconstants "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/constants" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/testing/wrappers" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/test/util" +) + +const ( + BatchJobNameLabel = "batch.kubernetes.io/job-name" + BatchJobCompletionIndexLabel = "batch.kubernetes.io/job-completion-index" +) + +var _ = ginkgo.Describe("Slurm", ginkgo.Ordered, func() { + var ( + ns *corev1.Namespace + profile *v1alpha1.ApplicationProfile + jobTemplate *v1alpha1.JobTemplate + ) + + ginkgo.BeforeEach(func() { + ns = &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "e2e-slurm-", + }, + } + gomega.Expect(k8sClient.Create(ctx, ns)).To(gomega.Succeed()) + + jobTemplate = wrappers.MakeJobTemplate("job-template", ns.Name). + RestartPolicy(corev1.RestartPolicyNever). + BackoffLimitPerIndex(0). + WithContainer(*wrappers.MakeContainer("c1", util.E2eTestBashImage).Obj()). + Obj() + gomega.Expect(k8sClient.Create(ctx, jobTemplate)).To(gomega.Succeed()) + + profile = wrappers.MakeApplicationProfile("profile", ns.Name). + WithSupportedMode(*wrappers.MakeSupportedMode(v1alpha1.SlurmMode, "job-template").Obj()). + Obj() + gomega.Expect(k8sClient.Create(ctx, profile)).To(gomega.Succeed()) + }) + + ginkgo.AfterEach(func() { + gomega.Expect(util.DeleteNamespace(ctx, k8sClient, ns)).To(gomega.Succeed()) + }) + + ginkgo.DescribeTable("should be created", func( + args, slurmArgs []string, + expectCompletions, expectParallelism int32, + expectCommonVars map[string]string, expectPods []map[string]map[string]string, + withFirstNodeIP bool, + ) { + ginkgo.By("Create temporary file") + script, err := os.CreateTemp("", "e2e-slurm-") + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + defer script.Close() + defer os.Remove(script.Name()) + + ginkgo.By("Prepare script", func() { + _, err := script.WriteString("#!/bin/bash\nwhile true; do printenv | grep SLURM_ > /env.out; sleep 0.25; done") + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + }) + + var jobName, configMapName, serviceName string + + ginkgo.By("Create slurm", func() { + cmdArgs := []string{"create", "slurm", "-n", ns.Name, "--profile", profile.Name} + cmdArgs = append(cmdArgs, args...) + cmdArgs = append(cmdArgs, "--", script.Name()) + cmdArgs = append(cmdArgs, slurmArgs...) + + cmd := exec.Command(kjobctlPath, cmdArgs...) + out, err := util.Run(cmd) + gomega.Expect(err).NotTo(gomega.HaveOccurred(), "%s: %s", err, out) + gomega.Expect(out).NotTo(gomega.BeEmpty()) + + jobName, configMapName, serviceName, err = parseSlurmCreateOutput(out, profile.Name) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(jobName).NotTo(gomega.BeEmpty()) + gomega.Expect(configMapName).NotTo(gomega.BeEmpty()) + gomega.Expect(serviceName).NotTo(gomega.BeEmpty()) + }) + + job := &batchv1.Job{} + configMap := &corev1.Service{} + service := &corev1.Service{} + + ginkgo.By("Check slurm is created", func() { + gomega.Expect(k8sClient.Get(ctx, client.ObjectKey{Namespace: ns.Name, Name: jobName}, job)).To(gomega.Succeed()) + gomega.Expect(ptr.Deref(job.Spec.Completions, 1)).To(gomega.Equal(expectCompletions)) + gomega.Expect(ptr.Deref(job.Spec.Parallelism, 1)).To(gomega.Equal(expectParallelism)) + gomega.Expect(job.Annotations).To(gomega.HaveKeyWithValue(kjobctlconstants.ScriptAnnotation, script.Name())) + gomega.Expect(k8sClient.Get(ctx, client.ObjectKey{Namespace: ns.Name, Name: configMapName}, configMap)).To(gomega.Succeed()) + gomega.Expect(k8sClient.Get(ctx, client.ObjectKey{Namespace: ns.Name, Name: serviceName}, service)).To(gomega.Succeed()) + }) + + var firstPod *corev1.Pod + + for completionIndex, expectPod := range expectPods { + podList := &corev1.PodList{} + + ginkgo.By(fmt.Sprintf("Wait for pod with completion index %d is running", completionIndex), func() { + gomega.Eventually(func(g gomega.Gomega) { + g.Expect(k8sClient.List(ctx, podList, client.InNamespace(ns.Name), client.MatchingLabels(map[string]string{ + BatchJobNameLabel: job.Name, + BatchJobCompletionIndexLabel: strconv.Itoa(completionIndex), + }))).Should(gomega.Succeed()) + g.Expect(podList.Items).Should(gomega.HaveLen(1)) + g.Expect(podList.Items[0].Status.Phase).To(gomega.Equal(corev1.PodRunning)) + if completionIndex == 0 { + firstPod = &podList.Items[0] + } + }, util.LongTimeout, util.Interval).Should(gomega.Succeed()) + }) + + pod := podList.Items[0] + + for containerName, expectContainerVars := range expectPod { + ginkgo.By(fmt.Sprintf("Check env variables in index %d and container name %s", completionIndex, containerName), func() { + wantOut := maps.MergeKeepFirst(expectContainerVars, expectCommonVars) + if withFirstNodeIP { + wantOut["SLURM_JOB_FIRST_NODE_IP"] = firstPod.Status.PodIP + } + + gomega.Eventually(func(g gomega.Gomega) { + out, outErr, err := util.KExecute(ctx, cfg, restClient, ns.Name, pod.Name, containerName) + g.Expect(err).NotTo(gomega.HaveOccurred()) + g.Expect(string(outErr)).To(gomega.BeEmpty()) + g.Expect(parseSlurmEnvOutput(out)).To(gomega.BeComparableTo(wantOut, + cmpopts.AcyclicTransformer("RemoveGeneratedNameSuffixInMap", func(m map[string]string) map[string]string { + for key, val := range m { + m[key] = regexp.MustCompile("(profile-slurm)(-.{5})").ReplaceAllString(val, "$1") + } + return m + }), + )) + }, util.Timeout, util.Interval).Should(gomega.Succeed()) + }) + } + + if completionIndex != 0 || !withFirstNodeIP { + gomega.Expect(k8sClient.Delete(ctx, &pod)).To(gomega.Succeed()) + } + } + }, + ginkgo.Entry( + "without arguments", + []string{}, []string{}, + int32(1), int32(1), + map[string]string{ + "SLURM_NTASKS_PER_NODE": "1", + "SLURM_ARRAY_JOB_ID": "1", + "SLURM_MEM_PER_CPU": "", + "SLURM_GPUS": "", + "SLURM_NNODES": "1", + "SLURM_MEM_PER_GPU": "", + "SLURM_NTASKS": "1", + "SLURM_ARRAY_TASK_COUNT": "1", + "SLURM_TASKS_PER_NODE": "1", + "SLURM_CPUS_PER_TASK": "", + "SLURM_ARRAY_TASK_MAX": "0", + "SLURM_CPUS_PER_GPU": "", + "SLURM_SUBMIT_DIR": "/slurm/scripts", + "SLURM_NPROCS": "1", + "SLURM_CPUS_ON_NODE": "", + "SLURM_ARRAY_TASK_MIN": "0", + "SLURM_JOB_NODELIST": "profile-slurm-xxxxx-0.profile-slurm-xxxxx", + "SLURM_JOB_CPUS_PER_NODE": "", + "SLURM_JOB_FIRST_NODE": "profile-slurm-xxxxx-1.profile-slurm-xxxxx", + "SLURM_MEM_PER_NODE": "", + "SLURM_JOB_FIRST_NODE_IP": "", + }, + []map[string]map[string]string{ + { + "c1": { + "SLURM_ARRAY_TASK_ID": "0", + "SLURM_JOB_ID": "1", + "SLURM_JOBID": "1", + "SLURM_SUBMIT_HOST": "profile-slurm-xxxxx-0", + }, + }, + }, + false, + ), + ginkgo.Entry( + "with --first-node-ip", + []string{"--first-node-ip"}, []string{"--array", "1-5%2", "--nodes", "2", "--ntasks", "2"}, + int32(3), int32(2), + map[string]string{ + "SLURM_NTASKS_PER_NODE": "2", + "SLURM_ARRAY_JOB_ID": "1", + "SLURM_MEM_PER_CPU": "", + "SLURM_GPUS": "", + "SLURM_NNODES": "2", + "SLURM_MEM_PER_GPU": "", + "SLURM_NTASKS": "2", + "SLURM_ARRAY_TASK_COUNT": "5", + "SLURM_TASKS_PER_NODE": "2", + "SLURM_CPUS_PER_TASK": "", + "SLURM_ARRAY_TASK_MAX": "5", + "SLURM_CPUS_PER_GPU": "", + "SLURM_SUBMIT_DIR": "/slurm/scripts", + "SLURM_NPROCS": "2", + "SLURM_CPUS_ON_NODE": "", + "SLURM_ARRAY_TASK_MIN": "1", + "SLURM_JOB_NODELIST": "profile-slurm-xxxxx-0.profile-slurm-xxxxx,profile-slurm-xxxxx-1.profile-slurm-xxxxx", + "SLURM_JOB_CPUS_PER_NODE": "", + "SLURM_JOB_FIRST_NODE": "profile-slurm-xxxxx-1.profile-slurm-xxxxx", + "SLURM_MEM_PER_NODE": "", + "SLURM_JOB_FIRST_NODE_IP": "", + }, + []map[string]map[string]string{ + { + "c1-0": { + "SLURM_ARRAY_TASK_ID": "1", + "SLURM_JOB_ID": "1", + "SLURM_JOBID": "1", + "SLURM_SUBMIT_HOST": "profile-slurm-xxxxx-0", + }, + "c1-1": { + "SLURM_ARRAY_TASK_ID": "2", + "SLURM_JOB_ID": "2", + "SLURM_JOBID": "2", + "SLURM_SUBMIT_HOST": "profile-slurm-xxxxx-0", + }, + }, + { + "c1-0": { + "SLURM_ARRAY_TASK_ID": "3", + "SLURM_JOB_ID": "3", + "SLURM_JOBID": "3", + "SLURM_SUBMIT_HOST": "profile-slurm-xxxxx-1", + }, + "c1-1": { + "SLURM_ARRAY_TASK_ID": "4", + "SLURM_JOB_ID": "4", + "SLURM_JOBID": "4", + "SLURM_SUBMIT_HOST": "profile-slurm-xxxxx-1", + }, + }, + { + "c1-0": { + "SLURM_ARRAY_TASK_ID": "5", + "SLURM_JOB_ID": "5", + "SLURM_JOBID": "5", + "SLURM_SUBMIT_HOST": "profile-slurm-xxxxx-2", + }, + }, + }, + true, + ), + ) + + ginkgo.When("delete", func() { + ginkgo.It("should delete job and child objects", func() { + ginkgo.By("Create temporary file") + script, err := os.CreateTemp("", "e2e-slurm-") + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + defer script.Close() + defer os.Remove(script.Name()) + + ginkgo.By("Prepare script", func() { + _, err := script.WriteString("#!/bin/bash\nsleep 600") + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + }) + + var jobName, configMapName, serviceName string + + ginkgo.By("Create slurm", func() { + cmdArgs := []string{"create", "slurm", "-n", ns.Name, "--profile", profile.Name} + cmdArgs = append(cmdArgs, "--", script.Name()) + + cmd := exec.Command(kjobctlPath, cmdArgs...) + out, err := util.Run(cmd) + gomega.Expect(err).NotTo(gomega.HaveOccurred(), "%s: %s", err, out) + gomega.Expect(out).NotTo(gomega.BeEmpty()) + + jobName, configMapName, serviceName, err = parseSlurmCreateOutput(out, profile.Name) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(jobName).NotTo(gomega.BeEmpty()) + gomega.Expect(configMapName).NotTo(gomega.BeEmpty()) + gomega.Expect(serviceName).NotTo(gomega.BeEmpty()) + }) + + job := &batchv1.Job{} + configMap := &corev1.Service{} + service := &corev1.Service{} + + ginkgo.By("Check slurm is created", func() { + gomega.Expect(k8sClient.Get(ctx, client.ObjectKey{Namespace: ns.Name, Name: jobName}, job)).To(gomega.Succeed()) + gomega.Expect(k8sClient.Get(ctx, client.ObjectKey{Namespace: ns.Name, Name: configMapName}, configMap)).To(gomega.Succeed()) + gomega.Expect(k8sClient.Get(ctx, client.ObjectKey{Namespace: ns.Name, Name: serviceName}, service)).To(gomega.Succeed()) + }) + + ginkgo.By("Delete slurm", func() { + cmd := exec.Command(kjobctlPath, "delete", "slurm", "-n", ns.Name, jobName) + out, err := util.Run(cmd) + gomega.Expect(err).NotTo(gomega.HaveOccurred(), "%s: %s", err, out) + gomega.Expect(string(out)).To(gomega.Equal(fmt.Sprintf("job.batch/%s deleted\n", jobName))) + }) + + ginkgo.By("Check job and child objects are deleted", func() { + gomega.Eventually(func(g gomega.Gomega) { + g.Expect(errors.IsNotFound(k8sClient.Get(ctx, client.ObjectKeyFromObject(job), job))).To(gomega.BeTrue()) + g.Expect(errors.IsNotFound(k8sClient.Get(ctx, client.ObjectKeyFromObject(configMap), configMap))).To(gomega.BeTrue()) + g.Expect(errors.IsNotFound(k8sClient.Get(ctx, client.ObjectKeyFromObject(service), service))).To(gomega.BeTrue()) + }, util.Timeout, util.Interval).Should(gomega.Succeed()) + }) + }) + }) +}) + +func parseSlurmCreateOutput(output []byte, profileName string) (string, string, string, error) { + output = bytes.ReplaceAll(output, []byte("\n"), []byte("")) + re := regexp.MustCompile(fmt.Sprintf(`^job\.batch\/(%[1]s-slurm-.{5}) createdconfigmap\/(%[1]s-slurm-.{5}) createdservice\/(%[1]s-slurm-.{5}) created$`, profileName)) + matches := re.FindSubmatch(output) + + if len(matches) < 4 { + return "", "", "", fmt.Errorf("unexpected output format: %s", output) + } + + return string(matches[1]), string(matches[2]), string(matches[3]), nil +} + +func parseSlurmEnvOutput(output []byte) map[string]string { + parts := bytes.Split(output, []byte{'\n'}) + gotOut := make(map[string]string, len(parts)) + for _, part := range parts { + pair := bytes.SplitN(part, []byte{'='}, 2) + if len(pair) == 2 { + gotOut[string(pair[0])] = string(pair[1]) + } + } + return gotOut +} diff --git a/test/e2e/suite_test.go b/test/e2e/suite_test.go new file mode 100644 index 0000000..3f98444 --- /dev/null +++ b/test/e2e/suite_test.go @@ -0,0 +1,54 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "context" + "path/filepath" + "testing" + + "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client" + + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/test/util" +) + +var ( + kjobctlPath string + cfg *rest.Config + k8sClient client.Client + restClient *rest.RESTClient + ctx context.Context +) + +// Run e2e tests using the Ginkgo runner. +func TestE2E(t *testing.T) { + suiteName := "End To End Suite" + gomega.RegisterFailHandler(ginkgo.Fail) + ginkgo.RunSpecs(t, suiteName) +} + +var _ = ginkgo.BeforeSuite(func() { + dir, _ := util.GetProjectDir() + kjobctlPath = filepath.Join(dir, "bin", "kubectl-kjob") + cfg = util.GetConfigWithContext("") + k8sClient = util.CreateClient(cfg) + restClient = util.CreateRestClient(cfg) + ctx = context.Background() +}) diff --git a/test/framework/framework.go b/test/framework/framework.go new file mode 100644 index 0000000..e28179c --- /dev/null +++ b/test/framework/framework.go @@ -0,0 +1,80 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package framework + +import ( + "context" + + "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" + rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/envtest" + + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/apis/v1alpha1" +) + +type Framework struct { + CRDPath string + DepCRDPaths []string + testEnv *envtest.Environment + cancel context.CancelFunc +} + +func (f *Framework) Init() *rest.Config { + ginkgo.By("bootstrapping test environment") + + f.testEnv = &envtest.Environment{ + CRDDirectoryPaths: append(f.DepCRDPaths, f.CRDPath), + ErrorIfCRDPathMissing: true, + } + + cfg, err := f.testEnv.Start() + + gomega.ExpectWithOffset(1, err).NotTo(gomega.HaveOccurred()) + gomega.ExpectWithOffset(1, cfg).NotTo(gomega.BeNil()) + + return cfg +} + +func (f *Framework) SetupClient(cfg *rest.Config) (context.Context, client.Client) { + err := v1alpha1.AddToScheme(scheme.Scheme) + gomega.ExpectWithOffset(1, err).NotTo(gomega.HaveOccurred()) + + err = rayv1.AddToScheme(scheme.Scheme) + gomega.ExpectWithOffset(1, err).NotTo(gomega.HaveOccurred()) + + k8sClient, err := client.New(cfg, client.Options{Scheme: scheme.Scheme}) + gomega.ExpectWithOffset(1, err).NotTo(gomega.HaveOccurred()) + gomega.ExpectWithOffset(1, k8sClient).NotTo(gomega.BeNil()) + + ctx, cancel := context.WithCancel(context.Background()) + f.cancel = cancel + + return ctx, k8sClient +} + +func (f *Framework) Teardown() { + ginkgo.By("tearing down the test environment") + if f.cancel != nil { + f.cancel() + } + err := f.testEnv.Stop() + gomega.ExpectWithOffset(1, err).NotTo(gomega.HaveOccurred()) +} diff --git a/test/integration/kjobctl/create_test.go b/test/integration/kjobctl/create_test.go new file mode 100644 index 0000000..e6e7543 --- /dev/null +++ b/test/integration/kjobctl/create_test.go @@ -0,0 +1,172 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kjobctl + +import ( + "fmt" + "time" + + "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/cli-runtime/pkg/genericiooptions" + testingclock "k8s.io/utils/clock/testing" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" + kueueconstants "sigs.k8s.io/kueue/pkg/controller/constants" + + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/apis/v1alpha1" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/cmd" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/constants" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/testing/wrappers" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/test/util" +) + +var _ = ginkgo.Describe("Kjobctl Create", ginkgo.Ordered, ginkgo.ContinueOnFailure, func() { + var ns *corev1.Namespace + + ginkgo.BeforeEach(func() { + ns = &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{GenerateName: "ns-"}} + gomega.Expect(k8sClient.Create(ctx, ns)).To(gomega.Succeed()) + }) + + ginkgo.AfterEach(func() { + gomega.Expect(util.DeleteNamespace(ctx, k8sClient, ns)).To(gomega.Succeed()) + }) + + ginkgo.When("Create a Job", func() { + var ( + jobTemplate *v1alpha1.JobTemplate + profile *v1alpha1.ApplicationProfile + ) + + ginkgo.BeforeEach(func() { + jobTemplate = wrappers.MakeJobTemplate("job-template", ns.Name). + RestartPolicy(corev1.RestartPolicyOnFailure). + WithContainer(*wrappers.MakeContainer("c1", "sleep").Obj()). + WithContainer(*wrappers.MakeContainer("c2", "sleep").Obj()). + Obj() + gomega.Expect(k8sClient.Create(ctx, jobTemplate)).To(gomega.Succeed()) + + profile = wrappers.MakeApplicationProfile("profile", ns.Name). + WithSupportedMode(*wrappers.MakeSupportedMode(v1alpha1.JobMode, "job-template").Obj()). + Obj() + gomega.Expect(k8sClient.Create(ctx, profile)).To(gomega.Succeed()) + }) + + ginkgo.It("Should create job", func() { + testStartTime := time.Now() + + ginkgo.By("Create a Job", func() { + streams, _, out, outErr := genericiooptions.NewTestIOStreams() + configFlags := CreateConfigFlagsWithRestConfig(cfg, streams) + + kjobctlCmd := cmd.NewKjobctlCmd(cmd.KjobctlOptions{ + ConfigFlags: configFlags, + IOStreams: streams, + Clock: testingclock.NewFakeClock(testStartTime), + }) + kjobctlCmd.SetOut(out) + kjobctlCmd.SetErr(outErr) + kjobctlCmd.SetArgs([]string{ + "create", "job", + "-n", ns.Name, + "--profile", profile.Name, + "--cmd", "sleep 60s", + "--parallelism", "2", + "--completions", "3", + "--request", "cpu=100m,memory=4Gi", + "--localqueue", "lq1", + "--skip-localqueue-validation", + }) + + err := kjobctlCmd.Execute() + gomega.Expect(err).NotTo(gomega.HaveOccurred(), "%s: %s", err, out) + gomega.Expect(outErr.String()).Should(gomega.BeEmpty()) + gomega.Expect(out.String()).Should(gomega.MatchRegexp( + fmt.Sprintf("^job.batch\\/%s-%s-[a-zA-Z0-9]+ created\\n$", profile.Name, "job"))) + }) + + ginkgo.By("Check that Job created", func() { + timestamp := testStartTime.Format(time.RFC3339) + expectedTaskName := fmt.Sprintf("%s_%s", ns.Name, profile.Name) + expectedProfileName := fmt.Sprintf("%s_%s", ns.Name, profile.Name) + expectedTaskID := fmt.Sprintf("%s_%s_%s_%s", userID, timestamp, ns.Name, profile.Name) + + jobList := &batchv1.JobList{} + gomega.Expect(k8sClient.List(ctx, jobList, client.InNamespace(ns.Name))).To(gomega.Succeed()) + gomega.Expect(jobList.Items).To(gomega.HaveLen(1)) + gomega.Expect(jobList.Items[0].Labels[constants.ProfileLabel]).To(gomega.Equal(profile.Name)) + gomega.Expect(jobList.Items[0].Labels[kueueconstants.QueueLabel]).To(gomega.Equal("lq1")) + gomega.Expect(jobList.Items[0].Name).To(gomega.HavePrefix(profile.Name)) + gomega.Expect(jobList.Items[0].Spec.Parallelism).To(gomega.Equal(ptr.To[int32](2))) + gomega.Expect(jobList.Items[0].Spec.Completions).To(gomega.Equal(ptr.To[int32](3))) + gomega.Expect(jobList.Items[0].Spec.Template.Spec.RestartPolicy).To(gomega.Equal(corev1.RestartPolicyOnFailure)) + gomega.Expect(jobList.Items[0].Spec.Template.Spec.Containers).To(gomega.HaveLen(2)) + gomega.Expect(jobList.Items[0].Spec.Template.Spec.Containers[0].Command).To(gomega.Equal([]string{"sleep", "60s"})) + gomega.Expect(jobList.Items[0].Spec.Template.Spec.Containers[0].Resources.Requests).To(gomega.Equal(corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("100m"), + corev1.ResourceMemory: resource.MustParse("4Gi"), + })) + gomega.Expect(jobList.Items[0].Spec.Template.Spec.Containers[0].Env).To(gomega.Equal([]corev1.EnvVar{ + {Name: constants.EnvVarNameUserID, Value: userID}, + {Name: constants.EnvVarTaskName, Value: expectedTaskName}, + {Name: constants.EnvVarTaskID, Value: expectedTaskID}, + {Name: constants.EnvVarNameProfile, Value: expectedProfileName}, + {Name: constants.EnvVarNameTimestamp, Value: timestamp}, + })) + gomega.Expect(jobList.Items[0].Spec.Template.Spec.Containers[1].Command).To(gomega.BeNil()) + gomega.Expect(jobList.Items[0].Spec.Template.Spec.Containers[1].Resources.Requests).To(gomega.BeNil()) + gomega.Expect(jobList.Items[0].Spec.Template.Spec.Containers[1].Env).To(gomega.Equal([]corev1.EnvVar{ + {Name: constants.EnvVarNameUserID, Value: userID}, + {Name: constants.EnvVarTaskName, Value: expectedTaskName}, + {Name: constants.EnvVarTaskID, Value: expectedTaskID}, + {Name: constants.EnvVarNameProfile, Value: expectedProfileName}, + {Name: constants.EnvVarNameTimestamp, Value: timestamp}, + })) + }) + }) + + // SimpleDynamicClient didn't allow to check server dry run flag. + ginkgo.It("Shouldn't create job with server dry run", func() { + ginkgo.By("Create a Job", func() { + streams, _, out, outErr := genericiooptions.NewTestIOStreams() + configFlags := CreateConfigFlagsWithRestConfig(cfg, streams) + + kjobctlCmd := cmd.NewKjobctlCmd(cmd.KjobctlOptions{ConfigFlags: configFlags, IOStreams: streams}) + kjobctlCmd.SetOut(out) + kjobctlCmd.SetErr(outErr) + kjobctlCmd.SetArgs([]string{"create", "job", "-n", ns.Name, "--profile", profile.Name, "--dry-run", "server"}) + + err := kjobctlCmd.Execute() + gomega.Expect(err).NotTo(gomega.HaveOccurred(), "%s: %s", err, out) + gomega.Expect(outErr.String()).Should(gomega.BeEmpty()) + gomega.Expect(out.String()).Should(gomega.MatchRegexp( + fmt.Sprintf("job.batch\\/%s-%s-[a-zA-Z0-9]+ created \\(server dry run\\)", profile.Name, "job"))) + }) + + ginkgo.By("Check that Job not created", func() { + jobList := &batchv1.JobList{} + gomega.Expect(k8sClient.List(ctx, jobList, client.InNamespace(ns.Name))).To(gomega.Succeed()) + gomega.Expect(jobList.Items).To(gomega.BeEmpty()) + }) + }) + }) +}) diff --git a/test/integration/kjobctl/list_test.go b/test/integration/kjobctl/list_test.go new file mode 100644 index 0000000..cae7e9c --- /dev/null +++ b/test/integration/kjobctl/list_test.go @@ -0,0 +1,346 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kjobctl + +import ( + "fmt" + "os" + "time" + + "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" + rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/duration" + "k8s.io/cli-runtime/pkg/genericiooptions" + testingclock "k8s.io/utils/clock/testing" + + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/apis/v1alpha1" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/cmd" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/cmd/list" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/testing/wrappers" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/test/util" +) + +var _ = ginkgo.Describe("Kjobctl List", ginkgo.Ordered, ginkgo.ContinueOnFailure, func() { + var ns *corev1.Namespace + + ginkgo.BeforeEach(func() { + ns = &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{GenerateName: "ns-"}} + gomega.Expect(k8sClient.Create(ctx, ns)).To(gomega.Succeed()) + }) + + ginkgo.AfterEach(func() { + gomega.Expect(util.DeleteNamespace(ctx, k8sClient, ns)).To(gomega.Succeed()) + os.Unsetenv(list.KjobctlListRequestLimitEnvName) + }) + + ginkgo.When("List Jobs", func() { + var ( + j1 *batchv1.Job + j2 *batchv1.Job + j3 *batchv1.Job + ) + + ginkgo.JustBeforeEach(func() { + j1 = wrappers.MakeJob("j1", ns.Name). + Profile("profile1"). + Mode(v1alpha1.JobMode). + Completions(3). + WithContainer(*wrappers.MakeContainer("c1", "sleep").Obj()). + RestartPolicy(corev1.RestartPolicyOnFailure). + Obj() + gomega.Expect(k8sClient.Create(ctx, j1)).To(gomega.Succeed()) + + j2 = wrappers.MakeJob("j2", ns.Name). + Profile("profile1"). + Mode(v1alpha1.JobMode). + Completions(3). + WithContainer(*wrappers.MakeContainer("c1", "sleep").Obj()). + RestartPolicy(corev1.RestartPolicyOnFailure). + Obj() + gomega.Expect(k8sClient.Create(ctx, j2)).To(gomega.Succeed()) + + j3 = wrappers.MakeJob("very-long-job-name", ns.Name). + Profile("profile1"). + Mode(v1alpha1.JobMode). + Completions(3). + WithContainer(*wrappers.MakeContainer("c1", "sleep").Obj()). + RestartPolicy(corev1.RestartPolicyOnFailure). + Obj() + gomega.Expect(k8sClient.Create(ctx, j3)).To(gomega.Succeed()) + }) + + // Simple client set that are using on unit tests not allow paging. + ginkgo.It("Should print jobs list with paging", func() { + streams, _, output, errOutput := genericiooptions.NewTestIOStreams() + configFlags := CreateConfigFlagsWithRestConfig(cfg, streams) + executeTime := time.Now() + kjobctl := cmd.NewKjobctlCmd(cmd.KjobctlOptions{ConfigFlags: configFlags, IOStreams: streams, + Clock: testingclock.NewFakeClock(executeTime)}) + + os.Setenv(list.KjobctlListRequestLimitEnvName, "1") + kjobctl.SetArgs([]string{"list", "job", "--namespace", ns.Name}) + err := kjobctl.Execute() + + gomega.Expect(err).NotTo(gomega.HaveOccurred(), "%s: %s", err, output) + gomega.Expect(errOutput.String()).Should(gomega.BeEmpty()) + gomega.Expect(output.String()).Should(gomega.Equal(fmt.Sprintf(`NAME PROFILE LOCAL QUEUE COMPLETIONS DURATION AGE +j1 profile1 0/3 %s +j2 profile1 0/3 %s +very-long-job-name profile1 0/3 %s +`, + duration.HumanDuration(executeTime.Sub(j1.CreationTimestamp.Time)), + duration.HumanDuration(executeTime.Sub(j2.CreationTimestamp.Time)), + duration.HumanDuration(executeTime.Sub(j3.CreationTimestamp.Time)), + ))) + }) + }) + + ginkgo.When("List RayJobs", func() { + var ( + rj1 *rayv1.RayJob + rj2 *rayv1.RayJob + rj3 *rayv1.RayJob + ) + + ginkgo.JustBeforeEach(func() { + rj1 = wrappers.MakeRayJob("rj1", ns.Name). + Profile("profile1"). + Suspend(true). + Obj() + gomega.Expect(k8sClient.Create(ctx, rj1)).To(gomega.Succeed()) + + rj2 = wrappers.MakeRayJob("rj2", ns.Name). + Profile("profile1"). + Suspend(true). + Obj() + gomega.Expect(k8sClient.Create(ctx, rj2)).To(gomega.Succeed()) + + rj3 = wrappers.MakeRayJob("very-long-job-name", ns.Name). + Profile("profile1"). + Suspend(true). + Obj() + gomega.Expect(k8sClient.Create(ctx, rj3)).To(gomega.Succeed()) + }) + + // Simple client set that are using on unit tests not allow paging. + ginkgo.It("Should print jobs list with paging", func() { + streams, _, output, errOutput := genericiooptions.NewTestIOStreams() + configFlags := CreateConfigFlagsWithRestConfig(cfg, streams) + executeTime := time.Now() + kjobctl := cmd.NewKjobctlCmd(cmd.KjobctlOptions{ConfigFlags: configFlags, IOStreams: streams, + Clock: testingclock.NewFakeClock(executeTime)}) + + os.Setenv(list.KjobctlListRequestLimitEnvName, "1") + kjobctl.SetArgs([]string{"list", "rayjob", "--namespace", ns.Name}) + err := kjobctl.Execute() + + gomega.Expect(err).NotTo(gomega.HaveOccurred(), "%s: %s", err, output) + gomega.Expect(errOutput.String()).Should(gomega.BeEmpty()) + gomega.Expect(output.String()).Should(gomega.Equal(fmt.Sprintf(`NAME PROFILE LOCAL QUEUE RAY CLUSTER NAME JOB STATUS DEPLOYMENT STATUS START TIME END TIME AGE +rj1 profile1 %s +rj2 profile1 %s +very-long-job-name profile1 %s +`, + duration.HumanDuration(executeTime.Sub(rj1.CreationTimestamp.Time)), + duration.HumanDuration(executeTime.Sub(rj2.CreationTimestamp.Time)), + duration.HumanDuration(executeTime.Sub(rj3.CreationTimestamp.Time)), + ))) + }) + }) + + ginkgo.When("List RayClusters", func() { + var ( + rc1 *rayv1.RayCluster + rc2 *rayv1.RayCluster + rc3 *rayv1.RayCluster + ) + + ginkgo.JustBeforeEach(func() { + rc1 = wrappers.MakeRayCluster("rc1", ns.Name). + Profile("profile1"). + Spec(*wrappers.MakeRayClusterSpec(). + HeadGroupSpec(rayv1.HeadGroupSpec{ + RayStartParams: make(map[string]string), + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + {Name: "ray-head", Image: "rayproject/ray:2.9.0"}, + }, + }, + }, + }). + WithWorkerGroupSpec(rayv1.WorkerGroupSpec{ + RayStartParams: make(map[string]string), + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + {Name: "ray-worker", Image: "rayproject/ray:2.9.0"}, + }, + }, + }, + }). + Obj()). + Obj() + gomega.Expect(k8sClient.Create(ctx, rc1)).To(gomega.Succeed()) + + rc2 = wrappers.MakeRayCluster("rc2", ns.Name). + Profile("profile1"). + Spec(*wrappers.MakeRayClusterSpec(). + HeadGroupSpec(rayv1.HeadGroupSpec{ + RayStartParams: make(map[string]string), + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + {Name: "ray-head", Image: "rayproject/ray:2.9.0"}, + }, + }, + }, + }). + WithWorkerGroupSpec(rayv1.WorkerGroupSpec{ + RayStartParams: make(map[string]string), + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + {Name: "ray-worker", Image: "rayproject/ray:2.9.0"}, + }, + }, + }, + }). + Obj()). + Obj() + gomega.Expect(k8sClient.Create(ctx, rc2)).To(gomega.Succeed()) + + rc3 = wrappers.MakeRayCluster("very-long-ray-cluster-name", ns.Name). + Profile("profile1"). + Spec(*wrappers.MakeRayClusterSpec(). + HeadGroupSpec(rayv1.HeadGroupSpec{ + RayStartParams: make(map[string]string), + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + {Name: "ray-head", Image: "rayproject/ray:2.9.0"}, + }, + }, + }, + }). + WithWorkerGroupSpec(rayv1.WorkerGroupSpec{ + RayStartParams: make(map[string]string), + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + {Name: "ray-worker", Image: "rayproject/ray:2.9.0"}, + }, + }, + }, + }). + Obj()). + Obj() + gomega.Expect(k8sClient.Create(ctx, rc3)).To(gomega.Succeed()) + }) + + // Simple client set that are using on unit tests not allow paging. + ginkgo.It("Should print ray clusters list with paging", func() { + streams, _, output, errOutput := genericiooptions.NewTestIOStreams() + configFlags := CreateConfigFlagsWithRestConfig(cfg, streams) + executeTime := time.Now() + kjobctl := cmd.NewKjobctlCmd(cmd.KjobctlOptions{ConfigFlags: configFlags, IOStreams: streams, + Clock: testingclock.NewFakeClock(executeTime)}) + + os.Setenv(list.KjobctlListRequestLimitEnvName, "1") + kjobctl.SetArgs([]string{"list", "raycluster", "--namespace", ns.Name}) + err := kjobctl.Execute() + + gomega.Expect(err).NotTo(gomega.HaveOccurred(), "%s: %s", err, output) + gomega.Expect(errOutput.String()).Should(gomega.BeEmpty()) + gomega.Expect(output.String()).Should(gomega.Equal(fmt.Sprintf(`NAME PROFILE LOCAL QUEUE DESIRED WORKERS AVAILABLE WORKERS CPUS MEMORY GPUS STATUS AGE +rc1 profile1 0 0 0 0 0 %s +rc2 profile1 0 0 0 0 0 %s +very-long-ray-cluster-name profile1 0 0 0 0 0 %s +`, + duration.HumanDuration(executeTime.Sub(rc1.CreationTimestamp.Time)), + duration.HumanDuration(executeTime.Sub(rc2.CreationTimestamp.Time)), + duration.HumanDuration(executeTime.Sub(rc3.CreationTimestamp.Time)), + ))) + }) + }) + + ginkgo.When("List Slurm Jobs", func() { + var ( + j1 *batchv1.Job + j2 *batchv1.Job + j3 *batchv1.Job + ) + + ginkgo.JustBeforeEach(func() { + j1 = wrappers.MakeJob("j1", ns.Name). + Profile("profile1"). + Mode(v1alpha1.SlurmMode). + Completions(3). + WithContainer(*wrappers.MakeContainer("c1", "sleep").Obj()). + RestartPolicy(corev1.RestartPolicyOnFailure). + Obj() + gomega.Expect(k8sClient.Create(ctx, j1)).To(gomega.Succeed()) + + j2 = wrappers.MakeJob("j2", ns.Name). + Profile("profile1"). + Mode(v1alpha1.SlurmMode). + Completions(3). + WithContainer(*wrappers.MakeContainer("c1", "sleep").Obj()). + RestartPolicy(corev1.RestartPolicyOnFailure). + Obj() + gomega.Expect(k8sClient.Create(ctx, j2)).To(gomega.Succeed()) + + j3 = wrappers.MakeJob("very-long-job-name", ns.Name). + Profile("profile1"). + Mode(v1alpha1.SlurmMode). + Completions(3). + WithContainer(*wrappers.MakeContainer("c1", "sleep").Obj()). + RestartPolicy(corev1.RestartPolicyOnFailure). + Obj() + gomega.Expect(k8sClient.Create(ctx, j3)).To(gomega.Succeed()) + }) + + // Simple client set that are using on unit tests not allow paging. + ginkgo.It("Should print slurm jobs list with paging", func() { + streams, _, output, errOutput := genericiooptions.NewTestIOStreams() + configFlags := CreateConfigFlagsWithRestConfig(cfg, streams) + executeTime := time.Now() + kjobctl := cmd.NewKjobctlCmd(cmd.KjobctlOptions{ConfigFlags: configFlags, IOStreams: streams, + Clock: testingclock.NewFakeClock(executeTime)}) + + os.Setenv(list.KjobctlListRequestLimitEnvName, "1") + kjobctl.SetArgs([]string{"list", "slurm", "--namespace", ns.Name}) + err := kjobctl.Execute() + + gomega.Expect(err).NotTo(gomega.HaveOccurred(), "%s: %s", err, output) + gomega.Expect(errOutput.String()).Should(gomega.BeEmpty()) + gomega.Expect(output.String()).Should(gomega.Equal(fmt.Sprintf(`NAME PROFILE LOCAL QUEUE COMPLETIONS DURATION AGE +j1 profile1 0/3 %s +j2 profile1 0/3 %s +very-long-job-name profile1 0/3 %s +`, + duration.HumanDuration(executeTime.Sub(j1.CreationTimestamp.Time)), + duration.HumanDuration(executeTime.Sub(j2.CreationTimestamp.Time)), + duration.HumanDuration(executeTime.Sub(j3.CreationTimestamp.Time)), + ))) + }) + }) +}) diff --git a/test/integration/kjobctl/suite_test.go b/test/integration/kjobctl/suite_test.go new file mode 100644 index 0000000..c89946a --- /dev/null +++ b/test/integration/kjobctl/suite_test.go @@ -0,0 +1,61 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kjobctl + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client" + + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/pkg/constants" + "sigs.k8s.io/kueue/cmd/experimental/kjobctl/test/framework" +) + +var ( + userID string + cfg *rest.Config + k8sClient client.Client + ctx context.Context + fwk *framework.Framework + crdPath = filepath.Join("..", "..", "..", "config", "crd", "bases") + rayCrdPath = filepath.Join("..", "..", "..", "dep-crds", "ray-operator") +) + +func TestKjobctl(t *testing.T) { + userID = os.Getenv(constants.SystemEnvVarNameUser) + gomega.RegisterFailHandler(ginkgo.Fail) + ginkgo.RunSpecs(t, "Kjobctl Suite") +} + +var _ = ginkgo.BeforeSuite(func() { + fwk = &framework.Framework{ + CRDPath: crdPath, + DepCRDPaths: []string{rayCrdPath}, + } + cfg = fwk.Init() + ctx, k8sClient = fwk.SetupClient(cfg) +}) + +var _ = ginkgo.AfterSuite(func() { + fwk.Teardown() +}) diff --git a/test/integration/kjobctl/util.go b/test/integration/kjobctl/util.go new file mode 100644 index 0000000..02002ad --- /dev/null +++ b/test/integration/kjobctl/util.go @@ -0,0 +1,33 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kjobctl + +import ( + "k8s.io/cli-runtime/pkg/genericclioptions" + "k8s.io/cli-runtime/pkg/genericiooptions" + "k8s.io/client-go/rest" +) + +func CreateConfigFlagsWithRestConfig(cfg *rest.Config, streams genericiooptions.IOStreams) *genericclioptions.ConfigFlags { + return genericclioptions. + NewConfigFlags(true). + WithDiscoveryQPS(50.0). + WithWarningPrinter(streams). + WithWrapConfigFn(func(*rest.Config) *rest.Config { + return cfg + }) +} diff --git a/test/util/constants.go b/test/util/constants.go new file mode 100644 index 0000000..efe9d95 --- /dev/null +++ b/test/util/constants.go @@ -0,0 +1,27 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import "time" + +const ( + Timeout = 5 * time.Second + LongTimeout = 45 * time.Second + Interval = time.Millisecond * 250 + + E2eTestBashImage = "registry.k8s.io/alpine-with-bash:1.0@sha256:0955672451201896cf9e2e5ce30bec0c7f10757af33bf78b7a6afa5672c596f5" +) diff --git a/test/util/util.go b/test/util/util.go new file mode 100644 index 0000000..e0041df --- /dev/null +++ b/test/util/util.go @@ -0,0 +1,171 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import ( + "bytes" + "context" + "fmt" + "os" + "os/exec" + "path" + "slices" + "strings" + + "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/remotecommand" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/config" + + kjob "sigs.k8s.io/kueue/cmd/experimental/kjobctl/apis/v1alpha1" +) + +// DeleteNamespace deletes all objects the tests typically create in the namespace. +func DeleteNamespace(ctx context.Context, c client.Client, ns *corev1.Namespace) error { + if ns == nil { + return nil + } + if err := c.DeleteAllOf(ctx, &batchv1.Job{}, client.InNamespace(ns.Name)); err != nil && !apierrors.IsNotFound(err) { + return err + } + if err := c.DeleteAllOf(ctx, &kjob.ApplicationProfile{}, client.InNamespace(ns.Name)); err != nil && !apierrors.IsNotFound(err) { + return err + } + if err := c.DeleteAllOf(ctx, &kjob.VolumeBundle{}, client.InNamespace(ns.Name)); err != nil && !apierrors.IsNotFound(err) { + return err + } + if err := c.DeleteAllOf(ctx, &kjob.JobTemplate{}, client.InNamespace(ns.Name)); err != nil && !apierrors.IsNotFound(err) { + return err + } + if err := c.DeleteAllOf(ctx, &corev1.Pod{}, client.InNamespace(ns.Name)); err != nil && !apierrors.IsNotFound(err) { + return err + } + if err := c.DeleteAllOf(ctx, &corev1.ConfigMap{}, client.InNamespace(ns.Name)); err != nil && !apierrors.IsNotFound(err) { + return err + } + if err := c.DeleteAllOf(ctx, &corev1.Service{}, client.InNamespace(ns.Name)); err != nil && !apierrors.IsNotFound(err) { + return err + } + if err := c.Delete(ctx, ns); err != nil && !apierrors.IsNotFound(err) { + return err + } + return nil +} + +// Run executes the provided command +func Run(cmd *exec.Cmd) ([]byte, error) { + dir, _ := GetProjectDir() + cmd.Dir = dir + + command := cmd.String() + fmt.Fprintf(ginkgo.GinkgoWriter, "running: %s\n", command) + output, err := cmd.CombinedOutput() + if err != nil { + return output, fmt.Errorf("%s failed with error: %v", command, err) + } + + return output, nil +} + +// GetNonEmptyLines converts given command output string into individual objects +// according to line breakers, and ignores the empty elements in it. +func GetNonEmptyLines(output string) []string { + return slices.DeleteFunc(strings.Split(output, "\n"), func(s string) bool { + return s == "" + }) +} + +// GetProjectDir will return the directory where the project is +func GetProjectDir() (string, error) { + wd, err := os.Getwd() + if err != nil { + return wd, err + } + return path.Clean(wd + "/../.."), nil +} + +// GetConfigWithContext creates a *rest.Config for talking to a Kubernetes API server +// with a specific context. +func GetConfigWithContext(kContext string) *rest.Config { + cfg, err := config.GetConfigWithContext(kContext) + if err != nil { + fmt.Printf("unable to get kubeconfig for context %q: %s", kContext, err) + os.Exit(1) + } + gomega.ExpectWithOffset(1, cfg).NotTo(gomega.BeNil()) + + cfg.APIPath = "/api" + cfg.ContentConfig.GroupVersion = &schema.GroupVersion{Group: "", Version: "v1"} + cfg.ContentConfig.NegotiatedSerializer = scheme.Codecs.WithoutConversion() + + return cfg +} + +// CreateClient creates a client.Client using the provided config. +func CreateClient(cfg *rest.Config) client.Client { + err := kjob.AddToScheme(scheme.Scheme) + gomega.ExpectWithOffset(1, err).NotTo(gomega.HaveOccurred()) + + client, err := client.New(cfg, client.Options{Scheme: scheme.Scheme}) + gomega.ExpectWithOffset(1, err).NotTo(gomega.HaveOccurred()) + gomega.ExpectWithOffset(1, client).NotTo(gomega.BeNil()) + + return client +} + +// CreateRestClient creates a *rest.RESTClient using the provided config. +func CreateRestClient(cfg *rest.Config) *rest.RESTClient { + restClient, err := rest.RESTClientFor(cfg) + gomega.ExpectWithOffset(1, err).Should(gomega.Succeed()) + gomega.ExpectWithOffset(1, restClient).NotTo(gomega.BeNil()) + + return restClient +} + +func KExecute(ctx context.Context, cfg *rest.Config, client *rest.RESTClient, ns, pod, container string) ([]byte, []byte, error) { + var out, outErr bytes.Buffer + + req := client.Post(). + Resource("pods"). + Namespace(ns). + Name(pod). + SubResource("exec"). + VersionedParams(&corev1.PodExecOptions{ + Container: container, + Command: []string{"cat", "/env.out"}, + Stdout: true, + Stderr: true, + }, scheme.ParameterCodec) + + executor, err := remotecommand.NewSPDYExecutor(cfg, "POST", req.URL()) + if err != nil { + return nil, nil, err + } + + if err = executor.StreamWithContext(ctx, remotecommand.StreamOptions{Stdout: &out, Stderr: &outErr}); err != nil { + return nil, nil, err + } + + return out.Bytes(), outErr.Bytes(), nil +}