Merge branch 'kubeflow:master' into commonize-webhook-validations

.github/workflows/publish-core-images.yaml

@@ -26,10 +26,18 @@ jobs:
             dockerfile: build/images/training-operator/Dockerfile
             platforms: linux/amd64,linux/arm64,linux/ppc64le
             tag-prefix: v1
-          - component-name: training-operator
+          - component-name: training-operator-v2
             dockerfile: cmd/training-operator.v2alpha1/Dockerfile
             platforms: linux/amd64,linux/arm64,linux/ppc64le
             tag-prefix: v2alpha1
+          - component-name: model-initializer-v2
+            dockerfile: cmd/initializer_v2/model/Dockerfile
+            platforms: linux/amd64,linux/arm64
+            tag-prefix: v2
+          - component-name: dataset-initializer-v2
+            dockerfile: cmd/initializer_v2/dataset/Dockerfile
+            platforms: linux/amd64,linux/arm64
+            tag-prefix: v2
           - component-name: kubectl-delivery
             dockerfile: build/images/kubectl-delivery/Dockerfile
             platforms: linux/amd64,linux/arm64,linux/ppc64le

.github/workflows/publish-example-images.yaml

@@ -73,3 +73,7 @@ jobs:
             platforms: linux/amd64,linux/arm64
             dockerfile: examples/jax/cpu-demo/Dockerfile
             context: examples/jax/cpu-demo
+          - component-name: pytorch-deepspeed-demo
+            platforms: linux/amd64
+            dockerfile: examples/pytorch/deepspeed-demo/Dockerfile
+            context: examples/pytorch/deepspeed-demo

.github/workflows/test-go.yaml

@@ -29,14 +29,10 @@ jobs:
         run: |
           go mod tidy && pushd hack/swagger && go mod tidy && popd && git add go.* &&
           git diff --cached --exit-code || (echo 'Please run "go mod tidy" to sync Go modules' && exit 1);
-      - name: Check manifests
+      - name: Check auto-generated assets
         run: |
-          make manifests && git add manifests &&
-          git diff --cached --exit-code || (echo 'Please run "make manifests" to generate manifests' && exit 1);
-      - name: Check auto-generated codes
-        run: |
-          make generate && git add pkg sdk &&
-          git diff --cached --exit-code || (echo 'Please run "make generate" to generate Go codes' && exit 1);
+          make generate && git add pkg sdk manifests &&
+          git diff --cached --exit-code || (echo 'Please run "make generate" to generate assets' && exit 1);
       - name: Verify gofmt
         run: |
           make fmt && git add pkg cmd &&

.github/workflows/unittests.yaml

@@ -37,6 +37,7 @@ jobs:
 
       - name: Run Go test for v2
         run: |
+          make testv2
           make test-integrationv2 ENVTEST_K8S_VERSION=${{ matrix.kubernetes-version }}
 
       - name: Coveralls report

.gitignore

@@ -4,14 +4,15 @@ bin/
 /tf-operator
 vendor/
 testbin/*
+manifests/external-crds/
 cover.out
 
 # IDEs
 .vscode/
 __debug_bin
 
-# Compiled python files.
-*.pyc
+# Python cache files
+__pycache__/
 
 # Emacs temporary files
 *~
@@ -41,6 +42,7 @@ examples/.ipynb_checkpoints/
 
 # openapi-codegen tools and auto generated files but useless
 hack/python-sdk/openapi-generator-cli.jar
+hack/python-sdk-v2/openapi-generator-cli.jar
 
 # Coverage
 cover.out

.pre-commit-config.yaml

@@ -17,42 +17,26 @@ repos:
     rev: 24.2.0
     hooks:
       - id: black
-        files: (sdk|examples)/.*
-        exclude: |
-          (?x)^(
-            /*kubeflow_org_v1*|
-            __init__.py|
-            api_client.py|
-            configuration.py|
-            exceptions.py|
-            rest.py
-          )$
   - repo: https://github.com/pycqa/flake8
     rev: 7.1.1
     hooks:
       - id: flake8
-        files: ^(sdk|examples)/.*$
-        exclude: |
-          (?x)^(
-            .*kubeflow_org_v1.*|
-            __init__\.py|
-            api_client\.py|
-            configuration\.py|
-            exceptions\.py|
-            rest\.py
-          )$
 exclude: |
   (?x)^(
     pkg/apis/kubeflow.org/v1/openapi_generated.go|
     pkg/apis/kubeflow.org/v1/zz_.*|
     pkg/client/.*|
     test_job/apis/test_job/v1/.*generated.*.go|
     test_job/client/.*|
+    docs/api/.*|
+    hack/python-sdk/post_gen.py|
     sdk/python/kubeflow/training/[^/]*.py|
     sdk/python/kubeflow/training/models/.*|
     sdk/python/test/.*|
-    docs/api/.*|
     sdk/python/docs/.*|
     sdk/python/.openapi-generator/VERSION|
-    sdk/python/kubeflow/__init__.py
+    sdk/python/kubeflow/__init__.py|
+    sdk_v2/kubeflow/training/[^/]*.py|
+    sdk_v2/kubeflow/training/models/.*|
+    sdk_v2/docs/.*
   )$

Makefile

@@ -41,14 +41,17 @@ manifests: controller-gen ## Generate WebhookConfiguration, ClusterRole and Cust
 		output:crd:artifacts:config=manifests/base/crds \
 		output:rbac:artifacts:config=manifests/base/rbac \
 		output:webhook:artifacts:config=manifests/base/webhook
-	$(CONTROLLER_GEN) "crd:generateEmbeddedObjectMeta=true" "webhook" paths="./pkg/apis/kubeflow.org/v2alpha1/...;./pkg/webhook.v2/..." \
+	$(CONTROLLER_GEN) "crd:generateEmbeddedObjectMeta=true" rbac:roleName=training-operator-v2 webhook \
+		paths="./pkg/apis/kubeflow.org/v2alpha1/...;./pkg/controller.v2/...;./pkg/runtime.v2/...;./pkg/webhook.v2/...;./pkg/cert/..." \
 		output:crd:artifacts:config=manifests/v2/base/crds \
+		output:rbac:artifacts:config=manifests/v2/base/rbac \
 		output:webhook:artifacts:config=manifests/v2/base/webhook
 
-generate: controller-gen ## Generate apidoc, sdk and code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations.
+generate: controller-gen manifests ## Generate apidoc, sdk and code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations.
 	$(CONTROLLER_GEN) object:headerFile="hack/boilerplate/boilerplate.go.txt" paths="./pkg/apis/..."
 	hack/update-codegen.sh
 	hack/python-sdk/gen-sdk.sh
+	hack/python-sdk-v2/gen-sdk.sh
 	$(MAKE) apidoc
 
 apidoc:
@@ -74,12 +77,17 @@ HAS_SETUP_ENVTEST := $(shell command -v setup-envtest;)
 testall: manifests generate fmt vet golangci-lint test ## Run tests.
 
 test: envtest
-	KUBEBUILDER_ASSETS="$(shell setup-envtest use $(ENVTEST_K8S_VERSION) -p path)" go test ./... -coverprofile cover.out
+	KUBEBUILDER_ASSETS="$(shell setup-envtest use $(ENVTEST_K8S_VERSION) -p path)" \
+		go test ./pkg/apis/kubeflow.org/v1/... ./pkg/cert/... ./pkg/common/... ./pkg/config/... ./pkg/controller.v1/... ./pkg/core/... ./pkg/util/... ./pkg/webhooks/... -coverprofile cover.out
 
 .PHONY: test-integrationv2
-test-integrationv2: envtest
+test-integrationv2: envtest jobset-operator-crd scheduler-plugins-crd
 	KUBEBUILDER_ASSETS="$(shell setup-envtest use $(ENVTEST_K8S_VERSION) -p path)" go test ./test/... -coverprofile cover.out
 
+.PHONY: testv2
+testv2:
+	go test ./pkg/apis/kubeflow.org/v2alpha1/... ./pkg/controller.v2/... ./pkg/runtime.v2/... ./pkg/webhook.v2/... ./pkg/util.v2/... -coverprofile cover.out
+
 envtest:
 ifndef HAS_SETUP_ENVTEST
 	go install sigs.k8s.io/controller-runtime/tools/setup-envtest@bf15e44028f908c790721fc8fe67c7bf2d06a611 # v0.17.2
@@ -123,3 +131,18 @@ controller-gen: ## Download controller-gen locally if necessary.
 KUSTOMIZE = $(shell pwd)/bin/kustomize
 kustomize: ## Download kustomize locally if necessary.
 	GOBIN=$(PROJECT_DIR)/bin go install sigs.k8s.io/kustomize/kustomize/[email protected]
+
+## Download external CRDs for the integration testings.
+EXTERNAL_CRDS_DIR ?= $(PROJECT_DIR)/manifests/external-crds
+
+JOBSET_ROOT = $(shell go list -m -mod=readonly -f "{{.Dir}}" sigs.k8s.io/jobset)
+.PHONY: jobset-operator-crd
+jobset-operator-crd: ## Copy the CRDs from the jobset-operator to the manifests/external-crds directory.
+	mkdir -p $(EXTERNAL_CRDS_DIR)/jobset-operator/
+	cp -f $(JOBSET_ROOT)/config/components/crd/bases/* $(EXTERNAL_CRDS_DIR)/jobset-operator/
+
+SCHEDULER_PLUGINS_ROOT = $(shell go list -m -f "{{.Dir}}" sigs.k8s.io/scheduler-plugins)
+.PHONY: scheduler-plugins-crd
+scheduler-plugins-crd:
+	mkdir -p $(EXTERNAL_CRDS_DIR)/scheduler-plugins/
+	cp -f $(SCHEDULER_PLUGINS_ROOT)/manifests/coscheduling/* $(EXTERNAL_CRDS_DIR)/scheduler-plugins

api.v2/openapi-spec/README.md

@@ -0,0 +1,4 @@
+# Kubeflow Training V2 OpenAPI Specification
+
+This folder contains an [OpenAPI specification](https://github.com/OAI/OpenAPI-Specification)
+for Training V2 API.