rancher · mallardduck · Oct 9, 2024 · Sep 11, 2024 · Sep 11, 2024 · Sep 11, 2024
@@ -0,0 +1,55 @@
+#!/bin/bash
+
+# Exit immediately if a command exits with a non-zero status
+set -e
+
+# Function to get the previous tag
+getPreviousTag() {
+  local tagPrefix="$1"
+  # List all tags and filter ones that start with tagPrefix, sort by creation date
+  git tag --sort=-creatordate | grep "^${tagPrefix}" | head -n 1
+}
+
+# Determine if we're in a GitHub Actions environment
+if [ -n "$GITHUB_REF" ] && [ -n "$GITHUB_SHA" ]; then
+  # Use GHA environment variables
+  ref="$GITHUB_REF"
+  commitSha="${GITHUB_SHA:0:7}"
+else
+  # Fallback to local Git repo
+  if [ ! -d ".git" ]; then
+    echo "This script must be run from the root of a Git repository or GitHub Actions."
+    exit 1
+  fi
+  ref=$(git symbolic-ref HEAD)
+  commitSha=$(git rev-parse --short HEAD)
+fi
+
+branchTag=""
+branchStaticTag=""
+prevTag=""
+
+if [ "$ref" == "refs/heads/main" ]; then
+  branchTag="head"
+  branchStaticTag="main-${commitSha}"
+  prevTag=$(getPreviousTag "main-")
+elif [[ "$ref" == refs/heads/release/* ]]; then
+  version="${ref#refs/heads/release/}"  # Extract "vX.0"
+  branchTag="${version}-head"
+  branchStaticTag="${version}-head-${commitSha}"
+  prevTag=$(getPreviousTag "${version}-head-")
+else
+  gitTag=$(git tag -l --contains HEAD | head -n 1)
+  if [[ -n "$gitTag" ]]; then
+      branchTag="${gitTag}"
+      branchStaticTag="${gitTag}-${commitSha}"
+  else
+      branchTag="dev-${commitSha}"
+      branchStaticTag="dev-${commitSha}"
+  fi
+fi
+
+# Output the results
+echo "branch_tag=${branchTag}"
+echo "branch_static_tag=${branchStaticTag}"
+echo "prev_static_tag=${prevTag}"
@@ -32,23 +32,41 @@ env:
   YQ_VERSION: v4.25.1
   E2E_CI: true
   REPO: rancher
-  TAG: dev
   APISERVER_PORT: 8001
   DEFAULT_SLEEP_TIMEOUT_SECONDS: 10
   KUBECTL_WAIT_TIMEOUT: 300s
   DEBUG: ${{ github.event.inputs.debug || false }}
+  CLUSTER_NAME: 'e2e-ci-prometheus-federator'
 
 permissions:
   contents: write
 
 jobs:
+  prebuild-env:
+    name: Prebuild needed Env vars
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out the repository to the runner
+        uses: actions/checkout@v4
+      - name: Set Branch Tag and Other Variables
+        id: set-vars
+        run: bash ./.github/scripts/branch-tags.sh >> $GITHUB_OUTPUT
+    outputs:
+      branch_tag: ${{ steps.set-vars.outputs.branch_tag }}
+      branch_static_tag: ${{ steps.set-vars.outputs.branch_static_tag }}
+      prev_tag: ${{ steps.set-vars.outputs.prev_tag }}
   e2e-prometheus-federator:
+    needs: [
+      prebuild-env,
+    ]
     runs-on: ubuntu-latest
+    env:
+      TAG: ${{ needs.prebuild-env.outputs.branch_static_tag }}
     strategy:
       matrix:
         k3s_version:
           # k3d version list k3s | sed 's/+/-/' | sort -h
-          - ${{ github.event.inputs.k3s_version || 'v1.28.4-k3s2' }}
+          - ${{ github.event.inputs.k3s_version || 'v1.28.14-k3s1' }}
     steps:
       -
         uses: actions/checkout@v3
@@ -66,28 +84,24 @@ jobs:
         run: |
           sudo wget https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_amd64 -O /usr/bin/yq && sudo chmod +x /usr/bin/yq;
       -
-        name: Perform CI
+        name: Perform pre-e2e image build
         run: |
-          REPO=${REPO} TAG=${TAG} ./scripts/build;
-          REPO=${REPO} TAG=${TAG} ./scripts/package;
+          EMBEDED_CHART_VERSION=0.3.4 REPO=${REPO} TAG=${TAG} make build;
+          REPO=${REPO} TAG=${TAG} make package;
       -
-        name: Provision k3d Cluster
-        uses: AbsaOSS/k3d-action@v2
-        # k3d will automatically create a network named k3d-test-cluster-1 with the range 172.18.0.0/16
-        with:
-          cluster-name: "e2e-ci-prometheus-federator"
-          args: >-
-            --agents 1
-            --network "nw01"
-            --image docker.io/rancher/k3s:${{matrix.k3s_version}}
+        name : Install k3d
+        run : ./.github/workflows/e2e/scripts/install-k3d.sh
+      -
+        name : Setup k3d cluster
+        run : K3S_VERSION=${{ matrix.k3s_version }} ./.github/workflows/e2e/scripts/setup-cluster.sh
       -
         name: Import Images Into k3d
         run: |
-          k3d image import ${REPO}/prometheus-federator:${TAG} -c e2e-ci-prometheus-federator;
+          k3d image import ${REPO}/prometheus-federator:${TAG} -c $CLUSTER_NAME;
       -
         name: Setup kubectl context
         run: |
-          kubectl config use-context k3d-e2e-ci-prometheus-federator;
+          kubectl config use-context "k3d-$CLUSTER_NAME";
       -
         name: Set Up Tmate Debug Session
         if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.enable_tmate == 'true' }}
@@ -110,39 +124,39 @@ jobs:
       -
         name: Check if Project Registration Namespace is auto-created on namespace detection
         run: ./.github/workflows/e2e/scripts/create-project-namespace.sh;
-
-      # Commenting out for failure in CI but not locally
-      # -
-      #   name: Create Project Monitoring Stack via ProjectHelmChart CR
-      #   run: ./.github/workflows/e2e/scripts/create-projecthelmchart.sh;
-      # - 
-      #   name: Check if the Project Prometheus Stack is up
-      #   run: ./.github/workflows/e2e/scripts/validate-project-monitoring.sh;
-      # - 
-      #   name: Wait for 8 minutes for enough scraping to be done to continue
-      #   run: |
-      #     for i in {1..48}; do sleep 10; echo "Waited $((i*10)) seconds for metrics to be populated"...; done;
-      # -
-      #   name: Validate Project Prometheus Targets
-      #   run: ./.github/workflows/e2e/scripts/validate-project-prometheus-targets.sh;
-      # -
-      #   name: Validate Project Grafana Datasources
-      #   run: ./.github/workflows/e2e/scripts/validate-project-grafana-datasource.sh;
-      # -
-      #   name: Validate Project Grafana Dashboards
-      #   run: ./.github/workflows/e2e/scripts/validate-project-grafana-dashboards.sh;
-      # #-
-      #   #name: Validate Project Grafana Dashboard Data
-      #   #run: ./.github/workflows/e2e/scripts/validate-project-grafana-dashboard-data.sh;
-      # -
-      #   name: Validate Project Prometheus Alerts
-      #   run: ./.github/workflows/e2e/scripts/validate-project-prometheus-alerts.sh;
-      # -
-      #   name: Validate Project Alertmanager
-      #   run: ./.github/workflows/e2e/scripts/validate-project-alertmanager.sh;
-      # - 
-      #   name: Delete Project Prometheus Stack
-      #   run: ./.github/workflows/e2e/scripts/delete-projecthelmchart.sh;
+      -
+        name: Create Project Monitoring Stack via ProjectHelmChart CR
+        run: DEFAULT_SLEEP_TIMEOUT_SECONDS=20 ./.github/workflows/e2e/scripts/create-projecthelmchart.sh;
+      -
+        name: Check if the Project Prometheus Stack is up
+        run: ./.github/workflows/e2e/scripts/validate-project-monitoring.sh;
+      -
+        name: Wait for 8 minutes for enough scraping to be done to continue
+        run: |
+          for i in {1..48}; do sleep 10; echo "Waited $((i*10)) seconds for metrics to be populated"...; done;
+      -
+        name: Validate Project Prometheus Targets
+        run: ./.github/workflows/e2e/scripts/validate-project-prometheus-targets.sh;
+      -
+        name: Validate Project Grafana Datasources
+        run: ./.github/workflows/e2e/scripts/validate-project-grafana-datasource.sh;
+      -
+        name: Validate Project Grafana Dashboards
+        run: ./.github/workflows/e2e/scripts/validate-project-grafana-dashboards.sh;
+      # Re-disable this as it's been broken since Jun 28, 2023
+      # More context: https://github.com/rancher/prometheus-federator/pull/73
+      #  -
+      #    name: Validate Project Grafana Dashboard Data
+      #    run: ./.github/workflows/e2e/scripts/validate-project-grafana-dashboard-data.sh;
+      -
+        name: Validate Project Prometheus Alerts
+        run: ./.github/workflows/e2e/scripts/validate-project-prometheus-alerts.sh;
+      -
+        name: Validate Project Alertmanager
+        run: ./.github/workflows/e2e/scripts/validate-project-alertmanager.sh;
+      -
+        name: Delete Project Prometheus Stack
+        run: ./.github/workflows/e2e/scripts/delete-projecthelmchart.sh;
       - 
         name: Uninstall Prometheus Federator
         run: ./.github/workflows/e2e/scripts/uninstall-federator.sh;

@@ -6,8 +6,13 @@ source $(dirname $0)/entry
 
 cd $(dirname $0)/../../../..
 
-kubectl create namespace e2e-prometheus-federator || true
-kubectl label namespace e2e-prometheus-federator field.cattle.io/projectId=p-example --overwrite
+USE_RANCHER=${USE_RANCHER:-"false"}
+if [ "$USE_RANCHER" = "true" ]; then
+  kubectl apply -f ./examples/ci/project.yaml
+fi
+
+kubectl apply -f ./examples/ci/namespace.yaml
+
 sleep "${DEFAULT_SLEEP_TIMEOUT_SECONDS}"
 if ! kubectl get namespace cattle-project-p-example; then
     echo "ERROR: Expected cattle-project-p-example namespace to exist after ${DEFAULT_SLEEP_TIMEOUT_SECONDS} seconds, not found"

@@ -7,14 +7,14 @@ source $(dirname $0)/entry
 cd $(dirname $0)/../../../..
 
 if [[ "${E2E_CI}" == "true" ]]; then
-    kubectl apply -f ./examples/ci-example.yaml
+    kubectl apply -f ./examples/ci/project-helm-chart.yaml
 else
-    kubectl apply -f ./examples/example.yaml
+    kubectl apply -f ./examples/project-helm-chart.yaml
 fi
 sleep ${DEFAULT_SLEEP_TIMEOUT_SECONDS};
 
 if ! kubectl get -n cattle-monitoring-system job/helm-install-cattle-project-p-example-monitoring; then
-    echo "ERROR: Helm Install Job for Project Monitoring Stack was never created after ${KUBECTL_WAIT_TIMEOUT} seconds"
+    echo "ERROR: Helm Install Job for Project Monitoring Stack was never created after ${DEFAULT_SLEEP_TIMEOUT_SECONDS} seconds"
     exit 1
 fi
 

@@ -6,7 +6,11 @@ source $(dirname $0)/entry
 
 cd $(dirname $0)/../../../..
 
-kubectl delete -f ./examples/ci-example.yaml
+if [[ "${E2E_CI}" == "true" ]]; then
+    kubectl delete -f ./examples/ci/project-helm-chart.yaml
+else
+    kubectl delete -f ./examples/project-helm-chart.yaml
+fi
 if kubectl get -n cattle-monitoring-system job/helm-delete-cattle-project-p-example-monitoring --ignore-not-found; then
     if ! kubectl wait --for=condition=complete --timeout="${KUBECTL_WAIT_TIMEOUT}" -n cattle-monitoring-system job/helm-delete-cattle-project-p-example-monitoring; then
         echo "ERROR: Helm Uninstall Job for Project Monitoring Stack never completed after ${KUBECTL_WAIT_TIMEOUT}"

@@ -38,17 +38,41 @@ MANIFEST_DIRECTORY=${ARTIFACT_DIRECTORY}/manifests
 LOG_DIRECTORY=${ARTIFACT_DIRECTORY}/logs
 
 # Manifests
-
 mkdir -p ${MANIFEST_DIRECTORY}
-kubectl get pods -n cattle-monitoring-system -o yaml > ${MANIFEST_DIRECTORY}/monitoring_pods.yaml || true
-kubectl get pods -n cattle-project-p-example -o yaml > ${MANIFEST_DIRECTORY}/project_pods.yaml || true
+mkdir -p ${MANIFEST_DIRECTORY}/helmcharts
+mkdir -p ${MANIFEST_DIRECTORY}/helmreleases
+mkdir -p ${MANIFEST_DIRECTORY}/daemonsets
+mkdir -p ${MANIFEST_DIRECTORY}/deployments
+mkdir -p ${MANIFEST_DIRECTORY}/jobs
+mkdir -p ${MANIFEST_DIRECTORY}/statefulsets
+mkdir -p ${MANIFEST_DIRECTORY}/pods
+mkdir -p ${MANIFEST_DIRECTORY}/projecthelmcharts
+
 kubectl get namespaces -o yaml > ${MANIFEST_DIRECTORY}/namespaces.yaml || true
-kubectl get projecthelmchart -n cattle-project-p-example -o yaml > ${MANIFEST_DIRECTORY}/projecthelmcharts.yaml || true
-kubectl get helmcharts -n cattle-monitoring-system -o yaml > ${MANIFEST_DIRECTORY}/helmcharts.yaml || true
-kubectl get helmreleases -n cattle-monitoring-system -o yaml > ${MANIFEST_DIRECTORY}/helmreleases.yaml || true
+kubectl get helmcharts -A > ${MANIFEST_DIRECTORY}/helmcharts-list.txt || true
+kubectl get services -A > ${MANIFEST_DIRECTORY}/services-list.txt || true
 
-# Logs
+## cattle-monitoring-system ns manifests
+kubectl get helmcharts -n cattle-monitoring-system -o yaml > ${MANIFEST_DIRECTORY}/helmcharts/cattle-monitoring-system.yaml || true
+kubectl get helmreleases -n cattle-monitoring-system -o yaml > ${MANIFEST_DIRECTORY}/helmreleases/cattle-monitoring-system.yaml || true
+kubectl get daemonset -n cattle-monitoring-system -o yaml > ${MANIFEST_DIRECTORY}/daemonsets/cattle-monitoring-system.yaml || true
+kubectl get deployment -n cattle-monitoring-system -o yaml > ${MANIFEST_DIRECTORY}/deployments/cattle-monitoring-system.yaml || true
+kubectl get job -n cattle-monitoring-system -o yaml > ${MANIFEST_DIRECTORY}/jobs/cattle-monitoring-system.yaml || true
+kubectl get statefulset -n cattle-monitoring-system -o yaml > ${MANIFEST_DIRECTORY}/statefulsets/cattle-monitoring-system.yaml || true
+kubectl get pods -n cattle-monitoring-system -o yaml > ${MANIFEST_DIRECTORY}/pods/cattle-monitoring-system.yaml || true
 
+## cattle-project-p-example ns manifests
+kubectl get deployment -n cattle-project-p-example -o yaml > ${MANIFEST_DIRECTORY}/deployments/cattle-project-p-example.yaml || true
+kubectl get projecthelmchart -n cattle-project-p-example -o yaml > ${MANIFEST_DIRECTORY}/projecthelmcharts/cattle-project-p-example.yaml || true
+kubectl get statefulset -n cattle-project-p-example -o yaml > ${MANIFEST_DIRECTORY}/statefulsets/cattle-project-p-example.yaml || true
+kubectl get pods -n cattle-project-p-example -o yaml > ${MANIFEST_DIRECTORY}/pods/cattle-project-p-example.yaml || true
+
+## cattle-project-p-example-monitoring ns manifests
+kubectl get deployment -n cattle-project-p-example-monitoring -o yaml > ${MANIFEST_DIRECTORY}/deployments/cattle-project-p-example-monitoring.yaml || true
+kubectl get statefulset -n cattle-project-p-example-monitoring -o yaml > ${MANIFEST_DIRECTORY}/statefulsets/cattle-project-p-example-monitoring.yaml || true
+kubectl get pods -n cattle-project-p-example-monitoring -o yaml > ${MANIFEST_DIRECTORY}/pods/cattle-project-p-example-monitoring.yaml || true
+
+# Logs
 mkdir -p ${LOG_DIRECTORY}/rancher-monitoring
 
 ## Rancher Monitoring

@@ -25,7 +25,7 @@ case "${KUBERNETES_DISTRIBUTION_TYPE}" in
             cluster_args="--set helmProjectOperator.helmController.enabled=false"
         fi
         ;;
-    *)
+    v1.25.*)
         embedded_helm_controller_fixed_version="v1.25.4"
         if [[ $(echo ${kubernetes_version} ${embedded_helm_controller_fixed_version} | tr " " "\n" | sort -rV | head -n 1 ) == "${embedded_helm_controller_fixed_version}" ]]; then
             cluster_args="--set helmProjectOperator.helmController.enabled=false"
@@ -52,7 +52,7 @@ case "${KUBERNETES_DISTRIBUTION_TYPE}" in
             cluster_args="--set helmProjectOperator.helmController.enabled=false"
         fi
         ;;
-    *)
+    v1.25.*)
         embedded_helm_controller_fixed_version="v1.25.4"
         if [[ $(echo ${kubernetes_version} ${embedded_helm_controller_fixed_version} | tr " " "\n" | sort -rV | head -n 1 ) == "${embedded_helm_controller_fixed_version}" ]]; then
             cluster_args="--set helmProjectOperator.helmController.enabled=false"

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+set -e
+set -x
+
+K3D_URL=https://raw.githubusercontent.com/k3d-io/k3d/main/install.sh
+DEFAULT_K3D_VERSION=v5.7.4
+
+install_k3d(){
+  local k3dVersion=${K3D_VERSION:-${DEFAULT_K3D_VERSION}}
+  echo -e "Downloading k3d@${k3dVersion} see: ${K3D_URL}"
+  curl --silent --fail ${K3D_URL} | TAG=${k3dVersion} bash
+}
+
+install_k3d
+
+k3d version
@@ -13,6 +13,9 @@ helm version
 helm repo add ${HELM_REPO} https://charts.rancher.io
 helm repo update
 
+echo "Create required \`cattle-fleet-system\` namespace"
+kubectl create namespace cattle-fleet-system
+
 echo "Installing rancher monitoring crd with :\n"
 
 helm search repo ${HELM_REPO}/rancher-monitoring-crd --versions --max-col-width=0 | head -n 2