Merge pull request #171 from mallardduck/fix-e2e-tags-main

fix(ci): GHA e2e bug depending on branch/tag

.github/workflows/e2e/scripts/validate-project-alertmanager.sh

@@ -10,27 +10,59 @@ tmp_alerts_yaml=$(mktemp)
 trap 'cleanup' EXIT
 cleanup() {
     set +e
-    rm ${tmp_alerts_yaml}
+    rm "${tmp_alerts_yaml}"
 }
 
-if [[ -z "${RANCHER_TOKEN}" ]]; then
-    curl -s ${API_SERVER_URL}/api/v1/namespaces/cattle-project-p-example/services/http:cattle-project-p-example-m-alertmanager:9093/proxy/api/v2/alerts | yq -P - > ${tmp_alerts_yaml}
-else
-    curl -s ${API_SERVER_URL}/api/v1/namespaces/cattle-project-p-example/services/http:cattle-project-p-example-m-alertmanager:9093/proxy/api/v2/alerts -k -H "Authorization: Bearer ${RANCHER_TOKEN}" | yq -P - > ${tmp_alerts_yaml}
-fi
-
-if [[ $(yq '. | length' "${tmp_alerts_yaml}") != "1" ]]; then
-    echo "ERROR: Found the wrong number of alerts in Project Alertmanager, expected only 'Watchdog'"
-    cat ${tmp_alerts_yaml}
-    exit 1
-fi
-
-if [[ $(yq '.[0].labels.alertname' "${tmp_alerts_yaml}") != "Watchdog" ]]; then
-    echo "ERROR: Expected the only alert to be triggered on the Project Alertmanager to be 'Watchdog'"
-    cat ${tmp_alerts_yaml}
-    exit 1
-fi
-
-cat ${tmp_alerts_yaml}
+checkData() {
+  if [[ -z "${RANCHER_TOKEN}" ]]; then
+      curl -s ${API_SERVER_URL}/api/v1/namespaces/cattle-project-p-example/services/http:cattle-project-p-example-m-alertmanager:9093/proxy/api/v2/alerts | yq -P - > "${tmp_alerts_yaml}"
+  else
+      curl -s ${API_SERVER_URL}/api/v1/namespaces/cattle-project-p-example/services/http:cattle-project-p-example-m-alertmanager:9093/proxy/api/v2/alerts -k -H "Authorization: Bearer ${RANCHER_TOKEN}" | yq -P - > "${tmp_alerts_yaml}"
+  fi
+}
+
+WAIT_TIMEOUT="${KUBECTL_WAIT_TIMEOUT%s}"
+START_TIME=$(date +%s)
+while true; do
+  checkData
+  CHECKS_PASSED=0
+
+  # Check if timeout has been reached
+  CURRENT_TIME=$(date +%s)
+  ELAPSED_TIME=$((CURRENT_TIME - START_TIME))
+  if [[ $ELAPSED_TIME -ge $WAIT_TIMEOUT ]]; then
+      echo "ERROR: Timeout reached, condition not met."
+      exit 1
+  fi
+
+  if [[ $(yq '. | length' "${tmp_alerts_yaml}") != "1" ]]; then
+      echo "ERROR: Found the wrong number of alerts in Project Alertmanager, expected only 'Watchdog'"
+      cat "${tmp_alerts_yaml}"
+
+    echo "Retrying in $DEFAULT_SLEEP_TIMEOUT_SECONDS seconds..."
+    sleep "$DEFAULT_SLEEP_TIMEOUT_SECONDS"
+    continue
+  fi
+  CHECKS_PASSED=$((CHECKS_PASSED+1))
+
+
+  if [[ $(yq '.[0].labels.alertname' "${tmp_alerts_yaml}") != "Watchdog" ]]; then
+      echo "ERROR: Expected the only alert to be triggered on the Project Alertmanager to be 'Watchdog'"
+      cat "${tmp_alerts_yaml}"
+
+      echo "Retrying in $DEFAULT_SLEEP_TIMEOUT_SECONDS seconds..."
+      sleep "$DEFAULT_SLEEP_TIMEOUT_SECONDS"
+      continue
+  fi
+  CHECKS_PASSED=$((CHECKS_PASSED+1))
+
+  if [[ $CHECKS_PASSED -eq 2 ]];then
+    # Get final elapsed time
+    ELAPSED_TIME=$((CURRENT_TIME - START_TIME))
+    break
+  fi
+done
+
+cat "${tmp_alerts_yaml}"
 
 echo "PASS: Project Alertmanager is up and running"

.github/workflows/e2e/scripts/validate-project-grafana-dashboards.sh

@@ -52,7 +52,7 @@ while true; do
   CURRENT_TIME=$(date +%s)
   ELAPSED_TIME=$((CURRENT_TIME - START_TIME))
   if [[ $ELAPSED_TIME -ge $WAIT_TIMEOUT ]]; then
-      echo "Error: Timeout reached, condition not met."
+      echo "ERROR: Timeout reached, condition not met."
       exit 1
   fi
 

.github/workflows/e2e/scripts/validate-project-prometheus-alerts.sh

@@ -35,7 +35,7 @@ while true; do
   CURRENT_TIME=$(date +%s)
   ELAPSED_TIME=$((CURRENT_TIME - START_TIME))
   if [[ $ELAPSED_TIME -ge $WAIT_TIMEOUT ]]; then
-      echo "Error: Timeout reached, condition not met."
+      echo "ERROR: Timeout reached, condition not met."
       exit 1
   fi
 

.github/workflows/e2e/scripts/validate-project-prometheus-targets.sh

@@ -34,7 +34,7 @@ while true; do
   CURRENT_TIME=$(date +%s)
   ELAPSED_TIME=$((CURRENT_TIME - START_TIME))
   if [[ $ELAPSED_TIME -ge $WAIT_TIMEOUT ]]; then
-      echo "Error: Timeout reached, condition not met."
+      echo "ERROR: Timeout reached, condition not met."
       exit 1
   fi
 
@@ -58,6 +58,13 @@ while true; do
           sleep "$DEFAULT_SLEEP_TIMEOUT_SECONDS"
           break
       fi
+      if ! grep "${expected_target}" "${tmp_targets_up_yaml}" | grep up; then
+          echo "ERROR: Expected '${expected_target}' to exist in 'up' state"
+
+          echo "Retrying in $DEFAULT_SLEEP_TIMEOUT_SECONDS seconds..."
+          sleep "$DEFAULT_SLEEP_TIMEOUT_SECONDS"
+          break
+      fi
       FOUND_TARGETS=$((FOUND_TARGETS+1))
   done
 

.github/workflows/prom-fed-e2e-ci.yaml

@@ -46,28 +46,7 @@ permissions:
   contents: write
 
 jobs:
-  prebuild-env:
-    name: Prebuild needed Env vars
-    strategy:
-      matrix:
-        arch:
-          - x64
-          - arm64
-    runs-on : runs-on,image=ubuntu22-full-${{ matrix.arch }},runner=4cpu-linux-${{ matrix.arch }},run-id=${{ github.run_id }}
-    steps:
-      - name: Check out the repository to the runner
-        uses: actions/checkout@v4
-      - name: Set Branch Tag and Other Variables
-        id: set-vars
-        run: bash ./.github/scripts/branch-tags.sh >> $GITHUB_OUTPUT
-    outputs:
-      branch_tag: ${{ steps.set-vars.outputs.branch_tag }}
-      branch_static_tag: ${{ steps.set-vars.outputs.branch_static_tag }}
-      prev_tag: ${{ steps.set-vars.outputs.prev_tag }}
   e2e-prometheus-federator:
-    needs: [
-      prebuild-env,
-    ]
     strategy:
       matrix:
         arch:
@@ -77,8 +56,6 @@ jobs:
           # k3d version list k3s | sed 's/+/-/' | sort -h
           - ${{ github.event.inputs.k3s_version || 'v1.28.14-k3s1' }}
     runs-on : runs-on,image=ubuntu22-full-${{ matrix.arch }},runner=4cpu-linux-${{ matrix.arch }},run-id=${{ github.run_id }}
-    env:
-      TAG: ${{ needs.prebuild-env.outputs.branch_static_tag }}
     steps:
       -
         uses: actions/checkout@v4
@@ -95,11 +72,16 @@ jobs:
         name: Install mikefarah/yq
         run: |
           sudo wget https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_${{ matrix.arch == 'x64' && 'amd64' || matrix.arch  }} -O /usr/bin/yq && sudo chmod +x /usr/bin/yq;
+      - name : Export image version
+        run : |
+          source ./scripts/version
+          echo TAG=$TAG >> $GITHUB_ENV
+          echo IMAGE=$IMAGE >> $GITHUB_ENV
       -
         name: Perform pre-e2e image build
         run: |
-          REPO=${REPO} TAG=${TAG} make build;
-          REPO=${REPO} TAG=${TAG} make package;
+          make build;
+          make package;
       -
         name : Install k3d
         run : ./.github/workflows/e2e/scripts/install-k3d.sh
@@ -109,7 +91,7 @@ jobs:
       -
         name: Import Images Into k3d
         run: |
-          k3d image import ${REPO}/prometheus-federator:${TAG} -c $CLUSTER_NAME;
+          k3d image import ${IMAGE} -c $CLUSTER_NAME;
       -
         name: Setup kubectl context
         run: |
@@ -154,7 +136,7 @@ jobs:
         run: KUBECTL_WAIT_TIMEOUT=480 ./.github/workflows/e2e/scripts/validate-project-prometheus-alerts.sh;
       -
         name: Validate Project Alertmanager
-        run: ./.github/workflows/e2e/scripts/validate-project-alertmanager.sh;
+        run: KUBECTL_WAIT_TIMEOUT=480 ./.github/workflows/e2e/scripts/validate-project-alertmanager.sh;
       -
         name: Delete Project Prometheus Stack
         run: ./.github/workflows/e2e/scripts/delete-projecthelmchart.sh;