Merge pull request #170 from mallardduck/fix-versions-tag-bug

fix(ci): corrects multiple tags bug & add a `local-ci` script
rancher · Feb 13, 2025 · 60f28e7 · 60f28e7
2 parents 57444ba + 885cb06
commit 60f28e7
Show file tree

Hide file tree

Showing 9 changed files with 297 additions and 73 deletions.
diff --git a/.github/scripts/branch-tags.sh b/.github/scripts/branch-tags.sh
@@ -44,8 +44,8 @@ else
       branchTag="${gitTag}"
       branchStaticTag="${gitTag}-${commitSha}"
   else
-      branchTag="dev-${commitSha}"
-      branchStaticTag="dev-${commitSha}"
+      branchTag="0.0.0-dev"
+      branchStaticTag="0.0.0-dev-${commitSha}"
   fi
 fi
 

diff --git a/.github/workflows/e2e/scripts/install-federator.sh b/.github/workflows/e2e/scripts/install-federator.sh
@@ -8,10 +8,7 @@ source $(dirname $0)/cluster-args.sh
 cd $(dirname $0)/../../../..
 source "$(pwd)/scripts/util-team-charts"
 
-NEWEST_CHART_VERSION=$(newest-chart-version "prometheus-federator")
-fetch-team-chart "prometheus-federator" "$NEWEST_CHART_VERSION"
-LATEST_CHART_PATH="./build/charts/prometheus-federator-${NEWEST_CHART_VERSION}.tgz"
-tar -xvzf "$LATEST_CHART_PATH" -C ./build/charts/
+make package-helm
 
 helm upgrade --install --create-namespace -n cattle-monitoring-system prometheus-federator --set helmProjectOperator.image.repository=${REPO:-rancher}/prometheus-federator --set helmProjectOperator.image.tag=${TAG:-dev} ${cluster_args} ${RANCHER_HELM_ARGS} ./build/charts/prometheus-federator
 

diff --git a/.github/workflows/e2e/scripts/validate-project-grafana-dashboards.sh b/.github/workflows/e2e/scripts/validate-project-grafana-dashboards.sh
@@ -13,11 +13,13 @@ cleanup() {
     rm ${tmp_dashboards_yaml}
 }
 
-if [[ -z "${RANCHER_TOKEN}" ]]; then
-    curl -s ${API_SERVER_URL}/api/v1/namespaces/cattle-project-p-example/services/http:cattle-project-p-example-monitoring-grafana:80/proxy/api/search | yq -P - > ${tmp_dashboards_yaml}
-else
-    curl -s ${API_SERVER_URL}/api/v1/namespaces/cattle-project-p-example/services/http:cattle-project-p-example-monitoring-grafana:80/proxy/api/search -k -H "Authorization: Bearer ${RANCHER_TOKEN}" | yq -P - > ${tmp_dashboards_yaml}
-fi
+checkData() {
+  if [[ -z "${RANCHER_TOKEN}" ]]; then
+      curl -s ${API_SERVER_URL}/api/v1/namespaces/cattle-project-p-example/services/http:cattle-project-p-example-monitoring-grafana:80/proxy/api/search | yq -P - > ${tmp_dashboards_yaml}
+  else
+      curl -s ${API_SERVER_URL}/api/v1/namespaces/cattle-project-p-example/services/http:cattle-project-p-example-monitoring-grafana:80/proxy/api/search -k -H "Authorization: Bearer ${RANCHER_TOKEN}" | yq -P - > ${tmp_dashboards_yaml}
+  fi
+}
 
 expected_dashboards=(
     db/alertmanager-overview
@@ -41,18 +43,42 @@ expected_dashboards=(
     db/rancher-workload-pods
 );
 
-if [[ $(yq '.[].uri' ${tmp_dashboards_yaml} | wc -l | xargs) != "${#expected_dashboards[@]}" ]]; then
-    echo "ERROR: Found the wrong number of dashboards in Project Grafana, expected only the following: ${expected_dashboards[@]}"
-    cat ${tmp_dashboards_yaml}
-    exit 1
-fi      
-
-for dashboard in "${expected_dashboards[@]}"; do
-    if ! yq '.[].uri' ${tmp_dashboards_yaml} | grep "${dashboard}" 1>/dev/null; then
-        echo "ERROR: Expected '${dashboard}' to exist amongst ${#expected_dashboards[@]} dashboards in Project Grafana"
-        cat ${tmp_dashboards_yaml}
-        exit 1
-    fi
+WAIT_TIMEOUT="${KUBECTL_WAIT_TIMEOUT%s}"
+START_TIME=$(date +%s)
+while true; do
+  checkData
+
+  # Check if timeout has been reached
+  CURRENT_TIME=$(date +%s)
+  ELAPSED_TIME=$((CURRENT_TIME - START_TIME))
+  if [[ $ELAPSED_TIME -ge $WAIT_TIMEOUT ]]; then
+      echo "Error: Timeout reached, condition not met."
+      exit 1
+  fi
+
+  if [[ $(yq '.[].uri' ${tmp_dashboards_yaml} | wc -l | xargs) != "${#expected_dashboards[@]}" ]]; then
+    echo "Retrying in $DEFAULT_SLEEP_TIMEOUT_SECONDS seconds..."
+    sleep "$DEFAULT_SLEEP_TIMEOUT_SECONDS"
+    continue
+  fi
+
+  FOUND_DASHBOARDS=0
+  for dashboard in "${expected_dashboards[@]}"; do
+      if ! yq '.[].uri' ${tmp_dashboards_yaml} | grep "${dashboard}" 1>/dev/null; then
+          echo "ERROR: Expected '${dashboard}' to exist amongst ${#expected_dashboards[@]} dashboards in Project Grafana"
+          cat ${tmp_dashboards_yaml}
+          echo "Retrying in $DEFAULT_SLEEP_TIMEOUT_SECONDS seconds..."
+          sleep "$DEFAULT_SLEEP_TIMEOUT_SECONDS"
+          break
+      fi
+      FOUND_DASHBOARDS=$((FOUND_DASHBOARDS+1))
+  done
+
+  if [[ FOUND_DASHBOARDS -eq 19 ]];then
+    # Get final elapsed time
+    ELAPSED_TIME=$((CURRENT_TIME - START_TIME))
+    break
+  fi
 done
 
 cat ${tmp_dashboards_yaml}

diff --git a/.github/workflows/e2e/scripts/validate-project-prometheus-alerts.sh b/.github/workflows/e2e/scripts/validate-project-prometheus-alerts.sh
@@ -11,32 +11,64 @@ tmp_alert_rules_yaml=$(mktemp)
 trap 'cleanup' EXIT
 cleanup() {
     set +e
-    rm ${tmp_rules_yaml}
-    rm ${tmp_alert_rules_yaml}
+    rm "${tmp_rules_yaml}"
+    rm "${tmp_alert_rules_yaml}"
 }
 
-if [[ -z "${RANCHER_TOKEN}" ]]; then
-    curl -s ${API_SERVER_URL}/api/v1/namespaces/cattle-project-p-example/services/http:cattle-project-p-example-m-prometheus:9090/proxy/api/v1/alerts | yq -P - > ${tmp_rules_yaml}
-else
-    curl -s ${API_SERVER_URL}/api/v1/namespaces/cattle-project-p-example/services/http:cattle-project-p-example-m-prometheus:9090/proxy/api/v1/alerts -k -H "Authorization: Bearer ${RANCHER_TOKEN}" | yq -P - > ${tmp_rules_yaml}
-fi
+checkData() {
+  if [[ -z "${RANCHER_TOKEN}" ]]; then
+      curl -s "${API_SERVER_URL}/api/v1/namespaces/cattle-project-p-example/services/http:cattle-project-p-example-m-prometheus:9090/proxy/api/v1/alerts" | yq -P - > "${tmp_rules_yaml}"
+  else
+      curl -s "${API_SERVER_URL}/api/v1/namespaces/cattle-project-p-example/services/http:cattle-project-p-example-m-prometheus:9090/proxy/api/v1/alerts" -k -H "Authorization: Bearer ${RANCHER_TOKEN}" | yq -P - > "${tmp_rules_yaml}"
+  fi
 
-yq '.data.alerts' ${tmp_rules_yaml} > ${tmp_alert_rules_yaml}
+  yq '.data.alerts' "${tmp_rules_yaml}" > "${tmp_alert_rules_yaml}"
+}
+
+WAIT_TIMEOUT="${KUBECTL_WAIT_TIMEOUT%s}"
+START_TIME=$(date +%s)
+while true; do
+  checkData
+  CHECKS_PASSED=0
 
-if [[ $(yq '. | length' ${tmp_alert_rules_yaml}) != "1" ]]; then
+  # Check if timeout has been reached
+  CURRENT_TIME=$(date +%s)
+  ELAPSED_TIME=$((CURRENT_TIME - START_TIME))
+  if [[ $ELAPSED_TIME -ge $WAIT_TIMEOUT ]]; then
+      echo "Error: Timeout reached, condition not met."
+      exit 1
+  fi
+
+  if [[ $(yq '. | length' "${tmp_alert_rules_yaml}") != "1" ]]; then
     echo "ERROR: Found the wrong number of alerts in Project Prometheus, expected only 'Watchdog'"
     echo "ALERT RULES:"
-    cat ${tmp_alert_rules_yaml}
-    exit 1
-fi
+    cat "${tmp_alert_rules_yaml}"
 
-if [[ $(yq '.[0].labels.alertname' ${tmp_alert_rules_yaml}) != "Watchdog" ]]; then
-    echo "ERROR: Expected the only alert to be triggered on the Project Prometheus to be 'Watchdog'"
-    echo "ALERT RULES:"
-    cat ${tmp_alert_rules_yaml}
-    exit 1
-fi
+    echo "Retrying in $DEFAULT_SLEEP_TIMEOUT_SECONDS seconds..."
+    sleep "$DEFAULT_SLEEP_TIMEOUT_SECONDS"
+    continue
+  fi
+  CHECKS_PASSED=$((CHECKS_PASSED+1))
+
+
+  if [[ $(yq '.[0].labels.alertname' "${tmp_alert_rules_yaml}") != "Watchdog" ]]; then
+      echo "ERROR: Expected the only alert to be triggered on the Project Prometheus to be 'Watchdog'"
+      echo "ALERT RULES:"
+      cat "${tmp_alert_rules_yaml}"
+
+      echo "Retrying in $DEFAULT_SLEEP_TIMEOUT_SECONDS seconds..."
+      sleep "$DEFAULT_SLEEP_TIMEOUT_SECONDS"
+      continue
+  fi
+  CHECKS_PASSED=$((CHECKS_PASSED+1))
+
+  if [[ $CHECKS_PASSED -eq 2 ]];then
+    # Get final elapsed time
+    ELAPSED_TIME=$((CURRENT_TIME - START_TIME))
+    break
+  fi
+done
 
-cat ${tmp_alert_rules_yaml};
+cat "${tmp_alert_rules_yaml}";
 
 echo "PASS: Project Prometheus has exactly one alert (Watchdog) active"
diff --git a/.github/workflows/e2e/scripts/validate-project-prometheus-targets.sh b/.github/workflows/e2e/scripts/validate-project-prometheus-targets.sh
@@ -11,34 +11,61 @@ tmp_targets_up_yaml=$(mktemp)
 trap 'cleanup' EXIT
 cleanup() {
     set +e
-    rm ${tmp_targets_yaml}
-    rm ${tmp_targets_up_yaml}
+    rm "${tmp_targets_yaml}"
+    rm "${tmp_targets_up_yaml}"
 }
 
-if [[ -z "${RANCHER_TOKEN}" ]]; then
-    curl -s ${API_SERVER_URL}/api/v1/namespaces/cattle-project-p-example/services/http:cattle-project-p-example-m-prometheus:9090/proxy/api/v1/targets | yq -P - > ${tmp_targets_yaml}
-else
-    curl -s ${API_SERVER_URL}/api/v1/namespaces/cattle-project-p-example/services/http:cattle-project-p-example-m-prometheus:9090/proxy/api/v1/targets -k -H "Authorization: Bearer ${RANCHER_TOKEN}" | yq -P - > ${tmp_targets_yaml}
-fi
+checkData() {
+  if [[ -z "${RANCHER_TOKEN}" ]]; then
+      curl -s ${API_SERVER_URL}/api/v1/namespaces/cattle-project-p-example/services/http:cattle-project-p-example-m-prometheus:9090/proxy/api/v1/targets | yq -P - > "${tmp_targets_yaml}"
+  else
+      curl -s ${API_SERVER_URL}/api/v1/namespaces/cattle-project-p-example/services/http:cattle-project-p-example-m-prometheus:9090/proxy/api/v1/targets -k -H "Authorization: Bearer ${RANCHER_TOKEN}" | yq -P - > "${tmp_targets_yaml}"
+  fi
 
-yq '.data.activeTargets[] | {.labels.job: .health}' ${tmp_targets_yaml} > ${tmp_targets_up_yaml};
+  yq '.data.activeTargets[] | {.labels.job: .health}' "${tmp_targets_yaml}" > "${tmp_targets_up_yaml}";
+}
+
+WAIT_TIMEOUT="${KUBECTL_WAIT_TIMEOUT%s}"
+START_TIME=$(date +%s)
+while true; do
+  checkData
 
-echo "TARGETS:";
-if [[ $(yq '. | length' ${tmp_targets_up_yaml}) != "4" ]]; then
-    echo "ERROR: Expected exactly 4 targets but found $(yq '. | length' ${tmp_targets_up_yaml})."
+  # Check if timeout has been reached
+  CURRENT_TIME=$(date +%s)
+  ELAPSED_TIME=$((CURRENT_TIME - START_TIME))
+  if [[ $ELAPSED_TIME -ge $WAIT_TIMEOUT ]]; then
+      echo "Error: Timeout reached, condition not met."
+      exit 1
+  fi
+
+  if [[ $(yq '. | length' "${tmp_targets_up_yaml}") != "4" ]]; then
+    echo "ERROR: Expected exactly 4 targets but found $(yq '. | length' "${tmp_targets_up_yaml}")."
     echo "Expected Targets in Project Prometheus: federate, cattle-project-p-example-m-alertmanager, cattle-project-p-example-m-prometheus, cattle-project-p-example-monitoring-grafana"
     echo "TARGETS:"
-    cat ${tmp_targets_up_yaml}
-    exit 1
-fi
-
-for expected_target in federate cattle-project-p-example-m-alertmanager cattle-project-p-example-m-prometheus cattle-project-p-example-monitoring-grafana; do
-    if ! grep "${expected_target}" ${tmp_targets_up_yaml}; then
-        echo "ERROR: Expected '${expected_target}' to exist amongst 4 targets in Project Prometheus"
-        echo "TARGETS:"
-        cat ${tmp_targets_up_yaml}
-        exit 1
-    fi
+    cat "${tmp_targets_up_yaml}"
+
+    echo "Retrying in $DEFAULT_SLEEP_TIMEOUT_SECONDS seconds..."
+    sleep "$DEFAULT_SLEEP_TIMEOUT_SECONDS"
+    continue
+  fi
+
+  FOUND_TARGETS=0
+  for expected_target in federate cattle-project-p-example-m-alertmanager cattle-project-p-example-m-prometheus cattle-project-p-example-monitoring-grafana; do
+      if ! grep "${expected_target}" "${tmp_targets_up_yaml}"; then
+          echo "ERROR: Expected '${expected_target}' to exist amongst 4 targets in Project Prometheus"
+
+          echo "Retrying in $DEFAULT_SLEEP_TIMEOUT_SECONDS seconds..."
+          sleep "$DEFAULT_SLEEP_TIMEOUT_SECONDS"
+          break
+      fi
+      FOUND_TARGETS=$((FOUND_TARGETS+1))
+  done
+
+  if [[ $FOUND_TARGETS -eq 4 ]];then
+    # Get final elapsed time
+    ELAPSED_TIME=$((CURRENT_TIME - START_TIME))
+    break
+  fi
 done
 
-echo "PASS: Project Prometheus has all targets healthy"
+echo "PASS: Project Prometheus has all targets healthy after ${ELAPSED_TIME}s"
diff --git a/.github/workflows/prom-fed-e2e-ci.yaml b/.github/workflows/prom-fed-e2e-ci.yaml
@@ -135,13 +135,9 @@ jobs:
       -
         name: Check if the Project Prometheus Stack is up
         run: ./.github/workflows/e2e/scripts/validate-project-monitoring.sh;
-      -
-        name: Wait for 8 minutes for enough scraping to be done to continue
-        run: |
-          for i in {1..48}; do sleep 10; echo "Waited $((i*10)) seconds for metrics to be populated"...; done;
       -
         name: Validate Project Prometheus Targets
-        run: ./.github/workflows/e2e/scripts/validate-project-prometheus-targets.sh;
+        run: KUBECTL_WAIT_TIMEOUT=480 ./.github/workflows/e2e/scripts/validate-project-prometheus-targets.sh;
       -
         name: Validate Project Grafana Datasources
         run: ./.github/workflows/e2e/scripts/validate-project-grafana-datasource.sh;
@@ -155,7 +151,7 @@ jobs:
       #    run: ./.github/workflows/e2e/scripts/validate-project-grafana-dashboard-data.sh;
       -
         name: Validate Project Prometheus Alerts
-        run: ./.github/workflows/e2e/scripts/validate-project-prometheus-alerts.sh;
+        run: KUBECTL_WAIT_TIMEOUT=480 ./.github/workflows/e2e/scripts/validate-project-prometheus-alerts.sh;
       -
         name: Validate Project Alertmanager
         run: ./.github/workflows/e2e/scripts/validate-project-alertmanager.sh;

diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml
@@ -137,6 +137,8 @@ jobs:
           username: ${{ env.DOCKER_USERNAME }}
           password: ${{ env.DOCKER_PASSWORD }}
       - name : Export image version
+        env:
+          GIT_TAG: ${{ github.ref_name }}
         run : |
           source ./scripts/version
           echo IMAGE=$IMAGE >> $GITHUB_ENV