From e7d0ae7430a4583690d1a40c24ca9416c946e254 Mon Sep 17 00:00:00 2001
From: nqb <nquiniou@akamai.com>
Date: Wed, 15 Mar 2023 14:37:48 +0100
Subject: [PATCH] don't use after_script on test jobs to handle correctly
 timeouts

workaround for https://gitlab.com/gitlab-org/gitlab-runner/-/issues/2716
see GitLab tip to handle error code https://docs.gitlab.com/ee/ci/yaml/script.html#ignore-non-zero-exit-codes

if test job failed, clean-test should exit with proper code
---
 .gitlab-ci.yml                        | 19 +++++++++++--------
 ci/lib/test/clean-test-environment.sh |  9 +++++----
 2 files changed, 16 insertions(+), 12 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 50e9070a4195..6d4ee766ef91 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -26,7 +26,6 @@ stages:
 variables:
   # synced with Pipeline timeout in GitLab UI
   PIPELINE_TIMEOUT_SCRIPT: 160m
-  PIPELINE_TIMEOUT_AFTER_SCRIPT: 10m
   BUILD_PFAPPSERVER_VUE: "yes"
   PFBUILD_CENTOS_8_IMG: ghcr.io/inverse-inc/packetfence/pfbuild-centos-8
   PFBUILD_DEB_BULLSEYE_IMG: ghcr.io/inverse-inc/packetfence/pfbuild-debian-bullseye
@@ -349,8 +348,9 @@ variables:
     BOX_DESC: ${CI_PIPELINE_URL}
   script:
     - timeout ${PIPELINE_TIMEOUT_SCRIPT} make -e -C ${VAGRANT_IMG_DIR} ${BOX_NAME}
+  # implicit timeout = 5 minutes, see https://gitlab.com/gitlab-org/gitlab-runner/-/issues/2716
   after_script:
-    - timeout ${PIPELINE_TIMEOUT_AFTER_SCRIPT} make -e -C ${VAGRANT_IMG_DIR} clean
+    - make -e -C ${VAGRANT_IMG_DIR} clean
   tags:
     - shell
     - inverse.ca
@@ -518,8 +518,9 @@ variables:
   environment:
     name: sourceforge
     url: ${SF_ZEN_REPO_URL}/${CI_COMMIT_REF_NAME}
+  # implicit timeout = 5 minutes, see https://gitlab.com/gitlab-org/gitlab-runner/-/issues/2716
   after_script:
-    - timeout ${PIPELINE_TIMEOUT_AFTER_SCRIPT} make -e -C ${ZENDIR} clean_all
+    - make -e -C ${ZENDIR} clean_all
   dependencies: []
   tags:
     - shell
@@ -528,8 +529,9 @@ variables:
   stage: build_pf_img
   environment:
     name: sourceforge
+  # implicit timeout = 5 minutes, see https://gitlab.com/gitlab-org/gitlab-runner/-/issues/2716
   after_script:
-    - timeout ${PIPELINE_TIMEOUT_AFTER_SCRIPT} make -e -C ${ISODIR} clean
+    - make -e -C ${ISODIR} clean
   dependencies: []
   tags:
     - shell
@@ -541,8 +543,9 @@ variables:
     BOX_DESC: ${CI_PIPELINE_URL}
   script:
     - timeout ${PIPELINE_TIMEOUT_SCRIPT} make -e -C ${VAGRANT_IMG_DIR} ${BOX_NAME}
+  # implicit timeout = 5 minutes, see https://gitlab.com/gitlab-org/gitlab-runner/-/issues/2716
   after_script:
-    - timeout ${PIPELINE_TIMEOUT_AFTER_SCRIPT} make -e -C ${VAGRANT_IMG_DIR} clean
+    - make -e -C ${VAGRANT_IMG_DIR} clean
   dependencies: []
   tags:
     - inverse.ca
@@ -560,9 +563,9 @@ variables:
   variables:
     VAGRANT_COMMON_DOTFILE_PATH: /var/local/gitlab-runner/vagrant/vagrant-common-${CI_COMMIT_REF_SLUG}
   script:
-    - timeout ${PIPELINE_TIMEOUT_SCRIPT} make -e -C ${TESTDIR} MAKE_TARGET=run ${CI_JOB_NAME}
-  after_script:
-    - timeout ${PIPELINE_TIMEOUT_AFTER_SCRIPT} ${TESTCIDIR}/clean-test-environment.sh
+    # || EXIT_CODE=$? prevents script to finish at first command
+    - timeout ${PIPELINE_TIMEOUT_SCRIPT} make -e -C ${TESTDIR} MAKE_TARGET=run ${CI_JOB_NAME} || EXIT_CODE=$?
+    - JOB_STATUS=$EXIT_CODE ${TESTCIDIR}/clean-test-environment.sh
 
 ################################################################################
 # JOBS
diff --git a/ci/lib/test/clean-test-environment.sh b/ci/lib/test/clean-test-environment.sh
index 1803b7c6f8f1..3b8ca210bd6c 100755
--- a/ci/lib/test/clean-test-environment.sh
+++ b/ci/lib/test/clean-test-environment.sh
@@ -17,13 +17,13 @@ source ${FUNCTIONS_FILE}
 
 
 configure_and_check() {
-    CI_JOB_STATUS=${CI_JOB_STATUS:-}
+    JOB_STATUS=${JOB_STATUS:-}
     CI_JOB_NAME=${CI_JOB_NAME:-}
     KEEP_VMS=${KEEP_VMS:-no}
 
-    if [ "$CI_JOB_STATUS" = "success" ]; then
+    if [ "$JOB_STATUS" -eq 0 ]; then
         echo "Passed tests"
-        if [ "KEEP_VMS" = "yes" ]; then
+        if [ "$KEEP_VMS" = "yes" ]; then
             echo "Keeping VM according to 'KEEP_VMS' value"
             MAKE_TARGET=halt make -e -C ${TEST_DIR} ${CI_JOB_NAME}
         else
@@ -34,9 +34,10 @@ configure_and_check() {
         echo 'Failed tests: cancelling jobs not started and halting VM'
         ${PF_SRC_DIR}/ci/lib/test/cancel-created-jobs.sh
         MAKE_TARGET=halt make -e -C ${TEST_DIR} ${CI_JOB_NAME}
+        exit $JOB_STATUS
     fi
 
-    declare -p CI_JOB_STATUS CI_JOB_NAME
+    declare -p JOB_STATUS CI_JOB_NAME
     declare -p TEST_DIR
 }