add coverage to fontional tests

.github/workflows/ai4sim-ci-cd.yaml

@@ -59,6 +59,10 @@ jobs:
       working-directory: ${{ matrix.model-project }}
       run: nox -s train_test -v -- clean_data
 
+    - name: Run functional tests
+      working-directory: ${{ matrix.model-project }}
+      run: nox -s coverage_report -R -v -- combine
+
     - name: Coverage Commentator
       uses: 5monkeys/cobertura-action@master
       with:

reactive-flows/cnf-combustion/gnns/.coveragerc

@@ -1,2 +1,3 @@
 [run]
+include = ./*.py
 omit = noxfile.py,tests/*

reactive-flows/cnf-combustion/gnns/.gitignore

@@ -1,5 +1,32 @@
-data/*
-experiments/
-_/
-*.ipynb_checkpoints
-**/__pycache__
+# Byte-compiled / optimized / DLL files
+__pycache__/
+
+# Build tool
+.nox/
+.ci-reports/
+
+# Unit test / coverage reports
+.pytest_cache/
+.coverage
+.coverage-*
+.coverage.*
+coverage.xml
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+.vscode
+
+# Data
+data/
+
+# Experiments
+experiments/

reactive-flows/cnf-combustion/gnns/ci/run.sh

@@ -11,7 +11,23 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-python3 trainer.py --config ./ci/configs/gat_test.yaml || exit 1
-python3 trainer.py --config ./ci/configs/gcn_test.yaml || exit 2
-python3 trainer.py --config ./ci/configs/gin_test.yaml || exit 3
-python3 trainer.py --config ./ci/configs/gunet_test.yaml  || exit 4
+RUNNER="python3"
+# Parse script options
+while [[ $# -gt 0 ]]; do
+  case $1 in
+    -r|--runner)
+      RUNNER="$2"
+      shift # past argument
+      shift # past value
+      ;;
+    -*|--*)
+      echo "Unknown option $1"
+      exit 1
+      ;;
+  esac
+done
+
+${RUNNER} trainer.py --config ./ci/configs/gat_test.yaml || exit 1
+${RUNNER} trainer.py --config ./ci/configs/gcn_test.yaml || exit 2
+${RUNNER} trainer.py --config ./ci/configs/gin_test.yaml || exit 3
+${RUNNER} trainer.py --config ./ci/configs/gunet_test.yaml  || exit 4

reactive-flows/cnf-combustion/unets/.coveragerc

@@ -1,2 +1,3 @@
 [run]
+include = ./*.py
 omit = noxfile.py,tests/*

reactive-flows/cnf-combustion/unets/.gitignore

@@ -8,6 +8,7 @@ __pycache__/
 # Unit test / coverage reports
 .pytest_cache/
 .coverage
+.coverage-*
 .coverage.*
 coverage.xml
 

reactive-flows/cnf-combustion/unets/ci/run.sh

@@ -11,4 +11,20 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-python3 trainer.py --config ./ci/configs/unets_test.yaml || exit 1
+RUNNER="python3"
+# Parse script options
+while [[ $# -gt 0 ]]; do
+  case $1 in
+    -r|--runner)
+      RUNNER="$2"
+      shift # past argument
+      shift # past value
+      ;;
+    -*|--*)
+      echo "Unknown option $1"
+      exit 1
+      ;;
+  esac
+done
+
+${RUNNER} trainer.py --config ./ci/configs/unets_test.yaml || exit 1

tools/nox/nox_ref_file.py

@@ -20,9 +20,12 @@
 import os
 import sys
 import re
+import glob
 import nox
 
 REPORTS_DIR = ".ci-reports/"
+COV_FT_FILE = ".coverage-ft"
+COV_UT_FILE = ".coverage-ut"
 ROOT_PATH = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 FLAKE8_CFG = os.path.join(ROOT_PATH, 'tools', 'flake8', 'flake8.cfg')
 
@@ -122,15 +125,31 @@ def tests(session):
     # Install use-case python dependencies
     dev_dependencies(session)
     session.run("python3", "-m", "pip", "install", "pytest-cov")
+    session.run('rm','-rf', COV_UT_FILE, external=True)
     session.run("python3", "-m", "pytest", "--cache-clear", "--cov=./", "-v")
-    session.notify("coverage_report")
+    session.run('mv','.coverage', COV_UT_FILE, external=True)
+    session.notify("coverage_report", ['data-file', f'{COV_UT_FILE}'])
+
+
+def coverage_install(session):
+    session.run("python3", "-m", "pip", "install", "coverage")
 
 
 @nox.session
 def coverage_report(session):
     """Target to generate coverage report from test results."""
-    session.run("python3", "-m", "pip", "install", "coverage")
-    session.run("coverage", "xml", "-o", f"{REPORTS_DIR}/pycoverage.xml")
+    coverage_install(session)
+
+    cov_file=".coverage"
+    if "combine" in session.posargs:
+        session.run("coverage", "combine", "--keep", "--append", *glob.glob(".coverage-*"))
+    elif "data-file" in session.posargs:
+        # set the coverage input coverage datafile name, if provided in the command line with the "data-file" key word
+        # ex: nox -s coverage_report -- data-file .coverage-out
+        cov_file = session.posargs[session.posargs.index("data-file") + 1]
+
+    session.run("coverage", "xml", f"--data-file={cov_file}", "-o", f"{REPORTS_DIR}/py{cov_file.lstrip('.')}.xml")
+    session.run("coverage", "report", "-m", f"--data-file={cov_file}")
 
 
 @nox.session
@@ -204,7 +223,10 @@ def train_test(session):
     # Generate the synthetic dataset required for the functional tests
     generate_synthetic_data(session)
 
-    # Run te functional tests
-    session.run('bash','./ci/run.sh')
+    # Run te functional tests with coverage
+    coverage_install(session)
+    session.run('rm','-rf', COV_FT_FILE, external=True)
+    session.run('bash', './ci/run.sh', '--runner', f'coverage run --append --data-file={COV_FT_FILE}', external=True)
     if "clean_data" in session.posargs:
         session.run('rm','-rf', './data')
+    session.notify("coverage_report", ['data-file', f'{COV_FT_FILE}'])

weather-forecast/ecrad-3d-correction/unets/.coveragerc

@@ -1,2 +1,3 @@
 [run]
+include = ./*.py
 omit = noxfile.py,tests/*

weather-forecast/ecrad-3d-correction/unets/.gitignore

@@ -8,6 +8,7 @@ __pycache__/
 # Unit test / coverage reports
 .pytest_cache/
 .coverage
+.coverage-*
 .coverage.*
 coverage.xml
 

weather-forecast/ecrad-3d-correction/unets/ci/run.sh

@@ -11,4 +11,20 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-python3 trainer.py --config ./ci/configs/unet_test.yaml || exit 1
+RUNNER="python3"
+# Parse script options
+while [[ $# -gt 0 ]]; do
+  case $1 in
+    -r|--runner)
+      RUNNER="$2"
+      shift # past argument
+      shift # past value
+      ;;
+    -*|--*)
+      echo "Unknown option $1"
+      exit 1
+      ;;
+  esac
+done
+
+${RUNNER} trainer.py --config ./ci/configs/unet_test.yaml || exit 1

weather-forecast/gravity-wave-drag/cnns/.coveragerc

@@ -1,2 +1,3 @@
 [run]
+include = ./*.py
 omit = noxfile.py,tests/*

weather-forecast/gravity-wave-drag/cnns/.gitignore

@@ -8,6 +8,7 @@ __pycache__/
 # Unit test / coverage reports
 .pytest_cache/
 .coverage
+.coverage-*
 .coverage.*
 coverage.xml
 

weather-forecast/gravity-wave-drag/cnns/ci/run.sh

@@ -11,5 +11,21 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-python3 trainer.py --config ./ci/configs/cnn_test.yaml || exit 1
-python3 trainer.py --config ./ci/configs/mlp_test.yaml || exit 2
+RUNNER="python3"
+# Parse script options
+while [[ $# -gt 0 ]]; do
+  case $1 in
+    -r|--runner)
+      RUNNER="$2"
+      shift # past argument
+      shift # past value
+      ;;
+    -*|--*)
+      echo "Unknown option $1"
+      exit 1
+      ;;
+  esac
+done
+
+${RUNNER} trainer.py --config ./ci/configs/cnn_test.yaml || exit 1
+${RUNNER} trainer.py --config ./ci/configs/mlp_test.yaml || exit 2