From 03762c9bf9a687acce08d9d66198a86fecfc897a Mon Sep 17 00:00:00 2001 From: EmilyBourne Date: Mon, 11 Mar 2024 11:46:33 +0100 Subject: [PATCH] Add cuda workflow to test cuda developments on CI --- .github/actions/coverage_install/action.yml | 2 +- .github/actions/linux_install/action.yml | 10 +-- .github/actions/pytest_run/action.yml | 4 +- .github/actions/pytest_run_cuda/action.yml | 17 +++++ .github/actions/python_install/action.yml | 17 +++++ .github/workflows/cuda.yml | 85 +++++++++++++++++++++ ci_tools/bot_messages/show_tests.txt | 1 + ci_tools/bot_tools/bot_funcs.py | 12 +-- ci_tools/devel_branch_tests.py | 1 + ci_tools/json_pytest_output.py | 2 +- 10 files changed, 137 insertions(+), 14 deletions(-) create mode 100644 .github/actions/pytest_run_cuda/action.yml create mode 100644 .github/actions/python_install/action.yml create mode 100644 .github/workflows/cuda.yml diff --git a/.github/actions/coverage_install/action.yml b/.github/actions/coverage_install/action.yml index ac5294e542..5732baee34 100644 --- a/.github/actions/coverage_install/action.yml +++ b/.github/actions/coverage_install/action.yml @@ -15,7 +15,7 @@ runs: - name: Directory Creation run: | INSTALL_DIR=$(cd tests; python -c "import pyccel; print(pyccel.__path__[0])") - SITE_DIR=$(python -c 'import sysconfig; print(sysconfig.get_paths()["purelib"])') + SITE_DIR=$(dirname ${INSTALL_DIR}) echo -e "import coverage; coverage.process_startup()" > ${SITE_DIR}/pyccel_cov.pth echo -e "[run]\nparallel = True\nsource = ${INSTALL_DIR}\ndata_file = $(pwd)/.coverage\n[report]\ninclude = ${INSTALL_DIR}/*\n[xml]\noutput = cobertura.xml" > .coveragerc echo "SITE_DIR=${SITE_DIR}" >> $GITHUB_ENV diff --git a/.github/actions/linux_install/action.yml b/.github/actions/linux_install/action.yml index 8fb5cd8505..0ef9a69b8e 100644 --- a/.github/actions/linux_install/action.yml +++ b/.github/actions/linux_install/action.yml @@ -9,22 +9,22 @@ runs: shell: bash - name: Install fortran run: - sudo apt-get install gfortran + sudo apt-get install -y gfortran shell: bash - name: Install LaPack run: - sudo apt-get install libblas-dev liblapack-dev + sudo apt-get install -y libblas-dev liblapack-dev shell: bash - name: Install MPI run: | - sudo apt-get install libopenmpi-dev openmpi-bin + sudo apt-get install -y libopenmpi-dev openmpi-bin echo "MPI_OPTS=--oversubscribe" >> $GITHUB_ENV shell: bash - name: Install OpenMP run: - sudo apt-get install libomp-dev libomp5 + sudo apt-get install -y libomp-dev libomp5 shell: bash - name: Install Valgrind run: - sudo apt-get install valgrind + sudo apt-get install -y valgrind shell: bash diff --git a/.github/actions/pytest_run/action.yml b/.github/actions/pytest_run/action.yml index 0b6f0f988d..b0bdc31f16 100644 --- a/.github/actions/pytest_run/action.yml +++ b/.github/actions/pytest_run/action.yml @@ -51,13 +51,13 @@ runs: working-directory: ./tests id: pytest_3 - name: Test Fortran translations - run: python -m pytest -n auto -rX ${FLAGS} -m "not (parallel or xdist_incompatible) and not (c or python) ${{ inputs.pytest_mark }}" --ignore=ndarrays 2>&1 | tee s4_outfile.out + run: python -m pytest -n auto -rX ${FLAGS} -m "not (parallel or xdist_incompatible) and not (c or python or ccuda) ${{ inputs.pytest_mark }}" --ignore=ndarrays 2>&1 | tee s4_outfile.out shell: ${{ inputs.shell_cmd }} working-directory: ./tests id: pytest_4 - name: Test multi-file Fortran translations run: | - python -m pytest -rX ${FLAGS} -m "xdist_incompatible and not parallel and not (c or python) ${{ inputs.pytest_mark }}" --ignore=ndarrays 2>&1 | tee s5_outfile.out + python -m pytest -rX ${FLAGS} -m "xdist_incompatible and not parallel and not (c or python or ccuda) ${{ inputs.pytest_mark }}" --ignore=ndarrays 2>&1 | tee s5_outfile.out pyccel-clean shell: ${{ inputs.shell_cmd }} working-directory: ./tests diff --git a/.github/actions/pytest_run_cuda/action.yml b/.github/actions/pytest_run_cuda/action.yml new file mode 100644 index 0000000000..52092a6e02 --- /dev/null +++ b/.github/actions/pytest_run_cuda/action.yml @@ -0,0 +1,17 @@ +name: 'Pyccel pytest commands generating Ccuda' +inputs: + shell_cmd: + description: 'Specifies the shell command (different for anaconda)' + required: false + default: "bash" + +runs: + using: "composite" + steps: + - name: Ccuda tests with pytest + run: | + # Catch exit 5 (no tests found) + sh -c 'python -m pytest -n auto -rx -m "not (parallel or xdist_incompatible) and ccuda" --ignore=symbolic --ignore=ndarrays; ret=$?; [ $ret = 5 ] && exit 0 || exit $ret' + pyccel-clean + shell: ${{ inputs.shell_cmd }} + working-directory: ./tests diff --git a/.github/actions/python_install/action.yml b/.github/actions/python_install/action.yml new file mode 100644 index 0000000000..f9b720e3e1 --- /dev/null +++ b/.github/actions/python_install/action.yml @@ -0,0 +1,17 @@ +name: 'Python installation commands' + +runs: + using: "composite" + steps: + - name: Install python + run: + sudo apt-get -y install python3-dev + shell: bash + - name: python as python3 + run: + sudo apt-get -y install python-is-python3 + shell: bash + - name: Install Pip + run: + sudo apt-get -y install python3-pip + shell: bash diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml new file mode 100644 index 0000000000..a2e21c2f55 --- /dev/null +++ b/.github/workflows/cuda.yml @@ -0,0 +1,85 @@ +name: Cuda unit tests + +on: + workflow_dispatch: + inputs: + python_version: + required: false + type: string + ref: + required: false + type: string + check_run_id: + required: false + type: string + pr_repo: + required: false + type: string + push: + branches: [devel, main] + +env: + COMMIT: ${{ inputs.ref || github.event.ref }} + PEM: ${{ secrets.BOT_PEM }} + GITHUB_RUN_ID: ${{ github.run_id }} + GITHUB_CHECK_RUN_ID: ${{ inputs.check_run_id }} + PR_REPO: ${{ inputs.pr_repo || github.repository }} + +jobs: + Cuda: + + runs-on: ubuntu-20.04 + name: Unit tests + + container: nvidia/cuda:11.7.1-devel-ubuntu20.04 + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ env.COMMIT }} + repository: ${{ env.PR_REPO }} + submodules: true + - name: Prepare docker + run: | + apt update && apt install sudo + TZ=Europe/France + ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone + DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends tzdata + shell: bash + - name: Install python (setup-python action doesn't work with containers) + uses: ./.github/actions/python_install + - name: "Setup" + id: token + run: | + pip install jwt requests + python ci_tools/setup_check_run.py cuda + - name: CUDA Version + run: nvcc --version # cuda install check + - name: Install dependencies + uses: ./.github/actions/linux_install + - name: Install Pyccel with tests + run: | + PATH=${PATH}:$HOME/.local/bin + echo "PATH=${PATH}" >> $GITHUB_ENV + python -m pip install --upgrade pip + python -m pip install --user .[test] + shell: bash + - name: Coverage install + uses: ./.github/actions/coverage_install + - name: Ccuda tests with pytest + id: cuda_pytest + uses: ./.github/actions/pytest_run_cuda + - name: Collect coverage information + continue-on-error: True + uses: ./.github/actions/coverage_collection + - name: Save code coverage report + uses: actions/upload-artifact@v3 + with: + name: coverage-artifact + path: .coverage + retention-days: 1 + include-hidden-files: true + - name: "Post completed" + if: always() + run: + python ci_tools/complete_check_run.py ${{ steps.cuda_pytest.outcome }} + diff --git a/ci_tools/bot_messages/show_tests.txt b/ci_tools/bot_messages/show_tests.txt index adc07e8431..eb15492d2e 100644 --- a/ci_tools/bot_messages/show_tests.txt +++ b/ci_tools/bot_messages/show_tests.txt @@ -2,6 +2,7 @@ The following is a list of keywords which can be used to run tests. Tests in bol - **linux** : Runs the unit tests on a Linux system. - **windows** : Runs the unit tests on a Windows system. - **macosx** : Runs the unit tests on a MacOS X system. +- **cuda** : Runs the cuda unit tests on a Linux system. - **coverage** : Runs the unit tests on a Linux system and checks the coverage of the tests. - **docs** : Checks if the documentation follows the numpydoc format. - **pylint** : Runs pylint on files which are too big to be handled by codacy. diff --git a/ci_tools/bot_tools/bot_funcs.py b/ci_tools/bot_tools/bot_funcs.py index f10f19bd9a..a86364ca8e 100644 --- a/ci_tools/bot_tools/bot_funcs.py +++ b/ci_tools/bot_tools/bot_funcs.py @@ -23,7 +23,8 @@ 'pyccel_lint': '3.8', 'pylint': '3.8', 'spelling': '3.8', - 'windows': '3.8' + 'windows': '3.8', + 'cuda': '-' } test_names = { @@ -40,15 +41,16 @@ 'pyccel_lint': "Pyccel best practices", 'pylint': "Python linting", 'spelling': "Spelling verification", - 'windows': "Unit tests on Windows" + 'windows': "Unit tests on Windows", + 'cuda': "Unit tests on Linux with cuda" } -test_dependencies = {'coverage':['linux']} +test_dependencies = {'coverage':['linux', 'cuda']} tests_with_base = ('coverage', 'docs', 'pyccel_lint', 'pylint') pr_test_keys = ('linux', 'windows', 'macosx', 'coverage', 'docs', 'pylint', - 'pyccel_lint', 'spelling') + 'pyccel_lint', 'spelling', 'cuda') review_stage_labels = ["needs_initial_review", "Ready_for_review", "Ready_to_merge"] @@ -430,7 +432,7 @@ def is_test_required(self, commit_log, name, key, state): True if the test should be run, False otherwise. """ print("Checking : ", name, key) - if key in ('linux', 'windows', 'macosx', 'anaconda_linux', 'anaconda_windows', 'intel'): + if key in ('linux', 'windows', 'macosx', 'anaconda_linux', 'anaconda_windows', 'intel', 'cuda'): has_relevant_change = lambda diff: any((f.startswith('pyccel/') or f.startswith('tests/')) #pylint: disable=unnecessary-lambda-assignment and f.endswith('.py') and f != 'pyccel/version.py' for f in diff) diff --git a/ci_tools/devel_branch_tests.py b/ci_tools/devel_branch_tests.py index 1102ef9e92..ec67b6c49a 100644 --- a/ci_tools/devel_branch_tests.py +++ b/ci_tools/devel_branch_tests.py @@ -15,3 +15,4 @@ bot.run_tests(['anaconda_linux'], '3.10', force_run = True) bot.run_tests(['anaconda_windows'], '3.10', force_run = True) bot.run_tests(['intel'], '3.9', force_run = True) + bot.run_tests(['cuda'], '-', force_run = True) diff --git a/ci_tools/json_pytest_output.py b/ci_tools/json_pytest_output.py index 409ae76d72..b84f4a4c09 100644 --- a/ci_tools/json_pytest_output.py +++ b/ci_tools/json_pytest_output.py @@ -61,7 +61,7 @@ def mini_md_summary(title, outcome, failed_tests): summary = "" failed_pattern = re.compile(r".*FAILED.*") - languages = ('c', 'fortran', 'python') + languages = ('c', 'fortran', 'python', 'cuda') pattern = {lang: re.compile(r".*\["+lang+r"\]\ \_.*") for lang in languages} for i in p_args.tests: