Merge branch 'main' into name-conflict-error

.coveragerc

@@ -1,6 +1,7 @@
 [run]
 branch = True
 source = cwltool
+omit = cwltool/run_job.py
 
 [report]
 exclude_lines =

.dockerignore

@@ -1,6 +1,28 @@
+.coverage
+coverage.xml
 .tox/
 .eggs/
 .vscode/
 build/
 dist/
+.swp
+.mypy_cache/
+.git/
+.pytest_cache/
+*.whl
+env*/
+testenv*/
+*.img
+*.sif
+*.so
+.github/
+cwltool/*.so
 *.Dockerfile
+build-cwltool-docker.sh
+__pycache__/
+*/__pycache__/
+**/__pycache__/
+*.egg-info/
+*.orig
+.dockerignore
+cache*

.github/dependabot.yml

@@ -0,0 +1,16 @@
+# To get started with Dependabot version updates, you'll need to specify which
+# package ecosystems to update and where the package manifests are located.
+# Please see the documentation for all configuration options:
+# https://help.github.com/github/administering-a-repository/configuration-options-for-dependency-updates
+
+version: 2
+updates:
+  - package-ecosystem: "pip" # See documentation for possible values
+    directory: "/" # Location of package manifests
+    schedule:
+      interval: "daily"
+  # Maintain dependencies for GitHub Actions
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "daily"

.github/workflows/ci-tests.yml

@@ -0,0 +1,189 @@
+name: Continous integration tests
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+  workflow_dispatch:
+
+env:
+  singularity_version: 3.6.4
+
+jobs:
+
+  tox:
+    name: CI tests via Tox
+
+    runs-on: ubuntu-20.04
+
+    strategy:
+      matrix:
+        py-ver-major: [3]
+        py-ver-minor: [6, 7, 8, 9]
+        step: [lint, unit, bandit, mypy]
+
+    env:
+      py-semver: ${{ format('{0}.{1}', matrix.py-ver-major, matrix.py-ver-minor) }}
+      TOXENV: ${{ format('py{0}{1}-{2}', matrix.py-ver-major, matrix.py-ver-minor, matrix.step) }}
+
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+
+      - name: Set up Singularity
+        if: ${{ matrix.step == 'unit' || matrix.step == 'mypy' }}
+        uses: eWaterCycle/setup-singularity@v6
+        with:
+          singularity-version: ${{ env.singularity_version }}
+
+      - name: Give the test runner user a name to make provenance happy.
+        if: ${{ matrix.step == 'unit' || matrix.step == 'mypy' }}
+        run: sudo usermod -c 'CI Runner' $(whoami)
+
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ env.py-semver }}
+
+      - name: Cache for pip
+        uses: actions/cache@v2
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-${{ matrix.step }}-${{ hashFiles('requirements.txt', 'tox.ini') }}
+
+      - name: Upgrade setuptools and install tox
+        run: |
+          pip install -U pip setuptools wheel
+          pip install tox tox-gh-actions
+
+      - name: MyPy cache
+        if: ${{ matrix.step == 'mypy' }}
+        uses: actions/cache@v2
+        with:
+          path: .mypy_cache/${{ env.py-semver }}
+          key: mypy-${{ env.py-semver }}
+
+      - name: Test with tox
+        run: tox
+
+      - name: Upload coverage to Codecov
+        if: ${{ matrix.step == 'unit' }}
+        uses: codecov/[email protected]
+        with:
+          fail_ci_if_error: true
+
+  tox-style:
+    name: CI linters via Tox
+
+    runs-on: ubuntu-20.04
+
+    strategy:
+      matrix:
+        step: [lintreadme, shellcheck, pydocstyle]
+
+    env:
+      py-semver: 3.9
+      TOXENV: ${{ format('py39-{0}', matrix.step) }}
+
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ env.py-semver }}
+
+      - name: Cache for pip
+        uses: actions/cache@v2
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-${{ matrix.step }}-${{ hashFiles('requirements.txt') }}
+
+      - name: Upgrade setuptools and install tox
+        run: |
+          pip install -U pip setuptools wheel
+          pip install tox tox-gh-actions
+
+      - if: ${{ matrix.step == 'pydocstyle' && github.event_name == 'pull_request'}}
+        name: Create local branch for diff-quality for PRs
+        run: git branch ${{github.base_ref}} origin/${{github.base_ref}}
+
+      - name: Test with tox
+        run: tox
+
+  conformance_tests:
+    name: CWL spec conformance tests
+
+    runs-on: ubuntu-20.04
+
+    strategy:
+      matrix:
+        cwl-version: [v1.0, v1.1, v1.2]
+        container: [docker, singularity]
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Set up Singularity
+        uses: eWaterCycle/setup-singularity@v6
+        with:
+          singularity-version: ${{ env.singularity_version }}
+
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.9
+
+      - name: Cache for pip
+        uses: actions/cache@v2
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-conformance-${{ matrix.step }}-${{ hashFiles('requirements.txt') }}
+
+      - name: Run CWL conformance tests ${{ matrix.cwl-version }}
+        env:
+          version: ${{ matrix.cwl-version }}
+          container: ${{ matrix.container }}
+          spec_branch: main
+        run: ./conformance-test.sh
+
+  release_test:
+    name: cwltool release test
+
+    runs-on: ubuntu-20.04
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Set up Singularity
+        uses: eWaterCycle/setup-singularity@v6
+        with:
+          singularity-version: ${{ env.singularity_version }}
+
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.9
+
+      - name: Give the test runner user a name to make provenance happy.
+        run: sudo usermod -c 'CI Runner' $(whoami)
+
+      - name: Cache for pip
+        uses: actions/cache@v2
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-release-${{ hashFiles('requirements.txt', 'test-requirements.txt') }}
+
+      - name: Install packages
+        run: |
+          pip install -U pip setuptools wheel
+          pip install virtualenv
+
+      - name: Release test
+        env:
+          RELEASE_SKIP: head
+        run: ./release-test.sh

.github/workflows/codeql-analysis.yml

@@ -2,7 +2,9 @@ name: "Code scanning - action"
 
 on:
   push:
+    branches: [main]
   pull_request:
+    branches: [main]
   schedule:
     - cron: '0 10 * * 2'
 
@@ -11,19 +13,14 @@ jobs:
 
     runs-on: ubuntu-latest
 
+    permissions:
+      # required for all workflows
+      security-events: write
+
     steps:
     - name: Checkout repository
       uses: actions/checkout@v2
-      with:
-        # We must fetch at least the immediate parents so that if this is
-        # a pull request then we can checkout the head.
-        fetch-depth: 2
 
-    # If this run was triggered by a pull request event, then checkout
-    # the head of the pull request instead of the merge commit.
-    - run: git checkout HEAD^2
-      if: ${{ github.event_name == 'pull_request' }}
-
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
       uses: github/codeql-action/init@v1

.github/workflows/quay-publish.yml

@@ -0,0 +1,40 @@
+name: publish-quay
+on:
+  push:
+    tags:
+      - '*'
+  workflow_dispatch: {}
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@master
+      - name: Get image tags
+        id: image_tags
+        run: |
+          echo -n ::set-output name=IMAGE_TAGS::${GITHUB_REF#refs/*/}
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+      - name: Login to Quay.io
+        uses: docker/login-action@v1
+        with:
+          registry: ${{ secrets.REGISTRY_SERVER }}
+          username: ${{ secrets.REGISTRY_USERNAME }}
+          password: ${{ secrets.REGISTRY_PASSWORD }}
+      - name: Build and publish cwltool_module image to Quay
+        uses: docker/build-push-action@v2
+        with:
+          file: cwltool.Dockerfile
+          tags: quay.io/commonwl/cwltool_module:${{ steps.image_tags.outputs.IMAGE_TAGS }}
+          target: module
+          push: true
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+      - name: Build and publish cwltool image to Quay
+        uses: docker/build-push-action@v2
+        with:
+          file: cwltool.Dockerfile
+          tags: quay.io/commonwl/cwltool:${{ steps.image_tags.outputs.IMAGE_TAGS }}
+          push: true
+          cache-from: type=gha
+          cache-to: type=gha,mode=max

.mergify.yml

@@ -2,18 +2,22 @@ pull_request_rules:
   - name: Automatic merge on approval and when when GitHub branch protection passes on main
     conditions:
       - "#approved-reviews-by>=1"
+      - -draft
       - base=main
     actions:
       merge:
         method: merge
         strict: smart+fasttrack
 
 pull_request_rules:
-  - name: Automatic merge for leadership team members when there are no reviewers
+  - name: Automatic merge for leadership team members when there are no reviewers and the label is "ready"
     conditions:
       - "#review-requested=0"
+      - "#changes-requested-reviews-by<1"
+      - -draft
       - base=main
       - author=@leadership
+      - label=ready
     actions:
       merge:
         method: merge

.snyk

@@ -0,0 +1,10 @@
+# Snyk (https://snyk.io) policy file, patches or ignores known vulnerabilities.
+version: v1.19.0
+# ignores vulnerabilities until expiry date; change duration by modifying expiry date
+ignore:
+  SNYK-PYTHON-NETWORKX-1062709:
+    - '*':
+        reason: Prov does not use the affected code path
+        expires: 2022-08-08T15:27:21.289Z
+        created: 2021-07-09T15:27:21.300Z
+patch: {}