From ac60dc1df0c23e54ecee99bc0d989da410851d2e Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" <1330696+mr-c@users.noreply.github.com> Date: Thu, 3 Dec 2020 19:26:17 +0100 Subject: [PATCH 001/111] Mergify: configuration update (#1385) --- .mergify.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.mergify.yml b/.mergify.yml index f2a542e6d0..85cbc7d268 100644 --- a/.mergify.yml +++ b/.mergify.yml @@ -2,6 +2,7 @@ pull_request_rules: - name: Automatic merge on approval and when when GitHub branch protection passes on main conditions: - "#approved-reviews-by>=1" + - -draft - base=main actions: merge: @@ -9,11 +10,14 @@ pull_request_rules: strict: smart+fasttrack pull_request_rules: - - name: Automatic merge for leadership team members when there are no reviewers + - name: Automatic merge for leadership team members when there are no reviewers and the label is "ready" conditions: - "#review-requested=0" + - "#changes-requested-reviews-by<1" + - -draft - base=main - author=@leadership + - label=ready actions: merge: method: merge From 5f30c8eb4ac95ebcc86b2fc72330efbf98b7ddf5 Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" Date: Mon, 4 Jan 2021 10:34:16 +0100 Subject: [PATCH 002/111] tests: replace py.path and NamedTemporaryFile with pathlib.Path --- Makefile | 2 +- cwltool/executors.py | 3 +- cwltool/task_queue.py | 1 - tests/test_docker.py | 10 +- tests/test_empty_input.py | 22 +- tests/test_examples.py | 114 +++++----- tests/test_ext.py | 250 +++++++++++----------- tests/test_http_input.py | 11 +- tests/test_iwdr.py | 354 ++++++++++++++++---------------- tests/test_pack.py | 57 +++-- tests/test_parallel.py | 7 +- tests/test_provenance.py | 196 +++++++++--------- tests/test_relax_path_checks.py | 40 ++-- tests/test_relocate.py | 18 +- tests/test_singularity.py | 12 +- tests/test_toolargparse.py | 37 ++-- tests/util.py | 14 +- 17 files changed, 563 insertions(+), 585 deletions(-) diff --git a/Makefile b/Makefile index 8f985a6293..9f3487eb07 100644 --- a/Makefile +++ b/Makefile @@ -92,7 +92,7 @@ pep257: pydocstyle pydocstyle: $(PYSOURCES) pydocstyle --add-ignore=D100,D101,D102,D103 $^ || true -pydocstyle_report.txt: $(PYSOURCES) +pydocstyle_report.txt: $(filter-out tests/%,${PYSOURCES}) pydocstyle setup.py $^ > $@ 2>&1 || true diff_pydocstyle_report: pydocstyle_report.txt diff --git a/cwltool/executors.py b/cwltool/executors.py index 53bf1ee44d..0e6c35535e 100644 --- a/cwltool/executors.py +++ b/cwltool/executors.py @@ -32,11 +32,10 @@ from .mutation import MutationManager from .process import Process, cleanIntermediate, relocateOutputs from .provenance_profile import ProvenanceProfile +from .task_queue import TaskQueue from .utils import CWLObjectType, JobsType from .workflow import Workflow from .workflow_job import WorkflowJob, WorkflowJobStep -from .task_queue import TaskQueue - TMPDIR_LOCK = Lock() diff --git a/cwltool/task_queue.py b/cwltool/task_queue.py index ba834008b9..987ce56ade 100644 --- a/cwltool/task_queue.py +++ b/cwltool/task_queue.py @@ -5,7 +5,6 @@ import queue import threading - from typing import Callable, Optional from .loghandler import _logger diff --git a/tests/test_docker.py b/tests/test_docker.py index f0399e2b94..0767bb8f06 100644 --- a/tests/test_docker.py +++ b/tests/test_docker.py @@ -1,6 +1,5 @@ from distutils import spawn - -import py.path +from pathlib import Path from cwltool.main import main @@ -8,20 +7,21 @@ @needs_docker -def test_docker_workflow(tmpdir: py.path.local) -> None: +def test_docker_workflow(tmp_path: Path) -> None: + """Basic test for docker with a CWL Workflow.""" result_code, _, stderr = get_main_output( [ "--default-container", "debian", "--outdir", - str(tmpdir), + str(tmp_path), get_data("tests/wf/hello-workflow.cwl"), "--usermessage", "hello", ] ) assert "completed success" in stderr - assert (tmpdir / "response.txt").read_text("utf-8") == "hello" + assert (tmp_path / "response.txt").read_text("utf-8") == "hello" assert result_code == 0 diff --git a/tests/test_empty_input.py b/tests/test_empty_input.py index dbcc357bd6..a29e03c26b 100644 --- a/tests/test_empty_input.py +++ b/tests/test_empty_input.py @@ -1,19 +1,25 @@ from io import StringIO +from pathlib import Path from cwltool.main import main -from .util import get_data, temp_dir, windows_needs_docker +from .util import get_data, windows_needs_docker @windows_needs_docker -def test_empty_input() -> None: +def test_empty_input(tmp_path: Path) -> None: + """Affirm that an empty input works.""" empty_json = "{}" empty_input = StringIO(empty_json) - with temp_dir() as tmpdir: - params = ["--outdir", tmpdir, get_data("tests/wf/no-parameters-echo.cwl"), "-"] + params = [ + "--outdir", + str(tmp_path), + get_data("tests/wf/no-parameters-echo.cwl"), + "-", + ] - try: - assert main(params, stdin=empty_input) == 0 - except SystemExit as err: - assert err.code == 0 + try: + assert main(params, stdin=empty_input) == 0 + except SystemExit as err: + assert err.code == 0 diff --git a/tests/test_examples.py b/tests/test_examples.py index 65a46627aa..296c07b25c 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -9,7 +9,6 @@ from typing import Any, Dict, List, Union, cast from urllib.parse import urlparse -import py.path import pydot # type: ignore import pytest from ruamel.yaml.comments import CommentedMap, CommentedSeq @@ -34,7 +33,6 @@ get_main_output, get_windows_safe_factory, needs_docker, - temp_dir, windows_needs_docker, ) @@ -1053,62 +1051,64 @@ def test_no_js_console(factor: str) -> None: @needs_docker @pytest.mark.parametrize("factor", test_factors) -def test_cid_file_dir(tmpdir: py.path.local, factor: str) -> None: +def test_cid_file_dir(tmp_path: Path, factor: str) -> None: + """Test --cidfile-dir option works.""" test_file = "cache_test_workflow.cwl" - cwd = tmpdir.chdir() + cwd = Path.cwd() + os.chdir(tmp_path) commands = factor.split() - commands.extend(["--cidfile-dir", str(tmpdir), get_data("tests/wf/" + test_file)]) + commands.extend(["--cidfile-dir", str(tmp_path), get_data("tests/wf/" + test_file)]) error_code, stdout, stderr = get_main_output(commands) assert "completed success" in stderr assert error_code == 0 - cidfiles_count = sum(1 for _ in tmpdir.visit(fil="*")) + cidfiles_count = sum(1 for _ in tmp_path.glob("**/*")) assert cidfiles_count == 2 - cwd.chdir() - tmpdir.remove(ignore_errors=True) + os.chdir(cwd) @needs_docker @pytest.mark.parametrize("factor", test_factors) -def test_cid_file_dir_arg_is_file_instead_of_dir( - tmpdir: py.path.local, factor: str -) -> None: +def test_cid_file_dir_arg_is_file_instead_of_dir(tmp_path: Path, factor: str) -> None: + """Test --cidfile-dir with a file produces the correct error.""" test_file = "cache_test_workflow.cwl" - bad_cidfile_dir = str(tmpdir.ensure("cidfile-dir-actually-a-file")) + bad_cidfile_dir = tmp_path / "cidfile-dir-actually-a-file" + bad_cidfile_dir.touch() commands = factor.split() commands.extend( - ["--cidfile-dir", bad_cidfile_dir, get_data("tests/wf/" + test_file)] + ["--cidfile-dir", str(bad_cidfile_dir), get_data("tests/wf/" + test_file)] ) error_code, _, stderr = get_main_output(commands) assert "is not a directory, please check it first" in stderr, stderr assert error_code == 2 or error_code == 1, stderr - tmpdir.remove(ignore_errors=True) @needs_docker @pytest.mark.parametrize("factor", test_factors) -def test_cid_file_non_existing_dir(tmpdir: py.path.local, factor: str) -> None: +def test_cid_file_non_existing_dir(tmp_path: Path, factor: str) -> None: + """Test that --cachedir with a bad path should produce a specific error.""" test_file = "cache_test_workflow.cwl" - bad_cidfile_dir = str(tmpdir.join("cidfile-dir-badpath")) + bad_cidfile_dir = tmp_path / "cidfile-dir-badpath" commands = factor.split() commands.extend( [ "--record-container-id", "--cidfile-dir", - bad_cidfile_dir, + str(bad_cidfile_dir), get_data("tests/wf/" + test_file), ] ) error_code, _, stderr = get_main_output(commands) assert "directory doesn't exist, please create it first" in stderr, stderr assert error_code == 2 or error_code == 1, stderr - tmpdir.remove(ignore_errors=True) @needs_docker @pytest.mark.parametrize("factor", test_factors) -def test_cid_file_w_prefix(tmpdir: py.path.local, factor: str) -> None: +def test_cid_file_w_prefix(tmp_path: Path, factor: str) -> None: + """Test that --cidfile-prefix works.""" test_file = "cache_test_workflow.cwl" - cwd = tmpdir.chdir() + cwd = Path.cwd() + os.chdir(tmp_path) try: commands = factor.split() commands.extend( @@ -1120,13 +1120,12 @@ def test_cid_file_w_prefix(tmpdir: py.path.local, factor: str) -> None: ) error_code, stdout, stderr = get_main_output(commands) finally: - listing = tmpdir.listdir() - cwd.chdir() - cidfiles_count = sum(1 for _ in tmpdir.visit(fil="pytestcid*")) - tmpdir.remove(ignore_errors=True) + listing = tmp_path.iterdir() + os.chdir(cwd) + cidfiles_count = sum(1 for _ in tmp_path.glob("**/pytestcid*")) assert "completed success" in stderr assert error_code == 0 - assert cidfiles_count == 2, "{}/n{}".format(listing, stderr) + assert cidfiles_count == 2, "{}/n{}".format(list(listing), stderr) @needs_docker @@ -1178,22 +1177,23 @@ def test_secondary_files_v1_0(factor: str) -> None: @needs_docker @pytest.mark.parametrize("factor", test_factors) -def test_wf_without_container(tmpdir: py.path.local, factor: str) -> None: +def test_wf_without_container(tmp_path: Path, factor: str) -> None: + """Confirm that we can run a workflow without a container.""" test_file = "hello-workflow.cwl" - with temp_dir("cwltool_cache") as cache_dir: - commands = factor.split() - commands.extend( - [ - "--cachedir", - cache_dir, - "--outdir", - str(tmpdir), - get_data("tests/wf/" + test_file), - "--usermessage", - "hello", - ] - ) - error_code, _, stderr = get_main_output(commands) + cache_dir = str(tmp_path / "cwltool_cache") + commands = factor.split() + commands.extend( + [ + "--cachedir", + cache_dir, + "--outdir", + str(tmp_path / "outdir"), + get_data("tests/wf/" + test_file), + "--usermessage", + "hello", + ] + ) + error_code, _, stderr = get_main_output(commands) assert "completed success" in stderr assert error_code == 0 @@ -1201,22 +1201,23 @@ def test_wf_without_container(tmpdir: py.path.local, factor: str) -> None: @needs_docker @pytest.mark.parametrize("factor", test_factors) -def test_issue_740_fixed(factor: str) -> None: +def test_issue_740_fixed(tmp_path: Path, factor: str) -> None: + """Confirm that re-running a particular workflow with caching suceeds.""" test_file = "cache_test_workflow.cwl" - with temp_dir("cwltool_cache") as cache_dir: - commands = factor.split() - commands.extend(["--cachedir", cache_dir, get_data("tests/wf/" + test_file)]) - error_code, _, stderr = get_main_output(commands) + cache_dir = str(tmp_path / "cwltool_cache") + commands = factor.split() + commands.extend(["--cachedir", cache_dir, get_data("tests/wf/" + test_file)]) + error_code, _, stderr = get_main_output(commands) - assert "completed success" in stderr - assert error_code == 0 + assert "completed success" in stderr + assert error_code == 0 - commands = factor.split() - commands.extend(["--cachedir", cache_dir, get_data("tests/wf/" + test_file)]) - error_code, _, stderr = get_main_output(commands) + commands = factor.split() + commands.extend(["--cachedir", cache_dir, get_data("tests/wf/" + test_file)]) + error_code, _, stderr = get_main_output(commands) - assert "Output of job will be cached in" not in stderr - assert error_code == 0, stderr + assert "Output of job will be cached in" not in stderr + assert error_code == 0, stderr @needs_docker @@ -1238,7 +1239,7 @@ def test_compute_checksum() -> None: @needs_docker @pytest.mark.parametrize("factor", test_factors) -def test_no_compute_chcksum(tmpdir: py.path.local, factor: str) -> None: +def test_no_compute_chcksum(tmp_path: Path, factor: str) -> None: test_file = "tests/wf/wc-tool.cwl" job_file = "tests/wf/wc-job.json" commands = factor.split() @@ -1246,7 +1247,7 @@ def test_no_compute_chcksum(tmpdir: py.path.local, factor: str) -> None: [ "--no-compute-checksum", "--outdir", - str(tmpdir), + str(tmp_path), get_data(test_file), get_data(job_file), ] @@ -1379,9 +1380,10 @@ def test_v1_0_arg_empty_prefix_separate_false() -> None: assert error_code == 0 -def test_scatter_output_filenames(tmpdir: py.path.local) -> None: +def test_scatter_output_filenames(tmp_path: Path) -> None: """If a scatter step produces identically named output then confirm that the final output is renamed correctly.""" - cwd = tmpdir.chdir() + cwd = Path.cwd() + os.chdir(tmp_path) rtc = RuntimeContext() rtc.outdir = str(cwd) factory = cwltool.factory.Factory(runtime_context=rtc) diff --git a/tests/test_ext.py b/tests/test_ext.py index 8959a252a7..235e60fc0b 100644 --- a/tests/test_ext.py +++ b/tests/test_ext.py @@ -1,14 +1,14 @@ import os import re from io import StringIO +from pathlib import Path -import py.path import pytest import cwltool.process from cwltool.main import main -from .util import get_data, needs_docker, temp_dir, windows_needs_docker +from .util import get_data, needs_docker, windows_needs_docker @needs_docker @@ -90,162 +90,168 @@ def test_listing_v1_1() -> None: @needs_docker -def test_double_overwrite(tmpdir: py.path.local) -> None: - with temp_dir() as tmp: - tmp_name = os.path.join(tmp, "value") +def test_double_overwrite(tmp_path: Path) -> None: + """Test that overwriting an input using cwltool:InplaceUpdateRequirement works.""" + tmp_name = str(tmp_path / "value") - before_value, expected_value = "1", "3" + before_value, expected_value = "1", "3" - with open(tmp_name, "w") as f: - f.write(before_value) + with open(tmp_name, "w") as f: + f.write(before_value) - assert ( - main( - [ - "--enable-ext", - "--outdir", - str(tmpdir), - get_data("tests/wf/mut2.cwl"), - "-a", - tmp_name, - ] - ) - == 0 + assert ( + main( + [ + "--enable-ext", + "--outdir", + str(tmp_path / "outdir"), + get_data("tests/wf/mut2.cwl"), + "-a", + tmp_name, + ] ) + == 0 + ) - with open(tmp_name, "r") as f: - actual_value = f.read() + with open(tmp_name, "r") as f: + actual_value = f.read() - assert actual_value == expected_value + assert actual_value == expected_value @needs_docker -def test_disable_file_overwrite_without_ext() -> None: - with temp_dir() as tmp: - with temp_dir() as out: - tmp_name = os.path.join(tmp, "value") - out_name = os.path.join(out, "value") - - before_value, expected_value = "1", "2" - - with open(tmp_name, "w") as f: - f.write(before_value) - - assert ( - main( - [ - "--outdir", - out, - get_data("tests/wf/updateval.cwl"), - "-r", - tmp_name, - ] - ) - == 0 - ) - - with open(tmp_name, "r") as f: - tmp_value = f.read() - with open(out_name, "r") as f: - out_value = f.read() - - assert tmp_value == before_value - assert out_value == expected_value +def test_disable_file_overwrite_without_ext(tmp_path: Path) -> None: + """Test that overwriting an input using an unprefixed InplaceUpdateRequirement works.""" + tmpdir = tmp_path / "tmp" + tmpdir.mkdir() + tmp_name = tmpdir / "value" + outdir = tmp_path / "out" + outdir.mkdir() + out_name = outdir / "value" + before_value, expected_value = "1", "2" + + with open(tmp_name, "w") as f: + f.write(before_value) + assert ( + main( + [ + "--outdir", + str(outdir), + get_data("tests/wf/updateval.cwl"), + "-r", + str(tmp_name), + ] + ) + == 0 + ) -@needs_docker -def test_disable_dir_overwrite_without_ext() -> None: - with temp_dir() as tmp: - with temp_dir() as out: - - assert ( - main(["--outdir", out, get_data("tests/wf/updatedir.cwl"), "-r", tmp]) - == 0 - ) + with open(tmp_name, "r") as f: + tmp_value = f.read() + with open(out_name, "r") as f: + out_value = f.read() - assert not os.listdir(tmp) - assert os.listdir(out) + assert tmp_value == before_value + assert out_value == expected_value @needs_docker -def test_disable_file_creation_in_outdir_with_ext() -> None: - with temp_dir() as tmp: - with temp_dir() as out: +def test_disable_dir_overwrite_without_ext(tmp_path: Path) -> None: + """Test that we can write into a "writable" input Directory w/o ext.""" + tmp = tmp_path / "tmp" + out = tmp_path / "outdir" + tmp.mkdir() + out.mkdir() + assert ( + main(["--outdir", str(out), get_data("tests/wf/updatedir.cwl"), "-r", str(tmp)]) + == 0 + ) + + assert not os.listdir(tmp) + assert os.listdir(out) - tmp_name = os.path.join(tmp, "value") - out_name = os.path.join(out, "value") - before_value, expected_value = "1", "2" +@needs_docker +def test_disable_file_creation_in_outdir_with_ext(tmp_path: Path) -> None: + tmp = tmp_path / "tmp" + tmp.mkdir() + out = tmp_path / "outdir" + tmp_name = tmp / "value" + out_name = out / "value" - with open(tmp_name, "w") as f: - f.write(before_value) + before_value, expected_value = "1", "2" - params = [ - "--enable-ext", - "--leave-outputs", - "--outdir", - out, - get_data("tests/wf/updateval_inplace.cwl"), - "-r", - tmp_name, - ] - assert main(params) == 0 + with open(tmp_name, "w") as f: + f.write(before_value) - with open(tmp_name, "r") as f: - tmp_value = f.read() + params = [ + "--enable-ext", + "--leave-outputs", + "--outdir", + str(out), + get_data("tests/wf/updateval_inplace.cwl"), + "-r", + str(tmp_name), + ] + assert main(params) == 0 - assert tmp_value == expected_value - assert not os.path.exists(out_name) + with open(tmp_name, "r") as f: + tmp_value = f.read() + + assert tmp_value == expected_value + assert not out_name.exists() @needs_docker -def test_disable_dir_creation_in_outdir_with_ext() -> None: - with temp_dir() as tmp: - with temp_dir() as out: - params = [ - "--enable-ext", - "--leave-outputs", - "--outdir", - out, - get_data("tests/wf/updatedir_inplace.cwl"), - "-r", - tmp, - ] - assert main(params) == 0 +def test_disable_dir_creation_in_outdir_with_ext(tmp_path: Path) -> None: + tmp = tmp_path / "tmp" + tmp.mkdir() + out = tmp_path / "outdir" + out.mkdir() + params = [ + "--enable-ext", + "--leave-outputs", + "--outdir", + str(out), + get_data("tests/wf/updatedir_inplace.cwl"), + "-r", + str(tmp), + ] + assert main(params) == 0 - assert os.listdir(tmp) - assert not os.listdir(out) + assert os.listdir(tmp) + assert not os.listdir(out) @needs_docker -def test_write_write_conflict() -> None: - with temp_dir("tmp") as tmp: - tmp_name = os.path.join(tmp, "value") +def test_write_write_conflict(tmp_path: Path) -> None: + tmp_name = tmp_path / "value" - before_value, expected_value = "1", "2" + before_value, expected_value = "1", "2" - with open(tmp_name, "w") as f: - f.write(before_value) + with open(tmp_name, "w") as f: + f.write(before_value) - assert main(["--enable-ext", get_data("tests/wf/mut.cwl"), "-a", tmp_name]) != 0 + assert ( + main(["--enable-ext", get_data("tests/wf/mut.cwl"), "-a", str(tmp_name)]) != 0 + ) - with open(tmp_name, "r") as f: - tmp_value = f.read() + with open(tmp_name, "r") as f: + tmp_value = f.read() - assert tmp_value == expected_value + assert tmp_value == expected_value @pytest.mark.skip(reason="This test is non-deterministic") -def test_read_write_conflict() -> None: - with temp_dir("tmp") as tmp: - tmp_name = os.path.join(tmp, "value") +def test_read_write_conflict(tmp_path: Path) -> None: + tmp_name = tmp_path / "value" - with open(tmp_name, "w") as f: - f.write("1") + with open(tmp_name, "w") as f: + f.write("1") - assert ( - main(["--enable-ext", get_data("tests/wf/mut3.cwl"), "-a", tmp_name]) != 0 - ) + assert ( + main(["--enable-ext", get_data("tests/wf/mut3.cwl"), "-a", str(tmp_name)]) != 0 + ) @needs_docker @@ -256,13 +262,13 @@ def test_require_prefix_networkaccess() -> None: @needs_docker -def test_require_prefix_workreuse(tmpdir: py.path.local) -> None: +def test_require_prefix_workreuse(tmp_path: Path) -> None: assert ( main( [ "--enable-ext", "--outdir", - str(tmpdir), + str(tmp_path), get_data("tests/wf/workreuse.cwl"), ] ) diff --git a/tests/test_http_input.py b/tests/test_http_input.py index 4c967f7367..ac0e3fd969 100644 --- a/tests/test_http_input.py +++ b/tests/test_http_input.py @@ -1,23 +1,22 @@ import os -import tempfile +from pathlib import Path from typing import List from cwltool.pathmapper import PathMapper from cwltool.utils import CWLObjectType -def test_http_path_mapping() -> None: +def test_http_path_mapping(tmp_path: Path) -> None: input_file_path = "https://raw.githubusercontent.com/common-workflow-language/cwltool/main/tests/2.fasta" - tempdir = tempfile.mkdtemp() - base_file = [ + base_file: List[CWLObjectType] = [ { "class": "File", "location": "https://raw.githubusercontent.com/common-workflow-language/cwltool/main/tests/2.fasta", "basename": "chr20.fa", } - ] # type: List[CWLObjectType] - pathmap = PathMapper(base_file, os.getcwd(), tempdir)._pathmap + ] + pathmap = PathMapper(base_file, os.getcwd(), str(tmp_path))._pathmap assert input_file_path in pathmap assert os.path.exists(pathmap[input_file_path].resolved) diff --git a/tests/test_iwdr.py b/tests/test_iwdr.py index 84d76fa0d6..0ba5f19010 100644 --- a/tests/test_iwdr.py +++ b/tests/test_iwdr.py @@ -1,6 +1,7 @@ import os import tempfile from pathlib import Path +from typing import Any from cwltool.main import main @@ -10,7 +11,6 @@ get_windows_safe_factory, needs_docker, needs_singularity, - temp_dir, windows_needs_docker, ) @@ -45,188 +45,188 @@ def test_directory_literal_with_real_inputs_inside(tmp_path: Path) -> None: @needs_docker -def test_iwdr_permutations() -> None: - saved_tempdir = tempfile.tempdir - with temp_dir() as misc: - tempfile.tempdir = os.path.realpath(misc) - with temp_dir() as fifth: - with temp_dir() as sixth: - with temp_dir() as seventh: - with temp_dir() as eighth: - with tempfile.NamedTemporaryFile() as first: - with tempfile.NamedTemporaryFile() as second: - with tempfile.NamedTemporaryFile() as third: - with tempfile.NamedTemporaryFile() as fourth: - with temp_dir() as outdir: - assert ( - main( - [ - "--outdir", - outdir, - "--enable-dev", - get_data( - "tests/wf/iwdr_permutations.cwl" - ), - "--first", - first.name, - "--second", - second.name, - "--third", - third.name, - "--fourth", - fourth.name, - "--fifth", - fifth, - "--sixth", - sixth, - "--seventh", - seventh, - "--eighth", - eighth, - ] - ) - == 0 - ) - tempfile.tempdir = saved_tempdir +def test_iwdr_permutations(tmp_path_factory: Any) -> None: + misc = tmp_path_factory.mktemp("misc") + fifth = str(tmp_path_factory.mktemp("fifth")) + sixth = str(tmp_path_factory.mktemp("sixth")) + seventh = str(tmp_path_factory.mktemp("seventh")) + eighth = str(tmp_path_factory.mktemp("eighth")) + first = misc / "first" + first.touch() + second = misc / "second" + second.touch() + third = misc / "third" + third.touch() + fourth = misc / "fourth" + fourth.touch() + outdir = str(tmp_path_factory.mktemp("outdir")) + assert ( + main( + [ + "--outdir", + outdir, + "--enable-dev", + get_data("tests/wf/iwdr_permutations.cwl"), + "--first", + str(first), + "--second", + str(second), + "--third", + str(third), + "--fourth", + str(fourth), + "--fifth", + fifth, + "--sixth", + sixth, + "--seventh", + seventh, + "--eighth", + eighth, + ] + ) + == 0 + ) @needs_docker -def test_iwdr_permutations_inplace() -> None: - saved_tempdir = tempfile.tempdir - with temp_dir() as misc: - tempfile.tempdir = os.path.realpath(misc) - with temp_dir() as fifth: - with temp_dir() as sixth: - with temp_dir() as seventh: - with temp_dir() as eighth: - with tempfile.NamedTemporaryFile() as first: - with tempfile.NamedTemporaryFile() as second: - with tempfile.NamedTemporaryFile() as third: - with tempfile.NamedTemporaryFile() as fourth: - with temp_dir() as outdir: - assert ( - main( - [ - "--outdir", - outdir, - "--enable-ext", - "--enable-dev", - "--overrides", - get_data( - "tests/wf/iwdr_permutations_inplace.yml" - ), - get_data( - "tests/wf/iwdr_permutations.cwl" - ), - "--first", - first.name, - "--second", - second.name, - "--third", - third.name, - "--fourth", - fourth.name, - "--fifth", - fifth, - "--sixth", - sixth, - "--seventh", - seventh, - "--eighth", - eighth, - ] - ) - == 0 - ) - tempfile.tempdir = saved_tempdir +def test_iwdr_permutations_inplace(tmp_path_factory: Any) -> None: + misc = tmp_path_factory.mktemp("misc") + fifth = str(tmp_path_factory.mktemp("fifth")) + sixth = str(tmp_path_factory.mktemp("sixth")) + seventh = str(tmp_path_factory.mktemp("seventh")) + eighth = str(tmp_path_factory.mktemp("eighth")) + first = misc / "first" + first.touch() + second = misc / "second" + second.touch() + third = misc / "third" + third.touch() + fourth = misc / "fourth" + fourth.touch() + outdir = str(tmp_path_factory.mktemp("outdir")) + assert ( + main( + [ + "--outdir", + outdir, + "--enable-ext", + "--enable-dev", + "--overrides", + get_data("tests/wf/iwdr_permutations_inplace.yml"), + get_data("tests/wf/iwdr_permutations.cwl"), + "--first", + str(first), + "--second", + str(second), + "--third", + str(third), + "--fourth", + str(fourth), + "--fifth", + fifth, + "--sixth", + sixth, + "--seventh", + seventh, + "--eighth", + eighth, + ] + ) + == 0 + ) @needs_singularity -def test_iwdr_permutations_singularity() -> None: - with temp_dir() as fifth: - with temp_dir() as sixth: - with temp_dir() as seventh: - with temp_dir() as eighth: - with tempfile.NamedTemporaryFile() as first: - with tempfile.NamedTemporaryFile() as second: - with tempfile.NamedTemporaryFile() as third: - with tempfile.NamedTemporaryFile() as fourth: - with temp_dir() as outdir: - assert ( - main( - [ - "--outdir", - outdir, - "--enable-dev", - "--singularity", - get_data( - "tests/wf/iwdr_permutations.cwl" - ), - "--first", - first.name, - "--second", - second.name, - "--third", - third.name, - "--fourth", - fourth.name, - "--fifth", - fifth, - "--sixth", - sixth, - "--seventh", - seventh, - "--eighth", - eighth, - ] - ) - == 0 - ) +def test_iwdr_permutations_singularity(tmp_path_factory: Any) -> None: + misc = tmp_path_factory.mktemp("misc") + fifth = str(tmp_path_factory.mktemp("fifth")) + sixth = str(tmp_path_factory.mktemp("sixth")) + seventh = str(tmp_path_factory.mktemp("seventh")) + eighth = str(tmp_path_factory.mktemp("eighth")) + first = misc / "first" + first.touch() + second = misc / "second" + second.touch() + third = misc / "third" + third.touch() + fourth = misc / "fourth" + fourth.touch() + outdir = str(tmp_path_factory.mktemp("outdir")) + assert ( + main( + [ + "--outdir", + outdir, + "--enable-dev", + "--singularity", + get_data("tests/wf/iwdr_permutations.cwl"), + "--first", + str(first), + "--second", + str(second), + "--third", + str(third), + "--fourth", + str(fourth), + "--fifth", + fifth, + "--sixth", + sixth, + "--seventh", + seventh, + "--eighth", + eighth, + ] + ) + == 0 + ) @needs_singularity -def test_iwdr_permutations_singularity_inplace() -> None: - with temp_dir() as fifth: - with temp_dir() as sixth: - with temp_dir() as seventh: - with temp_dir() as eighth: - with tempfile.NamedTemporaryFile() as first: - with tempfile.NamedTemporaryFile() as second: - with tempfile.NamedTemporaryFile() as third: - with tempfile.NamedTemporaryFile() as fourth: - with temp_dir() as outdir: - assert ( - main( - [ - "--outdir", - outdir, - "--singularity", - "--enable-ext", - "--enable-dev", - "--overrides", - get_data( - "tests/wf/iwdr_permutations_inplace.yml" - ), - get_data( - "tests/wf/iwdr_permutations.cwl" - ), - "--first", - first.name, - "--second", - second.name, - "--third", - third.name, - "--fourth", - fourth.name, - "--fifth", - fifth, - "--sixth", - sixth, - "--seventh", - seventh, - "--eighth", - eighth, - ] - ) - == 0 - ) +def test_iwdr_permutations_singularity_inplace(tmp_path_factory: Any) -> None: + misc = tmp_path_factory.mktemp("misc") + fifth = str(tmp_path_factory.mktemp("fifth")) + sixth = str(tmp_path_factory.mktemp("sixth")) + seventh = str(tmp_path_factory.mktemp("seventh")) + eighth = str(tmp_path_factory.mktemp("eighth")) + first = misc / "first" + first.touch() + second = misc / "second" + second.touch() + third = misc / "third" + third.touch() + fourth = misc / "fourth" + fourth.touch() + outdir = str(tmp_path_factory.mktemp("outdir")) + assert ( + main( + [ + "--outdir", + outdir, + "--singularity", + "--enable-ext", + "--enable-dev", + "--overrides", + get_data("tests/wf/iwdr_permutations_inplace.yml"), + get_data("tests/wf/iwdr_permutations.cwl"), + "--first", + str(first), + "--second", + str(second), + "--third", + str(third), + "--fourth", + str(fourth), + "--fifth", + fifth, + "--sixth", + sixth, + "--seventh", + seventh, + "--eighth", + eighth, + ] + ) + == 0 + ) diff --git a/tests/test_pack.py b/tests/test_pack.py index a32479f189..45b3340926 100644 --- a/tests/test_pack.py +++ b/tests/test_pack.py @@ -4,10 +4,9 @@ from collections.abc import Sized from functools import partial from io import StringIO -from tempfile import NamedTemporaryFile +from pathlib import Path from typing import Dict -import py.path import pytest from ruamel import yaml @@ -155,17 +154,17 @@ def test_pack_missing_cwlVersion(cwl_path: str) -> None: assert packed["cwlVersion"] == "v1.0" -def test_pack_idempotence_tool() -> None: +def test_pack_idempotence_tool(tmp_path: Path) -> None: """Ensure that pack produces exactly the same document for an already packed CommandLineTool.""" - _pack_idempotently("tests/wf/hello_single_tool.cwl") + _pack_idempotently("tests/wf/hello_single_tool.cwl", tmp_path) -def test_pack_idempotence_workflow() -> None: +def test_pack_idempotence_workflow(tmp_path: Path) -> None: """Ensure that pack produces exactly the same document for an already packed workflow.""" - _pack_idempotently("tests/wf/count-lines1-wf.cwl") + _pack_idempotently("tests/wf/count-lines1-wf.cwl", tmp_path) -def _pack_idempotently(document: str) -> None: +def _pack_idempotently(document: str, tmp_path: Path) -> None: loadingContext, workflowobj, uri = fetch_document(get_data(document)) loadingContext.do_update = False loadingContext, uri = resolve_and_validate_document( @@ -179,26 +178,24 @@ def _pack_idempotently(document: str) -> None: packed_text = print_pack(loadingContext, uri) packed = json.loads(packed_text) - tmp = NamedTemporaryFile(mode="w", delete=False) - try: - tmp.write(packed_text) - tmp.flush() - tmp.close() - - loadingContext, workflowobj, uri2 = fetch_document(tmp.name) - loadingContext.do_update = False - loadingContext, uri2 = resolve_and_validate_document( - loadingContext, workflowobj, uri2 - ) - loader2 = loadingContext.loader - assert loader2 - loader2.resolve_ref(uri2)[0] - - # generate pack output dict - packed_text = print_pack(loadingContext, uri2) - double_packed = json.loads(packed_text) - finally: - os.remove(tmp.name) + tmp_name = tmp_path / "packed.cwl" + tmp = tmp_name.open(mode="w") + tmp.write(packed_text) + tmp.flush() + tmp.close() + + loadingContext, workflowobj, uri2 = fetch_document(tmp.name) + loadingContext.do_update = False + loadingContext, uri2 = resolve_and_validate_document( + loadingContext, workflowobj, uri2 + ) + loader2 = loadingContext.loader + assert loader2 + loader2.resolve_ref(uri2)[0] + + # generate pack output dict + packed_text = print_pack(loadingContext, uri2) + double_packed = json.loads(packed_text) assert uri != uri2 assert packed == double_packed @@ -213,7 +210,7 @@ def _pack_idempotently(document: str) -> None: @needs_docker @pytest.mark.parametrize("wf_path,job_path,namespaced", cwl_to_run) def test_packed_workflow_execution( - wf_path: str, job_path: str, namespaced: bool, tmpdir: py.path.local + wf_path: str, job_path: str, namespaced: bool, tmp_path: Path ) -> None: loadingContext = LoadingContext() loadingContext.resolver = tool_resolver @@ -236,10 +233,10 @@ def test_packed_workflow_execution( normal_output = StringIO() packed_output = StringIO() - normal_params = ["--outdir", str(tmpdir), get_data(wf_path), get_data(job_path)] + normal_params = ["--outdir", str(tmp_path), get_data(wf_path), get_data(job_path)] packed_params = [ "--outdir", - str(tmpdir), + str(tmp_path), "--debug", wf_packed_path, get_data(job_path), diff --git a/tests/test_parallel.py b/tests/test_parallel.py index 2ff52c3c56..5995f6c27f 100644 --- a/tests/test_parallel.py +++ b/tests/test_parallel.py @@ -1,6 +1,5 @@ import json - -import py.path +from pathlib import Path from cwltool.context import RuntimeContext from cwltool.executors import MultithreadedJobExecutor @@ -9,11 +8,11 @@ @windows_needs_docker -def test_sequential_workflow(tmpdir: py.path.local) -> None: +def test_sequential_workflow(tmp_path: Path) -> None: test_file = "tests/wf/count-lines1-wf.cwl" executor = MultithreadedJobExecutor() runtime_context = RuntimeContext() - runtime_context.outdir = str(tmpdir) + runtime_context.outdir = str(tmp_path) runtime_context.select_resources = executor.select_resources factory = get_windows_safe_factory( executor=executor, runtime_context=runtime_context diff --git a/tests/test_provenance.py b/tests/test_provenance.py index 0f7c3287ab..97df6aa11a 100644 --- a/tests/test_provenance.py +++ b/tests/test_provenance.py @@ -3,14 +3,12 @@ import pickle import shutil import sys -import tempfile import urllib from pathlib import Path from typing import Any, Generator, cast import arcp import bagit -import py.path import pytest from rdflib import Graph, Namespace, URIRef from rdflib.namespace import DC, DCTERMS, RDF @@ -22,7 +20,7 @@ from cwltool.provenance import ResearchObject from cwltool.stdfsaccess import StdFsAccess -from .util import get_data, needs_docker, temp_dir, working_directory +from .util import get_data, needs_docker, working_directory # RDF namespaces we'll query for later ORE = Namespace("http://www.openarchives.org/ore/terms/") @@ -35,63 +33,65 @@ OA = Namespace("http://www.w3.org/ns/oa#") -@pytest.fixture -def folder(tmpdir: py.path.local) -> Generator[str, None, None]: - directory = str(tmpdir) - yield directory - - if not os.environ.get("DEBUG"): - shutil.rmtree(directory) - - -def cwltool(folder: str, *args: Any) -> None: - new_args = ["--provenance", folder] +def cwltool(tmp_path: Path, *args: Any) -> Path: + prov_folder = tmp_path / "provenance" + prov_folder.mkdir() + new_args = ["--provenance", str(prov_folder)] new_args.extend(args) # Run within a temporary directory to not pollute git checkout - with temp_dir("cwltool-run") as tmp_dir: - with working_directory(tmp_dir): - status = main(new_args) - assert status == 0, "Failed: cwltool.main(%r)" % (args) + tmp_dir = tmp_path / "cwltool-run" + tmp_dir.mkdir() + with working_directory(tmp_dir): + status = main(new_args) + assert status == 0, "Failed: cwltool.main(%r)" % (args) + return prov_folder @needs_docker -def test_hello_workflow(folder: str) -> None: - cwltool( - folder, - get_data("tests/wf/hello-workflow.cwl"), - "--usermessage", - "Hello workflow", +def test_hello_workflow(tmp_path: Path) -> None: + check_provenance( + cwltool( + tmp_path, + get_data("tests/wf/hello-workflow.cwl"), + "--usermessage", + "Hello workflow", + ) ) - check_provenance(folder) @needs_docker -def test_hello_single_tool(folder: str) -> None: - cwltool( - folder, get_data("tests/wf/hello_single_tool.cwl"), "--message", "Hello tool" +def test_hello_single_tool(tmp_path: Path) -> None: + check_provenance( + cwltool( + tmp_path, + get_data("tests/wf/hello_single_tool.cwl"), + "--message", + "Hello tool", + ), + single_tool=True, ) - check_provenance(folder, single_tool=True) @needs_docker -def test_revsort_workflow(folder: str) -> None: - cwltool( - folder, get_data("tests/wf/revsort.cwl"), get_data("tests/wf/revsort-job.json") +def test_revsort_workflow(tmp_path: Path) -> None: + folder = cwltool( + tmp_path, + get_data("tests/wf/revsort.cwl"), + get_data("tests/wf/revsort-job.json"), ) check_output_object(folder) check_provenance(folder) @needs_docker -def test_nested_workflow(folder: str) -> None: - cwltool(folder, get_data("tests/wf/nested.cwl")) - check_provenance(folder, nested=True) +def test_nested_workflow(tmp_path: Path) -> None: + check_provenance(cwltool(tmp_path, get_data("tests/wf/nested.cwl")), nested=True) @needs_docker -def test_secondary_files_implicit(folder: str, tmpdir: py.path.local) -> None: - file1 = tmpdir.join("foo1.txt") - file1idx = tmpdir.join("foo1.txt.idx") +def test_secondary_files_implicit(tmp_path: Path) -> None: + file1 = tmp_path / "foo1.txt" + file1idx = tmp_path / "foo1.txt.idx" with open(str(file1), "w", encoding="ascii") as f: f.write("foo") @@ -99,18 +99,20 @@ def test_secondary_files_implicit(folder: str, tmpdir: py.path.local) -> None: f.write("bar") # secondary will be picked up by .idx - cwltool(folder, get_data("tests/wf/sec-wf.cwl"), "--file1", str(file1)) + folder = cwltool(tmp_path, get_data("tests/wf/sec-wf.cwl"), "--file1", str(file1)) check_provenance(folder, secondary_files=True) check_secondary_files(folder) @needs_docker -def test_secondary_files_explicit(folder: str, tmpdir: py.path.local) -> None: - orig_tempdir = tempfile.tempdir - tempfile.tempdir = str(tmpdir) +def test_secondary_files_explicit(tmp_path: Path) -> None: # Deliberately do NOT have common basename or extension - file1 = tempfile.mktemp("foo") - file1idx = tempfile.mktemp("bar") + file1dir = tmp_path / "foo" + file1dir.mkdir() + file1 = file1dir / "foo" + file1idxdir = tmp_path / "bar" + file1idxdir.mkdir() + file1idx = file1idxdir / "bar" with open(file1, "w", encoding="ascii") as f: f.write("foo") @@ -121,41 +123,41 @@ def test_secondary_files_explicit(folder: str, tmpdir: py.path.local) -> None: job = { "file1": { "class": "File", - "path": file1, + "path": str(file1), "basename": "foo1.txt", "secondaryFiles": [ { "class": "File", - "path": file1idx, + "path": str(file1idx), "basename": "foo1.txt.idx", } ], } } - jobJson = tempfile.mktemp("job.json") + + jobJson = tmp_path / "job.json" with open(jobJson, "wb") as fp: j = json.dumps(job, ensure_ascii=True) fp.write(j.encode("ascii")) - cwltool(folder, get_data("tests/wf/sec-wf.cwl"), jobJson) + folder = cwltool(tmp_path, get_data("tests/wf/sec-wf.cwl"), str(jobJson)) check_provenance(folder, secondary_files=True) check_secondary_files(folder) - tempfile.tempdir = orig_tempdir @needs_docker -def test_secondary_files_output(folder: str) -> None: +def test_secondary_files_output(tmp_path: Path) -> None: # secondary will be picked up by .idx - cwltool(folder, get_data("tests/wf/sec-wf-out.cwl")) + folder = cwltool(tmp_path, get_data("tests/wf/sec-wf-out.cwl")) check_provenance(folder, secondary_files=True) # Skipped, not the same secondary files as above # self.check_secondary_files() @needs_docker -def test_directory_workflow(folder: str, tmpdir: py.path.local) -> None: - dir2 = tmpdir.join("dir2") - os.makedirs(str(dir2)) +def test_directory_workflow(tmp_path: Path) -> None: + dir2 = tmp_path / "dir2" + dir2.mkdir() sha1 = { # Expected hashes of ASCII letters (no linefeed) # as returned from: @@ -166,34 +168,35 @@ def test_directory_workflow(folder: str, tmpdir: py.path.local) -> None: } for x in "abc": # Make test files with predictable hashes - with open(str(dir2.join(x)), "w", encoding="ascii") as f: + with open(dir2 / x, "w", encoding="ascii") as f: f.write(x) - cwltool(folder, get_data("tests/wf/directory.cwl"), "--dir", str(dir2)) + folder = cwltool(tmp_path, get_data("tests/wf/directory.cwl"), "--dir", str(dir2)) check_provenance(folder, directory=True) # Output should include ls stdout of filenames a b c on each line - file_list = os.path.join( - folder, - "data", + file_list = ( + folder + / "data" + / # checksum as returned from: # echo -e "a\nb\nc" | sha1sum # 3ca69e8d6c234a469d16ac28a4a658c92267c423 - - "3c", - "3ca69e8d6c234a469d16ac28a4a658c92267c423", + "3c" + / "3ca69e8d6c234a469d16ac28a4a658c92267c423" ) - assert os.path.isfile(file_list) + assert file_list.is_file() # Input files should be captured by hash value, # even if they were inside a class: Directory for (l, l_hash) in sha1.items(): prefix = l_hash[:2] # first 2 letters - p = os.path.join(folder, "data", prefix, l_hash) - assert os.path.isfile(p), "Could not find %s as %s" % (l, p) + p = folder / "data" / prefix / l_hash + assert p.is_file(), "Could not find %s as %s" % (l, p) -def check_output_object(base_path: str) -> None: - output_obj = os.path.join(base_path, "workflow", "primary-output.json") +def check_output_object(base_path: Path) -> None: + output_obj = base_path / "workflow" / "primary-output.json" compare_checksum = "sha1$b9214658cc453331b62c2282b772a5c063dbd284" compare_location = "../data/b9/b9214658cc453331b62c2282b772a5c063dbd284" with open(output_obj) as fp: @@ -203,23 +206,22 @@ def check_output_object(base_path: str) -> None: assert f1["location"] == compare_location -def check_secondary_files(base_path: str) -> None: - foo_data = os.path.join( - base_path, - "data", +def check_secondary_files(base_path: Path) -> None: + foo_data = ( + base_path + / "data" + / # checksum as returned from: # $ echo -n foo | sha1sum # 0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33 - - "0b", - "0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33", - ) - bar_data = os.path.join( - base_path, "data", "62", "62cdb7020ff920e5aa642c3d4066950dd1f01f4d" + "0b" + / "0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33" ) - assert os.path.isfile(foo_data), "Did not capture file.txt 'foo'" - assert os.path.isfile(bar_data), "Did not capture secondary file.txt.idx 'bar" + bar_data = base_path / "data" / "62" / "62cdb7020ff920e5aa642c3d4066950dd1f01f4d" + assert foo_data.is_file(), "Did not capture file.txt 'foo'" + assert bar_data.is_file(), "Did not capture secondary file.txt.idx 'bar" - primary_job = os.path.join(base_path, "workflow", "primary-job.json") + primary_job = base_path / "workflow" / "primary-job.json" with open(primary_job) as fp: job_json = json.load(fp) # TODO: Verify secondaryFile in primary-job.json @@ -235,7 +237,7 @@ def check_secondary_files(base_path: str) -> None: def check_provenance( - base_path: str, + base_path: Path, nested: bool = False, single_tool: bool = False, directory: bool = False, @@ -253,7 +255,7 @@ def check_provenance( ) -def check_folders(base_path: str) -> None: +def check_folders(base_path: Path) -> None: required_folders = [ "data", "snapshot", @@ -263,10 +265,10 @@ def check_folders(base_path: str) -> None: ] for folder in required_folders: - assert os.path.isdir(os.path.join(base_path, folder)) + assert (base_path / folder).is_dir() -def check_bagit(base_path: str) -> None: +def check_bagit(base_path: Path) -> None: # check bagit structure required_files = [ "bagit.txt", @@ -277,10 +279,9 @@ def check_bagit(base_path: str) -> None: ] for basename in required_files: - file_path = os.path.join(base_path, basename) - assert os.path.isfile(file_path) + assert (base_path / basename).is_file() - bag = bagit.Bag(base_path) + bag = bagit.Bag(str(base_path)) assert bag.has_oxum() (only_manifest, only_fs) = bag.compare_manifests_with_fs() assert not list(only_manifest), "Some files only in manifest" @@ -292,16 +293,16 @@ def check_bagit(base_path: str) -> None: assert arcp.is_arcp_uri(bag.info.get("External-Identifier")) -def find_arcp(base_path: str) -> str: +def find_arcp(base_path: Path) -> str: # First try to find External-Identifier - bag = bagit.Bag(base_path) + bag = bagit.Bag(str(base_path)) ext_id = bag.info.get("External-Identifier") if arcp.is_arcp_uri(ext_id): - return cast(str, ext_id) + return str(ext_id) raise Exception("Can't find External-Identifier") -def _arcp2file(base_path: str, uri: str) -> str: +def _arcp2file(base_path: Path, uri: str) -> Path: parsed = arcp.parse_arcp(uri) # arcp URIs, ensure they are local to our RO assert ( @@ -310,13 +311,12 @@ def _arcp2file(base_path: str, uri: str) -> str: path = parsed.path[1:] # Strip first / # Convert to local path, in case it uses \ on Windows - lpath = str(Path(path)) - return os.path.join(base_path, lpath) + return base_path / Path(path) -def check_ro(base_path: str, nested: bool = False) -> None: - manifest_file = os.path.join(base_path, "metadata", "manifest.json") - assert os.path.isfile(manifest_file), "Can't find " + manifest_file +def check_ro(base_path: Path, nested: bool = False) -> None: + manifest_file = base_path / "metadata" / "manifest.json" + assert manifest_file.is_file(), "Can't find {}".format(manifest_file) arcp_root = find_arcp(base_path) base = urllib.parse.urljoin(arcp_root, "metadata/manifest.json") g = Graph() @@ -357,7 +357,7 @@ def check_ro(base_path: str, nested: bool = False) -> None: continue lfile = _arcp2file(base_path, aggregate) paths.append(os.path.relpath(lfile, base_path)) - assert os.path.isfile(lfile), "Can't find aggregated " + lfile + assert os.path.isfile(lfile), "Can't find aggregated {}".format(lfile) assert paths, "Didn't find any arcp aggregates" assert externals, "Didn't find any data URIs" @@ -436,14 +436,14 @@ def check_ro(base_path: str, nested: bool = False) -> None: def check_prov( - base_path: str, + base_path: Path, nested: bool = False, single_tool: bool = False, directory: bool = False, secondary_files: bool = False, ) -> None: - prov_file = os.path.join(base_path, "metadata", "provenance", "primary.cwlprov.nt") - assert os.path.isfile(prov_file), "Can't find " + prov_file + prov_file = base_path / "metadata" / "provenance" / "primary.cwlprov.nt" + assert prov_file.is_file(), "Can't find {}".format(prov_file) arcp_root = find_arcp(base_path) # Note: We don't need to include metadata/provnance in base URI # as .nt always use absolute URIs diff --git a/tests/test_relax_path_checks.py b/tests/test_relax_path_checks.py index f5e4579cf9..a605ed9c05 100644 --- a/tests/test_relax_path_checks.py +++ b/tests/test_relax_path_checks.py @@ -1,7 +1,5 @@ import os -from tempfile import NamedTemporaryFile - -import py.path +from pathlib import Path from cwltool.main import main @@ -27,26 +25,20 @@ @needs_docker -def test_spaces_in_input_files(tmpdir: py.path.local) -> None: - try: - script_file = NamedTemporaryFile(mode="w", delete=False) +def test_spaces_in_input_files(tmp_path: Path) -> None: + script_name = tmp_path / "script" + spaces = tmp_path / "test with spaces" + spaces.touch() + with script_name.open(mode="w") as script_file: script_file.write(script) - script_file.flush() - script_file.close() - - spaces = NamedTemporaryFile(prefix="test with spaces", delete=False) - spaces.close() - params = [ - "--debug", - "--outdir", - str(tmpdir), - script_file.name, - "--input", - spaces.name, - ] - assert main(params) == 1 - assert main(["--relax-path-checks"] + params) == 0 - finally: - os.remove(script_file.name) - os.remove(spaces.name) + params = [ + "--debug", + "--outdir", + str(tmp_path / "outdir"), + str(script_name), + "--input", + str(spaces), + ] + assert main(params) == 1 + assert main(["--relax-path-checks"] + params) == 0 diff --git a/tests/test_relocate.py b/tests/test_relocate.py index a2d5872e4d..1d0ffb56cb 100644 --- a/tests/test_relocate.py +++ b/tests/test_relocate.py @@ -1,9 +1,10 @@ import json import sys +from pathlib import Path from cwltool.main import main -from .util import get_data, needs_docker, temp_dir +from .util import get_data, needs_docker if sys.version_info[0] < 3: from StringIO import StringIO @@ -18,17 +19,16 @@ def test_for_910() -> None: @needs_docker -def test_for_conflict_file_names() -> None: +def test_for_conflict_file_names(tmp_path: Path) -> None: stream = StringIO() - with temp_dir() as tmp: - assert ( - main( - ["--debug", "--outdir", tmp, get_data("tests/wf/conflict.cwl")], - stdout=stream, - ) - == 0 + assert ( + main( + ["--debug", "--outdir", str(tmp_path), get_data("tests/wf/conflict.cwl")], + stdout=stream, ) + == 0 + ) out = json.loads(stream.getvalue()) assert out["b1"]["basename"] == out["b2"]["basename"] diff --git a/tests/test_singularity.py b/tests/test_singularity.py index 28fec9462f..74c0f2e309 100644 --- a/tests/test_singularity.py +++ b/tests/test_singularity.py @@ -3,8 +3,6 @@ import sys from pathlib import Path -import py.path - from cwltool.main import main from .util import ( @@ -22,7 +20,7 @@ def test_singularity_pullfolder(tmp_path: Path) -> None: workdir = tmp_path / "working_dir_new" workdir.mkdir() - os.chdir(str(workdir)) + os.chdir(workdir) pullfolder = tmp_path / "pullfolder" pullfolder.mkdir() env = os.environ.copy() @@ -44,8 +42,8 @@ def test_singularity_pullfolder(tmp_path: Path) -> None: @needs_singularity -def test_singularity_workflow(tmpdir: py.path.local) -> None: - with working_directory(str(tmpdir)): +def test_singularity_workflow(tmp_path: Path) -> None: + with working_directory(tmp_path): error_code, _, stderr = get_main_output( [ "--singularity", @@ -98,7 +96,7 @@ def test_singularity_incorrect_image_pull() -> None: def test_singularity_local(tmp_path: Path) -> None: workdir = tmp_path / "working_dir" workdir.mkdir() - os.chdir(str(workdir)) + os.chdir(workdir) result_code, stdout, stderr = get_main_output( [ "--singularity", @@ -114,7 +112,7 @@ def test_singularity_local(tmp_path: Path) -> None: def test_singularity_docker_image_id_in_tool(tmp_path: Path) -> None: workdir = tmp_path / "working_dir" workdir.mkdir() - os.chdir(str(workdir)) + os.chdir(workdir) result_code, stdout, stderr = get_main_output( [ "--singularity", diff --git a/tests/test_toolargparse.py b/tests/test_toolargparse.py index d05d5964be..feff5cafee 100644 --- a/tests/test_toolargparse.py +++ b/tests/test_toolargparse.py @@ -1,10 +1,9 @@ import argparse import os from io import StringIO -from tempfile import NamedTemporaryFile +from pathlib import Path from typing import Callable -import py.path import pytest import cwltool.executors @@ -87,37 +86,32 @@ @needs_docker @pytest.mark.parametrize("name,script_contents,params", scripts_argparse_params) def test_argparse( - name: str, script_contents: str, params: Callable[[str], str], tmpdir: py.path.local + name: str, script_contents: str, params: Callable[[str], str], tmp_path: Path ) -> None: - script = None + script_name = tmp_path / "script" try: - script = NamedTemporaryFile(mode="w", delete=False) - script.write(script_contents) - script.close() + with script_name.open(mode="w") as script: + script.write(script_contents) - my_params = ["--outdir", str(tmpdir)] + my_params = ["--outdir", str(tmp_path / "outdir")] my_params.extend(params(script.name)) assert main(my_params) == 0, name except SystemExit as err: assert err.code == 0, name - finally: - if script and script.name and os.path.exists(script.name): - os.unlink(script.name) -def test_dont_require_inputs() -> None: +def test_dont_require_inputs(tmp_path: Path) -> None: stream = StringIO() - script = None + script_name = tmp_path / "script" try: - script = NamedTemporaryFile(mode="w", delete=False) - script.write(script_a) - script.close() + with script_name.open(mode="w") as script: + script.write(script_a) assert ( main( - argsl=["--debug", script.name, "--input", script.name], + argsl=["--debug", str(script_name), "--input", str(script_name)], executor=cwltool.executors.NoopJobExecutor(), stdout=stream, ) @@ -125,7 +119,7 @@ def test_dont_require_inputs() -> None: ) assert ( main( - argsl=["--debug", script.name], + argsl=["--debug", str(script_name)], executor=cwltool.executors.NoopJobExecutor(), stdout=stream, ) @@ -133,7 +127,7 @@ def test_dont_require_inputs() -> None: ) assert ( main( - argsl=["--debug", script.name], + argsl=["--debug", str(script_name)], executor=cwltool.executors.NoopJobExecutor(), input_required=False, stdout=stream, @@ -142,10 +136,7 @@ def test_dont_require_inputs() -> None: ) except SystemExit as err: - assert err.code == 0, script.name if script else None - finally: - if script and script.name and os.path.exists(script.name): - os.unlink(script.name) + assert err.code == 0, script_name if script else None def test_argparser_with_doc() -> None: diff --git a/tests/util.py b/tests/util.py index f79fdefdc5..b3e650601d 100644 --- a/tests/util.py +++ b/tests/util.py @@ -102,22 +102,12 @@ def get_main_output( ) -@contextlib.contextmanager -def temp_dir(suffix: str = "") -> Generator[str, None, None]: - c_dir = tempfile.mkdtemp(suffix, dir=os.curdir) - try: - yield c_dir - finally: - shutil.rmtree(c_dir, ignore_errors=True) - - @contextlib.contextmanager def working_directory(path: Union[str, Path]) -> Generator[None, None, None]: """Change working directory and returns to previous on exit.""" prev_cwd = Path.cwd() - # before python 3.6 chdir doesn't support paths from pathlib - os.chdir(str(path)) + os.chdir(path) try: yield finally: - os.chdir(str(prev_cwd)) + os.chdir(prev_cwd) From 2a922ab11b2f9c21ae4955c74563543b87717b93 Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" Date: Mon, 4 Jan 2021 14:11:09 +0100 Subject: [PATCH 003/111] pin pytest version to Debian's development distribution version --- setup.py | 2 +- test-requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index b62c0ffd78..1c40d92b38 100755 --- a/setup.py +++ b/setup.py @@ -121,7 +121,7 @@ setup_requires=PYTEST_RUNNER, test_suite="tests", tests_require=[ - "pytest < 7", + "pytest == 6.0.2", "mock >= 2.0.0", "pytest-mock >= 1.10.0", "arcp >= 0.2.0", diff --git a/test-requirements.txt b/test-requirements.txt index 0a54cd8045..440fd237d3 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,4 +1,4 @@ -pytest < 7 +pytest == 6.0.2 mock >= 2.0.0 pytest-mock >= 1.10.0 pytest-cov From 5bdb3d3dd47d8d1b3a1685220b4b6ce0f94c055e Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" Date: Sun, 24 Jan 2021 10:06:49 +0100 Subject: [PATCH 004/111] upgrade acceptable rdflib version --- requirements.txt | 4 ++-- setup.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/requirements.txt b/requirements.txt index 4a6b1c4539..f57195b609 100755 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,8 @@ requests>=2.4.3 ruamel.yaml>=0.12.4,<=0.16.5 -rdflib>=4.2.2,<4.3 +rdflib>=4.2.2,<5.1 shellescape>=3.4.1,<3.5 -schema-salad>=7,<8 +schema-salad>=7.0.20210124093443,<8 prov==1.5.1 bagit==1.6.4 mypy-extensions diff --git a/setup.py b/setup.py index 1c40d92b38..ba64e06164 100755 --- a/setup.py +++ b/setup.py @@ -95,9 +95,9 @@ "requests >= 2.6.1", # >= 2.6.1 to workaround # https://github.com/ionrock/cachecontrol/issues/137 "ruamel.yaml >= 0.12.4, <= 0.16.5", - "rdflib >= 4.2.2, < 4.3.0", + "rdflib >= 4.2.2, < 5.1.0", "shellescape >= 3.4.1, < 3.5", - "schema-salad >= 7, < 8", + "schema-salad >= 7.0.20210124093443, < 8", "mypy-extensions", "psutil", "prov == 1.5.1", From dd997396b336565ce2a48159708612b9377d08da Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" <1330696+mr-c@users.noreply.github.com> Date: Fri, 29 Jan 2021 16:13:25 +0100 Subject: [PATCH 005/111] Update installation instructions --- README.rst | 72 ++++++++++++++++++++++++++++++++++-------------------- 1 file changed, 46 insertions(+), 26 deletions(-) diff --git a/README.rst b/README.rst index abd0397580..7fb5342568 100644 --- a/README.rst +++ b/README.rst @@ -21,7 +21,7 @@ intended to be feature complete and provide comprehensive validation of CWL files as well as provide other tools related to working with CWL. This is written and tested for -`Python `_ ``3.x {x = 6, 7, 8}`` +`Python `_ ``3.x {x = 6, 7, 8, 9}`` The reference implementation consists of two packages. The ``cwltool`` package is the primary Python module containing the reference implementation in the @@ -31,61 +31,81 @@ The ``cwlref-runner`` package is optional and provides an additional entry point under the alias ``cwl-runner``, which is the implementation-agnostic name for the default CWL interpreter installed on a host. -``cwltool`` is provided by the CWL project, `a member project of Software Freedom Conservancy `_ and our `many contributors `_. +``cwltool`` is provided by the CWL project, `a member project of Software Freedom Conservancy `_ +and our `many contributors `_. Install ------- -Your operating system may offer cwltool directly. For `Debian `_ or `Ubuntu `_ try +``cwltool`` packages +^^^^^^^^^^^^^^^^^^^^ + +Your operating system may offer cwltool directly. For `Debian `_, `Ubuntu `_, +and similar Linux distribution try .. code:: bash - apt-get install cwltool + sudo apt-get install cwltool -For MacOS X, other UNIXes or Windows packages prepared by the conda-forge project. Please follow instructions of conda-forge (https://conda-forge.org/#about) for its installation, then perform: +If you are running MacOS X or other UNIXes and you want to use packages prepared by the conda-forge project, then +please follow the install instructions for `conda-forge `_ (if you haven't already) and then .. code:: bash conda install -c conda-forge cwltool - -Under the hood, conda setups virtual environments before installing `cwltool` to -avoid conflicting versions of the same library. When installing cwltool directly, -it is recommended to do the same manually: + +All of the above methods of installing ``cwltool`` use packages which might contain bugs already fixed in newer versions, or be missing features that you desire. +If the packaged version of ``cwltool`` available to you is too old, then we recommend installing using ``pip`` and ``venv`:: .. code:: bash - virtualenv -p python3 venv # Create a virtual environment - source venv/bin/activate # Activate environment before installing `cwltool` + python3 -m venv env # Create a virtual environment named 'env' in the current directory + source env/bin/activate # Activate environment before installing `cwltool` -Installing the official package from PyPi (will install "cwltool" package as +Then install the latest ``cwlref-runner`` package from PyPi (which will install the latest ``cwltool`` package as well) .. code:: bash pip install cwlref-runner -If installing alongside another CWL implementation then +If installing alongside another CWL implementation (like ``toil-cwl-runner`` or ``arvados-cwl-runner``) then instead run .. code:: bash pip install cwltool -Or you can install from source: +MS Windows users +^^^^^^^^^^^^^^^^ + +1. Install `"Windows Subsystem for Linux 2" (WSL2) and Docker Desktop `_ +2. Install `Debian from the Microsoft Store `_ +3. Set Debian as your default WSL 2 distro: ``wsl --set-default debian`` +4. Reboot if you have not yet already. +5. Launch Debian and follow the Linux instructions above (``apt-get install cwltool`` or use the ``venv`` method) + +``cwltool`` development version +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Or you can skip the direct ``pip`` commands above and install the latest development version of ``cwltool``: .. code:: bash - git clone https://github.com/common-workflow-language/cwltool.git # clone cwltool repo - cd cwltool # Switch to source directory - pip install . # Install `cwltool` from source + git clone https://github.com/common-workflow-language/cwltool.git # clone (copy) the cwltool git repository + cd cwltool # Change to source directory that git clone just downloaded + pip install . # Installs ``cwltool`` from source cwltool --version # Check if the installation works correctly Remember, if co-installing multiple CWL implementations then you need to maintain which implementation ``cwl-runner`` points to via a symbolic file system link or `another facility `_. +Recommended Software +^^^^^^^^^^^^^^^^^^^^ + You may also want to have the following installed: -node.js -Docker, udocker, or Singularity (optional) +- `node.js `_ +- Docker, udocker, or Singularity (optional) Without these, some examples in the CWL tutorials at http://www.commonwl.org/user_guide/ may not work. @@ -97,7 +117,7 @@ Simple command:: cwl-runner [tool-or-workflow-description] [input-job-settings] Or if you have multiple CWL implementations installed and you want to override -the default cwl-runner use:: +the default cwl-runner then use:: cwltool [tool-or-workflow-description] [input-job-settings] @@ -106,8 +126,8 @@ these will be inserted at the beginning of the command line:: export CWLTOOL_OPTIONS="--debug" -Use with boot2docker --------------------- +Use with boot2docker on macOS +----------------------------- boot2docker runs Docker inside a virtual machine and it only mounts ``Users`` on it. The default behavior of CWL is to create temporary directories under e.g. ``/Var`` which is not accessible to Docker containers. @@ -117,8 +137,8 @@ and ``--tmp-outdir-prefix`` to somewhere under ``/Users``:: $ cwl-runner --tmp-outdir-prefix=/Users/username/project --tmpdir-prefix=/Users/username/project wc-tool.cwl wc-job.json -Using user-space replacements for Docker ----------------------------------------- +Using uDocker +------------- Some shared computing environments don't support Docker software containers for technical or policy reasons. As a work around, the CWL reference runner supports using alternative ``docker`` implementations on Linux @@ -134,7 +154,7 @@ Run `cwltool` just as you normally would, but with the new option, e.g. from the cwltool --user-space-docker-cmd=udocker https://raw.githubusercontent.com/common-workflow-language/common-workflow-language/main/v1.0/v1.0/test-cwl-out2.cwl https://github.com/common-workflow-language/common-workflow-language/blob/main/v1.0/v1.0/empty.json -``cwltool`` can use `Singularity `_ version 2.6.1 +``cwltool`` can also use `Singularity `_ version 2.6.1 or later as a Docker container runtime. ``cwltool`` with Singularity will run software containers specified in ``DockerRequirement`` and therefore works with Docker images only, native @@ -143,7 +163,7 @@ runtime, provide ``--singularity`` command line option to ``cwltool``. With Singularity, ``cwltool`` can pass all CWL v1.0 conformance tests, except those involving Docker container ENTRYPOINTs. - +Example: .. code:: bash cwltool --singularity https://raw.githubusercontent.com/common-workflow-language/common-workflow-language/main/v1.0/v1.0/v1.0/cat3-tool-mediumcut.cwl https://github.com/common-workflow-language/common-workflow-language/blob/main/v1.0/v1.0/cat-job.json From 72733d9f97d791f17e8b20ee6601cbcdf69be14c Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" <1330696+mr-c@users.noreply.github.com> Date: Fri, 5 Feb 2021 12:12:07 +0100 Subject: [PATCH 006/111] fix wrong URL --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 7fb5342568..8bfe7207b2 100644 --- a/README.rst +++ b/README.rst @@ -152,7 +152,7 @@ Run `cwltool` just as you normally would, but with the new option, e.g. from the .. code:: bash - cwltool --user-space-docker-cmd=udocker https://raw.githubusercontent.com/common-workflow-language/common-workflow-language/main/v1.0/v1.0/test-cwl-out2.cwl https://github.com/common-workflow-language/common-workflow-language/blob/main/v1.0/v1.0/empty.json + cwltool --user-space-docker-cmd=udocker https://raw.githubusercontent.com/common-workflow-language/common-workflow-language/main/v1.0/v1.0/test-cwl-out2.cwl https://github.com/common-workflow-language/common-workflow-language/raw/main/v1.0/v1.0/empty.json ``cwltool`` can also use `Singularity `_ version 2.6.1 or later as a Docker container runtime. From ab713b6c3bd2c590deb725bd7ce0aab7b2764811 Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" <1330696+mr-c@users.noreply.github.com> Date: Mon, 22 Feb 2021 11:54:36 +0100 Subject: [PATCH 007/111] Add Debian package badges --- README.rst | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 8bfe7207b2..01004c0f14 100644 --- a/README.rst +++ b/README.rst @@ -2,11 +2,17 @@ Common Workflow Language tool description reference implementation ================================================================== -|Linux Status| |Windows Status| |Coverage Status| |Downloads| +|Linux Status| |Debian Stable package| |Debian Testing package| |Windows Status| |Coverage Status| |Downloads| .. |Linux Status| image:: https://img.shields.io/travis/common-workflow-language/cwltool/main.svg?label=Linux%20builds :target: https://travis-ci.org/common-workflow-language/cwltool +.. |Debian Stable package| image:: https://badges.debian.net/badges/debian/stable/cwltool/version.svg + :target: https://packages.debian.org/stable/cwltool + +.. |Debian Testing package| image:: https://badges.debian.net/badges/debian/testing/cwltool/version.svg + :target: https://packages.debian.org/testing/cwltool + .. |Windows Status| image:: https://img.shields.io/appveyor/ci/mr-c/cwltool/main.svg?label=Windows%20builds :target: https://ci.appveyor.com/project/mr-c/cwltool From 193c5759417c73a62104182ed0110eb65699276e Mon Sep 17 00:00:00 2001 From: Rupert Nash Date: Thu, 4 Mar 2021 10:52:32 +0000 Subject: [PATCH 008/111] add pydot to requirements.txt, version as in setup.py --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index f57195b609..2329e6a4f1 100755 --- a/requirements.txt +++ b/requirements.txt @@ -9,3 +9,4 @@ mypy-extensions psutil typing-extensions coloredlogs +pydot>=1.4.1 From 3a32a8f0622ef7967df2a30a3bc5ffd684ef674e Mon Sep 17 00:00:00 2001 From: Rupert Nash Date: Thu, 4 Mar 2021 18:39:04 +0000 Subject: [PATCH 009/111] add argcomplete to requirements --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 2329e6a4f1..500dc1a07f 100755 --- a/requirements.txt +++ b/requirements.txt @@ -10,3 +10,4 @@ psutil typing-extensions coloredlogs pydot>=1.4.1 +argcomplete>=1.12.0 From 506ac9d7d8b7ea9a4e326f968c70632fed0d044d Mon Sep 17 00:00:00 2001 From: Rupert Nash Date: Mon, 8 Mar 2021 17:33:07 +0000 Subject: [PATCH 010/111] Enable CI via GitHub Actions (#1411) --- .github/workflows/ci-tests.yml | 180 +++++++++++++++++++++++++++++ .travis.singularity_key.txt | 70 ----------- .travis.yml | 50 -------- Makefile | 2 +- README.rst | 6 +- travis.bash => conformance-test.sh | 0 release-test.sh | 13 ++- requirements.txt | 2 +- tox.ini | 13 +-- 9 files changed, 201 insertions(+), 135 deletions(-) create mode 100644 .github/workflows/ci-tests.yml delete mode 100644 .travis.singularity_key.txt delete mode 100644 .travis.yml rename travis.bash => conformance-test.sh (100%) diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml new file mode 100644 index 0000000000..a23055517c --- /dev/null +++ b/.github/workflows/ci-tests.yml @@ -0,0 +1,180 @@ +name: Continous integration tests + +on: + push: + pull_request: + +env: + singularity_version: 3.6.4 + +jobs: + + tox: + name: CI tests via Tox + + runs-on: ubuntu-20.04 + + strategy: + matrix: + py-ver-major: [3] + py-ver-minor: [6, 7, 8, 9] + step: [lint, unit, bandit, mypy] + exclude: + - py-ver-major: 3 + py-ver-minor: 9 + step: mypy + + env: + py-semver: ${{ format('{0}.{1}', matrix.py-ver-major, matrix.py-ver-minor) }} + TOXENV: ${{ format('py{0}{1}-{2}', matrix.py-ver-major, matrix.py-ver-minor, matrix.step) }} + + steps: + - uses: actions/checkout@v2 + + - name: Set up Singularity + if: ${{ matrix.step == 'unit' || matrix.step == 'mypy' }} + uses: eWaterCycle/setup-singularity@v6 + with: + singularity-version: ${{ env.singularity_version }} + + - name: Give the test runner user a name to make provenance happy. + if: ${{ matrix.step == 'unit' || matrix.step == 'mypy' }} + run: sudo usermod -c 'CI Runner' $(whoami) + + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: ${{ env.py-semver }} + + - name: Cache for pip + uses: actions/cache@v2 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ matrix.step }}-${{ hashFiles('requirements.txt', 'tox.ini') }} + + - name: Upgrade setuptools and install tox + run: | + pip install -U pip setuptools wheel + pip install tox tox-gh-actions + + - name: MyPy cache + if: ${{ matrix.step == 'mypy' }} + uses: actions/cache@v2 + with: + path: .mypy_cache/${{ env.py-semver }} + key: mypy-${{ env.py-semver }} + + - name: Test with tox + run: tox + + - name: Upload coverage to Codecov + if: ${{ matrix.step == 'unit' }} + uses: codecov/codecov-action@v1 + with: + fail_ci_if_error: true + + tox-style: + name: CI linters via Tox + + runs-on: ubuntu-20.04 + + strategy: + matrix: + step: [lint-readme, shellcheck, pydocstyle] + + env: + py-semver: 3.9 + TOXENV: ${{ format('py39-{0}', matrix.step) }} + + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: ${{ env.py-semver }} + + - name: Cache for pip + uses: actions/cache@v2 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ matrix.step }}-${{ hashFiles('requirements.txt') }} + + - name: Upgrade setuptools and install tox + run: | + pip install -U pip setuptools wheel + pip install tox tox-gh-actions + + - if: ${{ matrix.step == 'pydocstyle' }} + name: Create local branch main == origin/main for diff-quality + run: git branch main origin/main + + - name: Test with tox + run: tox + + conformance_tests: + name: CWL spec conformance tests + + runs-on: ubuntu-20.04 + + strategy: + matrix: + cwl-version: [v1.0, v1.1, v1.2] + + steps: + - uses: actions/checkout@v2 + + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: 3.9 + + - name: Cache for pip + uses: actions/cache@v2 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-conformance-${{ matrix.step }}-${{ hashFiles('requirements.txt') }} + + - name: Run CWL conformance tests ${{ matrix.cwl-version }} + env: + version: ${{ matrix.cwl-version }} + run: ./conformance-test.sh + + release_test: + name: CWL release test + + runs-on: ubuntu-20.04 + + steps: + - uses: actions/checkout@v2 + + - name: Set up Singularity + uses: eWaterCycle/setup-singularity@v6 + with: + singularity-version: ${{ env.singularity_version }} + + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: 3.9 + + - name: Give the test runner user a name to make provenance happy. + run: sudo usermod -c 'CI Runner' $(whoami) + + - name: Cache for pip + uses: actions/cache@v2 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-release-${{ hashFiles('requirements.txt', 'test-requirements.txt') }} + + - name: Install packages + run: | + pip install -U pip setuptools wheel + pip install virtualenv + + - name: Release test + env: + RELEASE_SKIP: head + run: ./release-test.sh diff --git a/.travis.singularity_key.txt b/.travis.singularity_key.txt deleted file mode 100644 index b9d4dcaebc..0000000000 --- a/.travis.singularity_key.txt +++ /dev/null @@ -1,70 +0,0 @@ ------BEGIN PGP PUBLIC KEY BLOCK----- - -mQGiBEQ7TOgRBADvaRsIZ3VZ6Qy7PlDpdMm97m0OfvouOj/HhjOM4M3ECbGn4cYh -vN1gK586s3sUsUcNQ8LuWvNsYhxYsVTZymCReJMEDxod0U6/z/oIbpWv5svF3kpl -ogA66Ju/6cZx62RiCSOkskI6A3Waj6xHyEo8AGOPfzbMoOOQ1TS1u9s2FwCgxziL -wADvKYlDZnWM03QtqIJVD8UEAOks9Q2OqFoqKarj6xTRdOYIBVEp2jhozZUZmLmz -pKL9E4NKGfixqxdVimFcRUGM5h7R2w7ORqXjCzpiPmgdv3jJLWDnmHLmMYRYQc8p -5nqo8mxuO3zJugxBemWoacBDd1MJaH7nK20Hsk9L/jvU/qLxPJotMStTnwO+EpsK -HlihA/9ZpvzR1QWNUd9nSuNR3byJhaXvxqQltsM7tLqAT4qAOJIcMjxr+qESdEbx -NHM5M1Y21ZynrsQw+Fb1WHXNbP79vzOxHoZR0+OXe8uUpkri2d9iOocre3NUdpOO -JHtl6cGGTFILt8tSuOVxMT/+nlo038JQB2jARe4B85O0tkPIPbQybmV1cm8uZGVi -aWFuLm5ldCBhcmNoaXZlIDxtaWNoYWVsLmhhbmtlQGdtYWlsLmNvbT6IRgQQEQgA -BgUCTVHJKwAKCRCNEUVjdcAkyOvzAJ0abJz+f2a6VZG1c9T8NHMTYh1atwCgt0EE -3ZZd/2in64jSzu0miqhXbOKISgQQEQIACgUCSotRlwMFAXgACgkQ93+NsjFEvg8n -JgCfWcdJbILBtpLZCocvOzlLPqJ0Fn0AoI4EpJRxoUnrtzBGUC1MqecU7WsDiGAE -ExECACAFAkqLUWcCGwMGCwkIBwMCBBUCCAMEFgIDAQIeAQIXgAAKCRCl0y8BJkml -qVklAJ4h2V6MdQkSAThF5c2Gkq6eSoIQYQCeM0DWyB9Bl+tTPSTYXwwZi2uoif20 -QmFwc3kuZ3NlLnVuaS1tYWdkZWJ1cmcuZGUgRGViaWFuIEFyY2hpdmUgPG1pY2hh -ZWwuaGFua2VAZ21haWwuY29tPohGBBARAgAGBQJEO03FAAoJEPd/jbIxRL4PU18A -n3tn7i4qdlMi8kHbYWFoabsKc9beAJ9sl/leZNCYNMGhz+u6BQgyeLKw94heBBMR -AgAeBQJEO0zoAhsDBgsJCAcDAgMVAgMDFgIBAh4BAheAAAoJEKXTLwEmSaWpVdoA -n27DvtZizNEbhz3wRUPQMiQjtqdvAJ9rS9YdPe5h5o5gHx3mw3BSkOttdYheBBMR -AgAeBQJEO0zoAhsDBgsJCAcDAgMVAgMDFgIBAh4BAheAAAoJEKXTLwEmSaWpVdoA -oLhwWL+E+2I9lrUf4Lf26quOK9vLAKC9ZpIF2tUirFFkBWnQvu13/TA0SokCHAQQ -AQIABgUCTSNBgQAKCRDAc9Iof/uem4NpEACQ8jxmaCaS/qk/Y4GiwLA5bvKosG3B -iARZ2v5UWqCZQ1tS56yKse/lCIzXQqU9BnYW6wOI2rvFf9meLfd8h96peG6oKscs -fbclLDIf68bBvGBQaD0VYFi/Fk/rxmTQBOCQ3AJZs8O5rIM4gPGE0QGvSZ1h7VRw -3Uyeg4jKXLIeJn2xEmOJgt3auAR2FyKbzHaX9JCoByJZ/eU23akNl9hgt7ePlpXo -74KNYC58auuMUhCq3BQDB+II4ERYMcmFp1N5ZG05Cl6jcaRRHDXz+Ax6DWprRI1+ -RH/Yyae6LmKpeJNwd+vM14aawnNO9h8IAQ+aJ3oYZdRhGyybbin3giJ10hmWveg/ -Pey91Nh9vBCHdDkdPU0s9zE7z/PHT0c5ccZRukxfZfkrlWQ5iqu3V064ku5f4PBy -8UPSkETcjYgDnrdnwqIAO+oVg/SFlfsOzftnwUrvwIcZlXAgtP6MEEAs/38e/JIN -g4VrpdAy7HMGEUsh6Ah6lvGQr+zBnG44XwKfl7e0uCYkrAzUJRGM5vx9iXvFMcMu -jv9EBNNBOU8/Y6MBDzGZhgaoeI27nrUvaveJXjAiDKAQWBLjtQjINZ8I9uaSGOul -8kpbFavE4eS3+KhISrSHe4DuAa3dk9zI+FiPvXY1ZyfQBtNpR+gYFY6VxMbHhY1U -lSLHO2eUIQLdYbRITmV1cm9EZWJpYW4gQXJjaGl2ZSBLZXkgPHBrZy1leHBwc3kt -bWFpbnRhaW5lcnNAbGlzdHMuYWxpb3RoLmRlYmlhbi5vcmc+iEYEEBEIAAYFAk1R -yQYACgkQjRFFY3XAJMgEWwCggx4Gqlcrt76TSMlbU94cESo55AEAoJ3asQEMpe8t -QUX+5aikw3z1AUoCiEoEEBECAAoFAkqf/3cDBQF4AAoJEPd/jbIxRL4PxyMAoKUI -RPWlHCj/+HSFfwhos68wcSwmAKChuC00qutDro+AOo+uuq6YoHXj+ohgBBMRAgAg -BQJKn/8bAhsDBgsJCAcDAgQVAggDBBYCAwECHgECF4AACgkQpdMvASZJpalDggCe -KF9KOgOPdQbFnKXl8KtHory4EEwAnA7jxgorE6kk2QHEXFSF8LzOOH4GiGMEExEC -ACMCGwMGCwkIBwMCBBUCCAMEFgIDAQIeAQIXgAUCSp//RgIZAQAKCRCl0y8BJkml -qekFAKCRyt4+FoCzmBbRUUP3Cr8PzH++IgCgkno4vdjsWdyAey8e0KpITTXMFrmJ -AhwEEAECAAYFAk0jQYEACgkQwHPSKH/7npsFfw/+P8B8hpM3+T1fgboBa4R32deu -n8m6b8vZMXwuo/awQtMpzjem8JGXSUQm8iiX4hDtjq6ZoPrlN8T4jNmviBt/F5jI -Jji/PYmhq+Zn9s++mfx+aF4IJrcHJWFkg/6kJzn4oSdl/YlvKf4VRCcQNtj4xV87 -GsdamnzU17XapLVMbSaVKh+6Af7ZLDerEH+iAq733HsYaTK+1xKmN7EFVXgS7bZ1 -9C4LTzc97bVHSywpT9yIrg9QQs/1kshfVIHDKyhjF6IwzSVbeGAIL3Oqo5zOMkWv -7JlEIkkhTyl+FETxNMTMYjAk+Uei3kRodneq3YBF2uFYSEzrXQgHAyn37geiaMYj -h8wu6a85nG1NS0SdxiZDIePmbvD9vWxFZUWYJ/h9ifsLivWcVXlvHoQ0emd+n2ai -FhAck2xsuyHgnGIZMHww5IkQdu/TMqvbcR6d8Xulh+C4Tq7ppy+oTLADSBKII++p -JQioYydRD529EUJgVlhyH27X6YAk3FuRD3zYZRYS2QECiKXvS665o3JRJ0ZSqNgv -YOom8M0zz6bI9grnUoivMI4o7ISpE4ZwffEd37HVzmraaUHDXRhkulFSf1ImtXoj -V9nNSM5p/+9eP7OioTZhSote6Vj6Ja1SZeRkXZK7BwqPbdO0VsYOb7G//ZiOlqs+ -paRr92G/pwBfj5Dq8EK5Ag0ERDtM9RAIAN0EJqBPvLN0tEin/y4Fe0R4n+E+zNXg -bBsq4WidwyUFy3h/6u86FYvegXwUqVS2OsEs5MwPcCVJOfaEthF7I89QJnP9Nfx7 -V5I9yFB53o9ii38BN7X+9gSjpfwXOvf/wIDfggxX8/wRFel37GRB7TiiABRArBez -s5x+zTXvT++WPhElySj0uY8bjVR6tso+d65K0UesvAa7PPWeRS+3nhqABSFLuTTT -MMbnVXCGesBrYHlFVXClAYrSIOX8Ub/UnuEYs9+hIV7U4jKzRF9WJhIC1cXHPmOh -vleAf/I9h/0KahD7HLYud40pNBo5tW8jSfp2/Q8TIE0xxshd51/xy4MAAwUH+wWn -zsYVk981OKUEXul8JPyPxbw05fOd6gF4MJ3YodO+6dfoyIl3bewk+11KXZQALKaO -1xmkAEO1RqizPeetoadBVkQBp5xPudsVElUTOX0pTYhkUd3iBilsCYKK1/KQ9KzD -I+O/lRsm6L9lc6rV0IgPU00P4BAwR+x8Rw7TJFbuS0miR3lP1NSguz+/kpjxzmGP -LyHJ+LVDYFkk6t0jPXhqFdUY6McUTBDEvavTGlVO062l9APTmmSMVFDsPN/rBes2 -rYhuuT+lDp+gcaS1UoaYCIm9kKOteQBnowX9V74Z+HKEYLtwILaSnNe6/fNSTvyj -g0z+R+sPCY4nHewbVC+ISQQYEQIACQUCRDtM9QIbDAAKCRCl0y8BJkmlqbecAJ9B -UdSKVg9H+fQNyP5sbOjj4RDtdACfXHrRHa2+XjJP0dhpvJ8IfvYnQsU= -=fAJZ ------END PGP PUBLIC KEY BLOCK----- diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 838266f5ce..0000000000 --- a/.travis.yml +++ /dev/null @@ -1,50 +0,0 @@ -dist: xenial -services: - - docker - -before_install: -- wget -O- http://neuro.debian.net/lists/xenial.us-nh.full | sudo tee /etc/apt/sources.list.d/neurodebian.sources.list -- sudo apt-key add .travis.singularity_key.txt -- sudo apt-get update -- sudo apt-get install -y singularity-container - -language: python -cache: - pip: true - directories: - - .mypy_cache - -os: - - linux -install: - - pip install tox-travis -jobs: - include: - - python: "3.6" - - python: "3.7" - - python: "3.8" - - python: "3.9-dev" - - python: "3.8" - name: "CWL v1.0 conformance tests" - script: ${TRAVIS_BUILD_DIR}/travis.bash - env: - - version=v1.0 - - python: "3.8" - name: "CWL v1.1 conformance tests" - env: - - version=v1.1 - script: ${TRAVIS_BUILD_DIR}/travis.bash - - python: "3.7" - script: RELEASE_SKIP=head ${TRAVIS_BUILD_DIR}/release-test.sh - name: 'release test' - - python: "3.8" - name: "CWL v1.2 conformance tests" - env: - - version=v1.2.0-dev4 - script: ${TRAVIS_BUILD_DIR}/travis.bash -script: tox -branches: - only: - - main -notifications: - email: false diff --git a/Makefile b/Makefile index 9f3487eb07..c4e8fee467 100644 --- a/Makefile +++ b/Makefile @@ -172,7 +172,7 @@ mypyc: ${PYSOURCES} MYPYPATH=typeshed/2and3/:typeshed/3 CWLTOOL_USE_MYPYC=1 pip install --verbose -e . && pytest --ignore cwltool/schemas --basetemp ./tmp shellcheck: FORCE - shellcheck build-cwl-docker.sh cwl-docker.sh release-test.sh travis.bash \ + shellcheck build-cwl-docker.sh cwl-docker.sh release-test.sh conformance-test.sh \ cwltool-in-docker.sh release-test: FORCE diff --git a/README.rst b/README.rst index 01004c0f14..9c39e6ef6e 100644 --- a/README.rst +++ b/README.rst @@ -4,8 +4,8 @@ Common Workflow Language tool description reference implementation |Linux Status| |Debian Stable package| |Debian Testing package| |Windows Status| |Coverage Status| |Downloads| -.. |Linux Status| image:: https://img.shields.io/travis/common-workflow-language/cwltool/main.svg?label=Linux%20builds - :target: https://travis-ci.org/common-workflow-language/cwltool +.. |Linux Status| image:: https://github.com/common-workflow-language/cwltool/actions/workflows/ci-tests.yml/badge.svg?branch=main + :target: https://github.com/common-workflow-language/cwltool/actions/workflows/ci-tests.yml .. |Debian Stable package| image:: https://badges.debian.net/badges/debian/stable/cwltool/version.svg :target: https://packages.debian.org/stable/cwltool @@ -61,7 +61,7 @@ please follow the install instructions for `conda-forge /dev/null && pwd )" rm -Rf testenv? || /bin/true -export HEAD=${TRAVIS_PULL_REQUEST_SHA:-$(git rev-parse HEAD)} if [ "${RELEASE_SKIP}" != "head" ] then diff --git a/requirements.txt b/requirements.txt index 500dc1a07f..5fa84198c0 100755 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,7 @@ rdflib>=4.2.2,<5.1 shellescape>=3.4.1,<3.5 schema-salad>=7.0.20210124093443,<8 prov==1.5.1 -bagit==1.6.4 +bagit==1.7.0 mypy-extensions psutil typing-extensions diff --git a/tox.ini b/tox.ini index 4d4b4a5cd4..629f5a709d 100644 --- a/tox.ini +++ b/tox.ini @@ -11,12 +11,12 @@ envlist = skipsdist = True skip_missing_interpreters = True -[travis] +[gh-actions] python = - 3.6: py36 - 3.7: py37 - 3.8: py38 - 3.9: py39 + 3.6: py36 + 3.7: py37 + 3.8: py38 + 3.9: py39 [testenv] description = @@ -29,8 +29,7 @@ description = passenv = CI - TRAVIS - TRAVIS_* + GITHUB_* PROOT_NO_SECCOMP deps = py{36,37,38,39}-{unit,lint,bandit,mypy}: -rrequirements.txt From be00bebfafb5c753bb2e8e87fa75124c40607ee1 Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" Date: Sun, 24 Jan 2021 17:15:29 +0100 Subject: [PATCH 011/111] upgrade to mypy 0.800 --- .github/workflows/ci-tests.yml | 4 ---- cwltool/context.py | 3 ++- cwltool/cwlrdf.py | 2 +- cwltool/main.py | 4 +--- cwltool/mpi.py | 2 +- cwltool/mutation.py | 2 +- cwltool/pack.py | 3 ++- cwltool/validate_js.py | 2 +- mypy.ini | 7 ++++--- setup.py | 2 +- test-requirements.txt | 2 +- tests/test_mpi.py | 2 +- tests/test_provenance.py | 6 +++--- tox.ini | 12 ++++++------ 14 files changed, 25 insertions(+), 28 deletions(-) diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index a23055517c..831ac4f31a 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -19,10 +19,6 @@ jobs: py-ver-major: [3] py-ver-minor: [6, 7, 8, 9] step: [lint, unit, bandit, mypy] - exclude: - - py-ver-major: 3 - py-ver-minor: 9 - step: mypy env: py-semver: ${{ format('{0}.{1}', matrix.py-ver-major, matrix.py-ver-minor) }} diff --git a/cwltool/context.py b/cwltool/context.py index 898554f7b9..5e58c779e4 100644 --- a/cwltool/context.py +++ b/cwltool/context.py @@ -8,7 +8,8 @@ # move to a regular typing import when Python 3.3-3.6 is no longer supported from ruamel.yaml.comments import CommentedMap from schema_salad.avro.schema import Names -from schema_salad.ref_resolver import FetcherCallableType, Loader +from schema_salad.ref_resolver import Loader +from schema_salad.utils import FetcherCallableType from typing_extensions import TYPE_CHECKING from .builder import Builder, HasReqsHints diff --git a/cwltool/cwlrdf.py b/cwltool/cwlrdf.py index 3df6dfdd64..86b87877ee 100644 --- a/cwltool/cwlrdf.py +++ b/cwltool/cwlrdf.py @@ -5,7 +5,7 @@ from rdflib import Graph from ruamel.yaml.comments import CommentedMap from schema_salad.jsonld_context import makerdf -from schema_salad.ref_resolver import ContextType +from schema_salad.utils import ContextType from .cwlviewer import CWLViewer from .process import Process diff --git a/cwltool/main.py b/cwltool/main.py index 78821028b7..02fc003fca 100755 --- a/cwltool/main.py +++ b/cwltool/main.py @@ -37,14 +37,12 @@ from ruamel.yaml.comments import CommentedMap, CommentedSeq from schema_salad.exceptions import ValidationException from schema_salad.ref_resolver import ( - ContextType, - FetcherCallableType, Loader, file_uri, uri_file_path, ) from schema_salad.sourceline import strip_dup_lineno -from schema_salad.utils import json_dumps +from schema_salad.utils import json_dumps, ContextType, FetcherCallableType from . import command_line_tool, workflow from .argparser import arg_parser, generate_parser, get_default_args diff --git a/cwltool/mpi.py b/cwltool/mpi.py index 3ec23d23ac..369e2ce7a9 100644 --- a/cwltool/mpi.py +++ b/cwltool/mpi.py @@ -53,7 +53,7 @@ def load(cls: Type[MpiConfigT], config_file_name: str) -> MpiConfigT: optional). """ with open(config_file_name) as cf: - data = yaml.round_trip_load(cf) + data = yaml.main.round_trip_load(cf) try: return cls(**data) except TypeError as e: diff --git a/cwltool/mutation.py b/cwltool/mutation.py index b1dc88a56c..8088232302 100644 --- a/cwltool/mutation.py +++ b/cwltool/mutation.py @@ -4,7 +4,7 @@ from .errors import WorkflowException from .utils import CWLObjectType -MutationState = namedtuple("MutationTracker", ["generation", "readers", "stepname"]) +MutationState = namedtuple("MutationState", ["generation", "readers", "stepname"]) _generation = "http://commonwl.org/cwltool#generation" diff --git a/cwltool/pack.py b/cwltool/pack.py index 92d864bc7b..1de051055a 100644 --- a/cwltool/pack.py +++ b/cwltool/pack.py @@ -15,7 +15,8 @@ ) from ruamel.yaml.comments import CommentedMap, CommentedSeq -from schema_salad.ref_resolver import Loader, ResolveType, SubLoader +from schema_salad.ref_resolver import Loader, SubLoader +from schema_salad.utils import ResolveType from .context import LoadingContext from .load_tool import fetch_document, resolve_and_validate_document diff --git a/cwltool/validate_js.py b/cwltool/validate_js.py index 3a2d757202..1f1415b8e2 100644 --- a/cwltool/validate_js.py +++ b/cwltool/validate_js.py @@ -116,7 +116,7 @@ def get_expressions( return [] -JSHintJSReturn = namedtuple("jshint_return", ["errors", "globals"]) +JSHintJSReturn = namedtuple("JSHintJSReturn", ["errors", "globals"]) def jshint_js( diff --git a/mypy.ini b/mypy.ini index 3a6c4faab6..25ed405f07 100644 --- a/mypy.ini +++ b/mypy.ini @@ -1,8 +1,8 @@ [mypy] -# --strict options as of mypy 0.720 +# --strict options as of mypy 0.800 warn_unused_configs = True -disallow_subclassing_any = True disallow_any_generics = True +disallow_subclassing_any = True disallow_untyped_calls = True disallow_untyped_defs = True disallow_incomplete_defs = True @@ -12,7 +12,8 @@ no_implicit_optional = True warn_redundant_casts = True warn_unused_ignores = True warn_return_any = True -implicit_reexport = True +implicit_reexport = False +strict_equality = True [mypy-schema_salad.tests.*] ignore_errors = True diff --git a/setup.py b/setup.py index ba64e06164..03da0f330a 100755 --- a/setup.py +++ b/setup.py @@ -121,7 +121,7 @@ setup_requires=PYTEST_RUNNER, test_suite="tests", tests_require=[ - "pytest == 6.0.2", + "pytest >= 6.0.2, < 6.2", "mock >= 2.0.0", "pytest-mock >= 1.10.0", "arcp >= 0.2.0", diff --git a/test-requirements.txt b/test-requirements.txt index 440fd237d3..dea3f4c9ff 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,4 +1,4 @@ -pytest == 6.0.2 +pytest >= 6.0.2, < 6.2 mock >= 2.0.0 pytest-mock >= 1.10.0 pytest-cov diff --git a/tests/test_mpi.py b/tests/test_mpi.py index 6e7a2a6c16..35061e07a4 100644 --- a/tests/test_mpi.py +++ b/tests/test_mpi.py @@ -104,7 +104,7 @@ def run_many(self, n: int, args: List[str]): "env_pass": ["USER"], } plat_conf_file = mpitmp / "plat_mpi.yml" - plat_conf_file.write_text(yaml.round_trip_dump(plat_conf)) + plat_conf_file.write_text(yaml.main.round_trip_dump(plat_conf)) yield str(plat_conf_file) diff --git a/tests/test_provenance.py b/tests/test_provenance.py index 97df6aa11a..e0ab67758a 100644 --- a/tests/test_provenance.py +++ b/tests/test_provenance.py @@ -14,8 +14,8 @@ from rdflib.namespace import DC, DCTERMS, RDF from rdflib.term import Literal -# Module to be tested from cwltool import provenance +from cwltool import provenance_constants from cwltool.main import main from cwltool.provenance import ResearchObject from cwltool.stdfsaccess import StdFsAccess @@ -343,7 +343,7 @@ def check_ro(base_path: Path, nested: bool = False) -> None: profile = dc break assert profile is not None, "Can't find profile with dct:conformsTo" - assert profile == URIRef(provenance.CWLPROV_VERSION), ( + assert profile == URIRef(provenance_constants.CWLPROV_VERSION), ( "Unexpected cwlprov version " + profile ) @@ -407,7 +407,7 @@ def check_ro(base_path: Path, nested: bool = False) -> None: # Check all prov elements are listed formats = set() for prov in g.objects(p, OA.hasBody): - assert (prov, DCTERMS.conformsTo, URIRef(provenance.CWLPROV_VERSION)) in g + assert (prov, DCTERMS.conformsTo, URIRef(provenance_constants.CWLPROV_VERSION)) in g # NOTE: DC.format is a Namespace method and does not resolve like other terms formats.update(set(g.objects(prov, DC["format"]))) assert formats, "Could not find media types" diff --git a/tox.ini b/tox.ini index 629f5a709d..20e5be3e92 100644 --- a/tox.ini +++ b/tox.ini @@ -3,7 +3,7 @@ envlist = py{36,37,38,39}-lint, py{36,37,38,39}-unit, py{36,37,38,39}-bandit, - py{36,37,38}-mypy, + py{36,37,38,39}-mypy, py38-lint-readme, py38-shellcheck, py38-pydocstyle @@ -23,7 +23,7 @@ description = py{36,37,38,39}-unit: Run the unit tests py{36,37,38,39}-lint: Lint the Python code py{36,37,38,39}-bandit: Search for common security issues - py{36,37,38}-mypy: Check for type safety + py{36,37,38,39}-mypy: Check for type safety py38-pydocstyle: docstring style checker py38-shellcheck: syntax check for shell scripts @@ -41,7 +41,7 @@ deps = py{36,37,38,39}-lint: flake8-bugbear py{36,37,38,39}-lint: black py{36,37,38,39}-bandit: bandit - py{36,37,38}-mypy: mypy==0.790 + py{36,37,38,39}-mypy: mypy==0.800 setenv = py{36,37,38,39}-unit: LC_ALL = C @@ -57,14 +57,14 @@ commands = py{36,37,38,39}-unit: codecov --file coverage.xml py{36,37,38,39}-bandit: bandit -r cwltool py{36,37,38,39}-lint: flake8 cwltool setup.py - py{36,37,38}-mypy: make mypy - py{36,37,38}-mypy: make mypyc + py{36,37,38,39}-mypy: make mypy + py{36,37,38,39}-mypy: make mypyc py38-shellcheck: make shellcheck whitelist_externals = py{36,37,38,39}-lint: flake8 py{36,37,38,39}-lint: black - py{36,37,38}-{mypy,shellcheck}: make + py{36,37,38,39}-{mypy,shellcheck}: make py38-shellcheck: shellcheck [testenv:py38-pydocstyle] From 0b78dec610c38adc1f8ff1fd77aad349b4212463 Mon Sep 17 00:00:00 2001 From: Rupert Nash Date: Mon, 8 Mar 2021 19:56:20 +0000 Subject: [PATCH 012/111] CI: only create a local branch for PRs --- .github/workflows/ci-tests.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index 831ac4f31a..da993434c6 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -103,9 +103,9 @@ jobs: pip install -U pip setuptools wheel pip install tox tox-gh-actions - - if: ${{ matrix.step == 'pydocstyle' }} - name: Create local branch main == origin/main for diff-quality - run: git branch main origin/main + - if: ${{ matrix.step == 'pydocstyle' && github.event_name == 'pull_request'}} + name: Create local branch for diff-quality for PRs + run: git branch ${{github.base_ref}} origin/${{github.base_ref}} - name: Test with tox run: tox From ea6273d8969516621cadc709f7ed382257f963d1 Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" Date: Tue, 9 Mar 2021 10:22:11 +0100 Subject: [PATCH 013/111] minimum setuptools & pip versions for Python3.9 --- .github/workflows/ci-tests.yml | 4 +++- release-test.sh | 22 ++++++++++------------ 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index da993434c6..187c788977 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -2,7 +2,9 @@ name: Continous integration tests on: push: + branches: [ main ] pull_request: + branches: [ main ] env: singularity_version: 3.6.4 @@ -139,7 +141,7 @@ jobs: run: ./conformance-test.sh release_test: - name: CWL release test + name: cwltool release test runs-on: ubuntu-20.04 diff --git a/release-test.sh b/release-test.sh index 320ffc81d5..1daa20e325 100755 --- a/release-test.sh +++ b/release-test.sh @@ -11,12 +11,11 @@ module=cwltool if [ "$GITHUB_ACTIONS" = "true" ]; then # We are running as a GH Action repo=${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}.git - HEAD=${GITHUB_SHA} + HEAD=${GITHUB_REF} else repo=https://github.com/common-workflow-language/cwltool.git HEAD=$(git rev-parse HEAD) fi - test_prefix="" run_tests() { local mod_loc @@ -25,9 +24,8 @@ run_tests() { "${test_prefix}"bin/py.test "--ignore=${mod_loc}/schemas/" \ --pyargs -x ${module} -n auto --dist=loadfile } -pipver=7.0.2 # minimum required version of pip -setuptoolsver=24.2.0 # required to generate correct metadata for - # python_requires +pipver=20.3b1 # minimum required version of pip for Python 3.9 +setuptoolsver=41.1.0 # required for Python 3.9 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" rm -Rf testenv? || /bin/true @@ -35,12 +33,12 @@ rm -Rf testenv? || /bin/true if [ "${RELEASE_SKIP}" != "head" ] then - virtualenv testenv1 -p python3 + python3 -m venv testenv_1 # First we test the head # shellcheck source=/dev/null source testenv1/bin/activate rm -Rf testenv1/local - rm testenv1/lib/python-wheels/setuptools* \ + rm -f testenv1/lib/python-wheels/setuptools* \ && pip install --force-reinstall -U pip==${pipver} \ && pip install setuptools==${setuptoolsver} wheel make install-dep @@ -55,9 +53,9 @@ then test_prefix=../ run_tests; popd fi -virtualenv testenv2 -p python3 -virtualenv testenv3 -p python3 -virtualenv testenv4 -p python3 +python3 -m venv testenv2 +python3 -m venv testenv3 +python3 -m venv testenv4 rm -Rf testenv[234]/local # Secondly we test via pip @@ -65,7 +63,7 @@ rm -Rf testenv[234]/local cd testenv2 # shellcheck source=/dev/null source bin/activate -rm lib/python-wheels/setuptools* \ +rm -f lib/python-wheels/setuptools* \ && pip install --force-reinstall -U pip==${pipver} \ && pip install setuptools==${setuptoolsver} wheel pip install -e "git+${repo}@${HEAD}#egg=${package}" #[deps] @@ -85,7 +83,7 @@ run_tests cd ../testenv3/ # shellcheck source=/dev/null source bin/activate -rm lib/python-wheels/setuptools* \ +rm -f lib/python-wheels/setuptools* \ && pip install --force-reinstall -U pip==${pipver} \ && pip install setuptools==${setuptoolsver} wheel package_tar=$(find . -name "${package}*tar.gz") From 39496f3e685f94266aab0092ce0b41e80e596f56 Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" <1330696+mr-c@users.noreply.github.com> Date: Fri, 5 Feb 2021 13:05:03 +0100 Subject: [PATCH 014/111] Detect some types of bad URLs for the job/cwl inputs --- cwltool/__init__.py | 9 +++++++++ cwltool/load_tool.py | 7 +++++-- cwltool/main.py | 16 ++++++++-------- requirements.txt | 2 +- setup.py | 2 +- tests/test_content_type.py | 18 ++++++++++++++++++ tests/test_fetch.py | 4 ++-- tests/test_iwdr.py | 2 -- tests/test_provenance.py | 12 +++++++----- tests/test_relax_path_checks.py | 1 - tests/test_toolargparse.py | 1 - tests/test_trs.py | 5 +++-- tests/test_udocker.py | 2 -- tests/util.py | 2 -- 14 files changed, 54 insertions(+), 29 deletions(-) create mode 100644 tests/test_content_type.py diff --git a/cwltool/__init__.py b/cwltool/__init__.py index 057094f459..030ec0c1fa 100644 --- a/cwltool/__init__.py +++ b/cwltool/__init__.py @@ -1,3 +1,12 @@ """Reference implementation of the CWL standards.""" __author__ = "pamstutz@veritasgenetics.com" + +CWL_CONTENT_TYPES = [ + "text/plain", + "application/json", + "text/vnd.yaml", + "text/yaml", + "text/x-yaml", + "application/x-yaml", +] diff --git a/cwltool/load_tool.py b/cwltool/load_tool.py index ed6cde383f..4a23f117e0 100644 --- a/cwltool/load_tool.py +++ b/cwltool/load_tool.py @@ -31,7 +31,7 @@ json_dumps, ) -from . import process, update +from . import CWL_CONTENT_TYPES, process, update from .context import LoadingContext from .errors import WorkflowException from .loghandler import _logger @@ -127,7 +127,10 @@ def fetch_document( resolver=loadingContext.resolver, document_loader=loadingContext.loader, ) - workflowobj = cast(CommentedMap, loadingContext.loader.fetch(fileuri)) + workflowobj = cast( + CommentedMap, + loadingContext.loader.fetch(fileuri, content_types=CWL_CONTENT_TYPES), + ) return loadingContext, workflowobj, uri if isinstance(argsworkflow, MutableMapping): uri = ( diff --git a/cwltool/main.py b/cwltool/main.py index 02fc003fca..0d221c47f8 100755 --- a/cwltool/main.py +++ b/cwltool/main.py @@ -36,15 +36,11 @@ from ruamel import yaml from ruamel.yaml.comments import CommentedMap, CommentedSeq from schema_salad.exceptions import ValidationException -from schema_salad.ref_resolver import ( - Loader, - file_uri, - uri_file_path, -) +from schema_salad.ref_resolver import Loader, file_uri, uri_file_path from schema_salad.sourceline import strip_dup_lineno -from schema_salad.utils import json_dumps, ContextType, FetcherCallableType +from schema_salad.utils import ContextType, FetcherCallableType, json_dumps -from . import command_line_tool, workflow +from . import CWL_CONTENT_TYPES, command_line_tool, workflow from .argparser import arg_parser, generate_parser, get_default_args from .builder import HasReqsHints from .context import LoadingContext, RuntimeContext, getdefault @@ -332,7 +328,11 @@ def load_job_order( if args.basedir else os.path.abspath(os.path.dirname(job_order_file)) ) - job_order_object, _ = loader.resolve_ref(job_order_file, checklinks=False) + job_order_object, _ = loader.resolve_ref( + job_order_file, + checklinks=False, + content_types=CWL_CONTENT_TYPES, + ) if ( job_order_object is not None diff --git a/requirements.txt b/requirements.txt index 5fa84198c0..8711d83431 100755 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ requests>=2.4.3 ruamel.yaml>=0.12.4,<=0.16.5 rdflib>=4.2.2,<5.1 shellescape>=3.4.1,<3.5 -schema-salad>=7.0.20210124093443,<8 +schema-salad>=7.1,<8 prov==1.5.1 bagit==1.7.0 mypy-extensions diff --git a/setup.py b/setup.py index 03da0f330a..64c2820feb 100755 --- a/setup.py +++ b/setup.py @@ -97,7 +97,7 @@ "ruamel.yaml >= 0.12.4, <= 0.16.5", "rdflib >= 4.2.2, < 5.1.0", "shellescape >= 3.4.1, < 3.5", - "schema-salad >= 7.0.20210124093443, < 8", + "schema-salad >= 7.1, < 8", "mypy-extensions", "psutil", "prov == 1.5.1", diff --git a/tests/test_content_type.py b/tests/test_content_type.py new file mode 100644 index 0000000000..b609143f53 --- /dev/null +++ b/tests/test_content_type.py @@ -0,0 +1,18 @@ +import pydot # type: ignore + + +from .util import ( + get_main_output, +) + + +def test_content_types() -> None: + for test_file in ("js_output.cwl", "js_output_workflow.cwl"): + commands = [ + "https://raw.githubusercontent.com/common-workflow-language/common-workflow-language/main/v1.0/v1.0/test-cwl-out2.cwl", + "https://github.com/common-workflow-language/common-workflow-language/blob/main/v1.0/v1.0/empty.json", + ] + error_code, _, stderr = get_main_output(commands) + + assert "got content-type of 'text/html'" in stderr + assert error_code == 1, stderr diff --git a/tests/test_fetch.py b/tests/test_fetch.py index 57cb3186b5..2923ed2dcc 100644 --- a/tests/test_fetch.py +++ b/tests/test_fetch.py @@ -1,6 +1,6 @@ import os from pathlib import Path -from typing import Any, Optional +from typing import Any, List, Optional from urllib.parse import urljoin, urlsplit import pytest @@ -26,7 +26,7 @@ def __init__( ) -> None: """Create a Fetcher that provides a fixed result for testing purposes.""" - def fetch_text(self, url): # type: (str) -> str + def fetch_text(self, url: str, content_types: Optional[List[str]] = None) -> str: if url == "baz:bar/foo.cwl": return """ cwlVersion: v1.0 diff --git a/tests/test_iwdr.py b/tests/test_iwdr.py index 0ba5f19010..fe4fcbfd7a 100644 --- a/tests/test_iwdr.py +++ b/tests/test_iwdr.py @@ -1,5 +1,3 @@ -import os -import tempfile from pathlib import Path from typing import Any diff --git a/tests/test_provenance.py b/tests/test_provenance.py index e0ab67758a..0239c42578 100644 --- a/tests/test_provenance.py +++ b/tests/test_provenance.py @@ -1,11 +1,10 @@ import json import os import pickle -import shutil import sys import urllib from pathlib import Path -from typing import Any, Generator, cast +from typing import Any, Generator import arcp import bagit @@ -14,8 +13,7 @@ from rdflib.namespace import DC, DCTERMS, RDF from rdflib.term import Literal -from cwltool import provenance -from cwltool import provenance_constants +from cwltool import provenance, provenance_constants from cwltool.main import main from cwltool.provenance import ResearchObject from cwltool.stdfsaccess import StdFsAccess @@ -407,7 +405,11 @@ def check_ro(base_path: Path, nested: bool = False) -> None: # Check all prov elements are listed formats = set() for prov in g.objects(p, OA.hasBody): - assert (prov, DCTERMS.conformsTo, URIRef(provenance_constants.CWLPROV_VERSION)) in g + assert ( + prov, + DCTERMS.conformsTo, + URIRef(provenance_constants.CWLPROV_VERSION), + ) in g # NOTE: DC.format is a Namespace method and does not resolve like other terms formats.update(set(g.objects(prov, DC["format"]))) assert formats, "Could not find media types" diff --git a/tests/test_relax_path_checks.py b/tests/test_relax_path_checks.py index a605ed9c05..eab7fcb993 100644 --- a/tests/test_relax_path_checks.py +++ b/tests/test_relax_path_checks.py @@ -1,4 +1,3 @@ -import os from pathlib import Path from cwltool.main import main diff --git a/tests/test_toolargparse.py b/tests/test_toolargparse.py index feff5cafee..1ac723d52e 100644 --- a/tests/test_toolargparse.py +++ b/tests/test_toolargparse.py @@ -1,5 +1,4 @@ import argparse -import os from io import StringIO from pathlib import Path from typing import Callable diff --git a/tests/test_trs.py b/tests/test_trs.py index 7ddd6a8629..a1641a8394 100644 --- a/tests/test_trs.py +++ b/tests/test_trs.py @@ -22,7 +22,7 @@ def json(self) -> Any: return self.json_data -def mocked_requests_head(*args): # type: (*Any) -> MockResponse1 +def mocked_requests_head(*args: Any) -> MockResponse1: return MockResponse1(None, 200) @@ -41,9 +41,10 @@ def __init__( def json(self) -> Any: return self.json_data + headers = {"content-type": "text/plain"} -def mocked_requests_get(*args): # type: (*Any) -> MockResponse2 +def mocked_requests_get(*args: Any, **kwargs: Any) -> MockResponse2: if ( args[0] == "https://dockstore.org/api/api/ga4gh/v2/tools/quay.io%2Fbriandoconnor%2Fdockstore-tool-md5sum/versions/1.0.4/CWL/files" diff --git a/tests/test_udocker.py b/tests/test_udocker.py index cad2f948fc..603d29a606 100644 --- a/tests/test_udocker.py +++ b/tests/test_udocker.py @@ -1,7 +1,6 @@ """Test optional udocker feature.""" import copy import os -import shutil import subprocess import sys @@ -12,7 +11,6 @@ from pathlib import Path -from typing import Generator import pytest from _pytest.tmpdir import TempPathFactory diff --git a/tests/util.py b/tests/util.py index b3e650601d..8fe5b9385f 100644 --- a/tests/util.py +++ b/tests/util.py @@ -2,10 +2,8 @@ import distutils.spawn # pylint: disable=no-name-in-module,import-error import functools import os -import shutil import subprocess import sys -import tempfile from pathlib import Path from typing import Generator, List, Mapping, Optional, Tuple, Union From f288e677bff7700238994171de6a484f0fb72442 Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" Date: Mon, 25 Jan 2021 09:51:06 +0100 Subject: [PATCH 015/111] Add missing interpreter lines --- tests/arg-empty-prefix-separate-false.cwl | 3 ++- tests/iwdr_dir_literal_real_file.cwl | 1 + tests/wf/conflict.cwl | 1 + tests/wf/generator/pytoolgen.cwl | 1 + tests/wf/literalfile.cwl | 1 + tests/wf/packed-with-loadlisting.cwl | 1 + tests/wf/scatter2.cwl | 1 + tests/wf/scatter2_subwf.cwl | 1 + tests/with_doc.cwl | 1 + tests/without_doc.cwl | 1 + 10 files changed, 11 insertions(+), 1 deletion(-) diff --git a/tests/arg-empty-prefix-separate-false.cwl b/tests/arg-empty-prefix-separate-false.cwl index e86842fadc..7bee9f1a76 100644 --- a/tests/arg-empty-prefix-separate-false.cwl +++ b/tests/arg-empty-prefix-separate-false.cwl @@ -1,3 +1,4 @@ +#!/usr/bin/env cwl-runner class: CommandLineTool cwlVersion: v1.0 baseCommand: [] @@ -15,4 +16,4 @@ arguments: shellQuote: false valueFrom: "echo" requirements: -- class: ShellCommandRequirement \ No newline at end of file +- class: ShellCommandRequirement diff --git a/tests/iwdr_dir_literal_real_file.cwl b/tests/iwdr_dir_literal_real_file.cwl index 6cca7c4a06..105b9b62b5 100644 --- a/tests/iwdr_dir_literal_real_file.cwl +++ b/tests/iwdr_dir_literal_real_file.cwl @@ -1,3 +1,4 @@ +#!/usr/bin/env cwl-runner class: CommandLineTool cwlVersion: v1.0 diff --git a/tests/wf/conflict.cwl b/tests/wf/conflict.cwl index 9f052cbc3c..21ed20b1fe 100644 --- a/tests/wf/conflict.cwl +++ b/tests/wf/conflict.cwl @@ -1,3 +1,4 @@ +#!/usr/bin/env cwl-runner cwlVersion: v1.1 $graph: - class: CommandLineTool diff --git a/tests/wf/generator/pytoolgen.cwl b/tests/wf/generator/pytoolgen.cwl index 8e39c3c394..35ccd145db 100644 --- a/tests/wf/generator/pytoolgen.cwl +++ b/tests/wf/generator/pytoolgen.cwl @@ -1,3 +1,4 @@ +#!/usr/bin/env cwl-runner cwlVersion: v1.0 $namespaces: cwltool: "http://commonwl.org/cwltool#" diff --git a/tests/wf/literalfile.cwl b/tests/wf/literalfile.cwl index 489b096bf9..6c212314e1 100644 --- a/tests/wf/literalfile.cwl +++ b/tests/wf/literalfile.cwl @@ -1,3 +1,4 @@ +#!/usr/bin/env cwl-runner cwlVersion: v1.0 class: CommandLineTool inputs: diff --git a/tests/wf/packed-with-loadlisting.cwl b/tests/wf/packed-with-loadlisting.cwl index a9dd178535..dff5c4ad9b 100644 --- a/tests/wf/packed-with-loadlisting.cwl +++ b/tests/wf/packed-with-loadlisting.cwl @@ -1,3 +1,4 @@ +#!/usr/bin/env cwl-runner { "$graph": [ { diff --git a/tests/wf/scatter2.cwl b/tests/wf/scatter2.cwl index 79c3854d8e..5c829102b4 100644 --- a/tests/wf/scatter2.cwl +++ b/tests/wf/scatter2.cwl @@ -1,3 +1,4 @@ +#!/usr/bin/env cwl-runner # Copyright (C) The Arvados Authors. All rights reserved. # # SPDX-License-Identifier: Apache-2.0 diff --git a/tests/wf/scatter2_subwf.cwl b/tests/wf/scatter2_subwf.cwl index 2c41367fde..7fb35d50fd 100644 --- a/tests/wf/scatter2_subwf.cwl +++ b/tests/wf/scatter2_subwf.cwl @@ -1,3 +1,4 @@ +#!/usr/bin/env cwl-runner # Copyright (C) The Arvados Authors. All rights reserved. # # SPDX-License-Identifier: Apache-2.0 diff --git a/tests/with_doc.cwl b/tests/with_doc.cwl index a68c8e076b..77c6c0f4b7 100644 --- a/tests/with_doc.cwl +++ b/tests/with_doc.cwl @@ -1,3 +1,4 @@ +#!/usr/bin/env cwl-runner cwlVersion: v1.0 class: CommandLineTool inputs: [] diff --git a/tests/without_doc.cwl b/tests/without_doc.cwl index ef1688a8d3..85765e7d82 100644 --- a/tests/without_doc.cwl +++ b/tests/without_doc.cwl @@ -1,3 +1,4 @@ +#!/usr/bin/env cwl-runner cwlVersion: v1.0 class: CommandLineTool inputs: [] From e9ee4151154132ce6d403649bb14ee34496eb2cb Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" Date: Tue, 9 Mar 2021 15:53:55 +0100 Subject: [PATCH 016/111] increase checkout depth for codecov.io --- .github/workflows/ci-tests.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index 187c788977..2a0455f2c0 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -28,6 +28,8 @@ jobs: steps: - uses: actions/checkout@v2 + with: + fetch-depth: 0 - name: Set up Singularity if: ${{ matrix.step == 'unit' || matrix.step == 'mypy' }} From 828e48a21c189b0265be98e85a3fb7e95cf62ef2 Mon Sep 17 00:00:00 2001 From: Michael Kotliar Date: Tue, 9 Mar 2021 11:24:24 -0500 Subject: [PATCH 017/111] Kill Docker container if --cidfile is specified (#996) --- cwltool/main.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/cwltool/main.py b/cwltool/main.py index 0d221c47f8..74141b1c0e 100755 --- a/cwltool/main.py +++ b/cwltool/main.py @@ -8,6 +8,7 @@ import logging import os import signal +import subprocess # nosec import sys import time import urllib @@ -112,7 +113,18 @@ def _terminate_processes() -> None: # It's possible that another thread will spawn a new task while # we're executing, so it's not safe to use a for loop here. while processes_to_kill: - processes_to_kill.popleft().kill() + process = processes_to_kill.popleft() + cidfile = [str(arg).split("=")[1] for arg in process.args if "--cidfile" in str(arg)] + if cidfile: + try: + with open(cidfile[0], "r") as inp_stream: + p = subprocess.Popen(["docker", "kill", inp_stream.read()], shell=False) # nosec + try: + p.wait(timeout=10) + except subprocess.TimeoutExpired: + p.kill() + except FileNotFoundError: + pass def _signal_handler(signum: int, _: Any) -> None: From 667943cab463891612d36094cf8ea4d7289b80b7 Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" Date: Tue, 9 Mar 2021 19:49:53 +0100 Subject: [PATCH 018/111] --single-step to extract/run a single step --- cwltool/argparser.py | 114 +++++++++++++++++-------------- cwltool/main.py | 51 ++++++++++++-- cwltool/subgraph.py | 76 ++++++++++++++++----- tests/subgraph/single_step1.json | 23 +++++++ tests/subgraph/single_step2.json | 23 +++++++ tests/subgraph/single_step3.json | 24 +++++++ tests/subgraph/single_step4.json | 24 +++++++ tests/subgraph/single_step5.json | 30 ++++++++ tests/test_content_type.py | 5 +- tests/test_subgraph.py | 33 ++++++++- tox.ini | 1 + 11 files changed, 327 insertions(+), 77 deletions(-) create mode 100644 tests/subgraph/single_step1.json create mode 100644 tests/subgraph/single_step2.json create mode 100644 tests/subgraph/single_step3.json create mode 100644 tests/subgraph/single_step4.json create mode 100644 tests/subgraph/single_step5.json diff --git a/cwltool/argparser.py b/cwltool/argparser.py index 205565ad68..4f60a6ca40 100644 --- a/cwltool/argparser.py +++ b/cwltool/argparser.py @@ -64,8 +64,8 @@ def arg_parser() -> argparse.ArgumentParser: dest="preserve_entire_environment", ) - exgroup = parser.add_mutually_exclusive_group() - exgroup.add_argument( + containergroup = parser.add_mutually_exclusive_group() + containergroup.add_argument( "--rm-container", action="store_true", default=True, @@ -73,7 +73,7 @@ def arg_parser() -> argparse.ArgumentParser: dest="rm_container", ) - exgroup.add_argument( + containergroup.add_argument( "--leave-container", action="store_false", default=True, @@ -123,8 +123,8 @@ def arg_parser() -> argparse.ArgumentParser: default=DEFAULT_TMP_PREFIX, ) - exgroup = parser.add_mutually_exclusive_group() - exgroup.add_argument( + intgroup = parser.add_mutually_exclusive_group() + intgroup.add_argument( "--tmp-outdir-prefix", type=str, help="Path prefix for intermediate output directories. Defaults to the " @@ -132,7 +132,7 @@ def arg_parser() -> argparse.ArgumentParser: default="", ) - exgroup.add_argument( + intgroup.add_argument( "--cachedir", type=str, default="", @@ -141,8 +141,8 @@ def arg_parser() -> argparse.ArgumentParser: "troubleshooting of CWL documents.", ) - exgroup = parser.add_mutually_exclusive_group() - exgroup.add_argument( + tmpgroup = parser.add_mutually_exclusive_group() + tmpgroup.add_argument( "--rm-tmpdir", action="store_true", default=True, @@ -150,7 +150,7 @@ def arg_parser() -> argparse.ArgumentParser: dest="rm_tmpdir", ) - exgroup.add_argument( + tmpgroup.add_argument( "--leave-tmpdir", action="store_false", default=True, @@ -158,8 +158,8 @@ def arg_parser() -> argparse.ArgumentParser: dest="rm_tmpdir", ) - exgroup = parser.add_mutually_exclusive_group() - exgroup.add_argument( + outgroup = parser.add_mutually_exclusive_group() + outgroup.add_argument( "--move-outputs", action="store_const", const="move", @@ -169,7 +169,7 @@ def arg_parser() -> argparse.ArgumentParser: dest="move_outputs", ) - exgroup.add_argument( + outgroup.add_argument( "--leave-outputs", action="store_const", const="leave", @@ -178,7 +178,7 @@ def arg_parser() -> argparse.ArgumentParser: dest="move_outputs", ) - exgroup.add_argument( + outgroup.add_argument( "--copy-outputs", action="store_const", const="copy", @@ -188,8 +188,8 @@ def arg_parser() -> argparse.ArgumentParser: dest="move_outputs", ) - exgroup = parser.add_mutually_exclusive_group() - exgroup.add_argument( + pullgroup = parser.add_mutually_exclusive_group() + pullgroup.add_argument( "--enable-pull", default=True, action="store_true", @@ -197,7 +197,7 @@ def arg_parser() -> argparse.ArgumentParser: dest="pull_image", ) - exgroup.add_argument( + pullgroup.add_argument( "--disable-pull", default=True, action="store_false", @@ -280,65 +280,68 @@ def arg_parser() -> argparse.ArgumentParser: type=str, ) - exgroup = parser.add_mutually_exclusive_group() - exgroup.add_argument( + printgroup = parser.add_mutually_exclusive_group() + printgroup.add_argument( "--print-rdf", action="store_true", help="Print corresponding RDF graph for workflow and exit", ) - exgroup.add_argument( + printgroup.add_argument( "--print-dot", action="store_true", help="Print workflow visualization in graphviz format and exit", ) - exgroup.add_argument( + printgroup.add_argument( "--print-pre", action="store_true", help="Print CWL document after preprocessing.", ) - exgroup.add_argument( + printgroup.add_argument( "--print-deps", action="store_true", help="Print CWL document dependencies." ) - exgroup.add_argument( + printgroup.add_argument( "--print-input-deps", action="store_true", help="Print input object document dependencies.", ) - exgroup.add_argument( + printgroup.add_argument( "--pack", action="store_true", help="Combine components into single document and print.", ) - exgroup.add_argument( + printgroup.add_argument( "--version", action="store_true", help="Print version and exit" ) - exgroup.add_argument( + printgroup.add_argument( "--validate", action="store_true", help="Validate CWL document only." ) - exgroup.add_argument( + printgroup.add_argument( "--print-supported-versions", action="store_true", help="Print supported CWL specs.", ) - exgroup.add_argument( + printgroup.add_argument( "--print-subgraph", action="store_true", help="Print workflow subgraph that will execute. Can combined with " - "--target.", + "--target or --single-step", ) - exgroup.add_argument( + printgroup.add_argument( "--print-targets", action="store_true", help="Print targets (output parameters)" ) + printgroup.add_argument( + "--make-template", action="store_true", help="Generate a template input object" + ) - exgroup = parser.add_mutually_exclusive_group() - exgroup.add_argument( + strictgroup = parser.add_mutually_exclusive_group() + strictgroup.add_argument( "--strict", action="store_true", help="Strict validation (unrecognized or out of place fields are error)", default=True, dest="strict", ) - exgroup.add_argument( + strictgroup.add_argument( "--non-strict", action="store_false", help="Lenient validation (ignore unrecognized fields)", @@ -354,15 +357,15 @@ def arg_parser() -> argparse.ArgumentParser: dest="skip_schemas", ) - exgroup = parser.add_mutually_exclusive_group() - exgroup.add_argument( + doccachegroup = parser.add_mutually_exclusive_group() + doccachegroup.add_argument( "--no-doc-cache", action="store_false", help="Disable disk cache for documents loaded over HTTP", default=True, dest="doc_cache", ) - exgroup.add_argument( + doccachegroup.add_argument( "--doc-cache", action="store_true", help="Enable disk cache for documents loaded over HTTP", @@ -370,12 +373,14 @@ def arg_parser() -> argparse.ArgumentParser: dest="doc_cache", ) - exgroup = parser.add_mutually_exclusive_group() - exgroup.add_argument("--verbose", action="store_true", help="Default logging") - exgroup.add_argument( + volumegroup = parser.add_mutually_exclusive_group() + volumegroup.add_argument("--verbose", action="store_true", help="Default logging") + volumegroup.add_argument( "--quiet", action="store_true", help="Only print warnings and errors." ) - exgroup.add_argument("--debug", action="store_true", help="Print even more logging") + volumegroup.add_argument( + "--debug", action="store_true", help="Print even more logging" + ) parser.add_argument( "--strict-memory-limit", @@ -506,14 +511,14 @@ def arg_parser() -> argparse.ArgumentParser: default=False, ) - exgroup = parser.add_mutually_exclusive_group() - exgroup.add_argument( + colorgroup = parser.add_mutually_exclusive_group() + colorgroup.add_argument( "--enable-color", action="store_true", help="Enable logging color (default enabled)", default=not onWindows(), ) - exgroup.add_argument( + colorgroup.add_argument( "--disable-color", action="store_false", dest="enable_color", @@ -545,15 +550,15 @@ def arg_parser() -> argparse.ArgumentParser: help=argparse.SUPPRESS, ) - exgroup = parser.add_mutually_exclusive_group() - exgroup.add_argument( + reggroup = parser.add_mutually_exclusive_group() + reggroup.add_argument( "--enable-ga4gh-tool-registry", action="store_true", help="Enable tool resolution using GA4GH tool registry API", dest="enable_ga4gh_tool_registry", default=True, ) - exgroup.add_argument( + reggroup.add_argument( "--disable-ga4gh-tool-registry", action="store_false", help="Disable tool resolution using GA4GH tool registry API", @@ -579,15 +584,15 @@ def arg_parser() -> argparse.ArgumentParser: choices=("stop", "continue"), ) - exgroup = parser.add_mutually_exclusive_group() - exgroup.add_argument( + checkgroup = parser.add_mutually_exclusive_group() + checkgroup.add_argument( "--compute-checksum", action="store_true", default=True, help="Compute checksum of contents while collecting outputs", dest="compute_checksum", ) - exgroup.add_argument( + checkgroup.add_argument( "--no-compute-checksum", action="store_false", help="Do not compute checksum of contents while collecting outputs", @@ -602,9 +607,6 @@ def arg_parser() -> argparse.ArgumentParser: "spaces and hash characters.", dest="relax_path_checks", ) - exgroup.add_argument( - "--make-template", action="store_true", help="Generate a template input object" - ) parser.add_argument( "--force-docker-pull", @@ -628,13 +630,21 @@ def arg_parser() -> argparse.ArgumentParser: help="Read process requirement overrides from file.", ) - parser.add_argument( + subgroup = parser.add_mutually_exclusive_group() + subgroup.add_argument( "--target", "-t", action="append", help="Only execute steps that contribute to listed targets (can be " "provided more than once).", ) + subgroup.add_argument( + "--single-step", + type=str, + default=None, + help="Only executes a single step in a workflow. The input object must " + "match that step's inputs. Can be combined with --print-subgraph.", + ) parser.add_argument( "--mpi-config-file", diff --git a/cwltool/main.py b/cwltool/main.py index 74141b1c0e..90b0800daa 100755 --- a/cwltool/main.py +++ b/cwltool/main.py @@ -80,7 +80,7 @@ get_container_from_software_requirements, ) from .stdfsaccess import StdFsAccess -from .subgraph import get_subgraph +from .subgraph import get_step, get_subgraph from .update import ALLUPDATES, UPDATES from .utils import ( DEFAULT_TMP_PREFIX, @@ -114,11 +114,15 @@ def _terminate_processes() -> None: # we're executing, so it's not safe to use a for loop here. while processes_to_kill: process = processes_to_kill.popleft() - cidfile = [str(arg).split("=")[1] for arg in process.args if "--cidfile" in str(arg)] + cidfile = [ + str(arg).split("=")[1] for arg in process.args if "--cidfile" in str(arg) + ] if cidfile: try: with open(cidfile[0], "r") as inp_stream: - p = subprocess.Popen(["docker", "kill", inp_stream.read()], shell=False) # nosec + p = subprocess.Popen( # nosec + ["docker", "kill", inp_stream.read()], shell=False # nosec + ) try: p.wait(timeout=10) except subprocess.TimeoutExpired: @@ -764,7 +768,7 @@ def choose_target( tool: Process, loadingContext: LoadingContext, ) -> Optional[Process]: - """Walk the given Workflow and find the process that matches args.target.""" + """Walk the Workflow, extract the subset matches all the args.targets.""" if loadingContext.loader is None: raise Exception("loadingContext.loader cannot be None") @@ -794,6 +798,38 @@ def choose_target( return tool +def choose_step( + args: argparse.Namespace, + tool: Process, + loadingContext: LoadingContext, +) -> Optional[Process]: + """Walk the given Workflow and extract just args.single_step.""" + if loadingContext.loader is None: + raise Exception("loadingContext.loader cannot be None") + + if isinstance(tool, Workflow): + url = urllib.parse.urlparse(tool.tool["id"]) + if url.fragment: + extracted = get_step(tool, tool.tool["id"] + "/" + args.singe_step) + else: + extracted = get_step( + tool, + loadingContext.loader.fetcher.urljoin( + tool.tool["id"], "#" + args.single_step + ), + ) + else: + _logger.error("Can only use --single-step on Workflows") + return None + if isinstance(loadingContext.loader.idx, MutableMapping): + loadingContext.loader.idx[extracted["id"]] = extracted + tool = make_tool(extracted["id"], loadingContext) + else: + raise Exception("Missing loadingContext.loader.idx!") + + return tool + + def check_working_directories( runtimeContext: RuntimeContext, ) -> Optional[int]: @@ -1033,6 +1069,13 @@ def main( else: tool = ctool + elif args.single_step: + ctool = choose_step(args, tool, loadingContext) + if ctool is None: + return 1 + else: + tool = ctool + if args.print_subgraph: if "name" in tool.tool: del tool.tool["name"] diff --git a/cwltool/subgraph.py b/cwltool/subgraph.py index 2df2f206d9..92795eedf7 100644 --- a/cwltool/subgraph.py +++ b/cwltool/subgraph.py @@ -1,11 +1,20 @@ import urllib from collections import namedtuple -from typing import Dict, MutableMapping, MutableSequence, Optional, Set, Tuple, cast +from typing import ( + Dict, + List, + MutableMapping, + MutableSequence, + Optional, + Set, + Tuple, + cast, +) from ruamel.yaml.comments import CommentedMap from .utils import CWLObjectType, aslist -from .workflow import Workflow +from .workflow import Workflow, WorkflowStep Node = namedtuple("Node", ("up", "down", "type")) UP = "up" @@ -44,11 +53,18 @@ def declare_node(nodes: Dict[str, Node], nodeid: str, tp: Optional[str]) -> Node return nodes[nodeid] +def find_step(steps: List[WorkflowStep], stepid: str) -> Optional[CWLObjectType]: + for st in steps: + if st.tool["id"] == stepid: + return st.tool + return None + + def get_subgraph(roots: MutableSequence[str], tool: Workflow) -> CommentedMap: if tool.tool["class"] != "Workflow": raise Exception("Can only extract subgraph from workflow") - nodes = {} # type: Dict[str, Node] + nodes: Dict[str, Node] = {} for inp in tool.tool["inputs"]: declare_node(nodes, inp["id"], INPUT) @@ -81,22 +97,16 @@ def get_subgraph(roots: MutableSequence[str], tool: Workflow) -> CommentedMap: nodes[out].up.append(st["id"]) # Find all the downstream nodes from the starting points - visited_down = set() # type: Set[str] + visited_down: Set[str] = set() for r in roots: if nodes[r].type == OUTPUT: subgraph_visit(r, nodes, visited_down, UP) else: subgraph_visit(r, nodes, visited_down, DOWN) - def find_step(stepid: str) -> Optional[CWLObjectType]: - for st in tool.steps: - if st.tool["id"] == stepid: - return st.tool - return None - # Now make sure all the nodes are connected to upstream inputs - visited = set() # type: Set[str] - rewire = {} # type: Dict[str, Tuple[str, str]] + visited: Set[str] = set() + rewire: Dict[str, Tuple[str, CWLObjectType]] = {} for v in visited_down: visited.add(v) if nodes[v].type in (STEP, OUTPUT): @@ -108,15 +118,17 @@ def find_step(stepid: str) -> Optional[CWLObjectType]: else: # rewire df = urllib.parse.urldefrag(u) - rn = df[0] + "#" + df[1].replace("/", "_") + rn = str(df[0] + "#" + df[1].replace("/", "_")) if nodes[v].type == STEP: - wfstep = find_step(v) + wfstep = find_step(tool.steps, v) if wfstep is not None: for inp in cast( MutableSequence[CWLObjectType], wfstep["inputs"] ): - if u in inp["source"]: - rewire[u] = (rn, inp["type"]) + if "source" in inp and u in cast( + CWLObjectType, inp["source"] + ): + rewire[u] = (rn, cast(CWLObjectType, inp["type"])) break else: raise Exception("Could not find step %s" % v) @@ -147,3 +159,35 @@ def find_step(stepid: str) -> Optional[CWLObjectType]: extracted["inputs"].append({"id": rv[0], "type": rv[1]}) return extracted + + +def get_step(tool: Workflow, step_id: str) -> CommentedMap: + + extracted = CommentedMap() + + step = find_step(tool.steps, step_id) + if step is None: + raise Exception(f"Step {step_id} was not found") + + extracted["steps"] = [step] + extracted["inputs"] = [] + extracted["outputs"] = [] + + for inport in cast(List[CWLObjectType], step["in"]): + name = cast(str, inport["id"]).split("#")[-1].split("/")[-1] + extracted["inputs"].append({"id": name, "type": "Any"}) + inport["source"] = name + if "linkMerge" in inport: + del inport["linkMerge"] + + for outport in cast(List[str], step["out"]): + name = outport.split("#")[-1].split("/")[-1] + extracted["outputs"].append( + {"id": name, "type": "Any", "outputSource": f"{step_id}/{name}"} + ) + + for f in tool.tool: + if f not in ("steps", "inputs", "outputs"): + extracted[f] = tool.tool[f] + + return extracted diff --git a/tests/subgraph/single_step1.json b/tests/subgraph/single_step1.json new file mode 100644 index 0000000000..8f140addae --- /dev/null +++ b/tests/subgraph/single_step1.json @@ -0,0 +1,23 @@ +{"class": "Workflow", + "cwlVersion": "v1.0", + "id": "count-lines1-wf.cwl", + "inputs": [{"id": "file1", "type": "Any"}], + "outputs": [{"id": "output", + "outputSource": "count-lines1-wf.cwl#step1/output", + "type": "Any"}], + "steps": [{"id": "count-lines1-wf.cwl#step1", + "in": [{"id": "count-lines1-wf.cwl#step1/file1", + "source": "file1"}], + "inputs": [{"_tool_entry": {"id": "wc-tool.cwl#file1", + "type": "File"}, + "id": "count-lines1-wf.cwl#step1/file1", + "source": "count-lines1-wf.cwl#file1", + "type": "File"}], + "out": ["count-lines1-wf.cwl#step1/output"], + "outputs": [{"_tool_entry": {"id": "wc-tool.cwl#output", + "outputBinding": {"glob": "output"}, + "type": "File"}, + "id": "count-lines1-wf.cwl#step1/output", + "outputBinding": {"glob": "output"}, + "type": "File"}], + "run": "wc-tool.cwl"}]} diff --git a/tests/subgraph/single_step2.json b/tests/subgraph/single_step2.json new file mode 100644 index 0000000000..68d2216880 --- /dev/null +++ b/tests/subgraph/single_step2.json @@ -0,0 +1,23 @@ +{"class": "Workflow", + "cwlVersion": "v1.0", + "id": "count-lines1-wf.cwl", + "inputs": [{"id": "file1", "type": "Any"}], + "outputs": [{"id": "output", + "outputSource": "count-lines1-wf.cwl#step2/output", + "type": "Any"}], + "steps": [{"id": "count-lines1-wf.cwl#step2", + "in": [{"id": "count-lines1-wf.cwl#step2/file1", + "source": "file1"}], + "inputs": [{"_tool_entry": {"id": "parseInt-tool.cwl#file1", + "inputBinding": {"loadContents": true}, + "type": "File"}, + "id": "count-lines1-wf.cwl#step2/file1", + "inputBinding": {"loadContents": true}, + "source": "count-lines1-wf.cwl#step1/output", + "type": "File"}], + "out": ["count-lines1-wf.cwl#step2/output"], + "outputs": [{"_tool_entry": {"id": "parseInt-tool.cwl#output", + "type": "int"}, + "id": "count-lines1-wf.cwl#step2/output", + "type": "int"}], + "run": "parseInt-tool.cwl"}]} diff --git a/tests/subgraph/single_step3.json b/tests/subgraph/single_step3.json new file mode 100644 index 0000000000..5ef062b352 --- /dev/null +++ b/tests/subgraph/single_step3.json @@ -0,0 +1,24 @@ +{"class": "Workflow", + "cwlVersion": "v1.0", + "id": "count-lines1-wf.cwl", + "inputs": [{"id": "file1", "type": "Any"}], + "outputs": [{"id": "output", + "outputSource": "count-lines1-wf.cwl#step3/output", + "type": "Any"}], + "steps": [{"id": "count-lines1-wf.cwl#step3", + "in": [{"id": "count-lines1-wf.cwl#step3/file1", + "source": "file1"}], + "inputs": [{"_tool_entry": {"id": "wc-tool.cwl#file1", + "type": "File"}, + "id": "count-lines1-wf.cwl#step3/file1", + "source": "count-lines1-wf.cwl#file2", + "type": "File"}], + "label": "step that is independent of step1 and step2", + "out": ["count-lines1-wf.cwl#step3/output"], + "outputs": [{"_tool_entry": {"id": "wc-tool.cwl#output", + "outputBinding": {"glob": "output"}, + "type": "File"}, + "id": "count-lines1-wf.cwl#step3/output", + "outputBinding": {"glob": "output"}, + "type": "File"}], + "run": "wc-tool.cwl"}]} diff --git a/tests/subgraph/single_step4.json b/tests/subgraph/single_step4.json new file mode 100644 index 0000000000..b8c4979433 --- /dev/null +++ b/tests/subgraph/single_step4.json @@ -0,0 +1,24 @@ +{"class": "Workflow", + "cwlVersion": "v1.0", + "id": "count-lines1-wf.cwl", + "inputs": [{"id": "file1", "type": "Any"}], + "outputs": [{"id": "output", + "outputSource": "count-lines1-wf.cwl#step4/output", + "type": "Any"}], + "steps": [{"id": "count-lines1-wf.cwl#step4", + "in": [{"id": "count-lines1-wf.cwl#step4/file1", + "source": "file1"}], + "inputs": [{"_tool_entry": {"id": "parseInt-tool.cwl#file1", + "inputBinding": {"loadContents": true}, + "type": "File"}, + "id": "count-lines1-wf.cwl#step4/file1", + "inputBinding": {"loadContents": true}, + "source": "count-lines1-wf.cwl#step1/output", + "type": "File"}], + "label": "step that also depends on step1", + "out": ["count-lines1-wf.cwl#step4/output"], + "outputs": [{"_tool_entry": {"id": "parseInt-tool.cwl#output", + "type": "int"}, + "id": "count-lines1-wf.cwl#step4/output", + "type": "int"}], + "run": "parseInt-tool.cwl"}]} diff --git a/tests/subgraph/single_step5.json b/tests/subgraph/single_step5.json new file mode 100644 index 0000000000..07c91097a0 --- /dev/null +++ b/tests/subgraph/single_step5.json @@ -0,0 +1,30 @@ +{"class": "Workflow", + "cwlVersion": "v1.0", + "id": "count-lines1-wf.cwl", + "inputs": [{"id": "file1", "type": "Any"}, {"id": "file3", "type": "Any"}], + "outputs": [{"id": "output", + "outputSource": "count-lines1-wf.cwl#step5/output", + "type": "Any"}], + "steps": [{"id": "count-lines1-wf.cwl#step5", + "in": [{"id": "count-lines1-wf.cwl#step5/file1", "source": "file1"}, + {"id": "count-lines1-wf.cwl#step5/file3", + "source": "file3"}], + "inputs": [{"_tool_entry": {"id": "wc-tool.cwl#file1", + "type": "File"}, + "id": "count-lines1-wf.cwl#step5/file1", + "source": "count-lines1-wf.cwl#file1", + "type": "File"}, + {"id": "count-lines1-wf.cwl#step5/file3", + "not_connected": true, + "source": "count-lines1-wf.cwl#file3", + "type": "Any", + "used_by_step": false}], + "label": "step with two inputs", + "out": ["count-lines1-wf.cwl#step5/output"], + "outputs": [{"_tool_entry": {"id": "wc-tool.cwl#output", + "outputBinding": {"glob": "output"}, + "type": "File"}, + "id": "count-lines1-wf.cwl#step5/output", + "outputBinding": {"glob": "output"}, + "type": "File"}], + "run": "wc-tool.cwl"}]} diff --git a/tests/test_content_type.py b/tests/test_content_type.py index b609143f53..422565a7ba 100644 --- a/tests/test_content_type.py +++ b/tests/test_content_type.py @@ -1,9 +1,6 @@ import pydot # type: ignore - -from .util import ( - get_main_output, -) +from .util import get_main_output def test_content_types() -> None: diff --git a/tests/test_subgraph.py b/tests/test_subgraph.py index a2731c2c39..bd321eb762 100644 --- a/tests/test_subgraph.py +++ b/tests/test_subgraph.py @@ -6,7 +6,7 @@ from cwltool.context import LoadingContext from cwltool.load_tool import load_tool -from cwltool.subgraph import get_subgraph +from cwltool.subgraph import get_step, get_subgraph from cwltool.workflow import Workflow, default_make_tool from .test_fetch import norm @@ -49,3 +49,34 @@ def clean(val: Any) -> Any: extracted = get_subgraph([wf + "#" + a], tool) with open(get_data("tests/subgraph/extract_" + a + ".json")) as f: assert json.load(f) == clean(convert_to_dict(extracted)) + + +def test_get_step() -> None: + loadingContext = LoadingContext({"construct_tool_object": default_make_tool}) + wf = norm(Path(get_data("tests/subgraph/count-lines1-wf.cwl")).as_uri()) + loadingContext.do_update = False + tool = load_tool(wf, loadingContext) + assert isinstance(tool, Workflow) + + sg = norm(Path(get_data("tests/subgraph")).as_uri()) + + def clean(val: Any) -> Any: + if isinstance(val, str): + if val.startswith(sg): + return val[len(sg) + 1 :] + if isinstance(val, dict): + return {k: clean(v) for k, v in val.items()} + if isinstance(val, list): + return [clean(v) for v in val] + return val + + for a in ( + "step1", + "step2", + "step3", + "step4", + "step5", + ): + extracted = get_step(tool, wf + "#" + a) + with open(get_data("tests/subgraph/single_" + a + ".json")) as f: + assert json.load(f) == clean(convert_to_dict(extracted)) diff --git a/tox.ini b/tox.ini index 20e5be3e92..3242ff873d 100644 --- a/tox.ini +++ b/tox.ini @@ -57,6 +57,7 @@ commands = py{36,37,38,39}-unit: codecov --file coverage.xml py{36,37,38,39}-bandit: bandit -r cwltool py{36,37,38,39}-lint: flake8 cwltool setup.py + py{36,37,38,39}-lint: black --diff --exclude cwltool/schemas --check cwltool tests py{36,37,38,39}-mypy: make mypy py{36,37,38,39}-mypy: make mypyc py38-shellcheck: make shellcheck From 322d16c474473fc34afcf27020284a490329f654 Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" <1330696+mr-c@users.noreply.github.com> Date: Thu, 11 Mar 2021 13:06:04 +0100 Subject: [PATCH 019/111] Consolidate our typeshed (#1417) --- Makefile | 10 +- typeshed/2and3/ctypes/__init__.pyi | 277 ---- typeshed/2and3/ctypes/util.pyi | 8 - typeshed/2and3/ctypes/wintypes.pyi | 209 --- typeshed/2and3/distutils/__init__.pyi | 0 typeshed/2and3/distutils/spawn.pyi | 8 - typeshed/2and3/msvcrt.pyi | 8 - typeshed/2and3/pathlib2.pyi | 120 -- typeshed/2and3/select.pyi | 137 -- typeshed/2and3/uuid.pyi | 94 -- typeshed/3/logging/__init__.pyi | 561 -------- typeshed/3/py/__init__.pyi | 13 - typeshed/3/py/__metainfo.pyi | 3 - typeshed/3/py/_builtin.pyi | 16 - typeshed/3/py/_code/__init__.pyi | 0 typeshed/3/py/_code/_assertionnew.pyi | 31 - typeshed/3/py/_code/_assertionold.pyi | 110 -- typeshed/3/py/_code/_py2traceback.pyi | 3 - typeshed/3/py/_code/assertion.pyi | 10 - typeshed/3/py/_code/code.pyi | 162 --- typeshed/3/py/_code/source.pyi | 29 - typeshed/3/py/_error.pyi | 11 - typeshed/3/py/_io/__init__.pyi | 0 typeshed/3/py/_io/capture.pyi | 61 - typeshed/3/py/_io/saferepr.pyi | 11 - typeshed/3/py/_io/terminalwriter.pyi | 72 - typeshed/3/py/_log/__init__.pyi | 0 typeshed/3/py/_log/log.pyi | 47 - typeshed/3/py/_log/warning.pyi | 9 - typeshed/3/py/_path/__init__.pyi | 0 typeshed/3/py/_path/cacheutil.pyi | 27 - typeshed/3/py/_path/common.pyi | 66 - typeshed/3/py/_path/local.pyi | 89 -- typeshed/3/py/_path/svnurl.pyi | 43 - typeshed/3/py/_path/svnwc.pyi | 152 --- typeshed/3/py/_process/__init__.pyi | 0 typeshed/3/py/_process/cmdexec.pyi | 13 - typeshed/3/py/_process/forkedfunc.pyi | 25 - typeshed/3/py/_process/killproc.pyi | 4 - typeshed/3/py/_std.pyi | 10 - typeshed/3/py/_version.pyi | 1 - typeshed/3/py/_xmlgen.pyi | 61 - typeshed/3/py/path.pyi | 148 -- typeshed/3/subprocess.pyi | 1208 ----------------- typeshed/3/typing.pyi | 649 --------- typeshed/{3 => }/arcp/__init__.pyi | 0 typeshed/{3 => }/arcp/generate.pyi | 0 typeshed/{3 => }/arcp/parse.pyi | 0 typeshed/{3 => }/argcomplete/__init__.pyi | 0 typeshed/{3 => }/bagit.pyi | 0 .../{2and3 => }/cachecontrol/__init__.pyi | 0 typeshed/{2and3 => }/cachecontrol/adapter.pyi | 0 typeshed/{2and3 => }/cachecontrol/cache.pyi | 0 .../cachecontrol/caches/__init__.pyi | 0 .../cachecontrol/caches/file_cache.pyi | 0 typeshed/{2and3 => }/cachecontrol/compat.pyi | 0 .../{2and3 => }/cachecontrol/controller.pyi | 0 .../{2and3 => }/cachecontrol/filewrapper.pyi | 0 .../{2and3 => }/cachecontrol/serialize.pyi | 0 typeshed/{2and3 => }/cachecontrol/wrapper.pyi | 0 typeshed/{2and3 => }/coloredlogs/__init__.pyi | 0 typeshed/{2and3 => }/galaxy/__init__.pyi | 0 .../galaxy/exceptions/__init__.pyi | 0 .../galaxy/exceptions/error_codes.pyi | 0 typeshed/{2and3 => }/galaxy/jobs/__init__.pyi | 0 .../galaxy/jobs/metrics/__init__.pyi | 0 .../galaxy/jobs/metrics/collectl/__init__.pyi | 0 .../galaxy/jobs/metrics/collectl/cli.pyi | 0 .../jobs/metrics/collectl/processes.pyi | 0 .../galaxy/jobs/metrics/collectl/stats.pyi | 0 .../jobs/metrics/collectl/subsystems.pyi | 0 .../galaxy/jobs/metrics/formatting.pyi | 0 .../jobs/metrics/instrumenters/__init__.pyi | 0 .../jobs/metrics/instrumenters/collectl.pyi | 0 .../jobs/metrics/instrumenters/core.pyi | 0 .../jobs/metrics/instrumenters/cpuinfo.pyi | 0 .../galaxy/jobs/metrics/instrumenters/env.pyi | 0 .../jobs/metrics/instrumenters/meminfo.pyi | 0 .../jobs/metrics/instrumenters/uname.pyi | 0 .../galaxy/objectstore/__init__.pyi | 0 .../galaxy/objectstore/azure_blob.pyi | 0 .../{2and3 => }/galaxy/objectstore/pulsar.pyi | 0 .../{2and3 => }/galaxy/objectstore/rods.pyi | 0 .../{2and3 => }/galaxy/objectstore/s3.pyi | 0 .../objectstore/s3_multipart_upload.pyi | 0 .../{2and3 => }/galaxy/tool_util/__init__.pyi | 0 .../galaxy/tool_util/cwl/__init__.pyi | 0 .../galaxy/tool_util/cwl/cwltool_deps.pyi | 0 .../galaxy/tool_util/cwl/parser.pyi | 0 .../galaxy/tool_util/cwl/representation.pyi | 0 .../galaxy/tool_util/cwl/runtime_actions.pyi | 0 .../galaxy/tool_util/cwl/schema.pyi | 0 .../galaxy/tool_util/deps/__init__.pyi | 0 .../galaxy/tool_util/deps/brew_exts.pyi | 0 .../galaxy/tool_util/deps/brew_util.pyi | 0 .../galaxy/tool_util/deps/commands.pyi | 0 .../galaxy/tool_util/deps/conda_compat.pyi | 0 .../galaxy/tool_util/deps/conda_util.pyi | 0 .../deps/container_resolvers/__init__.pyi | 0 .../deps/container_resolvers/explicit.pyi | 0 .../deps/container_resolvers/mulled.pyi | 0 .../galaxy/tool_util/deps/containers.pyi | 0 .../galaxy/tool_util/deps/dependencies.pyi | 0 .../galaxy/tool_util/deps/docker_util.pyi | 0 .../galaxy/tool_util/deps/dockerfiles.pyi | 0 .../galaxy/tool_util/deps/installable.pyi | 0 .../galaxy/tool_util/deps/mulled/__init__.pyi | 0 .../galaxy/tool_util/deps/mulled/_cli.pyi | 0 .../tool_util/deps/mulled/mulled_build.pyi | 0 .../deps/mulled/mulled_build_channel.pyi | 0 .../deps/mulled/mulled_build_files.pyi | 0 .../deps/mulled/mulled_build_tool.pyi | 0 .../tool_util/deps/mulled/mulled_search.pyi | 0 .../galaxy/tool_util/deps/mulled/util.pyi | 0 .../galaxy/tool_util/deps/requirements.pyi | 0 .../tool_util/deps/resolvers/__init__.pyi | 0 .../resolvers/brewed_tool_shed_packages.pyi | 0 .../galaxy/tool_util/deps/resolvers/conda.pyi | 0 .../deps/resolvers/galaxy_packages.pyi | 0 .../tool_util/deps/resolvers/homebrew.pyi | 0 .../tool_util/deps/resolvers/modules.pyi | 0 .../deps/resolvers/resolver_mixins.pyi | 0 .../deps/resolvers/tool_shed_packages.pyi | 0 .../resolvers/unlinked_tool_shed_packages.pyi | 0 .../tool_util/deps/singularity_util.pyi | 0 .../galaxy/tool_util/deps/views.pyi | 0 .../{2and3 => }/galaxy/tool_util/fetcher.pyi | 0 .../{2and3 => }/galaxy/tool_util/lint.pyi | 0 .../galaxy/tool_util/lint_util.pyi | 0 .../galaxy/tool_util/linters/__init__.pyi | 0 .../galaxy/tool_util/linters/citations.pyi | 0 .../galaxy/tool_util/linters/command.pyi | 0 .../galaxy/tool_util/linters/cwl.pyi | 0 .../galaxy/tool_util/linters/general.pyi | 0 .../galaxy/tool_util/linters/help.pyi | 0 .../galaxy/tool_util/linters/inputs.pyi | 0 .../galaxy/tool_util/linters/outputs.pyi | 0 .../galaxy/tool_util/linters/stdio.pyi | 0 .../galaxy/tool_util/linters/tests.pyi | 0 .../galaxy/tool_util/linters/xml_order.pyi | 0 .../{2and3 => }/galaxy/tool_util/loader.pyi | 0 .../galaxy/tool_util/loader_directory.pyi | 0 .../galaxy/tool_util/locations/__init__.pyi | 0 .../galaxy/tool_util/locations/dockstore.pyi | 0 .../galaxy/tool_util/locations/file.pyi | 0 .../galaxy/tool_util/locations/http.pyi | 0 .../galaxy/tool_util/parser/__init__.pyi | 0 .../galaxy/tool_util/parser/cwl.pyi | 0 .../galaxy/tool_util/parser/factory.pyi | 0 .../galaxy/tool_util/parser/interface.pyi | 0 .../tool_util/parser/output_actions.pyi | 0 .../parser/output_collection_def.pyi | 0 .../tool_util/parser/output_objects.pyi | 0 .../galaxy/tool_util/parser/util.pyi | 0 .../galaxy/tool_util/parser/xml.pyi | 0 .../galaxy/tool_util/parser/yaml.pyi | 0 .../galaxy/tool_util/verify/__init__.pyi | 0 .../tool_util/verify/asserts/__init__.pyi | 0 .../tool_util/verify/asserts/tabular.pyi | 0 .../galaxy/tool_util/verify/asserts/text.pyi | 0 .../galaxy/tool_util/verify/asserts/xml.pyi | 0 .../galaxy/tool_util/verify/test_data.pyi | 0 typeshed/{2and3 => }/galaxy/util/__init__.pyi | 0 .../{2and3 => }/galaxy/util/aliaspickler.pyi | 0 typeshed/{2and3 => }/galaxy/util/bunch.pyi | 0 typeshed/{2and3 => }/galaxy/util/checkers.pyi | 0 .../galaxy/util/compression_utils.pyi | 0 .../{2and3 => }/galaxy/util/dictifiable.pyi | 0 .../{2and3 => }/galaxy/util/expressions.pyi | 0 typeshed/{2and3 => }/galaxy/util/filelock.pyi | 0 .../{2and3 => }/galaxy/util/hash_util.pyi | 0 .../{2and3 => }/galaxy/util/heartbeat.pyi | 0 .../{2and3 => }/galaxy/util/image_util.pyi | 0 .../{2and3 => }/galaxy/util/inflection.pyi | 0 typeshed/{2and3 => }/galaxy/util/json.pyi | 0 .../{2and3 => }/galaxy/util/lazy_process.pyi | 0 .../galaxy/util/object_wrapper.pyi | 0 typeshed/{2and3 => }/galaxy/util/odict.pyi | 0 typeshed/{2and3 => }/galaxy/util/oset.pyi | 0 .../{2and3 => }/galaxy/util/plugin_config.pyi | 0 .../{2and3 => }/galaxy/util/properties.pyi | 0 .../{2and3 => }/galaxy/util/simplegraph.pyi | 0 typeshed/{2and3 => }/galaxy/util/sleeper.pyi | 0 typeshed/{2and3 => }/galaxy/util/sockets.pyi | 0 typeshed/{2and3 => }/galaxy/util/specs.pyi | 0 typeshed/{2and3 => }/galaxy/util/sqlite.pyi | 0 .../{2and3 => }/galaxy/util/submodules.pyi | 0 typeshed/{2and3 => }/galaxy/util/topsort.pyi | 0 .../{2and3 => }/galaxy/util/xml_macros.pyi | 0 typeshed/{2and3 => }/graphviz/__init__.pyi | 0 typeshed/{2and3 => }/graphviz/_compat.pyi | 0 typeshed/{2and3 => }/graphviz/backend.pyi | 0 typeshed/{2and3 => }/graphviz/dot.pyi | 0 typeshed/{2and3 => }/graphviz/files.pyi | 0 typeshed/{2and3 => }/graphviz/lang.pyi | 0 typeshed/{2and3 => }/graphviz/tools.pyi | 0 typeshed/{2and3 => }/mistune.pyi | 0 typeshed/{2and3 => }/networkx/__init__.pyi | 0 .../networkx/algorithms/__init__.pyi | 0 .../algorithms/approximation/__init__.pyi | 0 .../algorithms/approximation/clique.pyi | 0 .../approximation/clustering_coefficient.pyi | 0 .../algorithms/approximation/connectivity.pyi | 0 .../approximation/dominating_set.pyi | 0 .../approximation/independent_set.pyi | 0 .../algorithms/approximation/kcomponents.pyi | 0 .../algorithms/approximation/matching.pyi | 0 .../algorithms/approximation/ramsey.pyi | 0 .../algorithms/approximation/steinertree.pyi | 0 .../algorithms/approximation/vertex_cover.pyi | 0 .../algorithms/assortativity/__init__.pyi | 0 .../algorithms/assortativity/connectivity.pyi | 0 .../algorithms/assortativity/correlation.pyi | 0 .../algorithms/assortativity/mixing.pyi | 0 .../assortativity/neighbor_degree.pyi | 0 .../algorithms/assortativity/pairs.pyi | 0 .../algorithms/bipartite/__init__.pyi | 0 .../networkx/algorithms/bipartite/basic.pyi | 0 .../algorithms/bipartite/centrality.pyi | 0 .../networkx/algorithms/bipartite/cluster.pyi | 0 .../algorithms/bipartite/covering.pyi | 0 .../algorithms/bipartite/edgelist.pyi | 0 .../algorithms/bipartite/generators.pyi | 0 .../algorithms/bipartite/matching.pyi | 0 .../networkx/algorithms/bipartite/matrix.pyi | 0 .../algorithms/bipartite/projection.pyi | 0 .../algorithms/bipartite/redundancy.pyi | 0 .../algorithms/bipartite/spectral.pyi | 0 .../networkx/algorithms/boundary.pyi | 0 .../networkx/algorithms/bridges.pyi | 0 .../algorithms/centrality/betweenness.pyi | 0 .../centrality/betweenness_subset.pyi | 0 .../algorithms/centrality/closeness.pyi | 0 .../centrality/current_flow_betweenness.pyi | 0 .../current_flow_betweenness_subset.pyi | 0 .../centrality/current_flow_closeness.pyi | 0 .../algorithms/centrality/degree_alg.pyi | 0 .../algorithms/centrality/dispersion.pyi | 0 .../algorithms/centrality/eigenvector.pyi | 0 .../algorithms/centrality/flow_matrix.pyi | 0 .../algorithms/centrality/harmonic.pyi | 0 .../networkx/algorithms/centrality/katz.pyi | 0 .../networkx/algorithms/centrality/load.pyi | 0 .../algorithms/centrality/reaching.pyi | 0 .../algorithms/centrality/subgraph_alg.pyi | 0 .../networkx/algorithms/chains.pyi | 0 .../networkx/algorithms/chordal.pyi | 0 .../networkx/algorithms/clique.pyi | 0 .../networkx/algorithms/cluster.pyi | 0 .../networkx/algorithms/coloring/__init__.pyi | 0 .../algorithms/coloring/greedy_coloring.pyi | 0 .../greedy_coloring_with_interchange.pyi | 0 .../algorithms/communicability_alg.pyi | 0 .../algorithms/community/__init__.pyi | 0 .../algorithms/community/asyn_fluidc.pyi | 0 .../algorithms/community/centrality.pyi | 0 .../community/community_generators.pyi | 0 .../algorithms/community/community_utils.pyi | 0 .../networkx/algorithms/community/kclique.pyi | 0 .../algorithms/community/kernighan_lin.pyi | 0 .../community/label_propagation.pyi | 0 .../networkx/algorithms/community/quality.pyi | 0 .../algorithms/components/__init__.pyi | 0 .../algorithms/components/attracting.pyi | 0 .../algorithms/components/biconnected.pyi | 0 .../algorithms/components/connected.pyi | 0 .../algorithms/components/semiconnected.pyi | 0 .../components/strongly_connected.pyi | 0 .../components/weakly_connected.pyi | 0 .../algorithms/connectivity/__init__.pyi | 0 .../algorithms/connectivity/connectivity.pyi | 0 .../networkx/algorithms/connectivity/cuts.pyi | 0 .../connectivity/disjoint_paths.pyi | 0 .../connectivity/edge_augmentation.pyi | 0 .../connectivity/edge_kcomponents.pyi | 0 .../algorithms/connectivity/kcomponents.pyi | 0 .../algorithms/connectivity/kcutsets.pyi | 0 .../algorithms/connectivity/stoerwagner.pyi | 0 .../algorithms/connectivity/utils.pyi | 0 .../{2and3 => }/networkx/algorithms/core.pyi | 0 .../networkx/algorithms/covering.pyi | 0 .../{2and3 => }/networkx/algorithms/cuts.pyi | 0 .../networkx/algorithms/cycles.pyi | 0 .../{2and3 => }/networkx/algorithms/dag.pyi | 0 .../networkx/algorithms/distance_measures.pyi | 0 .../networkx/algorithms/distance_regular.pyi | 0 .../networkx/algorithms/dominance.pyi | 0 .../networkx/algorithms/dominating.pyi | 0 .../networkx/algorithms/efficiency.pyi | 0 .../{2and3 => }/networkx/algorithms/euler.pyi | 0 .../networkx/algorithms/flow/__init__.pyi | 0 .../algorithms/flow/boykovkolmogorov.pyi | 0 .../algorithms/flow/capacityscaling.pyi | 0 .../networkx/algorithms/flow/dinitz_alg.pyi | 0 .../networkx/algorithms/flow/edmondskarp.pyi | 0 .../networkx/algorithms/flow/gomory_hu.pyi | 0 .../networkx/algorithms/flow/maxflow.pyi | 0 .../networkx/algorithms/flow/mincost.pyi | 0 .../algorithms/flow/networksimplex.pyi | 0 .../networkx/algorithms/flow/preflowpush.pyi | 0 .../flow/shortestaugmentingpath.pyi | 0 .../networkx/algorithms/flow/utils.pyi | 0 .../networkx/algorithms/graphical.pyi | 0 .../networkx/algorithms/hierarchy.pyi | 0 .../networkx/algorithms/hybrid.pyi | 0 .../networkx/algorithms/isolate.pyi | 0 .../algorithms/isomorphism/__init__.pyi | 0 .../algorithms/isomorphism/isomorph.pyi | 0 .../algorithms/isomorphism/isomorphvf2.pyi | 0 .../algorithms/isomorphism/matchhelpers.pyi | 0 .../isomorphism/temporalisomorphvf2.pyi | 0 .../algorithms/isomorphism/vf2userfunc.pyi | 0 .../algorithms/link_analysis/__init__.pyi | 0 .../algorithms/link_analysis/hits_alg.pyi | 0 .../algorithms/link_analysis/pagerank_alg.pyi | 0 .../networkx/algorithms/link_prediction.pyi | 0 .../algorithms/lowest_common_ancestors.pyi | 0 .../networkx/algorithms/matching.pyi | 0 .../networkx/algorithms/minors.pyi | 0 .../{2and3 => }/networkx/algorithms/mis.pyi | 0 .../algorithms/operators/__init__.pyi | 0 .../networkx/algorithms/operators/all.pyi | 0 .../networkx/algorithms/operators/binary.pyi | 0 .../networkx/algorithms/operators/product.pyi | 0 .../networkx/algorithms/operators/unary.pyi | 0 .../networkx/algorithms/reciprocity.pyi | 0 .../networkx/algorithms/richclub.pyi | 0 .../algorithms/shortest_paths/__init__.pyi | 0 .../algorithms/shortest_paths/astar.pyi | 0 .../algorithms/shortest_paths/dense.pyi | 0 .../algorithms/shortest_paths/generic.pyi | 0 .../algorithms/shortest_paths/unweighted.pyi | 0 .../algorithms/shortest_paths/weighted.pyi | 0 .../networkx/algorithms/similarity.pyi | 0 .../networkx/algorithms/simple_paths.pyi | 0 .../networkx/algorithms/smetric.pyi | 0 .../networkx/algorithms/structuralholes.pyi | 0 .../{2and3 => }/networkx/algorithms/swap.pyi | 0 .../networkx/algorithms/threshold.pyi | 0 .../networkx/algorithms/tournament.pyi | 0 .../algorithms/traversal/__init__.pyi | 0 .../algorithms/traversal/beamsearch.pyi | 0 .../traversal/breadth_first_search.pyi | 0 .../traversal/depth_first_search.pyi | 0 .../networkx/algorithms/traversal/edgedfs.pyi | 0 .../networkx/algorithms/tree/__init__.pyi | 0 .../networkx/algorithms/tree/branchings.pyi | 0 .../networkx/algorithms/tree/coding.pyi | 0 .../networkx/algorithms/tree/mst.pyi | 0 .../networkx/algorithms/tree/operations.pyi | 0 .../networkx/algorithms/tree/recognition.pyi | 0 .../networkx/algorithms/triads.pyi | 0 .../networkx/algorithms/vitality.pyi | 0 .../networkx/algorithms/voronoi.pyi | 0 .../networkx/algorithms/wiener.pyi | 0 .../{2and3 => }/networkx/classes/__init__.pyi | 0 .../networkx/classes/coreviews.pyi | 0 .../{2and3 => }/networkx/classes/digraph.pyi | 0 .../{2and3 => }/networkx/classes/filters.pyi | 0 .../{2and3 => }/networkx/classes/function.pyi | 0 .../{2and3 => }/networkx/classes/graph.pyi | 0 .../networkx/classes/graphviews.pyi | 0 .../networkx/classes/multidigraph.pyi | 0 .../networkx/classes/multigraph.pyi | 0 .../{2and3 => }/networkx/classes/ordered.pyi | 0 .../networkx/classes/reportviews.pyi | 0 typeshed/{2and3 => }/networkx/convert.pyi | 0 .../{2and3 => }/networkx/convert_matrix.pyi | 0 .../{2and3 => }/networkx/drawing/__init__.pyi | 0 .../{2and3 => }/networkx/drawing/layout.pyi | 0 .../networkx/drawing/nx_agraph.pyi | 0 .../{2and3 => }/networkx/drawing/nx_pydot.pyi | 0 .../{2and3 => }/networkx/drawing/nx_pylab.pyi | 0 typeshed/{2and3 => }/networkx/exception.pyi | 0 .../{2and3 => }/networkx/generators/atlas.pyi | 0 .../networkx/generators/classic.pyi | 0 .../networkx/generators/community.pyi | 0 .../networkx/generators/degree_seq.pyi | 0 .../networkx/generators/directed.pyi | 0 .../networkx/generators/duplication.pyi | 0 .../{2and3 => }/networkx/generators/ego.pyi | 0 .../networkx/generators/expanders.pyi | 0 .../networkx/generators/geometric.pyi | 0 .../networkx/generators/intersection.pyi | 0 .../networkx/generators/joint_degree_seq.pyi | 0 .../networkx/generators/lattice.pyi | 0 .../{2and3 => }/networkx/generators/line.pyi | 0 .../networkx/generators/mycielski.pyi | 0 .../generators/nonisomorphic_trees.pyi | 0 .../networkx/generators/random_clustered.pyi | 0 .../networkx/generators/random_graphs.pyi | 0 .../{2and3 => }/networkx/generators/small.pyi | 0 .../networkx/generators/social.pyi | 0 .../networkx/generators/stochastic.pyi | 0 .../{2and3 => }/networkx/generators/trees.pyi | 0 .../networkx/generators/triads.pyi | 0 .../{2and3 => }/networkx/linalg/__init__.pyi | 0 .../networkx/linalg/algebraicconnectivity.pyi | 0 .../networkx/linalg/attrmatrix.pyi | 0 .../networkx/linalg/graphmatrix.pyi | 0 .../networkx/linalg/laplacianmatrix.pyi | 0 .../networkx/linalg/modularitymatrix.pyi | 0 .../{2and3 => }/networkx/linalg/spectrum.pyi | 0 .../networkx/readwrite/__init__.pyi | 0 .../networkx/readwrite/adjlist.pyi | 0 .../networkx/readwrite/edgelist.pyi | 0 .../{2and3 => }/networkx/readwrite/gexf.pyi | 0 .../{2and3 => }/networkx/readwrite/gml.pyi | 0 .../networkx/readwrite/gpickle.pyi | 0 .../{2and3 => }/networkx/readwrite/graph6.pyi | 0 .../networkx/readwrite/graphml.pyi | 0 .../readwrite/json_graph/__init__.pyi | 0 .../readwrite/json_graph/adjacency.pyi | 0 .../readwrite/json_graph/cytoscape.pyi | 0 .../networkx/readwrite/json_graph/jit.pyi | 0 .../readwrite/json_graph/node_link.pyi | 0 .../networkx/readwrite/json_graph/tree.pyi | 0 .../{2and3 => }/networkx/readwrite/leda.pyi | 0 .../networkx/readwrite/multiline_adjlist.pyi | 0 .../{2and3 => }/networkx/readwrite/nx_shp.pyi | 0 .../networkx/readwrite/nx_yaml.pyi | 0 .../{2and3 => }/networkx/readwrite/p2g.pyi | 0 .../{2and3 => }/networkx/readwrite/pajek.pyi | 0 .../networkx/readwrite/sparse6.pyi | 0 typeshed/{2and3 => }/networkx/relabel.pyi | 0 typeshed/{2and3 => }/networkx/release.pyi | 0 .../{2and3 => }/networkx/tests/__init__.pyi | 0 typeshed/{2and3 => }/networkx/tests/test.pyi | 0 .../{2and3 => }/networkx/utils/__init__.pyi | 0 .../networkx/utils/contextmanagers.pyi | 0 .../{2and3 => }/networkx/utils/decorators.pyi | 0 typeshed/{2and3 => }/networkx/utils/heaps.pyi | 0 typeshed/{2and3 => }/networkx/utils/misc.pyi | 0 .../networkx/utils/random_sequence.pyi | 0 typeshed/{2and3 => }/networkx/utils/rcm.pyi | 0 .../{2and3 => }/networkx/utils/union_find.pyi | 0 typeshed/{2and3 => }/networkx/version.pyi | 0 typeshed/{2and3 => }/prov/__init__.pyi | 0 typeshed/{2and3 => }/prov/constants.pyi | 0 typeshed/{2and3 => }/prov/graph.pyi | 0 typeshed/{2and3 => }/prov/identifier.pyi | 0 typeshed/{2and3 => }/prov/model.pyi | 0 .../{2and3 => }/prov/serializers/__init__.pyi | 0 .../{2and3 => }/prov/serializers/provjson.pyi | 0 .../{2and3 => }/prov/serializers/provn.pyi | 0 .../{2and3 => }/prov/serializers/provrdf.pyi | 0 .../{2and3 => }/prov/serializers/provxml.pyi | 0 typeshed/{2and3 => }/psutil.pyi | 0 typeshed/{2and3 => }/rdflib/__init__.pyi | 0 typeshed/{2and3 => }/rdflib/events.pyi | 0 typeshed/{2and3 => }/rdflib/exceptions.pyi | 0 typeshed/{2and3 => }/rdflib/graph.pyi | 0 typeshed/{2and3 => }/rdflib/namespace.pyi | 0 typeshed/{2and3 => }/rdflib/parser.pyi | 0 typeshed/{2and3 => }/rdflib/plugin.pyi | 0 .../{2and3 => }/rdflib/plugins/__init__.pyi | 0 .../rdflib/plugins/parsers/__init__.pyi | 0 .../rdflib/plugins/parsers/notation3.pyi | 0 typeshed/{2and3 => }/rdflib/query.pyi | 0 typeshed/{2and3 => }/rdflib/serializer.pyi | 0 typeshed/{2and3 => }/rdflib/store.pyi | 0 typeshed/{2and3 => }/rdflib/term.pyi | 0 typeshed/{2and3 => }/rdflib/util.pyi | 0 typeshed/{2and3 => }/ruamel/__init__.py | 0 typeshed/{2and3 => }/shellescape/__init__.pyi | 0 typeshed/{2and3 => }/shellescape/main.pyi | 0 typeshed/subprocess.pyi | 1053 ++++++++++++++ 467 files changed, 1057 insertions(+), 4512 deletions(-) delete mode 100644 typeshed/2and3/ctypes/__init__.pyi delete mode 100644 typeshed/2and3/ctypes/util.pyi delete mode 100644 typeshed/2and3/ctypes/wintypes.pyi delete mode 100644 typeshed/2and3/distutils/__init__.pyi delete mode 100644 typeshed/2and3/distutils/spawn.pyi delete mode 100644 typeshed/2and3/msvcrt.pyi delete mode 100644 typeshed/2and3/pathlib2.pyi delete mode 100644 typeshed/2and3/select.pyi delete mode 100644 typeshed/2and3/uuid.pyi delete mode 100644 typeshed/3/logging/__init__.pyi delete mode 100644 typeshed/3/py/__init__.pyi delete mode 100644 typeshed/3/py/__metainfo.pyi delete mode 100644 typeshed/3/py/_builtin.pyi delete mode 100644 typeshed/3/py/_code/__init__.pyi delete mode 100644 typeshed/3/py/_code/_assertionnew.pyi delete mode 100644 typeshed/3/py/_code/_assertionold.pyi delete mode 100644 typeshed/3/py/_code/_py2traceback.pyi delete mode 100644 typeshed/3/py/_code/assertion.pyi delete mode 100644 typeshed/3/py/_code/code.pyi delete mode 100644 typeshed/3/py/_code/source.pyi delete mode 100644 typeshed/3/py/_error.pyi delete mode 100644 typeshed/3/py/_io/__init__.pyi delete mode 100644 typeshed/3/py/_io/capture.pyi delete mode 100644 typeshed/3/py/_io/saferepr.pyi delete mode 100644 typeshed/3/py/_io/terminalwriter.pyi delete mode 100644 typeshed/3/py/_log/__init__.pyi delete mode 100644 typeshed/3/py/_log/log.pyi delete mode 100644 typeshed/3/py/_log/warning.pyi delete mode 100644 typeshed/3/py/_path/__init__.pyi delete mode 100644 typeshed/3/py/_path/cacheutil.pyi delete mode 100644 typeshed/3/py/_path/common.pyi delete mode 100644 typeshed/3/py/_path/local.pyi delete mode 100644 typeshed/3/py/_path/svnurl.pyi delete mode 100644 typeshed/3/py/_path/svnwc.pyi delete mode 100644 typeshed/3/py/_process/__init__.pyi delete mode 100644 typeshed/3/py/_process/cmdexec.pyi delete mode 100644 typeshed/3/py/_process/forkedfunc.pyi delete mode 100644 typeshed/3/py/_process/killproc.pyi delete mode 100644 typeshed/3/py/_std.pyi delete mode 100644 typeshed/3/py/_version.pyi delete mode 100644 typeshed/3/py/_xmlgen.pyi delete mode 100644 typeshed/3/py/path.pyi delete mode 100644 typeshed/3/subprocess.pyi delete mode 100644 typeshed/3/typing.pyi rename typeshed/{3 => }/arcp/__init__.pyi (100%) rename typeshed/{3 => }/arcp/generate.pyi (100%) rename typeshed/{3 => }/arcp/parse.pyi (100%) rename typeshed/{3 => }/argcomplete/__init__.pyi (100%) rename typeshed/{3 => }/bagit.pyi (100%) rename typeshed/{2and3 => }/cachecontrol/__init__.pyi (100%) rename typeshed/{2and3 => }/cachecontrol/adapter.pyi (100%) rename typeshed/{2and3 => }/cachecontrol/cache.pyi (100%) rename typeshed/{2and3 => }/cachecontrol/caches/__init__.pyi (100%) rename typeshed/{2and3 => }/cachecontrol/caches/file_cache.pyi (100%) rename typeshed/{2and3 => }/cachecontrol/compat.pyi (100%) rename typeshed/{2and3 => }/cachecontrol/controller.pyi (100%) rename typeshed/{2and3 => }/cachecontrol/filewrapper.pyi (100%) rename typeshed/{2and3 => }/cachecontrol/serialize.pyi (100%) rename typeshed/{2and3 => }/cachecontrol/wrapper.pyi (100%) rename typeshed/{2and3 => }/coloredlogs/__init__.pyi (100%) rename typeshed/{2and3 => }/galaxy/__init__.pyi (100%) rename typeshed/{2and3 => }/galaxy/exceptions/__init__.pyi (100%) rename typeshed/{2and3 => }/galaxy/exceptions/error_codes.pyi (100%) rename typeshed/{2and3 => }/galaxy/jobs/__init__.pyi (100%) rename typeshed/{2and3 => }/galaxy/jobs/metrics/__init__.pyi (100%) rename typeshed/{2and3 => }/galaxy/jobs/metrics/collectl/__init__.pyi (100%) rename typeshed/{2and3 => }/galaxy/jobs/metrics/collectl/cli.pyi (100%) rename typeshed/{2and3 => }/galaxy/jobs/metrics/collectl/processes.pyi (100%) rename typeshed/{2and3 => }/galaxy/jobs/metrics/collectl/stats.pyi (100%) rename typeshed/{2and3 => }/galaxy/jobs/metrics/collectl/subsystems.pyi (100%) rename typeshed/{2and3 => }/galaxy/jobs/metrics/formatting.pyi (100%) rename typeshed/{2and3 => }/galaxy/jobs/metrics/instrumenters/__init__.pyi (100%) rename typeshed/{2and3 => }/galaxy/jobs/metrics/instrumenters/collectl.pyi (100%) rename typeshed/{2and3 => }/galaxy/jobs/metrics/instrumenters/core.pyi (100%) rename typeshed/{2and3 => }/galaxy/jobs/metrics/instrumenters/cpuinfo.pyi (100%) rename typeshed/{2and3 => }/galaxy/jobs/metrics/instrumenters/env.pyi (100%) rename typeshed/{2and3 => }/galaxy/jobs/metrics/instrumenters/meminfo.pyi (100%) rename typeshed/{2and3 => }/galaxy/jobs/metrics/instrumenters/uname.pyi (100%) rename typeshed/{2and3 => }/galaxy/objectstore/__init__.pyi (100%) rename typeshed/{2and3 => }/galaxy/objectstore/azure_blob.pyi (100%) rename typeshed/{2and3 => }/galaxy/objectstore/pulsar.pyi (100%) rename typeshed/{2and3 => }/galaxy/objectstore/rods.pyi (100%) rename typeshed/{2and3 => }/galaxy/objectstore/s3.pyi (100%) rename typeshed/{2and3 => }/galaxy/objectstore/s3_multipart_upload.pyi (100%) rename typeshed/{2and3 => }/galaxy/tool_util/__init__.pyi (100%) rename typeshed/{2and3 => }/galaxy/tool_util/cwl/__init__.pyi (100%) rename typeshed/{2and3 => }/galaxy/tool_util/cwl/cwltool_deps.pyi (100%) rename typeshed/{2and3 => }/galaxy/tool_util/cwl/parser.pyi (100%) rename typeshed/{2and3 => }/galaxy/tool_util/cwl/representation.pyi (100%) rename typeshed/{2and3 => }/galaxy/tool_util/cwl/runtime_actions.pyi (100%) rename typeshed/{2and3 => }/galaxy/tool_util/cwl/schema.pyi (100%) rename typeshed/{2and3 => }/galaxy/tool_util/deps/__init__.pyi (100%) rename typeshed/{2and3 => }/galaxy/tool_util/deps/brew_exts.pyi (100%) rename typeshed/{2and3 => }/galaxy/tool_util/deps/brew_util.pyi (100%) rename typeshed/{2and3 => }/galaxy/tool_util/deps/commands.pyi (100%) rename typeshed/{2and3 => }/galaxy/tool_util/deps/conda_compat.pyi (100%) rename typeshed/{2and3 => }/galaxy/tool_util/deps/conda_util.pyi (100%) rename typeshed/{2and3 => }/galaxy/tool_util/deps/container_resolvers/__init__.pyi (100%) rename typeshed/{2and3 => }/galaxy/tool_util/deps/container_resolvers/explicit.pyi (100%) rename typeshed/{2and3 => }/galaxy/tool_util/deps/container_resolvers/mulled.pyi (100%) rename typeshed/{2and3 => }/galaxy/tool_util/deps/containers.pyi (100%) rename typeshed/{2and3 => }/galaxy/tool_util/deps/dependencies.pyi (100%) rename typeshed/{2and3 => }/galaxy/tool_util/deps/docker_util.pyi (100%) rename typeshed/{2and3 => }/galaxy/tool_util/deps/dockerfiles.pyi (100%) rename typeshed/{2and3 => }/galaxy/tool_util/deps/installable.pyi (100%) rename typeshed/{2and3 => }/galaxy/tool_util/deps/mulled/__init__.pyi (100%) rename typeshed/{2and3 => }/galaxy/tool_util/deps/mulled/_cli.pyi (100%) rename typeshed/{2and3 => }/galaxy/tool_util/deps/mulled/mulled_build.pyi (100%) rename typeshed/{2and3 => }/galaxy/tool_util/deps/mulled/mulled_build_channel.pyi (100%) rename typeshed/{2and3 => }/galaxy/tool_util/deps/mulled/mulled_build_files.pyi (100%) rename typeshed/{2and3 => }/galaxy/tool_util/deps/mulled/mulled_build_tool.pyi (100%) rename typeshed/{2and3 => }/galaxy/tool_util/deps/mulled/mulled_search.pyi (100%) rename typeshed/{2and3 => }/galaxy/tool_util/deps/mulled/util.pyi (100%) rename typeshed/{2and3 => }/galaxy/tool_util/deps/requirements.pyi (100%) rename typeshed/{2and3 => }/galaxy/tool_util/deps/resolvers/__init__.pyi (100%) rename typeshed/{2and3 => }/galaxy/tool_util/deps/resolvers/brewed_tool_shed_packages.pyi (100%) rename typeshed/{2and3 => }/galaxy/tool_util/deps/resolvers/conda.pyi (100%) rename typeshed/{2and3 => }/galaxy/tool_util/deps/resolvers/galaxy_packages.pyi (100%) rename typeshed/{2and3 => }/galaxy/tool_util/deps/resolvers/homebrew.pyi (100%) rename typeshed/{2and3 => }/galaxy/tool_util/deps/resolvers/modules.pyi (100%) rename typeshed/{2and3 => }/galaxy/tool_util/deps/resolvers/resolver_mixins.pyi (100%) rename typeshed/{2and3 => }/galaxy/tool_util/deps/resolvers/tool_shed_packages.pyi (100%) rename typeshed/{2and3 => }/galaxy/tool_util/deps/resolvers/unlinked_tool_shed_packages.pyi (100%) rename typeshed/{2and3 => }/galaxy/tool_util/deps/singularity_util.pyi (100%) rename typeshed/{2and3 => }/galaxy/tool_util/deps/views.pyi (100%) rename typeshed/{2and3 => }/galaxy/tool_util/fetcher.pyi (100%) rename typeshed/{2and3 => }/galaxy/tool_util/lint.pyi (100%) rename typeshed/{2and3 => }/galaxy/tool_util/lint_util.pyi (100%) rename typeshed/{2and3 => }/galaxy/tool_util/linters/__init__.pyi (100%) rename typeshed/{2and3 => }/galaxy/tool_util/linters/citations.pyi (100%) rename typeshed/{2and3 => }/galaxy/tool_util/linters/command.pyi (100%) rename typeshed/{2and3 => }/galaxy/tool_util/linters/cwl.pyi (100%) rename typeshed/{2and3 => }/galaxy/tool_util/linters/general.pyi (100%) rename typeshed/{2and3 => }/galaxy/tool_util/linters/help.pyi (100%) rename typeshed/{2and3 => }/galaxy/tool_util/linters/inputs.pyi (100%) rename typeshed/{2and3 => }/galaxy/tool_util/linters/outputs.pyi (100%) rename typeshed/{2and3 => }/galaxy/tool_util/linters/stdio.pyi (100%) rename typeshed/{2and3 => }/galaxy/tool_util/linters/tests.pyi (100%) rename typeshed/{2and3 => }/galaxy/tool_util/linters/xml_order.pyi (100%) rename typeshed/{2and3 => }/galaxy/tool_util/loader.pyi (100%) rename typeshed/{2and3 => }/galaxy/tool_util/loader_directory.pyi (100%) rename typeshed/{2and3 => }/galaxy/tool_util/locations/__init__.pyi (100%) rename typeshed/{2and3 => }/galaxy/tool_util/locations/dockstore.pyi (100%) rename typeshed/{2and3 => }/galaxy/tool_util/locations/file.pyi (100%) rename typeshed/{2and3 => }/galaxy/tool_util/locations/http.pyi (100%) rename typeshed/{2and3 => }/galaxy/tool_util/parser/__init__.pyi (100%) rename typeshed/{2and3 => }/galaxy/tool_util/parser/cwl.pyi (100%) rename typeshed/{2and3 => }/galaxy/tool_util/parser/factory.pyi (100%) rename typeshed/{2and3 => }/galaxy/tool_util/parser/interface.pyi (100%) rename typeshed/{2and3 => }/galaxy/tool_util/parser/output_actions.pyi (100%) rename typeshed/{2and3 => }/galaxy/tool_util/parser/output_collection_def.pyi (100%) rename typeshed/{2and3 => }/galaxy/tool_util/parser/output_objects.pyi (100%) rename typeshed/{2and3 => }/galaxy/tool_util/parser/util.pyi (100%) rename typeshed/{2and3 => }/galaxy/tool_util/parser/xml.pyi (100%) rename typeshed/{2and3 => }/galaxy/tool_util/parser/yaml.pyi (100%) rename typeshed/{2and3 => }/galaxy/tool_util/verify/__init__.pyi (100%) rename typeshed/{2and3 => }/galaxy/tool_util/verify/asserts/__init__.pyi (100%) rename typeshed/{2and3 => }/galaxy/tool_util/verify/asserts/tabular.pyi (100%) rename typeshed/{2and3 => }/galaxy/tool_util/verify/asserts/text.pyi (100%) rename typeshed/{2and3 => }/galaxy/tool_util/verify/asserts/xml.pyi (100%) rename typeshed/{2and3 => }/galaxy/tool_util/verify/test_data.pyi (100%) rename typeshed/{2and3 => }/galaxy/util/__init__.pyi (100%) rename typeshed/{2and3 => }/galaxy/util/aliaspickler.pyi (100%) rename typeshed/{2and3 => }/galaxy/util/bunch.pyi (100%) rename typeshed/{2and3 => }/galaxy/util/checkers.pyi (100%) rename typeshed/{2and3 => }/galaxy/util/compression_utils.pyi (100%) rename typeshed/{2and3 => }/galaxy/util/dictifiable.pyi (100%) rename typeshed/{2and3 => }/galaxy/util/expressions.pyi (100%) rename typeshed/{2and3 => }/galaxy/util/filelock.pyi (100%) rename typeshed/{2and3 => }/galaxy/util/hash_util.pyi (100%) rename typeshed/{2and3 => }/galaxy/util/heartbeat.pyi (100%) rename typeshed/{2and3 => }/galaxy/util/image_util.pyi (100%) rename typeshed/{2and3 => }/galaxy/util/inflection.pyi (100%) rename typeshed/{2and3 => }/galaxy/util/json.pyi (100%) rename typeshed/{2and3 => }/galaxy/util/lazy_process.pyi (100%) rename typeshed/{2and3 => }/galaxy/util/object_wrapper.pyi (100%) rename typeshed/{2and3 => }/galaxy/util/odict.pyi (100%) rename typeshed/{2and3 => }/galaxy/util/oset.pyi (100%) rename typeshed/{2and3 => }/galaxy/util/plugin_config.pyi (100%) rename typeshed/{2and3 => }/galaxy/util/properties.pyi (100%) rename typeshed/{2and3 => }/galaxy/util/simplegraph.pyi (100%) rename typeshed/{2and3 => }/galaxy/util/sleeper.pyi (100%) rename typeshed/{2and3 => }/galaxy/util/sockets.pyi (100%) rename typeshed/{2and3 => }/galaxy/util/specs.pyi (100%) rename typeshed/{2and3 => }/galaxy/util/sqlite.pyi (100%) rename typeshed/{2and3 => }/galaxy/util/submodules.pyi (100%) rename typeshed/{2and3 => }/galaxy/util/topsort.pyi (100%) rename typeshed/{2and3 => }/galaxy/util/xml_macros.pyi (100%) rename typeshed/{2and3 => }/graphviz/__init__.pyi (100%) rename typeshed/{2and3 => }/graphviz/_compat.pyi (100%) rename typeshed/{2and3 => }/graphviz/backend.pyi (100%) rename typeshed/{2and3 => }/graphviz/dot.pyi (100%) rename typeshed/{2and3 => }/graphviz/files.pyi (100%) rename typeshed/{2and3 => }/graphviz/lang.pyi (100%) rename typeshed/{2and3 => }/graphviz/tools.pyi (100%) rename typeshed/{2and3 => }/mistune.pyi (100%) rename typeshed/{2and3 => }/networkx/__init__.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/__init__.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/approximation/__init__.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/approximation/clique.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/approximation/clustering_coefficient.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/approximation/connectivity.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/approximation/dominating_set.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/approximation/independent_set.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/approximation/kcomponents.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/approximation/matching.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/approximation/ramsey.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/approximation/steinertree.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/approximation/vertex_cover.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/assortativity/__init__.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/assortativity/connectivity.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/assortativity/correlation.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/assortativity/mixing.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/assortativity/neighbor_degree.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/assortativity/pairs.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/bipartite/__init__.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/bipartite/basic.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/bipartite/centrality.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/bipartite/cluster.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/bipartite/covering.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/bipartite/edgelist.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/bipartite/generators.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/bipartite/matching.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/bipartite/matrix.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/bipartite/projection.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/bipartite/redundancy.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/bipartite/spectral.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/boundary.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/bridges.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/centrality/betweenness.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/centrality/betweenness_subset.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/centrality/closeness.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/centrality/current_flow_betweenness.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/centrality/current_flow_betweenness_subset.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/centrality/current_flow_closeness.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/centrality/degree_alg.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/centrality/dispersion.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/centrality/eigenvector.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/centrality/flow_matrix.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/centrality/harmonic.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/centrality/katz.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/centrality/load.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/centrality/reaching.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/centrality/subgraph_alg.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/chains.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/chordal.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/clique.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/cluster.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/coloring/__init__.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/coloring/greedy_coloring.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/coloring/greedy_coloring_with_interchange.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/communicability_alg.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/community/__init__.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/community/asyn_fluidc.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/community/centrality.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/community/community_generators.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/community/community_utils.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/community/kclique.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/community/kernighan_lin.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/community/label_propagation.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/community/quality.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/components/__init__.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/components/attracting.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/components/biconnected.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/components/connected.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/components/semiconnected.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/components/strongly_connected.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/components/weakly_connected.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/connectivity/__init__.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/connectivity/connectivity.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/connectivity/cuts.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/connectivity/disjoint_paths.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/connectivity/edge_augmentation.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/connectivity/edge_kcomponents.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/connectivity/kcomponents.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/connectivity/kcutsets.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/connectivity/stoerwagner.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/connectivity/utils.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/core.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/covering.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/cuts.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/cycles.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/dag.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/distance_measures.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/distance_regular.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/dominance.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/dominating.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/efficiency.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/euler.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/flow/__init__.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/flow/boykovkolmogorov.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/flow/capacityscaling.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/flow/dinitz_alg.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/flow/edmondskarp.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/flow/gomory_hu.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/flow/maxflow.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/flow/mincost.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/flow/networksimplex.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/flow/preflowpush.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/flow/shortestaugmentingpath.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/flow/utils.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/graphical.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/hierarchy.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/hybrid.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/isolate.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/isomorphism/__init__.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/isomorphism/isomorph.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/isomorphism/isomorphvf2.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/isomorphism/matchhelpers.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/isomorphism/temporalisomorphvf2.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/isomorphism/vf2userfunc.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/link_analysis/__init__.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/link_analysis/hits_alg.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/link_analysis/pagerank_alg.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/link_prediction.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/lowest_common_ancestors.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/matching.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/minors.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/mis.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/operators/__init__.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/operators/all.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/operators/binary.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/operators/product.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/operators/unary.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/reciprocity.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/richclub.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/shortest_paths/__init__.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/shortest_paths/astar.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/shortest_paths/dense.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/shortest_paths/generic.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/shortest_paths/unweighted.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/shortest_paths/weighted.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/similarity.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/simple_paths.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/smetric.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/structuralholes.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/swap.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/threshold.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/tournament.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/traversal/__init__.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/traversal/beamsearch.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/traversal/breadth_first_search.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/traversal/depth_first_search.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/traversal/edgedfs.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/tree/__init__.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/tree/branchings.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/tree/coding.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/tree/mst.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/tree/operations.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/tree/recognition.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/triads.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/vitality.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/voronoi.pyi (100%) rename typeshed/{2and3 => }/networkx/algorithms/wiener.pyi (100%) rename typeshed/{2and3 => }/networkx/classes/__init__.pyi (100%) rename typeshed/{2and3 => }/networkx/classes/coreviews.pyi (100%) rename typeshed/{2and3 => }/networkx/classes/digraph.pyi (100%) rename typeshed/{2and3 => }/networkx/classes/filters.pyi (100%) rename typeshed/{2and3 => }/networkx/classes/function.pyi (100%) rename typeshed/{2and3 => }/networkx/classes/graph.pyi (100%) rename typeshed/{2and3 => }/networkx/classes/graphviews.pyi (100%) rename typeshed/{2and3 => }/networkx/classes/multidigraph.pyi (100%) rename typeshed/{2and3 => }/networkx/classes/multigraph.pyi (100%) rename typeshed/{2and3 => }/networkx/classes/ordered.pyi (100%) rename typeshed/{2and3 => }/networkx/classes/reportviews.pyi (100%) rename typeshed/{2and3 => }/networkx/convert.pyi (100%) rename typeshed/{2and3 => }/networkx/convert_matrix.pyi (100%) rename typeshed/{2and3 => }/networkx/drawing/__init__.pyi (100%) rename typeshed/{2and3 => }/networkx/drawing/layout.pyi (100%) rename typeshed/{2and3 => }/networkx/drawing/nx_agraph.pyi (100%) rename typeshed/{2and3 => }/networkx/drawing/nx_pydot.pyi (100%) rename typeshed/{2and3 => }/networkx/drawing/nx_pylab.pyi (100%) rename typeshed/{2and3 => }/networkx/exception.pyi (100%) rename typeshed/{2and3 => }/networkx/generators/atlas.pyi (100%) rename typeshed/{2and3 => }/networkx/generators/classic.pyi (100%) rename typeshed/{2and3 => }/networkx/generators/community.pyi (100%) rename typeshed/{2and3 => }/networkx/generators/degree_seq.pyi (100%) rename typeshed/{2and3 => }/networkx/generators/directed.pyi (100%) rename typeshed/{2and3 => }/networkx/generators/duplication.pyi (100%) rename typeshed/{2and3 => }/networkx/generators/ego.pyi (100%) rename typeshed/{2and3 => }/networkx/generators/expanders.pyi (100%) rename typeshed/{2and3 => }/networkx/generators/geometric.pyi (100%) rename typeshed/{2and3 => }/networkx/generators/intersection.pyi (100%) rename typeshed/{2and3 => }/networkx/generators/joint_degree_seq.pyi (100%) rename typeshed/{2and3 => }/networkx/generators/lattice.pyi (100%) rename typeshed/{2and3 => }/networkx/generators/line.pyi (100%) rename typeshed/{2and3 => }/networkx/generators/mycielski.pyi (100%) rename typeshed/{2and3 => }/networkx/generators/nonisomorphic_trees.pyi (100%) rename typeshed/{2and3 => }/networkx/generators/random_clustered.pyi (100%) rename typeshed/{2and3 => }/networkx/generators/random_graphs.pyi (100%) rename typeshed/{2and3 => }/networkx/generators/small.pyi (100%) rename typeshed/{2and3 => }/networkx/generators/social.pyi (100%) rename typeshed/{2and3 => }/networkx/generators/stochastic.pyi (100%) rename typeshed/{2and3 => }/networkx/generators/trees.pyi (100%) rename typeshed/{2and3 => }/networkx/generators/triads.pyi (100%) rename typeshed/{2and3 => }/networkx/linalg/__init__.pyi (100%) rename typeshed/{2and3 => }/networkx/linalg/algebraicconnectivity.pyi (100%) rename typeshed/{2and3 => }/networkx/linalg/attrmatrix.pyi (100%) rename typeshed/{2and3 => }/networkx/linalg/graphmatrix.pyi (100%) rename typeshed/{2and3 => }/networkx/linalg/laplacianmatrix.pyi (100%) rename typeshed/{2and3 => }/networkx/linalg/modularitymatrix.pyi (100%) rename typeshed/{2and3 => }/networkx/linalg/spectrum.pyi (100%) rename typeshed/{2and3 => }/networkx/readwrite/__init__.pyi (100%) rename typeshed/{2and3 => }/networkx/readwrite/adjlist.pyi (100%) rename typeshed/{2and3 => }/networkx/readwrite/edgelist.pyi (100%) rename typeshed/{2and3 => }/networkx/readwrite/gexf.pyi (100%) rename typeshed/{2and3 => }/networkx/readwrite/gml.pyi (100%) rename typeshed/{2and3 => }/networkx/readwrite/gpickle.pyi (100%) rename typeshed/{2and3 => }/networkx/readwrite/graph6.pyi (100%) rename typeshed/{2and3 => }/networkx/readwrite/graphml.pyi (100%) rename typeshed/{2and3 => }/networkx/readwrite/json_graph/__init__.pyi (100%) rename typeshed/{2and3 => }/networkx/readwrite/json_graph/adjacency.pyi (100%) rename typeshed/{2and3 => }/networkx/readwrite/json_graph/cytoscape.pyi (100%) rename typeshed/{2and3 => }/networkx/readwrite/json_graph/jit.pyi (100%) rename typeshed/{2and3 => }/networkx/readwrite/json_graph/node_link.pyi (100%) rename typeshed/{2and3 => }/networkx/readwrite/json_graph/tree.pyi (100%) rename typeshed/{2and3 => }/networkx/readwrite/leda.pyi (100%) rename typeshed/{2and3 => }/networkx/readwrite/multiline_adjlist.pyi (100%) rename typeshed/{2and3 => }/networkx/readwrite/nx_shp.pyi (100%) rename typeshed/{2and3 => }/networkx/readwrite/nx_yaml.pyi (100%) rename typeshed/{2and3 => }/networkx/readwrite/p2g.pyi (100%) rename typeshed/{2and3 => }/networkx/readwrite/pajek.pyi (100%) rename typeshed/{2and3 => }/networkx/readwrite/sparse6.pyi (100%) rename typeshed/{2and3 => }/networkx/relabel.pyi (100%) rename typeshed/{2and3 => }/networkx/release.pyi (100%) rename typeshed/{2and3 => }/networkx/tests/__init__.pyi (100%) rename typeshed/{2and3 => }/networkx/tests/test.pyi (100%) rename typeshed/{2and3 => }/networkx/utils/__init__.pyi (100%) rename typeshed/{2and3 => }/networkx/utils/contextmanagers.pyi (100%) rename typeshed/{2and3 => }/networkx/utils/decorators.pyi (100%) rename typeshed/{2and3 => }/networkx/utils/heaps.pyi (100%) rename typeshed/{2and3 => }/networkx/utils/misc.pyi (100%) rename typeshed/{2and3 => }/networkx/utils/random_sequence.pyi (100%) rename typeshed/{2and3 => }/networkx/utils/rcm.pyi (100%) rename typeshed/{2and3 => }/networkx/utils/union_find.pyi (100%) rename typeshed/{2and3 => }/networkx/version.pyi (100%) rename typeshed/{2and3 => }/prov/__init__.pyi (100%) rename typeshed/{2and3 => }/prov/constants.pyi (100%) rename typeshed/{2and3 => }/prov/graph.pyi (100%) rename typeshed/{2and3 => }/prov/identifier.pyi (100%) rename typeshed/{2and3 => }/prov/model.pyi (100%) rename typeshed/{2and3 => }/prov/serializers/__init__.pyi (100%) rename typeshed/{2and3 => }/prov/serializers/provjson.pyi (100%) rename typeshed/{2and3 => }/prov/serializers/provn.pyi (100%) rename typeshed/{2and3 => }/prov/serializers/provrdf.pyi (100%) rename typeshed/{2and3 => }/prov/serializers/provxml.pyi (100%) rename typeshed/{2and3 => }/psutil.pyi (100%) rename typeshed/{2and3 => }/rdflib/__init__.pyi (100%) rename typeshed/{2and3 => }/rdflib/events.pyi (100%) rename typeshed/{2and3 => }/rdflib/exceptions.pyi (100%) rename typeshed/{2and3 => }/rdflib/graph.pyi (100%) rename typeshed/{2and3 => }/rdflib/namespace.pyi (100%) rename typeshed/{2and3 => }/rdflib/parser.pyi (100%) rename typeshed/{2and3 => }/rdflib/plugin.pyi (100%) rename typeshed/{2and3 => }/rdflib/plugins/__init__.pyi (100%) rename typeshed/{2and3 => }/rdflib/plugins/parsers/__init__.pyi (100%) rename typeshed/{2and3 => }/rdflib/plugins/parsers/notation3.pyi (100%) rename typeshed/{2and3 => }/rdflib/query.pyi (100%) rename typeshed/{2and3 => }/rdflib/serializer.pyi (100%) rename typeshed/{2and3 => }/rdflib/store.pyi (100%) rename typeshed/{2and3 => }/rdflib/term.pyi (100%) rename typeshed/{2and3 => }/rdflib/util.pyi (100%) rename typeshed/{2and3 => }/ruamel/__init__.py (100%) rename typeshed/{2and3 => }/shellescape/__init__.pyi (100%) rename typeshed/{2and3 => }/shellescape/main.pyi (100%) create mode 100644 typeshed/subprocess.pyi diff --git a/Makefile b/Makefile index c4e8fee467..c11256d6bf 100644 --- a/Makefile +++ b/Makefile @@ -160,16 +160,14 @@ mypy3: mypy mypy: $(filter-out setup.py gittagger.py,${PYSOURCES}) if ! test -f $(shell python3 -c 'import ruamel.yaml; import os.path; print(os.path.dirname(ruamel.yaml.__file__))')/py.typed ; \ then \ - rm -Rf typeshed/2and3/ruamel/yaml ; \ + rm -Rf typeshed/ruamel/yaml ; \ ln -s $(shell python3 -c 'import ruamel.yaml; import os.path; print(os.path.dirname(ruamel.yaml.__file__))') \ - typeshed/2and3/ruamel/ ; \ + typeshed/ruamel/ ; \ fi # if minimally required ruamel.yaml version is 0.15.99 or greater, than the above can be removed - MYPYPATH=$$MYPYPATH:typeshed/3:typeshed/2and3 mypy --disallow-untyped-calls \ - --warn-redundant-casts \ - $^ + MYPYPATH=$$MYPYPATH:typeshed mypy $^ mypyc: ${PYSOURCES} - MYPYPATH=typeshed/2and3/:typeshed/3 CWLTOOL_USE_MYPYC=1 pip install --verbose -e . && pytest --ignore cwltool/schemas --basetemp ./tmp + MYPYPATH=typeshed CWLTOOL_USE_MYPYC=1 pip install --verbose -e . && pytest --ignore cwltool/schemas --basetemp ./tmp shellcheck: FORCE shellcheck build-cwl-docker.sh cwl-docker.sh release-test.sh conformance-test.sh \ diff --git a/typeshed/2and3/ctypes/__init__.pyi b/typeshed/2and3/ctypes/__init__.pyi deleted file mode 100644 index 8bd538af69..0000000000 --- a/typeshed/2and3/ctypes/__init__.pyi +++ /dev/null @@ -1,277 +0,0 @@ -# Stubs for ctypes - -from typing import ( - Any, Callable, ClassVar, Iterator, Iterable, List, Mapping, Optional, Sequence, Sized, Text, - Tuple, Type, Generic, TypeVar, overload, -) -from typing import Union as _UnionT -import sys - -_T = TypeVar('_T') -_DLLT = TypeVar('_DLLT', bound=CDLL) -_CT = TypeVar('_CT', bound=_CData) - - -RTLD_GLOBAL: int = ... -RTLD_LOCAL: int = ... -DEFAULT_MODE: int = ... - - -class CDLL(object): - _func_flags_: ClassVar[int] = ... - _func_restype_: ClassVar[_CData] = ... - _name: str = ... - _handle: int = ... - _FuncPtr: Type[_FuncPointer] = ... - def __init__(self, name: str, mode: int = ..., handle: Optional[int] = ..., - use_errno: bool = ..., use_last_error: bool = ...) -> None: ... - def __getattr__(self, name: str) -> _FuncPointer: ... - def __getitem__(self, name: str) -> _FuncPointer: ... -if sys.platform == 'win32': - class OleDLL(CDLL): ... - class WinDLL(CDLL): ... -class PyDLL(CDLL): ... - -class LibraryLoader(Generic[_DLLT]): - def __init__(self, dlltype: Type[_DLLT]) -> None: ... - def __getattr__(self, name: str) -> _DLLT: ... - def __getitem__(self, name: str) -> _DLLT: ... - def LoadLibrary(self, name: str) -> _DLLT: ... - -cdll: LibraryLoader[CDLL] = ... -if sys.platform == 'win32': - windll: LibraryLoader[WinDLL] = ... - oledll: LibraryLoader[OleDLL] = ... -pydll: LibraryLoader[PyDLL] = ... -pythonapi: PyDLL = ... - -class _CDataMeta(type): - # By default mypy complains about the following two methods, because strictly speaking cls - # might not be a Type[_CT]. However this can never actually happen, because the only class that - # uses _CDataMeta as its metaclass is _CData. So it's safe to ignore the errors here. - def __mul__(cls: Type[_CT], other: int) -> Type[Array[_CT]]: ... # type: ignore - def __rmul__(cls: Type[_CT], other: int) -> Type[Array[_CT]]: ... # type: ignore -class _CData(metaclass=_CDataMeta): - _b_base: int = ... - _b_needsfree_: bool = ... - _objects: Optional[Mapping[Any, int]] = ... - @classmethod - def from_buffer(cls: Type[_CT], source: bytearray, offset: int = ...) -> _CT: ... - @classmethod - def from_buffer_copy(cls: Type[_CT], source: bytearray, offset: int = ...) -> _CT: ... - @classmethod - def from_address(cls: Type[_CT], address: int) -> _CT: ... - @classmethod - def from_param(cls: Type[_CT], obj: Any) -> _UnionT[_CT, _CArgObject]: ... - @classmethod - def in_dll(cls: Type[_CT], library: CDLL, name: str) -> _CT: ... - -class _PointerLike(_CData): ... - -_ECT = Callable[[Optional[Type[_CData]], - _FuncPointer, - Tuple[_CData, ...]], - _CData] -_PF = _UnionT[ - Tuple[int], - Tuple[int, str], - Tuple[int, str, Any] -] -class _FuncPointer(_PointerLike, _CData): - restype: _UnionT[Type[_CData], Callable[[int], None], None] = ... - argtypes: Sequence[Type[_CData]] = ... - errcheck: _ECT = ... - @overload - def __init__(self, address: int) -> None: ... - @overload - def __init__(self, callable: Callable[..., Any]) -> None: ... - @overload - def __init__(self, func_spec: Tuple[_UnionT[str, int], CDLL], - paramflags: Tuple[_PF, ...] = ...) -> None: ... - @overload - def __init__(self, vtlb_index: int, name: str, - paramflags: Tuple[_PF, ...] = ..., - iid: pointer[c_int] = ...) -> None: ... - def __call__(self, *args: Any, **kwargs: Any) -> Any: ... - -class ArgumentError(Exception): ... - - -def CFUNCTYPE(restype: Optional[Type[_CData]], - *argtypes: Type[_CData], - use_errno: bool = ..., - use_last_error: bool = ...) -> Type[_FuncPointer]: ... -if sys.platform == 'win32': - def WINFUNCTYPE(restype: Optional[Type[_CData]], - *argtypes: Type[_CData], - use_errno: bool = ..., - use_last_error: bool = ...) -> Type[_FuncPointer]: ... -def PYFUNCTYPE(restype: Optional[Type[_CData]], - *argtypes: Type[_CData]) -> Type[_FuncPointer]: ... - -class _CArgObject: ... - -# Any type that can be implicitly converted to c_void_p when passed as a C function argument. -# (bytes is not included here, see below.) -_CVoidPLike = _UnionT[_PointerLike, Array[Any], _CArgObject, int] -# Same as above, but including types known to be read-only (i. e. bytes). -# This distinction is not strictly necessary (ctypes doesn't differentiate between const -# and non-const pointers), but it catches errors like memmove(b'foo', buf, 4) -# when memmove(buf, b'foo', 4) was intended. -_CVoidConstPLike = _UnionT[_CVoidPLike, bytes] - -def addressof(obj: _CData) -> int: ... -def alignment(obj_or_type: _UnionT[_CData, Type[_CData]]) -> int: ... -def byref(obj: _CData, offset: int = ...) -> _CArgObject: ... -_PT = TypeVar('_PT', bound=_PointerLike) -def cast(obj: _UnionT[_CData, _CArgObject], type: Type[_PT]) -> _PT: ... -def create_string_buffer(init_or_size: _UnionT[int, bytes], - size: Optional[int] = ...) -> Array[c_char]: ... -c_buffer = create_string_buffer -def create_unicode_buffer(init_or_size: _UnionT[int, Text], - size: Optional[int] = ...) -> Array[c_wchar]: ... -if sys.platform == 'win32': - def DllCanUnloadNow() -> int: ... - def DllGetClassObject(rclsid: Any, riid: Any, ppv: Any) -> int: ... # TODO not documented - def FormatError(code: int) -> str: ... - def GetLastError() -> int: ... -def get_errno() -> int: ... -if sys.platform == 'win32': - def get_last_error() -> int: ... -def memmove(dst: _CVoidPLike, src: _CVoidConstPLike, count: int) -> None: ... -def memset(dst: _CVoidPLike, c: int, count: int) -> None: ... -def POINTER(type: Type[_CT]) -> Type[pointer[_CT]]: ... - -# The real ctypes.pointer is a function, not a class. The stub version of pointer behaves like -# ctypes._Pointer in that it is the base class for all pointer types. Unlike the real _Pointer, -# it can be instantiated directly (to mimic the behavior of the real pointer function). -class pointer(Generic[_CT], _PointerLike, _CData): - _type_: ClassVar[Type[_CT]] = ... - contents: _CT = ... - def __init__(self, arg: _CT = ...) -> None: ... - @overload - def __getitem__(self, i: int) -> _CT: ... - @overload - def __getitem__(self, s: slice) -> List[_CT]: ... - @overload - def __setitem__(self, i: int, o: _CT) -> None: ... - @overload - def __setitem__(self, s: slice, o: Iterable[_CT]) -> None: ... - -def resize(obj: _CData, size: int) -> None: ... -if sys.version_info < (3,): - def set_conversion_mode(encoding: str, errors: str) -> Tuple[str, str]: ... -def set_errno(value: int) -> int: ... -if sys.platform == 'win32': - def set_last_error(value: int) -> int: ... -def sizeof(obj_or_type: _UnionT[_CData, Type[_CData]]) -> int: ... -def string_at(address: _CVoidConstPLike, size: int = ...) -> bytes: ... -if sys.platform == 'win32': - def WinError(code: Optional[int] = ..., - desc: Optional[str] = ...) -> WindowsError: ... -def wstring_at(address: _CVoidConstPLike, size: int = ...) -> str: ... - -class _SimpleCData(Generic[_T], _CData): - value: _T = ... - def __init__(self, value: _T = ...) -> None: ... - -class c_byte(_SimpleCData[int]): ... - -class c_char(_SimpleCData[bytes]): - def __init__(self, value: _UnionT[int, bytes] = ...) -> None: ... -class c_char_p(_PointerLike, _SimpleCData[Optional[bytes]]): - def __init__(self, value: Optional[_UnionT[int, bytes]] = ...) -> None: ... - -class c_double(_SimpleCData[float]): ... -class c_longdouble(_SimpleCData[float]): ... -class c_float(_SimpleCData[float]): ... - -class c_int(_SimpleCData[int]): ... -class c_int8(_SimpleCData[int]): ... -class c_int16(_SimpleCData[int]): ... -class c_int32(_SimpleCData[int]): ... -class c_int64(_SimpleCData[int]): ... - -class c_long(_SimpleCData[int]): ... -class c_longlong(_SimpleCData[int]): ... - -class c_short(_SimpleCData[int]): ... - -class c_size_t(_SimpleCData[int]): ... -class c_ssize_t(_SimpleCData[int]): ... - -class c_ubyte(_SimpleCData[int]): ... - -class c_uint(_SimpleCData[int]): ... -class c_uint8(_SimpleCData[int]): ... -class c_uint16(_SimpleCData[int]): ... -class c_uint32(_SimpleCData[int]): ... -class c_uint64(_SimpleCData[int]): ... - -class c_ulong(_SimpleCData[int]): ... -class c_ulonglong(_SimpleCData[int]): ... - -class c_ushort(_SimpleCData[int]): ... - -class c_void_p(_PointerLike, _SimpleCData[Optional[int]]): ... - -class c_wchar(_SimpleCData[Text]): ... -class c_wchar_p(_PointerLike, _SimpleCData[Optional[Text]]): - def __init__(self, value: Optional[_UnionT[int, Text]] = ...) -> None: ... - -class c_bool(_SimpleCData[bool]): - def __init__(self, value: bool) -> None: ... - -if sys.platform == 'win32': - class HRESULT(_SimpleCData[int]): ... # TODO undocumented - -class py_object(_SimpleCData[_T]): ... - -class _CField: - offset: int = ... - size: int = ... -class _StructUnionMeta(_CDataMeta): - _fields_: Sequence[_UnionT[Tuple[str, Type[_CData]], Tuple[str, Type[_CData], int]]] = ... - _pack_: int = ... - _anonymous_: Sequence[str] = ... - def __getattr__(self, name: str) -> _CField: ... -class _StructUnionBase(_CData, metaclass=_StructUnionMeta): - def __init__(self, *args: Any, **kw: Any) -> None: ... - def __getattr__(self, name: str) -> Any: ... - def __setattr__(self, name: str, value: Any) -> None: ... - -class Union(_StructUnionBase): ... -class Structure(_StructUnionBase): ... -class BigEndianStructure(Structure): ... -class LittleEndianStructure(Structure): ... - -class Array(Generic[_T], _CData): - _length_: ClassVar[int] = ... - _type_: ClassVar[Type[_T]] = ... - raw: bytes = ... # TODO only available with _T == c_char - value: bytes = ... # TODO only available with _T == c_char - # TODO These methods cannot be annotated correctly at the moment. - # All of these "Any"s stand for the array's element type, but it's not possible to use _T here, - # because of a special feature of ctypes. - # By default, when accessing an element of an Array[_T], the returned object has type _T. - # However, when _T is a "simple type" like c_int, ctypes automatically "unboxes" the object - # and converts it to the corresponding Python primitive. For example, when accessing an element - # of an Array[c_int], a Python int object is returned, not a c_int. - # This behavior does *not* apply to subclasses of "simple types". - # If MyInt is a subclass of c_int, then accessing an element of an Array[MyInt] returns - # a MyInt, not an int. - # This special behavior is not easy to model in a stub, so for now all places where - # the array element type would belong are annotated with Any instead. - def __init__(self, *args: Any) -> None: ... - @overload - def __getitem__(self, i: int) -> Any: ... - @overload - def __getitem__(self, s: slice) -> List[Any]: ... - @overload - def __setitem__(self, i: int, o: Any) -> None: ... - @overload - def __setitem__(self, s: slice, o: Iterable[Any]) -> None: ... - def __iter__(self) -> Iterator[Any]: ... - # Can't inherit from Sized because the metaclass conflict between - # Sized and _CData prevents using _CDataMeta. - def __len__(self) -> int: ... diff --git a/typeshed/2and3/ctypes/util.pyi b/typeshed/2and3/ctypes/util.pyi deleted file mode 100644 index 7077d9d2f1..0000000000 --- a/typeshed/2and3/ctypes/util.pyi +++ /dev/null @@ -1,8 +0,0 @@ -# Stubs for ctypes.util - -from typing import Optional -import sys - -def find_library(name: str) -> Optional[str]: ... -if sys.platform == 'win32': - def find_msvcrt() -> Optional[str]: ... diff --git a/typeshed/2and3/ctypes/wintypes.pyi b/typeshed/2and3/ctypes/wintypes.pyi deleted file mode 100644 index c5a6226b25..0000000000 --- a/typeshed/2and3/ctypes/wintypes.pyi +++ /dev/null @@ -1,209 +0,0 @@ -from ctypes import ( - _SimpleCData, Array, Structure, c_byte, c_char, c_char_p, c_double, c_float, c_int, c_long, - c_longlong, c_short, c_uint, c_ulong, c_ulonglong, c_ushort, c_void_p, c_wchar, c_wchar_p, - pointer, -) - -BYTE = c_byte -WORD = c_ushort -DWORD = c_ulong -CHAR = c_char -WCHAR = c_wchar -UINT = c_uint -INT = c_int -DOUBLE = c_double -FLOAT = c_float -BOOLEAN = BYTE -BOOL = c_long -class VARIANT_BOOL(_SimpleCData[bool]): ... -ULONG = c_ulong -LONG = c_long -USHORT = c_ushort -SHORT = c_short -LARGE_INTEGER = c_longlong -_LARGE_INTEGER = c_longlong -ULARGE_INTEGER = c_ulonglong -_ULARGE_INTEGER = c_ulonglong - -OLESTR = c_wchar_p -LPOLESTR = c_wchar_p -LPCOLESTR = c_wchar_p -LPWSTR = c_wchar_p -LPCWSTR = c_wchar_p -LPSTR = c_char_p -LPCSTR = c_char_p -LPVOID = c_void_p -LPCVOID = c_void_p - -# These two types are pointer-sized unsigned and signed ints, respectively. -# At runtime, they are either c_[u]long or c_[u]longlong, depending on the host's pointer size -# (they are not really separate classes). -class WPARAM(_SimpleCData[int]): ... -class LPARAM(_SimpleCData[int]): ... - -ATOM = WORD -LANGID = WORD -COLORREF = DWORD -LGRPID = DWORD -LCTYPE = DWORD -LCID = DWORD - -HANDLE = c_void_p -HACCEL = HANDLE -HBITMAP = HANDLE -HBRUSH = HANDLE -HCOLORSPACE = HANDLE -HDC = HANDLE -HDESK = HANDLE -HDWP = HANDLE -HENHMETAFILE = HANDLE -HFONT = HANDLE -HGDIOBJ = HANDLE -HGLOBAL = HANDLE -HHOOK = HANDLE -HICON = HANDLE -HINSTANCE = HANDLE -HKEY = HANDLE -HKL = HANDLE -HLOCAL = HANDLE -HMENU = HANDLE -HMETAFILE = HANDLE -HMODULE = HANDLE -HMONITOR = HANDLE -HPALETTE = HANDLE -HPEN = HANDLE -HRGN = HANDLE -HRSRC = HANDLE -HSTR = HANDLE -HTASK = HANDLE -HWINSTA = HANDLE -HWND = HANDLE -SC_HANDLE = HANDLE -SERVICE_STATUS_HANDLE = HANDLE - -class RECT(Structure): - left: LONG - top: LONG - right: LONG - bottom: LONG -RECTL = RECT -_RECTL = RECT -tagRECT = RECT - -class _SMALL_RECT(Structure): - Left: SHORT - Top: SHORT - Right: SHORT - Bottom: SHORT -SMALL_RECT = _SMALL_RECT - -class _COORD(Structure): - X: SHORT - Y: SHORT - -class POINT(Structure): - x: LONG - y: LONG -POINTL = POINT -_POINTL = POINT -tagPOINT = POINT - -class SIZE(Structure): - cx: LONG - cy: LONG -SIZEL = SIZE -tagSIZE = SIZE - -def RGB(red: int, green: int, blue: int) -> int: ... - -class FILETIME(Structure): - dwLowDateTime: DWORD - dwHighDateTime: DWORD -_FILETIME = FILETIME - -class MSG(Structure): - hWnd: HWND - message: UINT - wParam: WPARAM - lParam: LPARAM - time: DWORD - pt: POINT -tagMSG = MSG -MAX_PATH: int - -class WIN32_FIND_DATAA(Structure): - dwFileAttributes: DWORD - ftCreationTime: FILETIME - ftLastAccessTime: FILETIME - ftLastWriteTime: FILETIME - nFileSizeHigh: DWORD - nFileSizeLow: DWORD - dwReserved0: DWORD - dwReserved1: DWORD - cFileName: Array[CHAR] - cAlternateFileName: Array[CHAR] - -class WIN32_FIND_DATAW(Structure): - dwFileAttributes: DWORD - ftCreationTime: FILETIME - ftLastAccessTime: FILETIME - ftLastWriteTime: FILETIME - nFileSizeHigh: DWORD - nFileSizeLow: DWORD - dwReserved0: DWORD - dwReserved1: DWORD - cFileName: Array[WCHAR] - cAlternateFileName: Array[WCHAR] - -# These pointer type definitions use pointer[...] instead of POINTER(...), to allow them -# to be used in type annotations. -PBOOL = pointer[BOOL] -LPBOOL = pointer[BOOL] -PBOOLEAN = pointer[BOOLEAN] -PBYTE = pointer[BYTE] -LPBYTE = pointer[BYTE] -PCHAR = pointer[CHAR] -LPCOLORREF = pointer[COLORREF] -PDWORD = pointer[DWORD] -LPDWORD = pointer[DWORD] -PFILETIME = pointer[FILETIME] -LPFILETIME = pointer[FILETIME] -PFLOAT = pointer[FLOAT] -PHANDLE = pointer[HANDLE] -LPHANDLE = pointer[HANDLE] -PHKEY = pointer[HKEY] -LPHKL = pointer[HKL] -PINT = pointer[INT] -LPINT = pointer[INT] -PLARGE_INTEGER = pointer[LARGE_INTEGER] -PLCID = pointer[LCID] -PLONG = pointer[LONG] -LPLONG = pointer[LONG] -PMSG = pointer[MSG] -LPMSG = pointer[MSG] -PPOINT = pointer[POINT] -LPPOINT = pointer[POINT] -PPOINTL = pointer[POINTL] -PRECT = pointer[RECT] -LPRECT = pointer[RECT] -PRECTL = pointer[RECTL] -LPRECTL = pointer[RECTL] -LPSC_HANDLE = pointer[SC_HANDLE] -PSHORT = pointer[SHORT] -PSIZE = pointer[SIZE] -LPSIZE = pointer[SIZE] -PSIZEL = pointer[SIZEL] -LPSIZEL = pointer[SIZEL] -PSMALL_RECT = pointer[SMALL_RECT] -PUINT = pointer[UINT] -LPUINT = pointer[UINT] -PULARGE_INTEGER = pointer[ULARGE_INTEGER] -PULONG = pointer[ULONG] -PUSHORT = pointer[USHORT] -PWCHAR = pointer[WCHAR] -PWIN32_FIND_DATAA = pointer[WIN32_FIND_DATAA] -LPWIN32_FIND_DATAA = pointer[WIN32_FIND_DATAA] -PWIN32_FIND_DATAW = pointer[WIN32_FIND_DATAW] -LPWIN32_FIND_DATAW = pointer[WIN32_FIND_DATAW] -PWORD = pointer[WORD] -LPWORD = pointer[WORD] diff --git a/typeshed/2and3/distutils/__init__.pyi b/typeshed/2and3/distutils/__init__.pyi deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/typeshed/2and3/distutils/spawn.pyi b/typeshed/2and3/distutils/spawn.pyi deleted file mode 100644 index 8df9ebab7f..0000000000 --- a/typeshed/2and3/distutils/spawn.pyi +++ /dev/null @@ -1,8 +0,0 @@ -# Stubs for distutils.spawn - -from typing import List, Optional - -def spawn(cmd: List[str], search_path: bool = ..., - verbose: bool = ..., dry_run: bool = ...) -> None: ... -def find_executable(executable: str, - path: Optional[str] = ...) -> Optional[str]: ... diff --git a/typeshed/2and3/msvcrt.pyi b/typeshed/2and3/msvcrt.pyi deleted file mode 100644 index bcab64cd9a..0000000000 --- a/typeshed/2and3/msvcrt.pyi +++ /dev/null @@ -1,8 +0,0 @@ -# Stubs for msvcrt - -# NOTE: These are incomplete! - -from typing import overload, BinaryIO, TextIO - -def get_osfhandle(file: int) -> int: ... -def open_osfhandle(handle: int, flags: int) -> int: ... diff --git a/typeshed/2and3/pathlib2.pyi b/typeshed/2and3/pathlib2.pyi deleted file mode 100644 index 2e8a45a0da..0000000000 --- a/typeshed/2and3/pathlib2.pyi +++ /dev/null @@ -1,120 +0,0 @@ -from typing import Any, Generator, IO, Optional, Sequence, Text, Tuple, Type, TypeVar, Union, List -from types import TracebackType -import os -import sys - -_P = TypeVar('_P', bound='PurePath') - -if sys.version_info >= (3, 6): - _PurePathBase = os.PathLike[str] -else: - _PurePathBase = object - -class PurePath(_PurePathBase): - parts: Tuple[str, ...] - drive: str - root: str - anchor: str - name: str - suffix: str - suffixes: List[str] - stem: str - if sys.version_info < (3, 6): - def __new__(cls: Type[_P], *args: Union[Text, PurePath]) -> _P: ... - else: - def __new__(cls: Type[_P], *args: Union[Text, os.PathLike[str]]) -> _P: ... - def __hash__(self) -> int: ... - def __lt__(self, other: PurePath) -> bool: ... - def __le__(self, other: PurePath) -> bool: ... - def __gt__(self, other: PurePath) -> bool: ... - def __ge__(self, other: PurePath) -> bool: ... - def __truediv__(self: _P, key: Union[Text, PurePath]) -> _P: ... - if sys.version_info < (3,): - def __div__(self: _P, key: Union[Text, PurePath]) -> _P: ... - def __bytes__(self) -> bytes: ... - def as_posix(self) -> str: ... - def as_uri(self) -> str: ... - def is_absolute(self) -> bool: ... - def is_reserved(self) -> bool: ... - def match(self, path_pattern: str) -> bool: ... - def relative_to(self: _P, *other: Union[str, PurePath]) -> _P: ... - def with_name(self: _P, name: str) -> _P: ... - def with_suffix(self: _P, suffix: str) -> _P: ... - def joinpath(self: _P, *other: Union[str, PurePath]) -> _P: ... - - @property - def parents(self: _P) -> Sequence[_P]: ... - @property - def parent(self: _P) -> _P: ... - -class PurePosixPath(PurePath): ... -class PureWindowsPath(PurePath): ... - -class Path(PurePath): - def __enter__(self) -> Path: ... - def __exit__(self, exc_type: Optional[Type[BaseException]], - exc_value: Optional[BaseException], - traceback: Optional[TracebackType]) -> Optional[bool]: ... - @classmethod - def cwd(cls: Type[_P]) -> _P: ... - def stat(self) -> os.stat_result: ... - def chmod(self, mode: int) -> None: ... - def exists(self) -> bool: ... - def glob(self, pattern: str) -> Generator[Path, None, None]: ... - def group(self) -> str: ... - def is_dir(self) -> bool: ... - def is_file(self) -> bool: ... - def is_symlink(self) -> bool: ... - def is_socket(self) -> bool: ... - def is_fifo(self) -> bool: ... - def is_block_device(self) -> bool: ... - def is_char_device(self) -> bool: ... - def iterdir(self) -> Generator[Path, None, None]: ... - def lchmod(self, mode: int) -> None: ... - def lstat(self) -> os.stat_result: ... - if sys.version_info < (3, 5): - def mkdir(self, mode: int = ..., - parents: bool = ...) -> None: ... - else: - def mkdir(self, mode: int = ..., parents: bool = ..., - exist_ok: bool = ...) -> None: ... - def open(self, mode: str = ..., buffering: int = ..., - encoding: Optional[str] = ..., errors: Optional[str] = ..., - newline: Optional[str] = ...) -> IO[Any]: ... - def owner(self) -> str: ... - def rename(self, target: Union[str, PurePath]) -> None: ... - def replace(self, target: Union[str, PurePath]) -> None: ... - if sys.version_info < (3, 6): - def resolve(self: _P) -> _P: ... - else: - def resolve(self: _P, strict: bool = ...) -> _P: ... - def rglob(self, pattern: str) -> Generator[Path, None, None]: ... - def rmdir(self) -> None: ... - def symlink_to(self, target: Union[str, Path], - target_is_directory: bool = ...) -> None: ... - def touch(self, mode: int = ..., exist_ok: bool = ...) -> None: ... - def unlink(self) -> None: ... - - if sys.version_info >= (3, 5): - @classmethod - def home(cls: Type[_P]) -> _P: ... - if sys.version_info < (3, 6): - def __new__(cls: Type[_P], *args: Union[str, PurePath], - **kwargs: Any) -> _P: ... - else: - def __new__(cls: Type[_P], *args: Union[str, os.PathLike[str]], - **kwargs: Any) -> _P: ... - - def absolute(self: _P) -> _P: ... - def expanduser(self: _P) -> _P: ... - def read_bytes(self) -> bytes: ... - def read_text(self, encoding: Optional[str] = ..., - errors: Optional[str] = ...) -> str: ... - def samefile(self, other_path: Union[str, bytes, int, Path]) -> bool: ... - def write_bytes(self, data: bytes) -> int: ... - def write_text(self, data: str, encoding: Optional[str] = ..., - errors: Optional[str] = ...) -> int: ... - - -class PosixPath(Path, PurePosixPath): ... -class WindowsPath(Path, PureWindowsPath): ... diff --git a/typeshed/2and3/select.pyi b/typeshed/2and3/select.pyi deleted file mode 100644 index 094bb916ce..0000000000 --- a/typeshed/2and3/select.pyi +++ /dev/null @@ -1,137 +0,0 @@ -import sys -from typing import Any, Optional, Sequence, Tuple, Iterable, List, Union - -# When we have protocols, this should change to a protocol with a fileno method -# See https://docs.python.org/3/c-api/file.html#c.PyObject_AsFileDescriptor -_FileDescriptor = Union[int, Any] - -EPOLLERR: int -EPOLLET: int -EPOLLHUP: int -EPOLLIN: int -EPOLLMSG: int -EPOLLONESHOT: int -EPOLLOUT: int -EPOLLPRI: int -EPOLLRDBAND: int -EPOLLRDNORM: int -EPOLLWRBAND: int -EPOLLWRNORM: int -EPOLL_RDHUP: int -KQ_EV_ADD: int -KQ_EV_CLEAR: int -KQ_EV_DELETE: int -KQ_EV_DISABLE: int -KQ_EV_ENABLE: int -KQ_EV_EOF: int -KQ_EV_ERROR: int -KQ_EV_FLAG1: int -KQ_EV_ONESHOT: int -KQ_EV_SYSFLAGS: int -KQ_FILTER_AIO: int -KQ_FILTER_NETDEV: int -KQ_FILTER_PROC: int -KQ_FILTER_READ: int -KQ_FILTER_SIGNAL: int -KQ_FILTER_TIMER: int -KQ_FILTER_VNODE: int -KQ_FILTER_WRITE: int -KQ_NOTE_ATTRIB: int -KQ_NOTE_CHILD: int -KQ_NOTE_DELETE: int -KQ_NOTE_EXEC: int -KQ_NOTE_EXIT: int -KQ_NOTE_EXTEND: int -KQ_NOTE_FORK: int -KQ_NOTE_LINK: int -KQ_NOTE_LINKDOWN: int -KQ_NOTE_LINKINV: int -KQ_NOTE_LINKUP: int -KQ_NOTE_LOWAT: int -KQ_NOTE_PCTRLMASK: int -KQ_NOTE_PDATAMASK: int -KQ_NOTE_RENAME: int -KQ_NOTE_REVOKE: int -KQ_NOTE_TRACK: int -KQ_NOTE_TRACKERR: int -KQ_NOTE_WRITE: int -PIPE_BUF: int -POLLERR: int -POLLHUP: int -POLLIN: int -POLLMSG: int -POLLNVAL: int -POLLOUT: int -POLLPRI: int -POLLRDBAND: int -POLLRDNORM: int -POLLWRBAND: int -POLLWRNORM: int - -class poll: - def __init__(self) -> None: ... - def register(self, fd: _FileDescriptor, eventmask: int = ...) -> None: ... - def modify(self, fd: _FileDescriptor, eventmask: int) -> None: ... - def unregister(self, fd: _FileDescriptor) -> None: ... - def poll(self, timeout: Optional[float] = ...) -> List[Tuple[int, int]]: ... - -def select(rlist: Sequence[Any], wlist: Sequence[Any], xlist: Sequence[Any], - timeout: Optional[float] = ...) -> Tuple[List[Any], - List[Any], - List[Any]]: ... - -if sys.version_info >= (3, 3): - error = OSError -else: - class error(Exception): ... - -# BSD only -class kevent(object): - data: Any - fflags: int - filter: int - flags: int - ident: int - udata: Any - def __init__(self, ident: _FileDescriptor, filter: int = ..., flags: int = ..., fflags: int = ..., data: Any = ..., udata: Any = ...) -> None: ... - -# BSD only -class kqueue(object): - closed: bool - def __init__(self) -> None: ... - def close(self) -> None: ... - def control(self, changelist: Optional[Iterable[kevent]], max_events: int, timeout: float = ...) -> List[kevent]: ... - def fileno(self) -> int: ... - @classmethod - def fromfd(cls, fd: _FileDescriptor) -> kqueue: ... - -# Linux only -class epoll(object): - if sys.version_info >= (3, 3): - def __init__(self, sizehint: int = ..., flags: int = ...) -> None: ... - else: - def __init__(self, sizehint: int = ...) -> None: ... - if sys.version_info >= (3, 4): - def __enter__(self) -> epoll: ... - def __exit__(self, *args: Any) -> None: ... - def close(self) -> None: ... - closed: bool - def fileno(self) -> int: ... - def register(self, fd: _FileDescriptor, eventmask: int = ...) -> None: ... - def modify(self, fd: _FileDescriptor, eventmask: int) -> None: ... - def unregister(self, fd: _FileDescriptor) -> None: ... - def poll(self, timeout: float = ..., maxevents: int = ...) -> List[Tuple[int, int]]: ... - @classmethod - def fromfd(cls, fd: _FileDescriptor) -> epoll: ... - -if sys.version_info >= (3, 3): - # Solaris only - class devpoll: - if sys.version_info >= (3, 4): - def close(self) -> None: ... - closed: bool - def fileno(self) -> int: ... - def register(self, fd: _FileDescriptor, eventmask: int = ...) -> None: ... - def modify(self, fd: _FileDescriptor, eventmask: int = ...) -> None: ... - def unregister(self, fd: _FileDescriptor) -> None: ... - def poll(self, timeout: Optional[float] = ...) -> List[Tuple[int, int]]: ... \ No newline at end of file diff --git a/typeshed/2and3/uuid.pyi b/typeshed/2and3/uuid.pyi deleted file mode 100644 index 5828073c8e..0000000000 --- a/typeshed/2and3/uuid.pyi +++ /dev/null @@ -1,94 +0,0 @@ -# Stubs for uuid - -import sys -from typing import Tuple, Optional, Any, Union - -# Because UUID has properties called int and bytes we need to rename these temporarily. -_Int = int -_Bytes = bytes -_FieldsType = Tuple[int, int, int, int, int, int] -if sys.version_info >= (3,): - _Text = str -else: - _Text = Union[str, unicode] - -class UUID: - def __init__(self, hex: Optional[_Text] = ..., - bytes: Optional[_Bytes] = ..., - bytes_le: Optional[_Bytes] = ..., - fields: Optional[_FieldsType] = ..., - int: Optional[_Int] = ..., - version: Optional[_Int] = ...) -> None: ... - @property - def bytes(self) -> _Bytes: ... - @property - def bytes_le(self) -> _Bytes: ... - @property - def clock_seq(self) -> _Int: ... - @property - def clock_seq_hi_variant(self) -> _Int: ... - @property - def clock_seq_low(self) -> _Int: ... - @property - def fields(self) -> _FieldsType: ... - @property - def hex(self) -> str: ... - @property - def int(self) -> _Int: ... - @property - def node(self) -> _Int: ... - @property - def time(self) -> _Int: ... - @property - def time_hi_version(self) -> _Int: ... - @property - def time_low(self) -> _Int: ... - @property - def time_mid(self) -> _Int: ... - @property - def urn(self) -> str: ... - @property - def variant(self) -> str: ... - @property - def version(self) -> Optional[_Int]: ... - - def __int__(self) -> _Int: ... - - if sys.version_info >= (3,): - def __eq__(self, other: Any) -> bool: ... - def __lt__(self, other: Any) -> bool: ... - def __le__(self, other: Any) -> bool: ... - def __gt__(self, other: Any) -> bool: ... - def __ge__(self, other: Any) -> bool: ... - else: - def get_bytes(self) -> _Bytes: ... - def get_bytes_le(self) -> _Bytes: ... - def get_clock_seq(self) -> _Int: ... - def get_clock_seq_hi_variant(self) -> _Int: ... - def get_clock_seq_low(self) -> _Int: ... - def get_fields(self) -> _FieldsType: ... - def get_hex(self) -> str: ... - def get_node(self) -> _Int: ... - def get_time(self) -> _Int: ... - def get_time_hi_version(self) -> _Int: ... - def get_time_low(self) -> _Int: ... - def get_time_mid(self) -> _Int: ... - def get_urn(self) -> str: ... - def get_variant(self) -> str: ... - def get_version(self) -> Optional[_Int]: ... - def __cmp__(self, other: Any) -> _Int: ... - -def getnode() -> int: ... -def uuid1(node: Optional[_Int] = ..., clock_seq: Optional[_Int] = ...) -> UUID: ... -def uuid3(namespace: UUID, name: str) -> UUID: ... -def uuid4() -> UUID: ... -def uuid5(namespace: UUID, name: str) -> UUID: ... - -NAMESPACE_DNS = ... # type: UUID -NAMESPACE_URL = ... # type: UUID -NAMESPACE_OID = ... # type: UUID -NAMESPACE_X500 = ... # type: UUID -RESERVED_NCS = ... # type: str -RFC_4122 = ... # type: str -RESERVED_MICROSOFT = ... # type: str -RESERVED_FUTURE = ... # type: str diff --git a/typeshed/3/logging/__init__.pyi b/typeshed/3/logging/__init__.pyi deleted file mode 100644 index b104a2f102..0000000000 --- a/typeshed/3/logging/__init__.pyi +++ /dev/null @@ -1,561 +0,0 @@ -# Stubs for logging (Python 3.7) - -from typing import ( - Any, Callable, Dict, Iterable, List, Mapping, MutableMapping, Optional, IO, - Tuple, Text, Union, overload, -) -from string import Template -from time import struct_time -from types import TracebackType, FrameType -import sys -import threading - -_SysExcInfoType = Union[Tuple[type, BaseException, Optional[TracebackType]], - Tuple[None, None, None]] -if sys.version_info >= (3, 5): - _ExcInfoType = Union[None, bool, _SysExcInfoType, BaseException] -else: - _ExcInfoType = Union[None, bool, _SysExcInfoType] -_ArgsType = Union[Tuple[Any, ...], Mapping[str, Any]] -_FilterType = Union[Filter, Callable[[LogRecord], int]] -_Level = Union[int, Text] -if sys.version_info >= (3, 6): - from os import PathLike - _Path = Union[str, PathLike[str]] -else: - _Path = str - -raiseExceptions: bool -logThreads: bool -logMultiprocessing: bool -logProcesses: bool - -def currentframe() -> FrameType: ... - -if sys.version_info >= (3,): - _levelToName: Dict[int, str] - _nameToLevel: Dict[str, int] -else: - _levelNames: Dict[Union[int, str], Union[str, int]] # Union[int:str, str:int] - -class Filterer(object): - filters: List[Filter] - def __init__(self) -> None: ... - def addFilter(self, filter: Filter) -> None: ... - def removeFilter(self, filter: Filter) -> None: ... - def filter(self, record: LogRecord) -> bool: ... - -class Logger(Filterer): - name: str - level: int - parent: Union[Logger, PlaceHolder] - propagate: bool - handlers: List[Handler] - disabled: int - def __init__(self, name: str, level: _Level = ...) -> None: ... - def setLevel(self, level: _Level) -> None: ... - def isEnabledFor(self, level: int) -> bool: ... - def getEffectiveLevel(self) -> int: ... - def getChild(self, suffix: str) -> Logger: ... - if sys.version_info >= (3, 8): - def debug(self, msg: Any, *args: Any, exc_info: _ExcInfoType = ..., - stack_info: bool = ..., stacklevel: int = ..., extra: Optional[Dict[str, Any]] = ..., - **kwargs: Any) -> None: ... - def info(self, msg: Any, *args: Any, exc_info: _ExcInfoType = ..., - stack_info: bool = ..., stacklevel: int = ..., extra: Optional[Dict[str, Any]] = ..., - **kwargs: Any) -> None: ... - def warning(self, msg: Any, *args: Any, exc_info: _ExcInfoType = ..., - stack_info: bool = ..., stacklevel: int = ..., extra: Optional[Dict[str, Any]] = ..., - **kwargs: Any) -> None: ... - def warn(self, msg: Any, *args: Any, exc_info: _ExcInfoType = ..., - stack_info: bool = ..., stacklevel: int = ..., extra: Optional[Dict[str, Any]] = ..., - **kwargs: Any) -> None: ... - def error(self, msg: Any, *args: Any, exc_info: _ExcInfoType = ..., - stack_info: bool = ..., stacklevel: int = ..., extra: Optional[Dict[str, Any]] = ..., - **kwargs: Any) -> None: ... - def exception(self, msg: Any, *args: Any, exc_info: _ExcInfoType = ..., - stack_info: bool = ..., stacklevel: int = ..., extra: Optional[Dict[str, Any]] = ..., - **kwargs: Any) -> None: ... - def critical(self, msg: Any, *args: Any, exc_info: _ExcInfoType = ..., - stack_info: bool = ..., stacklevel: int = ..., extra: Optional[Dict[str, Any]] = ..., - **kwargs: Any) -> None: ... - def log(self, level: int, msg: Any, *args: Any, exc_info: _ExcInfoType = ..., - stack_info: bool = ..., stacklevel: int = ..., extra: Optional[Dict[str, Any]] = ..., - **kwargs: Any) -> None: ... - def _log( - self, - level: int, - msg: Any, - args: _ArgsType, - exc_info: Optional[_ExcInfoType] = ..., - extra: Optional[Dict[str, Any]] = ..., - stack_info: bool = ..., - stacklevel: int = ..., - ) -> None: ... # undocumented - elif sys.version_info >= (3,): - def debug(self, msg: Any, *args: Any, exc_info: _ExcInfoType = ..., - stack_info: bool = ..., extra: Optional[Dict[str, Any]] = ..., - **kwargs: Any) -> None: ... - def info(self, msg: Any, *args: Any, exc_info: _ExcInfoType = ..., - stack_info: bool = ..., extra: Optional[Dict[str, Any]] = ..., - **kwargs: Any) -> None: ... - def warning(self, msg: Any, *args: Any, exc_info: _ExcInfoType = ..., - stack_info: bool = ..., extra: Optional[Dict[str, Any]] = ..., - **kwargs: Any) -> None: ... - def warn(self, msg: Any, *args: Any, exc_info: _ExcInfoType = ..., - stack_info: bool = ..., extra: Optional[Dict[str, Any]] = ..., - **kwargs: Any) -> None: ... - def error(self, msg: Any, *args: Any, exc_info: _ExcInfoType = ..., - stack_info: bool = ..., extra: Optional[Dict[str, Any]] = ..., - **kwargs: Any) -> None: ... - def critical(self, msg: Any, *args: Any, exc_info: _ExcInfoType = ..., - stack_info: bool = ..., extra: Optional[Dict[str, Any]] = ..., - **kwargs: Any) -> None: ... - fatal = critical - def log(self, level: int, msg: Any, *args: Any, exc_info: _ExcInfoType = ..., - stack_info: bool = ..., extra: Optional[Dict[str, Any]] = ..., - **kwargs: Any) -> None: ... - def exception(self, msg: Any, *args: Any, exc_info: _ExcInfoType = ..., - stack_info: bool = ..., extra: Optional[Dict[str, Any]] = ..., - **kwargs: Any) -> None: ... - def _log( - self, - level: int, - msg: Any, - args: _ArgsType, - exc_info: Optional[_ExcInfoType] = ..., - extra: Optional[Dict[str, Any]] = ..., - stack_info: bool = ..., - ) -> None: ... # undocumented - else: - def debug(self, - msg: Any, *args: Any, exc_info: _ExcInfoType = ..., - extra: Optional[Dict[str, Any]] = ..., **kwargs: Any) -> None: ... - def info(self, - msg: Any, *args: Any, exc_info: _ExcInfoType = ..., - extra: Optional[Dict[str, Any]] = ..., **kwargs: Any) -> None: ... - def warning(self, - msg: Any, *args: Any, exc_info: _ExcInfoType = ..., - extra: Optional[Dict[str, Any]] = ..., **kwargs: Any) -> None: ... - warn = warning - def error(self, - msg: Any, *args: Any, exc_info: _ExcInfoType = ..., - extra: Optional[Dict[str, Any]] = ..., **kwargs: Any) -> None: ... - def critical(self, - msg: Any, *args: Any, exc_info: _ExcInfoType = ..., - extra: Optional[Dict[str, Any]] = ..., **kwargs: Any) -> None: ... - fatal = critical - def log(self, - level: int, msg: Any, *args: Any, exc_info: _ExcInfoType = ..., - extra: Optional[Dict[str, Any]] = ..., **kwargs: Any) -> None: ... - def exception(self, - msg: Any, *args: Any, exc_info: _ExcInfoType = ..., - extra: Optional[Dict[str, Any]] = ..., **kwargs: Any) -> None: ... - def _log( - self, - level: int, - msg: Any, - args: _ArgsType, - exc_info: Optional[_ExcInfoType] = ..., - extra: Optional[Dict[str, Any]] = ..., - ) -> None: ... # undocumented - def addFilter(self, filt: _FilterType) -> None: ... - def removeFilter(self, filt: _FilterType) -> None: ... - def filter(self, record: LogRecord) -> bool: ... - def addHandler(self, hdlr: Handler) -> None: ... - def removeHandler(self, hdlr: Handler) -> None: ... - if sys.version_info >= (3, 8): - def findCaller(self, stack_info: bool = ..., stacklevel: int = ...) -> Tuple[str, int, str, Optional[str]]: ... - elif sys.version_info >= (3,): - def findCaller(self, stack_info: bool = ...) -> Tuple[str, int, str, Optional[str]]: ... - else: - def findCaller(self) -> Tuple[str, int, str]: ... - def handle(self, record: LogRecord) -> None: ... - if sys.version_info >= (3,): - def makeRecord(self, name: str, level: int, fn: str, lno: int, msg: Any, - args: _ArgsType, - exc_info: Optional[_SysExcInfoType], - func: Optional[str] = ..., - extra: Optional[Mapping[str, Any]] = ..., - sinfo: Optional[str] = ...) -> LogRecord: ... - else: - def makeRecord(self, - name: str, level: int, fn: str, lno: int, msg: Any, - args: _ArgsType, - exc_info: Optional[_SysExcInfoType], - func: Optional[str] = ..., - extra: Optional[Mapping[str, Any]] = ...) -> LogRecord: ... - if sys.version_info >= (3,): - def hasHandlers(self) -> bool: ... - - -CRITICAL: int -FATAL: int -ERROR: int -WARNING: int -WARN: int -INFO: int -DEBUG: int -NOTSET: int - - -class Handler(Filterer): - level: int # undocumented - formatter: Optional[Formatter] # undocumented - lock: Optional[threading.Lock] # undocumented - name: Optional[str] # undocumented - def __init__(self, level: _Level = ...) -> None: ... - def createLock(self) -> None: ... - def acquire(self) -> None: ... - def release(self) -> None: ... - def setLevel(self, level: _Level) -> None: ... - def setFormatter(self, fmt: Formatter) -> None: ... - def addFilter(self, filt: _FilterType) -> None: ... - def removeFilter(self, filt: _FilterType) -> None: ... - def filter(self, record: LogRecord) -> bool: ... - def flush(self) -> None: ... - def close(self) -> None: ... - def handle(self, record: LogRecord) -> None: ... - def handleError(self, record: LogRecord) -> None: ... - def format(self, record: LogRecord) -> str: ... - def emit(self, record: LogRecord) -> None: ... - - -class Formatter: - converter: Callable[[Optional[float]], struct_time] - _fmt: Optional[str] - datefmt: Optional[str] - if sys.version_info >= (3,): - _style: PercentStyle - default_time_format: str - default_msec_format: str - - if sys.version_info >= (3, 8): - def __init__(self, fmt: Optional[str] = ..., - datefmt: Optional[str] = ..., - style: str = ..., validate: bool = ...) -> None: ... - elif sys.version_info >= (3,): - def __init__(self, fmt: Optional[str] = ..., - datefmt: Optional[str] = ..., - style: str = ...) -> None: ... - else: - def __init__(self, - fmt: Optional[str] = ..., - datefmt: Optional[str] = ...) -> None: ... - - def format(self, record: LogRecord) -> str: ... - def formatTime(self, record: LogRecord, datefmt: Optional[str] = ...) -> str: ... - def formatException(self, exc_info: _SysExcInfoType) -> str: ... - if sys.version_info >= (3,): - def formatMessage(self, record: LogRecord) -> str: ... # undocumented - def formatStack(self, stack_info: str) -> str: ... - - -class Filter: - def __init__(self, name: str = ...) -> None: ... - def filter(self, record: LogRecord) -> int: ... - - -class LogRecord: - args: _ArgsType - asctime: str - created: float - exc_info: Optional[_SysExcInfoType] - exc_text: Optional[str] - filename: str - funcName: str - levelname: str - levelno: int - lineno: int - module: str - msecs: float - message: str - msg: str - name: str - pathname: str - process: int - processName: str - relativeCreated: int - if sys.version_info >= (3,): - stack_info: Optional[str] - thread: int - threadName: str - if sys.version_info >= (3,): - def __init__(self, name: str, level: int, pathname: str, lineno: int, - msg: Any, args: _ArgsType, - exc_info: Optional[_SysExcInfoType], - func: Optional[str] = ..., - sinfo: Optional[str] = ...) -> None: ... - else: - def __init__(self, - name: str, level: int, pathname: str, lineno: int, - msg: Any, args: _ArgsType, - exc_info: Optional[_SysExcInfoType], - func: Optional[str] = ...) -> None: ... - def getMessage(self) -> str: ... - - -class LoggerAdapter: - logger: Logger - extra: Mapping[str, Any] - def __init__(self, logger: Logger, extra: Mapping[str, Any]) -> None: ... - def process(self, msg: Any, kwargs: MutableMapping[str, Any]) -> Tuple[Any, MutableMapping[str, Any]]: ... - if sys.version_info >= (3, 8): - def debug(self, msg: Any, *args: Any, exc_info: _ExcInfoType = ..., - stack_info: bool = ..., stacklevel: int = ..., extra: Optional[Dict[str, Any]] = ..., - **kwargs: Any) -> None: ... - def info(self, msg: Any, *args: Any, exc_info: _ExcInfoType = ..., - stack_info: bool = ..., stacklevel: int = ..., extra: Optional[Dict[str, Any]] = ..., - **kwargs: Any) -> None: ... - def warning(self, msg: Any, *args: Any, exc_info: _ExcInfoType = ..., - stack_info: bool = ..., stacklevel: int = ..., extra: Optional[Dict[str, Any]] = ..., - **kwargs: Any) -> None: ... - def warn(self, msg: Any, *args: Any, exc_info: _ExcInfoType = ..., - stack_info: bool = ..., stacklevel: int = ..., extra: Optional[Dict[str, Any]] = ..., - **kwargs: Any) -> None: ... - def error(self, msg: Any, *args: Any, exc_info: _ExcInfoType = ..., - stack_info: bool = ..., stacklevel: int = ..., extra: Optional[Dict[str, Any]] = ..., - **kwargs: Any) -> None: ... - def exception(self, msg: Any, *args: Any, exc_info: _ExcInfoType = ..., - stack_info: bool = ..., stacklevel: int = ..., extra: Optional[Dict[str, Any]] = ..., - **kwargs: Any) -> None: ... - def critical(self, msg: Any, *args: Any, exc_info: _ExcInfoType = ..., - stack_info: bool = ..., stacklevel: int = ..., extra: Optional[Dict[str, Any]] = ..., - **kwargs: Any) -> None: ... - def log(self, level: int, msg: Any, *args: Any, exc_info: _ExcInfoType = ..., - stack_info: bool = ..., stacklevel: int = ..., extra: Optional[Dict[str, Any]] = ..., - **kwargs: Any) -> None: ... - elif sys.version_info >= (3,): - def debug(self, msg: Any, *args: Any, exc_info: _ExcInfoType = ..., - stack_info: bool = ..., extra: Optional[Dict[str, Any]] = ..., - **kwargs: Any) -> None: ... - def info(self, msg: Any, *args: Any, exc_info: _ExcInfoType = ..., - stack_info: bool = ..., extra: Optional[Dict[str, Any]] = ..., - **kwargs: Any) -> None: ... - def warning(self, msg: Any, *args: Any, exc_info: _ExcInfoType = ..., - stack_info: bool = ..., extra: Optional[Dict[str, Any]] = ..., - **kwargs: Any) -> None: ... - def warn(self, msg: Any, *args: Any, exc_info: _ExcInfoType = ..., - stack_info: bool = ..., extra: Optional[Dict[str, Any]] = ..., - **kwargs: Any) -> None: ... - def error(self, msg: Any, *args: Any, exc_info: _ExcInfoType = ..., - stack_info: bool = ..., extra: Optional[Dict[str, Any]] = ..., - **kwargs: Any) -> None: ... - def exception(self, msg: Any, *args: Any, exc_info: _ExcInfoType = ..., - stack_info: bool = ..., extra: Optional[Dict[str, Any]] = ..., - **kwargs: Any) -> None: ... - def critical(self, msg: Any, *args: Any, exc_info: _ExcInfoType = ..., - stack_info: bool = ..., extra: Optional[Dict[str, Any]] = ..., - **kwargs: Any) -> None: ... - def log(self, level: int, msg: Any, *args: Any, exc_info: _ExcInfoType = ..., - stack_info: bool = ..., extra: Optional[Dict[str, Any]] = ..., - **kwargs: Any) -> None: ... - else: - def debug(self, - msg: Any, *args: Any, exc_info: _ExcInfoType = ..., - extra: Optional[Dict[str, Any]] = ..., **kwargs: Any) -> None: ... - def info(self, - msg: Any, *args: Any, exc_info: _ExcInfoType = ..., - extra: Optional[Dict[str, Any]] = ..., **kwargs: Any) -> None: ... - def warning(self, - msg: Any, *args: Any, exc_info: _ExcInfoType = ..., - extra: Optional[Dict[str, Any]] = ..., **kwargs: Any) -> None: ... - def error(self, - msg: Any, *args: Any, exc_info: _ExcInfoType = ..., - extra: Optional[Dict[str, Any]] = ..., **kwargs: Any) -> None: ... - def exception(self, - msg: Any, *args: Any, exc_info: _ExcInfoType = ..., - extra: Optional[Dict[str, Any]] = ..., **kwargs: Any) -> None: ... - def critical(self, - msg: Any, *args: Any, exc_info: _ExcInfoType = ..., - extra: Optional[Dict[str, Any]] = ..., **kwargs: Any) -> None: ... - def log(self, - level: int, msg: Any, *args: Any, exc_info: _ExcInfoType = ..., - extra: Optional[Dict[str, Any]] = ..., **kwargs: Any) -> None: ... - def isEnabledFor(self, lvl: int) -> bool: ... - if sys.version_info >= (3,): - def getEffectiveLevel(self) -> int: ... - def setLevel(self, lvl: Union[int, str]) -> None: ... - def hasHandlers(self) -> bool: ... - if sys.version_info >= (3, 6): - def _log( - self, - level: int, - msg: Any, - args: _ArgsType, - exc_info: Optional[_ExcInfoType] = ..., - extra: Optional[Dict[str, Any]] = ..., - stack_info: bool = ..., - ) -> None: ... # undocumented - -if sys.version_info >= (3,): - def getLogger(name: Optional[str] = ...) -> Logger: ... -else: - @overload - def getLogger() -> Logger: ... - @overload - def getLogger(name: Union[Text, str]) -> Logger: ... -def getLoggerClass() -> type: ... -if sys.version_info >= (3,): - def getLogRecordFactory() -> Callable[..., LogRecord]: ... - -if sys.version_info >= (3, 8): - def debug(msg: Any, *args: Any, exc_info: _ExcInfoType = ..., - stack_info: bool = ..., stacklevel: int = ..., extra: Optional[Dict[str, Any]] = ..., - **kwargs: Any) -> None: ... - def info(msg: Any, *args: Any, exc_info: _ExcInfoType = ..., - stack_info: bool = ..., stacklevel: int = ..., extra: Optional[Dict[str, Any]] = ..., - **kwargs: Any) -> None: ... - def warning(msg: Any, *args: Any, exc_info: _ExcInfoType = ..., - stack_info: bool = ..., stacklevel: int = ..., extra: Optional[Dict[str, Any]] = ..., - **kwargs: Any) -> None: ... - def warn(msg: Any, *args: Any, exc_info: _ExcInfoType = ..., - stack_info: bool = ..., stacklevel: int = ..., extra: Optional[Dict[str, Any]] = ..., - **kwargs: Any) -> None: ... - def error(msg: Any, *args: Any, exc_info: _ExcInfoType = ..., - stack_info: bool = ..., stacklevel: int = ..., extra: Optional[Dict[str, Any]] = ..., - **kwargs: Any) -> None: ... - def critical(msg: Any, *args: Any, exc_info: _ExcInfoType = ..., - stack_info: bool = ..., stacklevel: int = ..., extra: Optional[Dict[str, Any]] = ..., - **kwargs: Any) -> None: ... - def exception(msg: Any, *args: Any, exc_info: _ExcInfoType = ..., - stack_info: bool = ..., stacklevel: int = ..., extra: Optional[Dict[str, Any]] = ..., - **kwargs: Any) -> None: ... - def log(level: int, msg: Any, *args: Any, exc_info: _ExcInfoType = ..., - stack_info: bool = ..., stacklevel: int = ..., extra: Optional[Dict[str, Any]] = ..., - **kwargs: Any) -> None: ... -elif sys.version_info >= (3,): - def debug(msg: Any, *args: Any, exc_info: _ExcInfoType = ..., - stack_info: bool = ..., extra: Optional[Dict[str, Any]] = ..., - **kwargs: Any) -> None: ... - def info(msg: Any, *args: Any, exc_info: _ExcInfoType = ..., - stack_info: bool = ..., extra: Optional[Dict[str, Any]] = ..., - **kwargs: Any) -> None: ... - def warning(msg: Any, *args: Any, exc_info: _ExcInfoType = ..., - stack_info: bool = ..., extra: Optional[Dict[str, Any]] = ..., - **kwargs: Any) -> None: ... - def warn(msg: Any, *args: Any, exc_info: _ExcInfoType = ..., - stack_info: bool = ..., extra: Optional[Dict[str, Any]] = ..., - **kwargs: Any) -> None: ... - def error(msg: Any, *args: Any, exc_info: _ExcInfoType = ..., - stack_info: bool = ..., extra: Optional[Dict[str, Any]] = ..., - **kwargs: Any) -> None: ... - def critical(msg: Any, *args: Any, exc_info: _ExcInfoType = ..., - stack_info: bool = ..., extra: Optional[Dict[str, Any]] = ..., - **kwargs: Any) -> None: ... - def exception(msg: Any, *args: Any, exc_info: _ExcInfoType = ..., - stack_info: bool = ..., extra: Optional[Dict[str, Any]] = ..., - **kwargs: Any) -> None: ... - def log(level: int, msg: Any, *args: Any, exc_info: _ExcInfoType = ..., - stack_info: bool = ..., extra: Optional[Dict[str, Any]] = ..., - **kwargs: Any) -> None: ... -else: - def debug(msg: Any, *args: Any, exc_info: _ExcInfoType = ..., - extra: Optional[Dict[str, Any]] = ..., **kwargs: Any) -> None: ... - def info(msg: Any, *args: Any, exc_info: _ExcInfoType = ..., - extra: Optional[Dict[str, Any]] = ..., **kwargs: Any) -> None: ... - def warning(msg: Any, *args: Any, exc_info: _ExcInfoType = ..., - extra: Optional[Dict[str, Any]] = ..., **kwargs: Any) -> None: ... - warn = warning - def error(msg: Any, *args: Any, exc_info: _ExcInfoType = ..., - extra: Optional[Dict[str, Any]] = ..., **kwargs: Any) -> None: ... - def critical(msg: Any, *args: Any, exc_info: _ExcInfoType = ..., - extra: Optional[Dict[str, Any]] = ..., **kwargs: Any) -> None: ... - def exception(msg: Any, *args: Any, exc_info: _ExcInfoType = ..., - extra: Optional[Dict[str, Any]] = ..., **kwargs: Any) -> None: ... - def log(level: int, msg: Any, *args: Any, exc_info: _ExcInfoType = ..., - extra: Optional[Dict[str, Any]] = ..., **kwargs: Any) -> None: ... -fatal = critical - -def disable(lvl: int) -> None: ... -def addLevelName(lvl: int, levelName: str) -> None: ... -def getLevelName(lvl: Union[int, str]) -> Any: ... - -def makeLogRecord(attrdict: Mapping[str, Any]) -> LogRecord: ... - -if sys.version_info >= (3, 8): - def basicConfig(*, filename: Optional[_Path] = ..., filemode: str = ..., - format: str = ..., datefmt: Optional[str] = ..., style: str = ..., - level: Optional[_Level] = ..., stream: Optional[IO[str]] = ..., - handlers: Optional[Iterable[Handler]] = ..., force: bool = ...) -> None: ... -elif sys.version_info >= (3,): - def basicConfig(*, filename: Optional[_Path] = ..., filemode: str = ..., - format: str = ..., datefmt: Optional[str] = ..., style: str = ..., - level: Optional[_Level] = ..., stream: Optional[IO[str]] = ..., - handlers: Optional[Iterable[Handler]] = ...) -> None: ... -else: - @overload - def basicConfig() -> None: ... - @overload - def basicConfig(*, filename: Optional[str] = ..., filemode: str = ..., - format: str = ..., datefmt: Optional[str] = ..., - level: Optional[_Level] = ..., stream: IO[str] = ...) -> None: ... -def shutdown() -> None: ... - -def setLoggerClass(klass: type) -> None: ... - -def captureWarnings(capture: bool) -> None: ... - -if sys.version_info >= (3,): - def setLogRecordFactory(factory: Callable[..., LogRecord]) -> None: ... - - -if sys.version_info >= (3,): - lastResort: Optional[StreamHandler] - - -class StreamHandler(Handler): - stream: IO[str] # undocumented - if sys.version_info >= (3, 2): - terminator: str - def __init__(self, stream: Optional[IO[str]] = ...) -> None: ... - if sys.version_info >= (3, 7): - def setStream(self, stream: IO[str]) -> Optional[IO[str]]: ... - - -class FileHandler(StreamHandler): - baseFilename: str # undocumented - mode: str # undocumented - encoding: Optional[str] # undocumented - delay: bool # undocumented - def __init__(self, filename: _Path, mode: str = ..., - encoding: Optional[str] = ..., delay: bool = ...) -> None: ... - def _open(self) -> IO[Any]: ... - - -class NullHandler(Handler): ... - - -class PlaceHolder: - def __init__(self, alogger: Logger) -> None: ... - def append(self, alogger: Logger) -> None: ... - - -# Below aren't in module docs but still visible - -class RootLogger(Logger): ... - -root: RootLogger - - -if sys.version_info >= (3,): - class PercentStyle(object): - default_format: str - asctime_format: str - asctime_search: str - _fmt: str - - def __init__(self, fmt: str) -> None: ... - def usesTime(self) -> bool: ... - def format(self, record: Any) -> str: ... - - class StrFormatStyle(PercentStyle): - ... - - class StringTemplateStyle(PercentStyle): - _tpl: Template - - _STYLES: Dict[str, Tuple[PercentStyle, str]] - - -BASIC_FORMAT: str diff --git a/typeshed/3/py/__init__.pyi b/typeshed/3/py/__init__.pyi deleted file mode 100644 index 27b3076a7a..0000000000 --- a/typeshed/3/py/__init__.pyi +++ /dev/null @@ -1,13 +0,0 @@ -# Names in __all__ with no definition: -# _pydir -# builtin -# code -# iniconfig -# io -# log -# path -# process -# std -# test -# version -# xml diff --git a/typeshed/3/py/__metainfo.pyi b/typeshed/3/py/__metainfo.pyi deleted file mode 100644 index e6315ea2c8..0000000000 --- a/typeshed/3/py/__metainfo.pyi +++ /dev/null @@ -1,3 +0,0 @@ -from typing import Any - -pydir: Any diff --git a/typeshed/3/py/_builtin.pyi b/typeshed/3/py/_builtin.pyi deleted file mode 100644 index aedc6c5725..0000000000 --- a/typeshed/3/py/_builtin.pyi +++ /dev/null @@ -1,16 +0,0 @@ -from typing import Any, Optional - -BaseException = BaseException -GeneratorExit = GeneratorExit -all = all -any = any -callable = callable -enumerate = enumerate -reversed = reversed -set: Any -frozenset: Any -sorted = sorted -text = str -bytes = bytes - -def execfile(fn: Any, globs: Optional[Any] = ..., locs: Optional[Any] = ...) -> None: ... diff --git a/typeshed/3/py/_code/__init__.pyi b/typeshed/3/py/_code/__init__.pyi deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/typeshed/3/py/_code/_assertionnew.pyi b/typeshed/3/py/_code/_assertionnew.pyi deleted file mode 100644 index 9c4e0a76b8..0000000000 --- a/typeshed/3/py/_code/_assertionnew.pyi +++ /dev/null @@ -1,31 +0,0 @@ -import ast -from py._code.assertion import BuiltinAssertionError as BuiltinAssertionError -from typing import Any, Optional - -class Failure(Exception): - cause: Any = ... - explanation: Any = ... - def __init__(self, explanation: str = ...) -> None: ... - -def interpret(source: Any, frame: Any, should_fail: bool = ...): ... -def run(offending_line: Any, frame: Optional[Any] = ...): ... -def getfailure(failure: Any): ... - -operator_map: Any -unary_map: Any - -class DebugInterpreter(ast.NodeVisitor): - frame: Any = ... - def __init__(self, frame: Any) -> None: ... - def generic_visit(self, node: Any): ... - def visit_Expr(self, expr: Any): ... - def visit_Module(self, mod: Any) -> None: ... - def visit_Name(self, name: Any): ... - def visit_Compare(self, comp: Any): ... - def visit_BoolOp(self, boolop: Any): ... - def visit_UnaryOp(self, unary: Any): ... - def visit_BinOp(self, binop: Any): ... - def visit_Call(self, call: Any): ... - def visit_Attribute(self, attr: Any): ... - def visit_Assert(self, assrt: Any): ... - def visit_Assign(self, assign: Any): ... diff --git a/typeshed/3/py/_code/_assertionold.pyi b/typeshed/3/py/_code/_assertionold.pyi deleted file mode 100644 index a60491e6ae..0000000000 --- a/typeshed/3/py/_code/_assertionold.pyi +++ /dev/null @@ -1,110 +0,0 @@ -from py._code.assertion import BuiltinAssertionError as BuiltinAssertionError -from typing import Any, Optional - -passthroughex: Any - -class Failure: - node: Any = ... - def __init__(self, node: Any) -> None: ... - -class View: - __view__: Any = ... - __obj__: Any = ... - __rootclass__: Any = ... - __class__: Any = ... - def __new__(rootclass: Any, obj: Any, *args: Any, **kwds: Any): ... - def __getattr__(self, attr: Any): ... - def __viewkey__(self): ... - def __matchkey__(self, key: Any, subclasses: Any): ... - -def enumsubclasses(cls) -> None: ... - -class Interpretable(View): - explanation: Any = ... - def is_builtin(self, frame: Any): ... - result: Any = ... - def eval(self, frame: Any) -> None: ... - def run(self, frame: Any) -> None: ... - def nice_explanation(self): ... - -class Name(Interpretable): - __view__: Any = ... - def is_local(self, frame: Any): ... - def is_global(self, frame: Any): ... - def is_builtin(self, frame: Any): ... - explanation: Any = ... - def eval(self, frame: Any) -> None: ... - -class Compare(Interpretable): - __view__: Any = ... - explanation: Any = ... - result: Any = ... - def eval(self, frame: Any) -> None: ... - -class And(Interpretable): - __view__: Any = ... - result: Any = ... - explanation: Any = ... - def eval(self, frame: Any) -> None: ... - -class Or(Interpretable): - __view__: Any = ... - result: Any = ... - explanation: Any = ... - def eval(self, frame: Any) -> None: ... - -keepalive: Any - -class UnaryArith(Interpretable): - __view__: Any = ... - explanation: Any = ... - result: Any = ... - def eval(self, frame: Any, astpattern: Any = ...) -> None: ... - -class BinaryArith(Interpretable): - __view__: Any = ... - explanation: Any = ... - result: Any = ... - def eval(self, frame: Any, astpattern: Any = ...) -> None: ... - -class CallFunc(Interpretable): - __view__: Any = ... - def is_bool(self, frame: Any): ... - explanation: Any = ... - result: Any = ... - def eval(self, frame: Any) -> None: ... - -class Getattr(Interpretable): - __view__: Any = ... - result: Any = ... - explanation: Any = ... - def eval(self, frame: Any) -> None: ... - -class Assert(Interpretable): - __view__: Any = ... - result: Any = ... - explanation: Any = ... - def run(self, frame: Any) -> None: ... - -class Assign(Interpretable): - __view__: Any = ... - result: Any = ... - explanation: Any = ... - def run(self, frame: Any) -> None: ... - -class Discard(Interpretable): - __view__: Any = ... - result: Any = ... - explanation: Any = ... - def run(self, frame: Any) -> None: ... - -class Stmt(Interpretable): - __view__: Any = ... - def run(self, frame: Any) -> None: ... - -def report_failure(e: Any) -> None: ... -def check(s: Any, frame: Optional[Any] = ...) -> None: ... -def interpret(source: Any, frame: Any, should_fail: bool = ...): ... -def getmsg(excinfo: Any): ... -def getfailure(e: Any): ... -def run(s: Any, frame: Optional[Any] = ...) -> None: ... diff --git a/typeshed/3/py/_code/_py2traceback.pyi b/typeshed/3/py/_code/_py2traceback.pyi deleted file mode 100644 index c2e4595fda..0000000000 --- a/typeshed/3/py/_code/_py2traceback.pyi +++ /dev/null @@ -1,3 +0,0 @@ -from typing import Any - -def format_exception_only(etype: Any, value: Any): ... diff --git a/typeshed/3/py/_code/assertion.pyi b/typeshed/3/py/_code/assertion.pyi deleted file mode 100644 index d81fc60978..0000000000 --- a/typeshed/3/py/_code/assertion.pyi +++ /dev/null @@ -1,10 +0,0 @@ -from typing import Any - -BuiltinAssertionError: Any - -class AssertionError(BuiltinAssertionError): - msg: Any = ... - args: Any = ... - def __init__(self, *args: Any) -> None: ... - -reinterpret_old: str diff --git a/typeshed/3/py/_code/code.pyi b/typeshed/3/py/_code/code.pyi deleted file mode 100644 index c2387d6dc6..0000000000 --- a/typeshed/3/py/_code/code.pyi +++ /dev/null @@ -1,162 +0,0 @@ -from typing import Any, Optional - -builtin_repr = repr -reprlib: Any - -class Code: - filename: Any = ... - firstlineno: Any = ... - name: Any = ... - raw: Any = ... - def __init__(self, rawcode: Any) -> None: ... - def __eq__(self, other: Any) -> Any: ... - def __ne__(self, other: Any) -> Any: ... - @property - def path(self): ... - @property - def fullsource(self): ... - def source(self): ... - def getargs(self, var: bool = ...): ... - -class Frame: - lineno: Any = ... - f_globals: Any = ... - f_locals: Any = ... - raw: Any = ... - code: Any = ... - def __init__(self, frame: Any) -> None: ... - @property - def statement(self): ... - def eval(self, code: Any, **vars: Any): ... - def exec_(self, code: Any, **vars: Any) -> None: ... - def repr(self, object: Any): ... - def is_true(self, object: Any): ... - def getargs(self, var: bool = ...): ... - -class TracebackEntry: - exprinfo: Any = ... - lineno: Any = ... - def __init__(self, rawentry: Any) -> None: ... - def set_repr_style(self, mode: Any) -> None: ... - @property - def frame(self): ... - @property - def relline(self): ... - @property - def statement(self): ... - @property - def path(self): ... - def getlocals(self): ... - locals: Any = ... - def reinterpret(self): ... - def getfirstlinesource(self): ... - def getsource(self, astcache: Optional[Any] = ...): ... - source: Any = ... - def ishidden(self): ... - def name(self): ... - name: Any = ... - -class Traceback(list): - Entry: Any = ... - def __init__(self, tb: Any) -> None: ... - def cut(self, path: Optional[Any] = ..., lineno: Optional[Any] = ..., firstlineno: Optional[Any] = ..., excludepath: Optional[Any] = ...): ... - def __getitem__(self, key: Any): ... - def filter(self, fn: Any = ...): ... - def getcrashentry(self): ... - def recursionindex(self): ... - -co_equal: Any - -class ExceptionInfo: - type: Any = ... - value: Any = ... - tb: Any = ... - typename: Any = ... - traceback: Any = ... - def __init__(self, tup: Optional[Any] = ..., exprinfo: Optional[Any] = ...) -> None: ... - def exconly(self, tryshort: bool = ...): ... - def errisinstance(self, exc: Any): ... - def getrepr(self, showlocals: bool = ..., style: str = ..., abspath: bool = ..., tbfilter: bool = ..., funcargs: bool = ...): ... - def __unicode__(self): ... - -class FormattedExcinfo: - flow_marker: str = ... - fail_marker: str = ... - showlocals: Any = ... - style: Any = ... - tbfilter: Any = ... - funcargs: Any = ... - abspath: Any = ... - astcache: Any = ... - def __init__(self, showlocals: bool = ..., style: str = ..., abspath: bool = ..., tbfilter: bool = ..., funcargs: bool = ...) -> None: ... - def repr_args(self, entry: Any): ... - def get_source(self, source: Any, line_index: int = ..., excinfo: Optional[Any] = ..., short: bool = ...): ... - def get_exconly(self, excinfo: Any, indent: int = ..., markall: bool = ...): ... - def repr_locals(self, locals: Any): ... - def repr_traceback_entry(self, entry: Any, excinfo: Optional[Any] = ...): ... - def repr_traceback(self, excinfo: Any): ... - def repr_excinfo(self, excinfo: Any): ... - -class TerminalRepr: - def __unicode__(self): ... - -class ReprExceptionInfo(TerminalRepr): - reprtraceback: Any = ... - reprcrash: Any = ... - sections: Any = ... - def __init__(self, reprtraceback: Any, reprcrash: Any) -> None: ... - def addsection(self, name: Any, content: Any, sep: str = ...) -> None: ... - def toterminal(self, tw: Any) -> None: ... - -class ReprTraceback(TerminalRepr): - entrysep: str = ... - reprentries: Any = ... - extraline: Any = ... - style: Any = ... - def __init__(self, reprentries: Any, extraline: Any, style: Any) -> None: ... - def toterminal(self, tw: Any) -> None: ... - -class ReprTracebackNative(ReprTraceback): - style: str = ... - reprentries: Any = ... - extraline: Any = ... - def __init__(self, tblines: Any) -> None: ... - -class ReprEntryNative(TerminalRepr): - style: str = ... - lines: Any = ... - def __init__(self, tblines: Any) -> None: ... - def toterminal(self, tw: Any) -> None: ... - -class ReprEntry(TerminalRepr): - localssep: str = ... - lines: Any = ... - reprfuncargs: Any = ... - reprlocals: Any = ... - reprfileloc: Any = ... - style: Any = ... - def __init__(self, lines: Any, reprfuncargs: Any, reprlocals: Any, filelocrepr: Any, style: Any) -> None: ... - def toterminal(self, tw: Any) -> None: ... - -class ReprFileLocation(TerminalRepr): - path: Any = ... - lineno: Any = ... - message: Any = ... - def __init__(self, path: Any, lineno: Any, message: Any) -> None: ... - def toterminal(self, tw: Any) -> None: ... - -class ReprLocals(TerminalRepr): - lines: Any = ... - def __init__(self, lines: Any) -> None: ... - def toterminal(self, tw: Any) -> None: ... - -class ReprFuncArgs(TerminalRepr): - args: Any = ... - def __init__(self, args: Any) -> None: ... - def toterminal(self, tw: Any) -> None: ... - -oldbuiltins: Any - -def patch_builtins(assertion: bool = ..., compile: bool = ...) -> None: ... -def unpatch_builtins(assertion: bool = ..., compile: bool = ...) -> None: ... -def getrawcode(obj: Any, trycall: bool = ...): ... diff --git a/typeshed/3/py/_code/source.pyi b/typeshed/3/py/_code/source.pyi deleted file mode 100644 index b432c3990e..0000000000 --- a/typeshed/3/py/_code/source.pyi +++ /dev/null @@ -1,29 +0,0 @@ -from types import ModuleType as ModuleType -from typing import Any, Optional - -cpy_compile = compile - -class Source: - lines: Any = ... - def __init__(self, *parts: Any, **kwargs: Any) -> None: ... - def __eq__(self, other: Any) -> Any: ... - def __getitem__(self, key: Any): ... - def __len__(self): ... - def __getslice__(self, start: Any, end: Any): ... - def strip(self): ... - def putaround(self, before: str = ..., after: str = ..., indent: Any = ...): ... - def indent(self, indent: Any = ...): ... - def getstatement(self, lineno: Any, assertion: bool = ...): ... - def getstatementrange(self, lineno: Any, assertion: bool = ...): ... - def deindent(self, offset: Optional[Any] = ...): ... - def isparseable(self, deindent: bool = ...): ... - def compile(self, filename: Optional[Any] = ..., mode: str = ..., flag: Any = ..., dont_inherit: int = ..., _genframe: Optional[Any] = ...): ... - -def compile_(source: Any, filename: Optional[Any] = ..., mode: str = ..., flags: Any = ..., dont_inherit: int = ...): ... -def getfslineno(obj: Any): ... -def findsource(obj: Any): ... -def getsource(obj: Any, **kwargs: Any): ... -def deindent(lines: Any, offset: Optional[Any] = ...): ... -def get_statement_startend2(lineno: Any, node: Any): ... -def getstatementrange_ast(lineno: Any, source: Any, assertion: bool = ..., astnode: Optional[Any] = ...): ... -def getstatementrange_old(lineno: Any, source: Any, assertion: bool = ...): ... diff --git a/typeshed/3/py/_error.pyi b/typeshed/3/py/_error.pyi deleted file mode 100644 index 0b9451fd46..0000000000 --- a/typeshed/3/py/_error.pyi +++ /dev/null @@ -1,11 +0,0 @@ -from types import ModuleType -from typing import Any - -class Error(EnvironmentError): ... - -class ErrorMaker(ModuleType): - Error: Any = ... - def __getattr__(self, name: Any): ... - def checked_call(self, func: Any, *args: Any, **kwargs: Any): ... - -error: Any diff --git a/typeshed/3/py/_io/__init__.pyi b/typeshed/3/py/_io/__init__.pyi deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/typeshed/3/py/_io/capture.pyi b/typeshed/3/py/_io/capture.pyi deleted file mode 100644 index 12e02600f5..0000000000 --- a/typeshed/3/py/_io/capture.pyi +++ /dev/null @@ -1,61 +0,0 @@ -from StringIO import StringIO -from io import BytesIO as BytesIO -from typing import Any, Optional - -TextIO = StringIO - -class BytesIO(StringIO): - def write(self, data: Any) -> None: ... - -patchsysdict: Any - -class FDCapture: - targetfd: Any = ... - tmpfile: Any = ... - def __init__(self, targetfd: Any, tmpfile: Optional[Any] = ..., now: bool = ..., patchsys: bool = ...) -> None: ... - def start(self) -> None: ... - def done(self): ... - def writeorg(self, data: Any) -> None: ... - -def dupfile(f: Any, mode: Optional[Any] = ..., buffering: int = ..., raising: bool = ..., encoding: Optional[Any] = ...): ... - -class EncodedFile: - encoding: Any = ... - def __init__(self, _stream: Any, encoding: Any) -> None: ... - def write(self, obj: Any) -> None: ... - def writelines(self, linelist: Any) -> None: ... - def __getattr__(self, name: Any): ... - -class Capture: - def call(cls, func: Any, *args: Any, **kwargs: Any): ... - call: Any = ... - def reset(self): ... - def suspend(self): ... - -class StdCaptureFD(Capture): - def __init__(self, out: bool = ..., err: bool = ..., mixed: bool = ..., in_: bool = ..., patchsys: bool = ..., now: bool = ...) -> None: ... - def startall(self) -> None: ... - def resume(self) -> None: ... - def done(self, save: bool = ...): ... - def readouterr(self): ... - -class StdCapture(Capture): - out: Any = ... - err: Any = ... - in_: Any = ... - def __init__(self, out: bool = ..., err: bool = ..., in_: bool = ..., mixed: bool = ..., now: bool = ...) -> None: ... - def startall(self) -> None: ... - def done(self, save: bool = ...): ... - def resume(self) -> None: ... - def readouterr(self): ... - -class DontReadFromInput: - def read(self, *args: Any) -> None: ... - readline: Any = ... - readlines: Any = ... - __iter__: Any = ... - def fileno(self) -> None: ... - def isatty(self): ... - def close(self) -> None: ... - -devnullpath: Any diff --git a/typeshed/3/py/_io/saferepr.pyi b/typeshed/3/py/_io/saferepr.pyi deleted file mode 100644 index 1d305001d2..0000000000 --- a/typeshed/3/py/_io/saferepr.pyi +++ /dev/null @@ -1,11 +0,0 @@ -from typing import Any - -builtin_repr = repr -reprlib: Any - -class SafeRepr(reprlib.Repr): - def repr(self, x: Any): ... - def repr_unicode(self, x: Any, level: Any): ... - def repr_instance(self, x: Any, level: Any): ... - -def saferepr(obj: Any, maxsize: int = ...): ... diff --git a/typeshed/3/py/_io/terminalwriter.pyi b/typeshed/3/py/_io/terminalwriter.pyi deleted file mode 100644 index 23dc0e2eea..0000000000 --- a/typeshed/3/py/_io/terminalwriter.pyi +++ /dev/null @@ -1,72 +0,0 @@ -import ctypes -from py.builtin import bytes as bytes, text as text -from typing import Any, Optional - -py3k: Any -py33: Any -win32_and_ctypes: bool -colorama: Any - -def get_terminal_width(): ... - -terminal_width: Any -char_width: Any - -def get_line_width(text: Any): ... -def ansi_print(text: Any, esc: Any, file: Optional[Any] = ..., newline: bool = ..., flush: bool = ...) -> None: ... -def should_do_markup(file: Any): ... - -class TerminalWriter: - stringio: Any = ... - encoding: Any = ... - hasmarkup: Any = ... - def __init__(self, file: Optional[Any] = ..., stringio: bool = ..., encoding: Optional[Any] = ...) -> None: ... - @property - def fullwidth(self): ... - @fullwidth.setter - def fullwidth(self, value: Any) -> None: ... - @property - def chars_on_current_line(self): ... - @property - def width_of_current_line(self): ... - def markup(self, text: Any, **kw: Any): ... - def sep(self, sepchar: Any, title: Optional[Any] = ..., fullwidth: Optional[Any] = ..., **kw: Any) -> None: ... - def write(self, msg: Any, **kw: Any) -> None: ... - def line(self, s: str = ..., **kw: Any) -> None: ... - def reline(self, line: Any, **kw: Any) -> None: ... - -class Win32ConsoleWriter(TerminalWriter): - def write(self, msg: Any, **kw: Any) -> None: ... - -class WriteFile: - encoding: Any = ... - def __init__(self, writemethod: Any, encoding: Optional[Any] = ...) -> None: ... - def write(self, data: Any) -> None: ... - def flush(self) -> None: ... -TerminalWriter = Win32ConsoleWriter -STD_OUTPUT_HANDLE: int -STD_ERROR_HANDLE: int -FOREGROUND_BLACK: int -FOREGROUND_BLUE: int -FOREGROUND_GREEN: int -FOREGROUND_RED: int -FOREGROUND_WHITE: int -FOREGROUND_INTENSITY: int -BACKGROUND_BLACK: int -BACKGROUND_BLUE: int -BACKGROUND_GREEN: int -BACKGROUND_RED: int -BACKGROUND_WHITE: int -BACKGROUND_INTENSITY: int -SHORT = ctypes.c_short - -class COORD(ctypes.Structure): ... -class SMALL_RECT(ctypes.Structure): ... -class CONSOLE_SCREEN_BUFFER_INFO(ctypes.Structure): ... - -def GetStdHandle(kind: Any): ... - -SetConsoleTextAttribute: Any - -def GetConsoleInfo(handle: Any): ... -def write_out(fil: Any, msg: Any) -> None: ... diff --git a/typeshed/3/py/_log/__init__.pyi b/typeshed/3/py/_log/__init__.pyi deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/typeshed/3/py/_log/log.pyi b/typeshed/3/py/_log/log.pyi deleted file mode 100644 index f4c4b80635..0000000000 --- a/typeshed/3/py/_log/log.pyi +++ /dev/null @@ -1,47 +0,0 @@ -from typing import Any, Optional - -class Message: - keywords: Any = ... - args: Any = ... - def __init__(self, keywords: Any, args: Any) -> None: ... - def content(self): ... - def prefix(self): ... - -class Producer: - Message: Any = ... - keywords2consumer: Any = ... - def __init__(self, keywords: Any, keywordmapper: Optional[Any] = ..., **kw: Any) -> None: ... - def __getattr__(self, name: Any): ... - def __call__(self, *args: Any) -> None: ... - -class KeywordMapper: - keywords2consumer: Any = ... - def __init__(self) -> None: ... - def getstate(self): ... - def setstate(self, state: Any) -> None: ... - def getconsumer(self, keywords: Any): ... - def setconsumer(self, keywords: Any, consumer: Any) -> None: ... - -def default_consumer(msg: Any) -> None: ... - -default_keywordmapper: Any - -def setconsumer(keywords: Any, consumer: Any) -> None: ... -def setstate(state: Any) -> None: ... -def getstate(): ... - -class File: - def __init__(self, f: Any) -> None: ... - def __call__(self, msg: Any) -> None: ... - -class Path: - def __init__(self, filename: Any, append: bool = ..., delayed_create: bool = ..., buffering: bool = ...) -> None: ... - def __call__(self, msg: Any) -> None: ... - -def STDOUT(msg: Any) -> None: ... -def STDERR(msg: Any) -> None: ... - -class Syslog: - priority: Any = ... - def __init__(self, priority: Optional[Any] = ...) -> None: ... - def __call__(self, msg: Any) -> None: ... diff --git a/typeshed/3/py/_log/warning.pyi b/typeshed/3/py/_log/warning.pyi deleted file mode 100644 index cd707a61de..0000000000 --- a/typeshed/3/py/_log/warning.pyi +++ /dev/null @@ -1,9 +0,0 @@ -from typing import Any, Optional - -class DeprecationWarning(DeprecationWarning): - msg: Any = ... - path: Any = ... - lineno: Any = ... - def __init__(self, msg: Any, path: Any, lineno: Any) -> None: ... - -def warn(msg: Any, stacklevel: int = ..., function: Optional[Any] = ...) -> None: ... diff --git a/typeshed/3/py/_path/__init__.pyi b/typeshed/3/py/_path/__init__.pyi deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/typeshed/3/py/_path/cacheutil.pyi b/typeshed/3/py/_path/cacheutil.pyi deleted file mode 100644 index 996bac798a..0000000000 --- a/typeshed/3/py/_path/cacheutil.pyi +++ /dev/null @@ -1,27 +0,0 @@ -from typing import Any - -class BasicCache: - maxentries: Any = ... - prunenum: Any = ... - def __init__(self, maxentries: int = ...) -> None: ... - def clear(self) -> None: ... - def delentry(self, key: Any, raising: bool = ...) -> None: ... - def getorbuild(self, key: Any, builder: Any): ... - -class BuildcostAccessCache(BasicCache): ... - -class WeightedCountingEntry: - weight: Any = ... - def __init__(self, value: Any, oneweight: Any) -> None: ... - def value(self): ... - value: Any = ... - -class AgingCache(BasicCache): - maxseconds: Any = ... - def __init__(self, maxentries: int = ..., maxseconds: float = ...) -> None: ... - -class AgingEntry: - value: Any = ... - weight: Any = ... - def __init__(self, value: Any, expirationtime: Any) -> None: ... - def isexpired(self): ... diff --git a/typeshed/3/py/_path/common.pyi b/typeshed/3/py/_path/common.pyi deleted file mode 100644 index d94a22e5a8..0000000000 --- a/typeshed/3/py/_path/common.pyi +++ /dev/null @@ -1,66 +0,0 @@ -from typing import Any, Optional, Generator - -iswin32: Any -import_errors: Any - -class Checkers: - path: Any = ... - def __init__(self, path: Any) -> None: ... - def dir(self) -> None: ... - def file(self) -> None: ... - def dotfile(self): ... - def ext(self, arg: Any): ... - def exists(self) -> None: ... - def basename(self, arg: Any): ... - def basestarts(self, arg: Any): ... - def relto(self, arg: Any): ... - def fnmatch(self, arg: Any): ... - def endswith(self, arg: Any): ... - -class NeverRaised(Exception): ... - -class PathBase: - Checkers: Any = ... - def __div__(self, other: Any): ... - __truediv__: Any = ... - def basename(self): ... - def dirname(self): ... - def purebasename(self): ... - def ext(self): ... - def dirpath(self, *args: Any, **kwargs: Any): ... - def read_binary(self): ... - def read_text(self, encoding: Any): ... - def read(self, mode: str = ...): ... - def readlines(self, cr: int = ...): ... - def load(self): ... - def move(self, target: Any) -> None: ... - def check(self, **kw: Any): ... - def fnmatch(self, pattern: Any): ... - def relto(self, relpath: Any): ... - def ensure_dir(self, *args: Any): ... - def bestrelpath(self, dest: Any): ... - def exists(self): ... - def isdir(self): ... - def isfile(self): ... - def parts(self, reverse: bool = ...): ... - def common(self, other: Any): ... - def __add__(self, other: Any): ... - def __cmp__(self, other: Any): ... - def __lt__(self, other: Any) -> Any: ... - def visit(self, fil: Optional[Any] = ..., rec: Optional[Any] = ..., ignore: Any = ..., bf: bool = ..., sort: bool = ...) -> Generator[PathBase, None, None]: ... - def samefile(self, other: Any): ... - def __fspath__(self): ... - -class Visitor: - rec: Any = ... - fil: Any = ... - ignore: Any = ... - breadthfirst: Any = ... - optsort: Any = ... - def __init__(self, fil: Any, rec: Any, ignore: Any, bf: Any, sort: Any): ... - def gen(self, path: Any) -> None: ... - -class FNMatcher: - pattern: Any = ... - def __init__(self, pattern: Any) -> None: ... - def __call__(self, path: Any): ... diff --git a/typeshed/3/py/_path/local.pyi b/typeshed/3/py/_path/local.pyi deleted file mode 100644 index 2ce17166f2..0000000000 --- a/typeshed/3/py/_path/local.pyi +++ /dev/null @@ -1,89 +0,0 @@ -from py._path import common -from py._path.common import iswin32 -from typing import Any, Optional - - -def map_as_list(func: Any, iter: Any): ... - -ALLOW_IMPORTLIB_MODE: Any - -class Stat: - def __getattr__(self, name: Any): ... - path: Any = ... - def __init__(self, path: Any, osstatresult: Any) -> None: ... - @property - def owner(self): ... - @property - def group(self): ... - def isdir(self): ... - def isfile(self): ... - def islink(self): ... - -class PosixPath(common.PathBase): - def chown(self, user: Any, group: Any, rec: int = ...): ... - def readlink(self): ... - def mklinkto(self, oldname: Any) -> None: ... - def mksymlinkto(self, value: Any, absolute: int = ...) -> None: ... - -FSBase = PosixPath #or common.PathBase - -def getuserid(user: Any): ... -def getgroupid(group: Any): ... - -class LocalPath(FSBase): - class ImportMismatchError(ImportError): ... - sep: Any = ... - class Checkers(common.Checkers): - def dir(self): ... - def file(self): ... - def exists(self): ... - def link(self): ... - strpath: Any = ... - def __init__(self, path: Optional[Any] = ..., expanduser: bool = ...) -> None: ... - def __hash__(self) -> Any: ... - def __eq__(self, other: Any) -> Any: ... - def __ne__(self, other: Any) -> Any: ... - def __lt__(self, other: Any) -> Any: ... - def __gt__(self, other: Any) -> Any: ... - def samefile(self, other: Any): ... - def remove(self, rec: int = ..., ignore_errors: bool = ...) -> None: ... - def computehash(self, hashtype: str = ..., chunksize: int = ...): ... - def new(self, **kw: Any): ... - def dirpath(self, *args: Any, **kwargs: Any): ... - def join(self, *args: Any, **kwargs: Any): ... - def open(self, mode: str = ..., ensure: bool = ..., encoding: Optional[Any] = ...): ... - def islink(self): ... - def check(self, **kw: Any): ... - def listdir(self, fil: Optional[Any] = ..., sort: Optional[Any] = ...): ... - def size(self): ... - def mtime(self): ... - def copy(self, target: Any, mode: bool = ..., stat: bool = ...): ... - def rename(self, target: Any): ... - def dump(self, obj: Any, bin: int = ...) -> None: ... - def mkdir(self, *args: Any): ... - def write_binary(self, data: Any, ensure: bool = ...) -> None: ... - def write_text(self, data: Any, encoding: Any, ensure: bool = ...) -> None: ... - def write(self, data: Any, mode: str = ..., ensure: bool = ...) -> None: ... - def ensure(self, *args: Any, **kwargs: Any): ... - def stat(self, raising: bool = ...): ... - def lstat(self): ... - def setmtime(self, mtime: Optional[Any] = ...): ... - def chdir(self): ... - def as_cwd(self) -> None: ... - def realpath(self): ... - def atime(self): ... - def chmod(self, mode: Any, rec: int = ...) -> None: ... - def pypkgpath(self): ... - def pyimport(self, modname: Optional[Any] = ..., ensuresyspath: bool = ...): ... - def sysexec(self, *argv: Any, **popen_opts: Any): ... - def sysfind(cls, name: Any, checker: Optional[Any] = ..., paths: Optional[Any] = ...): ... - @classmethod - def get_temproot(cls): ... - @classmethod - def mkdtemp(cls, rootdir: Optional[Any] = ...): ... - def make_numbered_dir(cls, prefix: str = ..., rootdir: Optional[Any] = ..., keep: int = ..., lock_timeout: int = ...): ... - -def copymode(src: Any, dest: Any) -> None: ... -def copystat(src: Any, dest: Any) -> None: ... -def copychunked(src: Any, dest: Any) -> None: ... -def isimportable(name: Any): ... diff --git a/typeshed/3/py/_path/svnurl.pyi b/typeshed/3/py/_path/svnurl.pyi deleted file mode 100644 index 0fb1565fc4..0000000000 --- a/typeshed/3/py/_path/svnurl.pyi +++ /dev/null @@ -1,43 +0,0 @@ -from py import path as path, process as process -from py._path import common as common, svnwc as svncommon -from py._path.cacheutil import AgingCache as AgingCache, BuildcostAccessCache as BuildcostAccessCache -from typing import Any, Optional - -DEBUG: bool - -class SvnCommandPath(svncommon.SvnPathBase): - strpath: Any = ... - rev: Any = ... - auth: Any = ... - def __new__(cls, path: Any, rev: Optional[Any] = ..., auth: Optional[Any] = ...): ... - def open(self, mode: str = ...): ... - def dirpath(self, *args: Any, **kwargs: Any): ... - def mkdir(self, *args: Any, **kwargs: Any): ... - def copy(self, target: Any, msg: str = ...) -> None: ... - def rename(self, target: Any, msg: str = ...) -> None: ... - def remove(self, rec: int = ..., msg: str = ...) -> None: ... - def export(self, topath: Any): ... - def ensure(self, *args: Any, **kwargs: Any): ... - def info(self): ... - def listdir(self, fil: Optional[Any] = ..., sort: Optional[Any] = ...): ... - def log(self, rev_start: Optional[Any] = ..., rev_end: int = ..., verbose: bool = ...): ... - -class InfoSvnCommand: - lspattern: Any = ... - kind: str = ... - created_rev: Any = ... - last_author: Any = ... - size: Any = ... - mtime: Any = ... - time: Any = ... - def __init__(self, line: Any) -> None: ... - def __eq__(self, other: Any) -> Any: ... - -def parse_time_with_missing_year(timestr: Any): ... - -class PathEntry: - strpath: Any = ... - action: Any = ... - copyfrom_path: Any = ... - copyfrom_rev: Any = ... - def __init__(self, ppart: Any) -> None: ... diff --git a/typeshed/3/py/_path/svnwc.pyi b/typeshed/3/py/_path/svnwc.pyi deleted file mode 100644 index 73f89e412d..0000000000 --- a/typeshed/3/py/_path/svnwc.pyi +++ /dev/null @@ -1,152 +0,0 @@ -import py -from py._path import common as common -from typing import Any, Optional - -class cache: - proplist: Any = ... - info: Any = ... - entries: Any = ... - prop: Any = ... - -class RepoEntry: - url: Any = ... - rev: Any = ... - timestamp: Any = ... - def __init__(self, url: Any, rev: Any, timestamp: Any) -> None: ... - -class RepoCache: - timeout: int = ... - repos: Any = ... - def __init__(self) -> None: ... - def clear(self) -> None: ... - def put(self, url: Any, rev: Any, timestamp: Optional[Any] = ...) -> None: ... - def get(self, url: Any): ... - -repositories: Any -ALLOWED_CHARS: str -ALLOWED_CHARS_HOST: Any - -def checkbadchars(url: Any) -> None: ... - -class SvnPathBase(common.PathBase): - sep: str = ... - url: Any = ... - def __hash__(self) -> Any: ... - def new(self, **kw: Any): ... - def __eq__(self, other: Any) -> Any: ... - def __ne__(self, other: Any) -> Any: ... - def join(self, *args: Any): ... - def propget(self, name: Any): ... - def proplist(self): ... - def size(self): ... - def mtime(self): ... - class Checkers(common.Checkers): - def dir(self): ... - def file(self): ... - def exists(self): ... - -def parse_apr_time(timestr: Any): ... - -class PropListDict(dict): - path: Any = ... - def __init__(self, path: Any, keynames: Any) -> None: ... - def __getitem__(self, key: Any): ... - -def fixlocale(): ... - -ILLEGAL_CHARS: Any -ISWINDOWS: Any - -def path_to_fspath(path: Any, addat: bool = ...): ... -def url_from_path(path: Any): ... - -class SvnAuth: - username: Any = ... - password: Any = ... - cache_auth: Any = ... - interactive: Any = ... - def __init__(self, username: Any, password: Any, cache_auth: bool = ..., interactive: bool = ...) -> None: ... - def makecmdoptions(self): ... - -rex_blame: Any - -class SvnWCCommandPath(common.PathBase): - sep: Any = ... - localpath: Any = ... - auth: Any = ... - def __new__(cls, wcpath: Optional[Any] = ..., auth: Optional[Any] = ...): ... - strpath: Any = ... - rev: Any = ... - def __eq__(self, other: Any) -> Any: ... - url: Any = ... - def dump(self, obj: Any): ... - def svnurl(self): ... - def switch(self, url: Any) -> None: ... - def checkout(self, url: Optional[Any] = ..., rev: Optional[Any] = ...) -> None: ... - def update(self, rev: str = ..., interactive: bool = ...) -> None: ... - def write(self, content: Any, mode: str = ...) -> None: ... - def ensure(self, *args: Any, **kwargs: Any): ... - def mkdir(self, *args: Any): ... - def add(self) -> None: ... - def remove(self, rec: int = ..., force: int = ...) -> None: ... - def copy(self, target: Any) -> None: ... - def rename(self, target: Any) -> None: ... - def lock(self) -> None: ... - def unlock(self) -> None: ... - def cleanup(self) -> None: ... - def status(self, updates: int = ..., rec: int = ..., externals: int = ...): ... - def diff(self, rev: Optional[Any] = ...): ... - def blame(self): ... - def commit(self, msg: str = ..., rec: int = ...): ... - def propset(self, name: Any, value: Any, *args: Any) -> None: ... - def propget(self, name: Any): ... - def propdel(self, name: Any): ... - def proplist(self, rec: int = ...): ... - def revert(self, rec: int = ...): ... - def new(self, **kw: Any): ... - def join(self, *args: Any, **kwargs: Any): ... - def info(self, usecache: int = ...): ... - def listdir(self, fil: Optional[Any] = ..., sort: Optional[Any] = ...): ... - def open(self, mode: str = ...): ... - def log(self, rev_start: Optional[Any] = ..., rev_end: int = ..., verbose: bool = ...): ... - def size(self): ... - def mtime(self): ... - def __hash__(self) -> Any: ... - -class WCStatus: - attrnames: Any = ... - wcpath: Any = ... - rev: Any = ... - modrev: Any = ... - author: Any = ... - def __init__(self, wcpath: Any, rev: Optional[Any] = ..., modrev: Optional[Any] = ..., author: Optional[Any] = ...) -> None: ... - def allpath(self, sort: bool = ..., **kw: Any): ... - def fromstring(data: Any, rootwcpath: Any, rev: Optional[Any] = ..., modrev: Optional[Any] = ..., author: Optional[Any] = ...): ... - -class XMLWCStatus(WCStatus): - def fromstring(data: Any, rootwcpath: Any, rev: Optional[Any] = ..., modrev: Optional[Any] = ..., author: Optional[Any] = ...): ... - -class InfoSvnWCCommand: - url: Any = ... - kind: Any = ... - rev: Any = ... - path: Any = ... - size: Any = ... - created_rev: Any = ... - last_author: Any = ... - mtime: Any = ... - time: Any = ... - def __init__(self, output: Any) -> None: ... - def __eq__(self, other: Any) -> Any: ... - -def parse_wcinfotime(timestr: Any): ... -def make_recursive_propdict(wcroot: Any, output: Any, rex: Any = ...): ... -def importxml(cache: Any = ...): ... - -class LogEntry: - rev: Any = ... - author: Any = ... - msg: Any = ... - date: Any = ... - strpaths: Any = ... - def __init__(self, logentry: Any) -> None: ... diff --git a/typeshed/3/py/_process/__init__.pyi b/typeshed/3/py/_process/__init__.pyi deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/typeshed/3/py/_process/cmdexec.pyi b/typeshed/3/py/_process/cmdexec.pyi deleted file mode 100644 index f3f330f013..0000000000 --- a/typeshed/3/py/_process/cmdexec.pyi +++ /dev/null @@ -1,13 +0,0 @@ -import py -from subprocess import PIPE as PIPE, Popen as Popen -from typing import Any - -def cmdexec(cmd: Any): ... - -class ExecutionFailed(py.error.Error): - status: Any = ... - systemstatus: Any = ... - cmd: Any = ... - err: Any = ... - out: Any = ... - def __init__(self, status: Any, systemstatus: Any, cmd: Any, out: Any, err: Any) -> None: ... diff --git a/typeshed/3/py/_process/forkedfunc.pyi b/typeshed/3/py/_process/forkedfunc.pyi deleted file mode 100644 index 716c3acb83..0000000000 --- a/typeshed/3/py/_process/forkedfunc.pyi +++ /dev/null @@ -1,25 +0,0 @@ -from typing import Any, Optional - -def get_unbuffered_io(fd: Any, filename: Any): ... - -class ForkedFunc: - EXITSTATUS_EXCEPTION: int = ... - fun: Any = ... - args: Any = ... - kwargs: Any = ... - tempdir: Any = ... - RETVAL: Any = ... - STDOUT: Any = ... - STDERR: Any = ... - pid: Any = ... - def __init__(self, fun: Any, args: Optional[Any] = ..., kwargs: Optional[Any] = ..., nice_level: int = ..., child_on_start: Optional[Any] = ..., child_on_exit: Optional[Any] = ...) -> None: ... - def waitfinish(self, waiter: Any = ...): ... - def __del__(self) -> None: ... - -class Result: - exitstatus: Any = ... - signal: Any = ... - retval: Any = ... - out: Any = ... - err: Any = ... - def __init__(self, exitstatus: Any, signal: Any, retval: Any, stdout: Any, stderr: Any) -> None: ... diff --git a/typeshed/3/py/_process/killproc.pyi b/typeshed/3/py/_process/killproc.pyi deleted file mode 100644 index ed2f9b3d0f..0000000000 --- a/typeshed/3/py/_process/killproc.pyi +++ /dev/null @@ -1,4 +0,0 @@ -from typing import Any - -def dokill(pid: Any) -> None: ... -def kill(pid: Any) -> None: ... diff --git a/typeshed/3/py/_std.pyi b/typeshed/3/py/_std.pyi deleted file mode 100644 index 303052b061..0000000000 --- a/typeshed/3/py/_std.pyi +++ /dev/null @@ -1,10 +0,0 @@ -from typing import Any - -class PyStdIsDeprecatedWarning(DeprecationWarning): ... - -class Std: - __dict__: Any = ... - def __init__(self) -> None: ... - def __getattr__(self, name: Any): ... - -std: Any diff --git a/typeshed/3/py/_version.pyi b/typeshed/3/py/_version.pyi deleted file mode 100644 index c2ee2cab48..0000000000 --- a/typeshed/3/py/_version.pyi +++ /dev/null @@ -1 +0,0 @@ -version: str diff --git a/typeshed/3/py/_xmlgen.pyi b/typeshed/3/py/_xmlgen.pyi deleted file mode 100644 index 85d1297b80..0000000000 --- a/typeshed/3/py/_xmlgen.pyi +++ /dev/null @@ -1,61 +0,0 @@ -from typing import Any, Optional - -def u(s: Any): ... -def unicode(x: Any, errors: Optional[Any] = ...): ... - -class NamespaceMetaclass(type): - def __getattr__(self, name: Any): ... - -class Tag(list): - class Attr: - def __init__(self, **kwargs: Any) -> None: ... - attr: Any = ... - def __init__(self, *args: Any, **kwargs: Any) -> None: ... - def __unicode__(self): ... - def unicode(self, indent: int = ...): ... - -Namespace: Any - -class HtmlTag(Tag): - def unicode(self, indent: int = ...): ... - -class html(Namespace): - __tagclass__: Any = ... - __stickyname__: bool = ... - __tagspec__: Any = ... - class Style: - def __init__(self, **kw: Any) -> None: ... - -class raw: - uniobj: Any = ... - def __init__(self, uniobj: Any) -> None: ... - -class SimpleUnicodeVisitor: - write: Any = ... - cache: Any = ... - visited: Any = ... - indent: Any = ... - curindent: Any = ... - parents: Any = ... - shortempty: Any = ... - def __init__(self, write: Any, indent: int = ..., curindent: int = ..., shortempty: bool = ...) -> None: ... - def visit(self, node: Any) -> None: ... - def raw(self, obj: Any) -> None: ... - def list(self, obj: Any) -> None: ... - def Tag(self, tag: Any) -> None: ... - def attributes(self, tag: Any): ... - def repr_attribute(self, attrs: Any, name: Any): ... - def getstyle(self, tag: Any): ... - -class HtmlVisitor(SimpleUnicodeVisitor): - single: Any = ... - inline: Any = ... - def repr_attribute(self, attrs: Any, name: Any): ... - -class _escape: - escape: Any = ... - charef_rex: Any = ... - def __init__(self) -> None: ... - def __call__(self, ustring: Any): ... - -escape: Any diff --git a/typeshed/3/py/path.pyi b/typeshed/3/py/path.pyi deleted file mode 100644 index 3070f07f47..0000000000 --- a/typeshed/3/py/path.pyi +++ /dev/null @@ -1,148 +0,0 @@ -from typing import Any - -import py._path.common -import py._path.local -import py._path.svnwc -_ApiModule__doc: str - -class SvnAuth: - __init__: Any = ... - makecmdoptions: Any = ... - -class local(py._path.local.PosixPath): - __init__: Any = ... - Checkers: Any = ... - ImportMismatchError: Any = ... - _ensuredirs: Any = ... - _ensuresyspath: Any = ... - _fastjoin: Any = ... - _getbyspec: Any = ... - _patternchars: Any = ... - as_cwd: Any = ... - atime: Any = ... - chdir: Any = ... - check: Any = ... - chmod: Any = ... - computehash: Any = ... - copy: Any = ... - dirpath: Any = ... - dump: Any = ... - ensure: Any = ... - islink: Any = ... - join: Any = ... - listdir: Any = ... - lstat: Any = ... - mkdir: Any = ... - mtime: Any = ... - new: Any = ... - open: Any = ... - pyimport: Any = ... - pypkgpath: Any = ... - realpath: Any = ... - remove: Any = ... - rename: Any = ... - samefile: Any = ... - sep: Any = ... - setmtime: Any = ... - size: Any = ... - stat: Any = ... - sysexec: Any = ... - write: Any = ... - write_binary: Any = ... - write_text: Any = ... - __eq__: Any = ... - __gt__: Any = ... - __hash__: Any = ... - __lt__: Any = ... - __ne__: Any = ... - @classmethod - def _gethomedir(self, *args, **kwargs) -> Any: ... - @classmethod - def get_temproot(self, *args, **kwargs) -> Any: ... - @classmethod - def make_numbered_dir(self, *args, **kwargs) -> Any: ... - @classmethod - def mkdtemp(self, *args, **kwargs) -> Any: ... - @classmethod - def sysfind(self, *args, **kwargs) -> Any: ... - -class svnurl(py._path.svnwc.SvnPathBase): - _cmdexec: Any = ... - _encodedurl: Any = ... - _listdir_nameinfo: Any = ... - _lsnorevcache: Any = ... - _lsrevcache: Any = ... - _norev_delentry: Any = ... - _popen: Any = ... - _propget: Any = ... - _proplist: Any = ... - _svncmdexecauth: Any = ... - _svnpopenauth: Any = ... - _svnwithrev: Any = ... - _svnwrite: Any = ... - copy: Any = ... - dirpath: Any = ... - ensure: Any = ... - export: Any = ... - info: Any = ... - listdir: Any = ... - log: Any = ... - mkdir: Any = ... - open: Any = ... - remove: Any = ... - rename: Any = ... - def __init__(self, *args, **kwargs) -> None: ... - -class svnwc(py._path.common.PathBase): - Checkers: Any = ... - _authsvn: Any = ... - _ensuredirs: Any = ... - _escape: Any = ... - _getbyspec: Any = ... - _geturl: Any = ... - _makeauthoptions: Any = ... - _rex_commit: Any = ... - _svn: Any = ... - add: Any = ... - blame: Any = ... - checkout: Any = ... - cleanup: Any = ... - commit: Any = ... - copy: Any = ... - diff: Any = ... - dirpath: Any = ... - dump: Any = ... - ensure: Any = ... - info: Any = ... - join: Any = ... - listdir: Any = ... - lock: Any = ... - log: Any = ... - mkdir: Any = ... - mtime: Any = ... - new: Any = ... - open: Any = ... - propdel: Any = ... - propget: Any = ... - proplist: Any = ... - propset: Any = ... - remove: Any = ... - rename: Any = ... - revert: Any = ... - sep: Any = ... - size: Any = ... - status: Any = ... - svnurl: Any = ... - switch: Any = ... - unlock: Any = ... - update: Any = ... - write: Any = ... - __eq__: Any = ... - __hash__: Any = ... - def __init__(self, *args, **kwargs) -> None: ... - @property - def rev(self) -> Any: ... - @property - def strpath(self) -> Any: ... - @property - def url(self) -> Any: ... diff --git a/typeshed/3/subprocess.pyi b/typeshed/3/subprocess.pyi deleted file mode 100644 index 5494d927d0..0000000000 --- a/typeshed/3/subprocess.pyi +++ /dev/null @@ -1,1208 +0,0 @@ -# Stubs for subprocess - -# Based on http://docs.python.org/3.6/library/subprocess.html -import sys -from typing import Sequence, Any, Mapping, Callable, Tuple, IO, Optional, Union, Type, Text, Generic, TypeVar, AnyStr, overload -from types import TracebackType - -if sys.version_info >= (3, 8): - from typing import Literal -else: - from typing_extensions import Literal - -# We prefer to annotate inputs to methods (eg subprocess.check_call) with these -# union types. -# For outputs we use laborious literal based overloads to try to determine -# which specific return types to use, and prefer to fall back to Any when -# this does not work, so the caller does not have to use an assertion to confirm -# which type. -# -# For example: -# -# try: -# x = subprocess.check_output(["ls", "-l"]) -# reveal_type(x) # bytes, based on the overloads -# except TimeoutError as e: -# reveal_type(e.cmd) # Any, but morally is _CMD -_FILE = Union[None, int, IO[Any]] -_TXT = Union[bytes, Text] -if sys.version_info >= (3, 6): - from builtins import _PathLike - _PATH = Union[bytes, Text, _PathLike] -else: - _PATH = Union[bytes, Text] -# Python 3.6 does't support _CMD being a single PathLike. -# See: https://bugs.python.org/issue31961 -_CMD = Union[_TXT, Sequence[_PATH]] -_ENV = Union[Mapping[bytes, _TXT], Mapping[Text, _TXT]] - -_S = TypeVar('_S') -_T = TypeVar('_T') - -class CompletedProcess(Generic[_T]): - # morally: _CMD - args: Any - returncode: Optional[int] - # These are really both Optional, but requiring checks would be tedious - # and writing all the overloads would be horrific. - stdout: _T - stderr: _T - def __init__(self, args: _CMD, returncode: int, stdout: Optional[_T] = ..., stderr: Optional[_T] = ...) -> None: ... - def check_returncode(self) -> None: ... - -if sys.version_info >= (3, 7): - # Nearly the same args as for 3.6, except for capture_output and text - @overload - def run( - args: _CMD, - bufsize: int = ..., - executable: _PATH = ..., - stdin: _FILE = ..., - stdout: _FILE = ..., - stderr: _FILE = ..., - preexec_fn: Callable[[], Any] = ..., - close_fds: bool = ..., - shell: bool = ..., - cwd: Optional[_PATH] = ..., - env: Optional[_ENV] = ..., - universal_newlines: bool = ..., - startupinfo: Any = ..., - creationflags: int = ..., - restore_signals: bool = ..., - start_new_session: bool = ..., - pass_fds: Any = ..., - *, - capture_output: bool = ..., - check: bool = ..., - encoding: Optional[str] = ..., - errors: Optional[str] = ..., - input: Optional[str] = ..., - text: Literal[True], - timeout: Optional[float] = ..., - ) -> CompletedProcess[str]: ... - @overload - def run( - args: _CMD, - bufsize: int = ..., - executable: _PATH = ..., - stdin: _FILE = ..., - stdout: _FILE = ..., - stderr: _FILE = ..., - preexec_fn: Callable[[], Any] = ..., - close_fds: bool = ..., - shell: bool = ..., - cwd: Optional[_PATH] = ..., - env: Optional[_ENV] = ..., - universal_newlines: bool = ..., - startupinfo: Any = ..., - creationflags: int = ..., - restore_signals: bool = ..., - start_new_session: bool = ..., - pass_fds: Any = ..., - *, - capture_output: bool = ..., - check: bool = ..., - encoding: str, - errors: Optional[str] = ..., - input: Optional[str] = ..., - text: Optional[bool] = ..., - timeout: Optional[float] = ..., - ) -> CompletedProcess[str]: ... - @overload - def run( - args: _CMD, - bufsize: int = ..., - executable: _PATH = ..., - stdin: _FILE = ..., - stdout: _FILE = ..., - stderr: _FILE = ..., - preexec_fn: Callable[[], Any] = ..., - close_fds: bool = ..., - shell: bool = ..., - cwd: Optional[_PATH] = ..., - env: Optional[_ENV] = ..., - universal_newlines: bool = ..., - startupinfo: Any = ..., - creationflags: int = ..., - restore_signals: bool = ..., - start_new_session: bool = ..., - pass_fds: Any = ..., - *, - capture_output: bool = ..., - check: bool = ..., - encoding: Optional[str] = ..., - errors: str, - input: Optional[str] = ..., - text: Optional[bool] = ..., - timeout: Optional[float] = ..., - ) -> CompletedProcess[str]: ... - @overload - def run( - args: _CMD, - bufsize: int = ..., - executable: _PATH = ..., - stdin: _FILE = ..., - stdout: _FILE = ..., - stderr: _FILE = ..., - preexec_fn: Callable[[], Any] = ..., - close_fds: bool = ..., - shell: bool = ..., - cwd: Optional[_PATH] = ..., - env: Optional[_ENV] = ..., - *, - universal_newlines: Literal[True], - startupinfo: Any = ..., - creationflags: int = ..., - restore_signals: bool = ..., - start_new_session: bool = ..., - pass_fds: Any = ..., - # where the *real* keyword only args start - capture_output: bool = ..., - check: bool = ..., - encoding: Optional[str] = ..., - errors: Optional[str] = ..., - input: Optional[str] = ..., - text: Optional[bool] = ..., - timeout: Optional[float] = ..., - ) -> CompletedProcess[str]: ... - @overload - def run( - args: _CMD, - bufsize: int = ..., - executable: _PATH = ..., - stdin: _FILE = ..., - stdout: _FILE = ..., - stderr: _FILE = ..., - preexec_fn: Callable[[], Any] = ..., - close_fds: bool = ..., - shell: bool = ..., - cwd: Optional[_PATH] = ..., - env: Optional[_ENV] = ..., - universal_newlines: Literal[False] = ..., - startupinfo: Any = ..., - creationflags: int = ..., - restore_signals: bool = ..., - start_new_session: bool = ..., - pass_fds: Any = ..., - *, - capture_output: bool = ..., - check: bool = ..., - encoding: None = ..., - errors: None = ..., - input: Optional[bytes] = ..., - text: Literal[None, False] = ..., - timeout: Optional[float] = ..., - ) -> CompletedProcess[bytes]: ... - @overload - def run( - args: _CMD, - bufsize: int = ..., - executable: _PATH = ..., - stdin: _FILE = ..., - stdout: _FILE = ..., - stderr: _FILE = ..., - preexec_fn: Callable[[], Any] = ..., - close_fds: bool = ..., - shell: bool = ..., - cwd: Optional[_PATH] = ..., - env: Optional[_ENV] = ..., - universal_newlines: bool = ..., - startupinfo: Any = ..., - creationflags: int = ..., - restore_signals: bool = ..., - start_new_session: bool = ..., - pass_fds: Any = ..., - *, - capture_output: bool = ..., - check: bool = ..., - encoding: Optional[str] = ..., - errors: Optional[str] = ..., - input: Optional[_TXT] = ..., - text: Optional[bool] = ..., - timeout: Optional[float] = ..., - ) -> CompletedProcess[Any]: ... -elif sys.version_info >= (3, 6): - # Nearly same args as Popen.__init__ except for timeout, input, and check - @overload - def run( - args: _CMD, - bufsize: int = ..., - executable: _PATH = ..., - stdin: _FILE = ..., - stdout: _FILE = ..., - stderr: _FILE = ..., - preexec_fn: Callable[[], Any] = ..., - close_fds: bool = ..., - shell: bool = ..., - cwd: Optional[_PATH] = ..., - env: Optional[_ENV] = ..., - universal_newlines: bool = ..., - startupinfo: Any = ..., - creationflags: int = ..., - restore_signals: bool = ..., - start_new_session: bool = ..., - pass_fds: Any = ..., - *, - check: bool = ..., - encoding: str, - errors: Optional[str] = ..., - input: Optional[str] = ..., - timeout: Optional[float] = ..., - ) -> CompletedProcess[str]: ... - @overload - def run( - args: _CMD, - bufsize: int = ..., - executable: _PATH = ..., - stdin: _FILE = ..., - stdout: _FILE = ..., - stderr: _FILE = ..., - preexec_fn: Callable[[], Any] = ..., - close_fds: bool = ..., - shell: bool = ..., - cwd: Optional[_PATH] = ..., - env: Optional[_ENV] = ..., - universal_newlines: bool = ..., - startupinfo: Any = ..., - creationflags: int = ..., - restore_signals: bool = ..., - start_new_session: bool = ..., - pass_fds: Any = ..., - *, - check: bool = ..., - encoding: Optional[str] = ..., - errors: str, - input: Optional[str] = ..., - timeout: Optional[float] = ..., - ) -> CompletedProcess[str]: ... - @overload - def run( - args: _CMD, - bufsize: int = ..., - executable: _PATH = ..., - stdin: _FILE = ..., - stdout: _FILE = ..., - stderr: _FILE = ..., - preexec_fn: Callable[[], Any] = ..., - close_fds: bool = ..., - shell: bool = ..., - cwd: Optional[_PATH] = ..., - env: Optional[_ENV] = ..., - *, - universal_newlines: Literal[True], - startupinfo: Any = ..., - creationflags: int = ..., - restore_signals: bool = ..., - start_new_session: bool = ..., - pass_fds: Any = ..., - # where the *real* keyword only args start - check: bool = ..., - encoding: Optional[str] = ..., - errors: Optional[str] = ..., - input: Optional[str] = ..., - timeout: Optional[float] = ..., - ) -> CompletedProcess[str]: ... - @overload - def run( - args: _CMD, - bufsize: int = ..., - executable: _PATH = ..., - stdin: _FILE = ..., - stdout: _FILE = ..., - stderr: _FILE = ..., - preexec_fn: Callable[[], Any] = ..., - close_fds: bool = ..., - shell: bool = ..., - cwd: Optional[_PATH] = ..., - env: Optional[_ENV] = ..., - universal_newlines: Literal[False] = ..., - startupinfo: Any = ..., - creationflags: int = ..., - restore_signals: bool = ..., - start_new_session: bool = ..., - pass_fds: Any = ..., - *, - check: bool = ..., - encoding: None = ..., - errors: None = ..., - input: Optional[bytes] = ..., - timeout: Optional[float] = ..., - ) -> CompletedProcess[bytes]: ... - @overload - def run( - args: _CMD, - bufsize: int = ..., - executable: _PATH = ..., - stdin: _FILE = ..., - stdout: _FILE = ..., - stderr: _FILE = ..., - preexec_fn: Callable[[], Any] = ..., - close_fds: bool = ..., - shell: bool = ..., - cwd: Optional[_PATH] = ..., - env: Optional[_ENV] = ..., - universal_newlines: bool = ..., - startupinfo: Any = ..., - creationflags: int = ..., - restore_signals: bool = ..., - start_new_session: bool = ..., - pass_fds: Any = ..., - *, - check: bool = ..., - encoding: Optional[str] = ..., - errors: Optional[str] = ..., - input: Optional[_TXT] = ..., - timeout: Optional[float] = ..., - ) -> CompletedProcess[Any]: ... -else: - # Nearly same args as Popen.__init__ except for timeout, input, and check - @overload - def run( - args: _CMD, - bufsize: int = ..., - executable: _PATH = ..., - stdin: _FILE = ..., - stdout: _FILE = ..., - stderr: _FILE = ..., - preexec_fn: Callable[[], Any] = ..., - close_fds: bool = ..., - shell: bool = ..., - cwd: Optional[_PATH] = ..., - env: Optional[_ENV] = ..., - *, - universal_newlines: Literal[True], - startupinfo: Any = ..., - creationflags: int = ..., - restore_signals: bool = ..., - start_new_session: bool = ..., - pass_fds: Any = ..., - # where the *real* keyword only args start - check: bool = ..., - input: Optional[str] = ..., - timeout: Optional[float] = ..., - ) -> CompletedProcess[str]: ... - @overload - def run( - args: _CMD, - bufsize: int = ..., - executable: _PATH = ..., - stdin: _FILE = ..., - stdout: _FILE = ..., - stderr: _FILE = ..., - preexec_fn: Callable[[], Any] = ..., - close_fds: bool = ..., - shell: bool = ..., - cwd: Optional[_PATH] = ..., - env: Optional[_ENV] = ..., - universal_newlines: Literal[False] = ..., - startupinfo: Any = ..., - creationflags: int = ..., - restore_signals: bool = ..., - start_new_session: bool = ..., - pass_fds: Any = ..., - *, - check: bool = ..., - input: Optional[bytes] = ..., - timeout: Optional[float] = ..., - ) -> CompletedProcess[bytes]: ... - @overload - def run( - args: _CMD, - bufsize: int = ..., - executable: _PATH = ..., - stdin: _FILE = ..., - stdout: _FILE = ..., - stderr: _FILE = ..., - preexec_fn: Callable[[], Any] = ..., - close_fds: bool = ..., - shell: bool = ..., - cwd: Optional[_PATH] = ..., - env: Optional[_ENV] = ..., - universal_newlines: bool = ..., - startupinfo: Any = ..., - creationflags: int = ..., - restore_signals: bool = ..., - start_new_session: bool = ..., - pass_fds: Any = ..., - *, - check: bool = ..., - input: Optional[_TXT] = ..., - timeout: Optional[float] = ..., - ) -> CompletedProcess[Any]: ... - -# Same args as Popen.__init__ -def call(args: _CMD, - bufsize: int = ..., - executable: _PATH = ..., - stdin: _FILE = ..., - stdout: _FILE = ..., - stderr: _FILE = ..., - preexec_fn: Callable[[], Any] = ..., - close_fds: bool = ..., - shell: bool = ..., - cwd: Optional[_PATH] = ..., - env: Optional[_ENV] = ..., - universal_newlines: bool = ..., - startupinfo: Any = ..., - creationflags: int = ..., - restore_signals: bool = ..., - start_new_session: bool = ..., - pass_fds: Any = ..., - timeout: Optional[float] = ...) -> int: ... - -# Same args as Popen.__init__ -def check_call(args: _CMD, - bufsize: int = ..., - executable: _PATH = ..., - stdin: _FILE = ..., - stdout: _FILE = ..., - stderr: _FILE = ..., - preexec_fn: Callable[[], Any] = ..., - close_fds: bool = ..., - shell: bool = ..., - cwd: Optional[_PATH] = ..., - env: Optional[_ENV] = ..., - universal_newlines: bool = ..., - startupinfo: Any = ..., - creationflags: int = ..., - restore_signals: bool = ..., - start_new_session: bool = ..., - pass_fds: Any = ..., - timeout: Optional[float] = ...) -> int: ... - -if sys.version_info >= (3, 7): - # 3.7 added text - @overload - def check_output(args: _CMD, - bufsize: int = ..., - executable: _PATH = ..., - stdin: _FILE = ..., - stderr: _FILE = ..., - preexec_fn: Callable[[], Any] = ..., - close_fds: bool = ..., - shell: bool = ..., - cwd: Optional[_PATH] = ..., - env: Optional[_ENV] = ..., - universal_newlines: bool = ..., - startupinfo: Any = ..., - creationflags: int = ..., - restore_signals: bool = ..., - start_new_session: bool = ..., - pass_fds: Any = ..., - *, - timeout: Optional[float] = ..., - input: _TXT = ..., - encoding: Optional[str] = ..., - errors: Optional[str] = ..., - text: Literal[True], - ) -> str: ... - @overload - def check_output(args: _CMD, - bufsize: int = ..., - executable: _PATH = ..., - stdin: _FILE = ..., - stderr: _FILE = ..., - preexec_fn: Callable[[], Any] = ..., - close_fds: bool = ..., - shell: bool = ..., - cwd: Optional[_PATH] = ..., - env: Optional[_ENV] = ..., - universal_newlines: bool = ..., - startupinfo: Any = ..., - creationflags: int = ..., - restore_signals: bool = ..., - start_new_session: bool = ..., - pass_fds: Any = ..., - *, - timeout: Optional[float] = ..., - input: _TXT = ..., - encoding: str, - errors: Optional[str] = ..., - text: Optional[bool] = ..., - ) -> str: ... - @overload - def check_output(args: _CMD, - bufsize: int = ..., - executable: _PATH = ..., - stdin: _FILE = ..., - stderr: _FILE = ..., - preexec_fn: Callable[[], Any] = ..., - close_fds: bool = ..., - shell: bool = ..., - cwd: Optional[_PATH] = ..., - env: Optional[_ENV] = ..., - universal_newlines: bool = ..., - startupinfo: Any = ..., - creationflags: int = ..., - restore_signals: bool = ..., - start_new_session: bool = ..., - pass_fds: Any = ..., - *, - timeout: Optional[float] = ..., - input: _TXT = ..., - encoding: Optional[str] = ..., - errors: str, - text: Optional[bool] = ..., - ) -> str: ... - @overload - def check_output(args: _CMD, - bufsize: int = ..., - executable: _PATH = ..., - stdin: _FILE = ..., - stderr: _FILE = ..., - preexec_fn: Callable[[], Any] = ..., - close_fds: bool = ..., - shell: bool = ..., - cwd: Optional[_PATH] = ..., - env: Optional[_ENV] = ..., - *, - universal_newlines: Literal[True], - startupinfo: Any = ..., - creationflags: int = ..., - restore_signals: bool = ..., - start_new_session: bool = ..., - pass_fds: Any = ..., - # where the real keyword only ones start - timeout: Optional[float] = ..., - input: _TXT = ..., - encoding: Optional[str] = ..., - errors: Optional[str] = ..., - text: Optional[bool] = ..., - ) -> str: ... - @overload - def check_output(args: _CMD, - bufsize: int = ..., - executable: _PATH = ..., - stdin: _FILE = ..., - stderr: _FILE = ..., - preexec_fn: Callable[[], Any] = ..., - close_fds: bool = ..., - shell: bool = ..., - cwd: Optional[_PATH] = ..., - env: Optional[_ENV] = ..., - universal_newlines: Literal[False] = ..., - startupinfo: Any = ..., - creationflags: int = ..., - restore_signals: bool = ..., - start_new_session: bool = ..., - pass_fds: Any = ..., - *, - timeout: Optional[float] = ..., - input: _TXT = ..., - encoding: None = ..., - errors: None = ..., - text: Literal[None, False] = ..., - ) -> bytes: ... - @overload - def check_output(args: _CMD, - bufsize: int = ..., - executable: _PATH = ..., - stdin: _FILE = ..., - stderr: _FILE = ..., - preexec_fn: Callable[[], Any] = ..., - close_fds: bool = ..., - shell: bool = ..., - cwd: Optional[_PATH] = ..., - env: Optional[_ENV] = ..., - universal_newlines: bool = ..., - startupinfo: Any = ..., - creationflags: int = ..., - restore_signals: bool = ..., - start_new_session: bool = ..., - pass_fds: Any = ..., - *, - timeout: Optional[float] = ..., - input: _TXT = ..., - encoding: Optional[str] = ..., - errors: Optional[str] = ..., - text: Optional[bool] = ..., - ) -> Any: ... # morally: -> _TXT -elif sys.version_info >= (3, 6): - # 3.6 added encoding and errors - @overload - def check_output(args: _CMD, - bufsize: int = ..., - executable: _PATH = ..., - stdin: _FILE = ..., - stderr: _FILE = ..., - preexec_fn: Callable[[], Any] = ..., - close_fds: bool = ..., - shell: bool = ..., - cwd: Optional[_PATH] = ..., - env: Optional[_ENV] = ..., - universal_newlines: bool = ..., - startupinfo: Any = ..., - creationflags: int = ..., - restore_signals: bool = ..., - start_new_session: bool = ..., - pass_fds: Any = ..., - *, - timeout: Optional[float] = ..., - input: _TXT = ..., - encoding: str, - errors: Optional[str] = ..., - ) -> str: ... - @overload - def check_output(args: _CMD, - bufsize: int = ..., - executable: _PATH = ..., - stdin: _FILE = ..., - stderr: _FILE = ..., - preexec_fn: Callable[[], Any] = ..., - close_fds: bool = ..., - shell: bool = ..., - cwd: Optional[_PATH] = ..., - env: Optional[_ENV] = ..., - universal_newlines: bool = ..., - startupinfo: Any = ..., - creationflags: int = ..., - restore_signals: bool = ..., - start_new_session: bool = ..., - pass_fds: Any = ..., - *, - timeout: Optional[float] = ..., - input: _TXT = ..., - encoding: Optional[str] = ..., - errors: str, - ) -> str: ... - @overload - def check_output(args: _CMD, - bufsize: int = ..., - executable: _PATH = ..., - stdin: _FILE = ..., - stderr: _FILE = ..., - preexec_fn: Callable[[], Any] = ..., - close_fds: bool = ..., - shell: bool = ..., - cwd: Optional[_PATH] = ..., - env: Optional[_ENV] = ..., - startupinfo: Any = ..., - creationflags: int = ..., - restore_signals: bool = ..., - start_new_session: bool = ..., - pass_fds: Any = ..., - *, - universal_newlines: Literal[True], - timeout: Optional[float] = ..., - input: _TXT = ..., - encoding: Optional[str] = ..., - errors: Optional[str] = ..., - ) -> str: ... - @overload - def check_output(args: _CMD, - bufsize: int = ..., - executable: _PATH = ..., - stdin: _FILE = ..., - stderr: _FILE = ..., - preexec_fn: Callable[[], Any] = ..., - close_fds: bool = ..., - shell: bool = ..., - cwd: Optional[_PATH] = ..., - env: Optional[_ENV] = ..., - universal_newlines: Literal[False] = ..., - startupinfo: Any = ..., - creationflags: int = ..., - restore_signals: bool = ..., - start_new_session: bool = ..., - pass_fds: Any = ..., - *, - timeout: Optional[float] = ..., - input: _TXT = ..., - encoding: None = ..., - errors: None = ..., - ) -> bytes: ... - @overload - def check_output(args: _CMD, - bufsize: int = ..., - executable: _PATH = ..., - stdin: _FILE = ..., - stderr: _FILE = ..., - preexec_fn: Callable[[], Any] = ..., - close_fds: bool = ..., - shell: bool = ..., - cwd: Optional[_PATH] = ..., - env: Optional[_ENV] = ..., - universal_newlines: bool = ..., - startupinfo: Any = ..., - creationflags: int = ..., - restore_signals: bool = ..., - start_new_session: bool = ..., - pass_fds: Any = ..., - *, - timeout: Optional[float] = ..., - input: _TXT = ..., - encoding: Optional[str] = ..., - errors: Optional[str] = ..., - ) -> Any: ... # morally: -> _TXT -else: - @overload - def check_output(args: _CMD, - bufsize: int = ..., - executable: _PATH = ..., - stdin: _FILE = ..., - stderr: _FILE = ..., - preexec_fn: Callable[[], Any] = ..., - close_fds: bool = ..., - shell: bool = ..., - cwd: Optional[_PATH] = ..., - env: Optional[_ENV] = ..., - startupinfo: Any = ..., - creationflags: int = ..., - restore_signals: bool = ..., - start_new_session: bool = ..., - pass_fds: Any = ..., - timeout: Optional[float] = ..., - input: _TXT = ..., - *, - universal_newlines: Literal[True], - ) -> str: ... - @overload - def check_output(args: _CMD, - bufsize: int = ..., - executable: _PATH = ..., - stdin: _FILE = ..., - stderr: _FILE = ..., - preexec_fn: Callable[[], Any] = ..., - close_fds: bool = ..., - shell: bool = ..., - cwd: Optional[_PATH] = ..., - env: Optional[_ENV] = ..., - universal_newlines: Literal[False] = ..., - startupinfo: Any = ..., - creationflags: int = ..., - restore_signals: bool = ..., - start_new_session: bool = ..., - pass_fds: Any = ..., - timeout: Optional[float] = ..., - input: _TXT = ..., - ) -> bytes: ... - @overload - def check_output(args: _CMD, - bufsize: int = ..., - executable: _PATH = ..., - stdin: _FILE = ..., - stderr: _FILE = ..., - preexec_fn: Callable[[], Any] = ..., - close_fds: bool = ..., - shell: bool = ..., - cwd: Optional[_PATH] = ..., - env: Optional[_ENV] = ..., - universal_newlines: bool = ..., - startupinfo: Any = ..., - creationflags: int = ..., - restore_signals: bool = ..., - start_new_session: bool = ..., - pass_fds: Any = ..., - timeout: Optional[float] = ..., - input: _TXT = ..., - ) -> Any: ... # morally: -> _TXT - - -PIPE: int -STDOUT: int -DEVNULL: int -class SubprocessError(Exception): ... -class TimeoutExpired(SubprocessError): - def __init__(self, cmd: _CMD, timeout: float, output: Optional[_TXT] = ..., stderr: Optional[_TXT] = ...) -> None: ... - # morally: _CMD - cmd: Any - timeout: float - # morally: Optional[_TXT] - output: Any - stdout: Any - stderr: Any - - -class CalledProcessError(Exception): - returncode: Optional[int] - # morally: _CMD - cmd: Any - # morally: Optional[_TXT] - output: Any - - # morally: Optional[_TXT] - stdout: Any - stderr: Any - - def __init__(self, - returncode: int, - cmd: _CMD, - output: Optional[_TXT] = ..., - stderr: Optional[_TXT] = ...) -> None: ... - -class Popen(Generic[AnyStr]): - args: _CMD - stdin: IO[AnyStr] - stdout: IO[AnyStr] - stderr: IO[AnyStr] - pid: int - returncode: Optional[int] - - # Technically it is wrong that Popen provides __new__ instead of __init__ - # but this shouldn't come up hopefully? - - if sys.version_info >= (3, 7): - # text is added in 3.7 - @overload - def __new__(cls, - args: _CMD, - bufsize: int = ..., - executable: Optional[_PATH] = ..., - stdin: Optional[_FILE] = ..., - stdout: Optional[_FILE] = ..., - stderr: Optional[_FILE] = ..., - preexec_fn: Optional[Callable[[], Any]] = ..., - close_fds: bool = ..., - shell: bool = ..., - cwd: Optional[_PATH] = ..., - env: Optional[_ENV] = ..., - universal_newlines: bool = ..., - startupinfo: Optional[Any] = ..., - creationflags: int = ..., - restore_signals: bool = ..., - start_new_session: bool = ..., - pass_fds: Any = ..., - *, - text: Optional[bool] = ..., - encoding: str, - errors: Optional[str] = ...) -> Popen[str]: ... - @overload - def __new__(cls, - args: _CMD, - bufsize: int = ..., - executable: Optional[_PATH] = ..., - stdin: Optional[_FILE] = ..., - stdout: Optional[_FILE] = ..., - stderr: Optional[_FILE] = ..., - preexec_fn: Optional[Callable[[], Any]] = ..., - close_fds: bool = ..., - shell: bool = ..., - cwd: Optional[_PATH] = ..., - env: Optional[_ENV] = ..., - universal_newlines: bool = ..., - startupinfo: Optional[Any] = ..., - creationflags: int = ..., - restore_signals: bool = ..., - start_new_session: bool = ..., - pass_fds: Any = ..., - *, - text: Optional[bool] = ..., - encoding: Optional[str] = ..., - errors: str) -> Popen[str]: ... - @overload - def __new__(cls, - args: _CMD, - bufsize: int = ..., - executable: Optional[_PATH] = ..., - stdin: Optional[_FILE] = ..., - stdout: Optional[_FILE] = ..., - stderr: Optional[_FILE] = ..., - preexec_fn: Optional[Callable[[], Any]] = ..., - close_fds: bool = ..., - shell: bool = ..., - cwd: Optional[_PATH] = ..., - env: Optional[_ENV] = ..., - *, - universal_newlines: Literal[True], - startupinfo: Optional[Any] = ..., - creationflags: int = ..., - restore_signals: bool = ..., - start_new_session: bool = ..., - pass_fds: Any = ..., - # where the *real* keyword only args start - text: Optional[bool] = ..., - encoding: Optional[str] = ..., - errors: Optional[str] = ...) -> Popen[str]: ... - @overload - def __new__(cls, - args: _CMD, - bufsize: int = ..., - executable: Optional[_PATH] = ..., - stdin: Optional[_FILE] = ..., - stdout: Optional[_FILE] = ..., - stderr: Optional[_FILE] = ..., - preexec_fn: Optional[Callable[[], Any]] = ..., - close_fds: bool = ..., - shell: bool = ..., - cwd: Optional[_PATH] = ..., - env: Optional[_ENV] = ..., - universal_newlines: bool = ..., - startupinfo: Optional[Any] = ..., - creationflags: int = ..., - restore_signals: bool = ..., - start_new_session: bool = ..., - pass_fds: Any = ..., - *, - text: Literal[True], - encoding: Optional[str] = ..., - errors: Optional[str] = ...) -> Popen[str]: ... - @overload - def __new__(cls, - args: _CMD, - bufsize: int = ..., - executable: Optional[_PATH] = ..., - stdin: Optional[_FILE] = ..., - stdout: Optional[_FILE] = ..., - stderr: Optional[_FILE] = ..., - preexec_fn: Optional[Callable[[], Any]] = ..., - close_fds: bool = ..., - shell: bool = ..., - cwd: Optional[_PATH] = ..., - env: Optional[_ENV] = ..., - universal_newlines: Literal[False] = ..., - startupinfo: Optional[Any] = ..., - creationflags: int = ..., - restore_signals: bool = ..., - start_new_session: bool = ..., - pass_fds: Any = ..., - *, - text: Literal[None, False] = ..., - encoding: None = ..., - errors: None = ...) -> Popen[bytes]: ... - @overload - def __new__(cls, - args: _CMD, - bufsize: int = ..., - executable: Optional[_PATH] = ..., - stdin: Optional[_FILE] = ..., - stdout: Optional[_FILE] = ..., - stderr: Optional[_FILE] = ..., - preexec_fn: Optional[Callable[[], Any]] = ..., - close_fds: bool = ..., - shell: bool = ..., - cwd: Optional[_PATH] = ..., - env: Optional[_ENV] = ..., - universal_newlines: bool = ..., - startupinfo: Optional[Any] = ..., - creationflags: int = ..., - restore_signals: bool = ..., - start_new_session: bool = ..., - pass_fds: Any = ..., - *, - text: Optional[bool] = ..., - encoding: Optional[str] = ..., - errors: Optional[str] = ...) -> Popen[Any]: ... - elif sys.version_info >= (3, 6): - @overload - def __new__(cls, - args: _CMD, - bufsize: int = ..., - executable: Optional[_PATH] = ..., - stdin: Optional[_FILE] = ..., - stdout: Optional[_FILE] = ..., - stderr: Optional[_FILE] = ..., - preexec_fn: Optional[Callable[[], Any]] = ..., - close_fds: bool = ..., - shell: bool = ..., - cwd: Optional[_PATH] = ..., - env: Optional[_ENV] = ..., - universal_newlines: bool = ..., - startupinfo: Optional[Any] = ..., - creationflags: int = ..., - restore_signals: bool = ..., - start_new_session: bool = ..., - pass_fds: Any = ..., - *, - encoding: str, - errors: Optional[str] = ...) -> Popen[str]: ... - @overload - def __new__(cls, - args: _CMD, - bufsize: int = ..., - executable: Optional[_PATH] = ..., - stdin: Optional[_FILE] = ..., - stdout: Optional[_FILE] = ..., - stderr: Optional[_FILE] = ..., - preexec_fn: Optional[Callable[[], Any]] = ..., - close_fds: bool = ..., - shell: bool = ..., - cwd: Optional[_PATH] = ..., - env: Optional[_ENV] = ..., - universal_newlines: bool = ..., - startupinfo: Optional[Any] = ..., - creationflags: int = ..., - restore_signals: bool = ..., - start_new_session: bool = ..., - pass_fds: Any = ..., - *, - encoding: Optional[str] = ..., - errors: str) -> Popen[str]: ... - @overload - def __new__(cls, - args: _CMD, - bufsize: int = ..., - executable: Optional[_PATH] = ..., - stdin: Optional[_FILE] = ..., - stdout: Optional[_FILE] = ..., - stderr: Optional[_FILE] = ..., - preexec_fn: Optional[Callable[[], Any]] = ..., - close_fds: bool = ..., - shell: bool = ..., - cwd: Optional[_PATH] = ..., - env: Optional[_ENV] = ..., - *, - universal_newlines: Literal[True], - startupinfo: Optional[Any] = ..., - creationflags: int = ..., - restore_signals: bool = ..., - start_new_session: bool = ..., - pass_fds: Any = ..., - # where the *real* keyword only args start - encoding: Optional[str] = ..., - errors: Optional[str] = ...) -> Popen[str]: ... - @overload - def __new__(cls, - args: _CMD, - bufsize: int = ..., - executable: Optional[_PATH] = ..., - stdin: Optional[_FILE] = ..., - stdout: Optional[_FILE] = ..., - stderr: Optional[_FILE] = ..., - preexec_fn: Optional[Callable[[], Any]] = ..., - close_fds: bool = ..., - shell: bool = ..., - cwd: Optional[_PATH] = ..., - env: Optional[_ENV] = ..., - universal_newlines: Literal[False] = ..., - startupinfo: Optional[Any] = ..., - creationflags: int = ..., - restore_signals: bool = ..., - start_new_session: bool = ..., - pass_fds: Any = ..., - *, - encoding: None = ..., - errors: None = ...) -> Popen[bytes]: ... - @overload - def __new__(cls, - args: _CMD, - bufsize: int = ..., - executable: Optional[_PATH] = ..., - stdin: Optional[_FILE] = ..., - stdout: Optional[_FILE] = ..., - stderr: Optional[_FILE] = ..., - preexec_fn: Optional[Callable[[], Any]] = ..., - close_fds: bool = ..., - shell: bool = ..., - cwd: Optional[_PATH] = ..., - env: Optional[_ENV] = ..., - universal_newlines: bool = ..., - startupinfo: Optional[Any] = ..., - creationflags: int = ..., - restore_signals: bool = ..., - start_new_session: bool = ..., - pass_fds: Any = ..., - *, - encoding: Optional[str] = ..., - errors: Optional[str] = ...) -> Popen[Any]: ... - else: - @overload - def __new__(cls, - args: _CMD, - bufsize: int = ..., - executable: Optional[_PATH] = ..., - stdin: Optional[_FILE] = ..., - stdout: Optional[_FILE] = ..., - stderr: Optional[_FILE] = ..., - preexec_fn: Optional[Callable[[], Any]] = ..., - close_fds: bool = ..., - shell: bool = ..., - cwd: Optional[_PATH] = ..., - env: Optional[_ENV] = ..., - *, - universal_newlines: Literal[True], - startupinfo: Optional[Any] = ..., - creationflags: int = ..., - restore_signals: bool = ..., - start_new_session: bool = ..., - pass_fds: Any = ...) -> Popen[str]: ... - @overload - def __new__(cls, - args: _CMD, - bufsize: int = ..., - executable: Optional[_PATH] = ..., - stdin: Optional[_FILE] = ..., - stdout: Optional[_FILE] = ..., - stderr: Optional[_FILE] = ..., - preexec_fn: Optional[Callable[[], Any]] = ..., - close_fds: bool = ..., - shell: bool = ..., - cwd: Optional[_PATH] = ..., - env: Optional[_ENV] = ..., - *, - universal_newlines: Literal[False] = ..., - startupinfo: Optional[Any] = ..., - creationflags: int = ..., - restore_signals: bool = ..., - start_new_session: bool = ..., - pass_fds: Any = ...) -> Popen[bytes]: ... - @overload - def __new__(cls, - args: _CMD, - bufsize: int = ..., - executable: Optional[_PATH] = ..., - stdin: Optional[_FILE] = ..., - stdout: Optional[_FILE] = ..., - stderr: Optional[_FILE] = ..., - preexec_fn: Optional[Callable[[], Any]] = ..., - close_fds: bool = ..., - shell: bool = ..., - cwd: Optional[_PATH] = ..., - env: Optional[_ENV] = ..., - universal_newlines: bool = ..., - startupinfo: Optional[Any] = ..., - creationflags: int = ..., - restore_signals: bool = ..., - start_new_session: bool = ..., - pass_fds: Any = ...) -> Popen[Any]: ... - - def poll(self) -> int: ... - def wait(self, timeout: Optional[float] = ...) -> int: ... - # Return str/bytes - def communicate(self, - input: Optional[AnyStr] = ..., - timeout: Optional[float] = ..., - # morally this should be optional - ) -> Tuple[Optional[AnyStr], AnyStr]: ... - def send_signal(self, signal: int) -> None: ... - def terminate(self) -> None: ... - def kill(self) -> None: ... - def __enter__(self: _S) -> _S: ... - def __exit__(self, type: Optional[Type[BaseException]], value: Optional[BaseException], traceback: Optional[TracebackType]) -> None: ... - -# The result really is always a str. -def getstatusoutput(cmd: _TXT) -> Tuple[int, str]: ... -def getoutput(cmd: _TXT) -> str: ... - -def list2cmdline(seq: Sequence[str]) -> str: ... # undocumented - -if sys.platform == 'win32': - class STARTUPINFO: - if sys.version_info >= (3, 7): - def __init__(self, *, dwFlags: int = ..., hStdInput: Optional[Any] = ..., hStdOutput: Optional[Any] = ..., hStdError: Optional[Any] = ..., wShowWindow: int = ..., lpAttributeList: Optional[Mapping[str, Any]] = ...) -> None: ... - dwFlags: int - hStdInput: Optional[Any] - hStdOutput: Optional[Any] - hStdError: Optional[Any] - wShowWindow: int - if sys.version_info >= (3, 7): - lpAttributeList: Mapping[str, Any] - - STD_INPUT_HANDLE: Any - STD_OUTPUT_HANDLE: Any - STD_ERROR_HANDLE: Any - SW_HIDE: int - STARTF_USESTDHANDLES: int - STARTF_USESHOWWINDOW: int - CREATE_NEW_CONSOLE: int - CREATE_NEW_PROCESS_GROUP: int - if sys.version_info >= (3, 7): - ABOVE_NORMAL_PRIORITY_CLASS: int - BELOW_NORMAL_PRIORITY_CLASS: int - HIGH_PRIORITY_CLASS: int - IDLE_PRIORITY_CLASS: int - NORMAL_PRIORITY_CLASS: int - REALTIME_PRIORITY_CLASS: int - CREATE_NO_WINDOW: int - DETACHED_PROCESS: int - CREATE_DEFAULT_ERROR_MODE: int - CREATE_BREAKAWAY_FROM_JOB: int diff --git a/typeshed/3/typing.pyi b/typeshed/3/typing.pyi deleted file mode 100644 index 6fb66ad5f9..0000000000 --- a/typeshed/3/typing.pyi +++ /dev/null @@ -1,649 +0,0 @@ -# Stubs for typing - -import sys -from abc import abstractmethod, ABCMeta -from types import CodeType, FrameType, TracebackType -import collections # Needed by aliases like DefaultDict, see mypy issue 2986 - -# Definitions of special type checking related constructs. Their definitions -# are not used, so their value does not matter. - -overload = object() -Any = object() -TypeVar = object() -_promote = object() - -class _SpecialForm: - def __getitem__(self, typeargs: Any) -> Any: ... - -Tuple: _SpecialForm = ... -Generic: _SpecialForm = ... -Protocol: _SpecialForm = ... -Callable: _SpecialForm = ... -Type: _SpecialForm = ... -ClassVar: _SpecialForm = ... -if sys.version_info >= (3, 8): - Final: _SpecialForm = ... - _F = TypeVar('_F', bound=Callable[..., Any]) - def final(f: _F) -> _F: ... - Literal: _SpecialForm = ... - # TypedDict is a (non-subscriptable) special form. - TypedDict: object - -class GenericMeta(type): ... - -# Return type that indicates a function does not return. -# This type is equivalent to the None type, but the no-op Union is necessary to -# distinguish the None type from the None value. -NoReturn = Union[None] - -# These type variables are used by the container types. -_T = TypeVar('_T') -_S = TypeVar('_S') -_KT = TypeVar('_KT') # Key type. -_VT = TypeVar('_VT') # Value type. -_T_co = TypeVar('_T_co', covariant=True) # Any type covariant containers. -_V_co = TypeVar('_V_co', covariant=True) # Any type covariant containers. -_KT_co = TypeVar('_KT_co', covariant=True) # Key type covariant containers. -_VT_co = TypeVar('_VT_co', covariant=True) # Value type covariant containers. -_T_contra = TypeVar('_T_contra', contravariant=True) # Ditto contravariant. -_TC = TypeVar('_TC', bound=Type[object]) -_C = TypeVar("_C", bound=Callable[..., Any]) - -no_type_check = object() -def no_type_check_decorator(decorator: _C) -> _C: ... - -# Type aliases and type constructors - -class TypeAlias: - # Class for defining generic aliases for library types. - def __init__(self, target_type: type) -> None: ... - def __getitem__(self, typeargs: Any) -> Any: ... - -Union = TypeAlias(object) -Optional = TypeAlias(object) -List = TypeAlias(object) -Dict = TypeAlias(object) -DefaultDict = TypeAlias(object) -Set = TypeAlias(object) -FrozenSet = TypeAlias(object) -Counter = TypeAlias(object) -Deque = TypeAlias(object) -ChainMap = TypeAlias(object) - -if sys.version_info >= (3, 7): - OrderedDict = TypeAlias(object) - -# Predefined type variables. -AnyStr = TypeVar('AnyStr', str, bytes) - -# Abstract base classes. - -def runtime_checkable(cls: _TC) -> _TC: ... - -@runtime_checkable -class SupportsInt(Protocol, metaclass=ABCMeta): - @abstractmethod - def __int__(self) -> int: ... - -@runtime_checkable -class SupportsFloat(Protocol, metaclass=ABCMeta): - @abstractmethod - def __float__(self) -> float: ... - -@runtime_checkable -class SupportsComplex(Protocol, metaclass=ABCMeta): - @abstractmethod - def __complex__(self) -> complex: ... - -@runtime_checkable -class SupportsBytes(Protocol, metaclass=ABCMeta): - @abstractmethod - def __bytes__(self) -> bytes: ... - -if sys.version_info >= (3, 8): - @runtime_checkable - class SupportsIndex(Protocol, metaclass=ABCMeta): - @abstractmethod - def __index__(self) -> int: ... - -@runtime_checkable -class SupportsAbs(Protocol[_T_co]): - @abstractmethod - def __abs__(self) -> _T_co: ... - -@runtime_checkable -class SupportsRound(Protocol[_T_co]): - @overload - @abstractmethod - def __round__(self) -> int: ... - @overload - @abstractmethod - def __round__(self, ndigits: int) -> _T_co: ... - -@runtime_checkable -class Reversible(Protocol[_T_co]): - @abstractmethod - def __reversed__(self) -> Iterator[_T_co]: ... - -@runtime_checkable -class Sized(Protocol, metaclass=ABCMeta): - @abstractmethod - def __len__(self) -> int: ... - -@runtime_checkable -class Hashable(Protocol, metaclass=ABCMeta): - # TODO: This is special, in that a subclass of a hashable class may not be hashable - # (for example, list vs. object). It's not obvious how to represent this. This class - # is currently mostly useless for static checking. - @abstractmethod - def __hash__(self) -> int: ... - -@runtime_checkable -class Iterable(Protocol[_T_co]): - @abstractmethod - def __iter__(self) -> Iterator[_T_co]: ... - -@runtime_checkable -class Iterator(Iterable[_T_co], Protocol[_T_co]): - @abstractmethod - def __next__(self) -> _T_co: ... - def __iter__(self) -> Iterator[_T_co]: ... - -class Generator(Iterator[_T_co], Generic[_T_co, _T_contra, _V_co]): - @abstractmethod - def __next__(self) -> _T_co: ... - - @abstractmethod - def send(self, value: _T_contra) -> _T_co: ... - - @abstractmethod - def throw(self, typ: Type[BaseException], val: Optional[BaseException] = ..., - tb: Optional[TracebackType] = ...) -> _T_co: ... - - @abstractmethod - def close(self) -> None: ... - - @abstractmethod - def __iter__(self) -> Generator[_T_co, _T_contra, _V_co]: ... - - @property - def gi_code(self) -> CodeType: ... - @property - def gi_frame(self) -> FrameType: ... - @property - def gi_running(self) -> bool: ... - @property - def gi_yieldfrom(self) -> Optional[Generator[Any, Any, Any]]: ... - -@runtime_checkable -class Awaitable(Protocol[_T_co]): - @abstractmethod - def __await__(self) -> Generator[Any, None, _T_co]: ... - -class Coroutine(Awaitable[_V_co], Generic[_T_co, _T_contra, _V_co]): - @property - def cr_await(self) -> Optional[Any]: ... - @property - def cr_code(self) -> CodeType: ... - @property - def cr_frame(self) -> FrameType: ... - @property - def cr_running(self) -> bool: ... - - @abstractmethod - def send(self, value: _T_contra) -> _T_co: ... - - @abstractmethod - def throw(self, typ: Type[BaseException], val: Optional[BaseException] = ..., - tb: Optional[TracebackType] = ...) -> _T_co: ... - - @abstractmethod - def close(self) -> None: ... - - -# NOTE: This type does not exist in typing.py or PEP 484. -# The parameters correspond to Generator, but the 4th is the original type. -class AwaitableGenerator(Awaitable[_V_co], Generator[_T_co, _T_contra, _V_co], - Generic[_T_co, _T_contra, _V_co, _S], metaclass=ABCMeta): ... - -@runtime_checkable -class AsyncIterable(Protocol[_T_co]): - @abstractmethod - def __aiter__(self) -> AsyncIterator[_T_co]: ... - -@runtime_checkable -class AsyncIterator(AsyncIterable[_T_co], - Protocol[_T_co]): - @abstractmethod - def __anext__(self) -> Awaitable[_T_co]: ... - def __aiter__(self) -> AsyncIterator[_T_co]: ... - -if sys.version_info >= (3, 6): - class AsyncGenerator(AsyncIterator[_T_co], Generic[_T_co, _T_contra]): - @abstractmethod - def __anext__(self) -> Awaitable[_T_co]: ... - - @abstractmethod - def asend(self, value: _T_contra) -> Awaitable[_T_co]: ... - - @abstractmethod - def athrow(self, typ: Type[BaseException], val: Optional[BaseException] = ..., - tb: Any = ...) -> Awaitable[_T_co]: ... - - @abstractmethod - def aclose(self) -> Awaitable[None]: ... - - @abstractmethod - def __aiter__(self) -> AsyncGenerator[_T_co, _T_contra]: ... - - @property - def ag_await(self) -> Any: ... - @property - def ag_code(self) -> CodeType: ... - @property - def ag_frame(self) -> FrameType: ... - @property - def ag_running(self) -> bool: ... - -@runtime_checkable -class Container(Protocol[_T_co]): - @abstractmethod - def __contains__(self, __x: object) -> bool: ... - - -if sys.version_info >= (3, 6): - @runtime_checkable - class Collection(Iterable[_T_co], Container[_T_co], Protocol[_T_co]): - # Implement Sized (but don't have it as a base class). - @abstractmethod - def __len__(self) -> int: ... - - _Collection = Collection -else: - @runtime_checkable - class _Collection(Iterable[_T_co], Container[_T_co], Protocol[_T_co]): - # Implement Sized (but don't have it as a base class). - @abstractmethod - def __len__(self) -> int: ... - -class Sequence(_Collection[_T_co], Reversible[_T_co], Generic[_T_co]): - @overload - @abstractmethod - def __getitem__(self, i: int) -> _T_co: ... - @overload - @abstractmethod - def __getitem__(self, s: slice) -> Sequence[_T_co]: ... - # Mixin methods - def index(self, x: Any, start: int = ..., end: int = ...) -> int: ... - def count(self, x: Any) -> int: ... - def __contains__(self, x: object) -> bool: ... - def __iter__(self) -> Iterator[_T_co]: ... - def __reversed__(self) -> Iterator[_T_co]: ... - -class MutableSequence(Sequence[_T], Generic[_T]): - @abstractmethod - def insert(self, index: int, object: _T) -> None: ... - @overload - @abstractmethod - def __getitem__(self, i: int) -> _T: ... - @overload - @abstractmethod - def __getitem__(self, s: slice) -> MutableSequence[_T]: ... - @overload - @abstractmethod - def __setitem__(self, i: int, o: _T) -> None: ... - @overload - @abstractmethod - def __setitem__(self, s: slice, o: Iterable[_T]) -> None: ... - @overload - @abstractmethod - def __delitem__(self, i: int) -> None: ... - @overload - @abstractmethod - def __delitem__(self, i: slice) -> None: ... - # Mixin methods - def append(self, object: _T) -> None: ... - def clear(self) -> None: ... - def extend(self, iterable: Iterable[_T]) -> None: ... - def reverse(self) -> None: ... - def pop(self, index: int = ...) -> _T: ... - def remove(self, object: _T) -> None: ... - def __iadd__(self, x: Iterable[_T]) -> MutableSequence[_T]: ... - -class AbstractSet(_Collection[_T_co], Generic[_T_co]): - @abstractmethod - def __contains__(self, x: object) -> bool: ... - # Mixin methods - def __le__(self, s: AbstractSet[Any]) -> bool: ... - def __lt__(self, s: AbstractSet[Any]) -> bool: ... - def __gt__(self, s: AbstractSet[Any]) -> bool: ... - def __ge__(self, s: AbstractSet[Any]) -> bool: ... - def __and__(self, s: AbstractSet[Any]) -> AbstractSet[_T_co]: ... - def __or__(self, s: AbstractSet[_T]) -> AbstractSet[Union[_T_co, _T]]: ... - def __sub__(self, s: AbstractSet[Any]) -> AbstractSet[_T_co]: ... - def __xor__(self, s: AbstractSet[_T]) -> AbstractSet[Union[_T_co, _T]]: ... - def isdisjoint(self, s: Iterable[Any]) -> bool: ... - -class MutableSet(AbstractSet[_T], Generic[_T]): - @abstractmethod - def add(self, x: _T) -> None: ... - @abstractmethod - def discard(self, x: _T) -> None: ... - # Mixin methods - def clear(self) -> None: ... - def pop(self) -> _T: ... - def remove(self, element: _T) -> None: ... - def __ior__(self, s: AbstractSet[_S]) -> MutableSet[Union[_T, _S]]: ... - def __iand__(self, s: AbstractSet[Any]) -> MutableSet[_T]: ... - def __ixor__(self, s: AbstractSet[_S]) -> MutableSet[Union[_T, _S]]: ... - def __isub__(self, s: AbstractSet[Any]) -> MutableSet[_T]: ... - -class MappingView(Sized): - def __init__(self, mapping: Mapping[_KT_co, _VT_co]) -> None: ... # undocumented - def __len__(self) -> int: ... - -class ItemsView(MappingView, AbstractSet[Tuple[_KT_co, _VT_co]], Generic[_KT_co, _VT_co]): - def __init__(self, mapping: Mapping[_KT_co, _VT_co]) -> None: ... # undocumented - def __and__(self, o: Iterable[Any]) -> Set[Tuple[_KT_co, _VT_co]]: ... - def __rand__(self, o: Iterable[_T]) -> Set[_T]: ... - def __contains__(self, o: object) -> bool: ... - def __iter__(self) -> Iterator[Tuple[_KT_co, _VT_co]]: ... - if sys.version_info >= (3, 8): - def __reversed__(self) -> Iterator[Tuple[_KT_co, _VT_co]]: ... - def __or__(self, o: Iterable[_T]) -> Set[Union[Tuple[_KT_co, _VT_co], _T]]: ... - def __ror__(self, o: Iterable[_T]) -> Set[Union[Tuple[_KT_co, _VT_co], _T]]: ... - def __sub__(self, o: Iterable[Any]) -> Set[Tuple[_KT_co, _VT_co]]: ... - def __rsub__(self, o: Iterable[_T]) -> Set[_T]: ... - def __xor__(self, o: Iterable[_T]) -> Set[Union[Tuple[_KT_co, _VT_co], _T]]: ... - def __rxor__(self, o: Iterable[_T]) -> Set[Union[Tuple[_KT_co, _VT_co], _T]]: ... - -class KeysView(MappingView, AbstractSet[_KT_co], Generic[_KT_co]): - def __init__(self, mapping: Mapping[_KT_co, _VT_co]) -> None: ... # undocumented - def __and__(self, o: Iterable[Any]) -> Set[_KT_co]: ... - def __rand__(self, o: Iterable[_T]) -> Set[_T]: ... - def __contains__(self, o: object) -> bool: ... - def __iter__(self) -> Iterator[_KT_co]: ... - if sys.version_info >= (3, 8): - def __reversed__(self) -> Iterator[_KT_co]: ... - def __or__(self, o: Iterable[_T]) -> Set[Union[_KT_co, _T]]: ... - def __ror__(self, o: Iterable[_T]) -> Set[Union[_KT_co, _T]]: ... - def __sub__(self, o: Iterable[Any]) -> Set[_KT_co]: ... - def __rsub__(self, o: Iterable[_T]) -> Set[_T]: ... - def __xor__(self, o: Iterable[_T]) -> Set[Union[_KT_co, _T]]: ... - def __rxor__(self, o: Iterable[_T]) -> Set[Union[_KT_co, _T]]: ... - -class ValuesView(MappingView, Iterable[_VT_co], Generic[_VT_co]): - def __init__(self, mapping: Mapping[_KT_co, _VT_co]) -> None: ... # undocumented - def __contains__(self, o: object) -> bool: ... - def __iter__(self) -> Iterator[_VT_co]: ... - if sys.version_info >= (3, 8): - def __reversed__(self) -> Iterator[_VT_co]: ... - -@runtime_checkable -class ContextManager(Protocol[_T_co]): - def __enter__(self) -> _T_co: ... - def __exit__(self, __exc_type: Optional[Type[BaseException]], - __exc_value: Optional[BaseException], - __traceback: Optional[TracebackType]) -> Optional[bool]: ... - -@runtime_checkable -class AsyncContextManager(Protocol[_T_co]): - def __aenter__(self) -> Awaitable[_T_co]: ... - def __aexit__( - self, - exc_type: Optional[Type[BaseException]], - exc_value: Optional[BaseException], - traceback: Optional[TracebackType], - ) -> Awaitable[Optional[bool]]: ... - -class Mapping(_Collection[_KT], Generic[_KT, _VT_co]): - # TODO: We wish the key type could also be covariant, but that doesn't work, - # see discussion in https: //github.com/python/typing/pull/273. - @abstractmethod - def __getitem__(self, k: _KT) -> _VT_co: - ... - # Mixin methods - @overload - def get(self, k: _KT) -> Optional[_VT_co]: ... - @overload - def get(self, k: _KT, default: Union[_VT_co, _T]) -> Union[_VT_co, _T]: ... - def items(self) -> AbstractSet[Tuple[_KT, _VT_co]]: ... - def keys(self) -> AbstractSet[_KT]: ... - def values(self) -> ValuesView[_VT_co]: ... - def __contains__(self, o: object) -> bool: ... - -class MutableMapping(Mapping[_KT, _VT], Generic[_KT, _VT]): - @abstractmethod - def __setitem__(self, k: _KT, v: _VT) -> None: ... - @abstractmethod - def __delitem__(self, v: _KT) -> None: ... - - def clear(self) -> None: ... - @overload - def pop(self, k: _KT) -> _VT: ... - @overload - def pop(self, k: _KT, default: Union[_VT, _T] = ...) -> Union[_VT, _T]: ... - def popitem(self) -> Tuple[_KT, _VT]: ... - def setdefault(self, k: _KT, default: _VT = ...) -> _VT: ... - # 'update' used to take a Union, but using overloading is better. - # The second overloaded type here is a bit too general, because - # Mapping[Tuple[_KT, _VT], W] is a subclass of Iterable[Tuple[_KT, _VT]], - # but will always have the behavior of the first overloaded type - # at runtime, leading to keys of a mix of types _KT and Tuple[_KT, _VT]. - # We don't currently have any way of forcing all Mappings to use - # the first overload, but by using overloading rather than a Union, - # mypy will commit to using the first overload when the argument is - # known to be a Mapping with unknown type parameters, which is closer - # to the behavior we want. See mypy issue #1430. - @overload - def update(self, __m: Mapping[_KT, _VT], **kwargs: _VT) -> None: ... - @overload - def update(self, __m: Iterable[Tuple[_KT, _VT]], **kwargs: _VT) -> None: ... - @overload - def update(self, **kwargs: _VT) -> None: ... - -Text = str - -TYPE_CHECKING = True - -class IO(Iterator[AnyStr], Generic[AnyStr]): - # TODO detach - # TODO use abstract properties - @property - def mode(self) -> str: ... - @property - def name(self) -> str: ... - @abstractmethod - def close(self) -> None: ... - @property - def closed(self) -> bool: ... - @abstractmethod - def fileno(self) -> int: ... - @abstractmethod - def flush(self) -> None: ... - @abstractmethod - def isatty(self) -> bool: ... - # TODO what if n is None? - @abstractmethod - def read(self, n: int = ...) -> AnyStr: ... - @abstractmethod - def readable(self) -> bool: ... - @abstractmethod - def readline(self, limit: int = ...) -> AnyStr: ... - @abstractmethod - def readlines(self, hint: int = ...) -> list[AnyStr]: ... - @abstractmethod - def seek(self, offset: int, whence: int = ...) -> int: ... - @abstractmethod - def seekable(self) -> bool: ... - @abstractmethod - def tell(self) -> int: ... - @abstractmethod - def truncate(self, size: Optional[int] = ...) -> int: ... - @abstractmethod - def writable(self) -> bool: ... - # TODO buffer objects - @abstractmethod - def write(self, s: AnyStr) -> int: ... - @abstractmethod - def writelines(self, lines: Iterable[AnyStr]) -> None: ... - - @abstractmethod - def __next__(self) -> AnyStr: ... - @abstractmethod - def __iter__(self) -> Iterator[AnyStr]: ... - @abstractmethod - def __enter__(self) -> IO[AnyStr]: ... - @abstractmethod - def __exit__(self, t: Optional[Type[BaseException]], value: Optional[BaseException], - traceback: Optional[TracebackType]) -> Optional[bool]: ... - -class BinaryIO(IO[bytes]): - # TODO readinto - # TODO read1? - # TODO peek? - @overload - @abstractmethod - def write(self, s: bytearray) -> int: ... - @overload - @abstractmethod - def write(self, s: bytes) -> int: ... - - @abstractmethod - def __enter__(self) -> BinaryIO: ... - -class TextIO(IO[str]): - # TODO use abstractproperty - @property - def buffer(self) -> BinaryIO: ... - @property - def encoding(self) -> str: ... - @property - def errors(self) -> Optional[str]: ... - @property - def line_buffering(self) -> int: ... # int on PyPy, bool on CPython - @property - def newlines(self) -> Any: ... # None, str or tuple - @abstractmethod - def __enter__(self) -> TextIO: ... - -class ByteString(Sequence[int], metaclass=ABCMeta): ... - -class Match(Generic[AnyStr]): - pos: int - endpos: int - lastindex: Optional[int] - lastgroup: Optional[AnyStr] - string: AnyStr - - # The regular expression object whose match() or search() method produced - # this match instance. - re: Pattern[AnyStr] - - def expand(self, template: AnyStr) -> AnyStr: ... - - # TODO: The return for a group may be None, except if __group is 0 or not given. - @overload - def group(self, __group: Union[str, int] = ...) -> AnyStr: ... - @overload - def group( - self, - __group1: Union[str, int], - __group2: Union[str, int], - *groups: Union[str, int], - ) -> Tuple[AnyStr, ...]: ... - - def groups(self, default: AnyStr = ...) -> Sequence[AnyStr]: ... - def groupdict(self, default: AnyStr = ...) -> dict[str, AnyStr]: ... - def start(self, group: Union[int, str] = ...) -> int: ... - def end(self, group: Union[int, str] = ...) -> int: ... - def span(self, group: Union[int, str] = ...) -> Tuple[int, int]: ... - @property - def regs(self) -> Tuple[Tuple[int, int], ...]: ... # undocumented - if sys.version_info >= (3, 6): - def __getitem__(self, g: Union[int, str]) -> AnyStr: ... - -class Pattern(Generic[AnyStr]): - flags: int - groupindex: Mapping[str, int] - groups: int - pattern: AnyStr - - def search(self, string: AnyStr, pos: int = ..., - endpos: int = ...) -> Optional[Match[AnyStr]]: ... - def match(self, string: AnyStr, pos: int = ..., - endpos: int = ...) -> Optional[Match[AnyStr]]: ... - # New in Python 3.4 - def fullmatch(self, string: AnyStr, pos: int = ..., - endpos: int = ...) -> Optional[Match[AnyStr]]: ... - def split(self, string: AnyStr, maxsplit: int = ...) -> list[AnyStr]: ... - def findall(self, string: AnyStr, pos: int = ..., - endpos: int = ...) -> list[Any]: ... - def finditer(self, string: AnyStr, pos: int = ..., - endpos: int = ...) -> Iterator[Match[AnyStr]]: ... - - @overload - def sub(self, repl: AnyStr, string: AnyStr, - count: int = ...) -> AnyStr: ... - @overload - def sub(self, repl: Callable[[Match[AnyStr]], AnyStr], string: AnyStr, - count: int = ...) -> AnyStr: ... - - @overload - def subn(self, repl: AnyStr, string: AnyStr, - count: int = ...) -> Tuple[AnyStr, int]: ... - @overload - def subn(self, repl: Callable[[Match[AnyStr]], AnyStr], string: AnyStr, - count: int = ...) -> Tuple[AnyStr, int]: ... - -# Functions - -def get_type_hints( - obj: Callable[..., Any], globalns: Optional[Dict[str, Any]] = ..., localns: Optional[Dict[str, Any]] = ..., -) -> Dict[str, Any]: ... -if sys.version_info >= (3, 8): - def get_origin(tp: Any) -> Optional[Any]: ... - def get_args(tp: Any) -> Tuple[Any, ...]: ... - -@overload -def cast(tp: Type[_T], obj: Any) -> _T: ... -@overload -def cast(tp: str, obj: Any) -> Any: ... - -# Type constructors - -# NamedTuple is special-cased in the type checker -class NamedTuple(Tuple[Any, ...]): - _field_types: collections.OrderedDict[str, Type[Any]] - _field_defaults: Dict[str, Any] = ... - _fields: Tuple[str, ...] - _source: str - - def __init__(self, typename: str, fields: Iterable[Tuple[str, Any]] = ..., - **kwargs: Any) -> None: ... - - @classmethod - def _make(cls: Type[_T], iterable: Iterable[Any]) -> _T: ... - - if sys.version_info >= (3, 8): - def _asdict(self) -> Dict[str, Any]: ... - else: - def _asdict(self) -> collections.OrderedDict[str, Any]: ... - def _replace(self: _T, **kwargs: Any) -> _T: ... - -# Internal mypy fallback type for all typed dicts (does not exist at runtime) -class _TypedDict(Mapping[str, object], metaclass=ABCMeta): - def copy(self: _T) -> _T: ... - # Using NoReturn so that only calls using mypy plugin hook that specialize the signature - # can go through. - def setdefault(self, k: NoReturn, default: object) -> object: ... - # Mypy plugin hook for 'pop' expects that 'default' has a type variable type. - def pop(self, k: NoReturn, default: _T = ...) -> object: ... - def update(self: _T, __m: _T) -> None: ... - def __delitem__(self, k: NoReturn) -> None: ... - -def NewType(name: str, tp: Type[_T]) -> Type[_T]: ... - -# This itself is only available during type checking -def type_check_only(func_or_cls: _C) -> _C: ... diff --git a/typeshed/3/arcp/__init__.pyi b/typeshed/arcp/__init__.pyi similarity index 100% rename from typeshed/3/arcp/__init__.pyi rename to typeshed/arcp/__init__.pyi diff --git a/typeshed/3/arcp/generate.pyi b/typeshed/arcp/generate.pyi similarity index 100% rename from typeshed/3/arcp/generate.pyi rename to typeshed/arcp/generate.pyi diff --git a/typeshed/3/arcp/parse.pyi b/typeshed/arcp/parse.pyi similarity index 100% rename from typeshed/3/arcp/parse.pyi rename to typeshed/arcp/parse.pyi diff --git a/typeshed/3/argcomplete/__init__.pyi b/typeshed/argcomplete/__init__.pyi similarity index 100% rename from typeshed/3/argcomplete/__init__.pyi rename to typeshed/argcomplete/__init__.pyi diff --git a/typeshed/3/bagit.pyi b/typeshed/bagit.pyi similarity index 100% rename from typeshed/3/bagit.pyi rename to typeshed/bagit.pyi diff --git a/typeshed/2and3/cachecontrol/__init__.pyi b/typeshed/cachecontrol/__init__.pyi similarity index 100% rename from typeshed/2and3/cachecontrol/__init__.pyi rename to typeshed/cachecontrol/__init__.pyi diff --git a/typeshed/2and3/cachecontrol/adapter.pyi b/typeshed/cachecontrol/adapter.pyi similarity index 100% rename from typeshed/2and3/cachecontrol/adapter.pyi rename to typeshed/cachecontrol/adapter.pyi diff --git a/typeshed/2and3/cachecontrol/cache.pyi b/typeshed/cachecontrol/cache.pyi similarity index 100% rename from typeshed/2and3/cachecontrol/cache.pyi rename to typeshed/cachecontrol/cache.pyi diff --git a/typeshed/2and3/cachecontrol/caches/__init__.pyi b/typeshed/cachecontrol/caches/__init__.pyi similarity index 100% rename from typeshed/2and3/cachecontrol/caches/__init__.pyi rename to typeshed/cachecontrol/caches/__init__.pyi diff --git a/typeshed/2and3/cachecontrol/caches/file_cache.pyi b/typeshed/cachecontrol/caches/file_cache.pyi similarity index 100% rename from typeshed/2and3/cachecontrol/caches/file_cache.pyi rename to typeshed/cachecontrol/caches/file_cache.pyi diff --git a/typeshed/2and3/cachecontrol/compat.pyi b/typeshed/cachecontrol/compat.pyi similarity index 100% rename from typeshed/2and3/cachecontrol/compat.pyi rename to typeshed/cachecontrol/compat.pyi diff --git a/typeshed/2and3/cachecontrol/controller.pyi b/typeshed/cachecontrol/controller.pyi similarity index 100% rename from typeshed/2and3/cachecontrol/controller.pyi rename to typeshed/cachecontrol/controller.pyi diff --git a/typeshed/2and3/cachecontrol/filewrapper.pyi b/typeshed/cachecontrol/filewrapper.pyi similarity index 100% rename from typeshed/2and3/cachecontrol/filewrapper.pyi rename to typeshed/cachecontrol/filewrapper.pyi diff --git a/typeshed/2and3/cachecontrol/serialize.pyi b/typeshed/cachecontrol/serialize.pyi similarity index 100% rename from typeshed/2and3/cachecontrol/serialize.pyi rename to typeshed/cachecontrol/serialize.pyi diff --git a/typeshed/2and3/cachecontrol/wrapper.pyi b/typeshed/cachecontrol/wrapper.pyi similarity index 100% rename from typeshed/2and3/cachecontrol/wrapper.pyi rename to typeshed/cachecontrol/wrapper.pyi diff --git a/typeshed/2and3/coloredlogs/__init__.pyi b/typeshed/coloredlogs/__init__.pyi similarity index 100% rename from typeshed/2and3/coloredlogs/__init__.pyi rename to typeshed/coloredlogs/__init__.pyi diff --git a/typeshed/2and3/galaxy/__init__.pyi b/typeshed/galaxy/__init__.pyi similarity index 100% rename from typeshed/2and3/galaxy/__init__.pyi rename to typeshed/galaxy/__init__.pyi diff --git a/typeshed/2and3/galaxy/exceptions/__init__.pyi b/typeshed/galaxy/exceptions/__init__.pyi similarity index 100% rename from typeshed/2and3/galaxy/exceptions/__init__.pyi rename to typeshed/galaxy/exceptions/__init__.pyi diff --git a/typeshed/2and3/galaxy/exceptions/error_codes.pyi b/typeshed/galaxy/exceptions/error_codes.pyi similarity index 100% rename from typeshed/2and3/galaxy/exceptions/error_codes.pyi rename to typeshed/galaxy/exceptions/error_codes.pyi diff --git a/typeshed/2and3/galaxy/jobs/__init__.pyi b/typeshed/galaxy/jobs/__init__.pyi similarity index 100% rename from typeshed/2and3/galaxy/jobs/__init__.pyi rename to typeshed/galaxy/jobs/__init__.pyi diff --git a/typeshed/2and3/galaxy/jobs/metrics/__init__.pyi b/typeshed/galaxy/jobs/metrics/__init__.pyi similarity index 100% rename from typeshed/2and3/galaxy/jobs/metrics/__init__.pyi rename to typeshed/galaxy/jobs/metrics/__init__.pyi diff --git a/typeshed/2and3/galaxy/jobs/metrics/collectl/__init__.pyi b/typeshed/galaxy/jobs/metrics/collectl/__init__.pyi similarity index 100% rename from typeshed/2and3/galaxy/jobs/metrics/collectl/__init__.pyi rename to typeshed/galaxy/jobs/metrics/collectl/__init__.pyi diff --git a/typeshed/2and3/galaxy/jobs/metrics/collectl/cli.pyi b/typeshed/galaxy/jobs/metrics/collectl/cli.pyi similarity index 100% rename from typeshed/2and3/galaxy/jobs/metrics/collectl/cli.pyi rename to typeshed/galaxy/jobs/metrics/collectl/cli.pyi diff --git a/typeshed/2and3/galaxy/jobs/metrics/collectl/processes.pyi b/typeshed/galaxy/jobs/metrics/collectl/processes.pyi similarity index 100% rename from typeshed/2and3/galaxy/jobs/metrics/collectl/processes.pyi rename to typeshed/galaxy/jobs/metrics/collectl/processes.pyi diff --git a/typeshed/2and3/galaxy/jobs/metrics/collectl/stats.pyi b/typeshed/galaxy/jobs/metrics/collectl/stats.pyi similarity index 100% rename from typeshed/2and3/galaxy/jobs/metrics/collectl/stats.pyi rename to typeshed/galaxy/jobs/metrics/collectl/stats.pyi diff --git a/typeshed/2and3/galaxy/jobs/metrics/collectl/subsystems.pyi b/typeshed/galaxy/jobs/metrics/collectl/subsystems.pyi similarity index 100% rename from typeshed/2and3/galaxy/jobs/metrics/collectl/subsystems.pyi rename to typeshed/galaxy/jobs/metrics/collectl/subsystems.pyi diff --git a/typeshed/2and3/galaxy/jobs/metrics/formatting.pyi b/typeshed/galaxy/jobs/metrics/formatting.pyi similarity index 100% rename from typeshed/2and3/galaxy/jobs/metrics/formatting.pyi rename to typeshed/galaxy/jobs/metrics/formatting.pyi diff --git a/typeshed/2and3/galaxy/jobs/metrics/instrumenters/__init__.pyi b/typeshed/galaxy/jobs/metrics/instrumenters/__init__.pyi similarity index 100% rename from typeshed/2and3/galaxy/jobs/metrics/instrumenters/__init__.pyi rename to typeshed/galaxy/jobs/metrics/instrumenters/__init__.pyi diff --git a/typeshed/2and3/galaxy/jobs/metrics/instrumenters/collectl.pyi b/typeshed/galaxy/jobs/metrics/instrumenters/collectl.pyi similarity index 100% rename from typeshed/2and3/galaxy/jobs/metrics/instrumenters/collectl.pyi rename to typeshed/galaxy/jobs/metrics/instrumenters/collectl.pyi diff --git a/typeshed/2and3/galaxy/jobs/metrics/instrumenters/core.pyi b/typeshed/galaxy/jobs/metrics/instrumenters/core.pyi similarity index 100% rename from typeshed/2and3/galaxy/jobs/metrics/instrumenters/core.pyi rename to typeshed/galaxy/jobs/metrics/instrumenters/core.pyi diff --git a/typeshed/2and3/galaxy/jobs/metrics/instrumenters/cpuinfo.pyi b/typeshed/galaxy/jobs/metrics/instrumenters/cpuinfo.pyi similarity index 100% rename from typeshed/2and3/galaxy/jobs/metrics/instrumenters/cpuinfo.pyi rename to typeshed/galaxy/jobs/metrics/instrumenters/cpuinfo.pyi diff --git a/typeshed/2and3/galaxy/jobs/metrics/instrumenters/env.pyi b/typeshed/galaxy/jobs/metrics/instrumenters/env.pyi similarity index 100% rename from typeshed/2and3/galaxy/jobs/metrics/instrumenters/env.pyi rename to typeshed/galaxy/jobs/metrics/instrumenters/env.pyi diff --git a/typeshed/2and3/galaxy/jobs/metrics/instrumenters/meminfo.pyi b/typeshed/galaxy/jobs/metrics/instrumenters/meminfo.pyi similarity index 100% rename from typeshed/2and3/galaxy/jobs/metrics/instrumenters/meminfo.pyi rename to typeshed/galaxy/jobs/metrics/instrumenters/meminfo.pyi diff --git a/typeshed/2and3/galaxy/jobs/metrics/instrumenters/uname.pyi b/typeshed/galaxy/jobs/metrics/instrumenters/uname.pyi similarity index 100% rename from typeshed/2and3/galaxy/jobs/metrics/instrumenters/uname.pyi rename to typeshed/galaxy/jobs/metrics/instrumenters/uname.pyi diff --git a/typeshed/2and3/galaxy/objectstore/__init__.pyi b/typeshed/galaxy/objectstore/__init__.pyi similarity index 100% rename from typeshed/2and3/galaxy/objectstore/__init__.pyi rename to typeshed/galaxy/objectstore/__init__.pyi diff --git a/typeshed/2and3/galaxy/objectstore/azure_blob.pyi b/typeshed/galaxy/objectstore/azure_blob.pyi similarity index 100% rename from typeshed/2and3/galaxy/objectstore/azure_blob.pyi rename to typeshed/galaxy/objectstore/azure_blob.pyi diff --git a/typeshed/2and3/galaxy/objectstore/pulsar.pyi b/typeshed/galaxy/objectstore/pulsar.pyi similarity index 100% rename from typeshed/2and3/galaxy/objectstore/pulsar.pyi rename to typeshed/galaxy/objectstore/pulsar.pyi diff --git a/typeshed/2and3/galaxy/objectstore/rods.pyi b/typeshed/galaxy/objectstore/rods.pyi similarity index 100% rename from typeshed/2and3/galaxy/objectstore/rods.pyi rename to typeshed/galaxy/objectstore/rods.pyi diff --git a/typeshed/2and3/galaxy/objectstore/s3.pyi b/typeshed/galaxy/objectstore/s3.pyi similarity index 100% rename from typeshed/2and3/galaxy/objectstore/s3.pyi rename to typeshed/galaxy/objectstore/s3.pyi diff --git a/typeshed/2and3/galaxy/objectstore/s3_multipart_upload.pyi b/typeshed/galaxy/objectstore/s3_multipart_upload.pyi similarity index 100% rename from typeshed/2and3/galaxy/objectstore/s3_multipart_upload.pyi rename to typeshed/galaxy/objectstore/s3_multipart_upload.pyi diff --git a/typeshed/2and3/galaxy/tool_util/__init__.pyi b/typeshed/galaxy/tool_util/__init__.pyi similarity index 100% rename from typeshed/2and3/galaxy/tool_util/__init__.pyi rename to typeshed/galaxy/tool_util/__init__.pyi diff --git a/typeshed/2and3/galaxy/tool_util/cwl/__init__.pyi b/typeshed/galaxy/tool_util/cwl/__init__.pyi similarity index 100% rename from typeshed/2and3/galaxy/tool_util/cwl/__init__.pyi rename to typeshed/galaxy/tool_util/cwl/__init__.pyi diff --git a/typeshed/2and3/galaxy/tool_util/cwl/cwltool_deps.pyi b/typeshed/galaxy/tool_util/cwl/cwltool_deps.pyi similarity index 100% rename from typeshed/2and3/galaxy/tool_util/cwl/cwltool_deps.pyi rename to typeshed/galaxy/tool_util/cwl/cwltool_deps.pyi diff --git a/typeshed/2and3/galaxy/tool_util/cwl/parser.pyi b/typeshed/galaxy/tool_util/cwl/parser.pyi similarity index 100% rename from typeshed/2and3/galaxy/tool_util/cwl/parser.pyi rename to typeshed/galaxy/tool_util/cwl/parser.pyi diff --git a/typeshed/2and3/galaxy/tool_util/cwl/representation.pyi b/typeshed/galaxy/tool_util/cwl/representation.pyi similarity index 100% rename from typeshed/2and3/galaxy/tool_util/cwl/representation.pyi rename to typeshed/galaxy/tool_util/cwl/representation.pyi diff --git a/typeshed/2and3/galaxy/tool_util/cwl/runtime_actions.pyi b/typeshed/galaxy/tool_util/cwl/runtime_actions.pyi similarity index 100% rename from typeshed/2and3/galaxy/tool_util/cwl/runtime_actions.pyi rename to typeshed/galaxy/tool_util/cwl/runtime_actions.pyi diff --git a/typeshed/2and3/galaxy/tool_util/cwl/schema.pyi b/typeshed/galaxy/tool_util/cwl/schema.pyi similarity index 100% rename from typeshed/2and3/galaxy/tool_util/cwl/schema.pyi rename to typeshed/galaxy/tool_util/cwl/schema.pyi diff --git a/typeshed/2and3/galaxy/tool_util/deps/__init__.pyi b/typeshed/galaxy/tool_util/deps/__init__.pyi similarity index 100% rename from typeshed/2and3/galaxy/tool_util/deps/__init__.pyi rename to typeshed/galaxy/tool_util/deps/__init__.pyi diff --git a/typeshed/2and3/galaxy/tool_util/deps/brew_exts.pyi b/typeshed/galaxy/tool_util/deps/brew_exts.pyi similarity index 100% rename from typeshed/2and3/galaxy/tool_util/deps/brew_exts.pyi rename to typeshed/galaxy/tool_util/deps/brew_exts.pyi diff --git a/typeshed/2and3/galaxy/tool_util/deps/brew_util.pyi b/typeshed/galaxy/tool_util/deps/brew_util.pyi similarity index 100% rename from typeshed/2and3/galaxy/tool_util/deps/brew_util.pyi rename to typeshed/galaxy/tool_util/deps/brew_util.pyi diff --git a/typeshed/2and3/galaxy/tool_util/deps/commands.pyi b/typeshed/galaxy/tool_util/deps/commands.pyi similarity index 100% rename from typeshed/2and3/galaxy/tool_util/deps/commands.pyi rename to typeshed/galaxy/tool_util/deps/commands.pyi diff --git a/typeshed/2and3/galaxy/tool_util/deps/conda_compat.pyi b/typeshed/galaxy/tool_util/deps/conda_compat.pyi similarity index 100% rename from typeshed/2and3/galaxy/tool_util/deps/conda_compat.pyi rename to typeshed/galaxy/tool_util/deps/conda_compat.pyi diff --git a/typeshed/2and3/galaxy/tool_util/deps/conda_util.pyi b/typeshed/galaxy/tool_util/deps/conda_util.pyi similarity index 100% rename from typeshed/2and3/galaxy/tool_util/deps/conda_util.pyi rename to typeshed/galaxy/tool_util/deps/conda_util.pyi diff --git a/typeshed/2and3/galaxy/tool_util/deps/container_resolvers/__init__.pyi b/typeshed/galaxy/tool_util/deps/container_resolvers/__init__.pyi similarity index 100% rename from typeshed/2and3/galaxy/tool_util/deps/container_resolvers/__init__.pyi rename to typeshed/galaxy/tool_util/deps/container_resolvers/__init__.pyi diff --git a/typeshed/2and3/galaxy/tool_util/deps/container_resolvers/explicit.pyi b/typeshed/galaxy/tool_util/deps/container_resolvers/explicit.pyi similarity index 100% rename from typeshed/2and3/galaxy/tool_util/deps/container_resolvers/explicit.pyi rename to typeshed/galaxy/tool_util/deps/container_resolvers/explicit.pyi diff --git a/typeshed/2and3/galaxy/tool_util/deps/container_resolvers/mulled.pyi b/typeshed/galaxy/tool_util/deps/container_resolvers/mulled.pyi similarity index 100% rename from typeshed/2and3/galaxy/tool_util/deps/container_resolvers/mulled.pyi rename to typeshed/galaxy/tool_util/deps/container_resolvers/mulled.pyi diff --git a/typeshed/2and3/galaxy/tool_util/deps/containers.pyi b/typeshed/galaxy/tool_util/deps/containers.pyi similarity index 100% rename from typeshed/2and3/galaxy/tool_util/deps/containers.pyi rename to typeshed/galaxy/tool_util/deps/containers.pyi diff --git a/typeshed/2and3/galaxy/tool_util/deps/dependencies.pyi b/typeshed/galaxy/tool_util/deps/dependencies.pyi similarity index 100% rename from typeshed/2and3/galaxy/tool_util/deps/dependencies.pyi rename to typeshed/galaxy/tool_util/deps/dependencies.pyi diff --git a/typeshed/2and3/galaxy/tool_util/deps/docker_util.pyi b/typeshed/galaxy/tool_util/deps/docker_util.pyi similarity index 100% rename from typeshed/2and3/galaxy/tool_util/deps/docker_util.pyi rename to typeshed/galaxy/tool_util/deps/docker_util.pyi diff --git a/typeshed/2and3/galaxy/tool_util/deps/dockerfiles.pyi b/typeshed/galaxy/tool_util/deps/dockerfiles.pyi similarity index 100% rename from typeshed/2and3/galaxy/tool_util/deps/dockerfiles.pyi rename to typeshed/galaxy/tool_util/deps/dockerfiles.pyi diff --git a/typeshed/2and3/galaxy/tool_util/deps/installable.pyi b/typeshed/galaxy/tool_util/deps/installable.pyi similarity index 100% rename from typeshed/2and3/galaxy/tool_util/deps/installable.pyi rename to typeshed/galaxy/tool_util/deps/installable.pyi diff --git a/typeshed/2and3/galaxy/tool_util/deps/mulled/__init__.pyi b/typeshed/galaxy/tool_util/deps/mulled/__init__.pyi similarity index 100% rename from typeshed/2and3/galaxy/tool_util/deps/mulled/__init__.pyi rename to typeshed/galaxy/tool_util/deps/mulled/__init__.pyi diff --git a/typeshed/2and3/galaxy/tool_util/deps/mulled/_cli.pyi b/typeshed/galaxy/tool_util/deps/mulled/_cli.pyi similarity index 100% rename from typeshed/2and3/galaxy/tool_util/deps/mulled/_cli.pyi rename to typeshed/galaxy/tool_util/deps/mulled/_cli.pyi diff --git a/typeshed/2and3/galaxy/tool_util/deps/mulled/mulled_build.pyi b/typeshed/galaxy/tool_util/deps/mulled/mulled_build.pyi similarity index 100% rename from typeshed/2and3/galaxy/tool_util/deps/mulled/mulled_build.pyi rename to typeshed/galaxy/tool_util/deps/mulled/mulled_build.pyi diff --git a/typeshed/2and3/galaxy/tool_util/deps/mulled/mulled_build_channel.pyi b/typeshed/galaxy/tool_util/deps/mulled/mulled_build_channel.pyi similarity index 100% rename from typeshed/2and3/galaxy/tool_util/deps/mulled/mulled_build_channel.pyi rename to typeshed/galaxy/tool_util/deps/mulled/mulled_build_channel.pyi diff --git a/typeshed/2and3/galaxy/tool_util/deps/mulled/mulled_build_files.pyi b/typeshed/galaxy/tool_util/deps/mulled/mulled_build_files.pyi similarity index 100% rename from typeshed/2and3/galaxy/tool_util/deps/mulled/mulled_build_files.pyi rename to typeshed/galaxy/tool_util/deps/mulled/mulled_build_files.pyi diff --git a/typeshed/2and3/galaxy/tool_util/deps/mulled/mulled_build_tool.pyi b/typeshed/galaxy/tool_util/deps/mulled/mulled_build_tool.pyi similarity index 100% rename from typeshed/2and3/galaxy/tool_util/deps/mulled/mulled_build_tool.pyi rename to typeshed/galaxy/tool_util/deps/mulled/mulled_build_tool.pyi diff --git a/typeshed/2and3/galaxy/tool_util/deps/mulled/mulled_search.pyi b/typeshed/galaxy/tool_util/deps/mulled/mulled_search.pyi similarity index 100% rename from typeshed/2and3/galaxy/tool_util/deps/mulled/mulled_search.pyi rename to typeshed/galaxy/tool_util/deps/mulled/mulled_search.pyi diff --git a/typeshed/2and3/galaxy/tool_util/deps/mulled/util.pyi b/typeshed/galaxy/tool_util/deps/mulled/util.pyi similarity index 100% rename from typeshed/2and3/galaxy/tool_util/deps/mulled/util.pyi rename to typeshed/galaxy/tool_util/deps/mulled/util.pyi diff --git a/typeshed/2and3/galaxy/tool_util/deps/requirements.pyi b/typeshed/galaxy/tool_util/deps/requirements.pyi similarity index 100% rename from typeshed/2and3/galaxy/tool_util/deps/requirements.pyi rename to typeshed/galaxy/tool_util/deps/requirements.pyi diff --git a/typeshed/2and3/galaxy/tool_util/deps/resolvers/__init__.pyi b/typeshed/galaxy/tool_util/deps/resolvers/__init__.pyi similarity index 100% rename from typeshed/2and3/galaxy/tool_util/deps/resolvers/__init__.pyi rename to typeshed/galaxy/tool_util/deps/resolvers/__init__.pyi diff --git a/typeshed/2and3/galaxy/tool_util/deps/resolvers/brewed_tool_shed_packages.pyi b/typeshed/galaxy/tool_util/deps/resolvers/brewed_tool_shed_packages.pyi similarity index 100% rename from typeshed/2and3/galaxy/tool_util/deps/resolvers/brewed_tool_shed_packages.pyi rename to typeshed/galaxy/tool_util/deps/resolvers/brewed_tool_shed_packages.pyi diff --git a/typeshed/2and3/galaxy/tool_util/deps/resolvers/conda.pyi b/typeshed/galaxy/tool_util/deps/resolvers/conda.pyi similarity index 100% rename from typeshed/2and3/galaxy/tool_util/deps/resolvers/conda.pyi rename to typeshed/galaxy/tool_util/deps/resolvers/conda.pyi diff --git a/typeshed/2and3/galaxy/tool_util/deps/resolvers/galaxy_packages.pyi b/typeshed/galaxy/tool_util/deps/resolvers/galaxy_packages.pyi similarity index 100% rename from typeshed/2and3/galaxy/tool_util/deps/resolvers/galaxy_packages.pyi rename to typeshed/galaxy/tool_util/deps/resolvers/galaxy_packages.pyi diff --git a/typeshed/2and3/galaxy/tool_util/deps/resolvers/homebrew.pyi b/typeshed/galaxy/tool_util/deps/resolvers/homebrew.pyi similarity index 100% rename from typeshed/2and3/galaxy/tool_util/deps/resolvers/homebrew.pyi rename to typeshed/galaxy/tool_util/deps/resolvers/homebrew.pyi diff --git a/typeshed/2and3/galaxy/tool_util/deps/resolvers/modules.pyi b/typeshed/galaxy/tool_util/deps/resolvers/modules.pyi similarity index 100% rename from typeshed/2and3/galaxy/tool_util/deps/resolvers/modules.pyi rename to typeshed/galaxy/tool_util/deps/resolvers/modules.pyi diff --git a/typeshed/2and3/galaxy/tool_util/deps/resolvers/resolver_mixins.pyi b/typeshed/galaxy/tool_util/deps/resolvers/resolver_mixins.pyi similarity index 100% rename from typeshed/2and3/galaxy/tool_util/deps/resolvers/resolver_mixins.pyi rename to typeshed/galaxy/tool_util/deps/resolvers/resolver_mixins.pyi diff --git a/typeshed/2and3/galaxy/tool_util/deps/resolvers/tool_shed_packages.pyi b/typeshed/galaxy/tool_util/deps/resolvers/tool_shed_packages.pyi similarity index 100% rename from typeshed/2and3/galaxy/tool_util/deps/resolvers/tool_shed_packages.pyi rename to typeshed/galaxy/tool_util/deps/resolvers/tool_shed_packages.pyi diff --git a/typeshed/2and3/galaxy/tool_util/deps/resolvers/unlinked_tool_shed_packages.pyi b/typeshed/galaxy/tool_util/deps/resolvers/unlinked_tool_shed_packages.pyi similarity index 100% rename from typeshed/2and3/galaxy/tool_util/deps/resolvers/unlinked_tool_shed_packages.pyi rename to typeshed/galaxy/tool_util/deps/resolvers/unlinked_tool_shed_packages.pyi diff --git a/typeshed/2and3/galaxy/tool_util/deps/singularity_util.pyi b/typeshed/galaxy/tool_util/deps/singularity_util.pyi similarity index 100% rename from typeshed/2and3/galaxy/tool_util/deps/singularity_util.pyi rename to typeshed/galaxy/tool_util/deps/singularity_util.pyi diff --git a/typeshed/2and3/galaxy/tool_util/deps/views.pyi b/typeshed/galaxy/tool_util/deps/views.pyi similarity index 100% rename from typeshed/2and3/galaxy/tool_util/deps/views.pyi rename to typeshed/galaxy/tool_util/deps/views.pyi diff --git a/typeshed/2and3/galaxy/tool_util/fetcher.pyi b/typeshed/galaxy/tool_util/fetcher.pyi similarity index 100% rename from typeshed/2and3/galaxy/tool_util/fetcher.pyi rename to typeshed/galaxy/tool_util/fetcher.pyi diff --git a/typeshed/2and3/galaxy/tool_util/lint.pyi b/typeshed/galaxy/tool_util/lint.pyi similarity index 100% rename from typeshed/2and3/galaxy/tool_util/lint.pyi rename to typeshed/galaxy/tool_util/lint.pyi diff --git a/typeshed/2and3/galaxy/tool_util/lint_util.pyi b/typeshed/galaxy/tool_util/lint_util.pyi similarity index 100% rename from typeshed/2and3/galaxy/tool_util/lint_util.pyi rename to typeshed/galaxy/tool_util/lint_util.pyi diff --git a/typeshed/2and3/galaxy/tool_util/linters/__init__.pyi b/typeshed/galaxy/tool_util/linters/__init__.pyi similarity index 100% rename from typeshed/2and3/galaxy/tool_util/linters/__init__.pyi rename to typeshed/galaxy/tool_util/linters/__init__.pyi diff --git a/typeshed/2and3/galaxy/tool_util/linters/citations.pyi b/typeshed/galaxy/tool_util/linters/citations.pyi similarity index 100% rename from typeshed/2and3/galaxy/tool_util/linters/citations.pyi rename to typeshed/galaxy/tool_util/linters/citations.pyi diff --git a/typeshed/2and3/galaxy/tool_util/linters/command.pyi b/typeshed/galaxy/tool_util/linters/command.pyi similarity index 100% rename from typeshed/2and3/galaxy/tool_util/linters/command.pyi rename to typeshed/galaxy/tool_util/linters/command.pyi diff --git a/typeshed/2and3/galaxy/tool_util/linters/cwl.pyi b/typeshed/galaxy/tool_util/linters/cwl.pyi similarity index 100% rename from typeshed/2and3/galaxy/tool_util/linters/cwl.pyi rename to typeshed/galaxy/tool_util/linters/cwl.pyi diff --git a/typeshed/2and3/galaxy/tool_util/linters/general.pyi b/typeshed/galaxy/tool_util/linters/general.pyi similarity index 100% rename from typeshed/2and3/galaxy/tool_util/linters/general.pyi rename to typeshed/galaxy/tool_util/linters/general.pyi diff --git a/typeshed/2and3/galaxy/tool_util/linters/help.pyi b/typeshed/galaxy/tool_util/linters/help.pyi similarity index 100% rename from typeshed/2and3/galaxy/tool_util/linters/help.pyi rename to typeshed/galaxy/tool_util/linters/help.pyi diff --git a/typeshed/2and3/galaxy/tool_util/linters/inputs.pyi b/typeshed/galaxy/tool_util/linters/inputs.pyi similarity index 100% rename from typeshed/2and3/galaxy/tool_util/linters/inputs.pyi rename to typeshed/galaxy/tool_util/linters/inputs.pyi diff --git a/typeshed/2and3/galaxy/tool_util/linters/outputs.pyi b/typeshed/galaxy/tool_util/linters/outputs.pyi similarity index 100% rename from typeshed/2and3/galaxy/tool_util/linters/outputs.pyi rename to typeshed/galaxy/tool_util/linters/outputs.pyi diff --git a/typeshed/2and3/galaxy/tool_util/linters/stdio.pyi b/typeshed/galaxy/tool_util/linters/stdio.pyi similarity index 100% rename from typeshed/2and3/galaxy/tool_util/linters/stdio.pyi rename to typeshed/galaxy/tool_util/linters/stdio.pyi diff --git a/typeshed/2and3/galaxy/tool_util/linters/tests.pyi b/typeshed/galaxy/tool_util/linters/tests.pyi similarity index 100% rename from typeshed/2and3/galaxy/tool_util/linters/tests.pyi rename to typeshed/galaxy/tool_util/linters/tests.pyi diff --git a/typeshed/2and3/galaxy/tool_util/linters/xml_order.pyi b/typeshed/galaxy/tool_util/linters/xml_order.pyi similarity index 100% rename from typeshed/2and3/galaxy/tool_util/linters/xml_order.pyi rename to typeshed/galaxy/tool_util/linters/xml_order.pyi diff --git a/typeshed/2and3/galaxy/tool_util/loader.pyi b/typeshed/galaxy/tool_util/loader.pyi similarity index 100% rename from typeshed/2and3/galaxy/tool_util/loader.pyi rename to typeshed/galaxy/tool_util/loader.pyi diff --git a/typeshed/2and3/galaxy/tool_util/loader_directory.pyi b/typeshed/galaxy/tool_util/loader_directory.pyi similarity index 100% rename from typeshed/2and3/galaxy/tool_util/loader_directory.pyi rename to typeshed/galaxy/tool_util/loader_directory.pyi diff --git a/typeshed/2and3/galaxy/tool_util/locations/__init__.pyi b/typeshed/galaxy/tool_util/locations/__init__.pyi similarity index 100% rename from typeshed/2and3/galaxy/tool_util/locations/__init__.pyi rename to typeshed/galaxy/tool_util/locations/__init__.pyi diff --git a/typeshed/2and3/galaxy/tool_util/locations/dockstore.pyi b/typeshed/galaxy/tool_util/locations/dockstore.pyi similarity index 100% rename from typeshed/2and3/galaxy/tool_util/locations/dockstore.pyi rename to typeshed/galaxy/tool_util/locations/dockstore.pyi diff --git a/typeshed/2and3/galaxy/tool_util/locations/file.pyi b/typeshed/galaxy/tool_util/locations/file.pyi similarity index 100% rename from typeshed/2and3/galaxy/tool_util/locations/file.pyi rename to typeshed/galaxy/tool_util/locations/file.pyi diff --git a/typeshed/2and3/galaxy/tool_util/locations/http.pyi b/typeshed/galaxy/tool_util/locations/http.pyi similarity index 100% rename from typeshed/2and3/galaxy/tool_util/locations/http.pyi rename to typeshed/galaxy/tool_util/locations/http.pyi diff --git a/typeshed/2and3/galaxy/tool_util/parser/__init__.pyi b/typeshed/galaxy/tool_util/parser/__init__.pyi similarity index 100% rename from typeshed/2and3/galaxy/tool_util/parser/__init__.pyi rename to typeshed/galaxy/tool_util/parser/__init__.pyi diff --git a/typeshed/2and3/galaxy/tool_util/parser/cwl.pyi b/typeshed/galaxy/tool_util/parser/cwl.pyi similarity index 100% rename from typeshed/2and3/galaxy/tool_util/parser/cwl.pyi rename to typeshed/galaxy/tool_util/parser/cwl.pyi diff --git a/typeshed/2and3/galaxy/tool_util/parser/factory.pyi b/typeshed/galaxy/tool_util/parser/factory.pyi similarity index 100% rename from typeshed/2and3/galaxy/tool_util/parser/factory.pyi rename to typeshed/galaxy/tool_util/parser/factory.pyi diff --git a/typeshed/2and3/galaxy/tool_util/parser/interface.pyi b/typeshed/galaxy/tool_util/parser/interface.pyi similarity index 100% rename from typeshed/2and3/galaxy/tool_util/parser/interface.pyi rename to typeshed/galaxy/tool_util/parser/interface.pyi diff --git a/typeshed/2and3/galaxy/tool_util/parser/output_actions.pyi b/typeshed/galaxy/tool_util/parser/output_actions.pyi similarity index 100% rename from typeshed/2and3/galaxy/tool_util/parser/output_actions.pyi rename to typeshed/galaxy/tool_util/parser/output_actions.pyi diff --git a/typeshed/2and3/galaxy/tool_util/parser/output_collection_def.pyi b/typeshed/galaxy/tool_util/parser/output_collection_def.pyi similarity index 100% rename from typeshed/2and3/galaxy/tool_util/parser/output_collection_def.pyi rename to typeshed/galaxy/tool_util/parser/output_collection_def.pyi diff --git a/typeshed/2and3/galaxy/tool_util/parser/output_objects.pyi b/typeshed/galaxy/tool_util/parser/output_objects.pyi similarity index 100% rename from typeshed/2and3/galaxy/tool_util/parser/output_objects.pyi rename to typeshed/galaxy/tool_util/parser/output_objects.pyi diff --git a/typeshed/2and3/galaxy/tool_util/parser/util.pyi b/typeshed/galaxy/tool_util/parser/util.pyi similarity index 100% rename from typeshed/2and3/galaxy/tool_util/parser/util.pyi rename to typeshed/galaxy/tool_util/parser/util.pyi diff --git a/typeshed/2and3/galaxy/tool_util/parser/xml.pyi b/typeshed/galaxy/tool_util/parser/xml.pyi similarity index 100% rename from typeshed/2and3/galaxy/tool_util/parser/xml.pyi rename to typeshed/galaxy/tool_util/parser/xml.pyi diff --git a/typeshed/2and3/galaxy/tool_util/parser/yaml.pyi b/typeshed/galaxy/tool_util/parser/yaml.pyi similarity index 100% rename from typeshed/2and3/galaxy/tool_util/parser/yaml.pyi rename to typeshed/galaxy/tool_util/parser/yaml.pyi diff --git a/typeshed/2and3/galaxy/tool_util/verify/__init__.pyi b/typeshed/galaxy/tool_util/verify/__init__.pyi similarity index 100% rename from typeshed/2and3/galaxy/tool_util/verify/__init__.pyi rename to typeshed/galaxy/tool_util/verify/__init__.pyi diff --git a/typeshed/2and3/galaxy/tool_util/verify/asserts/__init__.pyi b/typeshed/galaxy/tool_util/verify/asserts/__init__.pyi similarity index 100% rename from typeshed/2and3/galaxy/tool_util/verify/asserts/__init__.pyi rename to typeshed/galaxy/tool_util/verify/asserts/__init__.pyi diff --git a/typeshed/2and3/galaxy/tool_util/verify/asserts/tabular.pyi b/typeshed/galaxy/tool_util/verify/asserts/tabular.pyi similarity index 100% rename from typeshed/2and3/galaxy/tool_util/verify/asserts/tabular.pyi rename to typeshed/galaxy/tool_util/verify/asserts/tabular.pyi diff --git a/typeshed/2and3/galaxy/tool_util/verify/asserts/text.pyi b/typeshed/galaxy/tool_util/verify/asserts/text.pyi similarity index 100% rename from typeshed/2and3/galaxy/tool_util/verify/asserts/text.pyi rename to typeshed/galaxy/tool_util/verify/asserts/text.pyi diff --git a/typeshed/2and3/galaxy/tool_util/verify/asserts/xml.pyi b/typeshed/galaxy/tool_util/verify/asserts/xml.pyi similarity index 100% rename from typeshed/2and3/galaxy/tool_util/verify/asserts/xml.pyi rename to typeshed/galaxy/tool_util/verify/asserts/xml.pyi diff --git a/typeshed/2and3/galaxy/tool_util/verify/test_data.pyi b/typeshed/galaxy/tool_util/verify/test_data.pyi similarity index 100% rename from typeshed/2and3/galaxy/tool_util/verify/test_data.pyi rename to typeshed/galaxy/tool_util/verify/test_data.pyi diff --git a/typeshed/2and3/galaxy/util/__init__.pyi b/typeshed/galaxy/util/__init__.pyi similarity index 100% rename from typeshed/2and3/galaxy/util/__init__.pyi rename to typeshed/galaxy/util/__init__.pyi diff --git a/typeshed/2and3/galaxy/util/aliaspickler.pyi b/typeshed/galaxy/util/aliaspickler.pyi similarity index 100% rename from typeshed/2and3/galaxy/util/aliaspickler.pyi rename to typeshed/galaxy/util/aliaspickler.pyi diff --git a/typeshed/2and3/galaxy/util/bunch.pyi b/typeshed/galaxy/util/bunch.pyi similarity index 100% rename from typeshed/2and3/galaxy/util/bunch.pyi rename to typeshed/galaxy/util/bunch.pyi diff --git a/typeshed/2and3/galaxy/util/checkers.pyi b/typeshed/galaxy/util/checkers.pyi similarity index 100% rename from typeshed/2and3/galaxy/util/checkers.pyi rename to typeshed/galaxy/util/checkers.pyi diff --git a/typeshed/2and3/galaxy/util/compression_utils.pyi b/typeshed/galaxy/util/compression_utils.pyi similarity index 100% rename from typeshed/2and3/galaxy/util/compression_utils.pyi rename to typeshed/galaxy/util/compression_utils.pyi diff --git a/typeshed/2and3/galaxy/util/dictifiable.pyi b/typeshed/galaxy/util/dictifiable.pyi similarity index 100% rename from typeshed/2and3/galaxy/util/dictifiable.pyi rename to typeshed/galaxy/util/dictifiable.pyi diff --git a/typeshed/2and3/galaxy/util/expressions.pyi b/typeshed/galaxy/util/expressions.pyi similarity index 100% rename from typeshed/2and3/galaxy/util/expressions.pyi rename to typeshed/galaxy/util/expressions.pyi diff --git a/typeshed/2and3/galaxy/util/filelock.pyi b/typeshed/galaxy/util/filelock.pyi similarity index 100% rename from typeshed/2and3/galaxy/util/filelock.pyi rename to typeshed/galaxy/util/filelock.pyi diff --git a/typeshed/2and3/galaxy/util/hash_util.pyi b/typeshed/galaxy/util/hash_util.pyi similarity index 100% rename from typeshed/2and3/galaxy/util/hash_util.pyi rename to typeshed/galaxy/util/hash_util.pyi diff --git a/typeshed/2and3/galaxy/util/heartbeat.pyi b/typeshed/galaxy/util/heartbeat.pyi similarity index 100% rename from typeshed/2and3/galaxy/util/heartbeat.pyi rename to typeshed/galaxy/util/heartbeat.pyi diff --git a/typeshed/2and3/galaxy/util/image_util.pyi b/typeshed/galaxy/util/image_util.pyi similarity index 100% rename from typeshed/2and3/galaxy/util/image_util.pyi rename to typeshed/galaxy/util/image_util.pyi diff --git a/typeshed/2and3/galaxy/util/inflection.pyi b/typeshed/galaxy/util/inflection.pyi similarity index 100% rename from typeshed/2and3/galaxy/util/inflection.pyi rename to typeshed/galaxy/util/inflection.pyi diff --git a/typeshed/2and3/galaxy/util/json.pyi b/typeshed/galaxy/util/json.pyi similarity index 100% rename from typeshed/2and3/galaxy/util/json.pyi rename to typeshed/galaxy/util/json.pyi diff --git a/typeshed/2and3/galaxy/util/lazy_process.pyi b/typeshed/galaxy/util/lazy_process.pyi similarity index 100% rename from typeshed/2and3/galaxy/util/lazy_process.pyi rename to typeshed/galaxy/util/lazy_process.pyi diff --git a/typeshed/2and3/galaxy/util/object_wrapper.pyi b/typeshed/galaxy/util/object_wrapper.pyi similarity index 100% rename from typeshed/2and3/galaxy/util/object_wrapper.pyi rename to typeshed/galaxy/util/object_wrapper.pyi diff --git a/typeshed/2and3/galaxy/util/odict.pyi b/typeshed/galaxy/util/odict.pyi similarity index 100% rename from typeshed/2and3/galaxy/util/odict.pyi rename to typeshed/galaxy/util/odict.pyi diff --git a/typeshed/2and3/galaxy/util/oset.pyi b/typeshed/galaxy/util/oset.pyi similarity index 100% rename from typeshed/2and3/galaxy/util/oset.pyi rename to typeshed/galaxy/util/oset.pyi diff --git a/typeshed/2and3/galaxy/util/plugin_config.pyi b/typeshed/galaxy/util/plugin_config.pyi similarity index 100% rename from typeshed/2and3/galaxy/util/plugin_config.pyi rename to typeshed/galaxy/util/plugin_config.pyi diff --git a/typeshed/2and3/galaxy/util/properties.pyi b/typeshed/galaxy/util/properties.pyi similarity index 100% rename from typeshed/2and3/galaxy/util/properties.pyi rename to typeshed/galaxy/util/properties.pyi diff --git a/typeshed/2and3/galaxy/util/simplegraph.pyi b/typeshed/galaxy/util/simplegraph.pyi similarity index 100% rename from typeshed/2and3/galaxy/util/simplegraph.pyi rename to typeshed/galaxy/util/simplegraph.pyi diff --git a/typeshed/2and3/galaxy/util/sleeper.pyi b/typeshed/galaxy/util/sleeper.pyi similarity index 100% rename from typeshed/2and3/galaxy/util/sleeper.pyi rename to typeshed/galaxy/util/sleeper.pyi diff --git a/typeshed/2and3/galaxy/util/sockets.pyi b/typeshed/galaxy/util/sockets.pyi similarity index 100% rename from typeshed/2and3/galaxy/util/sockets.pyi rename to typeshed/galaxy/util/sockets.pyi diff --git a/typeshed/2and3/galaxy/util/specs.pyi b/typeshed/galaxy/util/specs.pyi similarity index 100% rename from typeshed/2and3/galaxy/util/specs.pyi rename to typeshed/galaxy/util/specs.pyi diff --git a/typeshed/2and3/galaxy/util/sqlite.pyi b/typeshed/galaxy/util/sqlite.pyi similarity index 100% rename from typeshed/2and3/galaxy/util/sqlite.pyi rename to typeshed/galaxy/util/sqlite.pyi diff --git a/typeshed/2and3/galaxy/util/submodules.pyi b/typeshed/galaxy/util/submodules.pyi similarity index 100% rename from typeshed/2and3/galaxy/util/submodules.pyi rename to typeshed/galaxy/util/submodules.pyi diff --git a/typeshed/2and3/galaxy/util/topsort.pyi b/typeshed/galaxy/util/topsort.pyi similarity index 100% rename from typeshed/2and3/galaxy/util/topsort.pyi rename to typeshed/galaxy/util/topsort.pyi diff --git a/typeshed/2and3/galaxy/util/xml_macros.pyi b/typeshed/galaxy/util/xml_macros.pyi similarity index 100% rename from typeshed/2and3/galaxy/util/xml_macros.pyi rename to typeshed/galaxy/util/xml_macros.pyi diff --git a/typeshed/2and3/graphviz/__init__.pyi b/typeshed/graphviz/__init__.pyi similarity index 100% rename from typeshed/2and3/graphviz/__init__.pyi rename to typeshed/graphviz/__init__.pyi diff --git a/typeshed/2and3/graphviz/_compat.pyi b/typeshed/graphviz/_compat.pyi similarity index 100% rename from typeshed/2and3/graphviz/_compat.pyi rename to typeshed/graphviz/_compat.pyi diff --git a/typeshed/2and3/graphviz/backend.pyi b/typeshed/graphviz/backend.pyi similarity index 100% rename from typeshed/2and3/graphviz/backend.pyi rename to typeshed/graphviz/backend.pyi diff --git a/typeshed/2and3/graphviz/dot.pyi b/typeshed/graphviz/dot.pyi similarity index 100% rename from typeshed/2and3/graphviz/dot.pyi rename to typeshed/graphviz/dot.pyi diff --git a/typeshed/2and3/graphviz/files.pyi b/typeshed/graphviz/files.pyi similarity index 100% rename from typeshed/2and3/graphviz/files.pyi rename to typeshed/graphviz/files.pyi diff --git a/typeshed/2and3/graphviz/lang.pyi b/typeshed/graphviz/lang.pyi similarity index 100% rename from typeshed/2and3/graphviz/lang.pyi rename to typeshed/graphviz/lang.pyi diff --git a/typeshed/2and3/graphviz/tools.pyi b/typeshed/graphviz/tools.pyi similarity index 100% rename from typeshed/2and3/graphviz/tools.pyi rename to typeshed/graphviz/tools.pyi diff --git a/typeshed/2and3/mistune.pyi b/typeshed/mistune.pyi similarity index 100% rename from typeshed/2and3/mistune.pyi rename to typeshed/mistune.pyi diff --git a/typeshed/2and3/networkx/__init__.pyi b/typeshed/networkx/__init__.pyi similarity index 100% rename from typeshed/2and3/networkx/__init__.pyi rename to typeshed/networkx/__init__.pyi diff --git a/typeshed/2and3/networkx/algorithms/__init__.pyi b/typeshed/networkx/algorithms/__init__.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/__init__.pyi rename to typeshed/networkx/algorithms/__init__.pyi diff --git a/typeshed/2and3/networkx/algorithms/approximation/__init__.pyi b/typeshed/networkx/algorithms/approximation/__init__.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/approximation/__init__.pyi rename to typeshed/networkx/algorithms/approximation/__init__.pyi diff --git a/typeshed/2and3/networkx/algorithms/approximation/clique.pyi b/typeshed/networkx/algorithms/approximation/clique.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/approximation/clique.pyi rename to typeshed/networkx/algorithms/approximation/clique.pyi diff --git a/typeshed/2and3/networkx/algorithms/approximation/clustering_coefficient.pyi b/typeshed/networkx/algorithms/approximation/clustering_coefficient.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/approximation/clustering_coefficient.pyi rename to typeshed/networkx/algorithms/approximation/clustering_coefficient.pyi diff --git a/typeshed/2and3/networkx/algorithms/approximation/connectivity.pyi b/typeshed/networkx/algorithms/approximation/connectivity.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/approximation/connectivity.pyi rename to typeshed/networkx/algorithms/approximation/connectivity.pyi diff --git a/typeshed/2and3/networkx/algorithms/approximation/dominating_set.pyi b/typeshed/networkx/algorithms/approximation/dominating_set.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/approximation/dominating_set.pyi rename to typeshed/networkx/algorithms/approximation/dominating_set.pyi diff --git a/typeshed/2and3/networkx/algorithms/approximation/independent_set.pyi b/typeshed/networkx/algorithms/approximation/independent_set.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/approximation/independent_set.pyi rename to typeshed/networkx/algorithms/approximation/independent_set.pyi diff --git a/typeshed/2and3/networkx/algorithms/approximation/kcomponents.pyi b/typeshed/networkx/algorithms/approximation/kcomponents.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/approximation/kcomponents.pyi rename to typeshed/networkx/algorithms/approximation/kcomponents.pyi diff --git a/typeshed/2and3/networkx/algorithms/approximation/matching.pyi b/typeshed/networkx/algorithms/approximation/matching.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/approximation/matching.pyi rename to typeshed/networkx/algorithms/approximation/matching.pyi diff --git a/typeshed/2and3/networkx/algorithms/approximation/ramsey.pyi b/typeshed/networkx/algorithms/approximation/ramsey.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/approximation/ramsey.pyi rename to typeshed/networkx/algorithms/approximation/ramsey.pyi diff --git a/typeshed/2and3/networkx/algorithms/approximation/steinertree.pyi b/typeshed/networkx/algorithms/approximation/steinertree.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/approximation/steinertree.pyi rename to typeshed/networkx/algorithms/approximation/steinertree.pyi diff --git a/typeshed/2and3/networkx/algorithms/approximation/vertex_cover.pyi b/typeshed/networkx/algorithms/approximation/vertex_cover.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/approximation/vertex_cover.pyi rename to typeshed/networkx/algorithms/approximation/vertex_cover.pyi diff --git a/typeshed/2and3/networkx/algorithms/assortativity/__init__.pyi b/typeshed/networkx/algorithms/assortativity/__init__.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/assortativity/__init__.pyi rename to typeshed/networkx/algorithms/assortativity/__init__.pyi diff --git a/typeshed/2and3/networkx/algorithms/assortativity/connectivity.pyi b/typeshed/networkx/algorithms/assortativity/connectivity.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/assortativity/connectivity.pyi rename to typeshed/networkx/algorithms/assortativity/connectivity.pyi diff --git a/typeshed/2and3/networkx/algorithms/assortativity/correlation.pyi b/typeshed/networkx/algorithms/assortativity/correlation.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/assortativity/correlation.pyi rename to typeshed/networkx/algorithms/assortativity/correlation.pyi diff --git a/typeshed/2and3/networkx/algorithms/assortativity/mixing.pyi b/typeshed/networkx/algorithms/assortativity/mixing.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/assortativity/mixing.pyi rename to typeshed/networkx/algorithms/assortativity/mixing.pyi diff --git a/typeshed/2and3/networkx/algorithms/assortativity/neighbor_degree.pyi b/typeshed/networkx/algorithms/assortativity/neighbor_degree.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/assortativity/neighbor_degree.pyi rename to typeshed/networkx/algorithms/assortativity/neighbor_degree.pyi diff --git a/typeshed/2and3/networkx/algorithms/assortativity/pairs.pyi b/typeshed/networkx/algorithms/assortativity/pairs.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/assortativity/pairs.pyi rename to typeshed/networkx/algorithms/assortativity/pairs.pyi diff --git a/typeshed/2and3/networkx/algorithms/bipartite/__init__.pyi b/typeshed/networkx/algorithms/bipartite/__init__.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/bipartite/__init__.pyi rename to typeshed/networkx/algorithms/bipartite/__init__.pyi diff --git a/typeshed/2and3/networkx/algorithms/bipartite/basic.pyi b/typeshed/networkx/algorithms/bipartite/basic.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/bipartite/basic.pyi rename to typeshed/networkx/algorithms/bipartite/basic.pyi diff --git a/typeshed/2and3/networkx/algorithms/bipartite/centrality.pyi b/typeshed/networkx/algorithms/bipartite/centrality.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/bipartite/centrality.pyi rename to typeshed/networkx/algorithms/bipartite/centrality.pyi diff --git a/typeshed/2and3/networkx/algorithms/bipartite/cluster.pyi b/typeshed/networkx/algorithms/bipartite/cluster.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/bipartite/cluster.pyi rename to typeshed/networkx/algorithms/bipartite/cluster.pyi diff --git a/typeshed/2and3/networkx/algorithms/bipartite/covering.pyi b/typeshed/networkx/algorithms/bipartite/covering.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/bipartite/covering.pyi rename to typeshed/networkx/algorithms/bipartite/covering.pyi diff --git a/typeshed/2and3/networkx/algorithms/bipartite/edgelist.pyi b/typeshed/networkx/algorithms/bipartite/edgelist.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/bipartite/edgelist.pyi rename to typeshed/networkx/algorithms/bipartite/edgelist.pyi diff --git a/typeshed/2and3/networkx/algorithms/bipartite/generators.pyi b/typeshed/networkx/algorithms/bipartite/generators.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/bipartite/generators.pyi rename to typeshed/networkx/algorithms/bipartite/generators.pyi diff --git a/typeshed/2and3/networkx/algorithms/bipartite/matching.pyi b/typeshed/networkx/algorithms/bipartite/matching.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/bipartite/matching.pyi rename to typeshed/networkx/algorithms/bipartite/matching.pyi diff --git a/typeshed/2and3/networkx/algorithms/bipartite/matrix.pyi b/typeshed/networkx/algorithms/bipartite/matrix.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/bipartite/matrix.pyi rename to typeshed/networkx/algorithms/bipartite/matrix.pyi diff --git a/typeshed/2and3/networkx/algorithms/bipartite/projection.pyi b/typeshed/networkx/algorithms/bipartite/projection.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/bipartite/projection.pyi rename to typeshed/networkx/algorithms/bipartite/projection.pyi diff --git a/typeshed/2and3/networkx/algorithms/bipartite/redundancy.pyi b/typeshed/networkx/algorithms/bipartite/redundancy.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/bipartite/redundancy.pyi rename to typeshed/networkx/algorithms/bipartite/redundancy.pyi diff --git a/typeshed/2and3/networkx/algorithms/bipartite/spectral.pyi b/typeshed/networkx/algorithms/bipartite/spectral.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/bipartite/spectral.pyi rename to typeshed/networkx/algorithms/bipartite/spectral.pyi diff --git a/typeshed/2and3/networkx/algorithms/boundary.pyi b/typeshed/networkx/algorithms/boundary.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/boundary.pyi rename to typeshed/networkx/algorithms/boundary.pyi diff --git a/typeshed/2and3/networkx/algorithms/bridges.pyi b/typeshed/networkx/algorithms/bridges.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/bridges.pyi rename to typeshed/networkx/algorithms/bridges.pyi diff --git a/typeshed/2and3/networkx/algorithms/centrality/betweenness.pyi b/typeshed/networkx/algorithms/centrality/betweenness.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/centrality/betweenness.pyi rename to typeshed/networkx/algorithms/centrality/betweenness.pyi diff --git a/typeshed/2and3/networkx/algorithms/centrality/betweenness_subset.pyi b/typeshed/networkx/algorithms/centrality/betweenness_subset.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/centrality/betweenness_subset.pyi rename to typeshed/networkx/algorithms/centrality/betweenness_subset.pyi diff --git a/typeshed/2and3/networkx/algorithms/centrality/closeness.pyi b/typeshed/networkx/algorithms/centrality/closeness.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/centrality/closeness.pyi rename to typeshed/networkx/algorithms/centrality/closeness.pyi diff --git a/typeshed/2and3/networkx/algorithms/centrality/current_flow_betweenness.pyi b/typeshed/networkx/algorithms/centrality/current_flow_betweenness.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/centrality/current_flow_betweenness.pyi rename to typeshed/networkx/algorithms/centrality/current_flow_betweenness.pyi diff --git a/typeshed/2and3/networkx/algorithms/centrality/current_flow_betweenness_subset.pyi b/typeshed/networkx/algorithms/centrality/current_flow_betweenness_subset.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/centrality/current_flow_betweenness_subset.pyi rename to typeshed/networkx/algorithms/centrality/current_flow_betweenness_subset.pyi diff --git a/typeshed/2and3/networkx/algorithms/centrality/current_flow_closeness.pyi b/typeshed/networkx/algorithms/centrality/current_flow_closeness.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/centrality/current_flow_closeness.pyi rename to typeshed/networkx/algorithms/centrality/current_flow_closeness.pyi diff --git a/typeshed/2and3/networkx/algorithms/centrality/degree_alg.pyi b/typeshed/networkx/algorithms/centrality/degree_alg.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/centrality/degree_alg.pyi rename to typeshed/networkx/algorithms/centrality/degree_alg.pyi diff --git a/typeshed/2and3/networkx/algorithms/centrality/dispersion.pyi b/typeshed/networkx/algorithms/centrality/dispersion.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/centrality/dispersion.pyi rename to typeshed/networkx/algorithms/centrality/dispersion.pyi diff --git a/typeshed/2and3/networkx/algorithms/centrality/eigenvector.pyi b/typeshed/networkx/algorithms/centrality/eigenvector.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/centrality/eigenvector.pyi rename to typeshed/networkx/algorithms/centrality/eigenvector.pyi diff --git a/typeshed/2and3/networkx/algorithms/centrality/flow_matrix.pyi b/typeshed/networkx/algorithms/centrality/flow_matrix.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/centrality/flow_matrix.pyi rename to typeshed/networkx/algorithms/centrality/flow_matrix.pyi diff --git a/typeshed/2and3/networkx/algorithms/centrality/harmonic.pyi b/typeshed/networkx/algorithms/centrality/harmonic.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/centrality/harmonic.pyi rename to typeshed/networkx/algorithms/centrality/harmonic.pyi diff --git a/typeshed/2and3/networkx/algorithms/centrality/katz.pyi b/typeshed/networkx/algorithms/centrality/katz.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/centrality/katz.pyi rename to typeshed/networkx/algorithms/centrality/katz.pyi diff --git a/typeshed/2and3/networkx/algorithms/centrality/load.pyi b/typeshed/networkx/algorithms/centrality/load.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/centrality/load.pyi rename to typeshed/networkx/algorithms/centrality/load.pyi diff --git a/typeshed/2and3/networkx/algorithms/centrality/reaching.pyi b/typeshed/networkx/algorithms/centrality/reaching.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/centrality/reaching.pyi rename to typeshed/networkx/algorithms/centrality/reaching.pyi diff --git a/typeshed/2and3/networkx/algorithms/centrality/subgraph_alg.pyi b/typeshed/networkx/algorithms/centrality/subgraph_alg.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/centrality/subgraph_alg.pyi rename to typeshed/networkx/algorithms/centrality/subgraph_alg.pyi diff --git a/typeshed/2and3/networkx/algorithms/chains.pyi b/typeshed/networkx/algorithms/chains.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/chains.pyi rename to typeshed/networkx/algorithms/chains.pyi diff --git a/typeshed/2and3/networkx/algorithms/chordal.pyi b/typeshed/networkx/algorithms/chordal.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/chordal.pyi rename to typeshed/networkx/algorithms/chordal.pyi diff --git a/typeshed/2and3/networkx/algorithms/clique.pyi b/typeshed/networkx/algorithms/clique.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/clique.pyi rename to typeshed/networkx/algorithms/clique.pyi diff --git a/typeshed/2and3/networkx/algorithms/cluster.pyi b/typeshed/networkx/algorithms/cluster.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/cluster.pyi rename to typeshed/networkx/algorithms/cluster.pyi diff --git a/typeshed/2and3/networkx/algorithms/coloring/__init__.pyi b/typeshed/networkx/algorithms/coloring/__init__.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/coloring/__init__.pyi rename to typeshed/networkx/algorithms/coloring/__init__.pyi diff --git a/typeshed/2and3/networkx/algorithms/coloring/greedy_coloring.pyi b/typeshed/networkx/algorithms/coloring/greedy_coloring.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/coloring/greedy_coloring.pyi rename to typeshed/networkx/algorithms/coloring/greedy_coloring.pyi diff --git a/typeshed/2and3/networkx/algorithms/coloring/greedy_coloring_with_interchange.pyi b/typeshed/networkx/algorithms/coloring/greedy_coloring_with_interchange.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/coloring/greedy_coloring_with_interchange.pyi rename to typeshed/networkx/algorithms/coloring/greedy_coloring_with_interchange.pyi diff --git a/typeshed/2and3/networkx/algorithms/communicability_alg.pyi b/typeshed/networkx/algorithms/communicability_alg.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/communicability_alg.pyi rename to typeshed/networkx/algorithms/communicability_alg.pyi diff --git a/typeshed/2and3/networkx/algorithms/community/__init__.pyi b/typeshed/networkx/algorithms/community/__init__.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/community/__init__.pyi rename to typeshed/networkx/algorithms/community/__init__.pyi diff --git a/typeshed/2and3/networkx/algorithms/community/asyn_fluidc.pyi b/typeshed/networkx/algorithms/community/asyn_fluidc.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/community/asyn_fluidc.pyi rename to typeshed/networkx/algorithms/community/asyn_fluidc.pyi diff --git a/typeshed/2and3/networkx/algorithms/community/centrality.pyi b/typeshed/networkx/algorithms/community/centrality.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/community/centrality.pyi rename to typeshed/networkx/algorithms/community/centrality.pyi diff --git a/typeshed/2and3/networkx/algorithms/community/community_generators.pyi b/typeshed/networkx/algorithms/community/community_generators.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/community/community_generators.pyi rename to typeshed/networkx/algorithms/community/community_generators.pyi diff --git a/typeshed/2and3/networkx/algorithms/community/community_utils.pyi b/typeshed/networkx/algorithms/community/community_utils.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/community/community_utils.pyi rename to typeshed/networkx/algorithms/community/community_utils.pyi diff --git a/typeshed/2and3/networkx/algorithms/community/kclique.pyi b/typeshed/networkx/algorithms/community/kclique.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/community/kclique.pyi rename to typeshed/networkx/algorithms/community/kclique.pyi diff --git a/typeshed/2and3/networkx/algorithms/community/kernighan_lin.pyi b/typeshed/networkx/algorithms/community/kernighan_lin.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/community/kernighan_lin.pyi rename to typeshed/networkx/algorithms/community/kernighan_lin.pyi diff --git a/typeshed/2and3/networkx/algorithms/community/label_propagation.pyi b/typeshed/networkx/algorithms/community/label_propagation.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/community/label_propagation.pyi rename to typeshed/networkx/algorithms/community/label_propagation.pyi diff --git a/typeshed/2and3/networkx/algorithms/community/quality.pyi b/typeshed/networkx/algorithms/community/quality.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/community/quality.pyi rename to typeshed/networkx/algorithms/community/quality.pyi diff --git a/typeshed/2and3/networkx/algorithms/components/__init__.pyi b/typeshed/networkx/algorithms/components/__init__.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/components/__init__.pyi rename to typeshed/networkx/algorithms/components/__init__.pyi diff --git a/typeshed/2and3/networkx/algorithms/components/attracting.pyi b/typeshed/networkx/algorithms/components/attracting.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/components/attracting.pyi rename to typeshed/networkx/algorithms/components/attracting.pyi diff --git a/typeshed/2and3/networkx/algorithms/components/biconnected.pyi b/typeshed/networkx/algorithms/components/biconnected.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/components/biconnected.pyi rename to typeshed/networkx/algorithms/components/biconnected.pyi diff --git a/typeshed/2and3/networkx/algorithms/components/connected.pyi b/typeshed/networkx/algorithms/components/connected.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/components/connected.pyi rename to typeshed/networkx/algorithms/components/connected.pyi diff --git a/typeshed/2and3/networkx/algorithms/components/semiconnected.pyi b/typeshed/networkx/algorithms/components/semiconnected.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/components/semiconnected.pyi rename to typeshed/networkx/algorithms/components/semiconnected.pyi diff --git a/typeshed/2and3/networkx/algorithms/components/strongly_connected.pyi b/typeshed/networkx/algorithms/components/strongly_connected.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/components/strongly_connected.pyi rename to typeshed/networkx/algorithms/components/strongly_connected.pyi diff --git a/typeshed/2and3/networkx/algorithms/components/weakly_connected.pyi b/typeshed/networkx/algorithms/components/weakly_connected.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/components/weakly_connected.pyi rename to typeshed/networkx/algorithms/components/weakly_connected.pyi diff --git a/typeshed/2and3/networkx/algorithms/connectivity/__init__.pyi b/typeshed/networkx/algorithms/connectivity/__init__.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/connectivity/__init__.pyi rename to typeshed/networkx/algorithms/connectivity/__init__.pyi diff --git a/typeshed/2and3/networkx/algorithms/connectivity/connectivity.pyi b/typeshed/networkx/algorithms/connectivity/connectivity.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/connectivity/connectivity.pyi rename to typeshed/networkx/algorithms/connectivity/connectivity.pyi diff --git a/typeshed/2and3/networkx/algorithms/connectivity/cuts.pyi b/typeshed/networkx/algorithms/connectivity/cuts.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/connectivity/cuts.pyi rename to typeshed/networkx/algorithms/connectivity/cuts.pyi diff --git a/typeshed/2and3/networkx/algorithms/connectivity/disjoint_paths.pyi b/typeshed/networkx/algorithms/connectivity/disjoint_paths.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/connectivity/disjoint_paths.pyi rename to typeshed/networkx/algorithms/connectivity/disjoint_paths.pyi diff --git a/typeshed/2and3/networkx/algorithms/connectivity/edge_augmentation.pyi b/typeshed/networkx/algorithms/connectivity/edge_augmentation.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/connectivity/edge_augmentation.pyi rename to typeshed/networkx/algorithms/connectivity/edge_augmentation.pyi diff --git a/typeshed/2and3/networkx/algorithms/connectivity/edge_kcomponents.pyi b/typeshed/networkx/algorithms/connectivity/edge_kcomponents.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/connectivity/edge_kcomponents.pyi rename to typeshed/networkx/algorithms/connectivity/edge_kcomponents.pyi diff --git a/typeshed/2and3/networkx/algorithms/connectivity/kcomponents.pyi b/typeshed/networkx/algorithms/connectivity/kcomponents.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/connectivity/kcomponents.pyi rename to typeshed/networkx/algorithms/connectivity/kcomponents.pyi diff --git a/typeshed/2and3/networkx/algorithms/connectivity/kcutsets.pyi b/typeshed/networkx/algorithms/connectivity/kcutsets.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/connectivity/kcutsets.pyi rename to typeshed/networkx/algorithms/connectivity/kcutsets.pyi diff --git a/typeshed/2and3/networkx/algorithms/connectivity/stoerwagner.pyi b/typeshed/networkx/algorithms/connectivity/stoerwagner.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/connectivity/stoerwagner.pyi rename to typeshed/networkx/algorithms/connectivity/stoerwagner.pyi diff --git a/typeshed/2and3/networkx/algorithms/connectivity/utils.pyi b/typeshed/networkx/algorithms/connectivity/utils.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/connectivity/utils.pyi rename to typeshed/networkx/algorithms/connectivity/utils.pyi diff --git a/typeshed/2and3/networkx/algorithms/core.pyi b/typeshed/networkx/algorithms/core.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/core.pyi rename to typeshed/networkx/algorithms/core.pyi diff --git a/typeshed/2and3/networkx/algorithms/covering.pyi b/typeshed/networkx/algorithms/covering.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/covering.pyi rename to typeshed/networkx/algorithms/covering.pyi diff --git a/typeshed/2and3/networkx/algorithms/cuts.pyi b/typeshed/networkx/algorithms/cuts.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/cuts.pyi rename to typeshed/networkx/algorithms/cuts.pyi diff --git a/typeshed/2and3/networkx/algorithms/cycles.pyi b/typeshed/networkx/algorithms/cycles.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/cycles.pyi rename to typeshed/networkx/algorithms/cycles.pyi diff --git a/typeshed/2and3/networkx/algorithms/dag.pyi b/typeshed/networkx/algorithms/dag.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/dag.pyi rename to typeshed/networkx/algorithms/dag.pyi diff --git a/typeshed/2and3/networkx/algorithms/distance_measures.pyi b/typeshed/networkx/algorithms/distance_measures.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/distance_measures.pyi rename to typeshed/networkx/algorithms/distance_measures.pyi diff --git a/typeshed/2and3/networkx/algorithms/distance_regular.pyi b/typeshed/networkx/algorithms/distance_regular.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/distance_regular.pyi rename to typeshed/networkx/algorithms/distance_regular.pyi diff --git a/typeshed/2and3/networkx/algorithms/dominance.pyi b/typeshed/networkx/algorithms/dominance.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/dominance.pyi rename to typeshed/networkx/algorithms/dominance.pyi diff --git a/typeshed/2and3/networkx/algorithms/dominating.pyi b/typeshed/networkx/algorithms/dominating.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/dominating.pyi rename to typeshed/networkx/algorithms/dominating.pyi diff --git a/typeshed/2and3/networkx/algorithms/efficiency.pyi b/typeshed/networkx/algorithms/efficiency.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/efficiency.pyi rename to typeshed/networkx/algorithms/efficiency.pyi diff --git a/typeshed/2and3/networkx/algorithms/euler.pyi b/typeshed/networkx/algorithms/euler.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/euler.pyi rename to typeshed/networkx/algorithms/euler.pyi diff --git a/typeshed/2and3/networkx/algorithms/flow/__init__.pyi b/typeshed/networkx/algorithms/flow/__init__.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/flow/__init__.pyi rename to typeshed/networkx/algorithms/flow/__init__.pyi diff --git a/typeshed/2and3/networkx/algorithms/flow/boykovkolmogorov.pyi b/typeshed/networkx/algorithms/flow/boykovkolmogorov.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/flow/boykovkolmogorov.pyi rename to typeshed/networkx/algorithms/flow/boykovkolmogorov.pyi diff --git a/typeshed/2and3/networkx/algorithms/flow/capacityscaling.pyi b/typeshed/networkx/algorithms/flow/capacityscaling.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/flow/capacityscaling.pyi rename to typeshed/networkx/algorithms/flow/capacityscaling.pyi diff --git a/typeshed/2and3/networkx/algorithms/flow/dinitz_alg.pyi b/typeshed/networkx/algorithms/flow/dinitz_alg.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/flow/dinitz_alg.pyi rename to typeshed/networkx/algorithms/flow/dinitz_alg.pyi diff --git a/typeshed/2and3/networkx/algorithms/flow/edmondskarp.pyi b/typeshed/networkx/algorithms/flow/edmondskarp.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/flow/edmondskarp.pyi rename to typeshed/networkx/algorithms/flow/edmondskarp.pyi diff --git a/typeshed/2and3/networkx/algorithms/flow/gomory_hu.pyi b/typeshed/networkx/algorithms/flow/gomory_hu.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/flow/gomory_hu.pyi rename to typeshed/networkx/algorithms/flow/gomory_hu.pyi diff --git a/typeshed/2and3/networkx/algorithms/flow/maxflow.pyi b/typeshed/networkx/algorithms/flow/maxflow.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/flow/maxflow.pyi rename to typeshed/networkx/algorithms/flow/maxflow.pyi diff --git a/typeshed/2and3/networkx/algorithms/flow/mincost.pyi b/typeshed/networkx/algorithms/flow/mincost.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/flow/mincost.pyi rename to typeshed/networkx/algorithms/flow/mincost.pyi diff --git a/typeshed/2and3/networkx/algorithms/flow/networksimplex.pyi b/typeshed/networkx/algorithms/flow/networksimplex.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/flow/networksimplex.pyi rename to typeshed/networkx/algorithms/flow/networksimplex.pyi diff --git a/typeshed/2and3/networkx/algorithms/flow/preflowpush.pyi b/typeshed/networkx/algorithms/flow/preflowpush.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/flow/preflowpush.pyi rename to typeshed/networkx/algorithms/flow/preflowpush.pyi diff --git a/typeshed/2and3/networkx/algorithms/flow/shortestaugmentingpath.pyi b/typeshed/networkx/algorithms/flow/shortestaugmentingpath.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/flow/shortestaugmentingpath.pyi rename to typeshed/networkx/algorithms/flow/shortestaugmentingpath.pyi diff --git a/typeshed/2and3/networkx/algorithms/flow/utils.pyi b/typeshed/networkx/algorithms/flow/utils.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/flow/utils.pyi rename to typeshed/networkx/algorithms/flow/utils.pyi diff --git a/typeshed/2and3/networkx/algorithms/graphical.pyi b/typeshed/networkx/algorithms/graphical.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/graphical.pyi rename to typeshed/networkx/algorithms/graphical.pyi diff --git a/typeshed/2and3/networkx/algorithms/hierarchy.pyi b/typeshed/networkx/algorithms/hierarchy.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/hierarchy.pyi rename to typeshed/networkx/algorithms/hierarchy.pyi diff --git a/typeshed/2and3/networkx/algorithms/hybrid.pyi b/typeshed/networkx/algorithms/hybrid.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/hybrid.pyi rename to typeshed/networkx/algorithms/hybrid.pyi diff --git a/typeshed/2and3/networkx/algorithms/isolate.pyi b/typeshed/networkx/algorithms/isolate.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/isolate.pyi rename to typeshed/networkx/algorithms/isolate.pyi diff --git a/typeshed/2and3/networkx/algorithms/isomorphism/__init__.pyi b/typeshed/networkx/algorithms/isomorphism/__init__.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/isomorphism/__init__.pyi rename to typeshed/networkx/algorithms/isomorphism/__init__.pyi diff --git a/typeshed/2and3/networkx/algorithms/isomorphism/isomorph.pyi b/typeshed/networkx/algorithms/isomorphism/isomorph.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/isomorphism/isomorph.pyi rename to typeshed/networkx/algorithms/isomorphism/isomorph.pyi diff --git a/typeshed/2and3/networkx/algorithms/isomorphism/isomorphvf2.pyi b/typeshed/networkx/algorithms/isomorphism/isomorphvf2.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/isomorphism/isomorphvf2.pyi rename to typeshed/networkx/algorithms/isomorphism/isomorphvf2.pyi diff --git a/typeshed/2and3/networkx/algorithms/isomorphism/matchhelpers.pyi b/typeshed/networkx/algorithms/isomorphism/matchhelpers.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/isomorphism/matchhelpers.pyi rename to typeshed/networkx/algorithms/isomorphism/matchhelpers.pyi diff --git a/typeshed/2and3/networkx/algorithms/isomorphism/temporalisomorphvf2.pyi b/typeshed/networkx/algorithms/isomorphism/temporalisomorphvf2.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/isomorphism/temporalisomorphvf2.pyi rename to typeshed/networkx/algorithms/isomorphism/temporalisomorphvf2.pyi diff --git a/typeshed/2and3/networkx/algorithms/isomorphism/vf2userfunc.pyi b/typeshed/networkx/algorithms/isomorphism/vf2userfunc.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/isomorphism/vf2userfunc.pyi rename to typeshed/networkx/algorithms/isomorphism/vf2userfunc.pyi diff --git a/typeshed/2and3/networkx/algorithms/link_analysis/__init__.pyi b/typeshed/networkx/algorithms/link_analysis/__init__.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/link_analysis/__init__.pyi rename to typeshed/networkx/algorithms/link_analysis/__init__.pyi diff --git a/typeshed/2and3/networkx/algorithms/link_analysis/hits_alg.pyi b/typeshed/networkx/algorithms/link_analysis/hits_alg.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/link_analysis/hits_alg.pyi rename to typeshed/networkx/algorithms/link_analysis/hits_alg.pyi diff --git a/typeshed/2and3/networkx/algorithms/link_analysis/pagerank_alg.pyi b/typeshed/networkx/algorithms/link_analysis/pagerank_alg.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/link_analysis/pagerank_alg.pyi rename to typeshed/networkx/algorithms/link_analysis/pagerank_alg.pyi diff --git a/typeshed/2and3/networkx/algorithms/link_prediction.pyi b/typeshed/networkx/algorithms/link_prediction.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/link_prediction.pyi rename to typeshed/networkx/algorithms/link_prediction.pyi diff --git a/typeshed/2and3/networkx/algorithms/lowest_common_ancestors.pyi b/typeshed/networkx/algorithms/lowest_common_ancestors.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/lowest_common_ancestors.pyi rename to typeshed/networkx/algorithms/lowest_common_ancestors.pyi diff --git a/typeshed/2and3/networkx/algorithms/matching.pyi b/typeshed/networkx/algorithms/matching.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/matching.pyi rename to typeshed/networkx/algorithms/matching.pyi diff --git a/typeshed/2and3/networkx/algorithms/minors.pyi b/typeshed/networkx/algorithms/minors.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/minors.pyi rename to typeshed/networkx/algorithms/minors.pyi diff --git a/typeshed/2and3/networkx/algorithms/mis.pyi b/typeshed/networkx/algorithms/mis.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/mis.pyi rename to typeshed/networkx/algorithms/mis.pyi diff --git a/typeshed/2and3/networkx/algorithms/operators/__init__.pyi b/typeshed/networkx/algorithms/operators/__init__.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/operators/__init__.pyi rename to typeshed/networkx/algorithms/operators/__init__.pyi diff --git a/typeshed/2and3/networkx/algorithms/operators/all.pyi b/typeshed/networkx/algorithms/operators/all.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/operators/all.pyi rename to typeshed/networkx/algorithms/operators/all.pyi diff --git a/typeshed/2and3/networkx/algorithms/operators/binary.pyi b/typeshed/networkx/algorithms/operators/binary.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/operators/binary.pyi rename to typeshed/networkx/algorithms/operators/binary.pyi diff --git a/typeshed/2and3/networkx/algorithms/operators/product.pyi b/typeshed/networkx/algorithms/operators/product.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/operators/product.pyi rename to typeshed/networkx/algorithms/operators/product.pyi diff --git a/typeshed/2and3/networkx/algorithms/operators/unary.pyi b/typeshed/networkx/algorithms/operators/unary.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/operators/unary.pyi rename to typeshed/networkx/algorithms/operators/unary.pyi diff --git a/typeshed/2and3/networkx/algorithms/reciprocity.pyi b/typeshed/networkx/algorithms/reciprocity.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/reciprocity.pyi rename to typeshed/networkx/algorithms/reciprocity.pyi diff --git a/typeshed/2and3/networkx/algorithms/richclub.pyi b/typeshed/networkx/algorithms/richclub.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/richclub.pyi rename to typeshed/networkx/algorithms/richclub.pyi diff --git a/typeshed/2and3/networkx/algorithms/shortest_paths/__init__.pyi b/typeshed/networkx/algorithms/shortest_paths/__init__.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/shortest_paths/__init__.pyi rename to typeshed/networkx/algorithms/shortest_paths/__init__.pyi diff --git a/typeshed/2and3/networkx/algorithms/shortest_paths/astar.pyi b/typeshed/networkx/algorithms/shortest_paths/astar.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/shortest_paths/astar.pyi rename to typeshed/networkx/algorithms/shortest_paths/astar.pyi diff --git a/typeshed/2and3/networkx/algorithms/shortest_paths/dense.pyi b/typeshed/networkx/algorithms/shortest_paths/dense.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/shortest_paths/dense.pyi rename to typeshed/networkx/algorithms/shortest_paths/dense.pyi diff --git a/typeshed/2and3/networkx/algorithms/shortest_paths/generic.pyi b/typeshed/networkx/algorithms/shortest_paths/generic.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/shortest_paths/generic.pyi rename to typeshed/networkx/algorithms/shortest_paths/generic.pyi diff --git a/typeshed/2and3/networkx/algorithms/shortest_paths/unweighted.pyi b/typeshed/networkx/algorithms/shortest_paths/unweighted.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/shortest_paths/unweighted.pyi rename to typeshed/networkx/algorithms/shortest_paths/unweighted.pyi diff --git a/typeshed/2and3/networkx/algorithms/shortest_paths/weighted.pyi b/typeshed/networkx/algorithms/shortest_paths/weighted.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/shortest_paths/weighted.pyi rename to typeshed/networkx/algorithms/shortest_paths/weighted.pyi diff --git a/typeshed/2and3/networkx/algorithms/similarity.pyi b/typeshed/networkx/algorithms/similarity.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/similarity.pyi rename to typeshed/networkx/algorithms/similarity.pyi diff --git a/typeshed/2and3/networkx/algorithms/simple_paths.pyi b/typeshed/networkx/algorithms/simple_paths.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/simple_paths.pyi rename to typeshed/networkx/algorithms/simple_paths.pyi diff --git a/typeshed/2and3/networkx/algorithms/smetric.pyi b/typeshed/networkx/algorithms/smetric.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/smetric.pyi rename to typeshed/networkx/algorithms/smetric.pyi diff --git a/typeshed/2and3/networkx/algorithms/structuralholes.pyi b/typeshed/networkx/algorithms/structuralholes.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/structuralholes.pyi rename to typeshed/networkx/algorithms/structuralholes.pyi diff --git a/typeshed/2and3/networkx/algorithms/swap.pyi b/typeshed/networkx/algorithms/swap.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/swap.pyi rename to typeshed/networkx/algorithms/swap.pyi diff --git a/typeshed/2and3/networkx/algorithms/threshold.pyi b/typeshed/networkx/algorithms/threshold.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/threshold.pyi rename to typeshed/networkx/algorithms/threshold.pyi diff --git a/typeshed/2and3/networkx/algorithms/tournament.pyi b/typeshed/networkx/algorithms/tournament.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/tournament.pyi rename to typeshed/networkx/algorithms/tournament.pyi diff --git a/typeshed/2and3/networkx/algorithms/traversal/__init__.pyi b/typeshed/networkx/algorithms/traversal/__init__.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/traversal/__init__.pyi rename to typeshed/networkx/algorithms/traversal/__init__.pyi diff --git a/typeshed/2and3/networkx/algorithms/traversal/beamsearch.pyi b/typeshed/networkx/algorithms/traversal/beamsearch.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/traversal/beamsearch.pyi rename to typeshed/networkx/algorithms/traversal/beamsearch.pyi diff --git a/typeshed/2and3/networkx/algorithms/traversal/breadth_first_search.pyi b/typeshed/networkx/algorithms/traversal/breadth_first_search.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/traversal/breadth_first_search.pyi rename to typeshed/networkx/algorithms/traversal/breadth_first_search.pyi diff --git a/typeshed/2and3/networkx/algorithms/traversal/depth_first_search.pyi b/typeshed/networkx/algorithms/traversal/depth_first_search.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/traversal/depth_first_search.pyi rename to typeshed/networkx/algorithms/traversal/depth_first_search.pyi diff --git a/typeshed/2and3/networkx/algorithms/traversal/edgedfs.pyi b/typeshed/networkx/algorithms/traversal/edgedfs.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/traversal/edgedfs.pyi rename to typeshed/networkx/algorithms/traversal/edgedfs.pyi diff --git a/typeshed/2and3/networkx/algorithms/tree/__init__.pyi b/typeshed/networkx/algorithms/tree/__init__.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/tree/__init__.pyi rename to typeshed/networkx/algorithms/tree/__init__.pyi diff --git a/typeshed/2and3/networkx/algorithms/tree/branchings.pyi b/typeshed/networkx/algorithms/tree/branchings.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/tree/branchings.pyi rename to typeshed/networkx/algorithms/tree/branchings.pyi diff --git a/typeshed/2and3/networkx/algorithms/tree/coding.pyi b/typeshed/networkx/algorithms/tree/coding.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/tree/coding.pyi rename to typeshed/networkx/algorithms/tree/coding.pyi diff --git a/typeshed/2and3/networkx/algorithms/tree/mst.pyi b/typeshed/networkx/algorithms/tree/mst.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/tree/mst.pyi rename to typeshed/networkx/algorithms/tree/mst.pyi diff --git a/typeshed/2and3/networkx/algorithms/tree/operations.pyi b/typeshed/networkx/algorithms/tree/operations.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/tree/operations.pyi rename to typeshed/networkx/algorithms/tree/operations.pyi diff --git a/typeshed/2and3/networkx/algorithms/tree/recognition.pyi b/typeshed/networkx/algorithms/tree/recognition.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/tree/recognition.pyi rename to typeshed/networkx/algorithms/tree/recognition.pyi diff --git a/typeshed/2and3/networkx/algorithms/triads.pyi b/typeshed/networkx/algorithms/triads.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/triads.pyi rename to typeshed/networkx/algorithms/triads.pyi diff --git a/typeshed/2and3/networkx/algorithms/vitality.pyi b/typeshed/networkx/algorithms/vitality.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/vitality.pyi rename to typeshed/networkx/algorithms/vitality.pyi diff --git a/typeshed/2and3/networkx/algorithms/voronoi.pyi b/typeshed/networkx/algorithms/voronoi.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/voronoi.pyi rename to typeshed/networkx/algorithms/voronoi.pyi diff --git a/typeshed/2and3/networkx/algorithms/wiener.pyi b/typeshed/networkx/algorithms/wiener.pyi similarity index 100% rename from typeshed/2and3/networkx/algorithms/wiener.pyi rename to typeshed/networkx/algorithms/wiener.pyi diff --git a/typeshed/2and3/networkx/classes/__init__.pyi b/typeshed/networkx/classes/__init__.pyi similarity index 100% rename from typeshed/2and3/networkx/classes/__init__.pyi rename to typeshed/networkx/classes/__init__.pyi diff --git a/typeshed/2and3/networkx/classes/coreviews.pyi b/typeshed/networkx/classes/coreviews.pyi similarity index 100% rename from typeshed/2and3/networkx/classes/coreviews.pyi rename to typeshed/networkx/classes/coreviews.pyi diff --git a/typeshed/2and3/networkx/classes/digraph.pyi b/typeshed/networkx/classes/digraph.pyi similarity index 100% rename from typeshed/2and3/networkx/classes/digraph.pyi rename to typeshed/networkx/classes/digraph.pyi diff --git a/typeshed/2and3/networkx/classes/filters.pyi b/typeshed/networkx/classes/filters.pyi similarity index 100% rename from typeshed/2and3/networkx/classes/filters.pyi rename to typeshed/networkx/classes/filters.pyi diff --git a/typeshed/2and3/networkx/classes/function.pyi b/typeshed/networkx/classes/function.pyi similarity index 100% rename from typeshed/2and3/networkx/classes/function.pyi rename to typeshed/networkx/classes/function.pyi diff --git a/typeshed/2and3/networkx/classes/graph.pyi b/typeshed/networkx/classes/graph.pyi similarity index 100% rename from typeshed/2and3/networkx/classes/graph.pyi rename to typeshed/networkx/classes/graph.pyi diff --git a/typeshed/2and3/networkx/classes/graphviews.pyi b/typeshed/networkx/classes/graphviews.pyi similarity index 100% rename from typeshed/2and3/networkx/classes/graphviews.pyi rename to typeshed/networkx/classes/graphviews.pyi diff --git a/typeshed/2and3/networkx/classes/multidigraph.pyi b/typeshed/networkx/classes/multidigraph.pyi similarity index 100% rename from typeshed/2and3/networkx/classes/multidigraph.pyi rename to typeshed/networkx/classes/multidigraph.pyi diff --git a/typeshed/2and3/networkx/classes/multigraph.pyi b/typeshed/networkx/classes/multigraph.pyi similarity index 100% rename from typeshed/2and3/networkx/classes/multigraph.pyi rename to typeshed/networkx/classes/multigraph.pyi diff --git a/typeshed/2and3/networkx/classes/ordered.pyi b/typeshed/networkx/classes/ordered.pyi similarity index 100% rename from typeshed/2and3/networkx/classes/ordered.pyi rename to typeshed/networkx/classes/ordered.pyi diff --git a/typeshed/2and3/networkx/classes/reportviews.pyi b/typeshed/networkx/classes/reportviews.pyi similarity index 100% rename from typeshed/2and3/networkx/classes/reportviews.pyi rename to typeshed/networkx/classes/reportviews.pyi diff --git a/typeshed/2and3/networkx/convert.pyi b/typeshed/networkx/convert.pyi similarity index 100% rename from typeshed/2and3/networkx/convert.pyi rename to typeshed/networkx/convert.pyi diff --git a/typeshed/2and3/networkx/convert_matrix.pyi b/typeshed/networkx/convert_matrix.pyi similarity index 100% rename from typeshed/2and3/networkx/convert_matrix.pyi rename to typeshed/networkx/convert_matrix.pyi diff --git a/typeshed/2and3/networkx/drawing/__init__.pyi b/typeshed/networkx/drawing/__init__.pyi similarity index 100% rename from typeshed/2and3/networkx/drawing/__init__.pyi rename to typeshed/networkx/drawing/__init__.pyi diff --git a/typeshed/2and3/networkx/drawing/layout.pyi b/typeshed/networkx/drawing/layout.pyi similarity index 100% rename from typeshed/2and3/networkx/drawing/layout.pyi rename to typeshed/networkx/drawing/layout.pyi diff --git a/typeshed/2and3/networkx/drawing/nx_agraph.pyi b/typeshed/networkx/drawing/nx_agraph.pyi similarity index 100% rename from typeshed/2and3/networkx/drawing/nx_agraph.pyi rename to typeshed/networkx/drawing/nx_agraph.pyi diff --git a/typeshed/2and3/networkx/drawing/nx_pydot.pyi b/typeshed/networkx/drawing/nx_pydot.pyi similarity index 100% rename from typeshed/2and3/networkx/drawing/nx_pydot.pyi rename to typeshed/networkx/drawing/nx_pydot.pyi diff --git a/typeshed/2and3/networkx/drawing/nx_pylab.pyi b/typeshed/networkx/drawing/nx_pylab.pyi similarity index 100% rename from typeshed/2and3/networkx/drawing/nx_pylab.pyi rename to typeshed/networkx/drawing/nx_pylab.pyi diff --git a/typeshed/2and3/networkx/exception.pyi b/typeshed/networkx/exception.pyi similarity index 100% rename from typeshed/2and3/networkx/exception.pyi rename to typeshed/networkx/exception.pyi diff --git a/typeshed/2and3/networkx/generators/atlas.pyi b/typeshed/networkx/generators/atlas.pyi similarity index 100% rename from typeshed/2and3/networkx/generators/atlas.pyi rename to typeshed/networkx/generators/atlas.pyi diff --git a/typeshed/2and3/networkx/generators/classic.pyi b/typeshed/networkx/generators/classic.pyi similarity index 100% rename from typeshed/2and3/networkx/generators/classic.pyi rename to typeshed/networkx/generators/classic.pyi diff --git a/typeshed/2and3/networkx/generators/community.pyi b/typeshed/networkx/generators/community.pyi similarity index 100% rename from typeshed/2and3/networkx/generators/community.pyi rename to typeshed/networkx/generators/community.pyi diff --git a/typeshed/2and3/networkx/generators/degree_seq.pyi b/typeshed/networkx/generators/degree_seq.pyi similarity index 100% rename from typeshed/2and3/networkx/generators/degree_seq.pyi rename to typeshed/networkx/generators/degree_seq.pyi diff --git a/typeshed/2and3/networkx/generators/directed.pyi b/typeshed/networkx/generators/directed.pyi similarity index 100% rename from typeshed/2and3/networkx/generators/directed.pyi rename to typeshed/networkx/generators/directed.pyi diff --git a/typeshed/2and3/networkx/generators/duplication.pyi b/typeshed/networkx/generators/duplication.pyi similarity index 100% rename from typeshed/2and3/networkx/generators/duplication.pyi rename to typeshed/networkx/generators/duplication.pyi diff --git a/typeshed/2and3/networkx/generators/ego.pyi b/typeshed/networkx/generators/ego.pyi similarity index 100% rename from typeshed/2and3/networkx/generators/ego.pyi rename to typeshed/networkx/generators/ego.pyi diff --git a/typeshed/2and3/networkx/generators/expanders.pyi b/typeshed/networkx/generators/expanders.pyi similarity index 100% rename from typeshed/2and3/networkx/generators/expanders.pyi rename to typeshed/networkx/generators/expanders.pyi diff --git a/typeshed/2and3/networkx/generators/geometric.pyi b/typeshed/networkx/generators/geometric.pyi similarity index 100% rename from typeshed/2and3/networkx/generators/geometric.pyi rename to typeshed/networkx/generators/geometric.pyi diff --git a/typeshed/2and3/networkx/generators/intersection.pyi b/typeshed/networkx/generators/intersection.pyi similarity index 100% rename from typeshed/2and3/networkx/generators/intersection.pyi rename to typeshed/networkx/generators/intersection.pyi diff --git a/typeshed/2and3/networkx/generators/joint_degree_seq.pyi b/typeshed/networkx/generators/joint_degree_seq.pyi similarity index 100% rename from typeshed/2and3/networkx/generators/joint_degree_seq.pyi rename to typeshed/networkx/generators/joint_degree_seq.pyi diff --git a/typeshed/2and3/networkx/generators/lattice.pyi b/typeshed/networkx/generators/lattice.pyi similarity index 100% rename from typeshed/2and3/networkx/generators/lattice.pyi rename to typeshed/networkx/generators/lattice.pyi diff --git a/typeshed/2and3/networkx/generators/line.pyi b/typeshed/networkx/generators/line.pyi similarity index 100% rename from typeshed/2and3/networkx/generators/line.pyi rename to typeshed/networkx/generators/line.pyi diff --git a/typeshed/2and3/networkx/generators/mycielski.pyi b/typeshed/networkx/generators/mycielski.pyi similarity index 100% rename from typeshed/2and3/networkx/generators/mycielski.pyi rename to typeshed/networkx/generators/mycielski.pyi diff --git a/typeshed/2and3/networkx/generators/nonisomorphic_trees.pyi b/typeshed/networkx/generators/nonisomorphic_trees.pyi similarity index 100% rename from typeshed/2and3/networkx/generators/nonisomorphic_trees.pyi rename to typeshed/networkx/generators/nonisomorphic_trees.pyi diff --git a/typeshed/2and3/networkx/generators/random_clustered.pyi b/typeshed/networkx/generators/random_clustered.pyi similarity index 100% rename from typeshed/2and3/networkx/generators/random_clustered.pyi rename to typeshed/networkx/generators/random_clustered.pyi diff --git a/typeshed/2and3/networkx/generators/random_graphs.pyi b/typeshed/networkx/generators/random_graphs.pyi similarity index 100% rename from typeshed/2and3/networkx/generators/random_graphs.pyi rename to typeshed/networkx/generators/random_graphs.pyi diff --git a/typeshed/2and3/networkx/generators/small.pyi b/typeshed/networkx/generators/small.pyi similarity index 100% rename from typeshed/2and3/networkx/generators/small.pyi rename to typeshed/networkx/generators/small.pyi diff --git a/typeshed/2and3/networkx/generators/social.pyi b/typeshed/networkx/generators/social.pyi similarity index 100% rename from typeshed/2and3/networkx/generators/social.pyi rename to typeshed/networkx/generators/social.pyi diff --git a/typeshed/2and3/networkx/generators/stochastic.pyi b/typeshed/networkx/generators/stochastic.pyi similarity index 100% rename from typeshed/2and3/networkx/generators/stochastic.pyi rename to typeshed/networkx/generators/stochastic.pyi diff --git a/typeshed/2and3/networkx/generators/trees.pyi b/typeshed/networkx/generators/trees.pyi similarity index 100% rename from typeshed/2and3/networkx/generators/trees.pyi rename to typeshed/networkx/generators/trees.pyi diff --git a/typeshed/2and3/networkx/generators/triads.pyi b/typeshed/networkx/generators/triads.pyi similarity index 100% rename from typeshed/2and3/networkx/generators/triads.pyi rename to typeshed/networkx/generators/triads.pyi diff --git a/typeshed/2and3/networkx/linalg/__init__.pyi b/typeshed/networkx/linalg/__init__.pyi similarity index 100% rename from typeshed/2and3/networkx/linalg/__init__.pyi rename to typeshed/networkx/linalg/__init__.pyi diff --git a/typeshed/2and3/networkx/linalg/algebraicconnectivity.pyi b/typeshed/networkx/linalg/algebraicconnectivity.pyi similarity index 100% rename from typeshed/2and3/networkx/linalg/algebraicconnectivity.pyi rename to typeshed/networkx/linalg/algebraicconnectivity.pyi diff --git a/typeshed/2and3/networkx/linalg/attrmatrix.pyi b/typeshed/networkx/linalg/attrmatrix.pyi similarity index 100% rename from typeshed/2and3/networkx/linalg/attrmatrix.pyi rename to typeshed/networkx/linalg/attrmatrix.pyi diff --git a/typeshed/2and3/networkx/linalg/graphmatrix.pyi b/typeshed/networkx/linalg/graphmatrix.pyi similarity index 100% rename from typeshed/2and3/networkx/linalg/graphmatrix.pyi rename to typeshed/networkx/linalg/graphmatrix.pyi diff --git a/typeshed/2and3/networkx/linalg/laplacianmatrix.pyi b/typeshed/networkx/linalg/laplacianmatrix.pyi similarity index 100% rename from typeshed/2and3/networkx/linalg/laplacianmatrix.pyi rename to typeshed/networkx/linalg/laplacianmatrix.pyi diff --git a/typeshed/2and3/networkx/linalg/modularitymatrix.pyi b/typeshed/networkx/linalg/modularitymatrix.pyi similarity index 100% rename from typeshed/2and3/networkx/linalg/modularitymatrix.pyi rename to typeshed/networkx/linalg/modularitymatrix.pyi diff --git a/typeshed/2and3/networkx/linalg/spectrum.pyi b/typeshed/networkx/linalg/spectrum.pyi similarity index 100% rename from typeshed/2and3/networkx/linalg/spectrum.pyi rename to typeshed/networkx/linalg/spectrum.pyi diff --git a/typeshed/2and3/networkx/readwrite/__init__.pyi b/typeshed/networkx/readwrite/__init__.pyi similarity index 100% rename from typeshed/2and3/networkx/readwrite/__init__.pyi rename to typeshed/networkx/readwrite/__init__.pyi diff --git a/typeshed/2and3/networkx/readwrite/adjlist.pyi b/typeshed/networkx/readwrite/adjlist.pyi similarity index 100% rename from typeshed/2and3/networkx/readwrite/adjlist.pyi rename to typeshed/networkx/readwrite/adjlist.pyi diff --git a/typeshed/2and3/networkx/readwrite/edgelist.pyi b/typeshed/networkx/readwrite/edgelist.pyi similarity index 100% rename from typeshed/2and3/networkx/readwrite/edgelist.pyi rename to typeshed/networkx/readwrite/edgelist.pyi diff --git a/typeshed/2and3/networkx/readwrite/gexf.pyi b/typeshed/networkx/readwrite/gexf.pyi similarity index 100% rename from typeshed/2and3/networkx/readwrite/gexf.pyi rename to typeshed/networkx/readwrite/gexf.pyi diff --git a/typeshed/2and3/networkx/readwrite/gml.pyi b/typeshed/networkx/readwrite/gml.pyi similarity index 100% rename from typeshed/2and3/networkx/readwrite/gml.pyi rename to typeshed/networkx/readwrite/gml.pyi diff --git a/typeshed/2and3/networkx/readwrite/gpickle.pyi b/typeshed/networkx/readwrite/gpickle.pyi similarity index 100% rename from typeshed/2and3/networkx/readwrite/gpickle.pyi rename to typeshed/networkx/readwrite/gpickle.pyi diff --git a/typeshed/2and3/networkx/readwrite/graph6.pyi b/typeshed/networkx/readwrite/graph6.pyi similarity index 100% rename from typeshed/2and3/networkx/readwrite/graph6.pyi rename to typeshed/networkx/readwrite/graph6.pyi diff --git a/typeshed/2and3/networkx/readwrite/graphml.pyi b/typeshed/networkx/readwrite/graphml.pyi similarity index 100% rename from typeshed/2and3/networkx/readwrite/graphml.pyi rename to typeshed/networkx/readwrite/graphml.pyi diff --git a/typeshed/2and3/networkx/readwrite/json_graph/__init__.pyi b/typeshed/networkx/readwrite/json_graph/__init__.pyi similarity index 100% rename from typeshed/2and3/networkx/readwrite/json_graph/__init__.pyi rename to typeshed/networkx/readwrite/json_graph/__init__.pyi diff --git a/typeshed/2and3/networkx/readwrite/json_graph/adjacency.pyi b/typeshed/networkx/readwrite/json_graph/adjacency.pyi similarity index 100% rename from typeshed/2and3/networkx/readwrite/json_graph/adjacency.pyi rename to typeshed/networkx/readwrite/json_graph/adjacency.pyi diff --git a/typeshed/2and3/networkx/readwrite/json_graph/cytoscape.pyi b/typeshed/networkx/readwrite/json_graph/cytoscape.pyi similarity index 100% rename from typeshed/2and3/networkx/readwrite/json_graph/cytoscape.pyi rename to typeshed/networkx/readwrite/json_graph/cytoscape.pyi diff --git a/typeshed/2and3/networkx/readwrite/json_graph/jit.pyi b/typeshed/networkx/readwrite/json_graph/jit.pyi similarity index 100% rename from typeshed/2and3/networkx/readwrite/json_graph/jit.pyi rename to typeshed/networkx/readwrite/json_graph/jit.pyi diff --git a/typeshed/2and3/networkx/readwrite/json_graph/node_link.pyi b/typeshed/networkx/readwrite/json_graph/node_link.pyi similarity index 100% rename from typeshed/2and3/networkx/readwrite/json_graph/node_link.pyi rename to typeshed/networkx/readwrite/json_graph/node_link.pyi diff --git a/typeshed/2and3/networkx/readwrite/json_graph/tree.pyi b/typeshed/networkx/readwrite/json_graph/tree.pyi similarity index 100% rename from typeshed/2and3/networkx/readwrite/json_graph/tree.pyi rename to typeshed/networkx/readwrite/json_graph/tree.pyi diff --git a/typeshed/2and3/networkx/readwrite/leda.pyi b/typeshed/networkx/readwrite/leda.pyi similarity index 100% rename from typeshed/2and3/networkx/readwrite/leda.pyi rename to typeshed/networkx/readwrite/leda.pyi diff --git a/typeshed/2and3/networkx/readwrite/multiline_adjlist.pyi b/typeshed/networkx/readwrite/multiline_adjlist.pyi similarity index 100% rename from typeshed/2and3/networkx/readwrite/multiline_adjlist.pyi rename to typeshed/networkx/readwrite/multiline_adjlist.pyi diff --git a/typeshed/2and3/networkx/readwrite/nx_shp.pyi b/typeshed/networkx/readwrite/nx_shp.pyi similarity index 100% rename from typeshed/2and3/networkx/readwrite/nx_shp.pyi rename to typeshed/networkx/readwrite/nx_shp.pyi diff --git a/typeshed/2and3/networkx/readwrite/nx_yaml.pyi b/typeshed/networkx/readwrite/nx_yaml.pyi similarity index 100% rename from typeshed/2and3/networkx/readwrite/nx_yaml.pyi rename to typeshed/networkx/readwrite/nx_yaml.pyi diff --git a/typeshed/2and3/networkx/readwrite/p2g.pyi b/typeshed/networkx/readwrite/p2g.pyi similarity index 100% rename from typeshed/2and3/networkx/readwrite/p2g.pyi rename to typeshed/networkx/readwrite/p2g.pyi diff --git a/typeshed/2and3/networkx/readwrite/pajek.pyi b/typeshed/networkx/readwrite/pajek.pyi similarity index 100% rename from typeshed/2and3/networkx/readwrite/pajek.pyi rename to typeshed/networkx/readwrite/pajek.pyi diff --git a/typeshed/2and3/networkx/readwrite/sparse6.pyi b/typeshed/networkx/readwrite/sparse6.pyi similarity index 100% rename from typeshed/2and3/networkx/readwrite/sparse6.pyi rename to typeshed/networkx/readwrite/sparse6.pyi diff --git a/typeshed/2and3/networkx/relabel.pyi b/typeshed/networkx/relabel.pyi similarity index 100% rename from typeshed/2and3/networkx/relabel.pyi rename to typeshed/networkx/relabel.pyi diff --git a/typeshed/2and3/networkx/release.pyi b/typeshed/networkx/release.pyi similarity index 100% rename from typeshed/2and3/networkx/release.pyi rename to typeshed/networkx/release.pyi diff --git a/typeshed/2and3/networkx/tests/__init__.pyi b/typeshed/networkx/tests/__init__.pyi similarity index 100% rename from typeshed/2and3/networkx/tests/__init__.pyi rename to typeshed/networkx/tests/__init__.pyi diff --git a/typeshed/2and3/networkx/tests/test.pyi b/typeshed/networkx/tests/test.pyi similarity index 100% rename from typeshed/2and3/networkx/tests/test.pyi rename to typeshed/networkx/tests/test.pyi diff --git a/typeshed/2and3/networkx/utils/__init__.pyi b/typeshed/networkx/utils/__init__.pyi similarity index 100% rename from typeshed/2and3/networkx/utils/__init__.pyi rename to typeshed/networkx/utils/__init__.pyi diff --git a/typeshed/2and3/networkx/utils/contextmanagers.pyi b/typeshed/networkx/utils/contextmanagers.pyi similarity index 100% rename from typeshed/2and3/networkx/utils/contextmanagers.pyi rename to typeshed/networkx/utils/contextmanagers.pyi diff --git a/typeshed/2and3/networkx/utils/decorators.pyi b/typeshed/networkx/utils/decorators.pyi similarity index 100% rename from typeshed/2and3/networkx/utils/decorators.pyi rename to typeshed/networkx/utils/decorators.pyi diff --git a/typeshed/2and3/networkx/utils/heaps.pyi b/typeshed/networkx/utils/heaps.pyi similarity index 100% rename from typeshed/2and3/networkx/utils/heaps.pyi rename to typeshed/networkx/utils/heaps.pyi diff --git a/typeshed/2and3/networkx/utils/misc.pyi b/typeshed/networkx/utils/misc.pyi similarity index 100% rename from typeshed/2and3/networkx/utils/misc.pyi rename to typeshed/networkx/utils/misc.pyi diff --git a/typeshed/2and3/networkx/utils/random_sequence.pyi b/typeshed/networkx/utils/random_sequence.pyi similarity index 100% rename from typeshed/2and3/networkx/utils/random_sequence.pyi rename to typeshed/networkx/utils/random_sequence.pyi diff --git a/typeshed/2and3/networkx/utils/rcm.pyi b/typeshed/networkx/utils/rcm.pyi similarity index 100% rename from typeshed/2and3/networkx/utils/rcm.pyi rename to typeshed/networkx/utils/rcm.pyi diff --git a/typeshed/2and3/networkx/utils/union_find.pyi b/typeshed/networkx/utils/union_find.pyi similarity index 100% rename from typeshed/2and3/networkx/utils/union_find.pyi rename to typeshed/networkx/utils/union_find.pyi diff --git a/typeshed/2and3/networkx/version.pyi b/typeshed/networkx/version.pyi similarity index 100% rename from typeshed/2and3/networkx/version.pyi rename to typeshed/networkx/version.pyi diff --git a/typeshed/2and3/prov/__init__.pyi b/typeshed/prov/__init__.pyi similarity index 100% rename from typeshed/2and3/prov/__init__.pyi rename to typeshed/prov/__init__.pyi diff --git a/typeshed/2and3/prov/constants.pyi b/typeshed/prov/constants.pyi similarity index 100% rename from typeshed/2and3/prov/constants.pyi rename to typeshed/prov/constants.pyi diff --git a/typeshed/2and3/prov/graph.pyi b/typeshed/prov/graph.pyi similarity index 100% rename from typeshed/2and3/prov/graph.pyi rename to typeshed/prov/graph.pyi diff --git a/typeshed/2and3/prov/identifier.pyi b/typeshed/prov/identifier.pyi similarity index 100% rename from typeshed/2and3/prov/identifier.pyi rename to typeshed/prov/identifier.pyi diff --git a/typeshed/2and3/prov/model.pyi b/typeshed/prov/model.pyi similarity index 100% rename from typeshed/2and3/prov/model.pyi rename to typeshed/prov/model.pyi diff --git a/typeshed/2and3/prov/serializers/__init__.pyi b/typeshed/prov/serializers/__init__.pyi similarity index 100% rename from typeshed/2and3/prov/serializers/__init__.pyi rename to typeshed/prov/serializers/__init__.pyi diff --git a/typeshed/2and3/prov/serializers/provjson.pyi b/typeshed/prov/serializers/provjson.pyi similarity index 100% rename from typeshed/2and3/prov/serializers/provjson.pyi rename to typeshed/prov/serializers/provjson.pyi diff --git a/typeshed/2and3/prov/serializers/provn.pyi b/typeshed/prov/serializers/provn.pyi similarity index 100% rename from typeshed/2and3/prov/serializers/provn.pyi rename to typeshed/prov/serializers/provn.pyi diff --git a/typeshed/2and3/prov/serializers/provrdf.pyi b/typeshed/prov/serializers/provrdf.pyi similarity index 100% rename from typeshed/2and3/prov/serializers/provrdf.pyi rename to typeshed/prov/serializers/provrdf.pyi diff --git a/typeshed/2and3/prov/serializers/provxml.pyi b/typeshed/prov/serializers/provxml.pyi similarity index 100% rename from typeshed/2and3/prov/serializers/provxml.pyi rename to typeshed/prov/serializers/provxml.pyi diff --git a/typeshed/2and3/psutil.pyi b/typeshed/psutil.pyi similarity index 100% rename from typeshed/2and3/psutil.pyi rename to typeshed/psutil.pyi diff --git a/typeshed/2and3/rdflib/__init__.pyi b/typeshed/rdflib/__init__.pyi similarity index 100% rename from typeshed/2and3/rdflib/__init__.pyi rename to typeshed/rdflib/__init__.pyi diff --git a/typeshed/2and3/rdflib/events.pyi b/typeshed/rdflib/events.pyi similarity index 100% rename from typeshed/2and3/rdflib/events.pyi rename to typeshed/rdflib/events.pyi diff --git a/typeshed/2and3/rdflib/exceptions.pyi b/typeshed/rdflib/exceptions.pyi similarity index 100% rename from typeshed/2and3/rdflib/exceptions.pyi rename to typeshed/rdflib/exceptions.pyi diff --git a/typeshed/2and3/rdflib/graph.pyi b/typeshed/rdflib/graph.pyi similarity index 100% rename from typeshed/2and3/rdflib/graph.pyi rename to typeshed/rdflib/graph.pyi diff --git a/typeshed/2and3/rdflib/namespace.pyi b/typeshed/rdflib/namespace.pyi similarity index 100% rename from typeshed/2and3/rdflib/namespace.pyi rename to typeshed/rdflib/namespace.pyi diff --git a/typeshed/2and3/rdflib/parser.pyi b/typeshed/rdflib/parser.pyi similarity index 100% rename from typeshed/2and3/rdflib/parser.pyi rename to typeshed/rdflib/parser.pyi diff --git a/typeshed/2and3/rdflib/plugin.pyi b/typeshed/rdflib/plugin.pyi similarity index 100% rename from typeshed/2and3/rdflib/plugin.pyi rename to typeshed/rdflib/plugin.pyi diff --git a/typeshed/2and3/rdflib/plugins/__init__.pyi b/typeshed/rdflib/plugins/__init__.pyi similarity index 100% rename from typeshed/2and3/rdflib/plugins/__init__.pyi rename to typeshed/rdflib/plugins/__init__.pyi diff --git a/typeshed/2and3/rdflib/plugins/parsers/__init__.pyi b/typeshed/rdflib/plugins/parsers/__init__.pyi similarity index 100% rename from typeshed/2and3/rdflib/plugins/parsers/__init__.pyi rename to typeshed/rdflib/plugins/parsers/__init__.pyi diff --git a/typeshed/2and3/rdflib/plugins/parsers/notation3.pyi b/typeshed/rdflib/plugins/parsers/notation3.pyi similarity index 100% rename from typeshed/2and3/rdflib/plugins/parsers/notation3.pyi rename to typeshed/rdflib/plugins/parsers/notation3.pyi diff --git a/typeshed/2and3/rdflib/query.pyi b/typeshed/rdflib/query.pyi similarity index 100% rename from typeshed/2and3/rdflib/query.pyi rename to typeshed/rdflib/query.pyi diff --git a/typeshed/2and3/rdflib/serializer.pyi b/typeshed/rdflib/serializer.pyi similarity index 100% rename from typeshed/2and3/rdflib/serializer.pyi rename to typeshed/rdflib/serializer.pyi diff --git a/typeshed/2and3/rdflib/store.pyi b/typeshed/rdflib/store.pyi similarity index 100% rename from typeshed/2and3/rdflib/store.pyi rename to typeshed/rdflib/store.pyi diff --git a/typeshed/2and3/rdflib/term.pyi b/typeshed/rdflib/term.pyi similarity index 100% rename from typeshed/2and3/rdflib/term.pyi rename to typeshed/rdflib/term.pyi diff --git a/typeshed/2and3/rdflib/util.pyi b/typeshed/rdflib/util.pyi similarity index 100% rename from typeshed/2and3/rdflib/util.pyi rename to typeshed/rdflib/util.pyi diff --git a/typeshed/2and3/ruamel/__init__.py b/typeshed/ruamel/__init__.py similarity index 100% rename from typeshed/2and3/ruamel/__init__.py rename to typeshed/ruamel/__init__.py diff --git a/typeshed/2and3/shellescape/__init__.pyi b/typeshed/shellescape/__init__.pyi similarity index 100% rename from typeshed/2and3/shellescape/__init__.pyi rename to typeshed/shellescape/__init__.pyi diff --git a/typeshed/2and3/shellescape/main.pyi b/typeshed/shellescape/main.pyi similarity index 100% rename from typeshed/2and3/shellescape/main.pyi rename to typeshed/shellescape/main.pyi diff --git a/typeshed/subprocess.pyi b/typeshed/subprocess.pyi new file mode 100644 index 0000000000..5893763180 --- /dev/null +++ b/typeshed/subprocess.pyi @@ -0,0 +1,1053 @@ +import sys +from _typeshed import AnyPath +from types import TracebackType +from typing import IO, Any, AnyStr, Callable, Generic, Mapping, Optional, Sequence, Tuple, Type, TypeVar, Union, overload +from typing_extensions import Literal + +if sys.version_info >= (3, 9): + from types import GenericAlias + +# We prefer to annotate inputs to methods (eg subprocess.check_call) with these +# union types. +# For outputs we use laborious literal based overloads to try to determine +# which specific return types to use, and prefer to fall back to Any when +# this does not work, so the caller does not have to use an assertion to confirm +# which type. +# +# For example: +# +# try: +# x = subprocess.check_output(["ls", "-l"]) +# reveal_type(x) # bytes, based on the overloads +# except TimeoutError as e: +# reveal_type(e.cmd) # Any, but morally is _CMD +_FILE = Union[None, int, IO[Any]] +_TXT = Union[bytes, str] +# Python 3.6 does't support _CMD being a single PathLike. +# See: https://bugs.python.org/issue31961 +_CMD = Union[_TXT, Sequence[AnyPath]] +_ENV = Union[Mapping[bytes, _TXT], Mapping[str, _TXT]] + +_S = TypeVar("_S") +_T = TypeVar("_T") + +class CompletedProcess(Generic[_T]): + # morally: _CMD + args: Any + returncode: Optional[int] # this optional is REQUIRED for mypyc + # These are really both Optional, but requiring checks would be tedious + # and writing all the overloads would be horrific. + stdout: _T + stderr: _T + def __init__(self, args: _CMD, returncode: int, stdout: Optional[_T] = ..., stderr: Optional[_T] = ...) -> None: ... + def check_returncode(self) -> None: ... + if sys.version_info >= (3, 9): + def __class_getitem__(cls, item: Any) -> GenericAlias: ... + +if sys.version_info >= (3, 7): + # Nearly the same args as for 3.6, except for capture_output and text + @overload + def run( + args: _CMD, + bufsize: int = ..., + executable: Optional[AnyPath] = ..., + stdin: _FILE = ..., + stdout: _FILE = ..., + stderr: _FILE = ..., + preexec_fn: Optional[Callable[[], Any]] = ..., + close_fds: bool = ..., + shell: bool = ..., + cwd: Optional[AnyPath] = ..., + env: Optional[_ENV] = ..., + universal_newlines: bool = ..., + startupinfo: Any = ..., + creationflags: int = ..., + restore_signals: bool = ..., + start_new_session: bool = ..., + pass_fds: Any = ..., + *, + capture_output: bool = ..., + check: bool = ..., + encoding: Optional[str] = ..., + errors: Optional[str] = ..., + input: Optional[str] = ..., + text: Literal[True], + timeout: Optional[float] = ..., + ) -> CompletedProcess[str]: ... + @overload + def run( + args: _CMD, + bufsize: int = ..., + executable: Optional[AnyPath] = ..., + stdin: _FILE = ..., + stdout: _FILE = ..., + stderr: _FILE = ..., + preexec_fn: Optional[Callable[[], Any]] = ..., + close_fds: bool = ..., + shell: bool = ..., + cwd: Optional[AnyPath] = ..., + env: Optional[_ENV] = ..., + universal_newlines: bool = ..., + startupinfo: Any = ..., + creationflags: int = ..., + restore_signals: bool = ..., + start_new_session: bool = ..., + pass_fds: Any = ..., + *, + capture_output: bool = ..., + check: bool = ..., + encoding: str, + errors: Optional[str] = ..., + input: Optional[str] = ..., + text: Optional[bool] = ..., + timeout: Optional[float] = ..., + ) -> CompletedProcess[str]: ... + @overload + def run( + args: _CMD, + bufsize: int = ..., + executable: Optional[AnyPath] = ..., + stdin: _FILE = ..., + stdout: _FILE = ..., + stderr: _FILE = ..., + preexec_fn: Optional[Callable[[], Any]] = ..., + close_fds: bool = ..., + shell: bool = ..., + cwd: Optional[AnyPath] = ..., + env: Optional[_ENV] = ..., + universal_newlines: bool = ..., + startupinfo: Any = ..., + creationflags: int = ..., + restore_signals: bool = ..., + start_new_session: bool = ..., + pass_fds: Any = ..., + *, + capture_output: bool = ..., + check: bool = ..., + encoding: Optional[str] = ..., + errors: str, + input: Optional[str] = ..., + text: Optional[bool] = ..., + timeout: Optional[float] = ..., + ) -> CompletedProcess[str]: ... + @overload + def run( + args: _CMD, + bufsize: int = ..., + executable: Optional[AnyPath] = ..., + stdin: _FILE = ..., + stdout: _FILE = ..., + stderr: _FILE = ..., + preexec_fn: Optional[Callable[[], Any]] = ..., + close_fds: bool = ..., + shell: bool = ..., + cwd: Optional[AnyPath] = ..., + env: Optional[_ENV] = ..., + *, + universal_newlines: Literal[True], + startupinfo: Any = ..., + creationflags: int = ..., + restore_signals: bool = ..., + start_new_session: bool = ..., + pass_fds: Any = ..., + # where the *real* keyword only args start + capture_output: bool = ..., + check: bool = ..., + encoding: Optional[str] = ..., + errors: Optional[str] = ..., + input: Optional[str] = ..., + text: Optional[bool] = ..., + timeout: Optional[float] = ..., + ) -> CompletedProcess[str]: ... + @overload + def run( + args: _CMD, + bufsize: int = ..., + executable: Optional[AnyPath] = ..., + stdin: _FILE = ..., + stdout: _FILE = ..., + stderr: _FILE = ..., + preexec_fn: Optional[Callable[[], Any]] = ..., + close_fds: bool = ..., + shell: bool = ..., + cwd: Optional[AnyPath] = ..., + env: Optional[_ENV] = ..., + universal_newlines: Literal[False] = ..., + startupinfo: Any = ..., + creationflags: int = ..., + restore_signals: bool = ..., + start_new_session: bool = ..., + pass_fds: Any = ..., + *, + capture_output: bool = ..., + check: bool = ..., + encoding: None = ..., + errors: None = ..., + input: Optional[bytes] = ..., + text: Literal[None, False] = ..., + timeout: Optional[float] = ..., + ) -> CompletedProcess[bytes]: ... + @overload + def run( + args: _CMD, + bufsize: int = ..., + executable: Optional[AnyPath] = ..., + stdin: _FILE = ..., + stdout: _FILE = ..., + stderr: _FILE = ..., + preexec_fn: Optional[Callable[[], Any]] = ..., + close_fds: bool = ..., + shell: bool = ..., + cwd: Optional[AnyPath] = ..., + env: Optional[_ENV] = ..., + universal_newlines: bool = ..., + startupinfo: Any = ..., + creationflags: int = ..., + restore_signals: bool = ..., + start_new_session: bool = ..., + pass_fds: Any = ..., + *, + capture_output: bool = ..., + check: bool = ..., + encoding: Optional[str] = ..., + errors: Optional[str] = ..., + input: Optional[_TXT] = ..., + text: Optional[bool] = ..., + timeout: Optional[float] = ..., + ) -> CompletedProcess[Any]: ... + +else: + # Nearly same args as Popen.__init__ except for timeout, input, and check + @overload + def run( + args: _CMD, + bufsize: int = ..., + executable: Optional[AnyPath] = ..., + stdin: _FILE = ..., + stdout: _FILE = ..., + stderr: _FILE = ..., + preexec_fn: Optional[Callable[[], Any]] = ..., + close_fds: bool = ..., + shell: bool = ..., + cwd: Optional[AnyPath] = ..., + env: Optional[_ENV] = ..., + universal_newlines: bool = ..., + startupinfo: Any = ..., + creationflags: int = ..., + restore_signals: bool = ..., + start_new_session: bool = ..., + pass_fds: Any = ..., + *, + check: bool = ..., + encoding: str, + errors: Optional[str] = ..., + input: Optional[str] = ..., + timeout: Optional[float] = ..., + ) -> CompletedProcess[str]: ... + @overload + def run( + args: _CMD, + bufsize: int = ..., + executable: Optional[AnyPath] = ..., + stdin: _FILE = ..., + stdout: _FILE = ..., + stderr: _FILE = ..., + preexec_fn: Optional[Callable[[], Any]] = ..., + close_fds: bool = ..., + shell: bool = ..., + cwd: Optional[AnyPath] = ..., + env: Optional[_ENV] = ..., + universal_newlines: bool = ..., + startupinfo: Any = ..., + creationflags: int = ..., + restore_signals: bool = ..., + start_new_session: bool = ..., + pass_fds: Any = ..., + *, + check: bool = ..., + encoding: Optional[str] = ..., + errors: str, + input: Optional[str] = ..., + timeout: Optional[float] = ..., + ) -> CompletedProcess[str]: ... + @overload + def run( + args: _CMD, + bufsize: int = ..., + executable: Optional[AnyPath] = ..., + stdin: _FILE = ..., + stdout: _FILE = ..., + stderr: _FILE = ..., + preexec_fn: Optional[Callable[[], Any]] = ..., + close_fds: bool = ..., + shell: bool = ..., + cwd: Optional[AnyPath] = ..., + env: Optional[_ENV] = ..., + *, + universal_newlines: Literal[True], + startupinfo: Any = ..., + creationflags: int = ..., + restore_signals: bool = ..., + start_new_session: bool = ..., + pass_fds: Any = ..., + # where the *real* keyword only args start + check: bool = ..., + encoding: Optional[str] = ..., + errors: Optional[str] = ..., + input: Optional[str] = ..., + timeout: Optional[float] = ..., + ) -> CompletedProcess[str]: ... + @overload + def run( + args: _CMD, + bufsize: int = ..., + executable: Optional[AnyPath] = ..., + stdin: _FILE = ..., + stdout: _FILE = ..., + stderr: _FILE = ..., + preexec_fn: Optional[Callable[[], Any]] = ..., + close_fds: bool = ..., + shell: bool = ..., + cwd: Optional[AnyPath] = ..., + env: Optional[_ENV] = ..., + universal_newlines: Literal[False] = ..., + startupinfo: Any = ..., + creationflags: int = ..., + restore_signals: bool = ..., + start_new_session: bool = ..., + pass_fds: Any = ..., + *, + check: bool = ..., + encoding: None = ..., + errors: None = ..., + input: Optional[bytes] = ..., + timeout: Optional[float] = ..., + ) -> CompletedProcess[bytes]: ... + @overload + def run( + args: _CMD, + bufsize: int = ..., + executable: Optional[AnyPath] = ..., + stdin: _FILE = ..., + stdout: _FILE = ..., + stderr: _FILE = ..., + preexec_fn: Optional[Callable[[], Any]] = ..., + close_fds: bool = ..., + shell: bool = ..., + cwd: Optional[AnyPath] = ..., + env: Optional[_ENV] = ..., + universal_newlines: bool = ..., + startupinfo: Any = ..., + creationflags: int = ..., + restore_signals: bool = ..., + start_new_session: bool = ..., + pass_fds: Any = ..., + *, + check: bool = ..., + encoding: Optional[str] = ..., + errors: Optional[str] = ..., + input: Optional[_TXT] = ..., + timeout: Optional[float] = ..., + ) -> CompletedProcess[Any]: ... + +# Same args as Popen.__init__ +def call( + args: _CMD, + bufsize: int = ..., + executable: Optional[AnyPath] = ..., + stdin: _FILE = ..., + stdout: _FILE = ..., + stderr: _FILE = ..., + preexec_fn: Optional[Callable[[], Any]] = ..., + close_fds: bool = ..., + shell: bool = ..., + cwd: Optional[AnyPath] = ..., + env: Optional[_ENV] = ..., + universal_newlines: bool = ..., + startupinfo: Any = ..., + creationflags: int = ..., + restore_signals: bool = ..., + start_new_session: bool = ..., + pass_fds: Any = ..., + *, + timeout: Optional[float] = ..., +) -> int: ... + +# Same args as Popen.__init__ +def check_call( + args: _CMD, + bufsize: int = ..., + executable: AnyPath = ..., + stdin: _FILE = ..., + stdout: _FILE = ..., + stderr: _FILE = ..., + preexec_fn: Optional[Callable[[], Any]] = ..., + close_fds: bool = ..., + shell: bool = ..., + cwd: Optional[AnyPath] = ..., + env: Optional[_ENV] = ..., + universal_newlines: bool = ..., + startupinfo: Any = ..., + creationflags: int = ..., + restore_signals: bool = ..., + start_new_session: bool = ..., + pass_fds: Any = ..., + timeout: Optional[float] = ..., +) -> int: ... + +if sys.version_info >= (3, 7): + # 3.7 added text + @overload + def check_output( + args: _CMD, + bufsize: int = ..., + executable: Optional[AnyPath] = ..., + stdin: _FILE = ..., + stderr: _FILE = ..., + preexec_fn: Optional[Callable[[], Any]] = ..., + close_fds: bool = ..., + shell: bool = ..., + cwd: Optional[AnyPath] = ..., + env: Optional[_ENV] = ..., + universal_newlines: bool = ..., + startupinfo: Any = ..., + creationflags: int = ..., + restore_signals: bool = ..., + start_new_session: bool = ..., + pass_fds: Any = ..., + *, + timeout: Optional[float] = ..., + input: _TXT = ..., + encoding: Optional[str] = ..., + errors: Optional[str] = ..., + text: Literal[True], + ) -> str: ... + @overload + def check_output( + args: _CMD, + bufsize: int = ..., + executable: Optional[AnyPath] = ..., + stdin: _FILE = ..., + stderr: _FILE = ..., + preexec_fn: Optional[Callable[[], Any]] = ..., + close_fds: bool = ..., + shell: bool = ..., + cwd: Optional[AnyPath] = ..., + env: Optional[_ENV] = ..., + universal_newlines: bool = ..., + startupinfo: Any = ..., + creationflags: int = ..., + restore_signals: bool = ..., + start_new_session: bool = ..., + pass_fds: Any = ..., + *, + timeout: Optional[float] = ..., + input: _TXT = ..., + encoding: str, + errors: Optional[str] = ..., + text: Optional[bool] = ..., + ) -> str: ... + @overload + def check_output( + args: _CMD, + bufsize: int = ..., + executable: Optional[AnyPath] = ..., + stdin: _FILE = ..., + stderr: _FILE = ..., + preexec_fn: Optional[Callable[[], Any]] = ..., + close_fds: bool = ..., + shell: bool = ..., + cwd: Optional[AnyPath] = ..., + env: Optional[_ENV] = ..., + universal_newlines: bool = ..., + startupinfo: Any = ..., + creationflags: int = ..., + restore_signals: bool = ..., + start_new_session: bool = ..., + pass_fds: Any = ..., + *, + timeout: Optional[float] = ..., + input: _TXT = ..., + encoding: Optional[str] = ..., + errors: str, + text: Optional[bool] = ..., + ) -> str: ... + @overload + def check_output( + args: _CMD, + bufsize: int = ..., + executable: Optional[AnyPath] = ..., + stdin: _FILE = ..., + stderr: _FILE = ..., + preexec_fn: Optional[Callable[[], Any]] = ..., + close_fds: bool = ..., + shell: bool = ..., + cwd: Optional[AnyPath] = ..., + env: Optional[_ENV] = ..., + *, + universal_newlines: Literal[True], + startupinfo: Any = ..., + creationflags: int = ..., + restore_signals: bool = ..., + start_new_session: bool = ..., + pass_fds: Any = ..., + # where the real keyword only ones start + timeout: Optional[float] = ..., + input: _TXT = ..., + encoding: Optional[str] = ..., + errors: Optional[str] = ..., + text: Optional[bool] = ..., + ) -> str: ... + @overload + def check_output( + args: _CMD, + bufsize: int = ..., + executable: Optional[AnyPath] = ..., + stdin: _FILE = ..., + stderr: _FILE = ..., + preexec_fn: Optional[Callable[[], Any]] = ..., + close_fds: bool = ..., + shell: bool = ..., + cwd: Optional[AnyPath] = ..., + env: Optional[_ENV] = ..., + universal_newlines: Literal[False] = ..., + startupinfo: Any = ..., + creationflags: int = ..., + restore_signals: bool = ..., + start_new_session: bool = ..., + pass_fds: Any = ..., + *, + timeout: Optional[float] = ..., + input: _TXT = ..., + encoding: None = ..., + errors: None = ..., + text: Literal[None, False] = ..., + ) -> bytes: ... + @overload + def check_output( + args: _CMD, + bufsize: int = ..., + executable: Optional[AnyPath] = ..., + stdin: _FILE = ..., + stderr: _FILE = ..., + preexec_fn: Optional[Callable[[], Any]] = ..., + close_fds: bool = ..., + shell: bool = ..., + cwd: Optional[AnyPath] = ..., + env: Optional[_ENV] = ..., + universal_newlines: bool = ..., + startupinfo: Any = ..., + creationflags: int = ..., + restore_signals: bool = ..., + start_new_session: bool = ..., + pass_fds: Any = ..., + *, + timeout: Optional[float] = ..., + input: _TXT = ..., + encoding: Optional[str] = ..., + errors: Optional[str] = ..., + text: Optional[bool] = ..., + ) -> Any: ... # morally: -> _TXT + +else: + @overload + def check_output( + args: _CMD, + bufsize: int = ..., + executable: Optional[AnyPath] = ..., + stdin: _FILE = ..., + stderr: _FILE = ..., + preexec_fn: Optional[Callable[[], Any]] = ..., + close_fds: bool = ..., + shell: bool = ..., + cwd: Optional[AnyPath] = ..., + env: Optional[_ENV] = ..., + universal_newlines: bool = ..., + startupinfo: Any = ..., + creationflags: int = ..., + restore_signals: bool = ..., + start_new_session: bool = ..., + pass_fds: Any = ..., + *, + timeout: Optional[float] = ..., + input: _TXT = ..., + encoding: str, + errors: Optional[str] = ..., + ) -> str: ... + @overload + def check_output( + args: _CMD, + bufsize: int = ..., + executable: Optional[AnyPath] = ..., + stdin: _FILE = ..., + stderr: _FILE = ..., + preexec_fn: Optional[Callable[[], Any]] = ..., + close_fds: bool = ..., + shell: bool = ..., + cwd: Optional[AnyPath] = ..., + env: Optional[_ENV] = ..., + universal_newlines: bool = ..., + startupinfo: Any = ..., + creationflags: int = ..., + restore_signals: bool = ..., + start_new_session: bool = ..., + pass_fds: Any = ..., + *, + timeout: Optional[float] = ..., + input: _TXT = ..., + encoding: Optional[str] = ..., + errors: str, + ) -> str: ... + @overload + def check_output( + args: _CMD, + bufsize: int = ..., + executable: Optional[AnyPath] = ..., + stdin: _FILE = ..., + stderr: _FILE = ..., + preexec_fn: Optional[Callable[[], Any]] = ..., + close_fds: bool = ..., + shell: bool = ..., + cwd: Optional[AnyPath] = ..., + env: Optional[_ENV] = ..., + startupinfo: Any = ..., + creationflags: int = ..., + restore_signals: bool = ..., + start_new_session: bool = ..., + pass_fds: Any = ..., + *, + universal_newlines: Literal[True], + timeout: Optional[float] = ..., + input: _TXT = ..., + encoding: Optional[str] = ..., + errors: Optional[str] = ..., + ) -> str: ... + @overload + def check_output( + args: _CMD, + bufsize: int = ..., + executable: Optional[AnyPath] = ..., + stdin: _FILE = ..., + stderr: _FILE = ..., + preexec_fn: Optional[Callable[[], Any]] = ..., + close_fds: bool = ..., + shell: bool = ..., + cwd: Optional[AnyPath] = ..., + env: Optional[_ENV] = ..., + universal_newlines: Literal[False] = ..., + startupinfo: Any = ..., + creationflags: int = ..., + restore_signals: bool = ..., + start_new_session: bool = ..., + pass_fds: Any = ..., + *, + timeout: Optional[float] = ..., + input: _TXT = ..., + encoding: None = ..., + errors: None = ..., + ) -> bytes: ... + @overload + def check_output( + args: _CMD, + bufsize: int = ..., + executable: Optional[AnyPath] = ..., + stdin: _FILE = ..., + stderr: _FILE = ..., + preexec_fn: Optional[Callable[[], Any]] = ..., + close_fds: bool = ..., + shell: bool = ..., + cwd: Optional[AnyPath] = ..., + env: Optional[_ENV] = ..., + universal_newlines: bool = ..., + startupinfo: Any = ..., + creationflags: int = ..., + restore_signals: bool = ..., + start_new_session: bool = ..., + pass_fds: Any = ..., + *, + timeout: Optional[float] = ..., + input: _TXT = ..., + encoding: Optional[str] = ..., + errors: Optional[str] = ..., + ) -> Any: ... # morally: -> _TXT + +PIPE: int +STDOUT: int +DEVNULL: int + +class SubprocessError(Exception): ... + +class TimeoutExpired(SubprocessError): + def __init__(self, cmd: _CMD, timeout: float, output: Optional[_TXT] = ..., stderr: Optional[_TXT] = ...) -> None: ... + # morally: _CMD + cmd: Any + timeout: float + # morally: Optional[_TXT] + output: Any + stdout: Any + stderr: Any + +class CalledProcessError(SubprocessError): + returncode: int + # morally: _CMD + cmd: Any + # morally: Optional[_TXT] + output: Any + + # morally: Optional[_TXT] + stdout: Any + stderr: Any + def __init__(self, returncode: int, cmd: _CMD, output: Optional[_TXT] = ..., stderr: Optional[_TXT] = ...) -> None: ... + +class Popen(Generic[AnyStr]): + args: _CMD + stdin: Optional[IO[AnyStr]] + stdout: Optional[IO[AnyStr]] + stderr: Optional[IO[AnyStr]] + pid: int + returncode: Optional[int] # this is REQUIRED for mypyc + universal_newlines: bool + + # Technically it is wrong that Popen provides __new__ instead of __init__ + # but this shouldn't come up hopefully? + + if sys.version_info >= (3, 7): + # text is added in 3.7 + @overload + def __new__( + cls, + args: _CMD, + bufsize: int = ..., + executable: Optional[AnyPath] = ..., + stdin: Optional[_FILE] = ..., + stdout: Optional[_FILE] = ..., + stderr: Optional[_FILE] = ..., + preexec_fn: Optional[Callable[[], Any]] = ..., + close_fds: bool = ..., + shell: bool = ..., + cwd: Optional[AnyPath] = ..., + env: Optional[_ENV] = ..., + universal_newlines: bool = ..., + startupinfo: Optional[Any] = ..., + creationflags: int = ..., + restore_signals: bool = ..., + start_new_session: bool = ..., + pass_fds: Any = ..., + *, + text: Optional[bool] = ..., + encoding: str, + errors: Optional[str] = ..., + ) -> Popen[str]: ... + @overload + def __new__( + cls, + args: _CMD, + bufsize: int = ..., + executable: Optional[AnyPath] = ..., + stdin: Optional[_FILE] = ..., + stdout: Optional[_FILE] = ..., + stderr: Optional[_FILE] = ..., + preexec_fn: Optional[Callable[[], Any]] = ..., + close_fds: bool = ..., + shell: bool = ..., + cwd: Optional[AnyPath] = ..., + env: Optional[_ENV] = ..., + universal_newlines: bool = ..., + startupinfo: Optional[Any] = ..., + creationflags: int = ..., + restore_signals: bool = ..., + start_new_session: bool = ..., + pass_fds: Any = ..., + *, + text: Optional[bool] = ..., + encoding: Optional[str] = ..., + errors: str, + ) -> Popen[str]: ... + @overload + def __new__( + cls, + args: _CMD, + bufsize: int = ..., + executable: Optional[AnyPath] = ..., + stdin: Optional[_FILE] = ..., + stdout: Optional[_FILE] = ..., + stderr: Optional[_FILE] = ..., + preexec_fn: Optional[Callable[[], Any]] = ..., + close_fds: bool = ..., + shell: bool = ..., + cwd: Optional[AnyPath] = ..., + env: Optional[_ENV] = ..., + *, + universal_newlines: Literal[True], + startupinfo: Optional[Any] = ..., + creationflags: int = ..., + restore_signals: bool = ..., + start_new_session: bool = ..., + pass_fds: Any = ..., + # where the *real* keyword only args start + text: Optional[bool] = ..., + encoding: Optional[str] = ..., + errors: Optional[str] = ..., + ) -> Popen[str]: ... + @overload + def __new__( + cls, + args: _CMD, + bufsize: int = ..., + executable: Optional[AnyPath] = ..., + stdin: Optional[_FILE] = ..., + stdout: Optional[_FILE] = ..., + stderr: Optional[_FILE] = ..., + preexec_fn: Optional[Callable[[], Any]] = ..., + close_fds: bool = ..., + shell: bool = ..., + cwd: Optional[AnyPath] = ..., + env: Optional[_ENV] = ..., + universal_newlines: bool = ..., + startupinfo: Optional[Any] = ..., + creationflags: int = ..., + restore_signals: bool = ..., + start_new_session: bool = ..., + pass_fds: Any = ..., + *, + text: Literal[True], + encoding: Optional[str] = ..., + errors: Optional[str] = ..., + ) -> Popen[str]: ... + @overload + def __new__( + cls, + args: _CMD, + bufsize: int = ..., + executable: Optional[AnyPath] = ..., + stdin: Optional[_FILE] = ..., + stdout: Optional[_FILE] = ..., + stderr: Optional[_FILE] = ..., + preexec_fn: Optional[Callable[[], Any]] = ..., + close_fds: bool = ..., + shell: bool = ..., + cwd: Optional[AnyPath] = ..., + env: Optional[_ENV] = ..., + universal_newlines: Literal[False] = ..., + startupinfo: Optional[Any] = ..., + creationflags: int = ..., + restore_signals: bool = ..., + start_new_session: bool = ..., + pass_fds: Any = ..., + *, + text: Literal[None, False] = ..., + encoding: None = ..., + errors: None = ..., + ) -> Popen[bytes]: ... + @overload + def __new__( + cls, + args: _CMD, + bufsize: int = ..., + executable: Optional[AnyPath] = ..., + stdin: Optional[_FILE] = ..., + stdout: Optional[_FILE] = ..., + stderr: Optional[_FILE] = ..., + preexec_fn: Optional[Callable[[], Any]] = ..., + close_fds: bool = ..., + shell: bool = ..., + cwd: Optional[AnyPath] = ..., + env: Optional[_ENV] = ..., + universal_newlines: bool = ..., + startupinfo: Optional[Any] = ..., + creationflags: int = ..., + restore_signals: bool = ..., + start_new_session: bool = ..., + pass_fds: Any = ..., + *, + text: Optional[bool] = ..., + encoding: Optional[str] = ..., + errors: Optional[str] = ..., + ) -> Popen[Any]: ... + else: + @overload + def __new__( + cls, + args: _CMD, + bufsize: int = ..., + executable: Optional[AnyPath] = ..., + stdin: Optional[_FILE] = ..., + stdout: Optional[_FILE] = ..., + stderr: Optional[_FILE] = ..., + preexec_fn: Optional[Callable[[], Any]] = ..., + close_fds: bool = ..., + shell: bool = ..., + cwd: Optional[AnyPath] = ..., + env: Optional[_ENV] = ..., + universal_newlines: bool = ..., + startupinfo: Optional[Any] = ..., + creationflags: int = ..., + restore_signals: bool = ..., + start_new_session: bool = ..., + pass_fds: Any = ..., + *, + encoding: str, + errors: Optional[str] = ..., + ) -> Popen[str]: ... + @overload + def __new__( + cls, + args: _CMD, + bufsize: int = ..., + executable: Optional[AnyPath] = ..., + stdin: Optional[_FILE] = ..., + stdout: Optional[_FILE] = ..., + stderr: Optional[_FILE] = ..., + preexec_fn: Optional[Callable[[], Any]] = ..., + close_fds: bool = ..., + shell: bool = ..., + cwd: Optional[AnyPath] = ..., + env: Optional[_ENV] = ..., + universal_newlines: bool = ..., + startupinfo: Optional[Any] = ..., + creationflags: int = ..., + restore_signals: bool = ..., + start_new_session: bool = ..., + pass_fds: Any = ..., + *, + encoding: Optional[str] = ..., + errors: str, + ) -> Popen[str]: ... + @overload + def __new__( + cls, + args: _CMD, + bufsize: int = ..., + executable: Optional[AnyPath] = ..., + stdin: Optional[_FILE] = ..., + stdout: Optional[_FILE] = ..., + stderr: Optional[_FILE] = ..., + preexec_fn: Optional[Callable[[], Any]] = ..., + close_fds: bool = ..., + shell: bool = ..., + cwd: Optional[AnyPath] = ..., + env: Optional[_ENV] = ..., + *, + universal_newlines: Literal[True], + startupinfo: Optional[Any] = ..., + creationflags: int = ..., + restore_signals: bool = ..., + start_new_session: bool = ..., + pass_fds: Any = ..., + # where the *real* keyword only args start + encoding: Optional[str] = ..., + errors: Optional[str] = ..., + ) -> Popen[str]: ... + @overload + def __new__( + cls, + args: _CMD, + bufsize: int = ..., + executable: Optional[AnyPath] = ..., + stdin: Optional[_FILE] = ..., + stdout: Optional[_FILE] = ..., + stderr: Optional[_FILE] = ..., + preexec_fn: Optional[Callable[[], Any]] = ..., + close_fds: bool = ..., + shell: bool = ..., + cwd: Optional[AnyPath] = ..., + env: Optional[_ENV] = ..., + universal_newlines: Literal[False] = ..., + startupinfo: Optional[Any] = ..., + creationflags: int = ..., + restore_signals: bool = ..., + start_new_session: bool = ..., + pass_fds: Any = ..., + *, + encoding: None = ..., + errors: None = ..., + ) -> Popen[bytes]: ... + @overload + def __new__( + cls, + args: _CMD, + bufsize: int = ..., + executable: Optional[AnyPath] = ..., + stdin: Optional[_FILE] = ..., + stdout: Optional[_FILE] = ..., + stderr: Optional[_FILE] = ..., + preexec_fn: Optional[Callable[[], Any]] = ..., + close_fds: bool = ..., + shell: bool = ..., + cwd: Optional[AnyPath] = ..., + env: Optional[_ENV] = ..., + universal_newlines: bool = ..., + startupinfo: Optional[Any] = ..., + creationflags: int = ..., + restore_signals: bool = ..., + start_new_session: bool = ..., + pass_fds: Any = ..., + *, + encoding: Optional[str] = ..., + errors: Optional[str] = ..., + ) -> Popen[Any]: ... + def poll(self) -> Optional[int]: ... + if sys.version_info >= (3, 7): + def wait(self, timeout: Optional[float] = ...) -> int: ... + else: + def wait(self, timeout: Optional[float] = ..., endtime: Optional[float] = ...) -> int: ... + # Return str/bytes + def communicate( + self, + input: Optional[AnyStr] = ..., + timeout: Optional[float] = ..., + ) -> Tuple[Optional[AnyStr], AnyStr]: ... # this optional is REQUIRED for mypyc + def send_signal(self, sig: int) -> None: ... + def terminate(self) -> None: ... + def kill(self) -> None: ... + def __enter__(self: _S) -> _S: ... + def __exit__( + self, type: Optional[Type[BaseException]], value: Optional[BaseException], traceback: Optional[TracebackType] + ) -> None: ... + if sys.version_info >= (3, 9): + def __class_getitem__(cls, item: Any) -> GenericAlias: ... + +# The result really is always a str. +def getstatusoutput(cmd: _TXT) -> Tuple[int, str]: ... +def getoutput(cmd: _TXT) -> str: ... +def list2cmdline(seq: Sequence[str]) -> str: ... # undocumented + +if sys.platform == "win32": + class STARTUPINFO: + if sys.version_info >= (3, 7): + def __init__( + self, + *, + dwFlags: int = ..., + hStdInput: Optional[Any] = ..., + hStdOutput: Optional[Any] = ..., + hStdError: Optional[Any] = ..., + wShowWindow: int = ..., + lpAttributeList: Optional[Mapping[str, Any]] = ..., + ) -> None: ... + dwFlags: int + hStdInput: Optional[Any] + hStdOutput: Optional[Any] + hStdError: Optional[Any] + wShowWindow: int + if sys.version_info >= (3, 7): + lpAttributeList: Mapping[str, Any] + STD_INPUT_HANDLE: Any + STD_OUTPUT_HANDLE: Any + STD_ERROR_HANDLE: Any + SW_HIDE: int + STARTF_USESTDHANDLES: int + STARTF_USESHOWWINDOW: int + CREATE_NEW_CONSOLE: int + CREATE_NEW_PROCESS_GROUP: int + if sys.version_info >= (3, 7): + ABOVE_NORMAL_PRIORITY_CLASS: int + BELOW_NORMAL_PRIORITY_CLASS: int + HIGH_PRIORITY_CLASS: int + IDLE_PRIORITY_CLASS: int + NORMAL_PRIORITY_CLASS: int + REALTIME_PRIORITY_CLASS: int + CREATE_NO_WINDOW: int + DETACHED_PROCESS: int + CREATE_DEFAULT_ERROR_MODE: int + CREATE_BREAKAWAY_FROM_JOB: int From 10c2cb62e7e03ad1542018200adc09b705d37b5a Mon Sep 17 00:00:00 2001 From: Rupert Nash Date: Thu, 11 Mar 2021 13:33:25 +0000 Subject: [PATCH 020/111] make unit tests run in-process for speed (#1392) --- tests/test_content_type.py | 14 ++++++++++++-- tests/util.py | 28 ++++++++++++++++------------ 2 files changed, 28 insertions(+), 14 deletions(-) diff --git a/tests/test_content_type.py b/tests/test_content_type.py index 422565a7ba..7f8b3d340c 100644 --- a/tests/test_content_type.py +++ b/tests/test_content_type.py @@ -1,9 +1,11 @@ +from typing import Any + import pydot # type: ignore from .util import get_main_output -def test_content_types() -> None: +def test_content_types(caplog: Any) -> None: for test_file in ("js_output.cwl", "js_output_workflow.cwl"): commands = [ "https://raw.githubusercontent.com/common-workflow-language/common-workflow-language/main/v1.0/v1.0/test-cwl-out2.cwl", @@ -11,5 +13,13 @@ def test_content_types() -> None: ] error_code, _, stderr = get_main_output(commands) - assert "got content-type of 'text/html'" in stderr + found = False + for record in caplog.records: + if ( + record.name == "salad" + and "got content-type of 'text/html'" in record.message + ): + found = True + break + assert found assert error_code == 1, stderr diff --git a/tests/util.py b/tests/util.py index 8fe5b9385f..6370aa38fc 100644 --- a/tests/util.py +++ b/tests/util.py @@ -1,11 +1,12 @@ import contextlib import distutils.spawn # pylint: disable=no-name-in-module,import-error import functools +import io import os import subprocess import sys from pathlib import Path -from typing import Generator, List, Mapping, Optional, Tuple, Union +from typing import Any, Generator, List, Mapping, Optional, Tuple, Union import pytest from pkg_resources import Requirement, ResolutionError, resource_filename @@ -13,6 +14,7 @@ from cwltool.context import LoadingContext, RuntimeContext from cwltool.executors import JobExecutor from cwltool.factory import Factory +from cwltool.main import main from cwltool.singularity import is_version_2_6, is_version_3_or_newer from cwltool.utils import onWindows, windows_default_container_id @@ -84,19 +86,21 @@ def get_main_output( env: Union[ Mapping[bytes, Union[bytes, str]], Mapping[str, Union[bytes, str]], None ] = None, + monkeypatch: Any = None, ) -> Tuple[Optional[int], str, str]: - process = subprocess.Popen( - [sys.executable, "-m", "cwltool"] + args, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - env=env, - ) - - stdout, stderr = process.communicate() + stdout = io.StringIO() + stderr = io.StringIO() + if env is not None: + assert monkeypatch is not None + monkeypatch.setattr(os, "environ", env) + try: + rc = main(argsl=args, stdout=stdout, stderr=stderr) + except SystemExit as e: + rc = e.code return ( - process.returncode, - stdout.decode() if stdout else "", - stderr.decode() if stderr else "", + rc, + stdout.getvalue(), + stderr.getvalue(), ) From abac57517a883af5e906cc15932d2b864e4d3065 Mon Sep 17 00:00:00 2001 From: Rupert Nash Date: Thu, 11 Mar 2021 13:20:48 +0000 Subject: [PATCH 021/111] allow manual workflow triggering --- .github/workflows/ci-tests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index 2a0455f2c0..cf10f8a7e9 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -5,6 +5,7 @@ on: branches: [ main ] pull_request: branches: [ main ] + workflow_dispatch: env: singularity_version: 3.6.4 From 22a1fd2b245390f6380716540d95fea2cb2f56c8 Mon Sep 17 00:00:00 2001 From: Peter Amstutz Date: Fri, 12 Mar 2021 01:47:01 -0500 Subject: [PATCH 022/111] Secondaryfile deps (#1416) * Fix scandeps to resolve secondaryFiles that are File or Directory objects. --- cwltool/process.py | 16 +++++++++++++++- tests/test_examples.py | 17 +++++++++++++++++ tests/wf/indir/hello2.txt | 1 + tests/wf/trick_defaults2.cwl | 14 ++++++++++++++ 4 files changed, 47 insertions(+), 1 deletion(-) create mode 100644 tests/wf/indir/hello2.txt create mode 100644 tests/wf/trick_defaults2.cwl diff --git a/cwltool/process.py b/cwltool/process.py index ea2655e5d3..20d89d389f 100644 --- a/cwltool/process.py +++ b/cwltool/process.py @@ -1153,7 +1153,21 @@ def scandeps( if doc["class"] == "Directory" and "listing" in doc: deps["listing"] = doc["listing"] if doc["class"] == "File" and "secondaryFiles" in doc: - deps["secondaryFiles"] = doc["secondaryFiles"] + deps["secondaryFiles"] = cast( + CWLOutputAtomType, + scandeps( + base, + cast( + Union[CWLObjectType, MutableSequence[CWLObjectType]], + doc["secondaryFiles"], + ), + reffields, + urlfields, + loadref, + urljoin=urljoin, + nestdirs=nestdirs, + ), + ) if nestdirs: deps = nestdir(base, deps) r.append(deps) diff --git a/tests/test_examples.py b/tests/test_examples.py index 296c07b25c..24486867bb 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -528,6 +528,23 @@ def test_trick_scandeps() -> None: assert json.loads(stream.getvalue())["secondaryFiles"][0]["location"][:2] != "_:" +def test_scandeps_defaults_with_secondaryfiles() -> None: + stream = StringIO() + + main( + [ + "--print-deps", + "--relative-deps=cwd", + "--debug", + get_data("tests/wf/trick_defaults2.cwl"), + ], + stdout=stream, + ) + assert json.loads(stream.getvalue())["secondaryFiles"][0]["secondaryFiles"][0][ + "location" + ].endswith(os.path.join("tests", "wf", "indir1")) + + def test_input_deps() -> None: stream = StringIO() diff --git a/tests/wf/indir/hello2.txt b/tests/wf/indir/hello2.txt new file mode 100644 index 0000000000..14be0d41c6 --- /dev/null +++ b/tests/wf/indir/hello2.txt @@ -0,0 +1 @@ +hello2 diff --git a/tests/wf/trick_defaults2.cwl b/tests/wf/trick_defaults2.cwl new file mode 100644 index 0000000000..9b53fba927 --- /dev/null +++ b/tests/wf/trick_defaults2.cwl @@ -0,0 +1,14 @@ +#!/usr/bin/env cwl-runner +class: CommandLineTool +cwlVersion: v1.0 +inputs: + inp1: + type: File + default: + class: File + location: hello.txt + secondaryFiles: + - class: Directory + location: indir1 +outputs: [] +baseCommand: true From 8bdc1529fd49ccdfa4734daa6646bcf5f1a65cba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Mar=C3=ADa=20Fern=C3=A1ndez?= <2072150+jmfernandez@users.noreply.github.com> Date: Fri, 12 Mar 2021 12:24:11 +0100 Subject: [PATCH 023/111] Fixed singularity user namespace feature detection (#1419) --- cwltool/singularity.py | 1 + 1 file changed, 1 insertion(+) diff --git a/cwltool/singularity.py b/cwltool/singularity.py index 9361bc0169..9aaf65ba66 100644 --- a/cwltool/singularity.py +++ b/cwltool/singularity.py @@ -50,6 +50,7 @@ def _singularity_supports_userns() -> bool: _USERNS = ( "No valid /bin/sh" in result or "/bin/sh doesn't exist in container" in result + or "executable file not found in" in result ) except TimeoutExpired: _USERNS = False From 0053a86a5af15cbe02a95292bfa01fc980142eee Mon Sep 17 00:00:00 2001 From: Ralf Grubenmann Date: Tue, 16 Mar 2021 14:17:53 +0100 Subject: [PATCH 024/111] add unicode regex for paths and fix path encoding (#1420) --- cwltool/command_line_tool.py | 10 +++- requirements.txt | 2 +- tests/test_path_checks.py | 97 +++++++++++++++++++++++++++++++++ tests/test_relax_path_checks.py | 43 --------------- tox.ini | 2 +- 5 files changed, 107 insertions(+), 47 deletions(-) create mode 100644 tests/test_path_checks.py delete mode 100644 tests/test_relax_path_checks.py diff --git a/cwltool/command_line_tool.py b/cwltool/command_line_tool.py index 71d1e9866b..cf3a3afac5 100644 --- a/cwltool/command_line_tool.py +++ b/cwltool/command_line_tool.py @@ -10,6 +10,7 @@ import shutil import threading import urllib +import urllib.parse from functools import cmp_to_key, partial from typing import ( Any, @@ -83,7 +84,9 @@ if TYPE_CHECKING: from .provenance_profile import ProvenanceProfile # pylint: disable=unused-import -ACCEPTLIST_EN_STRICT_RE = re.compile(r"^[a-zA-Z0-9._+-]+$") +ACCEPTLIST_EN_STRICT_RE = re.compile( + r"^[\w.+\-\u2600-\u26FF\U0001f600-\U0001f64f]+$" +) # accept unicode word characters and emojis ACCEPTLIST_EN_RELAXED_RE = re.compile(r".*") # Accept anything ACCEPTLIST_RE = ACCEPTLIST_EN_STRICT_RE DEFAULT_CONTAINER_MSG = """ @@ -1178,7 +1181,10 @@ def collect_output( { "location": g, "path": fs_access.join( - builder.outdir, g[len(prefix[0]) + 1 :] + builder.outdir, + urllib.parse.unquote( + g[len(prefix[0]) + 1 :] + ), ), "basename": os.path.basename(g), "nameroot": os.path.splitext( diff --git a/requirements.txt b/requirements.txt index 8711d83431..01d8bfb85c 100755 --- a/requirements.txt +++ b/requirements.txt @@ -6,7 +6,7 @@ schema-salad>=7.1,<8 prov==1.5.1 bagit==1.7.0 mypy-extensions -psutil +psutil<5.8.0 typing-extensions coloredlogs pydot>=1.4.1 diff --git a/tests/test_path_checks.py b/tests/test_path_checks.py new file mode 100644 index 0000000000..1783a10ee9 --- /dev/null +++ b/tests/test_path_checks.py @@ -0,0 +1,97 @@ +# -*- coding: utf-8 -*- +import pytest +from pathlib import Path + +from cwltool.main import main + +from .util import needs_docker + +script = """ +#!/usr/bin/env cwl-runner +cwlVersion: v1.0 +class: CommandLineTool +inputs: + - id: input + type: File + inputBinding: + position: 0 + - id: output + type: string +outputs: + - id: output + type: File + outputBinding: + glob: "$(inputs.output)" +stdout: "$(inputs.output)" +baseCommand: [cat] +""" + + +@needs_docker +def test_spaces_in_input_files(tmp_path: Path) -> None: + script_name = tmp_path / "script" + spaces = tmp_path / "test with spaces" + spaces.touch() + with script_name.open(mode="w") as script_file: + script_file.write(script) + + params = [ + "--debug", + "--outdir", + str(tmp_path / "outdir"), + str(script_name), + "--input", + str(spaces), + "--output", + "test.txt", + ] + assert main(params) == 1 + assert main(["--relax-path-checks"] + params) == 0 + + +@needs_docker +@pytest.mark.parametrize( + "filename", ["測試", "그래프", "график", "𒁃", "☕😍", "امتحان", "abc+DEFGZ.z_12345-"] +) +def test_unicode_in_input_files(tmp_path: Path, filename: str) -> None: + script_name = tmp_path / "script" + inputfile = tmp_path / filename + inputfile.touch() + with script_name.open(mode="w") as script_file: + script_file.write(script) + + params = [ + "--debug", + "--outdir", + str(tmp_path / "outdir"), + str(script_name), + "--input", + str(inputfile), + "--output", + "test.txt", + ] + assert main(params) == 0 + + +@needs_docker +@pytest.mark.parametrize( + "filename", ["測試", "그래프", "график", "𒁃", "☕😍", "امتحان", "abc+DEFGZ.z_12345-"] +) +def test_unicode_in_output_files(tmp_path: Path, filename: str) -> None: + script_name = tmp_path / "script" + inputfile = tmp_path / "test" + inputfile.touch() + with script_name.open(mode="w") as script_file: + script_file.write(script) + + params = [ + "--debug", + "--outdir", + str(tmp_path / "outdir"), + str(script_name), + "--input", + str(inputfile), + "--output", + filename, + ] + assert main(params) == 0 diff --git a/tests/test_relax_path_checks.py b/tests/test_relax_path_checks.py deleted file mode 100644 index eab7fcb993..0000000000 --- a/tests/test_relax_path_checks.py +++ /dev/null @@ -1,43 +0,0 @@ -from pathlib import Path - -from cwltool.main import main - -from .util import needs_docker - -script = """ -#!/usr/bin/env cwl-runner -cwlVersion: v1.0 -class: CommandLineTool -inputs: - - id: input - type: File - inputBinding: - position: 0 -outputs: - - id: output - type: File - outputBinding: - glob: test.txt -stdout: test.txt -baseCommand: [cat] -""" - - -@needs_docker -def test_spaces_in_input_files(tmp_path: Path) -> None: - script_name = tmp_path / "script" - spaces = tmp_path / "test with spaces" - spaces.touch() - with script_name.open(mode="w") as script_file: - script_file.write(script) - - params = [ - "--debug", - "--outdir", - str(tmp_path / "outdir"), - str(script_name), - "--input", - str(spaces), - ] - assert main(params) == 1 - assert main(["--relax-path-checks"] + params) == 0 diff --git a/tox.ini b/tox.ini index 3242ff873d..6d742a295b 100644 --- a/tox.ini +++ b/tox.ini @@ -44,7 +44,7 @@ deps = py{36,37,38,39}-mypy: mypy==0.800 setenv = - py{36,37,38,39}-unit: LC_ALL = C + py{36,37,38,39}-unit: LC_ALL = C.UTF-8 commands = py{36,37,38,39}-unit: python3 -m pip install -U pip setuptools wheel From 83a489e7b3a2693f0665d28111543a61a7744e94 Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" Date: Tue, 16 Mar 2021 10:40:28 +0100 Subject: [PATCH 025/111] release-test, tox, makefile: converge with schema_salad --- Jenkinsfile | 29 ----------------------------- Makefile | 19 ++++++++++++++----- cwltool/main.py | 2 +- mypy_requirements.txt | 1 + release-test.sh | 34 ++++++++++++++++++---------------- test-requirements.txt | 2 +- tests/test_content_type.py | 2 +- tests/test_examples.py | 5 +---- tests/test_load_tool.py | 4 ++-- tests/test_provenance.py | 22 ++++++++++------------ tests/test_udocker.py | 17 +++++------------ tox.ini | 35 ++++++++++++++++------------------- 12 files changed, 70 insertions(+), 102 deletions(-) delete mode 100644 Jenkinsfile create mode 100644 mypy_requirements.txt diff --git a/Jenkinsfile b/Jenkinsfile deleted file mode 100644 index eba57b290e..0000000000 --- a/Jenkinsfile +++ /dev/null @@ -1,29 +0,0 @@ -pipeline { - agent { - node { - label 'windows' - } - - } - options { - timeout(30) - } - stages { - stage('build') { - steps { - withPythonEnv(pythonInstallation: 'Windows-CPython-36') { - pybat(script: 'pip install .', returnStdout: true) - pybat 'jenkins.bat' - } - - } - } - } - post { - always { - junit 'tests.xml' - - } - - } -} diff --git a/Makefile b/Makefile index c11256d6bf..b6266a0d7a 100644 --- a/Makefile +++ b/Makefile @@ -25,8 +25,8 @@ MODULE=cwltool # `SHELL=bash` doesn't work for some, so don't use BASH-isms like # `[[` conditional expressions. PYSOURCES=$(wildcard ${MODULE}/**.py tests/*.py) setup.py -DEVPKGS=diff_cover black pylint coverage pep257 pydocstyle flake8 mypy\ - pytest-xdist isort wheel autoflake -rtest-requirements.txt +DEVPKGS=diff_cover black pylint coverage pep257 pydocstyle flake8\ + isort wheel autoflake flake8-bugbear pyupgrade bandit -rtest-requirements.txt -rmypy_requirements.txt DEBDEVPKGS=pep8 python-autopep8 pylint python-coverage pydocstyle sloccount \ python-flake8 python-mock shellcheck VERSION=3.0.$(shell TZ=UTC git log --first-parent --max-count=1 \ @@ -81,8 +81,8 @@ clean: FORCE # Linting and code style related targets ## sorting imports using isort: https://github.com/timothycrosley/isort -sort_imports: - isort ${MODULE}/*.py tests/*.py setup.py +sort_imports: $(PYSOURCES) + isort $^ remove_unused_imports: $(PYSOURCES) autoflake --in-place --remove-all-unused-imports $^ @@ -92,7 +92,7 @@ pep257: pydocstyle pydocstyle: $(PYSOURCES) pydocstyle --add-ignore=D100,D101,D102,D103 $^ || true -pydocstyle_report.txt: $(filter-out tests/%,${PYSOURCES}) +pydocstyle_report.txt: $(PYSOURCES) pydocstyle setup.py $^ > $@ 2>&1 || true diff_pydocstyle_report: pydocstyle_report.txt @@ -102,6 +102,9 @@ diff_pydocstyle_report: pydocstyle_report.txt format: black --exclude cwltool/schemas setup.py cwltool.py cwltool tests +format-check: + black --diff --check --exclude cwltool/schemas setup.py cwltool.py cwltool tests + ## pylint : run static code analysis on Python code pylint: $(PYSOURCES) pylint --msg-template="{path}:{line}: [{msg_id}({symbol}), {obj}] {msg}" \ @@ -173,6 +176,9 @@ shellcheck: FORCE shellcheck build-cwl-docker.sh cwl-docker.sh release-test.sh conformance-test.sh \ cwltool-in-docker.sh +pyupgrade: ${PYSOURCES} + pyupgrade --exit-zero-even-if-changed --py36-plus $^ + release-test: FORCE git diff-index --quiet HEAD -- || ( echo You have uncommited changes, please commit them and try again; false ) ./release-test.sh @@ -184,6 +190,9 @@ release: release-test twine upload testenv2/src/${MODULE}/dist/* && \ git tag ${VERSION} && git push --tags +flake8: $(filter-out setup.py,${PYSOURCES}) + flake8 $^ + FORCE: # Use this to print the value of a Makefile variable diff --git a/cwltool/main.py b/cwltool/main.py index 90b0800daa..77a4d260c6 100755 --- a/cwltool/main.py +++ b/cwltool/main.py @@ -681,7 +681,7 @@ def __init__(self) -> None: def formatTime( self, record: logging.LogRecord, datefmt: Optional[str] = None ) -> str: - formatted_time = time.strftime("%Y-%m-%dT%H:%M:%S", time.gmtime(record.created)) + formatted_time = time.strftime("%Y-%m-%dT%H:%M:%S", time.gmtime(float(record.created))) with_msecs = "%s,%03f" % (formatted_time, record.msecs) return with_msecs diff --git a/mypy_requirements.txt b/mypy_requirements.txt new file mode 100644 index 0000000000..10316e95bc --- /dev/null +++ b/mypy_requirements.txt @@ -0,0 +1 @@ +mypy==0.812 diff --git a/release-test.sh b/release-test.sh index 1daa20e325..7164dd4879 100755 --- a/release-test.sh +++ b/release-test.sh @@ -40,10 +40,9 @@ then rm -Rf testenv1/local rm -f testenv1/lib/python-wheels/setuptools* \ && pip install --force-reinstall -U pip==${pipver} \ - && pip install setuptools==${setuptoolsver} wheel - make install-dep - pip install . - python setup.py test + && pip install setuptools==${setuptoolsver} wheel + pip install -rtest-requirements.txt + make test pip uninstall -y ${package} || true; pip uninstall -y ${package} || true; make install mkdir testenv1/not-${module} # if there is a subdir named '${module}' py.test will execute tests @@ -60,27 +59,29 @@ rm -Rf testenv[234]/local # Secondly we test via pip -cd testenv2 +pushd testenv2 # shellcheck source=/dev/null source bin/activate rm -f lib/python-wheels/setuptools* \ && pip install --force-reinstall -U pip==${pipver} \ && pip install setuptools==${setuptoolsver} wheel -pip install -e "git+${repo}@${HEAD}#egg=${package}" #[deps] -cd src/${package} -make install-dep +# The following can fail if you haven't pushed your commits to ${repo} +pip install -e "git+${repo}@${HEAD}#egg=${package}" +pushd src/${package} +pip install -rtest-requirements.txt make dist -python setup.py test +make test cp dist/${package}*tar.gz ../../../testenv3/ pip uninstall -y ${package} || true; pip uninstall -y ${package} || true; make install -cd ../.. # no subdir named ${proj} here, safe for py.testing the installed module +popd # ../.. no subdir named ${proj} here, safe for py.testing the installed module # shellcheck disable=SC2086 run_tests +popd -# Is the distribution in testenv2 complete enough to build another +# Is the source distribution in testenv2 complete enough to build another # functional distribution? -cd ../testenv3/ +pushd testenv3/ # shellcheck source=/dev/null source bin/activate rm -f lib/python-wheels/setuptools* \ @@ -88,15 +89,16 @@ rm -f lib/python-wheels/setuptools* \ && pip install setuptools==${setuptoolsver} wheel package_tar=$(find . -name "${package}*tar.gz") pip install "-r${DIR}/test-requirements.txt" -pip install "${package_tar}" # [deps] +pip install "${package_tar}" mkdir out tar --extract --directory=out -z -f ${package}*.tar.gz -cd out/${package}* -make install-dep +pushd out/${package}* make dist -python setup.py test +make test pip uninstall -y ${package} || true; pip uninstall -y ${package} || true; make install mkdir ../not-${module} pushd ../not-${module} # shellcheck disable=SC2086 test_prefix=../../ run_tests; popd +popd +popd diff --git a/test-requirements.txt b/test-requirements.txt index dea3f4c9ff..555bffe542 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,7 +1,7 @@ pytest >= 6.0.2, < 6.2 +pytest-xdist mock >= 2.0.0 pytest-mock >= 1.10.0 pytest-cov arcp >= 0.2.0 rdflib-jsonld >= 0.4.0 -attrs <19.2.0 diff --git a/tests/test_content_type.py b/tests/test_content_type.py index 7f8b3d340c..edc55d236f 100644 --- a/tests/test_content_type.py +++ b/tests/test_content_type.py @@ -6,7 +6,7 @@ def test_content_types(caplog: Any) -> None: - for test_file in ("js_output.cwl", "js_output_workflow.cwl"): + for _ in ("js_output.cwl", "js_output_workflow.cwl"): commands = [ "https://raw.githubusercontent.com/common-workflow-language/common-workflow-language/main/v1.0/v1.0/test-cwl-out2.cwl", "https://github.com/common-workflow-language/common-workflow-language/blob/main/v1.0/v1.0/empty.json", diff --git a/tests/test_examples.py b/tests/test_examples.py index 24486867bb..a79a550fb1 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -255,11 +255,8 @@ def test_parameter_to_expression_interpolate_escapebehavior( @pytest.mark.parametrize("pattern", interpolate_bad_parameters) def test_expression_interpolate_failures(pattern: str) -> None: result = None - try: + with pytest.raises(JavascriptException): result = expr.interpolate(pattern, interpolate_input) - except JavascriptException: - return - assert False, 'Should have produced a JavascriptException, got "{}".'.format(result) interpolate_escapebehavior = ( diff --git a/tests/test_load_tool.py b/tests/test_load_tool.py index 0d4c7e05e4..9ff5c6f5d6 100644 --- a/tests/test_load_tool.py +++ b/tests/test_load_tool.py @@ -23,13 +23,13 @@ def test_check_version() -> None: joborder = {"inp": "abc"} # type: CWLObjectType loadingContext = LoadingContext({"do_update": True}) tool = load_tool(get_data("tests/echo.cwl"), loadingContext) - for j in tool.job(joborder, None, RuntimeContext()): + for _ in tool.job(joborder, None, RuntimeContext()): pass loadingContext = LoadingContext({"do_update": False}) tool = load_tool(get_data("tests/echo.cwl"), loadingContext) with pytest.raises(WorkflowException): - for j in tool.job(joborder, None, RuntimeContext()): + for _ in tool.job(joborder, None, RuntimeContext()): pass diff --git a/tests/test_provenance.py b/tests/test_provenance.py index 0239c42578..bf00ca98c4 100644 --- a/tests/test_provenance.py +++ b/tests/test_provenance.py @@ -176,12 +176,11 @@ def test_directory_workflow(tmp_path: Path) -> None: file_list = ( folder / "data" - / + / "3c" + / "3ca69e8d6c234a469d16ac28a4a658c92267c423" # checksum as returned from: # echo -e "a\nb\nc" | sha1sum # 3ca69e8d6c234a469d16ac28a4a658c92267c423 - - "3c" - / "3ca69e8d6c234a469d16ac28a4a658c92267c423" ) assert file_list.is_file() @@ -208,12 +207,11 @@ def check_secondary_files(base_path: Path) -> None: foo_data = ( base_path / "data" - / + / "0b" + / "0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33" # checksum as returned from: # $ echo -n foo | sha1sum # 0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33 - - "0b" - / "0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33" ) bar_data = base_path / "data" / "62" / "62cdb7020ff920e5aa642c3d4066950dd1f01f4d" assert foo_data.is_file(), "Did not capture file.txt 'foo'" @@ -332,9 +330,9 @@ def check_ro(base_path: Path, nested: bool = False) -> None: g.serialize(sys.stdout, format="ttl") ro = None - for ro in g.subjects(ORE.isDescribedBy, URIRef(base)): + for _ro in g.subjects(ORE.isDescribedBy, URIRef(base)): break - assert ro is not None, "Can't find RO with ore:isDescribedBy" + assert _ro is not None, "Can't find RO with ore:isDescribedBy" profile = None for dc in g.objects(ro, DCTERMS.conformsTo): @@ -393,10 +391,10 @@ def check_ro(base_path: Path, nested: bool = False) -> None: assert (d, OA.hasTarget, URIRef(uuid.urn)) in g linked = set(g.subjects(OA.motivatedBy, OA.linking)) - for l in linked: - assert (l, OA.hasBody, URIRef(packed)) in g - assert (l, OA.hasBody, URIRef(primary_job)) in g - assert (l, OA.hasTarget, URIRef(uuid.urn)) in g + for link in linked: + assert (link, OA.hasBody, URIRef(packed)) in g + assert (link, OA.hasBody, URIRef(primary_job)) in g + assert (link, OA.hasTarget, URIRef(uuid.urn)) in g has_provenance = set(g.subjects(OA.hasBody, URIRef(primary_prov_nt))) for p in has_provenance: diff --git a/tests/test_udocker.py b/tests/test_udocker.py index 603d29a606..7f64ff2542 100644 --- a/tests/test_udocker.py +++ b/tests/test_udocker.py @@ -4,12 +4,6 @@ import subprocess import sys -try: - from psutil.tests import TRAVIS # type: ignore -except ImportError: - TRAVIS = True - - from pathlib import Path import pytest @@ -41,12 +35,12 @@ def udocker(tmp_path_factory: TempPathFactory) -> str: ], ] - os.environ["UDOCKER_DIR"] = os.path.join(docker_install_dir, ".udocker") - os.environ["HOME"] = docker_install_dir + test_environ["UDOCKER_DIR"] = os.path.join(docker_install_dir, ".udocker") + test_environ["HOME"] = docker_install_dir results = [] for _ in range(3): - results = [subprocess.call(cmds) for cmds in install_cmds] + results = [subprocess.call(cmds, env=test_environ) for cmds in install_cmds] if sum(results) == 0: break subprocess.call(["rm", "./udocker"]) @@ -55,7 +49,6 @@ def udocker(tmp_path_factory: TempPathFactory) -> str: udocker_path = os.path.join(docker_install_dir, "udocker") os.chdir(test_cwd) - os.environ = test_environ return udocker_path @@ -86,8 +79,8 @@ def test_udocker_usage_should_not_write_cid_file(udocker: str, tmp_path: Path) - @pytest.mark.skipif( - not LINUX or TRAVIS, - reason="Linux only & not reliable on single threaded test on Travis-CI.", + not LINUX or "GITHUB" in os.environ, + reason="Linux only", ) def test_udocker_should_display_memory_usage(udocker: str, tmp_path: Path) -> None: """Confirm that memory ussage is logged even with udocker.""" diff --git a/tox.ini b/tox.ini index 6d742a295b..566a18273c 100644 --- a/tox.ini +++ b/tox.ini @@ -4,19 +4,19 @@ envlist = py{36,37,38,39}-unit, py{36,37,38,39}-bandit, py{36,37,38,39}-mypy, - py38-lint-readme, - py38-shellcheck, - py38-pydocstyle + py39-lint-readme, + py39-shellcheck, + py39-pydocstyle skipsdist = True skip_missing_interpreters = True [gh-actions] python = - 3.6: py36 - 3.7: py37 - 3.8: py38 - 3.9: py39 + 3.6: py36 + 3.7: py37 + 3.8: py38 + 3.9: py39 [testenv] description = @@ -34,30 +34,27 @@ passenv = deps = py{36,37,38,39}-{unit,lint,bandit,mypy}: -rrequirements.txt py{36,37,38,39}-unit: codecov - py{36,37,38,39}-unit: pytest-xdist - py{36,37,38,39}-unit: pytest-cov py{36,37,38,39}-{unit,mypy}: -rtest-requirements.txt py{36,37,38,39}-unit: galaxy-tool-util py{36,37,38,39}-lint: flake8-bugbear py{36,37,38,39}-lint: black py{36,37,38,39}-bandit: bandit - py{36,37,38,39}-mypy: mypy==0.800 + py{36,37,38,39}-mypy: -rmypy_requirements.txt setenv = py{36,37,38,39}-unit: LC_ALL = C.UTF-8 commands = - py{36,37,38,39}-unit: python3 -m pip install -U pip setuptools wheel - py{36,37,38,39}-unit: python3 -m pip install -e .[deps] - py{36,37,38,39}-unit: python3 -m pip install -rtest-requirements.txt - py{36,37,38,39}-unit: coverage run --parallel-mode -m pytest --strict {posargs} + py{36,37,38,39}-unit: python -m pip install -U pip setuptools wheel + py{36,37,38,39}-unit: python -m pip install -e .[deps] + py{36,37,38,39}-unit: coverage run --parallel-mode -m pytest --strict-markers {posargs} py{36,37,38,39}-unit: coverage combine py{36,37,38,39}-unit: coverage report py{36,37,38,39}-unit: coverage xml py{36,37,38,39}-unit: codecov --file coverage.xml py{36,37,38,39}-bandit: bandit -r cwltool - py{36,37,38,39}-lint: flake8 cwltool setup.py - py{36,37,38,39}-lint: black --diff --exclude cwltool/schemas --check cwltool tests + py{36,37,38,39}-lint: make flake8 + py{36,37,38,39}-lint: make format-check py{36,37,38,39}-mypy: make mypy py{36,37,38,39}-mypy: make mypyc py38-shellcheck: make shellcheck @@ -65,17 +62,17 @@ commands = whitelist_externals = py{36,37,38,39}-lint: flake8 py{36,37,38,39}-lint: black - py{36,37,38,39}-{mypy,shellcheck}: make + py{36,37,38,39}-{mypy,shellcheck,lint}: make py38-shellcheck: shellcheck -[testenv:py38-pydocstyle] +[testenv:py39-pydocstyle] whitelist_externals = make commands = make diff_pydocstyle_report deps = pydocstyle diff-cover -[testenv:py38-lint-readme] +[testenv:py39-lint-readme] description = Lint the README.rst->.md conversion commands = python setup.py sdist From 9c702a9fff4fded16d9937da68575b2aced127e8 Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" Date: Tue, 16 Mar 2021 11:41:16 +0100 Subject: [PATCH 026/111] upgrade to py3.6+ syntax --- Makefile | 8 ++++--- cwltool/argparser.py | 4 ++-- cwltool/builder.py | 12 ++++++---- cwltool/checker.py | 6 ++--- cwltool/command_line_tool.py | 12 ++++++---- cwltool/context.py | 8 ++++--- cwltool/cwlrdf.py | 30 ++++++++++++------------ cwltool/docker.py | 12 ++++------ cwltool/executors.py | 9 ++++--- cwltool/expression.py | 22 ++++++------------ cwltool/factory.py | 11 +++++---- cwltool/job.py | 10 ++++---- cwltool/load_tool.py | 2 +- cwltool/main.py | 30 +++++++++++++----------- cwltool/mpi.py | 2 +- cwltool/mutation.py | 2 +- cwltool/pathmapper.py | 2 +- cwltool/process.py | 31 +++++++++++-------------- cwltool/procgenerator.py | 14 +++++------ cwltool/provenance.py | 32 ++++++++++++------------- cwltool/provenance_profile.py | 12 +++++----- cwltool/resolver.py | 6 ++--- cwltool/sandboxjs.py | 14 ++++++----- cwltool/secrets.py | 2 +- cwltool/singularity.py | 6 ++--- cwltool/software_requirements.py | 4 +++- cwltool/stdfsaccess.py | 2 +- cwltool/task_queue.py | 2 +- cwltool/update.py | 22 +++++++++--------- cwltool/utils.py | 4 ++-- cwltool/validate_js.py | 4 ++-- cwltool/workflow.py | 12 ++++------ cwltool/workflow_job.py | 34 +++++++++++++++------------ requirements.txt | 2 +- tests/test_examples.py | 2 +- tests/test_ext.py | 10 ++++---- tests/test_load_tool.py | 2 +- tests/test_pathmapper.py | 2 +- tests/test_provenance.py | 40 ++++++++++++++++---------------- tests/test_trs.py | 9 ++++--- tests/test_udocker.py | 1 - tox.ini | 1 - 42 files changed, 226 insertions(+), 226 deletions(-) diff --git a/Makefile b/Makefile index b6266a0d7a..ff78d90812 100644 --- a/Makefile +++ b/Makefile @@ -25,10 +25,12 @@ MODULE=cwltool # `SHELL=bash` doesn't work for some, so don't use BASH-isms like # `[[` conditional expressions. PYSOURCES=$(wildcard ${MODULE}/**.py tests/*.py) setup.py -DEVPKGS=diff_cover black pylint coverage pep257 pydocstyle flake8\ - isort wheel autoflake flake8-bugbear pyupgrade bandit -rtest-requirements.txt -rmypy_requirements.txt +DEVPKGS=diff_cover black pylint coverage pep257 pydocstyle flake8 tox\ + isort wheel autoflake flake8-bugbear pyupgrade bandit \ + -rtest-requirements.txt -rmypy_requirements.txt DEBDEVPKGS=pep8 python-autopep8 pylint python-coverage pydocstyle sloccount \ python-flake8 python-mock shellcheck + VERSION=3.0.$(shell TZ=UTC git log --first-parent --max-count=1 \ --format=format:%cd --date=format-local:%Y%m%d%H%M%S) mkfile_dir := $(dir $(abspath $(lastword $(MAKEFILE_LIST)))) @@ -190,7 +192,7 @@ release: release-test twine upload testenv2/src/${MODULE}/dist/* && \ git tag ${VERSION} && git push --tags -flake8: $(filter-out setup.py,${PYSOURCES}) +flake8: ${PYSOURCES} flake8 $^ FORCE: diff --git a/cwltool/argparser.py b/cwltool/argparser.py index 4f60a6ca40..89ccae4bbd 100644 --- a/cwltool/argparser.py +++ b/cwltool/argparser.py @@ -694,7 +694,7 @@ def __init__( """Fail if nargs is used.""" if nargs is not None: raise ValueError("nargs not allowed") - super(FSAction, self).__init__(option_strings, dest, **kwargs) + super().__init__(option_strings, dest, **kwargs) def __call__( self, @@ -722,7 +722,7 @@ def __init__( """Initialize.""" if nargs is not None: raise ValueError("nargs not allowed") - super(FSAppendAction, self).__init__(option_strings, dest, **kwargs) + super().__init__(option_strings, dest, **kwargs) def __call__( self, diff --git a/cwltool/builder.py b/cwltool/builder.py index 1cdb37df94..3688818e5b 100644 --- a/cwltool/builder.py +++ b/cwltool/builder.py @@ -130,7 +130,9 @@ def check_format( ) -class HasReqsHints(object): +class HasReqsHints: + """Base class for get_requirement().""" + def __init__(self) -> None: """Initialize this reqs decorator.""" self.requirements = [] # type: List[CWLObjectType] @@ -294,7 +296,7 @@ def bind_input( bound_input = True if not bound_input: raise ValidationException( - "'%s' is not a valid union %s" % (datum, schema["type"]) + "'{}' is not a valid union {}".format(datum, schema["type"]) ) elif isinstance(schema["type"], MutableMapping): st = copy.deepcopy(schema["type"]) @@ -398,7 +400,9 @@ def _capture_files(f: CWLObjectType) -> CWLObjectType: ) as f2: datum["contents"] = content_limit_respected_read(f2) except Exception as e: - raise Exception("Reading %s\n%s" % (datum["location"], e)) + raise Exception( + "Reading {}\n{}".format(datum["location"], e) + ) if "secondaryFiles" in schema: if "secondaryFiles" not in datum: @@ -545,7 +549,7 @@ def tostr(self, value: Union[MutableMapping[str, str], Any]) -> str: ): if "path" not in value: raise WorkflowException( - u'%s object missing "path": %s' % (value["class"], value) + '{} object missing "path": {}'.format(value["class"], value) ) # Path adjust for windows file path when passing to docker, docker accepts unix like path only diff --git a/cwltool/checker.py b/cwltool/checker.py index 0f9d39444a..ef30996016 100644 --- a/cwltool/checker.py +++ b/cwltool/checker.py @@ -60,7 +60,7 @@ def check_types( return check_types( merge_flatten_type(_get_type(srctype)), _get_type(sinktype), None, None ) - raise WorkflowException("Unrecognized linkMerge enum '{}'".format(linkMerge)) + raise WorkflowException(f"Unrecognized linkMerge enum '{linkMerge}'") def merge_flatten_type(src: SinkType) -> CWLOutputType: @@ -222,7 +222,7 @@ def static_checker( # by the source missing = missing_subset(srcsf, sinksf) if missing: - msg1 = "Parameter '%s' requires secondaryFiles %s but" % ( + msg1 = "Parameter '{}' requires secondaryFiles {} but".format( shortname(sink["id"]), missing, ) @@ -241,7 +241,7 @@ def static_checker( % shortname(sink["id"]) ) msg = SourceLine(sink).makeError( - "%s\n%s" % (msg1, bullets([msg3, msg4, msg5], " ")) + "{}\n{}".format(msg1, bullets([msg3, msg4, msg5], " ")) ) elif sink.get("not_connected"): if not sink.get("used_by_step"): diff --git a/cwltool/command_line_tool.py b/cwltool/command_line_tool.py index cf3a3afac5..20170b21fc 100644 --- a/cwltool/command_line_tool.py +++ b/cwltool/command_line_tool.py @@ -104,7 +104,7 @@ """ -class ExpressionJob(object): +class ExpressionJob: """Job for ExpressionTools.""" def __init__( @@ -267,7 +267,9 @@ def revmap_file( ) -class CallbackJob(object): +class CallbackJob: + """Callback Job class, used by CommandLine.job().""" + def __init__( self, job: "CommandLineTool", @@ -352,7 +354,7 @@ def check_valid_locations(fs_access: StdFsAccess, ob: CWLObjectType) -> None: class ParameterOutputWorkflowException(WorkflowException): def __init__(self, msg: str, port: CWLObjectType, **kwargs: Any) -> None: """Exception for when there was an error collecting output for a parameter.""" - super(ParameterOutputWorkflowException, self).__init__( + super().__init__( "Error collecting output for parameter '%s':\n%s" % (shortname(cast(str, port["id"])), msg), kwargs, @@ -364,7 +366,7 @@ def __init__( self, toolpath_object: CommentedMap, loadingContext: LoadingContext ) -> None: """Initialize this CommandLineTool.""" - super(CommandLineTool, self).__init__(toolpath_object, loadingContext) + super().__init__(toolpath_object, loadingContext) self.prov_obj = loadingContext.prov_obj def make_job_runner(self, runtimeContext: RuntimeContext) -> Type[JobBase]: @@ -792,7 +794,7 @@ def calc_checksum(location: str) -> Optional[str]: jobcache = os.path.join(runtimeContext.cachedir, cachekey) # Create a lockfile to manage cache status. - jobcachepending = "{}.status".format(jobcache) + jobcachepending = f"{jobcache}.status" jobcachelock = None jobstatus = None diff --git a/cwltool/context.py b/cwltool/context.py index 5e58c779e4..34749b3019 100644 --- a/cwltool/context.py +++ b/cwltool/context.py @@ -27,7 +27,9 @@ from .provenance_profile import ProvenanceProfile -class ContextBase(object): +class ContextBase: + """Shared kwargs based initilizer for {Runtime,Loading}Context.""" + def __init__(self, kwargs: Optional[Dict[str, Any]] = None) -> None: """Initialize.""" if kwargs: @@ -73,7 +75,7 @@ def __init__(self, kwargs: Optional[Dict[str, Any]] = None) -> None: self.jobdefaults = None # type: Optional[CommentedMap] self.doc_cache = True # type: bool - super(LoadingContext, self).__init__(kwargs) + super().__init__(kwargs) def copy(self): # type: () -> LoadingContext @@ -149,7 +151,7 @@ def __init__(self, kwargs: Optional[Dict[str, Any]] = None) -> None: self.mpi_config = MpiConfig() # type: MpiConfig self.default_stdout = None # type: Optional[Union[IO[bytes], TextIO]] self.default_stderr = None # type: Optional[Union[IO[bytes], TextIO]] - super(RuntimeContext, self).__init__(kwargs) + super().__init__(kwargs) if self.tmp_outdir_prefix == "": self.tmp_outdir_prefix = self.tmpdir_prefix diff --git a/cwltool/cwlrdf.py b/cwltool/cwlrdf.py index 86b87877ee..0c2f8357a1 100644 --- a/cwltool/cwlrdf.py +++ b/cwltool/cwlrdf.py @@ -47,8 +47,8 @@ def dot_with_parameters(g: Graph, stdout: Union[TextIO, StreamWriter]) -> None: for step, run, _ in qres: stdout.write( - u'"%s" [label="%s"]\n' - % (lastpart(step), "%s (%s)" % (lastpart(step), lastpart(run))) + '"%s" [label="%s"]\n' + % (lastpart(step), "{} ({})".format(lastpart(step), lastpart(run))) ) qres = g.query( @@ -61,12 +61,12 @@ def dot_with_parameters(g: Graph, stdout: Union[TextIO, StreamWriter]) -> None: ) for step, inp, source in qres: - stdout.write(u'"%s" [shape=box]\n' % (lastpart(inp))) + stdout.write('"%s" [shape=box]\n' % (lastpart(inp))) stdout.write( - u'"%s" -> "%s" [label="%s"]\n' % (lastpart(source), lastpart(inp), "") + '"{}" -> "{}" [label="{}"]\n'.format(lastpart(source), lastpart(inp), "") ) stdout.write( - u'"%s" -> "%s" [label="%s"]\n' % (lastpart(inp), lastpart(step), "") + '"{}" -> "{}" [label="{}"]\n'.format(lastpart(inp), lastpart(step), "") ) qres = g.query( @@ -78,9 +78,9 @@ def dot_with_parameters(g: Graph, stdout: Union[TextIO, StreamWriter]) -> None: ) for step, out in qres: - stdout.write(u'"%s" [shape=box]\n' % (lastpart(out))) + stdout.write('"%s" [shape=box]\n' % (lastpart(out))) stdout.write( - u'"%s" -> "%s" [label="%s"]\n' % (lastpart(step), lastpart(out), "") + '"{}" -> "{}" [label="{}"]\n'.format(lastpart(step), lastpart(out), "") ) qres = g.query( @@ -92,9 +92,9 @@ def dot_with_parameters(g: Graph, stdout: Union[TextIO, StreamWriter]) -> None: ) for out, source in qres: - stdout.write(u'"%s" [shape=octagon]\n' % (lastpart(out))) + stdout.write('"%s" [shape=octagon]\n' % (lastpart(out))) stdout.write( - u'"%s" -> "%s" [label="%s"]\n' % (lastpart(source), lastpart(out), "") + '"{}" -> "{}" [label="{}"]\n'.format(lastpart(source), lastpart(out), "") ) qres = g.query( @@ -106,7 +106,7 @@ def dot_with_parameters(g: Graph, stdout: Union[TextIO, StreamWriter]) -> None: ) for (inp,) in qres: - stdout.write(u'"%s" [shape=octagon]\n' % (lastpart(inp))) + stdout.write('"%s" [shape=octagon]\n' % (lastpart(inp))) def dot_without_parameters(g: Graph, stdout: Union[TextIO, StreamWriter]) -> None: @@ -150,7 +150,7 @@ def dot_without_parameters(g: Graph, stdout: Union[TextIO, StreamWriter]) -> Non if wf not in dotname: dotname[wf] = "cluster_" + lastpart(wf) stdout.write( - u'subgraph "%s" { label="%s"\n' % (dotname[wf], lastpart(wf)) + 'subgraph "{}" {{ label="{}"\n'.format(dotname[wf], lastpart(wf)) ) currentwf = wf clusternode[wf] = step @@ -159,7 +159,7 @@ def dot_without_parameters(g: Graph, stdout: Union[TextIO, StreamWriter]) -> Non if str(runtype) != "https://w3id.org/cwl/cwl#Workflow": stdout.write( - u'"%s" [label="%s"]\n' + '"%s" [label="%s"]\n' % (dotname[step], urllib.parse.urldefrag(str(step))[1]) ) @@ -182,12 +182,12 @@ def dot_without_parameters(g: Graph, stdout: Union[TextIO, StreamWriter]) -> Non for src, sink, srcrun, sinkrun in qres: attr = "" if srcrun in clusternode: - attr += u'ltail="%s"' % dotname[srcrun] + attr += 'ltail="%s"' % dotname[srcrun] src = clusternode[srcrun] if sinkrun in clusternode: - attr += u' lhead="%s"' % dotname[sinkrun] + attr += ' lhead="%s"' % dotname[sinkrun] sink = clusternode[sinkrun] - stdout.write(u'"%s" -> "%s" [%s]\n' % (dotname[src], dotname[sink], attr)) + stdout.write('"{}" -> "{}" [{}]\n'.format(dotname[src], dotname[sink], attr)) def printdot( diff --git a/cwltool/docker.py b/cwltool/docker.py index 38f2dd87da..c78e758abe 100644 --- a/cwltool/docker.py +++ b/cwltool/docker.py @@ -9,7 +9,7 @@ import sys import threading from distutils import spawn -from io import StringIO, open # pylint: disable=redefined-builtin +from io import StringIO # pylint: disable=redefined-builtin from typing import Callable, Dict, List, MutableMapping, Optional, Set, Tuple, cast import requests @@ -99,9 +99,7 @@ def __init__( name: str, ) -> None: """Initialize a command line builder using the Docker software container engine.""" - super(DockerCommandLineJob, self).__init__( - builder, joborder, make_path_mapper, requirements, hints, name - ) + super().__init__(builder, joborder, make_path_mapper, requirements, hints, name) @staticmethod def get_image( @@ -261,7 +259,7 @@ def append_volume( output = StringIO() csv.writer(output).writerow(options) mount_arg = output.getvalue().strip() - runtime.append("--mount={}".format(mount_arg)) + runtime.append(f"--mount={mount_arg}") # Unlike "--volume", "--mount" will fail if the volume doesn't already exist. if not os.path.exists(source): os.makedirs(source) @@ -382,7 +380,7 @@ def create_runtime( if self.networkaccess: if runtimeContext.custom_net: - runtime.append("--net={0}".format(runtimeContext.custom_net)) + runtime.append(f"--net={runtimeContext.custom_net}") else: runtime.append("--net=none") @@ -437,7 +435,7 @@ def create_runtime( cidfile_path = os.path.join(cidfile_dir, cidfile_name) runtime.append("--cidfile=%s" % cidfile_path) for key, value in self.environment.items(): - runtime.append("--env=%s=%s" % (key, value)) + runtime.append(f"--env={key}={value}") if runtimeContext.strict_memory_limit and not user_space_docker_cmd: ram = self.builder.resources["ram"] diff --git a/cwltool/executors.py b/cwltool/executors.py index 0e6c35535e..83817737e8 100644 --- a/cwltool/executors.py +++ b/cwltool/executors.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """Single and multi-threaded executors.""" import datetime import functools @@ -40,7 +39,7 @@ TMPDIR_LOCK = Lock() -class JobExecutor(object, metaclass=ABCMeta): +class JobExecutor(metaclass=ABCMeta): """Abstract base job executor.""" def __init__(self) -> None: @@ -278,7 +277,7 @@ class MultithreadedJobExecutor(JobExecutor): def __init__(self) -> None: """Initialize.""" - super(MultithreadedJobExecutor, self).__init__() + super().__init__() self.exceptions = [] # type: List[WorkflowException] self.pending_jobs = [] # type: List[JobsType] self.pending_jobs_lock = threading.Lock() @@ -332,10 +331,10 @@ def _runner(self, job, runtime_context, TMPDIR_LOCK): ) job.run(runtime_context, TMPDIR_LOCK) except WorkflowException as err: - _logger.exception("Got workflow error: {}".format(err)) + _logger.exception(f"Got workflow error: {err}") self.exceptions.append(err) except Exception as err: # pylint: disable=broad-except - _logger.exception("Got workflow error: {}".format(err)) + _logger.exception(f"Got workflow error: {err}") self.exceptions.append(WorkflowException(str(err))) finally: if runtime_context.workflow_eval_lock: diff --git a/cwltool/expression.py b/cwltool/expression.py index 89786d9cc9..4eb94e4695 100644 --- a/cwltool/expression.py +++ b/cwltool/expression.py @@ -47,9 +47,9 @@ def jshead(engine_config: List[str], rootvars: CWLObjectType) -> str: seg_single = r"""\['([^']|\\')+'\]""" seg_double = r"""\["([^"]|\\")+"\]""" seg_index = r"""\[[0-9]+\]""" -segments = r"(\.%s|%s|%s|%s)" % (seg_symbol, seg_single, seg_double, seg_index) +segments = fr"(\.{seg_symbol}|{seg_single}|{seg_double}|{seg_index})" segment_re = re.compile(segments, flags=re.UNICODE) -param_str = r"\((%s)%s*\)$" % (seg_symbol, segments) +param_str = fr"\(({seg_symbol}){segments}*\)$" param_re = re.compile(param_str, flags=re.UNICODE) @@ -163,9 +163,7 @@ def next_seg( % (parsed_string, type(current_value).__name__, key) ) if key not in current_value: - raise WorkflowException( - "%s does not contain key '%s'" % (parsed_string, key) - ) + raise WorkflowException(f"{parsed_string} does not contain key '{key}'") else: try: key = int(next_segment_str[1:-1]) @@ -189,9 +187,7 @@ def next_seg( cast(CWLOutputType, current_value[cast(str, key)]), ) except KeyError: - raise WorkflowException( - "%s doesn't have property %s" % (parsed_string, key) - ) + raise WorkflowException(f"{parsed_string} doesn't have property {key}") elif isinstance(current_value, list) and isinstance(key, int): try: return next_seg( @@ -200,13 +196,9 @@ def next_seg( current_value[key], ) except KeyError: - raise WorkflowException( - "%s doesn't have property %s" % (parsed_string, key) - ) + raise WorkflowException(f"{parsed_string} doesn't have property {key}") else: - raise WorkflowException( - "%s doesn't have property %s" % (parsed_string, key) - ) + raise WorkflowException(f"{parsed_string} doesn't have property {key}") else: return current_value @@ -358,7 +350,7 @@ def interpolate( scan = scan[w[1] :] w = scanner(scan) if convert_to_expression: - parts.append('"{}"'.format(scan)) + parts.append(f'"{scan}"') parts.append(";}") else: parts.append(scan) diff --git a/cwltool/factory.py b/cwltool/factory.py index d2a4104574..5c888c5b12 100644 --- a/cwltool/factory.py +++ b/cwltool/factory.py @@ -12,12 +12,14 @@ class WorkflowStatus(Exception): def __init__(self, out: Optional[CWLObjectType], status: str) -> None: """Signaling exception for the status of a Workflow.""" - super(WorkflowStatus, self).__init__("Completed %s" % status) + super().__init__("Completed %s" % status) self.out = out self.status = status -class Callable(object): +class Callable: + """Result of Factory.make().""" + def __init__(self, t: Process, factory: "Factory") -> None: """Initialize.""" self.t = t @@ -34,14 +36,15 @@ def __call__(self, **kwargs): return out -class Factory(object): +class Factory: + """Easy way to load a CWL document for execution.""" + def __init__( self, executor: Optional[JobExecutor] = None, loading_context: Optional[LoadingContext] = None, runtime_context: Optional[RuntimeContext] = None, ) -> None: - """Easy way to load a CWL document for execution.""" if executor is None: executor = SingleJobExecutor() self.executor = executor diff --git a/cwltool/job.py b/cwltool/job.py index c4132d5508..cbc1f3ae5b 100644 --- a/cwltool/job.py +++ b/cwltool/job.py @@ -188,7 +188,7 @@ def __init__( name: str, ) -> None: """Initialize the job object.""" - super(JobBase, self).__init__() + super().__init__() self.builder = builder self.joborder = joborder self.stdin = None # type: Optional[str] @@ -335,9 +335,7 @@ def _execute( if self.stdin is not None: rmap = self.pathmapper.reversemap(self.stdin) if rmap is None: - raise WorkflowException( - "{} missing from pathmapper".format(self.stdin) - ) + raise WorkflowException(f"{self.stdin} missing from pathmapper") else: stdin_path = rmap[1] @@ -807,7 +805,9 @@ def run( _logger.debug("%s error", container, exc_info=True) if docker_is_req: raise UnsupportedRequirement( - "%s is required to run this tool: %s" % (container, str(err)) + "{} is required to run this tool: {}".format( + container, str(err) + ) ) from err else: raise WorkflowException( diff --git a/cwltool/load_tool.py b/cwltool/load_tool.py index 4a23f117e0..901cfedf90 100644 --- a/cwltool/load_tool.py +++ b/cwltool/load_tool.py @@ -167,7 +167,7 @@ def _convert_stdstreams_to_files( ): if not isinstance(out, CommentedMap): raise ValidationException( - "Output '{}' is not a valid OutputParameter.".format(out) + f"Output '{out}' is not a valid OutputParameter." ) for streamtype in ["stdout", "stderr"]: if out.get("type") == streamtype: diff --git a/cwltool/main.py b/cwltool/main.py index 77a4d260c6..7ffb505c10 100755 --- a/cwltool/main.py +++ b/cwltool/main.py @@ -119,7 +119,7 @@ def _terminate_processes() -> None: ] if cidfile: try: - with open(cidfile[0], "r") as inp_stream: + with open(cidfile[0]) as inp_stream: p = subprocess.Popen( # nosec ["docker", "kill", inp_stream.read()], shell=False # nosec ) @@ -175,7 +175,7 @@ def generate_example_input( example, comment = generate_example_input(inptype[0], default) if optional: if comment: - comment = "{} (optional)".format(comment) + comment = f"{comment} (optional)" else: comment = "optional" else: @@ -194,7 +194,7 @@ def generate_example_input( # array of just an enum then list all the options example = first_item["symbols"] if "name" in first_item: - comment = u'array of type "{}".'.format(first_item["name"]) + comment = 'array of type "{}".'.format(first_item["name"]) else: value, comment = generate_example_input(inptype["items"], None) comment = "array of " + comment @@ -214,27 +214,27 @@ def generate_example_input( example = symbols[0] else: example = "{}_enum_value".format(inptype.get("name", "valid")) - comment = u'enum; valid values: "{}"'.format('", "'.join(symbols)) + comment = 'enum; valid values: "{}"'.format('", "'.join(symbols)) elif inptype["type"] == "record": example = yaml.comments.CommentedMap() if "name" in inptype: - comment = u'"{}" record type.'.format(inptype["name"]) + comment = '"{}" record type.'.format(inptype["name"]) for field in cast(List[CWLObjectType], inptype["fields"]): value, f_comment = generate_example_input(field["type"], None) example.insert(0, shortname(cast(str, field["name"])), value, f_comment) elif "default" in inptype: example = inptype["default"] - comment = u'default value of type "{}".'.format(inptype["type"]) + comment = 'default value of type "{}".'.format(inptype["type"]) else: example = defaults.get(cast(str, inptype["type"]), str(inptype)) - comment = u'type "{}".'.format(inptype["type"]) + comment = 'type "{}".'.format(inptype["type"]) else: if not default: example = defaults.get(str(inptype), str(inptype)) - comment = u'type "{}"'.format(inptype) + comment = f'type "{inptype}"' else: example = default - comment = u'default value of type "{}".'.format(inptype) + comment = f'default value of type "{inptype}".' return example, comment @@ -456,7 +456,7 @@ def init_job_order( if job_order_object is None: if process.tool["inputs"]: if toolparser is not None: - print("\nOptions for {} ".format(args.workflow)) + print(f"\nOptions for {args.workflow} ") toolparser.print_help() _logger.error("") _logger.error("Input object required, use --help for details") @@ -676,13 +676,15 @@ class ProvLogFormatter(logging.Formatter): def __init__(self) -> None: """Use the default formatter with our custom formatstring.""" - super(ProvLogFormatter, self).__init__("[%(asctime)sZ] %(message)s") + super().__init__("[%(asctime)sZ] %(message)s") def formatTime( self, record: logging.LogRecord, datefmt: Optional[str] = None ) -> str: - formatted_time = time.strftime("%Y-%m-%dT%H:%M:%S", time.gmtime(float(record.created))) - with_msecs = "%s,%03f" % (formatted_time, record.msecs) + formatted_time = time.strftime( + "%Y-%m-%dT%H:%M:%S", time.gmtime(float(record.created)) + ) + with_msecs = f"{formatted_time},{record.msecs:03f}" return with_msecs @@ -1038,7 +1040,7 @@ def main( return 0 if args.validate: - print("{} is valid CWL.".format(args.workflow)) + print(f"{args.workflow} is valid CWL.") return 0 if args.print_rdf: diff --git a/cwltool/mpi.py b/cwltool/mpi.py index 369e2ce7a9..bb1b16ffd7 100644 --- a/cwltool/mpi.py +++ b/cwltool/mpi.py @@ -58,7 +58,7 @@ def load(cls: Type[MpiConfigT], config_file_name: str) -> MpiConfigT: return cls(**data) except TypeError as e: unknown = set(data.keys()) - set(inspect.signature(cls).parameters) - raise ValueError("Unknown key(s) in MPI configuration: {}".format(unknown)) + raise ValueError(f"Unknown key(s) in MPI configuration: {unknown}") def pass_through_env_vars(self, env: MutableMapping[str, str]) -> None: """Take the configured list of environment variables and pass them to the executed process.""" diff --git a/cwltool/mutation.py b/cwltool/mutation.py index 8088232302..65f894e167 100644 --- a/cwltool/mutation.py +++ b/cwltool/mutation.py @@ -9,7 +9,7 @@ _generation = "http://commonwl.org/cwltool#generation" -class MutationManager(object): +class MutationManager: """Lock manager for checking correctness of in-place update of files. Used to validate that in-place file updates happen sequentially, and that a diff --git a/cwltool/pathmapper.py b/cwltool/pathmapper.py index f1f8714533..269d3ce9e5 100644 --- a/cwltool/pathmapper.py +++ b/cwltool/pathmapper.py @@ -19,7 +19,7 @@ ) -class PathMapper(object): +class PathMapper: """ Mapping of files from relative path provided in the file to a tuple. diff --git a/cwltool/process.py b/cwltool/process.py index 20d89d389f..a68414ad6b 100644 --- a/cwltool/process.py +++ b/cwltool/process.py @@ -12,7 +12,6 @@ import textwrap import urllib import uuid -from io import open from os import scandir from typing import ( Any, @@ -86,7 +85,7 @@ class LogAsDebugFilter(logging.Filter): def __init__(self, name: str, parent: logging.Logger) -> None: """Initialize.""" name = str(name) - super(LogAsDebugFilter, self).__init__(name) + super().__init__(name) self.parent = parent def filter(self, record: logging.LogRecord) -> bool: @@ -196,23 +195,23 @@ def get_schema( version = ".".join(version.split(".")[:-1]) for f in cwl_files: try: - res = resource_stream(__name__, "schemas/%s/%s" % (version, f)) + res = resource_stream(__name__, f"schemas/{version}/{f}") cache["https://w3id.org/cwl/" + f] = res.read().decode("UTF-8") res.close() - except IOError: + except OSError: pass for f in salad_files: try: res = resource_stream( __name__, - "schemas/{}/salad/schema_salad/metaschema/{}".format(version, f), + f"schemas/{version}/salad/schema_salad/metaschema/{f}", ) cache[ "https://w3id.org/cwl/salad/schema_salad/metaschema/" + f ] = res.read().decode("UTF-8") res.close() - except IOError: + except OSError: pass if version in custom_schemas: @@ -274,10 +273,10 @@ def stage_files( # find first key that does not clash with an existing entry in targets # start with entry.target + '_' + 2 and then keep incrementing the number till there is no clash i = 2 - tgt = "%s_%s" % (entry.target, i) + tgt = f"{entry.target}_{i}" while tgt in targets: i += 1 - tgt = "%s_%s" % (entry.target, i) + tgt = f"{entry.target}_{i}" targets[tgt] = pathmapper.update( key, entry.resolved, tgt, entry.type, entry.staged ) @@ -357,12 +356,10 @@ def _collectDirEntries( yield obj else: for sub_obj in obj.values(): - for dir_entry in _collectDirEntries(sub_obj): - yield dir_entry + yield from _collectDirEntries(sub_obj) elif isinstance(obj, MutableSequence): for sub_obj in obj: - for dir_entry in _collectDirEntries(sub_obj): - yield dir_entry + yield from _collectDirEntries(sub_obj) def _relocate(src: str, dst: str) -> None: if src == dst: @@ -568,7 +565,7 @@ def __init__( self, toolpath_object: CommentedMap, loadingContext: LoadingContext ) -> None: """Build a Process object from the provided dictionary.""" - super(Process, self).__init__() + super().__init__() self.metadata = getdefault(loadingContext.metadata, {}) # type: CWLObjectType self.provenance_object = None # type: Optional[ProvenanceProfile] self.parent_wf = None # type: Optional[ProvenanceProfile] @@ -1019,11 +1016,11 @@ def validate_hints( avsc_names.get_name(cast(str, r["class"]), None) is not None and self.doc_loader is not None ): - plain_hint = dict( - (key, r[key]) + plain_hint = { + key: r[key] for key in r if key not in self.doc_loader.identifiers - ) # strip identifiers + } # strip identifiers validate_ex( cast( Schema, @@ -1061,7 +1058,7 @@ def uniquename(stem: str, names: Optional[Set[str]] = None) -> str: u = stem while u in names: c += 1 - u = "%s_%s" % (stem, c) + u = f"{stem}_{c}" names.add(u) return u diff --git a/cwltool/procgenerator.py b/cwltool/procgenerator.py index b09cc3651b..69780e32cc 100644 --- a/cwltool/procgenerator.py +++ b/cwltool/procgenerator.py @@ -13,7 +13,9 @@ from .utils import CWLObjectType, JobsGeneratorType, OutputCallbackType -class ProcessGeneratorJob(object): +class ProcessGeneratorJob: + """Result of ProcessGenerator.job().""" + def __init__(self, procgenerator: "ProcessGenerator") -> None: """Create a ProccessGenerator Job.""" self.procgenerator = procgenerator @@ -34,10 +36,9 @@ def job( ) -> JobsGeneratorType: try: - for tool in self.procgenerator.embedded_tool.job( + yield from self.procgenerator.embedded_tool.job( job_order, self.receive_output, runtimeContext - ): - yield tool + ) while self.processStatus is None: yield None @@ -53,8 +54,7 @@ def job( job_order, self.jobout, runtimeContext ) - for tool in created_tool.job(runinputs, output_callbacks, runtimeContext): - yield tool + yield from created_tool.job(runinputs, output_callbacks, runtimeContext) except WorkflowException: raise @@ -70,7 +70,7 @@ def __init__( loadingContext: LoadingContext, ) -> None: """Create a ProcessGenerator from the given dictionary and context.""" - super(ProcessGenerator, self).__init__(toolpath_object, loadingContext) + super().__init__(toolpath_object, loadingContext) self.loadingContext = loadingContext # type: LoadingContext try: if isinstance(toolpath_object["run"], CommentedMap): diff --git a/cwltool/provenance.py b/cwltool/provenance.py index 318ff474eb..b2344aa6f9 100644 --- a/cwltool/provenance.py +++ b/cwltool/provenance.py @@ -11,7 +11,7 @@ from array import array from collections import OrderedDict from getpass import getuser -from io import FileIO, TextIOWrapper, open +from io import FileIO, TextIOWrapper from mmap import mmap from pathlib import Path, PurePosixPath from typing import ( @@ -126,7 +126,7 @@ def __init__(self, research_object: "ResearchObject", rel_path: str) -> None: if not path.startswith(os.path.abspath(research_object.folder)): raise ValueError("Path is outside Research Object: %s" % path) _logger.debug("[provenance] Creating WritableBagFile at %s.", path) - super(WritableBagFile, self).__init__(path, mode="w") + super().__init__(path, mode="w") def write(self, b: Any) -> int: """Write some content to the Bag.""" @@ -134,7 +134,7 @@ def write(self, b: Any) -> int: total = 0 length = len(real_b) while total < length: - ret = super(WritableBagFile, self).write(real_b) + ret = super().write(real_b) if ret: total += ret for val in self.hashes.values(): @@ -149,7 +149,7 @@ def close(self) -> None: else: self.research_object.tagfiles.add(self.rel_path) - super(WritableBagFile, self).close() + super().close() # { "sha1": "f572d396fae9206628714fb2ce00f72e94f2258f" } checksums = {} for name in self.hashes: @@ -171,7 +171,7 @@ def truncate(self, size: Optional[int] = None) -> int: # FIXME: This breaks contract IOBase, # as it means we would have to recalculate the hash if size is not None: - raise IOError("WritableBagFile can't truncate") + raise OSError("WritableBagFile can't truncate") return self.tell() @@ -243,14 +243,14 @@ def _valid_orcid(orcid: Optional[str]) -> str: "116780-structure-of-the-orcid-identifier" ) if not match: - raise ValueError("Invalid ORCID: %s\n%s" % (orcid, help_url)) + raise ValueError(f"Invalid ORCID: {orcid}\n{help_url}") # Conservative in what we produce: # a) Ensure any checksum digit is uppercase orcid_num = match.group("orcid").upper() # b) ..and correct if not _check_mod_11_2(orcid_num): - raise ValueError("Invalid ORCID checksum: %s\n%s" % (orcid_num, help_url)) + raise ValueError(f"Invalid ORCID checksum: {orcid_num}\n{help_url}") # c) Re-add the official prefix https://orcid.org/ return "https://orcid.org/%s" % orcid_num @@ -330,7 +330,7 @@ def self_check(self) -> None: def __str__(self) -> str: """Represent this RO as a string.""" - return "ResearchObject <{}> in <{}>".format(self.ro_uuid, self.folder) + return f"ResearchObject <{self.ro_uuid}> in <{self.folder}>" def _initialize(self) -> None: for research_obj_folder in ( @@ -366,8 +366,8 @@ def open_log_file_for_activity( name = "engine" else: name = "activity" - p = os.path.join(LOGS, "{}.{}.txt".format(name, activity_uuid)) - _logger.debug("[provenance] Opening log file for %s: %s" % (name, p)) + p = os.path.join(LOGS, f"{name}.{activity_uuid}.txt") + _logger.debug(f"[provenance] Opening log file for {name}: {p}") self.add_annotation(activity_uuid.urn, [p], CWLPROV["log"].uri) return self.write_bag_file(p) @@ -712,7 +712,7 @@ def _write_ro_manifest(self) -> None: manifest = OrderedDict( { "@context": [ - {"@base": "%s%s/" % (self.base_uri, posix_path(METADATA))}, + {"@base": "{}{}/".format(self.base_uri, posix_path(METADATA))}, "https://w3id.org/bundle/context", ], "id": "/", @@ -882,15 +882,13 @@ def add_to_manifest(self, rel_path: str, checksums: Dict[str, str]) -> None: for (method, hash_value) in checksums.items(): # File not in manifest because we bailed out on # existence in bagged_size above - manifestpath = os.path.join( - self.folder, "%s-%s.txt" % (manifest, method.lower()) - ) + manifestpath = os.path.join(self.folder, f"{manifest}-{method.lower()}.txt") # encoding: match Tag-File-Character-Encoding: UTF-8 # newline: ensure LF also on Windows with open( manifestpath, "a", encoding=ENCODING, newline="\n" ) as checksum_file: - line = "%s %s\n" % (hash_value, rel_path) + line = f"{hash_value} {rel_path}\n" _logger.debug("[provenance] Added to %s: %s", manifestpath, line) checksum_file.write(line) @@ -899,7 +897,7 @@ def _add_to_bagit(self, rel_path: str, **checksums: str) -> None: raise ValueError("rel_path must be relative: %s" % rel_path) lpath = os.path.join(self.folder, local_path(rel_path)) if not os.path.exists(lpath): - raise IOError("File %s does not exist within RO: %s" % (rel_path, lpath)) + raise OSError(f"File {rel_path} does not exist within RO: {lpath}") if rel_path in self.bagged_size: # Already added, assume checksum OK @@ -982,7 +980,7 @@ def _relativise_files( ) as fp: relative_path = self.add_data_file(fp) checksum = PurePosixPath(relative_path).name - structure["checksum"] = "%s$%s" % (SHA1, checksum) + structure["checksum"] = f"{SHA1}${checksum}" if relative_path is not None: # RO-relative path as new location structure["location"] = str(PurePosixPath("..") / relative_path) diff --git a/cwltool/provenance_profile.py b/cwltool/provenance_profile.py index c5ecbd77ba..0a39e45d2d 100644 --- a/cwltool/provenance_profile.py +++ b/cwltool/provenance_profile.py @@ -116,7 +116,7 @@ def __init__( def __str__(self) -> str: """Represent this Provenvance profile as a string.""" - return "ProvenanceProfile <%s> in <%s>" % ( + return "ProvenanceProfile <{}> in <{}>".format( self.workflow_run_uri, self.research_object, ) @@ -321,7 +321,7 @@ def declare_file(self, value: CWLObjectType) -> Tuple[ProvEntity, ProvEntity, st "data:" + checksum, {PROV_TYPE: WFPROV["Artifact"]} ) if "checksum" not in value: - value["checksum"] = "%s$%s" % (SHA1, checksum) + value["checksum"] = f"{SHA1}${checksum}" if not entity and "contents" in value: # Anonymous file, add content as string @@ -362,7 +362,7 @@ def declare_file(self, value: CWLObjectType) -> Tuple[ProvEntity, ProvEntity, st elif sec["class"] == "Directory": sec_entity = self.declare_directory(sec) else: - raise ValueError("Got unexpected secondaryFiles value: {}".format(sec)) + raise ValueError(f"Got unexpected secondaryFiles value: {sec}") # We don't know how/when/where the secondary file was generated, # but CWL convention is a kind of summary/index derived # from the original file. As its generally in a different format @@ -626,7 +626,7 @@ def used_artefacts( if name is not None: base += "/" + name for key, value in job_order.items(): - prov_role = self.wf_ns["%s/%s" % (base, key)] + prov_role = self.wf_ns[f"{base}/{key}"] try: entity = self.declare_artefact(value) self.document.used( @@ -661,7 +661,7 @@ def generate_output_prov( if name is not None: name = urllib.parse.quote(str(name), safe=":/,#") # FIXME: Probably not "main" in nested workflows - role = self.wf_ns["main/%s/%s" % (name, output)] + role = self.wf_ns[f"main/{name}/{output}"] else: role = self.wf_ns["main/%s" % output] @@ -737,7 +737,7 @@ def finalize_prov_profile(self, name): # workflows, but that's OK as we'll also include run uuid # which also covers thhe case of this step being run in # multiple places or iterations - filename = "%s.%s.cwlprov" % (wf_name, self.workflow_run_uuid) + filename = f"{wf_name}.{self.workflow_run_uuid}.cwlprov" basename = str(PurePosixPath(PROVENANCE) / filename) diff --git a/cwltool/resolver.py b/cwltool/resolver.py index f164bd8ffc..cb52281000 100644 --- a/cwltool/resolver.py +++ b/cwltool/resolver.py @@ -21,7 +21,7 @@ def resolve_local(document_loader: Optional[Loader], uri: str) -> Optional[str]: if pathobj.is_file(): if frag: - return "{}#{}".format(pathobj.as_uri(), frag) + return f"{pathobj.as_uri()}#{frag}" return pathobj.as_uri() sharepaths = [ @@ -39,8 +39,8 @@ def resolve_local(document_loader: Optional[Loader], uri: str) -> Optional[str]: for path in shares: if os.path.exists(path): return Path(uri).as_uri() - if os.path.exists("{}.cwl".format(path)): - return Path("{}.cwl".format(path)).as_uri() + if os.path.exists(f"{path}.cwl"): + return Path(f"{path}.cwl").as_uri() return None diff --git a/cwltool/sandboxjs.py b/cwltool/sandboxjs.py index f23086c1d6..778e3af3cf 100644 --- a/cwltool/sandboxjs.py +++ b/cwltool/sandboxjs.py @@ -344,7 +344,7 @@ def code_fragment_to_js(jscript: str, jslib: str = "") -> str: else: inner_js = "{return (%s);}" % jscript - return u'"use strict";\n{}\n(function(){})()'.format(jslib, inner_js) + return f'"use strict";\n{jslib}\n(function(){inner_js})()' def execjs( @@ -394,15 +394,17 @@ def fn_linenum() -> str: % (returncode, fn_linenum(), stdfmt(stdout), stdfmt(stderr)) ) else: - info = "Javascript expression was: %s\nstdout was: %s\nstderr was: %s" % ( - js, - stdfmt(stdout), - stdfmt(stderr), + info = ( + "Javascript expression was: {}\nstdout was: {}\nstderr was: {}".format( + js, + stdfmt(stdout), + stdfmt(stderr), + ) ) if returncode == -1: raise JavascriptException( - "Long-running script killed after {} seconds: {}".format(timeout, info) + f"Long-running script killed after {timeout} seconds: {info}" ) else: raise JavascriptException(info) diff --git a/cwltool/secrets.py b/cwltool/secrets.py index 0d0f5b23d8..7356c4c91d 100644 --- a/cwltool/secrets.py +++ b/cwltool/secrets.py @@ -5,7 +5,7 @@ from .utils import CWLObjectType, CWLOutputType -class SecretStore(object): +class SecretStore: """Minimal implementation of a secret storage.""" def __init__(self) -> None: diff --git a/cwltool/singularity.py b/cwltool/singularity.py index 9aaf65ba66..3257df4dfb 100644 --- a/cwltool/singularity.py +++ b/cwltool/singularity.py @@ -100,9 +100,7 @@ def __init__( name: str, ) -> None: """Builder for invoking the Singularty software container engine.""" - super(SingularityCommandLineJob, self).__init__( - builder, joborder, make_path_mapper, requirements, hints, name - ) + super().__init__(builder, joborder, make_path_mapper, requirements, hints, name) @staticmethod def get_image( @@ -449,5 +447,5 @@ def create_runtime( env["SINGULARITYENV_HOME"] = self.builder.outdir for name, value in self.environment.items(): - env["SINGULARITYENV_{}".format(name)] = str(value) + env[f"SINGULARITYENV_{name}"] = str(value) return (runtime, None) diff --git a/cwltool/software_requirements.py b/cwltool/software_requirements.py index 3ba602c56d..70d6eac184 100644 --- a/cwltool/software_requirements.py +++ b/cwltool/software_requirements.py @@ -36,7 +36,9 @@ ) -class DependenciesConfiguration(object): +class DependenciesConfiguration: + """Dependency configuration class, for RuntimeContext.job_script_provider.""" + def __init__(self, args: argparse.Namespace) -> None: """Initialize.""" conf_file = getattr(args, "beta_dependency_resolvers_configuration", None) diff --git a/cwltool/stdfsaccess.py b/cwltool/stdfsaccess.py index 338b21df32..c4c5dad52e 100644 --- a/cwltool/stdfsaccess.py +++ b/cwltool/stdfsaccess.py @@ -23,7 +23,7 @@ def abspath(src: str, basedir: str) -> str: return abpath -class StdFsAccess(object): +class StdFsAccess: """Local filesystem implementation.""" def __init__(self, basedir: str) -> None: diff --git a/cwltool/task_queue.py b/cwltool/task_queue.py index 987ce56ade..76cb760e5e 100644 --- a/cwltool/task_queue.py +++ b/cwltool/task_queue.py @@ -10,7 +10,7 @@ from .loghandler import _logger -class TaskQueue(object): +class TaskQueue: """A TaskQueue class. Uses a first-in, first-out queue of tasks executed on a fixed number of diff --git a/cwltool/update.py b/cwltool/update.py index 8e261baf0b..05e041b0a9 100644 --- a/cwltool/update.py +++ b/cwltool/update.py @@ -210,25 +210,25 @@ def v1_2_0dev5to1_2( ] UPDATES = { - u"v1.0": v1_0to1_1, - u"v1.1": v1_1to1_2, - u"v1.2": None, + "v1.0": v1_0to1_1, + "v1.1": v1_1to1_2, + "v1.2": None, } # type: Dict[str, Optional[Callable[[CommentedMap, Loader, str], Tuple[CommentedMap, str]]]] DEVUPDATES = { - u"v1.1.0-dev1": v1_1_0dev1to1_1, - u"v1.2.0-dev1": v1_2_0dev1todev2, - u"v1.2.0-dev2": v1_2_0dev2todev3, - u"v1.2.0-dev3": v1_2_0dev3todev4, - u"v1.2.0-dev4": v1_2_0dev4todev5, - u"v1.2.0-dev5": v1_2_0dev5to1_2, + "v1.1.0-dev1": v1_1_0dev1to1_1, + "v1.2.0-dev1": v1_2_0dev1todev2, + "v1.2.0-dev2": v1_2_0dev2todev3, + "v1.2.0-dev3": v1_2_0dev3todev4, + "v1.2.0-dev4": v1_2_0dev4todev5, + "v1.2.0-dev5": v1_2_0dev5to1_2, } # type: Dict[str, Optional[Callable[[CommentedMap, Loader, str], Tuple[CommentedMap, str]]]] ALLUPDATES = UPDATES.copy() ALLUPDATES.update(DEVUPDATES) -INTERNAL_VERSION = u"v1.2" +INTERNAL_VERSION = "v1.2" ORIGINAL_CWLVERSION = "http://commonwl.org/cwltool#original_cwlVersion" @@ -283,7 +283,7 @@ def checkversion( keys = list(UPDATES.keys()) keys.sort() raise ValidationException( - u"Version '%s' is a development or deprecated version.\n " + "Version '%s' is a development or deprecated version.\n " "Update your document to a stable version (%s) or use " "--enable-dev to enable support for development and " "deprecated versions." % (version, ", ".join(keys)) diff --git a/cwltool/utils.py b/cwltool/utils.py index 45d30c87ce..42a012ef96 100644 --- a/cwltool/utils.py +++ b/cwltool/utils.py @@ -120,8 +120,8 @@ def versionstring() -> str: """Version of CWLtool used to execute the workflow.""" pkg = pkg_resources.require("cwltool") if pkg: - return "%s %s" % (sys.argv[0], pkg[0].version) - return "%s %s" % (sys.argv[0], "unknown version") + return "{} {}".format(sys.argv[0], pkg[0].version) + return "{} {}".format(sys.argv[0], "unknown version") def aslist(thing: Any) -> MutableSequence[Any]: diff --git a/cwltool/validate_js.py b/cwltool/validate_js.py index 1f1415b8e2..8cb6b887d4 100644 --- a/cwltool/validate_js.py +++ b/cwltool/validate_js.py @@ -48,7 +48,7 @@ class SuppressLog(logging.Filter): def __init__(self, name): # type: (str) -> None """Initialize this log suppressor.""" name = str(name) - super(SuppressLog, self).__init__(name) + super().__init__(name) def filter(self, record): # type: (logging.LogRecord) -> bool return False @@ -182,7 +182,7 @@ def dump_jshint_error(): for jshint_error_obj in jshint_json.get("errors", []): text = "JSHINT: " + js_text_lines[jshint_error_obj["line"] - 1] + "\n" text += "JSHINT: " + " " * (jshint_error_obj["character"] - 1) + "^\n" - text += "JSHINT: %s: %s" % ( + text += "JSHINT: {}: {}".format( jshint_error_obj["code"], jshint_error_obj["reason"], ) diff --git a/cwltool/workflow.py b/cwltool/workflow.py index dafc3f8d2e..3b70ba5d38 100644 --- a/cwltool/workflow.py +++ b/cwltool/workflow.py @@ -71,7 +71,7 @@ def __init__( loadingContext: LoadingContext, ) -> None: """Initialize this Workflow.""" - super(Workflow, self).__init__(toolpath_object, loadingContext) + super().__init__(toolpath_object, loadingContext) self.provenance_object = None # type: Optional[ProvenanceProfile] if loadingContext.research_obj is not None: run_uuid = None # type: Optional[UUID] @@ -170,8 +170,7 @@ def job( runtimeContext.part_of = "workflow %s" % job.name runtimeContext.toplevel = False - for wjob in job.job(builder.job, output_callbacks, runtimeContext): - yield wjob + yield from job.job(builder.job, output_callbacks, runtimeContext) def visit(self, op: Callable[[CommentedMap], None]) -> None: op(self.tool) @@ -326,7 +325,7 @@ def __init__( if validation_errors: raise ValidationException("\n".join(validation_errors)) - super(WorkflowStep, self).__init__(toolpath_object, loadingContext) + super().__init__(toolpath_object, loadingContext) if self.embedded_tool.tool["class"] == "Workflow": (feature, _) = self.get_requirement("SubworkflowFeatureRequirement") @@ -431,12 +430,11 @@ def job( step_input[field] = job_order[inp["id"]] try: - for tool in self.embedded_tool.job( + yield from self.embedded_tool.job( step_input, functools.partial(self.receive_output, output_callbacks), runtimeContext, - ): - yield tool + ) except WorkflowException: _logger.error("Exception on step '%s'", runtimeContext.name) raise diff --git a/cwltool/workflow_job.py b/cwltool/workflow_job.py index d18ab6ea89..e67747caea 100644 --- a/cwltool/workflow_job.py +++ b/cwltool/workflow_job.py @@ -46,7 +46,9 @@ from .workflow import Workflow, WorkflowStep -class WorkflowJobStep(object): +class WorkflowJobStep: + """Generated for each step in Workflow.steps().""" + def __init__(self, step: "WorkflowStep") -> None: """Initialize this WorkflowJobStep.""" self.step = step @@ -71,11 +73,12 @@ def job( _logger.info("[%s] start", self.name) - for j in self.step.job(joborder, output_callback, runtimeContext): - yield j + yield from self.step.job(joborder, output_callback, runtimeContext) + +class ReceiveScatterOutput: + """Produced by the scatter generators.""" -class ReceiveScatterOutput(object): def __init__( self, output_callback: ScatterOutputCallbackType, @@ -438,7 +441,7 @@ def object_from_state( break if not found: raise WorkflowException( - u"All sources for '%s' are null" % (shortname(original_id)) + "All sources for '%s' are null" % (shortname(original_id)) ) elif inp["pickValue"] == "the_only_non_null": found = False @@ -446,14 +449,14 @@ def object_from_state( if v is not None: if found: raise WorkflowException( - u"Expected only one source for '%s' to be non-null, got %s" + "Expected only one source for '%s' to be non-null, got %s" % (shortname(original_id), seq) ) found = True inputobj[iid] = v if not found: raise WorkflowException( - u"All sources for '%s' are null" % (shortname(original_id)) + "All sources for '%s' are null" % (shortname(original_id)) ) elif inp["pickValue"] == "all_non_null": inputobj[iid] = [v for v in seq if v is not None] @@ -469,7 +472,9 @@ def object_from_state( return inputobj -class WorkflowJob(object): +class WorkflowJob: + """Generates steps from the Workflow.""" + def __init__(self, workflow: "Workflow", runtimeContext: RuntimeContext) -> None: """Initialize this WorkflowJob.""" self.workflow = workflow @@ -628,9 +633,9 @@ def try_make_job( i["id"]: i["valueFrom"] for i in step.tool["inputs"] if "valueFrom" in i } - loadContents = set( + loadContents = { i["id"] for i in step.tool["inputs"] if i.get("loadContents") - ) + } if len(valueFrom) > 0 and not bool( self.workflow.get_requirement("StepInputExpressionRequirement")[0] @@ -745,28 +750,27 @@ def valueFromFunc( else: if _logger.isEnabledFor(logging.DEBUG): _logger.debug( - u"[%s] job input %s", step.name, json_dumps(inputobj, indent=4) + "[%s] job input %s", step.name, json_dumps(inputobj, indent=4) ) inputobj = postScatterEval(inputobj) if inputobj is not None: if _logger.isEnabledFor(logging.DEBUG): _logger.debug( - u"[%s] evaluated job input to %s", + "[%s] evaluated job input to %s", step.name, json_dumps(inputobj, indent=4), ) jobs = step.job(inputobj, callback, runtimeContext) else: - _logger.info(u"[%s] will be skipped", step.name) + _logger.info("[%s] will be skipped", step.name) callback({k["id"]: None for k in outputparms}, "skipped") step.completed = True jobs = (_ for _ in ()) step.submitted = True - for j in jobs: - yield j + yield from jobs except WorkflowException: raise except Exception: diff --git a/requirements.txt b/requirements.txt index 01d8bfb85c..f1e7edb4de 100755 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,7 @@ rdflib>=4.2.2,<5.1 shellescape>=3.4.1,<3.5 schema-salad>=7.1,<8 prov==1.5.1 -bagit==1.7.0 +bagit==1.8.1 mypy-extensions psutil<5.8.0 typing-extensions diff --git a/tests/test_examples.py b/tests/test_examples.py index a79a550fb1..e7b46c2497 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -1413,4 +1413,4 @@ def test_scatter_output_filenames(tmp_path: Path) -> None: locations[0].endswith("output.txt") and locations[1].endswith("output.txt_2") and locations[2].endswith("output.txt_3") - ), "Locations {} do not end with {}".format(locations, output_names) + ), f"Locations {locations} do not end with {output_names}" diff --git a/tests/test_ext.py b/tests/test_ext.py index 235e60fc0b..becd77feff 100644 --- a/tests/test_ext.py +++ b/tests/test_ext.py @@ -113,7 +113,7 @@ def test_double_overwrite(tmp_path: Path) -> None: == 0 ) - with open(tmp_name, "r") as f: + with open(tmp_name) as f: actual_value = f.read() assert actual_value == expected_value @@ -146,9 +146,9 @@ def test_disable_file_overwrite_without_ext(tmp_path: Path) -> None: == 0 ) - with open(tmp_name, "r") as f: + with open(tmp_name) as f: tmp_value = f.read() - with open(out_name, "r") as f: + with open(out_name) as f: out_value = f.read() assert tmp_value == before_value @@ -195,7 +195,7 @@ def test_disable_file_creation_in_outdir_with_ext(tmp_path: Path) -> None: ] assert main(params) == 0 - with open(tmp_name, "r") as f: + with open(tmp_name) as f: tmp_value = f.read() assert tmp_value == expected_value @@ -236,7 +236,7 @@ def test_write_write_conflict(tmp_path: Path) -> None: main(["--enable-ext", get_data("tests/wf/mut.cwl"), "-a", str(tmp_name)]) != 0 ) - with open(tmp_name, "r") as f: + with open(tmp_name) as f: tmp_value = f.read() assert tmp_value == expected_value diff --git a/tests/test_load_tool.py b/tests/test_load_tool.py index 9ff5c6f5d6..629ed6c2aa 100644 --- a/tests/test_load_tool.py +++ b/tests/test_load_tool.py @@ -84,7 +84,7 @@ def test_load_graph_fragment_from_packed() -> None: loadingContext = LoadingContext() uri = Path(get_data("tests/wf/packed-with-loadlisting.cwl")).as_uri() + "#main" try: - with open(get_data("cwltool/extensions.yml"), "r") as res: + with open(get_data("cwltool/extensions.yml")) as res: use_custom_schema("v1.0", "http://commonwl.org/cwltool", res.read()) # The updater transforms LoadListingRequirement from an diff --git a/tests/test_pathmapper.py b/tests/test_pathmapper.py index b957d55dfe..fae1cb5d84 100644 --- a/tests/test_pathmapper.py +++ b/tests/test_pathmapper.py @@ -15,7 +15,7 @@ def __init__( stagedir: str, new: str, ): - super(SubPathMapper, self).__init__(referenced_files, basedir, stagedir) + super().__init__(referenced_files, basedir, stagedir) self.new = new pathmap = SubPathMapper([], "", "", "new") diff --git a/tests/test_provenance.py b/tests/test_provenance.py index bf00ca98c4..4f64a564dc 100644 --- a/tests/test_provenance.py +++ b/tests/test_provenance.py @@ -189,7 +189,7 @@ def test_directory_workflow(tmp_path: Path) -> None: for (l, l_hash) in sha1.items(): prefix = l_hash[:2] # first 2 letters p = folder / "data" / prefix / l_hash - assert p.is_file(), "Could not find %s as %s" % (l, p) + assert p.is_file(), f"Could not find {l} as {p}" def check_output_object(base_path: Path) -> None: @@ -312,7 +312,7 @@ def _arcp2file(base_path: Path, uri: str) -> Path: def check_ro(base_path: Path, nested: bool = False) -> None: manifest_file = base_path / "metadata" / "manifest.json" - assert manifest_file.is_file(), "Can't find {}".format(manifest_file) + assert manifest_file.is_file(), f"Can't find {manifest_file}" arcp_root = find_arcp(base_path) base = urllib.parse.urljoin(arcp_root, "metadata/manifest.json") g = Graph() @@ -320,7 +320,7 @@ def check_ro(base_path: Path, nested: bool = False) -> None: # Avoid resolving JSON-LD context https://w3id.org/bundle/context # so this test works offline context = Path(get_data("tests/bundle-context.jsonld")).as_uri() - with open(manifest_file, "r", encoding="UTF-8") as fh: + with open(manifest_file, encoding="UTF-8") as fh: jsonld = fh.read() # replace with file:/// URI jsonld = jsonld.replace("https://w3id.org/bundle/context", context) @@ -328,14 +328,14 @@ def check_ro(base_path: Path, nested: bool = False) -> None: if os.environ.get("DEBUG"): print("Parsed manifest:\n\n") g.serialize(sys.stdout, format="ttl") - ro = None + _ro = None for _ro in g.subjects(ORE.isDescribedBy, URIRef(base)): break assert _ro is not None, "Can't find RO with ore:isDescribedBy" profile = None - for dc in g.objects(ro, DCTERMS.conformsTo): + for dc in g.objects(_ro, DCTERMS.conformsTo): profile = dc break assert profile is not None, "Can't find profile with dct:conformsTo" @@ -345,7 +345,7 @@ def check_ro(base_path: Path, nested: bool = False) -> None: paths = [] externals = [] - for aggregate in g.objects(ro, ORE.aggregates): + for aggregate in g.objects(_ro, ORE.aggregates): if not arcp.is_arcp_uri(aggregate): externals.append(aggregate) # Won't check external URIs existence here @@ -353,7 +353,7 @@ def check_ro(base_path: Path, nested: bool = False) -> None: continue lfile = _arcp2file(base_path, aggregate) paths.append(os.path.relpath(lfile, base_path)) - assert os.path.isfile(lfile), "Can't find aggregated {}".format(lfile) + assert os.path.isfile(lfile), f"Can't find aggregated {lfile}" assert paths, "Didn't find any arcp aggregates" assert externals, "Didn't find any data URIs" @@ -411,7 +411,7 @@ def check_ro(base_path: Path, nested: bool = False) -> None: # NOTE: DC.format is a Namespace method and does not resolve like other terms formats.update(set(g.objects(prov, DC["format"]))) assert formats, "Could not find media types" - expected = set( + expected = { Literal(f) for f in ( "application/json", @@ -421,7 +421,7 @@ def check_ro(base_path: Path, nested: bool = False) -> None: 'text/turtle; charset="UTF-8"', "application/xml", ) - ) + } assert formats == expected, "Did not match expected PROV media types" if nested: @@ -443,7 +443,7 @@ def check_prov( secondary_files: bool = False, ) -> None: prov_file = base_path / "metadata" / "provenance" / "primary.cwlprov.nt" - assert prov_file.is_file(), "Can't find {}".format(prov_file) + assert prov_file.is_file(), f"Can't find {prov_file}" arcp_root = find_arcp(base_path) # Note: We don't need to include metadata/provnance in base URI # as .nt always use absolute URIs @@ -458,7 +458,7 @@ def check_prov( # main workflow run URI (as urn:uuid:) should correspond to arcp uuid part uuid = arcp.parse_arcp(arcp_root).uuid main_run = URIRef(uuid.urn) - assert main_run in runs, "Can't find run %s in %s" % (main_run, runs) + assert main_run in runs, f"Can't find run {main_run} in {runs}" # TODO: we should not need to parse arcp, but follow # the has_provenance annotations in manifest.json instead @@ -572,7 +572,7 @@ def check_prov( sec_basename = set(g.objects(sec, CWLPROV.basename)).pop() sec_nameroot = set(g.objects(sec, CWLPROV.nameroot)).pop() sec_nameext = set(g.objects(sec, CWLPROV.nameext)).pop() - assert str(sec_basename) == "%s%s" % (sec_nameroot, sec_nameext) + assert str(sec_basename) == f"{sec_nameroot}{sec_nameext}" # TODO: Check hash data file exist in RO # The primary entity should have the same, but different values @@ -580,7 +580,7 @@ def check_prov( prim_basename = set(g.objects(prim, CWLPROV.basename)).pop() prim_nameroot = set(g.objects(prim, CWLPROV.nameroot)).pop() prim_nameext = set(g.objects(prim, CWLPROV.nameext)).pop() - assert str(prim_basename) == "%s%s" % (prim_nameroot, prim_nameext) + assert str(prim_basename) == f"{prim_nameroot}{prim_nameext}" @pytest.fixture @@ -609,7 +609,7 @@ def test_writable_string(research_object: ResearchObject) -> None: sha1 = os.path.join(research_object.folder, "tagmanifest-sha1.txt") assert os.path.isfile(sha1) - with open(sha1, "r", encoding="UTF-8") as sha_file: + with open(sha1, encoding="UTF-8") as sha_file: stripped_sha = sha_file.readline().strip() assert stripped_sha.endswith("file.txt") # stain@biggie:~/src/cwltool$ echo Hello | sha1sum @@ -619,7 +619,7 @@ def test_writable_string(research_object: ResearchObject) -> None: sha256 = os.path.join(research_object.folder, "tagmanifest-sha256.txt") assert os.path.isfile(sha256) - with open(sha256, "r", encoding="UTF-8") as sha_file: + with open(sha256, encoding="UTF-8") as sha_file: stripped_sha = sha_file.readline().strip() assert stripped_sha.endswith("file.txt") @@ -640,7 +640,7 @@ def test_writable_unicode_string(research_object: ResearchObject) -> None: def test_writable_bytes(research_object: ResearchObject) -> None: - string = "Here is a snowman: \u2603 \n".encode("UTF-8") + string = "Here is a snowman: \u2603 \n".encode() with research_object.write_bag_file("file.txt", encoding=None) as fh: fh.write(string) # type: ignore @@ -655,7 +655,7 @@ def test_data(research_object: ResearchObject) -> None: # rather than tagmanifest sha1 = os.path.join(research_object.folder, "manifest-sha1.txt") assert os.path.isfile(sha1) - with open(sha1, "r", encoding="UTF-8") as fh2: + with open(sha1, encoding="UTF-8") as fh2: stripped_sha = fh2.readline().strip() assert stripped_sha.endswith("data/file.txt") @@ -663,14 +663,14 @@ def test_data(research_object: ResearchObject) -> None: def test_not_seekable(research_object: ResearchObject) -> None: with research_object.write_bag_file("file.txt") as fh: assert not fh.seekable() - with pytest.raises(IOError): + with pytest.raises(OSError): fh.seek(0) def test_not_readable(research_object: ResearchObject) -> None: with research_object.write_bag_file("file.txt") as fh: assert not fh.readable() - with pytest.raises(IOError): + with pytest.raises(OSError): fh.read() @@ -679,7 +679,7 @@ def test_truncate_fails(research_object: ResearchObject) -> None: fh.write("Hello there") fh.truncate() # OK as we're always at end # Will fail because the checksum can't rewind - with pytest.raises(IOError): + with pytest.raises(OSError): fh.truncate(0) diff --git a/tests/test_trs.py b/tests/test_trs.py index a1641a8394..acfee2f4ce 100644 --- a/tests/test_trs.py +++ b/tests/test_trs.py @@ -1,8 +1,7 @@ from typing import Any, Optional +from unittest import mock from unittest.mock import MagicMock -import mock - from cwltool.main import main from .util import get_data @@ -61,19 +60,19 @@ def mocked_requests_get(*args: Any, **kwargs: Any) -> MockResponse2: args[0] == "https://dockstore.org/api/api/ga4gh/v2/tools/quay.io%2Fbriandoconnor%2Fdockstore-tool-md5sum/versions/1.0.4/plain-CWL/descriptor/Dockstore.cwl" ): - string = open(get_data("tests/trs/Dockstore.cwl"), "r").read() + string = open(get_data("tests/trs/Dockstore.cwl")).read() return MockResponse2(string, 200) elif ( args[0] == "https://dockstore.org/api/api/ga4gh/v2/tools/%23workflow%2Fgithub.com%2Fdockstore-testing%2Fmd5sum-checker/versions/develop/plain-CWL/descriptor/md5sum-tool.cwl" ): - string = open(get_data("tests/trs/md5sum-tool.cwl"), "r").read() + string = open(get_data("tests/trs/md5sum-tool.cwl")).read() return MockResponse2(string, 200) elif ( args[0] == "https://dockstore.org/api/api/ga4gh/v2/tools/%23workflow%2Fgithub.com%2Fdockstore-testing%2Fmd5sum-checker/versions/develop/plain-CWL/descriptor/md5sum-workflow.cwl" ): - string = open(get_data("tests/trs/md5sum-workflow.cwl"), "r").read() + string = open(get_data("tests/trs/md5sum-workflow.cwl")).read() return MockResponse2(string, 200) elif ( args[0] diff --git a/tests/test_udocker.py b/tests/test_udocker.py index 7f64ff2542..fb036cf282 100644 --- a/tests/test_udocker.py +++ b/tests/test_udocker.py @@ -3,7 +3,6 @@ import os import subprocess import sys - from pathlib import Path import pytest diff --git a/tox.ini b/tox.ini index 566a18273c..56a6f7a79b 100644 --- a/tox.ini +++ b/tox.ini @@ -51,7 +51,6 @@ commands = py{36,37,38,39}-unit: coverage combine py{36,37,38,39}-unit: coverage report py{36,37,38,39}-unit: coverage xml - py{36,37,38,39}-unit: codecov --file coverage.xml py{36,37,38,39}-bandit: bandit -r cwltool py{36,37,38,39}-lint: make flake8 py{36,37,38,39}-lint: make format-check From dbde435f8c409fde68139fad896d1a11faa434ec Mon Sep 17 00:00:00 2001 From: Rupert Nash Date: Wed, 17 Mar 2021 11:55:04 +0000 Subject: [PATCH 027/111] remove relaxed/strict path checking global state to make in-process tests work consistently --- cwltool/command_line_tool.py | 39 +++++++++++++++++++++++++++--------- cwltool/context.py | 1 + cwltool/main.py | 4 +--- 3 files changed, 31 insertions(+), 13 deletions(-) diff --git a/cwltool/command_line_tool.py b/cwltool/command_line_tool.py index 20170b21fc..bd1fb92c6c 100644 --- a/cwltool/command_line_tool.py +++ b/cwltool/command_line_tool.py @@ -1,6 +1,7 @@ """Implementation of CommandLineTool.""" import copy +from enum import Enum import hashlib import json import locale @@ -22,6 +23,7 @@ MutableMapping, MutableSequence, Optional, + Pattern, Set, TextIO, Union, @@ -84,11 +86,19 @@ if TYPE_CHECKING: from .provenance_profile import ProvenanceProfile # pylint: disable=unused-import -ACCEPTLIST_EN_STRICT_RE = re.compile( - r"^[\w.+\-\u2600-\u26FF\U0001f600-\U0001f64f]+$" -) # accept unicode word characters and emojis -ACCEPTLIST_EN_RELAXED_RE = re.compile(r".*") # Accept anything -ACCEPTLIST_RE = ACCEPTLIST_EN_STRICT_RE + +class PathCheckingMode(Enum): + """What characters are allowed in path names. + + We have the strict, default mode and the relaxed mode. + """ + + STRICT = re.compile( + r"^[\w.+\-\u2600-\u26FF\U0001f600-\U0001f64f]+$" + ) # accept unicode word characters and emojis + RELAXED = re.compile(r".*") # Accept anything + + DEFAULT_CONTAINER_MSG = """ We are on Microsoft Windows and not all components of this CWL description have a container specified. This means that these steps will be executed in the default container, @@ -301,7 +311,9 @@ def run( ) -def check_adjust(builder: Builder, file_o: CWLObjectType) -> CWLObjectType: +def check_adjust( + accept_re: Pattern[str], builder: Builder, file_o: CWLObjectType +) -> CWLObjectType: """ Map files to assigned path inside a container. @@ -327,7 +339,7 @@ def check_adjust(builder: Builder, file_o: CWLObjectType) -> CWLObjectType: file_o["nameroot"] = str(nr) if file_o.get("nameext") != ne: file_o["nameext"] = str(ne) - if not ACCEPTLIST_RE.match(basename): + if not accept_re.match(basename): raise WorkflowException( "Invalid filename: '{}' contains illegal characters".format( file_o["basename"] @@ -368,6 +380,11 @@ def __init__( """Initialize this CommandLineTool.""" super().__init__(toolpath_object, loadingContext) self.prov_obj = loadingContext.prov_obj + self.path_check_mode = ( + PathCheckingMode.RELAXED + if loadingContext.relax_path_checks + else PathCheckingMode.STRICT + ) # type: PathCheckingMode def make_job_runner(self, runtimeContext: RuntimeContext) -> Type[JobBase]: dockerReq, dockerRequired = self.get_requirement("DockerRequirement") @@ -685,7 +702,7 @@ def remove_dirname(d: CWLObjectType) -> None: visit_class( [builder.files, builder.bindings], ("File", "Directory"), - partial(check_adjust, builder), + partial(check_adjust, self.path_check_mode.value, builder), ) def job( @@ -713,7 +730,9 @@ def job( cachebuilder.stagedir, separateDirs=False, ) - _check_adjust = partial(check_adjust, cachebuilder) + _check_adjust = partial( + check_adjust, self.path_check_mode.value, cachebuilder + ) visit_class( [cachebuilder.files, cachebuilder.bindings], ("File", "Directory"), @@ -887,7 +906,7 @@ def update_status_output_callback( ) builder.requirements = j.requirements - _check_adjust = partial(check_adjust, builder) + _check_adjust = partial(check_adjust, self.path_check_mode.value, builder) visit_class( [builder.files, builder.bindings], ("File", "Directory"), _check_adjust diff --git a/cwltool/context.py b/cwltool/context.py index 34749b3019..e7908d95e8 100644 --- a/cwltool/context.py +++ b/cwltool/context.py @@ -74,6 +74,7 @@ def __init__(self, kwargs: Optional[Dict[str, Any]] = None) -> None: self.do_update = None # type: Optional[bool] self.jobdefaults = None # type: Optional[CommentedMap] self.doc_cache = True # type: bool + self.relax_path_checks = False # type: bool super().__init__(kwargs) diff --git a/cwltool/main.py b/cwltool/main.py index 7ffb505c10..21498fbf65 100755 --- a/cwltool/main.py +++ b/cwltool/main.py @@ -41,7 +41,7 @@ from schema_salad.sourceline import strip_dup_lineno from schema_salad.utils import ContextType, FetcherCallableType, json_dumps -from . import CWL_CONTENT_TYPES, command_line_tool, workflow +from . import CWL_CONTENT_TYPES, workflow from .argparser import arg_parser, generate_parser, get_default_args from .builder import HasReqsHints from .context import LoadingContext, RuntimeContext, getdefault @@ -949,8 +949,6 @@ def main( _logger.error("CWL document required, no input file was provided") parser.print_help() return 1 - if args.relax_path_checks: - command_line_tool.ACCEPTLIST_RE = command_line_tool.ACCEPTLIST_EN_RELAXED_RE if args.ga4gh_tool_registries: ga4gh_tool_registries[:] = args.ga4gh_tool_registries From 615d3b9510d62202a9f53dfa7d96485c7680adf3 Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" <1330696+mr-c@users.noreply.github.com> Date: Fri, 19 Mar 2021 15:37:21 +0100 Subject: [PATCH 028/111] fix typo in release script (#1427) --- release-test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release-test.sh b/release-test.sh index 7164dd4879..72aa5cb6a3 100755 --- a/release-test.sh +++ b/release-test.sh @@ -33,7 +33,7 @@ rm -Rf testenv? || /bin/true if [ "${RELEASE_SKIP}" != "head" ] then - python3 -m venv testenv_1 + python3 -m venv testenv1 # First we test the head # shellcheck source=/dev/null source testenv1/bin/activate From 26530dc9839f650779173eeafc611ffb646be7fa Mon Sep 17 00:00:00 2001 From: Rupert Nash Date: Tue, 23 Mar 2021 15:29:40 +0000 Subject: [PATCH 029/111] set minimum version of galaxy-tool-util that copes with relaxed path setting --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 56a6f7a79b..d0fef6c97f 100644 --- a/tox.ini +++ b/tox.ini @@ -35,7 +35,7 @@ deps = py{36,37,38,39}-{unit,lint,bandit,mypy}: -rrequirements.txt py{36,37,38,39}-unit: codecov py{36,37,38,39}-{unit,mypy}: -rtest-requirements.txt - py{36,37,38,39}-unit: galaxy-tool-util + py{36,37,38,39}-unit: galaxy-tool-util>=21.1.0 py{36,37,38,39}-lint: flake8-bugbear py{36,37,38,39}-lint: black py{36,37,38,39}-bandit: bandit From 3758687feaceb7cecce02940d79360924af50934 Mon Sep 17 00:00:00 2001 From: Rupert Nash Date: Tue, 23 Mar 2021 15:36:44 +0000 Subject: [PATCH 030/111] Also setup.py, since older g-t-u won't work --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 64c2820feb..43f711de12 100755 --- a/setup.py +++ b/setup.py @@ -108,7 +108,7 @@ "argcomplete", ], extras_require={ - "deps": ["galaxy-tool-util"], + "deps": ["galaxy-tool-util >= 21.1.0"], "docs": [ "sphinx >= 2.2", "sphinx-rtd-theme", From 14c3f8f76c8c7c13ca378b7f1fcd8105fc80b88c Mon Sep 17 00:00:00 2001 From: Kushal Beniwal Date: Wed, 24 Mar 2021 17:41:50 +0530 Subject: [PATCH 031/111] Replace distutils.spawn usage (#1426) --- cwltool/docker.py | 3 +-- cwltool/singularity.py | 3 +-- tests/test_dependencies.py | 6 ++---- tests/test_docker.py | 5 +++-- tests/test_singularity.py | 5 +++-- tests/util.py | 13 ++++++------- 6 files changed, 16 insertions(+), 19 deletions(-) diff --git a/cwltool/docker.py b/cwltool/docker.py index c78e758abe..bd5c25cfa2 100644 --- a/cwltool/docker.py +++ b/cwltool/docker.py @@ -8,7 +8,6 @@ import subprocess # nosec import sys import threading -from distutils import spawn from io import StringIO # pylint: disable=redefined-builtin from typing import Callable, Dict, List, MutableMapping, Optional, Set, Tuple, cast @@ -235,7 +234,7 @@ def get_from_requirements( force_pull: bool, tmp_outdir_prefix: str, ) -> Optional[str]: - if not spawn.find_executable("docker"): + if not shutil.which("docker"): raise WorkflowException("docker executable is not available") if self.get_image( diff --git a/cwltool/singularity.py b/cwltool/singularity.py index 3257df4dfb..42b03e45d0 100644 --- a/cwltool/singularity.py +++ b/cwltool/singularity.py @@ -5,7 +5,6 @@ import re import shutil import sys -from distutils import spawn from subprocess import ( # nosec DEVNULL, PIPE, @@ -267,7 +266,7 @@ def get_from_requirements( (e.g. hello-world-latest.{img,sif}). """ - if not bool(spawn.find_executable("singularity")): + if not bool(shutil.which("singularity")): raise WorkflowException("singularity executable is not available") if not self.get_image(cast(Dict[str, str], r), pull_image, force_pull): diff --git a/tests/test_dependencies.py b/tests/test_dependencies.py index ea1e198314..c976edb92a 100644 --- a/tests/test_dependencies.py +++ b/tests/test_dependencies.py @@ -1,5 +1,5 @@ import os -from distutils import spawn +from shutil import which import pytest @@ -35,9 +35,7 @@ def test_bioconda() -> None: assert error_code == 0, stderr -@pytest.mark.skipif( - not spawn.find_executable("modulecmd"), reason="modulecmd not installed" -) +@pytest.mark.skipif(not which("modulecmd"), reason="modulecmd not installed") def test_modules() -> None: wflow = get_data("tests/random_lines.cwl") job = get_data("tests/random_lines_job.json") diff --git a/tests/test_docker.py b/tests/test_docker.py index 0767bb8f06..5093353735 100644 --- a/tests/test_docker.py +++ b/tests/test_docker.py @@ -1,4 +1,5 @@ -from distutils import spawn +"""Tests for docker engine.""" +from shutil import which from pathlib import Path from cwltool.main import main @@ -35,7 +36,7 @@ def test_docker_iwdr() -> None: "hello", ] ) - docker_installed = bool(spawn.find_executable("docker")) + docker_installed = bool(which("docker")) if docker_installed: assert result_code == 0 else: diff --git a/tests/test_singularity.py b/tests/test_singularity.py index 74c0f2e309..372aee2c6d 100644 --- a/tests/test_singularity.py +++ b/tests/test_singularity.py @@ -1,4 +1,5 @@ -import distutils.spawn +"""Tests to find local Singularity image.""" +import shutil import os import sys from pathlib import Path @@ -70,7 +71,7 @@ def test_singularity_iwdr() -> None: "hello", ] ) - singularity_installed = bool(distutils.spawn.find_executable("singularity")) + singularity_installed = bool(shutil.which("singularity")) if singularity_installed: assert result_code == 0 else: diff --git a/tests/util.py b/tests/util.py index 6370aa38fc..4afe6fedbb 100644 --- a/tests/util.py +++ b/tests/util.py @@ -1,5 +1,5 @@ import contextlib -import distutils.spawn # pylint: disable=no-name-in-module,import-error +import shutil # pylint: disable=no-name-in-module,import-error import functools import io import os @@ -53,29 +53,28 @@ def get_data(filename: str) -> str: needs_docker = pytest.mark.skipif( - not bool(distutils.spawn.find_executable("docker")), + not bool(shutil.which("docker")), reason="Requires the docker executable on the system path.", ) needs_singularity = pytest.mark.skipif( - not bool(distutils.spawn.find_executable("singularity")), + not bool(shutil.which("singularity")), reason="Requires the singularity executable on the system path.", ) needs_singularity_2_6 = pytest.mark.skipif( - not bool(distutils.spawn.find_executable("singularity") and is_version_2_6()), + not bool(shutil.which("singularity") and is_version_2_6()), reason="Requires that version 2.6.x of singularity executable version is on the system path.", ) needs_singularity_3_or_newer = pytest.mark.skipif( - (not bool(distutils.spawn.find_executable("singularity"))) - or (not is_version_3_or_newer()), + (not bool(shutil.which("singularity"))) or (not is_version_3_or_newer()), reason="Requires that version 3.x of singularity executable version is on the system path.", ) windows_needs_docker = pytest.mark.skipif( - onWindows() and not bool(distutils.spawn.find_executable("docker")), + onWindows() and not bool(shutil.which("docker")), reason="Running this test on MS Windows requires the docker executable " "on the system path.", ) From 29b66ed49d33941d3341f8d7f475637ffb935ac8 Mon Sep 17 00:00:00 2001 From: Rupert Nash Date: Fri, 4 Dec 2020 09:35:21 +0000 Subject: [PATCH 032/111] test_dependencies: fix import of galaxy tool_util --- tests/test_dependencies.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/test_dependencies.py b/tests/test_dependencies.py index c976edb92a..207b81ca55 100644 --- a/tests/test_dependencies.py +++ b/tests/test_dependencies.py @@ -1,5 +1,7 @@ import os from shutil import which +from types import ModuleType +from typing import Any, Optional import pytest @@ -7,10 +9,11 @@ from .util import get_data, get_main_output, needs_docker +deps = None # type: Optional[ModuleType] try: - from galaxy.tools import deps # type: ignore + from galaxy.tool_util import deps except ImportError: - deps = None + pass @needs_docker @@ -35,6 +38,7 @@ def test_bioconda() -> None: assert error_code == 0, stderr +@pytest.mark.skipif(not deps, reason="galaxy-lib is not installed") @pytest.mark.skipif(not which("modulecmd"), reason="modulecmd not installed") def test_modules() -> None: wflow = get_data("tests/random_lines.cwl") From bdb20faf898f42c98846543462bc270c15a928c9 Mon Sep 17 00:00:00 2001 From: Rupert Nash Date: Wed, 24 Mar 2021 13:18:33 +0000 Subject: [PATCH 033/111] tests: use monkeypatch to ensure test enviroment changes are undone --- tests/test_ext.py | 23 ++++++++----------- tests/test_procgenerator.py | 46 +++++++++++++++++-------------------- tests/test_rdfprint.py | 30 +++++++++++------------- 3 files changed, 43 insertions(+), 56 deletions(-) diff --git a/tests/test_ext.py b/tests/test_ext.py index becd77feff..1277e12b48 100644 --- a/tests/test_ext.py +++ b/tests/test_ext.py @@ -2,6 +2,7 @@ import re from io import StringIO from pathlib import Path +from typing import Any import pytest @@ -31,20 +32,14 @@ def test_listing_deep() -> None: @needs_docker -def test_cwltool_options() -> None: - try: - opt = os.environ.get("CWLTOOL_OPTIONS") - os.environ["CWLTOOL_OPTIONS"] = "--enable-ext" - params = [ - get_data("tests/wf/listing_deep.cwl"), - get_data("tests/listing-job.yml"), - ] - assert main(params) == 0 - finally: - if opt is not None: - os.environ["CWLTOOL_OPTIONS"] = opt - else: - del os.environ["CWLTOOL_OPTIONS"] +def test_cwltool_options(monkeypatch: Any) -> None: + """Check setting options via environment variable.""" + monkeypatch.setenv("CWLTOOL_OPTIONS", "--enable-ext") + params = [ + get_data("tests/wf/listing_deep.cwl"), + get_data("tests/listing-job.yml"), + ] + assert main(params) == 0 @needs_docker diff --git a/tests/test_procgenerator.py b/tests/test_procgenerator.py index 2c0d47a86c..deba27cad3 100644 --- a/tests/test_procgenerator.py +++ b/tests/test_procgenerator.py @@ -1,4 +1,5 @@ import os +from typing import Any from cwltool.main import main @@ -6,32 +7,27 @@ @windows_needs_docker -def test_missing_enable_ext() -> None: - # Requires --enable-ext and --enable-dev - try: - opt = os.environ.get("CWLTOOL_OPTIONS") +def test_missing_enable_ext(monkeypatch: Any) -> None: + """Test missing enable-ext option fails. - if "CWLTOOL_OPTIONS" in os.environ: - del os.environ["CWLTOOL_OPTIONS"] - assert main([get_data("tests/wf/generator/zing.cwl"), "--zing", "zipper"]) == 1 + Check that a workflow that needs `--enable-ext` and + `--enable-dev` fails without those options and passes with them. + """ + monkeypatch.delenv("CWLTOOL_OPTIONS", raising=False) + assert main([get_data("tests/wf/generator/zing.cwl"), "--zing", "zipper"]) == 1 - assert ( - main( - [ - "--enable-ext", - "--enable-dev", - get_data("tests/wf/generator/zing.cwl"), - "--zing", - "zipper", - ] - ) - == 0 + assert ( + main( + [ + "--enable-ext", + "--enable-dev", + get_data("tests/wf/generator/zing.cwl"), + "--zing", + "zipper", + ] ) + == 0 + ) - os.environ["CWLTOOL_OPTIONS"] = "--enable-ext --enable-dev" - assert main([get_data("tests/wf/generator/zing.cwl"), "--zing", "zipper"]) == 0 - finally: - if opt is not None: - os.environ["CWLTOOL_OPTIONS"] = opt - elif "CWLTOOL_OPTIONS" in os.environ: - del os.environ["CWLTOOL_OPTIONS"] + monkeypatch.setenv("CWLTOOL_OPTIONS", "--enable-ext --enable-dev") + assert main([get_data("tests/wf/generator/zing.cwl"), "--zing", "zipper"]) == 0 diff --git a/tests/test_rdfprint.py b/tests/test_rdfprint.py index 308265d913..f25310662f 100644 --- a/tests/test_rdfprint.py +++ b/tests/test_rdfprint.py @@ -1,6 +1,7 @@ import os import subprocess import sys +from typing import Any from cwltool.main import main @@ -11,21 +12,16 @@ def test_rdf_print() -> None: assert main(["--print-rdf", get_data("tests/wf/hello_single_tool.cwl")]) == 0 -def test_rdf_print_unicode() -> None: +def test_rdf_print_unicode(monkeypatch: Any) -> None: """Force ASCII encoding but load UTF file with --print-rdf.""" - try: - lc_all = os.environ.get("LC_ALL", None) - os.environ["LC_ALL"] = "C" - - params = [ - sys.executable, - "-m", - "cwltool", - "--print-rdf", - get_data("tests/utf_doc_example.cwl"), - ] - - assert subprocess.check_call(params) == 0 - finally: - if lc_all: - os.environ["LC_ALL"] = lc_all + monkeypatch.setenv("LC_ALL", "C") + + params = [ + sys.executable, + "-m", + "cwltool", + "--print-rdf", + get_data("tests/utf_doc_example.cwl"), + ] + + assert subprocess.check_call(params) == 0 From efda9e35c56b149c77b442dc84057c8f771f2bd6 Mon Sep 17 00:00:00 2001 From: Rupert Nash Date: Wed, 24 Mar 2021 15:15:53 +0000 Subject: [PATCH 034/111] tests: no manual os.chdir; use util.working_directory context manager --- tests/test_examples.py | 83 ++++++++++++++------------- tests/test_singularity.py | 89 +++++++++++++++-------------- tests/test_udocker.py | 116 ++++++++++++++++++-------------------- 3 files changed, 144 insertions(+), 144 deletions(-) diff --git a/tests/test_examples.py b/tests/test_examples.py index e7b46c2497..4e9020c365 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -34,6 +34,7 @@ get_windows_safe_factory, needs_docker, windows_needs_docker, + working_directory, ) sys.argv = [""] @@ -1068,16 +1069,16 @@ def test_no_js_console(factor: str) -> None: def test_cid_file_dir(tmp_path: Path, factor: str) -> None: """Test --cidfile-dir option works.""" test_file = "cache_test_workflow.cwl" - cwd = Path.cwd() - os.chdir(tmp_path) - commands = factor.split() - commands.extend(["--cidfile-dir", str(tmp_path), get_data("tests/wf/" + test_file)]) - error_code, stdout, stderr = get_main_output(commands) - assert "completed success" in stderr - assert error_code == 0 - cidfiles_count = sum(1 for _ in tmp_path.glob("**/*")) - assert cidfiles_count == 2 - os.chdir(cwd) + with working_directory(tmp_path): + commands = factor.split() + commands.extend( + ["--cidfile-dir", str(tmp_path), get_data("tests/wf/" + test_file)] + ) + error_code, stdout, stderr = get_main_output(commands) + assert "completed success" in stderr + assert error_code == 0 + cidfiles_count = sum(1 for _ in tmp_path.glob("**/*")) + assert cidfiles_count == 2 @needs_docker @@ -1121,22 +1122,20 @@ def test_cid_file_non_existing_dir(tmp_path: Path, factor: str) -> None: def test_cid_file_w_prefix(tmp_path: Path, factor: str) -> None: """Test that --cidfile-prefix works.""" test_file = "cache_test_workflow.cwl" - cwd = Path.cwd() - os.chdir(tmp_path) - try: - commands = factor.split() - commands.extend( - [ - "--record-container-id", - "--cidfile-prefix=pytestcid", - get_data("tests/wf/" + test_file), - ] - ) - error_code, stdout, stderr = get_main_output(commands) - finally: - listing = tmp_path.iterdir() - os.chdir(cwd) - cidfiles_count = sum(1 for _ in tmp_path.glob("**/pytestcid*")) + with working_directory(tmp_path): + try: + commands = factor.split() + commands.extend( + [ + "--record-container-id", + "--cidfile-prefix=pytestcid", + get_data("tests/wf/" + test_file), + ] + ) + error_code, stdout, stderr = get_main_output(commands) + finally: + listing = tmp_path.iterdir() + cidfiles_count = sum(1 for _ in tmp_path.glob("**/pytestcid*")) assert "completed success" in stderr assert error_code == 0 assert cidfiles_count == 2, "{}/n{}".format(list(listing), stderr) @@ -1397,20 +1396,20 @@ def test_v1_0_arg_empty_prefix_separate_false() -> None: def test_scatter_output_filenames(tmp_path: Path) -> None: """If a scatter step produces identically named output then confirm that the final output is renamed correctly.""" cwd = Path.cwd() - os.chdir(tmp_path) - rtc = RuntimeContext() - rtc.outdir = str(cwd) - factory = cwltool.factory.Factory(runtime_context=rtc) - output_names = ["output.txt", "output.txt_2", "output.txt_3"] - scatter_workflow = factory.make(get_data("tests/scatter_numbers.cwl")) - result = scatter_workflow(range=3) - assert isinstance(result, dict) - assert "output" in result - - locations = sorted([element["location"] for element in result["output"]]) + with working_directory(tmp_path): + rtc = RuntimeContext() + rtc.outdir = str(cwd) + factory = cwltool.factory.Factory(runtime_context=rtc) + output_names = ["output.txt", "output.txt_2", "output.txt_3"] + scatter_workflow = factory.make(get_data("tests/scatter_numbers.cwl")) + result = scatter_workflow(range=3) + assert isinstance(result, dict) + assert "output" in result + + locations = sorted([element["location"] for element in result["output"]]) - assert ( - locations[0].endswith("output.txt") - and locations[1].endswith("output.txt_2") - and locations[2].endswith("output.txt_3") - ), f"Locations {locations} do not end with {output_names}" + assert ( + locations[0].endswith("output.txt") + and locations[1].endswith("output.txt_2") + and locations[2].endswith("output.txt_3") + ), f"Locations {locations} do not end with {output_names}" diff --git a/tests/test_singularity.py b/tests/test_singularity.py index 372aee2c6d..c016b2fb86 100644 --- a/tests/test_singularity.py +++ b/tests/test_singularity.py @@ -21,25 +21,25 @@ def test_singularity_pullfolder(tmp_path: Path) -> None: workdir = tmp_path / "working_dir_new" workdir.mkdir() - os.chdir(workdir) - pullfolder = tmp_path / "pullfolder" - pullfolder.mkdir() - env = os.environ.copy() - env["SINGULARITY_PULLFOLDER"] = str(pullfolder) - result_code, stdout, stderr = get_main_output( - [ - "--singularity", - get_data("tests/sing_pullfolder_test.cwl"), - "--message", - "hello", - ], - env=env, - ) - print(stdout) - print(stderr) - assert result_code == 0 - image = pullfolder / "debian.img" - assert image.exists() + with working_directory(workdir): + pullfolder = tmp_path / "pullfolder" + pullfolder.mkdir() + env = os.environ.copy() + env["SINGULARITY_PULLFOLDER"] = str(pullfolder) + result_code, stdout, stderr = get_main_output( + [ + "--singularity", + get_data("tests/sing_pullfolder_test.cwl"), + "--message", + "hello", + ], + env=env, + ) + print(stdout) + print(stderr) + assert result_code == 0 + image = pullfolder / "debian.img" + assert image.exists() @needs_singularity @@ -97,32 +97,37 @@ def test_singularity_incorrect_image_pull() -> None: def test_singularity_local(tmp_path: Path) -> None: workdir = tmp_path / "working_dir" workdir.mkdir() - os.chdir(workdir) - result_code, stdout, stderr = get_main_output( - [ - "--singularity", - get_data("tests/sing_pullfolder_test.cwl"), - "--message", - "hello", - ] - ) - assert result_code == 0 + with working_directory(workdir): + result_code, stdout, stderr = get_main_output( + [ + "--singularity", + get_data("tests/sing_pullfolder_test.cwl"), + "--message", + "hello", + ] + ) + assert result_code == 0 @needs_singularity_2_6 def test_singularity_docker_image_id_in_tool(tmp_path: Path) -> None: workdir = tmp_path / "working_dir" workdir.mkdir() - os.chdir(workdir) - result_code, stdout, stderr = get_main_output( - [ - "--singularity", - get_data("tests/sing_pullfolder_test.cwl"), - "--message", - "hello", - ] - ) - result_code1, stdout, stderr = get_main_output( - ["--singularity", get_data("tests/debian_image_id.cwl"), "--message", "hello"] - ) - assert result_code1 == 0 + with working_directory(workdir): + result_code, stdout, stderr = get_main_output( + [ + "--singularity", + get_data("tests/sing_pullfolder_test.cwl"), + "--message", + "hello", + ] + ) + result_code1, stdout, stderr = get_main_output( + [ + "--singularity", + get_data("tests/debian_image_id.cwl"), + "--message", + "hello", + ] + ) + assert result_code1 == 0 diff --git a/tests/test_udocker.py b/tests/test_udocker.py index fb036cf282..f8febc1a0e 100644 --- a/tests/test_udocker.py +++ b/tests/test_udocker.py @@ -8,7 +8,7 @@ import pytest from _pytest.tmpdir import TempPathFactory -from .util import get_data, get_main_output +from .util import get_data, get_main_output, working_directory LINUX = sys.platform in ("linux", "linux2") @@ -19,59 +19,57 @@ def udocker(tmp_path_factory: TempPathFactory) -> str: test_cwd = os.getcwd() test_environ = copy.copy(os.environ) docker_install_dir = str(tmp_path_factory.mktemp("udocker")) - os.chdir(docker_install_dir) - - url = "https://raw.githubusercontent.com/jorge-lip/udocker-builds/master/tarballs/udocker-1.1.4.tar.gz" - install_cmds = [ - ["curl", url, "-o", "./udocker-tarball.tgz"], - ["tar", "xzvf", "udocker-tarball.tgz", "udocker"], - [ - "bash", - "-c", - "UDOCKER_TARBALL={}/udocker-tarball.tgz ./udocker install".format( - docker_install_dir - ), - ], - ] - - test_environ["UDOCKER_DIR"] = os.path.join(docker_install_dir, ".udocker") - test_environ["HOME"] = docker_install_dir - - results = [] - for _ in range(3): - results = [subprocess.call(cmds, env=test_environ) for cmds in install_cmds] - if sum(results) == 0: - break - subprocess.call(["rm", "./udocker"]) - - assert sum(results) == 0 - - udocker_path = os.path.join(docker_install_dir, "udocker") - os.chdir(test_cwd) + with working_directory(docker_install_dir): + + url = "https://raw.githubusercontent.com/jorge-lip/udocker-builds/master/tarballs/udocker-1.1.4.tar.gz" + install_cmds = [ + ["curl", url, "-o", "./udocker-tarball.tgz"], + ["tar", "xzvf", "udocker-tarball.tgz", "udocker"], + [ + "bash", + "-c", + "UDOCKER_TARBALL={}/udocker-tarball.tgz ./udocker install".format( + docker_install_dir + ), + ], + ] + + test_environ["UDOCKER_DIR"] = os.path.join(docker_install_dir, ".udocker") + test_environ["HOME"] = docker_install_dir + + results = [] + for _ in range(3): + results = [subprocess.call(cmds, env=test_environ) for cmds in install_cmds] + if sum(results) == 0: + break + subprocess.call(["rm", "./udocker"]) + + assert sum(results) == 0 + + udocker_path = os.path.join(docker_install_dir, "udocker") + return udocker_path @pytest.mark.skipif(not LINUX, reason="LINUX only") def test_udocker_usage_should_not_write_cid_file(udocker: str, tmp_path: Path) -> None: """Confirm that no cidfile is made when udocker is used.""" - cwd = Path.cwd() - os.chdir(tmp_path) - - test_file = "tests/wf/wc-tool.cwl" - job_file = "tests/wf/wc-job.json" - error_code, stdout, stderr = get_main_output( - [ - "--debug", - "--default-container", - "debian", - "--user-space-docker-cmd=" + udocker, - get_data(test_file), - get_data(job_file), - ] - ) - cidfiles_count = sum(1 for _ in tmp_path.glob("*.cid")) - os.chdir(cwd) + with working_directory(tmp_path): + test_file = "tests/wf/wc-tool.cwl" + job_file = "tests/wf/wc-job.json" + error_code, stdout, stderr = get_main_output( + [ + "--debug", + "--default-container", + "debian", + "--user-space-docker-cmd=" + udocker, + get_data(test_file), + get_data(job_file), + ] + ) + + cidfiles_count = sum(1 for _ in tmp_path.glob("*.cid")) assert "completed success" in stderr, stderr assert cidfiles_count == 0 @@ -83,19 +81,17 @@ def test_udocker_usage_should_not_write_cid_file(udocker: str, tmp_path: Path) - ) def test_udocker_should_display_memory_usage(udocker: str, tmp_path: Path) -> None: """Confirm that memory ussage is logged even with udocker.""" - cwd = Path.cwd() - os.chdir(tmp_path) - error_code, stdout, stderr = get_main_output( - [ - "--enable-ext", - "--default-container=debian", - "--user-space-docker-cmd=" + udocker, - get_data("tests/wf/timelimit.cwl"), - "--sleep_time", - "10", - ] - ) - os.chdir(cwd) + with working_directory(tmp_path): + error_code, stdout, stderr = get_main_output( + [ + "--enable-ext", + "--default-container=debian", + "--user-space-docker-cmd=" + udocker, + get_data("tests/wf/timelimit.cwl"), + "--sleep_time", + "10", + ] + ) assert "completed success" in stderr, stderr assert "Max memory" in stderr, stderr From 420b69f605864d54cab7e59becf1de7cb82caa81 Mon Sep 17 00:00:00 2001 From: Rupert Nash Date: Wed, 31 Mar 2021 11:12:08 +0100 Subject: [PATCH 035/111] test_singularity: remove sys.argv alteration --- tests/test_singularity.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/test_singularity.py b/tests/test_singularity.py index c016b2fb86..87395bb9de 100644 --- a/tests/test_singularity.py +++ b/tests/test_singularity.py @@ -1,7 +1,6 @@ """Tests to find local Singularity image.""" import shutil import os -import sys from pathlib import Path from cwltool.main import main @@ -14,8 +13,6 @@ working_directory, ) -sys.argv = [""] - @needs_singularity_2_6 def test_singularity_pullfolder(tmp_path: Path) -> None: From 99c8f1a6c03d6cb4d26c6d8d3a0c4df6db28d807 Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" Date: Wed, 17 Mar 2021 10:53:23 +0100 Subject: [PATCH 036/111] Drop MS Windows support in favor of WSL 2 --- Makefile | 31 ++++---- README.rst | 5 +- appveyor.yml | 49 ------------ cwltool/argparser.py | 5 +- cwltool/builder.py | 10 --- cwltool/command_line_tool.py | 38 +--------- cwltool/docker.py | 22 +----- cwltool/expression.py | 11 +-- cwltool/job.py | 48 +++--------- cwltool/main.py | 9 --- cwltool/pathmapper.py | 10 +-- cwltool/process.py | 12 +-- cwltool/provenance.py | 26 +------ cwltool/sandboxjs.py | 109 ++++----------------------- cwltool/singularity.py | 24 ++---- cwltool/stdfsaccess.py | 10 --- cwltool/utils.py | 80 -------------------- setup.cfg | 4 - setup.py | 8 +- tests/test_context.py | 8 +- tests/test_dependencies.py | 5 +- tests/test_docker.py | 2 +- tests/test_docker_warning.py | 28 ------- tests/test_empty_input.py | 3 +- tests/test_examples.py | 27 ++----- tests/test_ext.py | 3 +- tests/test_fetch.py | 11 +-- tests/test_iwdr.py | 13 +--- tests/test_js_sandbox.py | 10 +-- tests/test_load_tool.py | 11 +-- tests/test_mpi.py | 6 +- tests/test_parallel.py | 11 +-- tests/test_path_checks.py | 4 +- tests/test_procgenerator.py | 6 +- tests/test_provenance.py | 3 - tests/test_rdfprint.py | 1 - tests/test_subgraph.py | 9 +-- tests/test_target.py | 4 +- tests/util.py | 32 +------- tox.ini | 17 ++--- windowsdoc.md | 142 ----------------------------------- 41 files changed, 118 insertions(+), 749 deletions(-) delete mode 100644 appveyor.yml delete mode 100644 tests/test_docker_warning.py delete mode 100644 windowsdoc.md diff --git a/Makefile b/Makefile index ff78d90812..331866f6e8 100644 --- a/Makefile +++ b/Makefile @@ -25,20 +25,19 @@ MODULE=cwltool # `SHELL=bash` doesn't work for some, so don't use BASH-isms like # `[[` conditional expressions. PYSOURCES=$(wildcard ${MODULE}/**.py tests/*.py) setup.py -DEVPKGS=diff_cover black pylint coverage pep257 pydocstyle flake8 tox\ +DEVPKGS=diff_cover black pylint pep257 pydocstyle flake8 tox tox-pyenv \ isort wheel autoflake flake8-bugbear pyupgrade bandit \ -rtest-requirements.txt -rmypy_requirements.txt DEBDEVPKGS=pep8 python-autopep8 pylint python-coverage pydocstyle sloccount \ python-flake8 python-mock shellcheck -VERSION=3.0.$(shell TZ=UTC git log --first-parent --max-count=1 \ +VERSION=3.1.$(shell TZ=UTC git log --first-parent --max-count=1 \ --format=format:%cd --date=format-local:%Y%m%d%H%M%S) mkfile_dir := $(dir $(abspath $(lastword $(MAKEFILE_LIST)))) UNAME_S=$(shell uname -s) ## all : default task -all: - pip install -e . +all: dev ## help : print this help message and exit help: Makefile @@ -47,7 +46,7 @@ help: Makefile ## install-dep : install most of the development dependencies via pip install-dep: install-dependencies -install-dependencies: +install-dependencies: FORCE pip install --upgrade $(DEVPKGS) pip install -r requirements.txt @@ -112,7 +111,7 @@ pylint: $(PYSOURCES) pylint --msg-template="{path}:{line}: [{msg_id}({symbol}), {obj}] {msg}" \ $^ -j0|| true -pylint_report.txt: ${PYSOURCES} +pylint_report.txt: $(PYSOURCES) pylint --msg-template="{path}:{line}: [{msg_id}({symbol}), {obj}] {msg}" \ $^ -j0> $@ || true @@ -143,18 +142,18 @@ diff-cover.html: coverage.xml diff-cover $^ --html-report $@ ## test : run the ${MODULE} test suite -test: $(pysources) - python setup.py test --addopts "-n auto --dist=loadfile" +test: $(PYSOURCES) + python setup.py test ## testcov : run the ${MODULE} test suite and collect coverage -testcov: $(pysources) - python setup.py test --addopts "--cov cwltool -n auto --dist=loadfile" +testcov: $(PYSOURCES) + python setup.py test --addopts "--cov --cov-config=.coveragerc --cov-report= -n auto --dist=loadfile" -sloccount.sc: ${PYSOURCES} Makefile +sloccount.sc: $(PYSOURCES) Makefile sloccount --duplicates --wide --details $^ > $@ ## sloccount : count lines of code -sloccount: ${PYSOURCES} Makefile +sloccount: $(PYSOURCES) Makefile sloccount $^ list-author-emails: @@ -162,7 +161,7 @@ list-author-emails: @git log --format='%aN,%aE' | sort -u | grep -v 'root' mypy3: mypy -mypy: $(filter-out setup.py gittagger.py,${PYSOURCES}) +mypy: $(filter-out setup.py gittagger.py,$(PYSOURCES)) if ! test -f $(shell python3 -c 'import ruamel.yaml; import os.path; print(os.path.dirname(ruamel.yaml.__file__))')/py.typed ; \ then \ rm -Rf typeshed/ruamel/yaml ; \ @@ -171,14 +170,14 @@ mypy: $(filter-out setup.py gittagger.py,${PYSOURCES}) fi # if minimally required ruamel.yaml version is 0.15.99 or greater, than the above can be removed MYPYPATH=$$MYPYPATH:typeshed mypy $^ -mypyc: ${PYSOURCES} +mypyc: $(PYSOURCES) MYPYPATH=typeshed CWLTOOL_USE_MYPYC=1 pip install --verbose -e . && pytest --ignore cwltool/schemas --basetemp ./tmp shellcheck: FORCE shellcheck build-cwl-docker.sh cwl-docker.sh release-test.sh conformance-test.sh \ cwltool-in-docker.sh -pyupgrade: ${PYSOURCES} +pyupgrade: $(PYSOURCES) pyupgrade --exit-zero-even-if-changed --py36-plus $^ release-test: FORCE @@ -192,7 +191,7 @@ release: release-test twine upload testenv2/src/${MODULE}/dist/* && \ git tag ${VERSION} && git push --tags -flake8: ${PYSOURCES} +flake8: $(PYSOURCES) flake8 $^ FORCE: diff --git a/README.rst b/README.rst index 9c39e6ef6e..815eac4dde 100644 --- a/README.rst +++ b/README.rst @@ -2,7 +2,7 @@ Common Workflow Language tool description reference implementation ================================================================== -|Linux Status| |Debian Stable package| |Debian Testing package| |Windows Status| |Coverage Status| |Downloads| +|Linux Status| |Debian Stable package| |Debian Testing package| |Coverage Status| |Downloads| .. |Linux Status| image:: https://github.com/common-workflow-language/cwltool/actions/workflows/ci-tests.yml/badge.svg?branch=main :target: https://github.com/common-workflow-language/cwltool/actions/workflows/ci-tests.yml @@ -13,9 +13,6 @@ Common Workflow Language tool description reference implementation .. |Debian Testing package| image:: https://badges.debian.net/badges/debian/testing/cwltool/version.svg :target: https://packages.debian.org/testing/cwltool -.. |Windows Status| image:: https://img.shields.io/appveyor/ci/mr-c/cwltool/main.svg?label=Windows%20builds - :target: https://ci.appveyor.com/project/mr-c/cwltool - .. |Coverage Status| image:: https://img.shields.io/codecov/c/github/common-workflow-language/cwltool.svg :target: https://codecov.io/gh/common-workflow-language/cwltool diff --git a/appveyor.yml b/appveyor.yml deleted file mode 100644 index 66771f0ad4..0000000000 --- a/appveyor.yml +++ /dev/null @@ -1,49 +0,0 @@ -version: .{build}-{branch} - -cache: - - '%LOCALAPPDATA%\pip\Cache' - -environment: - - SYSTEMROOT: "C:\\WINDOWS" - - matrix: - - PYTHON: "C:\\Python36-x64" - PYTHON_VERSION: "3.6.x" - PYTHON_ARCH: "64" - - - PYTHON: "C:\\Python37-x64" - PYTHON_VERSION: "3.7.x" - PYTHON_ARCH: "64" - - - PYTHON: "C:\\Python38-x64" - PYTHON_VERSION: "3.8.x" - PYTHON_ARCH: "64" - -install: - - ps: 'Install-Product node 0.12 x64' - - "set PATH=%PYTHON%\\Scripts;%PATH%" - - "%PYTHON%\\python.exe -m pip install -U pip setuptools^>=20.3 wheel" - - "%PYTHON%\\python.exe -m pip install -U codecov -rtest-requirements.txt pytest-xdist" - # Note the use of a `^` to escape the `>` - -build_script: - - "%PYTHON%\\python.exe -m pip install -rrequirements.txt" - - "%PYTHON%\\python.exe -m pip install -e .[deps]" - -test_script: - - | - %PYTHON%\\python.exe -m coverage run --parallel-mode -m pytest --strict -p no:cacheprovider -p no:stepwise --junit-xml=tests.xml - - "%PYTHON%\\python.exe -m coverage combine" - - "%PYTHON%\\python.exe -m coverage report" - - "%PYTHON%\\python.exe -m coverage xml" - - "%PYTHON%\\python.exe -m codecov --file coverage.xml" - -on_finish: - - ps: | - $wc = New-Object 'System.Net.WebClient' - $wc.UploadFile("https://ci.appveyor.com/api/testresults/junit/$($Env:APPVEYOR_JOB_ID)", (Resolve-Path .\tests.xml)) - -branches: - only: - - main diff --git a/cwltool/argparser.py b/cwltool/argparser.py index 89ccae4bbd..5399cd9a2c 100644 --- a/cwltool/argparser.py +++ b/cwltool/argparser.py @@ -21,7 +21,7 @@ from .process import Process, shortname from .resolver import ga4gh_tool_registries from .software_requirements import SOFTWARE_REQUIREMENTS_ENABLED -from .utils import DEFAULT_TMP_PREFIX, onWindows +from .utils import DEFAULT_TMP_PREFIX def arg_parser() -> argparse.ArgumentParser: @@ -516,14 +516,13 @@ def arg_parser() -> argparse.ArgumentParser: "--enable-color", action="store_true", help="Enable logging color (default enabled)", - default=not onWindows(), + default=True, ) colorgroup.add_argument( "--disable-color", action="store_false", dest="enable_color", help="Disable colored logging (default false)", - default=onWindows(), ) parser.add_argument( diff --git a/cwltool/builder.py b/cwltool/builder.py index 3688818e5b..d206e66352 100644 --- a/cwltool/builder.py +++ b/cwltool/builder.py @@ -37,10 +37,8 @@ CWLObjectType, CWLOutputType, aslist, - docker_windows_path_adjust, get_listing, normalizeFilesDirs, - onWindows, visit_class, ) @@ -551,14 +549,6 @@ def tostr(self, value: Union[MutableMapping[str, str], Any]) -> str: raise WorkflowException( '{} object missing "path": {}'.format(value["class"], value) ) - - # Path adjust for windows file path when passing to docker, docker accepts unix like path only - (docker_req, docker_is_req) = self.get_requirement("DockerRequirement") - if onWindows() and docker_req is not None: - # docker_req is none only when there is no dockerRequirement - # mentioned in hints and Requirement - path = docker_windows_path_adjust(value["path"]) - return path return value["path"] else: return str(value) diff --git a/cwltool/command_line_tool.py b/cwltool/command_line_tool.py index bd1fb92c6c..68af83c97c 100644 --- a/cwltool/command_line_tool.py +++ b/cwltool/command_line_tool.py @@ -1,7 +1,6 @@ """Implementation of CommandLineTool.""" import copy -from enum import Enum import hashlib import json import locale @@ -12,6 +11,7 @@ import threading import urllib import urllib.parse +from enum import Enum from functools import cmp_to_key, partial from typing import ( Any, @@ -70,17 +70,13 @@ adjustDirObjs, adjustFileObjs, aslist, - convert_pathsep_to_unix, - docker_windows_path_adjust, get_listing, normalizeFilesDirs, - onWindows, random_outdir, shared_file_lock, trim_listing, upgrade_lock, visit_class, - windows_default_container_id, ) if TYPE_CHECKING: @@ -99,21 +95,6 @@ class PathCheckingMode(Enum): RELAXED = re.compile(r".*") # Accept anything -DEFAULT_CONTAINER_MSG = """ -We are on Microsoft Windows and not all components of this CWL description have a -container specified. This means that these steps will be executed in the default container, -which is %s. - -Note, this could affect portability if this CWL description relies on non-POSIX features -or commands in this container. For best results add the following to your CWL -description's hints section: - -hints: - DockerRequirement: - dockerPull: %s -""" - - class ExpressionJob: """Job for ExpressionTools.""" @@ -224,7 +205,7 @@ def revmap_file( if "location" in f and "path" not in f: location = cast(str, f["location"]) if location.startswith("file://"): - f["path"] = convert_pathsep_to_unix(uri_file_path(location)) + f["path"] = uri_file_path(location) else: return f @@ -324,9 +305,7 @@ def check_adjust( raise ValueError( "Do not call check_adjust using a builder that doesn't have a pathmapper." ) - file_o["path"] = path = docker_windows_path_adjust( - builder.pathmapper.mapper(cast(str, file_o["location"]))[1] - ) + file_o["path"] = path = builder.pathmapper.mapper(cast(str, file_o["location"]))[1] basename = cast(str, file_o.get("basename")) dn, bn = os.path.split(path) if file_o.get("dirname") != dn: @@ -405,17 +384,6 @@ def make_job_runner(self, runtimeContext: RuntimeContext) -> Type[JobBase]: self.requirements.insert(0, dockerReq) dockerRequired = True - if ( - default_container == windows_default_container_id - and runtimeContext.use_container - and onWindows() - ): - _logger.warning( - DEFAULT_CONTAINER_MSG, - windows_default_container_id, - windows_default_container_id, - ) - if dockerReq is not None and runtimeContext.use_container: if mpiReq is not None: _logger.warning("MPIRequirement with containers is a beta feature") diff --git a/cwltool/docker.py b/cwltool/docker.py index bd5c25cfa2..99a3f3ae8c 100644 --- a/cwltool/docker.py +++ b/cwltool/docker.py @@ -20,13 +20,7 @@ from .job import ContainerCommandLineJob from .loghandler import _logger from .pathmapper import MapperEnt, PathMapper -from .utils import ( - CWLObjectType, - create_tmp_dir, - docker_windows_path_adjust, - ensure_writable, - onWindows, -) +from .utils import CWLObjectType, create_tmp_dir, ensure_writable _IMAGES = set() # type: Set[str] _IMAGES_LOCK = threading.Lock() @@ -61,14 +55,10 @@ def _get_docker_machine_mounts() -> List[str]: def _check_docker_machine_path(path: Optional[str]) -> None: if path is None: return - if onWindows(): - path = path.lower() mounts = _get_docker_machine_mounts() found = False for mount in mounts: - if onWindows(): - mount = mount.lower() if path.startswith(mount): found = True break @@ -268,7 +258,7 @@ def add_file_or_directory_volume( ) -> None: """Append volume a file/dir mapping to the runtime option list.""" if not volume.resolved.startswith("_:"): - _check_docker_machine_path(docker_windows_path_adjust(volume.resolved)) + _check_docker_machine_path(volume.resolved) self.append_volume(runtime, volume.resolved, volume.target) def add_writable_file_volume( @@ -369,9 +359,7 @@ def create_runtime( runtime = [x.replace(":ro", "") for x in runtime] runtime = [x.replace(":rw", "") for x in runtime] - runtime.append( - "--workdir=%s" % (docker_windows_path_adjust(self.builder.outdir)) - ) + runtime.append("--workdir=%s" % (self.builder.outdir)) if not user_space_docker_cmd: if not runtimeContext.no_read_only: @@ -387,9 +375,7 @@ def create_runtime( runtime.append("--log-driver=none") euid, egid = docker_vm_id() - if not onWindows(): - # MS Windows does not have getuid() or geteuid() functions - euid, egid = euid or os.geteuid(), egid or os.getgid() + euid, egid = euid or os.geteuid(), egid or os.getgid() if runtimeContext.no_match_user is False and ( euid is not None and egid is not None diff --git a/cwltool/expression.py b/cwltool/expression.py index 4eb94e4695..03aed7ccca 100644 --- a/cwltool/expression.py +++ b/cwltool/expression.py @@ -21,12 +21,7 @@ from .errors import WorkflowException from .loghandler import _logger from .sandboxjs import JavascriptException, default_timeout, execjs -from .utils import ( - CWLObjectType, - CWLOutputType, - bytes2str_in_dicts, - docker_windows_path_adjust, -) +from .utils import CWLObjectType, CWLOutputType, bytes2str_in_dicts def jshead(engine_config: List[str], rootvars: CWLObjectType) -> str: @@ -378,8 +373,8 @@ def do_eval( ) -> Optional[CWLOutputType]: runtime = cast(MutableMapping[str, Union[int, str, None]], copy.deepcopy(resources)) - runtime["tmpdir"] = docker_windows_path_adjust(tmpdir) if tmpdir else None - runtime["outdir"] = docker_windows_path_adjust(outdir) if outdir else None + runtime["tmpdir"] = tmpdir if tmpdir else None + runtime["outdir"] = outdir if outdir else None rootvars = cast( CWLObjectType, diff --git a/cwltool/job.py b/cwltool/job.py index cbc1f3ae5b..d01f8d56ba 100644 --- a/cwltool/job.py +++ b/cwltool/job.py @@ -49,11 +49,9 @@ DirectoryType, OutputCallbackType, bytes2str_in_dicts, - copytree_with_merge, create_tmp_dir, ensure_non_writable, ensure_writable, - onWindows, processes_to_kill, ) @@ -71,13 +69,10 @@ import json import os import sys -if os.name == 'posix': - try: - import subprocess32 as subprocess # type: ignore - except Exception: - import subprocess -else: - import subprocess # type: ignore +try: + import subprocess32 as subprocess # type: ignore +except Exception: + import subprocess with open(sys.argv[1], "r") as f: popen_description = json.load(f) @@ -100,15 +95,9 @@ stderr = open(stderr_path, "wb") else: stderr = sys.stderr - if os.name == 'nt': - close_fds = False - for key, value in env.items(): - env[key] = str(value) - else: - close_fds = True sp = subprocess.Popen(commands, shell=False, - close_fds=close_fds, + close_fds=True, stdin=stdin, stdout=stdout, stderr=stderr, @@ -155,15 +144,7 @@ def relink_initialworkdir( pass elif os.path.isdir(host_outdir_tgt) and not vol.resolved.startswith("_:"): shutil.rmtree(host_outdir_tgt) - if onWindows(): - # If this becomes a big issue for someone then we could - # refactor the code to process output from a running container - # and avoid all the extra IO below - if vol.type in ("File", "WritableFile"): - shutil.copy(vol.resolved, host_outdir_tgt) - elif vol.type in ("Directory", "WritableDirectory"): - copytree_with_merge(vol.resolved, host_outdir_tgt) - elif not vol.resolved.startswith("_:"): + if not vol.resolved.startswith("_:"): try: os.symlink(vol.resolved, host_outdir_tgt) except FileExistsError: @@ -544,18 +525,11 @@ def run( if vars_to_preserve: for key, value in os.environ.items(): if key in vars_to_preserve and key not in env: - # On Windows, subprocess env can't handle unicode. - env[key] = str(value) if onWindows() else value - env["HOME"] = str(self.outdir) if onWindows() else self.outdir - env["TMPDIR"] = str(self.tmpdir) if onWindows() else self.tmpdir + env[key] = value + env["HOME"] = self.outdir + env["TMPDIR"] = self.tmpdir if "PATH" not in env: - env["PATH"] = str(os.environ["PATH"]) if onWindows() else os.environ["PATH"] - if "SYSTEMROOT" not in env and "SYSTEMROOT" in os.environ: - env["SYSTEMROOT"] = ( - str(os.environ["SYSTEMROOT"]) - if onWindows() - else os.environ["SYSTEMROOT"] - ) + env["PATH"] = os.environ["PATH"] stage_files( self.pathmapper, @@ -939,7 +913,7 @@ def _job_popen( sproc = subprocess.Popen( commands, shell=False, # nosec - close_fds=not onWindows(), + close_fds=True, stdin=stdin, stdout=stdout, stderr=stderr, diff --git a/cwltool/main.py b/cwltool/main.py index 21498fbf65..eb2491a02e 100755 --- a/cwltool/main.py +++ b/cwltool/main.py @@ -89,12 +89,10 @@ CWLOutputType, adjustDirObjs, normalizeFilesDirs, - onWindows, processes_to_kill, trim_listing, versionstring, visit_class, - windows_default_container_id, ) from .workflow import Workflow @@ -917,13 +915,6 @@ def main( else: runtimeContext = runtimeContext.copy() - # If on Windows platform, a default Docker Container is used if not - # explicitely provided by user - if onWindows() and not runtimeContext.default_container: - # This docker image is a minimal alpine image with bash installed - # (size 6 mb). source: https://github.com/frol/docker-alpine-bash - runtimeContext.default_container = windows_default_container_id - # If caller parsed its own arguments, it may not include every # cwltool option, so fill in defaults to avoid crashing when # dereferencing them in args. diff --git a/cwltool/pathmapper.py b/cwltool/pathmapper.py index 269d3ce9e5..c89fac5f4f 100644 --- a/cwltool/pathmapper.py +++ b/cwltool/pathmapper.py @@ -12,7 +12,7 @@ from .loghandler import _logger from .stdfsaccess import abspath -from .utils import CWLObjectType, convert_pathsep_to_unix, dedup, downloadHttpFile +from .utils import CWLObjectType, dedup, downloadHttpFile MapperEnt = collections.namedtuple( "MapperEnt", ["resolved", "target", "type", "staged"] @@ -95,11 +95,9 @@ def visit( staged: bool = False, ) -> None: stagedir = cast(Optional[str], obj.get("dirname")) or stagedir - tgt = convert_pathsep_to_unix( - os.path.join( - stagedir, - cast(str, obj["basename"]), - ) + tgt = os.path.join( + stagedir, + cast(str, obj["basename"]), ) if obj["location"] in self._pathmap: return diff --git a/cwltool/process.py b/cwltool/process.py index a68414ad6b..8e016ddfbf 100644 --- a/cwltool/process.py +++ b/cwltool/process.py @@ -67,11 +67,9 @@ adjustDirObjs, aslist, cmp_like_py2, - copytree_with_merge, ensure_writable, get_listing, normalizeFilesDirs, - onWindows, random_outdir, visit_class, ) @@ -293,15 +291,7 @@ def stage_files( os.makedirs(os.path.dirname(entry.target)) if entry.type in ("File", "Directory") and os.path.exists(entry.resolved): if symlink: # Use symlink func if allowed - if onWindows(): - if entry.type == "File": - shutil.copy(entry.resolved, entry.target) - elif entry.type == "Directory": - if os.path.exists(entry.target) and os.path.isdir(entry.target): - shutil.rmtree(entry.target) - copytree_with_merge(entry.resolved, entry.target) - else: - os.symlink(entry.resolved, entry.target) + os.symlink(entry.resolved, entry.target) elif stage_func is not None: stage_func(entry.resolved, entry.target) elif ( diff --git a/cwltool/provenance.py b/cwltool/provenance.py index b2344aa6f9..3de641e8c3 100644 --- a/cwltool/provenance.py +++ b/cwltool/provenance.py @@ -4,6 +4,7 @@ import datetime import hashlib import os +import pwd import re import shutil import tempfile @@ -63,20 +64,10 @@ CWLOutputType, create_tmp_dir, local_path, - onWindows, posix_path, versionstring, ) -# imports needed for retrieving user data -if onWindows(): - import ctypes # pylint: disable=unused-import -else: - try: - import pwd # pylint: disable=unused-import - except ImportError: - pass - if TYPE_CHECKING: from .command_line_tool import ( # pylint: disable=unused-import CommandLineTool, @@ -89,16 +80,7 @@ def _whoami() -> Tuple[str, str]: """Return the current operating system account as (username, fullname).""" username = getuser() try: - if onWindows(): - get_user_name = ctypes.windll.secur32.GetUserNameExW # type: ignore - size = ctypes.pointer(ctypes.c_ulong(0)) - get_user_name(3, None, size) - - name_buffer = ctypes.create_unicode_buffer(size.contents.value) - get_user_name(3, name_buffer, size) - fullname = str(name_buffer.value) - else: - fullname = pwd.getpwuid(os.getuid())[4].split(",")[0] + fullname = pwd.getpwuid(os.getuid())[4].split(",")[0] except (KeyError, IndexError): fullname = username @@ -349,7 +331,6 @@ def _initialize_bagit(self) -> None: self.self_check() bagit = os.path.join(self.folder, "bagit.txt") # encoding: always UTF-8 (although ASCII would suffice here) - # newline: ensure LF also on Windows with open(bagit, "w", encoding=ENCODING, newline="\n") as bag_it_file: # TODO: \n or \r\n ? bag_it_file.write("BagIt-Version: 0.97\n") @@ -424,7 +405,6 @@ def write_bag_file( bag_file = WritableBagFile(self, path) if encoding is not None: # encoding: match Tag-File-Character-Encoding: UTF-8 - # newline: ensure LF also on Windows return TextIOWrapper( cast(BinaryIO, bag_file), encoding=encoding, newline="\n" ) @@ -831,7 +811,6 @@ def add_data_file( os.rename(tmp.name, path) # Relative posix path - # (to avoid \ on Windows) rel_path = posix_path(os.path.relpath(path, self.folder)) # Register in bagit checksum @@ -884,7 +863,6 @@ def add_to_manifest(self, rel_path: str, checksums: Dict[str, str]) -> None: # existence in bagged_size above manifestpath = os.path.join(self.folder, f"{manifest}-{method.lower()}.txt") # encoding: match Tag-File-Character-Encoding: UTF-8 - # newline: ensure LF also on Windows with open( manifestpath, "a", encoding=ENCODING, newline="\n" ) as checksum_file: diff --git a/cwltool/sandboxjs.py b/cwltool/sandboxjs.py index 778e3af3cf..a677ba0dd9 100644 --- a/cwltool/sandboxjs.py +++ b/cwltool/sandboxjs.py @@ -3,11 +3,9 @@ import errno import json import os -import queue import re import select import subprocess # nosec -import sys import threading from io import BytesIO from typing import List, Optional, Tuple, cast @@ -16,7 +14,7 @@ from schema_salad.utils import json_dumps from .loghandler import _logger -from .utils import CWLOutputType, onWindows, processes_to_kill +from .utils import CWLOutputType, processes_to_kill class JavascriptException(Exception): @@ -180,7 +178,7 @@ def exec_js_process( else: nodejs = localdata.procs.get(js_engine) - if nodejs is None or nodejs.poll() is not None or onWindows(): + if nodejs is None or nodejs.poll() is not None: res = resource_stream(__name__, js_engine) js_engine_code = res.read().decode("utf-8") @@ -226,95 +224,20 @@ def process_finished() -> bool: PROCESS_FINISHED_STR ) and stderr_buf.getvalue().decode("utf-8").endswith(PROCESS_FINISHED_STR) - # On windows system standard input/output are not handled properly by select module - # (modules like pywin32, msvcrt, gevent don't work either) - if sys.platform == "win32": - READ_BYTES_SIZE = 512 - - # creating queue for reading from a thread to queue - input_queue = queue.Queue() - output_queue = queue.Queue() - error_queue = queue.Queue() - - # To tell threads that output has ended and threads can safely exit - no_more_output = threading.Lock() - no_more_output.acquire() - no_more_error = threading.Lock() - no_more_error.acquire() - - # put constructed command to input queue which then will be passed to nodejs's stdin - def put_input(input_queue): - while True: - buf = stdin_buf.read(READ_BYTES_SIZE) - if buf: - input_queue.put(buf) - else: - break - - # get the output from nodejs's stdout and continue till output ends - def get_output(output_queue): - while not no_more_output.acquire(False): - buf = os.read(nodejs.stdout.fileno(), READ_BYTES_SIZE) - if buf: - output_queue.put(buf) - - # get the output from nodejs's stderr and continue till error output ends - def get_error(error_queue): - while not no_more_error.acquire(False): - buf = os.read(nodejs.stderr.fileno(), READ_BYTES_SIZE) + while not process_finished() and timer.is_alive(): + rready, wready, _ = select.select(rselect, wselect, []) + try: + if nodejs.stdin in wready: + buf = stdin_buf.read(select.PIPE_BUF) if buf: - error_queue.put(buf) - - # Threads managing nodejs.stdin, nodejs.stdout and nodejs.stderr respectively - input_thread = threading.Thread(target=put_input, args=(input_queue,)) - input_thread.daemon = True - input_thread.start() - output_thread = threading.Thread(target=get_output, args=(output_queue,)) - output_thread.daemon = True - output_thread.start() - error_thread = threading.Thread(target=get_error, args=(error_queue,)) - error_thread.daemon = True - error_thread.start() - - finished = False - - while not finished and timer.is_alive(): - try: - if nodejs.stdin in wselect: - if not input_queue.empty(): - os.write(nodejs.stdin.fileno(), input_queue.get()) - elif not input_thread.is_alive(): - wselect = [] - if nodejs.stdout in rselect: - if not output_queue.empty(): - stdout_buf.write(output_queue.get()) - - if nodejs.stderr in rselect: - if not error_queue.empty(): - stderr_buf.write(error_queue.get()) - - if process_finished() and error_queue.empty() and output_queue.empty(): - finished = True - no_more_output.release() - no_more_error.release() - except OSError: - break - - else: - while not process_finished() and timer.is_alive(): - rready, wready, _ = select.select(rselect, wselect, []) - try: - if nodejs.stdin in wready: - buf = stdin_buf.read(select.PIPE_BUF) + os.write(nodejs.stdin.fileno(), buf) + for pipes in ((nodejs.stdout, stdout_buf), (nodejs.stderr, stderr_buf)): + if pipes[0] in rready: + buf = os.read(pipes[0].fileno(), select.PIPE_BUF) if buf: - os.write(nodejs.stdin.fileno(), buf) - for pipes in ((nodejs.stdout, stdout_buf), (nodejs.stderr, stderr_buf)): - if pipes[0] in rready: - buf = os.read(pipes[0].fileno(), select.PIPE_BUF) - if buf: - pipes[1].write(buf) - except OSError: - break + pipes[1].write(buf) + except OSError: + break timer.cancel() stdin_buf.close() @@ -330,10 +253,6 @@ def get_error(error_queue): returncode = nodejs.returncode else: returncode = 0 - # On windows currently a new instance of nodejs process is used due to - # problem with blocking on read operation on windows - if onWindows(): - nodejs.kill() return returncode, stdoutdata.decode("utf-8"), stderrdata.decode("utf-8") diff --git a/cwltool/singularity.py b/cwltool/singularity.py index 42b03e45d0..29646765bd 100644 --- a/cwltool/singularity.py +++ b/cwltool/singularity.py @@ -23,13 +23,7 @@ from .job import ContainerCommandLineJob from .loghandler import _logger from .pathmapper import MapperEnt, PathMapper -from .utils import ( - CWLObjectType, - create_tmp_dir, - docker_windows_path_adjust, - ensure_non_writable, - ensure_writable, -) +from .utils import CWLObjectType, create_tmp_dir, ensure_non_writable, ensure_writable _USERNS = None # type: Optional[bool] _SINGULARITY_VERSION = "" @@ -283,8 +277,8 @@ def append_volume( runtime.append("--bind") runtime.append( "{}:{}:{}".format( - docker_windows_path_adjust(source), - docker_windows_path_adjust(target), + source, + target, "rw" if writable else "ro", ) ) @@ -396,7 +390,7 @@ def create_runtime( runtime.append("--home") runtime.append( "{}:{}".format( - docker_windows_path_adjust(os.path.realpath(self.outdir)), + os.path.realpath(self.outdir), self.builder.outdir, ) ) @@ -404,17 +398,13 @@ def create_runtime( runtime.append("--bind") runtime.append( "{}:{}:rw".format( - docker_windows_path_adjust(os.path.realpath(self.outdir)), + os.path.realpath(self.outdir), self.builder.outdir, ) ) runtime.append("--bind") tmpdir = "/tmp" # nosec - runtime.append( - "{}:{}:rw".format( - docker_windows_path_adjust(os.path.realpath(self.tmpdir)), tmpdir - ) - ) + runtime.append("{}:{}:rw".format(os.path.realpath(self.tmpdir), tmpdir)) self.add_volumes( self.pathmapper, @@ -433,7 +423,7 @@ def create_runtime( ) runtime.append("--pwd") - runtime.append("%s" % (docker_windows_path_adjust(self.builder.outdir))) + runtime.append(self.builder.outdir) if runtime_context.custom_net: raise UnsupportedRequirement( diff --git a/cwltool/stdfsaccess.py b/cwltool/stdfsaccess.py index c4c5dad52e..0c8eea15d5 100644 --- a/cwltool/stdfsaccess.py +++ b/cwltool/stdfsaccess.py @@ -7,8 +7,6 @@ from schema_salad.ref_resolver import file_uri, uri_file_path -from .utils import onWindows - def abspath(src: str, basedir: str) -> str: if src.startswith("file://"): @@ -64,11 +62,3 @@ def join(self, path, *paths): # type: (str, *str) -> str def realpath(self, path: str) -> str: return os.path.realpath(path) - - # On windows os.path.realpath appends unecessary Drive, here we would avoid that - def docker_compatible_realpath(self, path: str) -> str: - if onWindows(): - if path.startswith("/"): - return path - return "/" + path - return self.realpath(path) diff --git a/cwltool/utils.py b/cwltool/utils.py index 42a012ef96..ef2c0d5d83 100644 --- a/cwltool/utils.py +++ b/cwltool/utils.py @@ -2,7 +2,6 @@ import collections import os -import platform import random import shutil import stat @@ -53,8 +52,6 @@ CONTENT_LIMIT = 64 * 1024 -windows_default_container_id = "frolvlad/alpine-bash" - DEFAULT_TMP_PREFIX = tempfile.gettempdir() + os.path.sep processes_to_kill = collections.deque() # type: Deque[subprocess.Popen[str]] @@ -145,83 +142,6 @@ def copytree_with_merge(src: str, dst: str) -> None: shutil.copy2(spath, dpath) -def docker_windows_path_adjust(path: str) -> str: - r""" - Adjust only windows paths for Docker. - - The docker run command treats them as unix paths. - - Example: 'C:\Users\foo to /C/Users/foo (Docker for Windows) or /c/Users/foo - (Docker toolbox). - """ - if onWindows(): - split = path.split(":") - if len(split) == 2: - if platform.win32_ver()[0] in ("7", "8"): - # Docker toolbox uses lowecase windows Drive letters - split[0] = split[0].lower() - else: - split[0] = split[0].capitalize() - # Docker for Windows uses uppercase windows Drive letters - path = ":".join(split) - path = path.replace(":", "").replace("\\", "/") - return path if path[0] == "/" else "/" + path - return path - - -def docker_windows_reverse_path_adjust(path: str) -> str: - r""" - Change docker path (only on windows os) appropriately back to Windows path. - - Example: /C/Users/foo to C:\Users\foo - """ - if path is not None and onWindows(): - if path[0] == "/": - path = path[1:] - else: - raise ValueError("not a docker path") - splitpath = path.split("/") - splitpath[0] = splitpath[0] + ":" - return "\\".join(splitpath) - return path - - -def docker_windows_reverse_fileuri_adjust(fileuri: str) -> str: - r""" - Convert fileuri to be MS Windows comptabile, if needed. - - On docker in windows fileuri do not contain : in path - To convert this file uri to windows compatible add : after drive letter, - so file:///E/var becomes file:///E:/var - """ - if fileuri is not None and onWindows(): - if urllib.parse.urlsplit(fileuri).scheme == "file": - filesplit = fileuri.split("/") - if filesplit[3][-1] != ":": - filesplit[3] = filesplit[3] + ":" - return "/".join(filesplit) - return fileuri - raise ValueError("not a file URI") - return fileuri - - -def onWindows() -> bool: - """Check if we are on Windows OS.""" - return os.name == "nt" - - -def convert_pathsep_to_unix(path: str) -> str: - """ - Convert path seperators to unix style. - - On windows os.path.join would use backslash to join path, since we would - use these paths in Docker we would convert it to use forward slashes: / - """ - if path is not None and onWindows(): - return path.replace("\\", "/") - return path - - def cmp_like_py2(dict1: Dict[str, Any], dict2: Dict[str, Any]) -> int: """ Compare in the same manner as Python2. diff --git a/setup.cfg b/setup.cfg index cd4a1c1167..4ead715c27 100644 --- a/setup.cfg +++ b/setup.cfg @@ -7,10 +7,6 @@ extend-ignore = E203, W503 [aliases] test=pytest -[tool:pytest] -addopts=--ignore cwltool/schemas --basetemp ./tmp -testpaths = tests - [tool:isort] multi_line_output = 3 include_trailing_comma = True diff --git a/setup.py b/setup.py index 43f711de12..1ad9a6674c 100755 --- a/setup.py +++ b/setup.py @@ -76,7 +76,7 @@ setup( name="cwltool", - version="3.0", + version="3.1", description="Common workflow language reference implementation", long_description=open(README).read(), long_description_content_type="text/x-rst", @@ -141,12 +141,6 @@ "Operating System :: MacOS :: MacOS X", "Operating System :: POSIX", "Operating System :: POSIX :: Linux", - "Operating System :: OS Independent", - "Operating System :: Microsoft :: Windows", - "Operating System :: Microsoft :: Windows :: Windows 10", - "Operating System :: Microsoft :: Windows :: Windows 8.1", - # 'Operating System :: Microsoft :: Windows :: Windows 8', # not tested - # 'Operating System :: Microsoft :: Windows :: Windows 7', # not tested "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", diff --git a/tests/test_context.py b/tests/test_context.py index 2dd517186c..7542160482 100644 --- a/tests/test_context.py +++ b/tests/test_context.py @@ -1,14 +1,14 @@ import subprocess +import sys from cwltool.context import RuntimeContext +from cwltool.factory import Factory -from .util import get_data, get_windows_safe_factory, windows_needs_docker +from .util import get_data -@windows_needs_docker def test_replace_default_stdout_stderr() -> None: """Test our ability to replace the default stdout/err.""" - import sys # break stdout & stderr original_stdout = sys.stdout @@ -20,7 +20,7 @@ def test_replace_default_stdout_stderr() -> None: runtime_context = RuntimeContext() runtime_context.default_stdout = subprocess.DEVNULL # type: ignore runtime_context.default_stderr = subprocess.DEVNULL # type: ignore - factory = get_windows_safe_factory(runtime_context=runtime_context) + factory = Factory(None, None, runtime_context) echo = factory.make(get_data("tests/echo.cwl")) assert echo(inp="foo") == {"out": "foo\n"} diff --git a/tests/test_dependencies.py b/tests/test_dependencies.py index 207b81ca55..67a3219840 100644 --- a/tests/test_dependencies.py +++ b/tests/test_dependencies.py @@ -1,12 +1,10 @@ import os from shutil import which from types import ModuleType -from typing import Any, Optional +from typing import Optional import pytest -from cwltool.utils import onWindows - from .util import get_data, get_main_output, needs_docker deps = None # type: Optional[ModuleType] @@ -26,7 +24,6 @@ def test_biocontainers() -> None: assert error_code == 0 -@pytest.mark.skipif(onWindows(), reason="bioconda currently not working on MS Windows") @pytest.mark.skipif(not deps, reason="galaxy-lib is not installed") def test_bioconda() -> None: wflow = get_data("tests/seqtk_seq.cwl") diff --git a/tests/test_docker.py b/tests/test_docker.py index 5093353735..57f2075e5b 100644 --- a/tests/test_docker.py +++ b/tests/test_docker.py @@ -1,6 +1,6 @@ """Tests for docker engine.""" -from shutil import which from pathlib import Path +from shutil import which from cwltool.main import main diff --git a/tests/test_docker_warning.py b/tests/test_docker_warning.py deleted file mode 100644 index 3aec636096..0000000000 --- a/tests/test_docker_warning.py +++ /dev/null @@ -1,28 +0,0 @@ -from typing import Any, cast - -import pytest -from ruamel.yaml.comments import CommentedMap -from schema_salad.sourceline import cmap - -from cwltool import command_line_tool -from cwltool.context import LoadingContext, RuntimeContext -from cwltool.utils import onWindows, windows_default_container_id - - -@pytest.mark.skip(not onWindows(), reason="MS Windows only") # type: ignore -def test_default_docker_warning(mocker: Any) -> None: - """Check warning when default docker Container is used on Windows.""" - mocker.patch("cwltool.command_line_tool._logger") - - tool = command_line_tool.CommandLineTool( - cast(CommentedMap, cmap({"inputs": [], "outputs": []})), LoadingContext() - ) - tool.make_job_runner( - RuntimeContext({"find_default_container": lambda x: "frolvlad/alpine-bash"}) - ) - - command_line_tool._logger.warning.assert_called_with( # type: ignore - command_line_tool.DEFAULT_CONTAINER_MSG, - windows_default_container_id, - windows_default_container_id, - ) diff --git a/tests/test_empty_input.py b/tests/test_empty_input.py index a29e03c26b..fc68600840 100644 --- a/tests/test_empty_input.py +++ b/tests/test_empty_input.py @@ -3,10 +3,9 @@ from cwltool.main import main -from .util import get_data, windows_needs_docker +from .util import get_data -@windows_needs_docker def test_empty_input(tmp_path: Path) -> None: """Affirm that an empty input works.""" empty_json = "{}" diff --git a/tests/test_examples.py b/tests/test_examples.py index 4e9020c365..f3de50c146 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -26,16 +26,9 @@ from cwltool.main import main from cwltool.process import CWL_IANA from cwltool.sandboxjs import JavascriptException -from cwltool.utils import CWLObjectType, dedup, onWindows - -from .util import ( - get_data, - get_main_output, - get_windows_safe_factory, - needs_docker, - windows_needs_docker, - working_directory, -) +from cwltool.utils import CWLObjectType, dedup + +from .util import get_data, get_main_output, needs_docker, working_directory sys.argv = [""] @@ -295,9 +288,8 @@ def test_expression_interpolate_escapebehavior( ) -@windows_needs_docker def test_factory() -> None: - factory = get_windows_safe_factory() + factory = cwltool.factory.Factory() echo = factory.make(get_data("tests/echo.cwl")) assert echo(inp="foo") == {"out": "foo\n"} @@ -1237,7 +1229,7 @@ def test_issue_740_fixed(tmp_path: Path, factor: str) -> None: def test_compute_checksum() -> None: runtime_context = RuntimeContext() runtime_context.compute_checksum = True - runtime_context.use_container = onWindows() + runtime_context.use_container = False factory = cwltool.factory.Factory(runtime_context=runtime_context) echo = factory.make(get_data("tests/wf/cat-tool.cwl")) output = echo( @@ -1271,7 +1263,6 @@ def test_no_compute_chcksum(tmp_path: Path, factor: str) -> None: assert "checksum" not in stdout -@pytest.mark.skipif(onWindows(), reason="udocker is Linux/macOS only") @pytest.mark.parametrize("factor", test_factors) def test_bad_userspace_runtime(factor: str) -> None: test_file = "tests/wf/wc-tool.cwl" @@ -1290,7 +1281,6 @@ def test_bad_userspace_runtime(factor: str) -> None: assert error_code == 1 -@windows_needs_docker @pytest.mark.parametrize("factor", test_factors) def test_bad_basecommand(factor: str) -> None: test_file = "tests/wf/missing-tool.cwl" @@ -1323,7 +1313,6 @@ def test_v1_0_position_expression(factor: str) -> None: assert error_code == 1 -@windows_needs_docker @pytest.mark.parametrize("factor", test_factors) def test_optional_numeric_output_0(factor: str) -> None: test_file = "tests/wf/optional-numerical-output-0.cwl" @@ -1337,7 +1326,6 @@ def test_optional_numeric_output_0(factor: str) -> None: @pytest.mark.parametrize("factor", test_factors) -@windows_needs_docker def test_env_filtering(factor: str) -> None: test_file = "tests/env.cwl" commands = factor.split() @@ -1367,9 +1355,7 @@ def test_env_filtering(factor: str) -> None: assert "completed success" in stderr, (error_code, stdout, stderr) assert error_code == 0, (error_code, stdout, stderr) - if onWindows(): - target = 5 - elif sh_name == "dash": + if sh_name == "dash": target = 4 else: # bash adds "SHLVL" and "_" environment variables target = 6 @@ -1383,7 +1369,6 @@ def test_env_filtering(factor: str) -> None: assert result == target, (error_code, sh_name, sh_name_err, details, stdout, stderr) -@windows_needs_docker def test_v1_0_arg_empty_prefix_separate_false() -> None: test_file = "tests/arg-empty-prefix-separate-false.cwl" error_code, stdout, stderr = get_main_output( diff --git a/tests/test_ext.py b/tests/test_ext.py index 1277e12b48..d8c4bab436 100644 --- a/tests/test_ext.py +++ b/tests/test_ext.py @@ -9,7 +9,7 @@ import cwltool.process from cwltool.main import main -from .util import get_data, needs_docker, windows_needs_docker +from .util import get_data, needs_docker @needs_docker @@ -273,7 +273,6 @@ def test_require_prefix_workreuse(tmp_path: Path) -> None: assert main(["--enable-ext", get_data("tests/wf/workreuse-fail.cwl")]) != 0 -@windows_needs_docker def test_require_prefix_timelimit() -> None: assert main(["--enable-ext", get_data("tests/wf/timelimit.cwl")]) == 0 assert main([get_data("tests/wf/timelimit.cwl")]) != 0 diff --git a/tests/test_fetch.py b/tests/test_fetch.py index 2923ed2dcc..2c06bbfc1b 100644 --- a/tests/test_fetch.py +++ b/tests/test_fetch.py @@ -12,7 +12,6 @@ from cwltool.load_tool import load_tool from cwltool.main import main from cwltool.resolver import resolve_local -from cwltool.utils import onWindows from cwltool.workflow import default_make_tool from .util import get_data, working_directory @@ -82,16 +81,10 @@ def test_resolver(d: Any, a: str) -> str: ] -def norm(uri: str) -> str: - if onWindows(): - return uri.lower() - return uri - - @pytest.mark.parametrize("path,expected_path", path_fragments) def test_resolve_local(path: str, expected_path: str) -> None: with working_directory(root): - expected = norm(root.as_uri() + expected_path) + expected = root.as_uri() + expected_path resolved = resolve_local(None, path) assert resolved - assert norm(resolved) == expected + assert resolved == expected diff --git a/tests/test_iwdr.py b/tests/test_iwdr.py index fe4fcbfd7a..22a0670c42 100644 --- a/tests/test_iwdr.py +++ b/tests/test_iwdr.py @@ -1,22 +1,15 @@ from pathlib import Path from typing import Any +from cwltool.factory import Factory from cwltool.main import main -from .util import ( - get_data, - get_main_output, - get_windows_safe_factory, - needs_docker, - needs_singularity, - windows_needs_docker, -) +from .util import get_data, get_main_output, needs_docker, needs_singularity -@windows_needs_docker def test_newline_in_entry() -> None: """Files in a InitialWorkingDirectory are created with a newline character.""" - factory = get_windows_safe_factory() + factory = Factory() echo = factory.make(get_data("tests/wf/iwdr-entry.cwl")) assert echo(message="hello") == {"out": "CONFIGVAR=hello\n"} diff --git a/tests/test_js_sandbox.py b/tests/test_js_sandbox.py index 21a65b468a..4efc302cd2 100644 --- a/tests/test_js_sandbox.py +++ b/tests/test_js_sandbox.py @@ -3,9 +3,9 @@ import pytest from cwltool import sandboxjs -from cwltool.utils import onWindows +from cwltool.factory import Factory -from .util import get_data, get_windows_safe_factory, windows_needs_docker +from .util import get_data node_versions = [ ("v0.8.26\n", False), @@ -24,18 +24,14 @@ def test_node_version(version: str, supported: bool, mocker: Any) -> None: assert sandboxjs.check_js_threshold_version("node") == supported -@windows_needs_docker def test_value_from_two_concatenated_expressions() -> None: - factory = get_windows_safe_factory() + factory = Factory() echo = factory.make(get_data("tests/wf/vf-concat.cwl")) file = {"class": "File", "location": get_data("tests/wf/whale.txt")} assert echo(file1=file) == {"out": "a string\n"} -@pytest.mark.skipif( - onWindows(), reason="Caching processes for windows is not supported." -) def test_caches_js_processes(mocker: Any) -> None: sandboxjs.exec_js_process("7", context="{}") diff --git a/tests/test_load_tool.py b/tests/test_load_tool.py index 629ed6c2aa..be2f9d7c3d 100644 --- a/tests/test_load_tool.py +++ b/tests/test_load_tool.py @@ -9,11 +9,9 @@ from cwltool.update import INTERNAL_VERSION from cwltool.utils import CWLObjectType -from .test_fetch import norm -from .util import get_data, windows_needs_docker +from .util import get_data -@windows_needs_docker def test_check_version() -> None: """ It is permitted to load without updating, but not execute. @@ -48,17 +46,16 @@ def test_use_metadata() -> None: def test_checklink_outputSource() -> None: """Is outputSource resolved correctly independent of value of do_validate.""" outsrc = ( - norm(Path(get_data("tests/wf/1st-workflow.cwl")).as_uri()) - + "#argument/classfile" + Path(get_data("tests/wf/1st-workflow.cwl")).as_uri() + "#argument/classfile" ) loadingContext = LoadingContext({"do_validate": True}) tool = load_tool(get_data("tests/wf/1st-workflow.cwl"), loadingContext) - assert norm(tool.tool["outputs"][0]["outputSource"]) == outsrc + assert tool.tool["outputs"][0]["outputSource"] == outsrc loadingContext = LoadingContext({"do_validate": False}) tool = load_tool(get_data("tests/wf/1st-workflow.cwl"), loadingContext) - assert norm(tool.tool["outputs"][0]["outputSource"]) == outsrc + assert tool.tool["outputs"][0]["outputSource"] == outsrc def test_load_graph_fragment() -> None: diff --git a/tests/test_mpi.py b/tests/test_mpi.py index 35061e07a4..72ff81eade 100644 --- a/tests/test_mpi.py +++ b/tests/test_mpi.py @@ -20,7 +20,7 @@ from cwltool.main import main from cwltool.mpi import MpiConfig, MPIRequirementName -from .util import get_data, windows_needs_docker, working_directory +from .util import get_data, working_directory def test_mpi_conf_defaults() -> None: @@ -134,7 +134,6 @@ def test_fake_mpi_config(self, fake_mpi_conf: str) -> None: assert conf_obj.default_nproc == 1 assert conf_obj.extra_flags == ["--no-fail"] - @windows_needs_docker def test_simple_mpi_tool(self, fake_mpi_conf: str, tmp_path: Path) -> None: stdout = StringIO() stderr = StringIO() @@ -153,7 +152,6 @@ def test_simple_mpi_tool(self, fake_mpi_conf: str, tmp_path: Path) -> None: pids = [int(line) for line in pidfile] assert len(pids) == 2 - @windows_needs_docker def test_simple_mpi_nproc_expr(self, fake_mpi_conf: str, tmp_path: Path) -> None: np = 4 input_file = make_processes_input(np, tmp_path) @@ -174,7 +172,6 @@ def test_simple_mpi_nproc_expr(self, fake_mpi_conf: str, tmp_path: Path) -> None pids = [int(line) for line in pidfile] assert len(pids) == np - @windows_needs_docker def test_mpi_workflow(self, fake_mpi_conf: str, tmp_path: Path) -> None: np = 3 input_file = make_processes_input(np, tmp_path) @@ -195,7 +192,6 @@ def test_mpi_workflow(self, fake_mpi_conf: str, tmp_path: Path) -> None: lc = int(lc_file.read()) assert lc == np - @windows_needs_docker def test_environment( self, fake_mpi_conf: str, tmp_path: Path, monkeypatch: Any ) -> None: diff --git a/tests/test_parallel.py b/tests/test_parallel.py index 5995f6c27f..e989e4f0a9 100644 --- a/tests/test_parallel.py +++ b/tests/test_parallel.py @@ -3,30 +3,27 @@ from cwltool.context import RuntimeContext from cwltool.executors import MultithreadedJobExecutor +from cwltool.factory import Factory -from .util import get_data, get_windows_safe_factory, windows_needs_docker +from .util import get_data -@windows_needs_docker def test_sequential_workflow(tmp_path: Path) -> None: test_file = "tests/wf/count-lines1-wf.cwl" executor = MultithreadedJobExecutor() runtime_context = RuntimeContext() runtime_context.outdir = str(tmp_path) runtime_context.select_resources = executor.select_resources - factory = get_windows_safe_factory( - executor=executor, runtime_context=runtime_context - ) + factory = Factory(executor, None, runtime_context) echo = factory.make(get_data(test_file)) file_contents = {"class": "File", "location": get_data("tests/wf/whale.txt")} assert echo(file1=file_contents) == {"count_output": 16} -@windows_needs_docker def test_scattered_workflow() -> None: test_file = "tests/wf/scatter-wf4.cwl" job_file = "tests/wf/scatter-job2.json" - factory = get_windows_safe_factory(executor=MultithreadedJobExecutor()) + factory = Factory(MultithreadedJobExecutor()) echo = factory.make(get_data(test_file)) with open(get_data(job_file)) as job: assert echo(**json.load(job)) == {"out": ["foo one three", "foo two four"]} diff --git a/tests/test_path_checks.py b/tests/test_path_checks.py index 1783a10ee9..e8a300fed5 100644 --- a/tests/test_path_checks.py +++ b/tests/test_path_checks.py @@ -1,7 +1,7 @@ -# -*- coding: utf-8 -*- -import pytest from pathlib import Path +import pytest + from cwltool.main import main from .util import needs_docker diff --git a/tests/test_procgenerator.py b/tests/test_procgenerator.py index deba27cad3..b7b0f3ebd9 100644 --- a/tests/test_procgenerator.py +++ b/tests/test_procgenerator.py @@ -1,12 +1,12 @@ -import os +"""ProcessGenerator related tests.""" + from typing import Any from cwltool.main import main -from .util import get_data, windows_needs_docker +from .util import get_data -@windows_needs_docker def test_missing_enable_ext(monkeypatch: Any) -> None: """Test missing enable-ext option fails. diff --git a/tests/test_provenance.py b/tests/test_provenance.py index 4f64a564dc..3d2d5d8082 100644 --- a/tests/test_provenance.py +++ b/tests/test_provenance.py @@ -306,7 +306,6 @@ def _arcp2file(base_path: Path, uri: str) -> Path: ), "arcp URI must be local to the research object" path = parsed.path[1:] # Strip first / - # Convert to local path, in case it uses \ on Windows return base_path / Path(path) @@ -604,7 +603,6 @@ def test_writable_string(research_object: ResearchObject) -> None: with research_object.write_bag_file("file.txt") as fh: assert fh.writable() fh.write("Hello\n") - # TODO: Check Windows does not modify \n to \r\n here sha1 = os.path.join(research_object.folder, "tagmanifest-sha1.txt") assert os.path.isfile(sha1) @@ -649,7 +647,6 @@ def test_data(research_object: ResearchObject) -> None: with research_object.write_bag_file("data/file.txt") as fh: assert fh.writable() fh.write("Hello\n") - # TODO: Check Windows does not modify \n to \r\n here # Because this is under data/ it should add to manifest # rather than tagmanifest diff --git a/tests/test_rdfprint.py b/tests/test_rdfprint.py index f25310662f..c69d77ab7d 100644 --- a/tests/test_rdfprint.py +++ b/tests/test_rdfprint.py @@ -1,4 +1,3 @@ -import os import subprocess import sys from typing import Any diff --git a/tests/test_subgraph.py b/tests/test_subgraph.py index bd321eb762..bae66b7ab4 100644 --- a/tests/test_subgraph.py +++ b/tests/test_subgraph.py @@ -9,17 +9,16 @@ from cwltool.subgraph import get_step, get_subgraph from cwltool.workflow import Workflow, default_make_tool -from .test_fetch import norm from .util import get_data def test_get_subgraph() -> None: loadingContext = LoadingContext({"construct_tool_object": default_make_tool}) - wf = norm(Path(get_data("tests/subgraph/count-lines1-wf.cwl")).as_uri()) + wf = Path(get_data("tests/subgraph/count-lines1-wf.cwl")).as_uri() loadingContext.do_update = False tool = load_tool(wf, loadingContext) - sg = norm(Path(get_data("tests/subgraph")).as_uri()) + sg = Path(get_data("tests/subgraph")).as_uri() def clean(val: Any) -> Any: if isinstance(val, str): @@ -53,12 +52,12 @@ def clean(val: Any) -> Any: def test_get_step() -> None: loadingContext = LoadingContext({"construct_tool_object": default_make_tool}) - wf = norm(Path(get_data("tests/subgraph/count-lines1-wf.cwl")).as_uri()) + wf = Path(get_data("tests/subgraph/count-lines1-wf.cwl")).as_uri() loadingContext.do_update = False tool = load_tool(wf, loadingContext) assert isinstance(tool, Workflow) - sg = norm(Path(get_data("tests/subgraph")).as_uri()) + sg = Path(get_data("tests/subgraph")).as_uri() def clean(val: Any) -> Any: if isinstance(val, str): diff --git a/tests/test_target.py b/tests/test_target.py index 37e3c362ae..400027ec83 100644 --- a/tests/test_target.py +++ b/tests/test_target.py @@ -1,9 +1,8 @@ from cwltool.main import main -from .util import get_data, windows_needs_docker +from .util import get_data -@windows_needs_docker def test_target() -> None: """Test --target option successful.""" test_file = "tests/wf/scatter-wf4.cwl" @@ -30,7 +29,6 @@ def test_wrong_target() -> None: assert exit_code == 1 -@windows_needs_docker def test_target_packed() -> None: """Test --target option with packed workflow schema.""" test_file = "tests/wf/scatter-wf4.json" diff --git a/tests/util.py b/tests/util.py index 4afe6fedbb..edc2518b98 100644 --- a/tests/util.py +++ b/tests/util.py @@ -1,38 +1,15 @@ import contextlib -import shutil # pylint: disable=no-name-in-module,import-error -import functools import io import os -import subprocess -import sys +import shutil from pathlib import Path from typing import Any, Generator, List, Mapping, Optional, Tuple, Union import pytest from pkg_resources import Requirement, ResolutionError, resource_filename -from cwltool.context import LoadingContext, RuntimeContext -from cwltool.executors import JobExecutor -from cwltool.factory import Factory from cwltool.main import main from cwltool.singularity import is_version_2_6, is_version_3_or_newer -from cwltool.utils import onWindows, windows_default_container_id - - -def get_windows_safe_factory( - runtime_context: Optional[RuntimeContext] = None, - loading_context: Optional[LoadingContext] = None, - executor: Optional[JobExecutor] = None, -) -> Factory: - if onWindows(): - if not runtime_context: - runtime_context = RuntimeContext() - runtime_context.find_default_container = functools.partial( - force_default_container, windows_default_container_id - ) - runtime_context.use_container = True - runtime_context.default_container = windows_default_container_id - return Factory(executor, loading_context, runtime_context) def force_default_container(default_container_id: str, _: str) -> str: @@ -73,13 +50,6 @@ def get_data(filename: str) -> str: ) -windows_needs_docker = pytest.mark.skipif( - onWindows() and not bool(shutil.which("docker")), - reason="Running this test on MS Windows requires the docker executable " - "on the system path.", -) - - def get_main_output( args: List[str], env: Union[ diff --git a/tox.ini b/tox.ini index d0fef6c97f..a9477f3d47 100644 --- a/tox.ini +++ b/tox.ini @@ -11,6 +11,10 @@ envlist = skipsdist = True skip_missing_interpreters = True +[pytest] +addopts=--ignore cwltool/schemas --basetemp ./tmp -n auto --dist=loadfile +testpaths = tests + [gh-actions] python = 3.6: py36 @@ -33,7 +37,6 @@ passenv = PROOT_NO_SECCOMP deps = py{36,37,38,39}-{unit,lint,bandit,mypy}: -rrequirements.txt - py{36,37,38,39}-unit: codecov py{36,37,38,39}-{unit,mypy}: -rtest-requirements.txt py{36,37,38,39}-unit: galaxy-tool-util>=21.1.0 py{36,37,38,39}-lint: flake8-bugbear @@ -46,23 +49,19 @@ setenv = commands = py{36,37,38,39}-unit: python -m pip install -U pip setuptools wheel - py{36,37,38,39}-unit: python -m pip install -e .[deps] - py{36,37,38,39}-unit: coverage run --parallel-mode -m pytest --strict-markers {posargs} - py{36,37,38,39}-unit: coverage combine - py{36,37,38,39}-unit: coverage report - py{36,37,38,39}-unit: coverage xml + py{36,37,38,39}-unit: make coverage-report coverage.xml py{36,37,38,39}-bandit: bandit -r cwltool py{36,37,38,39}-lint: make flake8 py{36,37,38,39}-lint: make format-check py{36,37,38,39}-mypy: make mypy py{36,37,38,39}-mypy: make mypyc - py38-shellcheck: make shellcheck + py39-shellcheck: make shellcheck whitelist_externals = py{36,37,38,39}-lint: flake8 py{36,37,38,39}-lint: black - py{36,37,38,39}-{mypy,shellcheck,lint}: make - py38-shellcheck: shellcheck + py{36,37,38,39}-{mypy,shellcheck,lint,unit}: make + py39-shellcheck: shellcheck [testenv:py39-pydocstyle] whitelist_externals = make diff --git a/windowsdoc.md b/windowsdoc.md deleted file mode 100644 index 215fa6fd69..0000000000 --- a/windowsdoc.md +++ /dev/null @@ -1,142 +0,0 @@ -# Windows Compatibility -The CWL reference runner, cwltool, is compatible with Microsoft Windows when -Docker is installed. On Windows, all CWL CommandLineTools are executed using -[Docker software containers](https://docs.docker.com/docker-for-windows/). The -default Docker Container is -[Alpine with Bash support](https://github.com/frol/docker-alpine-bash). You can -specify other Docker Containers for your tools and workflows using hints, -[requirements](http://www.commonwl.org/v1.0/CommandLineTool.html#DockerRequirement)), -or the `--default-container` cwltool option. - -## Supported Windows versions -* Windows 10 with native [Docker for Windows](https://docs.docker.com/docker-for-windows/). -* Windows 7, 8, and 8.1 with [Docker ToolBox](https://docs.docker.com/toolbox/toolbox_install_windows/). - -If you are using Docker Toolbox, then you must run cwltool in the Docker -Quickstart Terminal. - -## Installation - -You can install cwltool using pip or directly from source code. - -### Requirements - -Before installing cwltool, please install: - -* [Python 3](https://www.python.org/downloads/windows/) -* [Docker](https://docs.docker.com/docker-for-windows/install/) -* [Node.js](https://nodejs.org/en/download/) (optional, please install if your - workflows or tools contain [Javascript Expressions](http://www.commonwl.org/v1.0/CommandLineTool.html#InlineJavascriptRequirement)) - -### Install using pip (recommended) - -``` -pip install cwltool -``` - -### Install from source - -``` -git clone https://github.com/common-workflow-language/cwltool.git -cd cwltool -pip install . -``` - -***Note:*** In order to test if cwltool has been successfully installed on your -Windows system, run `cwltool` in `cmd`. If you see help instructions, cwltool was successfully installed. - -``` - CWL document required, no input file was provided - usage: cwltool [-h] [--basedir BASEDIR] [--outdir OUTDIR] [--no-container] - [--preserve-environment ENVVAR] [--preserve-entire-environment] - [--rm-container | --leave-container] - [--tmpdir-prefix TMPDIR_PREFIX] - ....................... -``` - -## Running tests - -There are two types of tests available for cwltool: unit tests and conformance tests. - -### Unit tests - -To run cwltool's unit tests, run the following command: -``` -python3 -m pytest --pyarg cwltool -``` - -Or go to the checkout of the cwltool Git repository on your system and run: - -``` -python3 setup.py test -``` - - - -### Conformance tests - -To run the CWL conformance tests, follow these instructions: - -``` -pip install cwltest -git clone https://github.com/common-workflow-language/common-workflow-language.git -cd common-workflow-language/v1.0 -cwltest --test conformance_test_v1.0.yaml -j 4 --tool cwltool -``` -The `-j` options is used to run multiple tests in parallel. - -## Troubleshooting - -You may encounter some problems with Docker on Windows. - -### Docker doesn't work on Windows, even outside cwltool - -Make sure you followed all instructions carefully while installing Docker. -Please check the Environment variables. If the problem persists, we recommend -consulting the [online Docker Community](https://forums.docker.com/). - -### Your local drives are not being shared with Docker Containers - -* ***On native Docker for Windows (supported by Windows 10):*** -On your tray, next to your clock, right-click on Docker, then click on Settings, -there you'll find the shared rdives: Here you can share your drives with Docker. -If you encounter a problem with your firewall, please -[refer this to post](https://blog.olandese.nl/2017/05/03/solve-docker-for-windows-error-a-firewall-is-blocking-file-sharing-between-windows-and-the-containers/). - -* ***On Docker Toolbox:*** -Docker Toolbox uses Virtualbox to create a linux base on which Docker machine runs. -Your Docker Container will be created inside Virtualbox. To share drives -in virtualbox, go to ****Virtualbox->settings->shared folders->Machine Folders**** -Here Map the drives you want to share with your Docker Container. -If you want to keep these settings permanent (Recommended!), You should mark the -`make permanent` checkbox or else these settings will be erased every time your -virtualbox closes. - -### In a Docker Container with shared drives, not all files are shown on `ls` - -This means your drives/folders are not shared properly. Docker uses caching, -which may result in not all files and folders being listed on ls. In order to -solve this problem, make your drive mapping settings permanent (see previous -question). - -### Can't create/modify a file in Docker when using cwltool - -When folders are shared with Docker Container, they inherit their current file -access permissions. If you can write to a folder (with your current privileges) -on your local machine, you should be able to write to that folder inside Docker -Container also (provided same user initiated Docker). In all it is a file -permission issue. - -### Workflows with Javascript Expressions occasionally give Timeout errors -To evaluate Javascript Expressions, cwltool looks for Nodejs on your system. -In case Nodejs isn't installed, JS expressions are executed in a Docker Container. -In order to avoid waiting forever in case error occurs, cwltool times out js -expression evaluation after a timeout period (by default 20 seconds). You can -provide a custom timeout period using `--eval-timeout` argument. So if you face -this error, the best option is to install Nodejs on your local system. If you -can't then use the `--eval-timeout` argument and set a higher timeout value. - -*If you still have problems with setting up and using Docker on Windows, please -consult the online Docker Community. If the problem is specific to cwltool, -create an [issue on cwltool](https://github.com/common-workflow-language/cwltool/issues).* - From 337edaa38a808942dc43437cf42142a0ac91b669 Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" Date: Tue, 30 Mar 2021 09:11:37 +0200 Subject: [PATCH 037/111] add warning for running/installing on MS Windows --- cwltool/main.py | 21 +++++++++++++++++++-- setup.py | 14 ++++++++++++++ tests/test_windows_warning.py | 23 +++++++++++++++++++++++ 3 files changed, 56 insertions(+), 2 deletions(-) create mode 100644 tests/test_windows_warning.py diff --git a/cwltool/main.py b/cwltool/main.py index eb2491a02e..1cf4a7ee03 100755 --- a/cwltool/main.py +++ b/cwltool/main.py @@ -12,6 +12,7 @@ import sys import time import urllib +import warnings from codecs import StreamWriter, getwriter from collections.abc import MutableMapping, MutableSequence from typing import ( @@ -1323,9 +1324,25 @@ def find_default_container( return default_container -def run(*args, **kwargs): - # type: (*Any, **Any) -> None +def windows_check() -> None: + """See if we are running on MS Windows and warn about the lack of support.""" + if os.name == "nt": + warnings.warn( + "The CWL reference runner (cwltool) no longer supports running " + "CWL workflows natively on MS Windows as its previous MS Windows " + "support was incomplete and untested. Instead, please see " + "https://pypi.org/project/cwltool/#ms-windows-users " + "for instructions on running cwltool via " + "Windows Subsystem for Linux 2 (WSL2). If don't need to execute " + "CWL documents, then you can ignore this warning, but please " + "consider migrating to https://pypi.org/project/cwl-utils/ " + "for your CWL document processing needs." + ) + + +def run(*args: Any, **kwargs: Any) -> None: """Run cwltool.""" + windows_check() signal.signal(signal.SIGTERM, _signal_handler) try: sys.exit(main(*args, **kwargs)) diff --git a/setup.py b/setup.py index 1ad9a6674c..9115a492bd 100755 --- a/setup.py +++ b/setup.py @@ -2,10 +2,24 @@ """Setup for the reference implementation of the CWL standards.""" import os import sys +import warnings import setuptools.command.egg_info as egg_info_cmd from setuptools import setup +if os.name == "nt": + warnings.warn( + "The CWL reference runner (cwltool) no longer supports running " + "CWL workflows natively on MS Windows as its previous MS Windows " + "support was incomplete and untested. Instead, please see " + "https://pypi.org/project/cwltool/#ms-windows-users " + "for instructions on running cwltool via " + "Windows Subsystem for Linux 2 (WSL2). If don't need to execute " + "CWL documents, then you can ignore this warning, but please " + "consider migrating to https://pypi.org/project/cwl-utils/ " + "for your CWL document processing needs." + ) + SETUP_DIR = os.path.dirname(__file__) README = os.path.join(SETUP_DIR, "README.rst") diff --git a/tests/test_windows_warning.py b/tests/test_windows_warning.py new file mode 100644 index 0000000000..a95d14abfb --- /dev/null +++ b/tests/test_windows_warning.py @@ -0,0 +1,23 @@ +"""Test user experience running on MS Windows.""" + +import os +import warnings + +import pytest + +from cwltool import main + +# Can't be just "import cwltool ; … cwltool.main.windows_check()" +# needs a direct import to avoid path traversal after os.name is set to "nt" + + +def test_windows_warning(monkeypatch: pytest.MonkeyPatch) -> None: + """Confirm that the windows warning is given.""" + with pytest.warns(UserWarning, match=r"Windows Subsystem for Linux 2"): + # would normally just use the MonkeyPatch object directly + # but if we don't use a context then os.name being "nt" causes problems + # for pytest on non-Windows systems. So the context unravels the change + # to os.name quickly, and then pytest will check for the desired warning + with monkeypatch.context() as m: + m.setattr(os, "name", "nt") + main.windows_check() From 782d5f4f45f318cd93fbedb614b792680cf3ffbe Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" Date: Tue, 30 Mar 2021 09:39:30 +0200 Subject: [PATCH 038/111] proper types for MonkeyPatch --- setup.py | 2 +- test-requirements.txt | 2 +- tests/test_ext.py | 3 +-- tests/test_mpi.py | 7 ++++--- tests/test_procgenerator.py | 5 +++-- tests/test_rdfprint.py | 5 +++-- tests/test_tmpdir.py | 6 ++++-- tests/test_windows_warning.py | 1 - tests/util.py | 4 ++-- 9 files changed, 19 insertions(+), 16 deletions(-) diff --git a/setup.py b/setup.py index 9115a492bd..1108115d7b 100755 --- a/setup.py +++ b/setup.py @@ -135,7 +135,7 @@ setup_requires=PYTEST_RUNNER, test_suite="tests", tests_require=[ - "pytest >= 6.0.2, < 6.2", + "pytest >= 6.2, < 6.3", "mock >= 2.0.0", "pytest-mock >= 1.10.0", "arcp >= 0.2.0", diff --git a/test-requirements.txt b/test-requirements.txt index 555bffe542..1f2ed9d003 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,4 +1,4 @@ -pytest >= 6.0.2, < 6.2 +pytest >= 6.2, < 6.3 pytest-xdist mock >= 2.0.0 pytest-mock >= 1.10.0 diff --git a/tests/test_ext.py b/tests/test_ext.py index d8c4bab436..1eb4091e5a 100644 --- a/tests/test_ext.py +++ b/tests/test_ext.py @@ -2,7 +2,6 @@ import re from io import StringIO from pathlib import Path -from typing import Any import pytest @@ -32,7 +31,7 @@ def test_listing_deep() -> None: @needs_docker -def test_cwltool_options(monkeypatch: Any) -> None: +def test_cwltool_options(monkeypatch: pytest.MonkeyPatch) -> None: """Check setting options via environment variable.""" monkeypatch.setenv("CWLTOOL_OPTIONS", "--enable-ext") params = [ diff --git a/tests/test_mpi.py b/tests/test_mpi.py index 72ff81eade..10e62e917d 100644 --- a/tests/test_mpi.py +++ b/tests/test_mpi.py @@ -193,7 +193,7 @@ def test_mpi_workflow(self, fake_mpi_conf: str, tmp_path: Path) -> None: assert lc == np def test_environment( - self, fake_mpi_conf: str, tmp_path: Path, monkeypatch: Any + self, fake_mpi_conf: str, tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: stdout = StringIO() stderr = StringIO() @@ -217,13 +217,14 @@ def test_environment( assert e["TEST_MPI_FOO"] == "bar" -def test_env_passing(monkeypatch: Any) -> None: +def test_env_passing(monkeypatch: pytest.MonkeyPatch) -> None: + """Confirm that MPI extension passes environment variables correctly.""" config = MpiConfig( env_pass=["A", "B", "LONG_NAME"], env_pass_regex=["TOOLNAME", "MPI_.*_CONF"], ) - env = {} # type: MutableMapping[str, str] + env: MutableMapping[str, str] = {} with monkeypatch.context() as m: m.setattr(os, "environ", {}) diff --git a/tests/test_procgenerator.py b/tests/test_procgenerator.py index b7b0f3ebd9..ad8c4d991a 100644 --- a/tests/test_procgenerator.py +++ b/tests/test_procgenerator.py @@ -1,13 +1,14 @@ """ProcessGenerator related tests.""" -from typing import Any + +import pytest from cwltool.main import main from .util import get_data -def test_missing_enable_ext(monkeypatch: Any) -> None: +def test_missing_enable_ext(monkeypatch: pytest.MonkeyPatch) -> None: """Test missing enable-ext option fails. Check that a workflow that needs `--enable-ext` and diff --git a/tests/test_rdfprint.py b/tests/test_rdfprint.py index c69d77ab7d..63adb1c74b 100644 --- a/tests/test_rdfprint.py +++ b/tests/test_rdfprint.py @@ -1,6 +1,7 @@ import subprocess import sys -from typing import Any + +import pytest from cwltool.main import main @@ -11,7 +12,7 @@ def test_rdf_print() -> None: assert main(["--print-rdf", get_data("tests/wf/hello_single_tool.cwl")]) == 0 -def test_rdf_print_unicode(monkeypatch: Any) -> None: +def test_rdf_print_unicode(monkeypatch: pytest.MonkeyPatch) -> None: """Force ASCII encoding but load UTF file with --print-rdf.""" monkeypatch.setenv("LC_ALL", "C") diff --git a/tests/test_tmpdir.py b/tests/test_tmpdir.py index e7a9b61e36..ab35578425 100644 --- a/tests/test_tmpdir.py +++ b/tests/test_tmpdir.py @@ -4,7 +4,7 @@ from pathlib import Path from typing import List, cast -from _pytest.monkeypatch import MonkeyPatch +import pytest from ruamel.yaml.comments import CommentedMap from schema_salad.avro import schema from schema_salad.sourceline import cmap @@ -105,7 +105,9 @@ def test_commandLineTool_job_tmpdir_prefix(tmp_path: Path) -> None: @needs_docker -def test_dockerfile_tmpdir_prefix(tmp_path: Path, monkeypatch: MonkeyPatch) -> None: +def test_dockerfile_tmpdir_prefix( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: """Test that DockerCommandLineJob.get_image respects temp directory directives.""" monkeypatch.setattr( target=subprocess, name="check_call", value=lambda *args, **kwargs: True diff --git a/tests/test_windows_warning.py b/tests/test_windows_warning.py index a95d14abfb..a480f8fb01 100644 --- a/tests/test_windows_warning.py +++ b/tests/test_windows_warning.py @@ -1,7 +1,6 @@ """Test user experience running on MS Windows.""" import os -import warnings import pytest diff --git a/tests/util.py b/tests/util.py index edc2518b98..b064ec389b 100644 --- a/tests/util.py +++ b/tests/util.py @@ -3,7 +3,7 @@ import os import shutil from pathlib import Path -from typing import Any, Generator, List, Mapping, Optional, Tuple, Union +from typing import Generator, List, Mapping, Optional, Tuple, Union import pytest from pkg_resources import Requirement, ResolutionError, resource_filename @@ -55,7 +55,7 @@ def get_main_output( env: Union[ Mapping[bytes, Union[bytes, str]], Mapping[str, Union[bytes, str]], None ] = None, - monkeypatch: Any = None, + monkeypatch: Optional[pytest.MonkeyPatch] = None, ) -> Tuple[Optional[int], str, str]: stdout = io.StringIO() stderr = io.StringIO() From 9dc966e054ad007dc0d9a9bfdd1dc899e1e8e59b Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" Date: Tue, 30 Mar 2021 09:47:04 +0200 Subject: [PATCH 039/111] tox config normalize --- .github/workflows/ci-tests.yml | 2 +- Makefile | 7 +-- README.rst | 12 ++--- tox.ini | 96 ++++++++++++++++------------------ 4 files changed, 57 insertions(+), 60 deletions(-) diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index cf10f8a7e9..d9ca39510a 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -81,7 +81,7 @@ jobs: strategy: matrix: - step: [lint-readme, shellcheck, pydocstyle] + step: [lintreadme, shellcheck, pydocstyle] env: py-semver: 3.9 diff --git a/Makefile b/Makefile index 331866f6e8..73baffe73e 100644 --- a/Makefile +++ b/Makefile @@ -143,11 +143,11 @@ diff-cover.html: coverage.xml ## test : run the ${MODULE} test suite test: $(PYSOURCES) - python setup.py test + python -m pytest ${PYTEST_EXTRA} ## testcov : run the ${MODULE} test suite and collect coverage testcov: $(PYSOURCES) - python setup.py test --addopts "--cov --cov-config=.coveragerc --cov-report= -n auto --dist=loadfile" + python -m pytest --cov --cov-config=.coveragerc --cov-report= ${PYTEST_EXTRA} sloccount.sc: $(PYSOURCES) Makefile sloccount --duplicates --wide --details $^ > $@ @@ -171,7 +171,8 @@ mypy: $(filter-out setup.py gittagger.py,$(PYSOURCES)) MYPYPATH=$$MYPYPATH:typeshed mypy $^ mypyc: $(PYSOURCES) - MYPYPATH=typeshed CWLTOOL_USE_MYPYC=1 pip install --verbose -e . && pytest --ignore cwltool/schemas --basetemp ./tmp + MYPYPATH=typeshed CWLTOOL_USE_MYPYC=1 pip install --verbose -e . \ + && pytest ${PYTEST_EXTRA} shellcheck: FORCE shellcheck build-cwl-docker.sh cwl-docker.sh release-test.sh conformance-test.sh \ diff --git a/README.rst b/README.rst index 815eac4dde..cfdeec8e1f 100644 --- a/README.rst +++ b/README.rst @@ -95,9 +95,9 @@ Or you can skip the direct ``pip`` commands above and install the latest develop .. code:: bash git clone https://github.com/common-workflow-language/cwltool.git # clone (copy) the cwltool git repository - cd cwltool # Change to source directory that git clone just downloaded - pip install . # Installs ``cwltool`` from source - cwltool --version # Check if the installation works correctly + cd cwltool # Change to source directory that git clone just downloaded + pip install .[deps] # Installs ``cwltool`` from source + cwltool --version # Check if the installation works correctly Remember, if co-installing multiple CWL implementations then you need to maintain which implementation ``cwl-runner`` points to via a symbolic file @@ -614,19 +614,19 @@ To run the basic tests after installing `cwltool` execute the following: .. code:: bash pip install -rtest-requirements.txt - py.test --ignore cwltool/schemas/ --pyarg cwltool + pytest To run various tests in all supported Python environments we use `tox `_. To run the test suite in all supported Python environments first downloading the complete code repository (see the ``git clone`` instructions above) and then run the following in the terminal: -``pip install tox; tox`` +``pip install tox; tox -p`` List of all environment can be seen using: ``tox --listenvs`` and running a specfic test env using: ``tox -e `` and additionally run a specific test using this format: -``tox -e py36-unit -- tests/test_examples.py::TestParamMatching`` +``tox -e py36-unit -- -v tests/test_examples.py::test_scandeps`` - Running the entire suite of CWL conformance tests: diff --git a/tox.ini b/tox.ini index a9477f3d47..f2f52fbb6b 100644 --- a/tox.ini +++ b/tox.ini @@ -1,14 +1,13 @@ [tox] envlist = - py{36,37,38,39}-lint, - py{36,37,38,39}-unit, - py{36,37,38,39}-bandit, - py{36,37,38,39}-mypy, - py39-lint-readme, - py39-shellcheck, + py3{6,7,8,9}-lint + py3{6,7,8,9}-unit + py3{6,7,8,9}-bandit + py3{6,7,8,9}-mypy + py39-lintreadme + py39-shellcheck py39-pydocstyle -skipsdist = True skip_missing_interpreters = True [pytest] @@ -23,60 +22,57 @@ python = 3.9: py39 [testenv] +skipsdist = + py3{5,7,8,9}-!{unit,mypy,lintreadme} = True + description = - py{36,37,38,39}-unit: Run the unit tests - py{36,37,38,39}-lint: Lint the Python code - py{36,37,38,39}-bandit: Search for common security issues - py{36,37,38,39}-mypy: Check for type safety - py38-pydocstyle: docstring style checker - py38-shellcheck: syntax check for shell scripts + py3{6,7,8,9}-unit: Run the unit tests + py3{6,7,8,9}-lint: Lint the Python code + py3{6,7,8,9}-bandit: Search for common security issues + py3{6,7,8,9}-mypy: Check for type safety + py39-pydocstyle: docstring style checker + py39-shellcheck: syntax check for shell scripts + py39-lintreadme: Lint the README.rst→.md conversion passenv = CI GITHUB_* PROOT_NO_SECCOMP + +extras = + py3{6,7,8,9}-unit: deps + deps = - py{36,37,38,39}-{unit,lint,bandit,mypy}: -rrequirements.txt - py{36,37,38,39}-{unit,mypy}: -rtest-requirements.txt - py{36,37,38,39}-unit: galaxy-tool-util>=21.1.0 - py{36,37,38,39}-lint: flake8-bugbear - py{36,37,38,39}-lint: black - py{36,37,38,39}-bandit: bandit - py{36,37,38,39}-mypy: -rmypy_requirements.txt + py3{6,7,8,9}-{unit,lint,bandit,mypy}: -rrequirements.txt + py3{6,7,8,9}-{unit,mypy}: -rtest-requirements.txt + py3{6,7,8,9}-lint: flake8-bugbear + py3{6,7,8,9}-lint: black + py3{6,7,8,9}-bandit: bandit + py3{6,7,8,9}-mypy: -rmypy_requirements.txt + py39-pydocstyle: pydocstyle + py39-pydocstyle: diff-cover + py39-lintreadme: twine + py39-lintreadme: wheel + py39-lintreadme: readme_renderer[md] setenv = - py{36,37,38,39}-unit: LC_ALL = C.UTF-8 + py3{6,7,8,9}-unit: LC_ALL = C.UTF-8 + +commands_pre = + py3{6,7,8,9}-unit: python -m pip install -U pip setuptools wheel + py39-lintreadme: python setup.py sdist --dist-dir {distdir} + py39-lintreadme: python setup.py bdist_wheel --dist-dir {distdir} commands = - py{36,37,38,39}-unit: python -m pip install -U pip setuptools wheel - py{36,37,38,39}-unit: make coverage-report coverage.xml - py{36,37,38,39}-bandit: bandit -r cwltool - py{36,37,38,39}-lint: make flake8 - py{36,37,38,39}-lint: make format-check - py{36,37,38,39}-mypy: make mypy - py{36,37,38,39}-mypy: make mypyc + py3{6,7,8,9}-unit: make coverage-report coverage.xml PYTEST_EXTRA={posargs} + py3{6,7,8,9}-bandit: bandit -r cwltool + py3{6,7,8,9}-lint: make flake8 format-check + py3{6,7,8,9}-mypy: make mypy mypyc PYTEST_EXTRA={posargs} py39-shellcheck: make shellcheck + py39-pydocstyle: make diff_pydocstyle_report + py39-lintreadme: twine check {distdir}/* -whitelist_externals = - py{36,37,38,39}-lint: flake8 - py{36,37,38,39}-lint: black - py{36,37,38,39}-{mypy,shellcheck,lint,unit}: make - py39-shellcheck: shellcheck +skip_install = + py3{6,7,8,9}-{bandit,lint,mypy,shellcheck,pydocstyle,lintreadme}: true -[testenv:py39-pydocstyle] -whitelist_externals = make -commands = make diff_pydocstyle_report -deps = - pydocstyle - diff-cover - -[testenv:py39-lint-readme] -description = Lint the README.rst->.md conversion -commands = - python setup.py sdist - python setup.py bdist_wheel - twine check dist/* -deps = - twine - wheel - readme_renderer[md] +allowlist_externals = make From e37134a90ac7a7c18254e30cff16da590b45c6d7 Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" <1330696+mr-c@users.noreply.github.com> Date: Mon, 5 Apr 2021 15:01:08 +0200 Subject: [PATCH 040/111] fix formatting --- README.rst | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/README.rst b/README.rst index cfdeec8e1f..64bcbc9927 100644 --- a/README.rst +++ b/README.rst @@ -48,7 +48,7 @@ and similar Linux distribution try .. code:: bash - sudo apt-get install cwltool + sudo apt-get install cwltool If you are running MacOS X or other UNIXes and you want to use packages prepared by the conda-forge project, then please follow the install instructions for `conda-forge `_ (if you haven't already) and then @@ -58,12 +58,12 @@ please follow the install instructions for `conda-forge Date: Sun, 18 Apr 2021 07:08:18 +0000 Subject: [PATCH 041/111] fix: cwltool.Dockerfile to reduce vulnerabilities The following vulnerabilities are fixed with an upgrade: - https://snyk.io/vuln/SNYK-ALPINE313-APKTOOLS-1246345 --- cwltool.Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cwltool.Dockerfile b/cwltool.Dockerfile index 8c483ab1e3..490efa0307 100644 --- a/cwltool.Dockerfile +++ b/cwltool.Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.6-alpine as builder +FROM python:3.8-alpine as builder RUN apk add --no-cache git gcc python3-dev libxml2-dev libxslt-dev libc-dev linux-headers @@ -9,13 +9,13 @@ RUN python setup.py bdist_wheel --dist-dir=/wheels RUN pip wheel -r requirements.txt --wheel-dir=/wheels RUN pip install --no-index --no-warn-script-location --root=/pythonroot/ /wheels/*.whl -FROM python:3.6-alpine as module +FROM python:3.8-alpine as module LABEL maintainer peter.amstutz@curoverse.com RUN apk add --no-cache docker nodejs graphviz libxml2 libxslt COPY --from=builder /pythonroot/ / -FROM python:3.6-alpine +FROM python:3.8-alpine LABEL maintainer peter.amstutz@curoverse.com RUN apk add --no-cache docker nodejs graphviz libxml2 libxslt From a3e3a5720f7b0131fa4f9c0b3f73b62a347278a6 Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" Date: Fri, 23 Apr 2021 16:21:19 +0200 Subject: [PATCH 042/111] check for empty hints/reqs --- cwltool/process.py | 20 ++++++++++++++++++-- tests/test_examples.py | 20 ++++++++++++++++++++ tests/wc-tool-bad-hints.cwl | 19 +++++++++++++++++++ tests/wc-tool-bad-reqs.cwl | 19 +++++++++++++++++++ 4 files changed, 76 insertions(+), 2 deletions(-) create mode 100644 tests/wc-tool-bad-hints.cwl create mode 100644 tests/wc-tool-bad-reqs.cwl diff --git a/cwltool/process.py b/cwltool/process.py index 8e016ddfbf..e444d04ddc 100644 --- a/cwltool/process.py +++ b/cwltool/process.py @@ -578,7 +578,15 @@ def __init__( self.names = make_avro_schema([SCHEMA_FILE, SCHEMA_DIR, SCHEMA_ANY], Loader({})) self.tool = toolpath_object self.requirements = copy.deepcopy(getdefault(loadingContext.requirements, [])) - self.requirements.extend(self.tool.get("requirements", [])) + tool_requirements = self.tool.get("requirements", []) + if tool_requirements is None: + raise ValidationException( + SourceLine(self.tool, "requirements").makeError( + "If 'requirements' is present then it must be a list " + "or map/dictionary, not empty." + ) + ) + self.requirements.extend(tool_requirements) if "id" not in self.tool: self.tool["id"] = "_:" + str(uuid.uuid4()) self.requirements.extend( @@ -590,7 +598,15 @@ def __init__( ) ) self.hints = copy.deepcopy(getdefault(loadingContext.hints, [])) - self.hints.extend(self.tool.get("hints", [])) + tool_hints = self.tool.get("hints", []) + if tool_hints is None: + raise ValidationException( + SourceLine(self.tool, "hints").makeError( + "If 'hints' is present then it must be a list " + "or map/dictionary, not empty." + ) + ) + self.hints.extend(tool_hints) # Versions of requirements and hints which aren't mutated. self.original_requirements = copy.deepcopy(self.requirements) self.original_hints = copy.deepcopy(self.hints) diff --git a/tests/test_examples.py b/tests/test_examples.py index f3de50c146..498bcaffff 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -1398,3 +1398,23 @@ def test_scatter_output_filenames(tmp_path: Path) -> None: and locations[1].endswith("output.txt_2") and locations[2].endswith("output.txt_3") ), f"Locations {locations} do not end with {output_names}" + + +def test_malformed_hints() -> None: + """Confirm that empty hints section is caught.""" + factory = cwltool.factory.Factory() + with pytest.raises( + ValidationException, + match=r".*wc-tool-bad-hints\.cwl:6:1: If 'hints' is\s*present\s*then\s*it\s*must\s*be\s*a\s*list.*", + ): + factory.make(get_data("tests/wc-tool-bad-hints.cwl")) + + +def test_malformed_reqs() -> None: + """Confirm that empty reqs section is caught.""" + factory = cwltool.factory.Factory() + with pytest.raises( + ValidationException, + match=r".*wc-tool-bad-reqs\.cwl:6:1: If 'requirements' is\s*present\s*then\s*it\s*must\s*be\s*a\s*list.*", + ): + factory.make(get_data("tests/wc-tool-bad-reqs.cwl")) diff --git a/tests/wc-tool-bad-hints.cwl b/tests/wc-tool-bad-hints.cwl new file mode 100644 index 0000000000..bbde0e2110 --- /dev/null +++ b/tests/wc-tool-bad-hints.cwl @@ -0,0 +1,19 @@ +#!/usr/bin/env cwl-runner + +class: CommandLineTool +cwlVersion: v1.1 + +hints: + +inputs: + file1: File + +outputs: + output: + type: File + outputBinding: { glob: output } + +baseCommand: [wc, -l] + +stdin: $(inputs.file1.path) +stdout: output diff --git a/tests/wc-tool-bad-reqs.cwl b/tests/wc-tool-bad-reqs.cwl new file mode 100644 index 0000000000..96e879b69c --- /dev/null +++ b/tests/wc-tool-bad-reqs.cwl @@ -0,0 +1,19 @@ +#!/usr/bin/env cwl-runner + +class: CommandLineTool +cwlVersion: v1.1 + +requirements: + +inputs: + file1: File + +outputs: + output: + type: File + outputBinding: { glob: output } + +baseCommand: [wc, -l] + +stdin: $(inputs.file1.path) +stdout: output From df79fd0139d4ec10051b4ffd8e06dda7269b4f57 Mon Sep 17 00:00:00 2001 From: Rupert Nash Date: Fri, 27 Nov 2020 11:12:27 +0000 Subject: [PATCH 043/111] Allow software requirements to modify env. First do basic env setup, run the modification commands in a script and record the environment. Have the run_job script then use this environment (ignoring some variables). --- Makefile | 6 +- README.rst | 9 ++- cwltool/job.py | 56 +------------------ cwltool/run_job.py | 96 ++++++++++++++++++++++++++++++++ cwltool/software_requirements.py | 5 +- 5 files changed, 114 insertions(+), 58 deletions(-) create mode 100644 cwltool/run_job.py diff --git a/Makefile b/Makefile index 73baffe73e..f2b807aae3 100644 --- a/Makefile +++ b/Makefile @@ -116,7 +116,7 @@ pylint_report.txt: $(PYSOURCES) $^ -j0> $@ || true diff_pylint_report: pylint_report.txt - diff-quality --violations=pylint pylint_report.txt + diff-quality --compare-branch=main --violations=pylint pylint_report.txt .coverage: testcov @@ -136,10 +136,10 @@ coverage-report: .coverage coverage report diff-cover: coverage.xml - diff-cover $^ + diff-cover --compare-branch=main $^ diff-cover.html: coverage.xml - diff-cover $^ --html-report $@ + diff-cover --compare-branch=main $^ --html-report $@ ## test : run the ${MODULE} test suite test: $(PYSOURCES) diff --git a/README.rst b/README.rst index 64bcbc9927..2c3e50504d 100644 --- a/README.rst +++ b/README.rst @@ -316,6 +316,13 @@ This option allows one to specify a dependency resolver's configuration file. This file may be specified as either XML or YAML and very simply describes various plugins to enable to "resolve" ``SoftwareRequirement`` dependencies. +Using these hints will allow cwltool to modify the environment in +which your tool runs, for example by loading one or more environment +modules. The enviroment is constructed as normal (i.e. standard CWL +runtime enviroment, ``EnvVarRequirement``, ``--preserve-environment`` +are applied), then this is modified. This currently means that you +cannot override variables set in this way. + To discuss some of these plugins and how to configure them, first consider the following ``hint`` definition for an example CWL tool. @@ -427,7 +434,7 @@ So consider the resolvers configuration file base_path: ./tests/test_deps_env mapping_files: ./tests/test_deps_mapping.yml -And the corresponding mapping configuraiton file (`tests/test_deps_mapping.yml`): +And the corresponding mapping configuration file (`tests/test_deps_mapping.yml`): .. code:: yaml diff --git a/cwltool/job.py b/cwltool/job.py index d01f8d56ba..6eae63f330 100644 --- a/cwltool/job.py +++ b/cwltool/job.py @@ -36,6 +36,7 @@ from schema_salad.utils import json_dump, json_dumps from typing_extensions import TYPE_CHECKING +from . import run_job from .builder import Builder, HasReqsHints from .context import RuntimeContext from .errors import UnsupportedRequirement, WorkflowException @@ -62,57 +63,7 @@ FORCE_SHELLED_POPEN = os.getenv("CWLTOOL_FORCE_SHELL_POPEN", "0") == "1" SHELL_COMMAND_TEMPLATE = """#!/bin/bash -python "run_job.py" "job.json" -""" - -PYTHON_RUN_SCRIPT = """ -import json -import os -import sys -try: - import subprocess32 as subprocess # type: ignore -except Exception: - import subprocess - -with open(sys.argv[1], "r") as f: - popen_description = json.load(f) - commands = popen_description["commands"] - cwd = popen_description["cwd"] - env = popen_description["env"] - env["PATH"] = os.environ.get("PATH") - stdin_path = popen_description["stdin_path"] - stdout_path = popen_description["stdout_path"] - stderr_path = popen_description["stderr_path"] - if stdin_path is not None: - stdin = open(stdin_path, "rb") - else: - stdin = subprocess.PIPE - if stdout_path is not None: - stdout = open(stdout_path, "wb") - else: - stdout = sys.stderr - if stderr_path is not None: - stderr = open(stderr_path, "wb") - else: - stderr = sys.stderr - sp = subprocess.Popen(commands, - shell=False, - close_fds=True, - stdin=stdin, - stdout=stdout, - stderr=stderr, - env=env, - cwd=cwd) - if sp.stdin: - sp.stdin.close() - rcode = sp.wait() - if stdin is not subprocess.PIPE: - stdin.close() - if stdout is not sys.stderr: - stdout.close() - if stderr is not sys.stderr: - stderr.close() - sys.exit(rcode) +python3 "run_job.py" "job.json" """ @@ -988,8 +939,7 @@ def terminate(): # type: () -> None with open(job_script, "wb") as _: _.write(job_script_contents.encode("utf-8")) job_run = os.path.join(job_dir, "run_job.py") - with open(job_run, "wb") as _: - _.write(PYTHON_RUN_SCRIPT.encode("utf-8")) + shutil.copyfile(run_job.__file__, job_run) sproc = subprocess.Popen( # nosec ["bash", job_script.encode("utf-8")], shell=False, # nosec diff --git a/cwltool/run_job.py b/cwltool/run_job.py new file mode 100644 index 0000000000..17250c366f --- /dev/null +++ b/cwltool/run_job.py @@ -0,0 +1,96 @@ +"""Only used when there is a job script or CWLTOOL_FORCE_SHELL_POPEN=1.""" +import json +import os +import subprocess # nosec +import sys +from typing import BinaryIO, Dict, List, Optional, TextIO, Union + + +def handle_software_environment(cwl_env: Dict[str, str], script: str) -> Dict[str, str]: + """Update the provided environment variables dictiony by running the script.""" + res = subprocess.run(["bash", script], shell=False, env=cwl_env) # nosec + if res.returncode != 0: + sys.stderr.write( + "Error while using SoftwareRequirements to modify environment\n" + ) + return cwl_env + env = cwl_env.copy() + with open("output_environment.bash") as env_file: + for line in env_file: + key, val = line.split("=", 1) + if key in ("_", "PWD", "SHLVL"): + # Skip some variables that are meaningful to the shell + continue + env[key] = val[:-1] # remove trailing newline + return env + + +def main(argv: List[str]) -> int: + """ + Read in the configuration JSON and execute the commands. + + The first argument is the path to the JSON dictionary file containing keys: + "commands": an array of strings that represents the command line to run + "cwd": A string specifying which directory to run in + "env": a dictionary of strings containing the environment variables to set + "stdin_path": a string (or a null) giving the path that should be piped to STDIN + "stdout_path": a string (or a null) giving the path that should receive the STDOUT + "stderr_path": a string (or a null) giving the path that should receive the STDERR + + The second argument is optional, it specifes a shell script to execute prior, + and the environment variables it sets will be combined with the environment + variables from the "env" key in the JSON dictionary from the first argument. + """ + with open(argv[1]) as f: + popen_description = json.load(f) + commands = popen_description["commands"] + cwd = popen_description["cwd"] + env = popen_description["env"] + env["PATH"] = os.environ.get("PATH") + stdin_path = popen_description["stdin_path"] + stdout_path = popen_description["stdout_path"] + stderr_path = popen_description["stderr_path"] + if stdin_path is not None: + stdin: Union[BinaryIO, int] = open(stdin_path, "rb") + else: + stdin = subprocess.PIPE + if stdout_path is not None: + stdout: Union[BinaryIO, TextIO] = open(stdout_path, "wb") + else: + stdout = sys.stderr + if stderr_path is not None: + stderr: Union[BinaryIO, TextIO] = open(stderr_path, "wb") + else: + stderr = sys.stderr + + try: + env_script: Optional[str] = argv[2] + except IndexError: + env_script = None + if env_script is not None: + env = handle_software_environment(env, env_script) + + sp = subprocess.Popen( # nosec + commands, + shell=False, + close_fds=True, + stdin=stdin, + stdout=stdout, + stderr=stderr, + env=env, + cwd=cwd, + ) + if sp.stdin: + sp.stdin.close() + rcode = sp.wait() + if not isinstance(stdin, int): + stdin.close() + if stdout is not sys.stderr: + stdout.close() + if stderr is not sys.stderr: + stderr.close() + return rcode + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/cwltool/software_requirements.py b/cwltool/software_requirements.py index 70d6eac184..d4e67f6286 100644 --- a/cwltool/software_requirements.py +++ b/cwltool/software_requirements.py @@ -30,8 +30,11 @@ COMMAND_WITH_DEPENDENCIES_TEMPLATE = string.Template( """#!/bin/bash +cat > modify_environment.bash <<'EOF' $handle_dependencies -python3 "run_job.py" "job.json" +env > output_environment.bash +EOF +python3 "run_job.py" "job.json" "modify_environment.bash" """ ) From 3f6b4553e0e1ee445380dea5fb70ba908b356d60 Mon Sep 17 00:00:00 2001 From: Rupert Nash Date: Fri, 27 Nov 2020 20:35:55 +0000 Subject: [PATCH 044/111] add a basic integration test for environment variables from modules --- cwltool/run_job.py | 3 +- tests/env3.cwl | 8 ++++ tests/env_with_software_req.yml | 7 +++ tests/test_dependencies.py | 48 +++++++++++++++++-- .../modulefiles/random-lines/1.0 | 1 + 5 files changed, 62 insertions(+), 5 deletions(-) create mode 100644 tests/env3.cwl create mode 100644 tests/env_with_software_req.yml diff --git a/cwltool/run_job.py b/cwltool/run_job.py index 17250c366f..f22788129b 100644 --- a/cwltool/run_job.py +++ b/cwltool/run_job.py @@ -18,8 +18,9 @@ def handle_software_environment(cwl_env: Dict[str, str], script: str) -> Dict[st with open("output_environment.bash") as env_file: for line in env_file: key, val = line.split("=", 1) - if key in ("_", "PWD", "SHLVL"): + if key in ("_", "PWD", "SHLVL", "TMPDIR", "HOME"): # Skip some variables that are meaningful to the shell + # or set specifically by the CWL runtime environment. continue env[key] = val[:-1] # remove trailing newline return env diff --git a/tests/env3.cwl b/tests/env3.cwl new file mode 100644 index 0000000000..d38826c2ea --- /dev/null +++ b/tests/env3.cwl @@ -0,0 +1,8 @@ +#!/usr/bin/env cwl-runner +cwlVersion: v1.0 +class: CommandLineTool +inputs: [] +baseCommand: env +outputs: + env: + type: stdout diff --git a/tests/env_with_software_req.yml b/tests/env_with_software_req.yml new file mode 100644 index 0000000000..797fc8b7db --- /dev/null +++ b/tests/env_with_software_req.yml @@ -0,0 +1,7 @@ +cwltool:overrides: + env3.cwl: + requirements: + SoftwareRequirement: + packages: + - package: 'random-lines' + version: '1.0' diff --git a/tests/test_dependencies.py b/tests/test_dependencies.py index 67a3219840..7031b061a0 100644 --- a/tests/test_dependencies.py +++ b/tests/test_dependencies.py @@ -1,11 +1,17 @@ +"""Tests of satisfying SoftwareRequirement via dependencies.""" +from io import StringIO +import json import os +from pathlib import Path from shutil import which from types import ModuleType from typing import Optional import pytest -from .util import get_data, get_main_output, needs_docker +from cwltool.main import main + +from .util import get_data, get_main_output, needs_docker, working_directory deps = None # type: Optional[ModuleType] try: @@ -37,11 +43,11 @@ def test_bioconda() -> None: @pytest.mark.skipif(not deps, reason="galaxy-lib is not installed") @pytest.mark.skipif(not which("modulecmd"), reason="modulecmd not installed") -def test_modules() -> None: +def test_modules(monkeypatch: pytest.MonkeyPatch) -> None: wflow = get_data("tests/random_lines.cwl") job = get_data("tests/random_lines_job.json") - os.environ["MODULEPATH"] = os.path.join( - os.getcwd(), "tests/test_deps_env/modulefiles" + monkeypatch.setenv( + "MODULEPATH", os.path.join(os.getcwd(), "tests/test_deps_env/modulefiles") ) error_code, _, stderr = get_main_output( [ @@ -54,3 +60,37 @@ def test_modules() -> None: ) assert error_code == 0, stderr + + +@pytest.mark.skipif(not deps, reason="galaxy-lib is not installed") +@pytest.mark.skipif(not which("modulecmd"), reason="modulecmd not installed") +def test_modules_environment(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv( + "MODULEPATH", os.path.join(os.getcwd(), "tests/test_deps_env/modulefiles") + ) + + stdout = StringIO() + stderr = StringIO() + with working_directory(tmp_path): + rc = main( + argsl=[ + "--beta-dependency-resolvers-configuration", + get_data("tests/test_deps_env_modules_resolvers_conf.yml"), + get_data("tests/env3.cwl"), + get_data("tests/env_with_software_req.yml"), + ], + stdout=stdout, + stderr=stderr, + ) + assert rc == 0 + + output = json.loads(stdout.getvalue()) + env_path = output["env"]["path"] + tool_env = {} + with open(env_path) as _: + for line in _: + key, val = line.split("=", 1) + tool_env[key] = val[:-1] + assert tool_env["TEST_VAR_MODULE"] == "environment variable ends in space " + tool_path = tool_env["PATH"].split(":") + assert get_data("tests/test_deps_env/random-lines/1.0/scripts") in tool_path diff --git a/tests/test_deps_env/modulefiles/random-lines/1.0 b/tests/test_deps_env/modulefiles/random-lines/1.0 index d3f358ba86..3f76238980 100644 --- a/tests/test_deps_env/modulefiles/random-lines/1.0 +++ b/tests/test_deps_env/modulefiles/random-lines/1.0 @@ -13,3 +13,4 @@ if {![file exists $prefix]} { } prepend-path PATH ${prefix}/scripts +setenv TEST_VAR_MODULE "environment variable ends in space " From a89c3298c1a6e9dce535df951c17aebd98b87650 Mon Sep 17 00:00:00 2001 From: Rupert Nash Date: Thu, 3 Dec 2020 12:42:36 +0000 Subject: [PATCH 045/111] Apply suggestions from code review Co-authored-by: Michael R. Crusoe <1330696+mr-c@users.noreply.github.com> --- .coveragerc | 1 + README.rst | 8 ++++---- tests/test_dependencies.py | 6 ++++++ 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/.coveragerc b/.coveragerc index ced2204062..5937dce43a 100644 --- a/.coveragerc +++ b/.coveragerc @@ -1,6 +1,7 @@ [run] branch = True source = cwltool +omit = cwltool/run_job.py [report] exclude_lines = diff --git a/README.rst b/README.rst index 2c3e50504d..987adfbd53 100644 --- a/README.rst +++ b/README.rst @@ -317,11 +317,11 @@ This file may be specified as either XML or YAML and very simply describes vario plugins to enable to "resolve" ``SoftwareRequirement`` dependencies. Using these hints will allow cwltool to modify the environment in -which your tool runs, for example by loading one or more environment -modules. The enviroment is constructed as normal (i.e. standard CWL +which your tool runs, for example by loading one or more Environment +Modules. The environment is constructed as normal (i.e. standard CWL runtime enviroment, ``EnvVarRequirement``, ``--preserve-environment`` -are applied), then this is modified. This currently means that you -cannot override variables set in this way. +are applied), then the environment may modified by the selected tool resolver. This currently means that you +cannot override any environment variables set by the selected tool resolver. To discuss some of these plugins and how to configure them, first consider the following ``hint`` definition for an example CWL tool. diff --git a/tests/test_dependencies.py b/tests/test_dependencies.py index 7031b061a0..e2a235a40b 100644 --- a/tests/test_dependencies.py +++ b/tests/test_dependencies.py @@ -44,6 +44,7 @@ def test_bioconda() -> None: @pytest.mark.skipif(not deps, reason="galaxy-lib is not installed") @pytest.mark.skipif(not which("modulecmd"), reason="modulecmd not installed") def test_modules(monkeypatch: pytest.MonkeyPatch) -> None: + """Do a basic smoke test using environment modules to satisfy a SoftwareRequirement.""" wflow = get_data("tests/random_lines.cwl") job = get_data("tests/random_lines_job.json") monkeypatch.setenv( @@ -65,6 +66,11 @@ def test_modules(monkeypatch: pytest.MonkeyPatch) -> None: @pytest.mark.skipif(not deps, reason="galaxy-lib is not installed") @pytest.mark.skipif(not which("modulecmd"), reason="modulecmd not installed") def test_modules_environment(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """ + Check that the environment variables set by a module are being propagated correctly. + + Do so by by running `env` as the tool and parsing its output. + """ monkeypatch.setenv( "MODULEPATH", os.path.join(os.getcwd(), "tests/test_deps_env/modulefiles") ) From 82030b5ba1f7e19d0d03953b1b4694caa4f51722 Mon Sep 17 00:00:00 2001 From: Rupert Nash Date: Fri, 26 Mar 2021 13:53:02 +0000 Subject: [PATCH 046/111] Put the environment setup code into (mostly) one place. --- README.rst | 33 ++++++++++++++++--- cwltool/command_line_tool.py | 9 +++--- cwltool/job.py | 61 +++++++++++++++++++++++------------- cwltool/run_job.py | 9 ++++-- 4 files changed, 80 insertions(+), 32 deletions(-) diff --git a/README.rst b/README.rst index 987adfbd53..a8792482fc 100644 --- a/README.rst +++ b/README.rst @@ -296,6 +296,30 @@ CWL documents can be expressed as RDF triple graphs. cwltool --print-rdf --rdf-serializer=turtle mywf.cwl +Environment Variables in cwltool +-------------------------------- + +This reference implementation supports several ways of setting +enviroment variables for tools, in addition to the standard +``EnvVarRequirement``. The sequence of steps applied to create the +enviroment is: + +0. If the ``--preserve-entire-environment`` flag is present, then begin with the current + environment, else begin with an empty environment. + +1. Add any variables specified by ``--preserve-environment`` option(s). + +2. Set ``TMPDIR`` and ``HOME`` per `the CWL v1.0+ CommandLineTool specification `_. + +3. Apply any ``EnvVarRequirement`` from the ``CommandLineTool`` description. + +4. Apply any manipulations required by any ``cwltool:MPIRequirement`` extensions. + +5. Substitute any secrets required by ``Secrets`` extension. + +6. Modify the environment in response to ``SoftwareRequirement`` (see below). + + Leveraging SoftwareRequirements (Beta) -------------------------------------- @@ -318,10 +342,11 @@ plugins to enable to "resolve" ``SoftwareRequirement`` dependencies. Using these hints will allow cwltool to modify the environment in which your tool runs, for example by loading one or more Environment -Modules. The environment is constructed as normal (i.e. standard CWL -runtime enviroment, ``EnvVarRequirement``, ``--preserve-environment`` -are applied), then the environment may modified by the selected tool resolver. This currently means that you -cannot override any environment variables set by the selected tool resolver. +Modules. The environment is constructed as above, then the environment +may modified by the selected tool resolver. This currently means that +you cannot override any environment variables set by the selected tool +resolver. Note that the enviroment given to the configured dependency +resolver has the variable `_CWLTOOL` set to `1` to allow introspection. To discuss some of these plugins and how to configure them, first consider the following ``hint`` definition for an example CWL tool. diff --git a/cwltool/command_line_tool.py b/cwltool/command_line_tool.py index 68af83c97c..03db249e24 100644 --- a/cwltool/command_line_tool.py +++ b/cwltool/command_line_tool.py @@ -995,13 +995,14 @@ def register_reader(f: CWLObjectType) -> None: "networkAccess must be a boolean, got: %s" % j.networkaccess ) - j.environment = {} + # Build a mapping to hold any EnvVarRequirement + required_env = {} evr, _ = self.get_requirement("EnvVarRequirement") if evr is not None: for t3 in cast(List[Dict[str, str]], evr["envDef"]): - j.environment[t3["envName"]] = cast( - str, builder.do_eval(t3["envValue"]) - ) + required_env[t3["envName"]] = cast(str, builder.do_eval(t3["envValue"])) + # Construct the env + j.prepare_environment(runtimeContext, required_env) shellcmd, _ = self.get_requirement("ShellCommandRequirement") if shellcmd is not None: diff --git a/cwltool/job.py b/cwltool/job.py index 6eae63f330..6c826f63d2 100644 --- a/cwltool/job.py +++ b/cwltool/job.py @@ -19,6 +19,7 @@ Callable, Iterable, List, + Mapping, Match, MutableMapping, MutableSequence, @@ -419,6 +420,42 @@ def _execute( ) shutil.rmtree(self.tmpdir, True) + def prepare_environment( + self, runtimeContext: RuntimeContext, envVarReq: Mapping[str, str] + ) -> None: + """Set up environment variables. + + Here we prepare the environment for the job, based on any + preserved variables and `EnvVarRequirement`. Later, changes due + to `MPIRequirement`, `Secrets`, or `SoftwareRequirement` are + applied (in that order). + """ + # Start empty + env = {} + + # Preserve any env vars + vars_to_preserve = runtimeContext.preserve_environment + if runtimeContext.preserve_entire_environment is not False: + vars_to_preserve = os.environ + if vars_to_preserve: + for key, value in os.environ.items(): + if key in vars_to_preserve and key not in env: + env[key] = value + + # Set required env vars + env["HOME"] = self.outdir + env["TMPDIR"] = self.tmpdir + if "PATH" not in env: + env["PATH"] = os.environ["PATH"] + if "SYSTEMROOT" not in env and "SYSTEMROOT" in os.environ: + env["SYSTEMROOT"] = os.environ["SYSTEMROOT"] + + # Apply EnvVarRequirement + env.update(envVarReq) + + # Set on ourselves + self.environment = env + def process_monitor(self, sproc): # type: (subprocess.Popen[str]) -> None monitor = psutil.Process(sproc.pid) # Value must be list rather than integer to utilise pass-by-reference in python @@ -469,19 +506,6 @@ def run( self._setup(runtimeContext) - env = self.environment - vars_to_preserve = runtimeContext.preserve_environment - if runtimeContext.preserve_entire_environment is not False: - vars_to_preserve = os.environ - if vars_to_preserve: - for key, value in os.environ.items(): - if key in vars_to_preserve and key not in env: - env[key] = value - env["HOME"] = self.outdir - env["TMPDIR"] = self.tmpdir - if "PATH" not in env: - env["PATH"] = os.environ["PATH"] - stage_files( self.pathmapper, ignore_writable=True, @@ -504,7 +528,7 @@ def run( monitor_function = functools.partial(self.process_monitor) - self._execute([], env, runtimeContext, monitor_function) + self._execute([], self.environment, runtimeContext, monitor_function) CONTROL_CODE_RE = r"\x1b\[[0-9;]*[a-zA-Z]" @@ -832,7 +856,7 @@ def _job_popen( stdin_path: Optional[str], stdout_path: Optional[str], stderr_path: Optional[str], - env: MutableMapping[str, str], + env: Mapping[str, str], cwd: str, make_job_dir: Callable[[], str], job_script_contents: Optional[str] = None, @@ -915,15 +939,10 @@ def terminate(): # type: () -> None if job_script_contents is None: job_script_contents = SHELL_COMMAND_TEMPLATE - env_copy = {} - key = None # type: Optional[str] - for key in env: - env_copy[key] = env[key] - job_description = { "commands": commands, "cwd": cwd, - "env": env_copy, + "env": env, "stdout_path": stdout_path, "stderr_path": stderr_path, "stdin_path": stdin_path, diff --git a/cwltool/run_job.py b/cwltool/run_job.py index f22788129b..f6f80a296e 100644 --- a/cwltool/run_job.py +++ b/cwltool/run_job.py @@ -7,18 +7,21 @@ def handle_software_environment(cwl_env: Dict[str, str], script: str) -> Dict[str, str]: - """Update the provided environment variables dictiony by running the script.""" - res = subprocess.run(["bash", script], shell=False, env=cwl_env) # nosec + """Update the provided environment dict by running the script.""" + exec_env = cwl_env.copy() + exec_env["_CWLTOOL"] = "1" + res = subprocess.run(["bash", script], shell=False, env=exec_env) # nosec if res.returncode != 0: sys.stderr.write( "Error while using SoftwareRequirements to modify environment\n" ) return cwl_env + env = cwl_env.copy() with open("output_environment.bash") as env_file: for line in env_file: key, val = line.split("=", 1) - if key in ("_", "PWD", "SHLVL", "TMPDIR", "HOME"): + if key in ("_", "PWD", "SHLVL", "TMPDIR", "HOME", "_CWLTOOL"): # Skip some variables that are meaningful to the shell # or set specifically by the CWL runtime environment. continue From 725174ea3dad79800212b5c148747654d877ae19 Mon Sep 17 00:00:00 2001 From: Rupert Nash Date: Tue, 30 Mar 2021 17:04:26 +0100 Subject: [PATCH 047/111] Enable (consistently) the preserve environment options with containers. --- cwltool/argparser.py | 6 ++-- cwltool/context.py | 2 +- cwltool/docker.py | 21 +++++++------ cwltool/job.py | 70 +++++++++++++++++++++++++++++++++--------- cwltool/singularity.py | 20 +++++++++--- 5 files changed, 86 insertions(+), 33 deletions(-) diff --git a/cwltool/argparser.py b/cwltool/argparser.py index 5399cd9a2c..cae924bcf3 100644 --- a/cwltool/argparser.py +++ b/cwltool/argparser.py @@ -49,10 +49,10 @@ def arg_parser() -> argparse.ArgumentParser: type=str, action="append", help="Preserve specific environment variable when running " - "CommandLineTools without a software container. May be provided " - "multiple times. The default is to preserve only the PATH.", + "CommandLineTools. May be provided multiple times. By default PATH is " + "preserved when not running in a container.", metavar="ENVVAR", - default=["PATH"], + default=[], dest="preserve_environment", ) envgroup.add_argument( diff --git a/cwltool/context.py b/cwltool/context.py index e7908d95e8..bf4154b574 100644 --- a/cwltool/context.py +++ b/cwltool/context.py @@ -95,7 +95,7 @@ def __init__(self, kwargs: Optional[Dict[str, Any]] = None) -> None: self.no_read_only = False # type: bool self.custom_net = "" # type: Optional[str] self.no_match_user = False # type: bool - self.preserve_environment = "" # type: Optional[Iterable[str]] + self.preserve_environment = None # type: Optional[Iterable[str]] self.preserve_entire_environment = False # type: bool self.use_container = True # type: bool self.force_docker_pull = False # type: bool diff --git a/cwltool/docker.py b/cwltool/docker.py index 99a3f3ae8c..42f38b082b 100644 --- a/cwltool/docker.py +++ b/cwltool/docker.py @@ -78,6 +78,8 @@ def _check_docker_machine_path(path: Optional[str]) -> None: class DockerCommandLineJob(ContainerCommandLineJob): """Runs a CommandLineJob in a sofware container using the Docker engine.""" + CONTAINER_TMPDIR: str = "/tmp" # nosec + def __init__( self, builder: Builder, @@ -318,6 +320,15 @@ def add_writable_directory_volume( shutil.copytree(volume.resolved, host_outdir_tgt) ensure_writable(host_outdir_tgt or new_dir) + def _required_env(self) -> Dict[str, str]: + # spec currently says "HOME must be set to the designated output + # directory." but spec might change to designated temp directory. + # runtime.append("--env=HOME=/tmp") + return { + "TMPDIR": self.CONTAINER_TMPDIR, + "HOME": self.builder.outdir, + } + def create_runtime( self, env: MutableMapping[str, str], runtimeContext: RuntimeContext ) -> Tuple[List[str], Optional[str]]: @@ -335,9 +346,8 @@ def create_runtime( self.append_volume( runtime, os.path.realpath(self.outdir), self.builder.outdir, writable=True ) - tmpdir = "/tmp" # nosec self.append_volume( - runtime, os.path.realpath(self.tmpdir), tmpdir, writable=True + runtime, os.path.realpath(self.tmpdir), self.CONTAINER_TMPDIR, writable=True ) self.add_volumes( self.pathmapper, @@ -385,13 +395,6 @@ def create_runtime( if runtimeContext.rm_container: runtime.append("--rm") - runtime.append("--env=TMPDIR=/tmp") - - # spec currently says "HOME must be set to the designated output - # directory." but spec might change to designated temp directory. - # runtime.append("--env=HOME=/tmp") - runtime.append("--env=HOME=%s" % self.builder.outdir) - cidfile_path = None # type: Optional[str] # add parameters to docker to write a container ID file if runtimeContext.user_space_docker_cmd is None: diff --git a/cwltool/job.py b/cwltool/job.py index 6c826f63d2..ebf18d3205 100644 --- a/cwltool/job.py +++ b/cwltool/job.py @@ -15,8 +15,9 @@ from io import IOBase from threading import Timer from typing import ( - IO, Callable, + Dict, + IO, Iterable, List, Mapping, @@ -420,6 +421,23 @@ def _execute( ) shutil.rmtree(self.tmpdir, True) + @abstractmethod + def _required_env(self) -> Dict[str, str]: + """Variables required by the CWL spec (HOME, TMPDIR, etc). + + Note that with containers, the paths will (likely) be those from + inside. + """ + pass + + def _preserve_environment_on_containers_warning( + self, varname: Optional[str] = None + ) -> None: + """When running in a container, issue a warning.""" + # By default, don't do anything; ContainerCommandLineJob below + # will issue a warning. + pass + def prepare_environment( self, runtimeContext: RuntimeContext, envVarReq: Mapping[str, str] ) -> None: @@ -431,24 +449,24 @@ def prepare_environment( applied (in that order). """ # Start empty - env = {} + env: Dict[str, str] = {} # Preserve any env vars - vars_to_preserve = runtimeContext.preserve_environment - if runtimeContext.preserve_entire_environment is not False: - vars_to_preserve = os.environ - if vars_to_preserve: - for key, value in os.environ.items(): - if key in vars_to_preserve and key not in env: - env[key] = value + if runtimeContext.preserve_entire_environment: + self._preserve_environment_on_containers_warning() + env.update(os.environ) + elif runtimeContext.preserve_environment: + for key in runtimeContext.preserve_environment: + self._preserve_environment_on_containers_warning(key) + try: + env[key] = os.environ[key] + except KeyError: + _logger.warning( + f"Attempting to preserve environment variable '{key}' which is not present" + ) # Set required env vars - env["HOME"] = self.outdir - env["TMPDIR"] = self.tmpdir - if "PATH" not in env: - env["PATH"] = os.environ["PATH"] - if "SYSTEMROOT" not in env and "SYSTEMROOT" in os.environ: - env["SYSTEMROOT"] = os.environ["SYSTEMROOT"] + env.update(self._required_env()) # Apply EnvVarRequirement env.update(envVarReq) @@ -530,6 +548,15 @@ def run( self._execute([], self.environment, runtimeContext, monitor_function) + def _required_env(self) -> Dict[str, str]: + env = {} + env["HOME"] = self.outdir + env["TMPDIR"] = self.tmpdir + env["PATH"] = os.environ["PATH"] + if "SYSTEMROOT" in os.environ: + env["SYSTEMROOT"] = os.environ["SYSTEMROOT"] + return env + CONTROL_CODE_RE = r"\x1b\[[0-9;]*[a-zA-Z]" @@ -588,6 +615,19 @@ def add_writable_directory_volume( ) -> None: """Append a writable directory mapping to the runtime option list.""" + def _preserve_environment_on_containers_warning( + self, varname: Optional[str] = None + ) -> None: + """When running in a container, issue a warning.""" + if varname is None: + _logger.warning( + "You have specified `--preserve-entire-environment` while running a container" + ) + else: + _logger.warning( + f"You have specified `--preserve-environment={varname}` while running a container" + ) + def create_file_and_add_volume( self, runtime: List[str], diff --git a/cwltool/singularity.py b/cwltool/singularity.py index 29646765bd..e1d83d33b8 100644 --- a/cwltool/singularity.py +++ b/cwltool/singularity.py @@ -83,6 +83,8 @@ def _normalize_sif_id(string: str) -> str: class SingularityCommandLineJob(ContainerCommandLineJob): + CONTAINER_TMPDIR: str = "/tmp" # nosec + def __init__( self, builder: Builder, @@ -370,6 +372,12 @@ def add_writable_directory_volume( self.append_volume(runtime, source, volume.target, writable=True) ensure_writable(source) + def _required_env(self) -> Dict[str, str]: + return { + "TMPDIR": self.CONTAINER_TMPDIR, + "HOME": self.builder.outdir, + } + def create_runtime( self, env: MutableMapping[str, str], runtime_context: RuntimeContext ) -> Tuple[List[str], Optional[str]]: @@ -381,6 +389,7 @@ def create_runtime( "exec", "--contain", "--ipc", + "--cleanenv", ] if _singularity_supports_userns(): runtime.append("--userns") @@ -403,8 +412,12 @@ def create_runtime( ) ) runtime.append("--bind") - tmpdir = "/tmp" # nosec - runtime.append("{}:{}:rw".format(os.path.realpath(self.tmpdir), tmpdir)) + runtime.append( + "{}:{}:rw".format( + os.path.realpath(self.tmpdir), + self.CONTAINER_TMPDIR, + ) + ) self.add_volumes( self.pathmapper, @@ -432,9 +445,6 @@ def create_runtime( elif runtime_context.disable_net: runtime.append("--net") - env["SINGULARITYENV_TMPDIR"] = tmpdir - env["SINGULARITYENV_HOME"] = self.builder.outdir - for name, value in self.environment.items(): env[f"SINGULARITYENV_{name}"] = str(value) return (runtime, None) From a523c2f496f8d6940ff3c8d4fc7729aaf1380af7 Mon Sep 17 00:00:00 2001 From: Rupert Nash Date: Tue, 30 Mar 2021 17:12:27 +0100 Subject: [PATCH 048/111] Add proper tests for env vars and fix a few resulting issues --- cwltool/docker.py | 2 - cwltool/env_to_stdout.py | 33 ++++ cwltool/job.py | 47 ++++-- cwltool/run_job.py | 17 +- cwltool/singularity.py | 45 +++--- cwltool/software_requirements.py | 7 +- tests/env3.cwl | 1 + tests/env4.cwl | 19 +++ tests/test_dependencies.py | 42 ++--- tests/test_environment.py | 270 +++++++++++++++++++++++++++++++ tests/test_singularity.py | 9 +- tests/util.py | 92 ++++++++++- 12 files changed, 503 insertions(+), 81 deletions(-) create mode 100644 cwltool/env_to_stdout.py create mode 100644 tests/env4.cwl create mode 100644 tests/test_environment.py diff --git a/cwltool/docker.py b/cwltool/docker.py index 42f38b082b..e5e2ae45b3 100644 --- a/cwltool/docker.py +++ b/cwltool/docker.py @@ -78,8 +78,6 @@ def _check_docker_machine_path(path: Optional[str]) -> None: class DockerCommandLineJob(ContainerCommandLineJob): """Runs a CommandLineJob in a sofware container using the Docker engine.""" - CONTAINER_TMPDIR: str = "/tmp" # nosec - def __init__( self, builder: Builder, diff --git a/cwltool/env_to_stdout.py b/cwltool/env_to_stdout.py new file mode 100644 index 0000000000..a05cd82302 --- /dev/null +++ b/cwltool/env_to_stdout.py @@ -0,0 +1,33 @@ +r"""Python script that acts like (GNU coreutils) env -0. + +When run as a script, it prints the the environment as +`(VARNAME=value\0)*`. + +Ideally we would just use `env -0`, because python (thanks to PEPs 538 +and 540) will set zero to two environment variables to better handle +Unicode-locale interactions, however BSD familiy implementations of +`env` do not all support the `-0` flag so we supply this script that +produces equivalent output. +""" + +import os +from typing import Dict + + +def deserialize_env(data: str) -> Dict[str, str]: + """Deserialize the output of `env -0` to dictionary.""" + ans = {} + for item in data.strip("\0").split("\0"): + key, val = item.split("=", 1) + ans[key] = val + return ans + + +def main() -> None: + """Print the null-separated enviroment to stdout.""" + for k, v in os.environ.items(): + print(f"{k}={v}", end="\0") + + +if __name__ == "__main__": + main() diff --git a/cwltool/job.py b/cwltool/job.py index ebf18d3205..885358807b 100644 --- a/cwltool/job.py +++ b/cwltool/job.py @@ -38,7 +38,7 @@ from schema_salad.utils import json_dump, json_dumps from typing_extensions import TYPE_CHECKING -from . import run_job +from . import env_to_stdout, run_job from .builder import Builder, HasReqsHints from .context import RuntimeContext from .errors import UnsupportedRequirement, WorkflowException @@ -212,7 +212,18 @@ def _execute( runtimeContext: RuntimeContext, monitor_function=None, # type: Optional[Callable[[subprocess.Popen[str]], None]] ) -> None: + """Execute the tool, either directly or via script. + Note: we are now at the point where self.environment is + ignored. The caller is responsible for correctly splitting that + into the runtime and env arguments. + + `runtime` is the list of arguments to put at the start of the + command (e.g. docker run). + + `env` is the enviroment to be set for running the resulting + command line. + """ scr = self.get_requirement("ShellCommandRequirement")[0] shouldquote = needs_shell_quoting_re.search @@ -431,7 +442,7 @@ def _required_env(self) -> Dict[str, str]: pass def _preserve_environment_on_containers_warning( - self, varname: Optional[str] = None + self, varname: Optional[Iterable[str]] = None ) -> None: """When running in a container, issue a warning.""" # By default, don't do anything; ContainerCommandLineJob below @@ -456,8 +467,10 @@ def prepare_environment( self._preserve_environment_on_containers_warning() env.update(os.environ) elif runtimeContext.preserve_environment: + self._preserve_environment_on_containers_warning( + runtimeContext.preserve_environment + ) for key in runtimeContext.preserve_environment: - self._preserve_environment_on_containers_warning(key) try: env[key] = os.environ[key] except KeyError: @@ -564,6 +577,8 @@ def _required_env(self) -> Dict[str, str]: class ContainerCommandLineJob(JobBase, metaclass=ABCMeta): """Commandline job using containers.""" + CONTAINER_TMPDIR: str = "/tmp" # nosec + @abstractmethod def get_from_requirements( self, @@ -616,17 +631,17 @@ def add_writable_directory_volume( """Append a writable directory mapping to the runtime option list.""" def _preserve_environment_on_containers_warning( - self, varname: Optional[str] = None + self, varnames: Optional[Iterable[str]] = None ) -> None: """When running in a container, issue a warning.""" - if varname is None: - _logger.warning( - "You have specified `--preserve-entire-environment` while running a container" - ) + if varnames is None: + flags = "--preserve-entire-environment" else: - _logger.warning( - f"You have specified `--preserve-environment={varname}` while running a container" - ) + flags = "--preserve-environment={" + ", ".join(varnames) + "}" + + _logger.warning( + f"You have specified `{flags}` while running a container which will override variables set in the container. This may break the container, be non-portable, and/or affect reproducibility." + ) def create_file_and_add_volume( self, @@ -713,7 +728,6 @@ def run( (docker_req, docker_is_req) = self.get_requirement("DockerRequirement") self.prov_obj = runtimeContext.prov_obj img_id = None - env = cast(MutableMapping[str, str], os.environ) user_space_docker_cmd = runtimeContext.user_space_docker_cmd if docker_req is not None and user_space_docker_cmd: # For user-space docker implementations, a local image name or ID @@ -807,7 +821,11 @@ def run( ) self._setup(runtimeContext) + + # Copy as don't want to modify our env + env = dict(os.environ) (runtime, cidfile) = self.create_runtime(env, runtimeContext) + runtime.append(str(img_id)) monitor_function = None if cidfile: @@ -997,8 +1015,13 @@ def terminate(): # type: () -> None job_script = os.path.join(job_dir, "run_job.bash") with open(job_script, "wb") as _: _.write(job_script_contents.encode("utf-8")) + job_run = os.path.join(job_dir, "run_job.py") shutil.copyfile(run_job.__file__, job_run) + + env_getter = os.path.join(job_dir, "env_to_stdout.py") + shutil.copyfile(env_to_stdout.__file__, env_getter) + sproc = subprocess.Popen( # nosec ["bash", job_script.encode("utf-8")], shell=False, # nosec diff --git a/cwltool/run_job.py b/cwltool/run_job.py index f6f80a296e..93b4e8a94b 100644 --- a/cwltool/run_job.py +++ b/cwltool/run_job.py @@ -18,14 +18,15 @@ def handle_software_environment(cwl_env: Dict[str, str], script: str) -> Dict[st return cwl_env env = cwl_env.copy() - with open("output_environment.bash") as env_file: - for line in env_file: - key, val = line.split("=", 1) - if key in ("_", "PWD", "SHLVL", "TMPDIR", "HOME", "_CWLTOOL"): - # Skip some variables that are meaningful to the shell - # or set specifically by the CWL runtime environment. - continue - env[key] = val[:-1] # remove trailing newline + with open("output_environment.dat") as _: + data = _.read().strip("\0") + for line in data.split("\0"): + key, val = line.split("=", 1) + if key in ("_", "PWD", "SHLVL", "TMPDIR", "HOME", "_CWLTOOL"): + # Skip some variables that are meaningful to the shell or set + # specifically by the CWL runtime environment. + continue + env[key] = val return env diff --git a/cwltool/singularity.py b/cwltool/singularity.py index e1d83d33b8..49b62bf562 100644 --- a/cwltool/singularity.py +++ b/cwltool/singularity.py @@ -83,8 +83,6 @@ def _normalize_sif_id(string: str) -> str: class SingularityCommandLineJob(ContainerCommandLineJob): - CONTAINER_TMPDIR: str = "/tmp" # nosec - def __init__( self, builder: Builder, @@ -277,13 +275,12 @@ def append_volume( runtime: List[str], source: str, target: str, writable: bool = False ) -> None: runtime.append("--bind") - runtime.append( - "{}:{}:{}".format( - source, - target, - "rw" if writable else "ro", - ) - ) + # Mounts are writable by default, so 'rw' is optional and not + # supported (due to a bug) in some 3.6 series releases. + vol = f"{source}:{target}" + if not writable: + vol += ":ro" + runtime.append(vol) def add_file_or_directory_volume( self, runtime: List[str], volume: MapperEnt, host_outdir_tgt: Optional[str] @@ -395,28 +392,28 @@ def create_runtime( runtime.append("--userns") else: runtime.append("--pid") + + container_HOME: Optional[str] = None if is_version_3_1_or_newer(): + # Remove HOME, as passed in a special way (restore it below) + container_HOME = self.environment.pop("HOME") runtime.append("--home") runtime.append( "{}:{}".format( os.path.realpath(self.outdir), - self.builder.outdir, + container_HOME, ) ) else: - runtime.append("--bind") - runtime.append( - "{}:{}:rw".format( - os.path.realpath(self.outdir), - self.builder.outdir, - ) - ) - runtime.append("--bind") - runtime.append( - "{}:{}:rw".format( - os.path.realpath(self.tmpdir), - self.CONTAINER_TMPDIR, + self.append_volume( + runtime, + os.path.realpath(self.outdir), + self.environment["HOME"], + writable=True, ) + + self.append_volume( + runtime, os.path.realpath(self.tmpdir), self.CONTAINER_TMPDIR, writable=True ) self.add_volumes( @@ -447,4 +444,8 @@ def create_runtime( for name, value in self.environment.items(): env[f"SINGULARITYENV_{name}"] = str(value) + + if container_HOME: + # Restore HOME if we removed it above. + self.environment["HOME"] = container_HOME return (runtime, None) diff --git a/cwltool/software_requirements.py b/cwltool/software_requirements.py index d4e67f6286..bdd216055a 100644 --- a/cwltool/software_requirements.py +++ b/cwltool/software_requirements.py @@ -32,7 +32,12 @@ """#!/bin/bash cat > modify_environment.bash <<'EOF' $handle_dependencies -env > output_environment.bash +# First try env -0 +if ! env -0 > "output_environment.dat" 2> /dev/null; then + # If that fails, use the python script. + # In some circumstances (see PEP 538) this will the add LC_CTYPE env var. + python3 "env_to_stdout.py" > "output_environment.dat" +fi EOF python3 "run_job.py" "job.json" "modify_environment.bash" """ diff --git a/tests/env3.cwl b/tests/env3.cwl index d38826c2ea..14718fd02a 100644 --- a/tests/env3.cwl +++ b/tests/env3.cwl @@ -3,6 +3,7 @@ cwlVersion: v1.0 class: CommandLineTool inputs: [] baseCommand: env +arguments: ["-0"] outputs: env: type: stdout diff --git a/tests/env4.cwl b/tests/env4.cwl new file mode 100644 index 0000000000..13f988b520 --- /dev/null +++ b/tests/env4.cwl @@ -0,0 +1,19 @@ +#!/usr/bin/env cwl-runner +cwlVersion: v1.0 +class: CommandLineTool + +requirements: + InitialWorkDirRequirement: + listing: + - entryname: env0.py + entry: | + import os + for k, v in os.environ.items(): + print(f"{k}={v}", end="\0") + +inputs: [] +baseCommand: python3 +arguments: ["env0.py"] +outputs: + env: + type: stdout diff --git a/tests/test_dependencies.py b/tests/test_dependencies.py index e2a235a40b..b5ad609396 100644 --- a/tests/test_dependencies.py +++ b/tests/test_dependencies.py @@ -1,17 +1,21 @@ """Tests of satisfying SoftwareRequirement via dependencies.""" -from io import StringIO -import json import os from pathlib import Path from shutil import which from types import ModuleType -from typing import Optional +from typing import Dict, List, Optional import pytest from cwltool.main import main -from .util import get_data, get_main_output, needs_docker, working_directory +from .util import ( + get_data, + get_main_output, + get_tool_env, + needs_docker, + working_directory, +) deps = None # type: Optional[ModuleType] try: @@ -74,29 +78,15 @@ def test_modules_environment(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> monkeypatch.setenv( "MODULEPATH", os.path.join(os.getcwd(), "tests/test_deps_env/modulefiles") ) + tool_env = get_tool_env( + tmp_path, + [ + "--beta-dependency-resolvers-configuration", + get_data("tests/test_deps_env_modules_resolvers_conf.yml"), + ], + get_data("tests/env_with_software_req.yml"), + ) - stdout = StringIO() - stderr = StringIO() - with working_directory(tmp_path): - rc = main( - argsl=[ - "--beta-dependency-resolvers-configuration", - get_data("tests/test_deps_env_modules_resolvers_conf.yml"), - get_data("tests/env3.cwl"), - get_data("tests/env_with_software_req.yml"), - ], - stdout=stdout, - stderr=stderr, - ) - assert rc == 0 - - output = json.loads(stdout.getvalue()) - env_path = output["env"]["path"] - tool_env = {} - with open(env_path) as _: - for line in _: - key, val = line.split("=", 1) - tool_env[key] = val[:-1] assert tool_env["TEST_VAR_MODULE"] == "environment variable ends in space " tool_path = tool_env["PATH"].split(":") assert get_data("tests/test_deps_env/random-lines/1.0/scripts") in tool_path diff --git a/tests/test_environment.py b/tests/test_environment.py new file mode 100644 index 0000000000..8352de98c4 --- /dev/null +++ b/tests/test_environment.py @@ -0,0 +1,270 @@ +"""Test passing of environment variables to tools.""" +from abc import ABC, abstractmethod +import os +from pathlib import Path +from typing import Any, Callable, Dict, Iterator, List, Mapping, Set, Union, cast + +import pytest + +from cwltool.singularity import get_version +from .util import env_accepts_null, get_tool_env, needs_docker, needs_singularity + +# None => accept anything, just require the key is present +# str => string equality +# Callable => call the function with the value - True => OK, False => fail +# TODO: maybe add regex? +Env = Mapping[str, str] +CheckerTypes = Union[None, str, Callable[[str, Env], bool]] +EnvChecks = Dict[str, CheckerTypes] + + +def assert_envvar_matches(check: CheckerTypes, k: str, env: Mapping[str, str]) -> None: + """Assert that the check is satisfied by the key in the env.""" + if check is None: + pass + else: + v = env[k] + if isinstance(check, str): + assert v == check, f'Environment variable {k} == "{v}" != "{check}"' + else: + assert check(v, env), f'Environment variable {k}="{v}" fails check' + + +def assert_env_matches( + checks: EnvChecks, env: Mapping[str, str], allow_unexpected: bool = False +) -> None: + """Assert that all checks are satisfied by the Mapping. + + Optional flag `allow_unexpected` (default = False) will allow the + Mapping to contain extra keys which are not checked. + """ + e = dict(env) + for k, check in checks.items(): + v = e.pop(k) + assert_envvar_matches(check, k, env) + + if not allow_unexpected: + # If we have to use env4.cwl, there may be unwanted variables + # (see cwltool.env_to_stdout docstrings). + # LC_CTYPE if platform has glibc + # __CF_USER_TEXT_ENCODING on macOS + if not env_accepts_null(): + e.pop("LC_CTYPE", None) + e.pop("__CF_USER_TEXT_ENCODING", None) + assert len(e) == 0, f"Unexpected environment variable(s): {', '.join(e.keys())}" + + +class CheckHolder(ABC): + """Base class for check factory functions and other data required to parametrize the tests below.""" + + @staticmethod + @abstractmethod + def checks(tmp_prefix: str) -> EnvChecks: + """Return a mapping from environment variable names to how to check for correctness.""" + pass + + # Any flags to pass to cwltool to force use of the correct container + flags: List[str] + + # Does the env tool (maybe in our container) accept a `-0` flag? + env_accepts_null: bool + + pass + + +class NoContainer(CheckHolder): + """No containers at all, just run in the host.""" + + @staticmethod + def checks(tmp_prefix: str) -> EnvChecks: + """Create checks.""" + return { + "TMPDIR": lambda v, _: v.startswith(tmp_prefix), + "HOME": lambda v, _: v.startswith(tmp_prefix), + "PATH": os.environ["PATH"], + } + + flags = ["--no-container"] + env_accepts_null = env_accepts_null() + + +class Docker(CheckHolder): + """Run in a Docker container.""" + + @staticmethod + def checks(tmp_prefix: str) -> EnvChecks: + """Create checks.""" + + def HOME(v: str, env: Env) -> bool: + # Want /whatever + parts = os.path.split(v) + return len(parts) == 2 and parts[0] == "/" + + return { + "HOME": HOME, + "TMPDIR": "/tmp", + "PATH": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", + "HOSTNAME": None, + } + + flags = ["--default-container=debian"] + env_accepts_null = True + + +class Singularity(CheckHolder): + """Run in a Singularity container.""" + + @staticmethod + def checks(tmp_prefix: str) -> EnvChecks: + """Create checks.""" + + def PWD(v: str, env: Env) -> bool: + return v == env["HOME"] + + ans: EnvChecks = { + "HOME": None, + "LANG": "C", + "LD_LIBRARY_PATH": None, + "PATH": None, + "PROMPT_COMMAND": None, + "PS1": None, + "PWD": PWD, + "TMPDIR": "/tmp", + } + + # Singularity variables appear to be in flux somewhat. + version = get_version().split(".") + vmajor = int(version[0]) + assert vmajor == 3, "Tests only work for Singularity 3" + vminor = int(version[1]) + sing_vars: EnvChecks = { + "SINGULARITY_CONTAINER": None, + "SINGULARITY_ENVIRONMENT": None, + "SINGULARITY_NAME": None, + } + if vminor < 6: + sing_vars["SINGULARITY_INIT"] = "1" + else: + sing_vars["SINGULARITY_COMMAND"] = "exec" + if vminor >= 7: + + def BIND(v: str, env: Env) -> bool: + return v.startswith(tmp_prefix) and v.endswith(":/tmp") + + sing_vars["SINGULARITY_BIND"] = BIND + + ans.update(sing_vars) + + # Singularity automatically passes some variables through, if + # they exist. This seems to be constant from 3.1 but isn't + # documented (see source /internal/pkg/util/env/clean.go). + autopass = ( + "ALL_PROXY", + "FTP_PROXY", + "HTTP_PROXY", + "HTTPS_PROXY", + "NO_PROXY", + "TERM", + ) + for vname in autopass: + if vname in os.environ: + ans[vname] = os.environ[vname] + + return ans + + flags = ["--default-container=debian", "--singularity"] + env_accepts_null = True + + +# CRT = container runtime +CRT_PARAMS = pytest.mark.parametrize( + "crt_params", + [ + NoContainer(), + pytest.param(Docker(), marks=needs_docker), + pytest.param(Singularity(), marks=needs_singularity), + ], +) + + +@CRT_PARAMS +def test_basic(crt_params: CheckHolder, tmp_path: Path, monkeypatch: Any) -> None: + """Test that basic env vars (only) show up.""" + tmp_prefix = str(tmp_path / "canary") + extra_env = { + "USEDVAR": "VARVAL", + "UNUSEDVAR": "VARVAL", + } + args = crt_params.flags + [f"--tmpdir-prefix={tmp_prefix}"] + env = get_tool_env( + tmp_path, + args, + extra_env=extra_env, + monkeypatch=monkeypatch, + runtime_env_accepts_null=crt_params.env_accepts_null, + ) + checks = crt_params.checks(tmp_prefix) + assert_env_matches(checks, env) + + +@CRT_PARAMS +def test_preserve_single( + crt_params: CheckHolder, tmp_path: Path, monkeypatch: Any +) -> None: + """Test that preserving a single env var works.""" + tmp_prefix = str(tmp_path / "canary") + extra_env = { + "USEDVAR": "VARVAL", + "UNUSEDVAR": "VARVAL", + } + args = crt_params.flags + [ + f"--tmpdir-prefix={tmp_prefix}", + "--preserve-environment=USEDVAR", + ] + env = get_tool_env( + tmp_path, + args, + extra_env=extra_env, + monkeypatch=monkeypatch, + runtime_env_accepts_null=crt_params.env_accepts_null, + ) + checks = crt_params.checks(tmp_prefix) + checks["USEDVAR"] = extra_env["USEDVAR"] + assert_env_matches(checks, env) + + +@CRT_PARAMS +def test_preserve_all( + crt_params: CheckHolder, tmp_path: Path, monkeypatch: Any +) -> None: + """Test that preserving all works.""" + tmp_prefix = str(tmp_path / "canary") + extra_env = { + "USEDVAR": "VARVAL", + "UNUSEDVAR": "VARVAL", + } + args = crt_params.flags + [ + f"--tmpdir-prefix={tmp_prefix}", + "--preserve-entire-environment", + ] + env = get_tool_env( + tmp_path, + args, + extra_env=extra_env, + monkeypatch=monkeypatch, + runtime_env_accepts_null=crt_params.env_accepts_null, + ) + checks = crt_params.checks(tmp_prefix) + checks.update(extra_env) + + for vname, val in env.items(): + try: + assert_envvar_matches(checks[vname], vname, env) + except KeyError: + assert val == os.environ[vname] + except AssertionError: + if vname == "HOME" or vname == "TMPDIR": + # These MUST be OK + raise + # Other variables can be overriden + assert val == os.environ[vname] diff --git a/tests/test_singularity.py b/tests/test_singularity.py index 87395bb9de..66fe094f10 100644 --- a/tests/test_singularity.py +++ b/tests/test_singularity.py @@ -2,6 +2,7 @@ import shutil import os from pathlib import Path +from typing import Any from cwltool.main import main @@ -15,14 +16,13 @@ @needs_singularity_2_6 -def test_singularity_pullfolder(tmp_path: Path) -> None: +def test_singularity_pullfolder(tmp_path: Path, monkeypatch: Any) -> None: + """Test singularity respects SINGULARITY_PULLFOLDER.""" workdir = tmp_path / "working_dir_new" workdir.mkdir() with working_directory(workdir): pullfolder = tmp_path / "pullfolder" pullfolder.mkdir() - env = os.environ.copy() - env["SINGULARITY_PULLFOLDER"] = str(pullfolder) result_code, stdout, stderr = get_main_output( [ "--singularity", @@ -30,7 +30,8 @@ def test_singularity_pullfolder(tmp_path: Path) -> None: "--message", "hello", ], - env=env, + extra_env={"SINGULARITY_PULLFOLDER": str(pullfolder)}, + monkeypatch=monkeypatch, ) print(stdout) print(stderr) diff --git a/tests/util.py b/tests/util.py index b064ec389b..5674107187 100644 --- a/tests/util.py +++ b/tests/util.py @@ -1,13 +1,18 @@ import contextlib +import functools import io +import json import os import shutil +import subprocess from pathlib import Path -from typing import Generator, List, Mapping, Optional, Tuple, Union +import sys +from typing import Dict, Generator, List, Mapping, Optional, Tuple, Union import pytest from pkg_resources import Requirement, ResolutionError, resource_filename +from cwltool.env_to_stdout import deserialize_env from cwltool.main import main from cwltool.singularity import is_version_2_6, is_version_3_or_newer @@ -49,19 +54,53 @@ def get_data(filename: str) -> str: reason="Requires that version 3.x of singularity executable version is on the system path.", ) +_env_accepts_null: Optional[bool] = None + + +def env_accepts_null() -> bool: + """Return True iff the env command on this host accepts `-0`.""" + global _env_accepts_null + if _env_accepts_null is None: + result = subprocess.run( + ["env", "-0"], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + encoding="utf-8", + ) + _env_accepts_null = result.returncode == 0 + + return _env_accepts_null + def get_main_output( args: List[str], - env: Union[ - Mapping[bytes, Union[bytes, str]], Mapping[str, Union[bytes, str]], None - ] = None, + replacement_env: Optional[Mapping[str, str]] = None, + extra_env: Optional[Mapping[str, str]] = None, monkeypatch: Optional[pytest.MonkeyPatch] = None, ) -> Tuple[Optional[int], str, str]: + """Run cwltool main. + + args: the command line args to call it with + + replacement_env: a total replacement of the environment + + extra_env: add these to the environment used + + monkeypatch: required if changing the environment + + Returns (return code, stdout, stderr) + """ stdout = io.StringIO() stderr = io.StringIO() - if env is not None: + if replacement_env is not None: assert monkeypatch is not None - monkeypatch.setattr(os, "environ", env) + monkeypatch.setattr(os, "environ", replacement_env) + + if extra_env is not None: + assert monkeypatch is not None + for k, v in extra_env.items(): + monkeypatch.setenv(k, v) + try: rc = main(argsl=args, stdout=stdout, stderr=stderr) except SystemExit as e: @@ -73,6 +112,47 @@ def get_main_output( ) +def get_tool_env( + tmp_path: Path, + flag_args: List[str], + inputs_file: Optional[str] = None, + replacement_env: Optional[Mapping[str, str]] = None, + extra_env: Optional[Mapping[str, str]] = None, + monkeypatch: Optional[pytest.MonkeyPatch] = None, + runtime_env_accepts_null: Optional[bool] = None, +) -> Dict[str, str]: + """Get the env vars for a tool's invocation.""" + # GNU env accepts the -0 option to end each variable's + # printing with "\0". No such luck on BSD-ish. + # + # runtime_env_accepts_null is None => figure it out, otherwise + # use wrapped bool (because containers). + if runtime_env_accepts_null is None: + runtime_env_accepts_null = env_accepts_null() + + args = flag_args.copy() + if runtime_env_accepts_null: + args.append(get_data("tests/env3.cwl")) + else: + args.append(get_data("tests/env4.cwl")) + + if inputs_file is not None: + args.append(inputs_file) + + with working_directory(tmp_path): + rc, stdout, _ = get_main_output( + args, + replacement_env=replacement_env, + extra_env=extra_env, + monkeypatch=monkeypatch, + ) + assert rc == 0 + + output = json.loads(stdout) + with open(output["env"]["path"]) as _: + return deserialize_env(_.read()) + + @contextlib.contextmanager def working_directory(path: Union[str, Path]) -> Generator[None, None, None]: """Change working directory and returns to previous on exit.""" From 4248e074709b9890b3f9c5a11d3b1a93b7307f50 Mon Sep 17 00:00:00 2001 From: Rupert Nash Date: Mon, 26 Apr 2021 13:16:47 +0100 Subject: [PATCH 049/111] Enable conformance tests with Singularity in CI Also a minor fix to NetworkAccess handling for singularity --- .github/workflows/ci-tests.yml | 8 ++ conformance-test.sh | 226 ++++++++++++++++++++------------- cwltool/context.py | 3 +- cwltool/singularity.py | 13 +- 4 files changed, 155 insertions(+), 95 deletions(-) diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index d9ca39510a..e2aafa2174 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -123,10 +123,16 @@ jobs: strategy: matrix: cwl-version: [v1.0, v1.1, v1.2] + container: [docker, singularity] steps: - uses: actions/checkout@v2 + - name: Set up Singularity + uses: eWaterCycle/setup-singularity@v6 + with: + singularity-version: ${{ env.singularity_version }} + - name: Set up Python uses: actions/setup-python@v2 with: @@ -141,6 +147,8 @@ jobs: - name: Run CWL conformance tests ${{ matrix.cwl-version }} env: version: ${{ matrix.cwl-version }} + container: ${{ matrix.container }} + spec_branch: main run: ./conformance-test.sh release_test: diff --git a/conformance-test.sh b/conformance-test.sh index 1d4bb628cb..e455055a2c 100755 --- a/conformance-test.sh +++ b/conformance-test.sh @@ -1,45 +1,70 @@ #!/bin/bash venv() { - if ! test -d "$1" ; then - virtualenv -p python3 "$1" - fi - # shellcheck source=/dev/null - source "$1"/bin/activate + if ! test -d "$1" ; then + if command -v virtualenv > /dev/null; then + virtualenv -p python3 "$1" + else + python3 -m venv "$1" + fi + fi + # shellcheck source=/dev/null + source "$1"/bin/activate } + +# Set these environment variables when running the script, e.g.: +# version=v1.1 spec_branch=new_test container=docker ./conformance_test.sh + +# Version of the standard to test against +# Current options: v1.0, v1.1, v1.2 version=${version:-v1.0} + +# Which branch of the standard's repo to use. +# This can be useful when adding new features +spec_branch=${spec_branch:-main} + +# Which container runtime to use +# Valid options: docker, singularity +container=${container:-docker} + +set -e +set -x + if [[ "$version" = "v1.0" ]] ; then - wget https://github.com/common-workflow-language/common-workflow-language/archive/main.tar.gz - tar xzf main.tar.gz && rm main.tar.gz + repo=common-workflow-language else # shellcheck disable=SC2001 - repo=$(echo "$version" | sed 's/\(v[0-9]*\.\)\([0-9]*\).*/\1\2/') - wget https://github.com/common-workflow-language/cwl-"${repo}"/archive/main.tar.gz - tar xzf main.tar.gz && rm main.tar.gz + repo=cwl-$(echo "$version" | sed 's/\(v[0-9]*\.\)\([0-9]*\).*/\1\2/') +fi + +if [ ! -d "${repo}-${spec_branch}" ]; then + if [ ! -f "${repo}-${spec_branch}.tar.gz" ]; then + wget "https://github.com/common-workflow-language/${repo}/archive/${spec_branch}.tar.gz" + fi + tar xzf "${spec_branch}.tar.gz" +fi + +if [ "${container}" == "docker" ]; then + docker pull node:slim fi -docker pull node:slim +venv cwltool-venv3 +pip3 install -U setuptools wheel pip +pip3 uninstall -y cwltool +pip3 install -e . +pip3 install codecov cwltest>=2.1 +pushd "${repo}-${spec_branch}" || exit 1 # shellcheck disable=SC2043 -for CONTAINER in docker -# for CONTAINER in docker singularity -# singularity having issues on ci.commonwl.org; tests pass with https://gist.github.com/mr-c/0ec90d717617d074017c0cb38b72d1a4 -do - venv cwltool-venv3 - pip3 install -U setuptools wheel pip - pip3 uninstall -y cwltool - pip3 install -e . - pip3 install "cwltest>=1.0.20180518074130" codecov - if [[ "$version" = "v1.0" ]] - then - DRAFT="DRAFT=v1.0" - pushd common-workflow-language-main || exit 1 - else - pushd cwl-"${repo}"-main || exit 1 - fi - rm -f .coverage* coverage.xml - source=$(realpath ../cwltool) - COVERAGE_RC=${PWD}/.coveragerc - cat > "${COVERAGE_RC}" < "${COVERAGE_RC}" < "${CWLTOOL_WITH_COV}" < "${CWLTOOL_WITH_COV}" < 0 )); then + EXCLUDE=-S$(IFS=,; echo "${exclusions[*]}") +else + EXCLUDE="" +fi +# shellcheck disable=SC2086 +LC_ALL=C.UTF-8 ./run_test.sh --junit-xml=result3.xml ${EXCLUDE} \ + RUNNER=${CWLTOOL_WITH_COV} "-j$(nproc)" ${BADGE} \ + ${DRAFT} "${EXTRA}" \ + "--classname=py3_${container}" +# LC_ALL=C is to work around junit-xml ASCII only bug + +# capture return code of ./run_test.sh +CODE=$? + +find . -name '.coverage.*' -print0 | xargs -0 coverage combine --rcfile="${COVERAGE_RC}" --append +coverage xml --rcfile="${COVERAGE_RC}" +codecov --file coverage.xml + +if [ -d conformance ] +then + rm -rf conformance/cwltool/cwl_"${version}"/cwltool_latest + cp -r conformance/cwltool/cwl_"${version}"/cwltool_"${tool_ver}" conformance/cwltool/cwl_"${version}"/cwltool_latest + git -C conformance add --all + git -C conformance diff-index --quiet HEAD || git -C conformance commit -m "${CONFORMANCE_MSG}" + git -C conformance push http://"${jenkins_cwl_conformance}":x-oauth-basic@github.com/common-workflow-language/conformance.git +fi + +popd || exit +deactivate - deactivate - popd || exit -done # build new docker container if [ "$GIT_BRANCH" = "origin/main" ] && [[ "$version" = "v1.0" ]] then diff --git a/cwltool/context.py b/cwltool/context.py index bf4154b574..83c098e12c 100644 --- a/cwltool/context.py +++ b/cwltool/context.py @@ -93,7 +93,7 @@ def __init__(self, kwargs: Optional[Dict[str, Any]] = None) -> None: self.user_space_docker_cmd = "" # type: Optional[str] self.secret_store = None # type: Optional[SecretStore] self.no_read_only = False # type: bool - self.custom_net = "" # type: Optional[str] + self.custom_net = None # type: Optional[str] self.no_match_user = False # type: bool self.preserve_environment = None # type: Optional[Iterable[str]] self.preserve_entire_environment = False # type: bool @@ -109,7 +109,6 @@ def __init__(self, kwargs: Optional[Dict[str, Any]] = None) -> None: self.move_outputs = "move" # type: str self.singularity = False # type: bool - self.disable_net = False # type: bool self.debug = False # type: bool self.compute_checksum = True # type: bool self.name = "" # type: str diff --git a/cwltool/singularity.py b/cwltool/singularity.py index 49b62bf562..7de8ee5658 100644 --- a/cwltool/singularity.py +++ b/cwltool/singularity.py @@ -19,7 +19,7 @@ from .builder import Builder from .context import RuntimeContext -from .errors import UnsupportedRequirement, WorkflowException +from .errors import WorkflowException from .job import ContainerCommandLineJob from .loghandler import _logger from .pathmapper import MapperEnt, PathMapper @@ -435,12 +435,11 @@ def create_runtime( runtime.append("--pwd") runtime.append(self.builder.outdir) - if runtime_context.custom_net: - raise UnsupportedRequirement( - "Singularity implementation does not support custom networking" - ) - elif runtime_context.disable_net: - runtime.append("--net") + if self.networkaccess: + if runtime_context.custom_net: + runtime.extend(["--net", "--network", runtime_context.custom_net]) + else: + runtime.extend(["--net", "--network", "none"]) for name, value in self.environment.items(): env[f"SINGULARITYENV_{name}"] = str(value) From 7bdc74dd20cdad688e8b46862aabc52afdbacbb2 Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" Date: Mon, 26 Apr 2021 15:25:15 +0200 Subject: [PATCH 050/111] include main requirements from the test requirements --- test-requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/test-requirements.txt b/test-requirements.txt index 1f2ed9d003..a608d13aa1 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -5,3 +5,4 @@ pytest-mock >= 1.10.0 pytest-cov arcp >= 0.2.0 rdflib-jsonld >= 0.4.0 +-rrequirements.txt From 9ffdfe50aa5c8006d35a2a6f0ba22b772567a57f Mon Sep 17 00:00:00 2001 From: Rupert Nash Date: Fri, 21 May 2021 11:34:31 +0100 Subject: [PATCH 051/111] update minimum required version of schema_salad --- requirements.txt | 2 +- setup.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index f1e7edb4de..5f768ab130 100755 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ requests>=2.4.3 ruamel.yaml>=0.12.4,<=0.16.5 rdflib>=4.2.2,<5.1 shellescape>=3.4.1,<3.5 -schema-salad>=7.1,<8 +schema-salad>=7.1.20210518142926,<8 prov==1.5.1 bagit==1.8.1 mypy-extensions diff --git a/setup.py b/setup.py index 1108115d7b..20a34c449a 100755 --- a/setup.py +++ b/setup.py @@ -111,7 +111,9 @@ "ruamel.yaml >= 0.12.4, <= 0.16.5", "rdflib >= 4.2.2, < 5.1.0", "shellescape >= 3.4.1, < 3.5", - "schema-salad >= 7.1, < 8", + # 7.1.20210518142926 or later required due to + # https://github.com/common-workflow-language/schema_salad/issues/385 + "schema-salad >= 7.1.20210518142926, < 8", "mypy-extensions", "psutil", "prov == 1.5.1", From 695a6fa9bd3f590c456d0a3150e8341f8a583dde Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" <1330696+mr-c@users.noreply.github.com> Date: Thu, 10 Jun 2021 11:51:28 +0200 Subject: [PATCH 052/111] enable dependabot updates --- .github/dependabot.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 .github/dependabot.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000000..cf7a39fb64 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,11 @@ +# To get started with Dependabot version updates, you'll need to specify which +# package ecosystems to update and where the package manifests are located. +# Please see the documentation for all configuration options: +# https://help.github.com/github/administering-a-repository/configuration-options-for-dependency-updates + +version: 2 +updates: + - package-ecosystem: "pip" # See documentation for possible values + directory: "/" # Location of package manifests + schedule: + interval: "daily" From 9c4d5f3169002f3b9321540cf31793e8e4cf0bb7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 10 Jun 2021 10:18:12 +0000 Subject: [PATCH 053/111] Update shellescape requirement from <3.5,>=3.4.1 to >=3.4.1,<3.9 Updates the requirements on [shellescape](https://github.com/chrissimpkins/shellescape) to permit the latest version. - [Release notes](https://github.com/chrissimpkins/shellescape/releases) - [Commits](https://github.com/chrissimpkins/shellescape/compare/v3.4.1...v3.8.1) --- updated-dependencies: - dependency-name: shellescape dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) mode change 100755 => 100644 requirements.txt mode change 100755 => 100644 setup.py diff --git a/requirements.txt b/requirements.txt old mode 100755 new mode 100644 index 5f768ab130..18090e8324 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ requests>=2.4.3 ruamel.yaml>=0.12.4,<=0.16.5 rdflib>=4.2.2,<5.1 -shellescape>=3.4.1,<3.5 +shellescape>=3.4.1,<3.9 schema-salad>=7.1.20210518142926,<8 prov==1.5.1 bagit==1.8.1 diff --git a/setup.py b/setup.py old mode 100755 new mode 100644 index 20a34c449a..65352187a2 --- a/setup.py +++ b/setup.py @@ -110,7 +110,7 @@ # https://github.com/ionrock/cachecontrol/issues/137 "ruamel.yaml >= 0.12.4, <= 0.16.5", "rdflib >= 4.2.2, < 5.1.0", - "shellescape >= 3.4.1, < 3.5", + "shellescape >= 3.4.1, < 3.9", # 7.1.20210518142926 or later required due to # https://github.com/common-workflow-language/schema_salad/issues/385 "schema-salad >= 7.1.20210518142926, < 8", From 1cdc689d1a523ec71a9757ddcef6b853111e9d0f Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" Date: Thu, 10 Jun 2021 12:51:15 +0200 Subject: [PATCH 054/111] fix release test --- Makefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index f2b807aae3..8d8438d4f8 100644 --- a/Makefile +++ b/Makefile @@ -67,7 +67,7 @@ dev: install-dep dist: dist/${MODULE}-$(VERSION).tar.gz dist/${MODULE}-$(VERSION).tar.gz: $(SOURCES) - ./setup.py sdist bdist_wheel + python setup.py sdist bdist_wheel ## docs : make the docs docs: FORCE @@ -76,7 +76,7 @@ docs: FORCE ## clean : clean up all temporary / machine-generated files clean: FORCE rm -f ${MODILE}/*.pyc tests/*.pyc - ./setup.py clean --all || true + python setup.py clean --all || true rm -Rf .coverage rm -f diff-cover.html @@ -187,7 +187,7 @@ release-test: FORCE release: release-test . testenv2/bin/activate && \ - testenv2/src/${MODULE}/setup.py sdist bdist_wheel && \ + python testenv2/src/${MODULE}/setup.py sdist bdist_wheel && \ pip install twine && \ twine upload testenv2/src/${MODULE}/dist/* && \ git tag ${VERSION} && git push --tags From 7a68ce4c94e9bb1c5359e11c0efb065c9ba728c4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 10 Jun 2021 11:29:08 +0000 Subject: [PATCH 055/111] Update psutil requirement from <5.8.0 to <5.9.0 Updates the requirements on [psutil](https://github.com/giampaolo/psutil) to permit the latest version. - [Release notes](https://github.com/giampaolo/psutil/releases) - [Changelog](https://github.com/giampaolo/psutil/blob/master/HISTORY.rst) - [Commits](https://github.com/giampaolo/psutil/compare/release-0.1.0...release-5.8.0) --- updated-dependencies: - dependency-name: psutil dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 18090e8324..037ac52b10 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,7 +6,7 @@ schema-salad>=7.1.20210518142926,<8 prov==1.5.1 bagit==1.8.1 mypy-extensions -psutil<5.8.0 +psutil<5.9.0 typing-extensions coloredlogs pydot>=1.4.1 From be0869afbc94202b66baac6ff8e6eb754c0a7880 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 10 Jun 2021 11:58:42 +0000 Subject: [PATCH 056/111] Update ruamel-yaml requirement Updates the requirements on [ruamel-yaml](https://sourceforge.net/p/ruamel-yaml/code/ci/default/tree) to permit the latest version. --- updated-dependencies: - dependency-name: ruamel-yaml dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 037ac52b10..2f57d7fc2b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ requests>=2.4.3 -ruamel.yaml>=0.12.4,<=0.16.5 +ruamel.yaml>=0.12.4,<0.17.10 rdflib>=4.2.2,<5.1 shellescape>=3.4.1,<3.9 schema-salad>=7.1.20210518142926,<8 diff --git a/setup.py b/setup.py index 65352187a2..2b6a0e57c9 100644 --- a/setup.py +++ b/setup.py @@ -108,7 +108,7 @@ "setuptools", "requests >= 2.6.1", # >= 2.6.1 to workaround # https://github.com/ionrock/cachecontrol/issues/137 - "ruamel.yaml >= 0.12.4, <= 0.16.5", + "ruamel.yaml >= 0.12.4, < 0.17.10", "rdflib >= 4.2.2, < 5.1.0", "shellescape >= 3.4.1, < 3.9", # 7.1.20210518142926 or later required due to From d91e812e8aafe58692991f04cbeeeb9b823cf7bc Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" <1330696+mr-c@users.noreply.github.com> Date: Thu, 10 Jun 2021 15:01:46 +0200 Subject: [PATCH 057/111] --single-step: fix typo ; add --single-process (#1444) --- cwltool/argparser.py | 10 ++++++++ cwltool/job.py | 2 +- cwltool/main.py | 48 +++++++++++++++++++++++++++++++++++++-- cwltool/subgraph.py | 16 +++++++++++++ tests/test_environment.py | 3 ++- tests/test_singularity.py | 2 +- tests/util.py | 2 +- 7 files changed, 77 insertions(+), 6 deletions(-) diff --git a/cwltool/argparser.py b/cwltool/argparser.py index cae924bcf3..fef777de86 100644 --- a/cwltool/argparser.py +++ b/cwltool/argparser.py @@ -644,6 +644,16 @@ def arg_parser() -> argparse.ArgumentParser: help="Only executes a single step in a workflow. The input object must " "match that step's inputs. Can be combined with --print-subgraph.", ) + subgroup.add_argument( + "--single-process", + type=str, + default=None, + help="Only executes the underlying Process (CommandLineTool, " + "ExpressionTool, or sub-Workflow) for the given step in a workflow. " + "This will not include any step-level processing: scatter, when, no " + "processing of step-level default, or valueFrom input modifiers. " + "The input object must match that Process's inputs.", + ) parser.add_argument( "--mpi-config-file", diff --git a/cwltool/job.py b/cwltool/job.py index 885358807b..b486a4017b 100644 --- a/cwltool/job.py +++ b/cwltool/job.py @@ -15,9 +15,9 @@ from io import IOBase from threading import Timer from typing import ( + IO, Callable, Dict, - IO, Iterable, List, Mapping, diff --git a/cwltool/main.py b/cwltool/main.py index 1cf4a7ee03..af52375d2e 100755 --- a/cwltool/main.py +++ b/cwltool/main.py @@ -81,7 +81,7 @@ get_container_from_software_requirements, ) from .stdfsaccess import StdFsAccess -from .subgraph import get_step, get_subgraph +from .subgraph import get_process, get_step, get_subgraph from .update import ALLUPDATES, UPDATES from .utils import ( DEFAULT_TMP_PREFIX, @@ -811,7 +811,7 @@ def choose_step( if isinstance(tool, Workflow): url = urllib.parse.urlparse(tool.tool["id"]) if url.fragment: - extracted = get_step(tool, tool.tool["id"] + "/" + args.singe_step) + extracted = get_step(tool, tool.tool["id"] + "/" + args.single_step) else: extracted = get_step( tool, @@ -831,6 +831,43 @@ def choose_step( return tool +def choose_process( + args: argparse.Namespace, + tool: Process, + loadingContext: LoadingContext, +) -> Optional[Process]: + """Walk the given Workflow and extract just args.single_step.""" + if loadingContext.loader is None: + raise Exception("loadingContext.loader cannot be None") + + if isinstance(tool, Workflow): + url = urllib.parse.urlparse(tool.tool["id"]) + if url.fragment: + extracted = get_process( + tool, + tool.tool["id"] + "/" + args.single_process, + loadingContext.loader.idx, + ) + else: + extracted = get_process( + tool, + loadingContext.loader.fetcher.urljoin( + tool.tool["id"], "#" + args.single_process + ), + loadingContext.loader.idx, + ) + else: + _logger.error("Can only use --single-process on Workflows") + return None + if isinstance(loadingContext.loader.idx, MutableMapping): + loadingContext.loader.idx[extracted["id"]] = extracted + tool = make_tool(extracted["id"], loadingContext) + else: + raise Exception("Missing loadingContext.loader.idx!") + + return tool + + def check_working_directories( runtimeContext: RuntimeContext, ) -> Optional[int]: @@ -1068,6 +1105,13 @@ def main( else: tool = ctool + elif args.single_process: + ctool = choose_process(args, tool, loadingContext) + if ctool is None: + return 1 + else: + tool = ctool + if args.print_subgraph: if "name" in tool.tool: del tool.tool["name"] diff --git a/cwltool/subgraph.py b/cwltool/subgraph.py index 92795eedf7..21da423116 100644 --- a/cwltool/subgraph.py +++ b/cwltool/subgraph.py @@ -1,8 +1,10 @@ import urllib from collections import namedtuple from typing import ( + Any, Dict, List, + Mapping, MutableMapping, MutableSequence, Optional, @@ -191,3 +193,17 @@ def get_step(tool: Workflow, step_id: str) -> CommentedMap: extracted[f] = tool.tool[f] return extracted + + +def get_process(tool: Workflow, step_id: str, index: Mapping[str, Any]) -> Any: + """Return just a single Process from a Workflow step.""" + step = find_step(tool.steps, step_id) + if step is None: + raise Exception(f"Step {step_id} was not found") + + run = step["run"] + + if isinstance(run, str): + return index[run] + else: + return run diff --git a/tests/test_environment.py b/tests/test_environment.py index 8352de98c4..2d33360d0d 100644 --- a/tests/test_environment.py +++ b/tests/test_environment.py @@ -1,12 +1,13 @@ """Test passing of environment variables to tools.""" -from abc import ABC, abstractmethod import os +from abc import ABC, abstractmethod from pathlib import Path from typing import Any, Callable, Dict, Iterator, List, Mapping, Set, Union, cast import pytest from cwltool.singularity import get_version + from .util import env_accepts_null, get_tool_env, needs_docker, needs_singularity # None => accept anything, just require the key is present diff --git a/tests/test_singularity.py b/tests/test_singularity.py index 66fe094f10..a5460661a2 100644 --- a/tests/test_singularity.py +++ b/tests/test_singularity.py @@ -1,6 +1,6 @@ """Tests to find local Singularity image.""" -import shutil import os +import shutil from pathlib import Path from typing import Any diff --git a/tests/util.py b/tests/util.py index 5674107187..6532d298fe 100644 --- a/tests/util.py +++ b/tests/util.py @@ -5,8 +5,8 @@ import os import shutil import subprocess -from pathlib import Path import sys +from pathlib import Path from typing import Dict, Generator, List, Mapping, Optional, Tuple, Union import pytest From 63a9d43be5dadc4f5ff7252c16282e324a32c153 Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" Date: Fri, 11 Jun 2021 11:08:30 +0200 Subject: [PATCH 058/111] skip psutil 5.6.1 Fixes #1119 --- requirements.txt | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 2f57d7fc2b..a0592b0142 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,7 +6,7 @@ schema-salad>=7.1.20210518142926,<8 prov==1.5.1 bagit==1.8.1 mypy-extensions -psutil<5.9.0 +psutil != 5.6.1, <5.9.0 typing-extensions coloredlogs pydot>=1.4.1 diff --git a/setup.py b/setup.py index 2b6a0e57c9..0fb2306875 100644 --- a/setup.py +++ b/setup.py @@ -115,7 +115,7 @@ # https://github.com/common-workflow-language/schema_salad/issues/385 "schema-salad >= 7.1.20210518142926, < 8", "mypy-extensions", - "psutil", + "psutil != 5.6.1", "prov == 1.5.1", "bagit >= 1.6.4", "typing-extensions", From dc5381e04fa2a9d18de5c78e0f8c2907919983ee Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" Date: Wed, 16 Jun 2021 15:21:01 +0200 Subject: [PATCH 059/111] avoid psutil CVE-2019-18874 --- requirements.txt | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index a0592b0142..af498c9bb0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,7 +6,7 @@ schema-salad>=7.1.20210518142926,<8 prov==1.5.1 bagit==1.8.1 mypy-extensions -psutil != 5.6.1, <5.9.0 +psutil>=5.6.6 typing-extensions coloredlogs pydot>=1.4.1 diff --git a/setup.py b/setup.py index 0fb2306875..3dac1cd54a 100644 --- a/setup.py +++ b/setup.py @@ -115,7 +115,7 @@ # https://github.com/common-workflow-language/schema_salad/issues/385 "schema-salad >= 7.1.20210518142926, < 8", "mypy-extensions", - "psutil != 5.6.1", + "psutil >= 5.6.6", "prov == 1.5.1", "bagit >= 1.6.4", "typing-extensions", From fcdd2c78493a4a760960a8c4a63c0a776b7626c9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 23 Jun 2021 07:12:01 +0000 Subject: [PATCH 060/111] Bump mypy from 0.812 to 0.910 Bumps [mypy](https://github.com/python/mypy) from 0.812 to 0.910. - [Release notes](https://github.com/python/mypy/releases) - [Commits](https://github.com/python/mypy/compare/v0.812...v0.910) --- updated-dependencies: - dependency-name: mypy dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- mypy_requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy_requirements.txt b/mypy_requirements.txt index 10316e95bc..13c640f740 100644 --- a/mypy_requirements.txt +++ b/mypy_requirements.txt @@ -1 +1 @@ -mypy==0.812 +mypy==0.910 From 8da32f2560c907bc21580d18477e93862f9696bd Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" Date: Wed, 23 Jun 2021 09:58:05 +0200 Subject: [PATCH 061/111] add external types packages --- cwltool/errors.py | 6 ++++++ cwltool/main.py | 22 ++++++++++++++-------- cwltool/validate_js.py | 2 +- mypy_requirements.txt | 2 ++ tests/test_dependencies.py | 4 +--- tests/test_environment.py | 2 +- tests/test_singularity.py | 1 - tests/util.py | 2 -- typeshed/galaxy/util/__init__.pyi | 1 - 9 files changed, 25 insertions(+), 17 deletions(-) diff --git a/cwltool/errors.py b/cwltool/errors.py index 9df1236faf..f355236a08 100644 --- a/cwltool/errors.py +++ b/cwltool/errors.py @@ -4,3 +4,9 @@ class WorkflowException(Exception): class UnsupportedRequirement(WorkflowException): pass + + +class ArgumentException(Exception): + """Mismatched command line arguments provided.""" + + pass diff --git a/cwltool/main.py b/cwltool/main.py index af52375d2e..f09f1a5a02 100755 --- a/cwltool/main.py +++ b/cwltool/main.py @@ -47,7 +47,7 @@ from .builder import HasReqsHints from .context import LoadingContext, RuntimeContext, getdefault from .cwlrdf import printdot, printrdf -from .errors import UnsupportedRequirement, WorkflowException +from .errors import ArgumentException, UnsupportedRequirement, WorkflowException from .executors import JobExecutor, MultithreadedJobExecutor, SingleJobExecutor from .load_tool import ( default_loader, @@ -73,7 +73,7 @@ use_standard_schema, ) from .procgenerator import ProcessGenerator -from .provenance import ResearchObject +from .provenance import ResearchObject, WritableBagFile from .resolver import ga4gh_tool_registries, tool_resolver from .secrets import SecretStore from .software_requirements import ( @@ -691,10 +691,10 @@ def setup_provenance( args: argparse.Namespace, argsl: List[str], runtimeContext: RuntimeContext, -) -> Optional[int]: +) -> Union[io.TextIOWrapper, WritableBagFile]: if not args.compute_checksum: _logger.error("--provenance incompatible with --no-compute-checksum") - return 1 + raise ArgumentException() ro = ResearchObject( getdefault(runtimeContext.make_fs_access, StdFsAccess)(""), temp_prefix_ro=args.tmpdir_prefix, @@ -703,7 +703,7 @@ def setup_provenance( ) runtimeContext.research_obj = ro log_file_io = ro.open_log_file_for_activity(ro.engine_uuid) - prov_log_handler = logging.StreamHandler(cast(IO[str], log_file_io)) + prov_log_handler = logging.StreamHandler(log_file_io) prov_log_handler.setFormatter(ProvLogFormatter()) _logger.addHandler(prov_log_handler) @@ -712,7 +712,7 @@ def setup_provenance( # Log cwltool command line options to provenance file _logger.info("[cwltool] %s %s", sys.argv[0], " ".join(argsl)) _logger.debug("[cwltool] Arguments: %s", args) - return None + return log_file_io def setup_loadingContext( @@ -989,10 +989,13 @@ def main( setup_schema(args, custom_schema_callback) + prov_log_stream: Optional[Union[io.TextIOWrapper, WritableBagFile]] = None if args.provenance: if argsl is None: raise Exception("argsl cannot be None") - if setup_provenance(args, argsl, runtimeContext) is not None: + try: + prov_log_stream = setup_provenance(args, argsl, runtimeContext) + except ArgumentException: return 1 loadingContext = setup_loadingContext(loadingContext, runtimeContext, args) @@ -1347,7 +1350,10 @@ def loc_to_path(obj: CWLObjectType) -> None: # Ensure last log lines are written out prov_log_handler.flush() # Underlying WritableBagFile will add the tagfile to the manifest - prov_log_handler.stream.close() + if prov_log_stream: + prov_log_stream.close() + # Why not use prov_log_handler.stream ? That is not part of the + # public API for logging.StreamHandler prov_log_handler.close() research_obj.close(args.provenance) diff --git a/cwltool/validate_js.py b/cwltool/validate_js.py index 8cb6b887d4..3cc81b1b76 100644 --- a/cwltool/validate_js.py +++ b/cwltool/validate_js.py @@ -88,7 +88,7 @@ def get_expressions( itertools.chain( *map( lambda x: get_expressions( - x[1], schema.items, SourceLine(tool, x[0]) # type: ignore + x[1], schema.items, SourceLine(tool, x[0]) ), enumerate(tool), ) diff --git a/mypy_requirements.txt b/mypy_requirements.txt index 13c640f740..5d241a30ef 100644 --- a/mypy_requirements.txt +++ b/mypy_requirements.txt @@ -1 +1,3 @@ mypy==0.910 +types-requests +types-setuptools diff --git a/tests/test_dependencies.py b/tests/test_dependencies.py index b5ad609396..624fb7d82d 100644 --- a/tests/test_dependencies.py +++ b/tests/test_dependencies.py @@ -3,18 +3,16 @@ from pathlib import Path from shutil import which from types import ModuleType -from typing import Dict, List, Optional +from typing import Optional import pytest -from cwltool.main import main from .util import ( get_data, get_main_output, get_tool_env, needs_docker, - working_directory, ) deps = None # type: Optional[ModuleType] diff --git a/tests/test_environment.py b/tests/test_environment.py index 2d33360d0d..397ab35eab 100644 --- a/tests/test_environment.py +++ b/tests/test_environment.py @@ -2,7 +2,7 @@ import os from abc import ABC, abstractmethod from pathlib import Path -from typing import Any, Callable, Dict, Iterator, List, Mapping, Set, Union, cast +from typing import Any, Callable, Dict, List, Mapping, Union import pytest diff --git a/tests/test_singularity.py b/tests/test_singularity.py index a5460661a2..2291b4a3de 100644 --- a/tests/test_singularity.py +++ b/tests/test_singularity.py @@ -1,5 +1,4 @@ """Tests to find local Singularity image.""" -import os import shutil from pathlib import Path from typing import Any diff --git a/tests/util.py b/tests/util.py index 6532d298fe..fbca2fb151 100644 --- a/tests/util.py +++ b/tests/util.py @@ -1,11 +1,9 @@ import contextlib -import functools import io import json import os import shutil import subprocess -import sys from pathlib import Path from typing import Dict, Generator, List, Mapping, Optional, Tuple, Union diff --git a/typeshed/galaxy/util/__init__.pyi b/typeshed/galaxy/util/__init__.pyi index d2c9a1ee58..d0e0292f3e 100644 --- a/typeshed/galaxy/util/__init__.pyi +++ b/typeshed/galaxy/util/__init__.pyi @@ -4,7 +4,6 @@ from typing import Any, Optional import collections -from six.moves.urllib import parse as urlparse, request as urlrequest from .inflection import English as English, Inflector as Inflector grp = ... # type: Any From 03ba4bb874d59f02c21139f2c1ebcf9f1ac040ee Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" Date: Wed, 23 Jun 2021 11:01:40 +0200 Subject: [PATCH 062/111] fix lambda typing for mypyc --- cwltool/validate_js.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/cwltool/validate_js.py b/cwltool/validate_js.py index 3cc81b1b76..14b9740227 100644 --- a/cwltool/validate_js.py +++ b/cwltool/validate_js.py @@ -16,7 +16,7 @@ ) from pkg_resources import resource_stream -from ruamel.yaml.comments import CommentedMap +from ruamel.yaml.comments import CommentedMap, CommentedSeq from schema_salad.avro.schema import ( ArraySchema, EnumSchema, @@ -59,7 +59,7 @@ def filter(self, record): # type: (logging.LogRecord) -> bool def get_expressions( - tool: Union[CommentedMap, str], + tool: Union[CommentedMap, str, CommentedSeq], schema: Optional[Union[Schema, ArraySchema]], source_line: Optional[SourceLine] = None, ) -> List[Tuple[str, Optional[SourceLine]]]: @@ -84,12 +84,18 @@ def get_expressions( if not isinstance(tool, MutableSequence): return [] + def tmp_expr( + x: Tuple[int, Union[CommentedMap, str, CommentedSeq]] + ) -> List[Tuple[str, Optional[SourceLine]]]: + # using a lambda for this broke mypyc v0.910 and before + return get_expressions( + x[1], cast(ArraySchema, schema).items, SourceLine(tool, x[0]) + ) + return list( itertools.chain( *map( - lambda x: get_expressions( - x[1], schema.items, SourceLine(tool, x[0]) - ), + tmp_expr, enumerate(tool), ) ) From 75795bf171171391f0e01ecedbf852f3d56f5238 Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" Date: Wed, 23 Jun 2021 11:12:46 +0200 Subject: [PATCH 063/111] fix some type errors that mypyc found --- cwltool/main.py | 2 +- cwltool/update.py | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/cwltool/main.py b/cwltool/main.py index f09f1a5a02..6966e7ced4 100755 --- a/cwltool/main.py +++ b/cwltool/main.py @@ -1045,7 +1045,7 @@ def main( if loadingContext.loader is None: raise Exception("Impossible code path.") processobj, metadata = loadingContext.loader.resolve_ref(uri) - processobj = cast(CommentedMap, processobj) + processobj = cast(Union[CommentedMap, CommentedSeq], processobj) if args.pack: stdout.write(print_pack(loadingContext, uri)) return 0 diff --git a/cwltool/update.py b/cwltool/update.py index 05e041b0a9..8a8434a2ee 100644 --- a/cwltool/update.py +++ b/cwltool/update.py @@ -26,9 +26,9 @@ def v1_1to1_2( """Public updater for v1.1 to v1.2.""" doc = copy.deepcopy(doc) - upd = doc + upd: Union[CommentedSeq, CommentedMap] = doc if isinstance(upd, MutableMapping) and "$graph" in upd: - upd = cast(CommentedMap, upd["$graph"]) + upd = upd["$graph"] for proc in aslist(upd): if "cwlVersion" in proc: del proc["cwlVersion"] @@ -119,9 +119,9 @@ def fix_inputBinding(t: CWLObjectType) -> None: visit_class(doc, ("ExpressionTool", "Workflow"), fix_inputBinding) visit_field(doc, "secondaryFiles", partial(update_secondaryFiles, top=True)) - upd = doc + upd: Union[CommentedMap, CommentedSeq] = doc if isinstance(upd, MutableMapping) and "$graph" in upd: - upd = cast(CommentedMap, upd["$graph"]) + upd = upd["$graph"] for proc in aslist(upd): proc.setdefault("hints", CommentedSeq()) proc["hints"].insert( @@ -167,9 +167,9 @@ def update_pickvalue(t: CWLObjectType) -> None: inp["pickValue"] = "the_only_non_null" visit_class(doc, "Workflow", update_pickvalue) - upd = doc + upd: Union[CommentedSeq, CommentedMap] = doc if isinstance(upd, MutableMapping) and "$graph" in upd: - upd = cast(CommentedMap, upd["$graph"]) + upd = upd["$graph"] for proc in aslist(upd): if "cwlVersion" in proc: del proc["cwlVersion"] From 79e2350c5d365d6c19e2714c5704772e90d26c51 Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" Date: Wed, 23 Jun 2021 11:28:35 +0200 Subject: [PATCH 064/111] fix ruamel.yaml PendingDeprecationWarnings --- cwltool/main.py | 27 ++++++++++++++++----------- cwltool/mpi.py | 5 +++-- requirements.txt | 2 +- setup.py | 2 +- tests/test_dependencies.py | 8 +------- tests/test_mpi.py | 5 +++-- tests/test_pack.py | 12 ++++++++---- tests/test_validate_js.py | 8 +++++--- 8 files changed, 38 insertions(+), 31 deletions(-) diff --git a/cwltool/main.py b/cwltool/main.py index 6966e7ced4..235344dd88 100755 --- a/cwltool/main.py +++ b/cwltool/main.py @@ -35,8 +35,9 @@ import argcomplete import coloredlogs import pkg_resources # part of setuptools -from ruamel import yaml +import ruamel.yaml from ruamel.yaml.comments import CommentedMap, CommentedSeq +from ruamel.yaml.main import YAML from schema_salad.exceptions import ValidationException from schema_salad.ref_resolver import Loader, file_uri, uri_file_path from schema_salad.sourceline import strip_dup_lineno @@ -158,10 +159,10 @@ def generate_example_input( "float": 0.1, "double": 0.1, "string": "a_string", - "File": yaml.comments.CommentedMap( + "File": ruamel.yaml.comments.CommentedMap( [("class", "File"), ("path", "a/file/path")] ), - "Directory": yaml.comments.CommentedMap( + "Directory": ruamel.yaml.comments.CommentedMap( [("class", "Directory"), ("path", "a/directory/path")] ), } # type: CWLObjectType @@ -215,7 +216,7 @@ def generate_example_input( example = "{}_enum_value".format(inptype.get("name", "valid")) comment = 'enum; valid values: "{}"'.format('", "'.join(symbols)) elif inptype["type"] == "record": - example = yaml.comments.CommentedMap() + example = ruamel.yaml.comments.CommentedMap() if "name" in inptype: comment = '"{}" record type.'.format(inptype["name"]) for field in cast(List[CWLObjectType], inptype["fields"]): @@ -303,7 +304,7 @@ def realize_input_schema( def generate_input_template(tool: Process) -> CWLObjectType: """Generate an example input object for the given CWL process.""" - template = yaml.comments.CommentedMap() + template = ruamel.yaml.comments.CommentedMap() for inp in realize_input_schema(tool.tool["inputs"], tool.schemaDefs): name = shortname(cast(str, inp["id"])) value, comment = generate_example_input(inp["type"], inp.get("default", None)) @@ -328,7 +329,8 @@ def load_job_order( if len(args.job_order) == 1 and args.job_order[0][0] != "-": job_order_file = args.job_order[0] elif len(args.job_order) == 1 and args.job_order[0] == "-": - job_order_object = yaml.main.round_trip_load(stdin) + yaml = YAML() + job_order_object = yaml.load(stdin) job_order_object, _ = loader.resolve_all( job_order_object, file_uri(os.getcwd()) + "/" ) @@ -754,13 +756,16 @@ def my_represent_none( """Force clean representation of 'null'.""" return self.represent_scalar("tag:yaml.org,2002:null", "null") - yaml.representer.RoundTripRepresenter.add_representer(type(None), my_represent_none) - yaml.main.round_trip_dump( + ruamel.yaml.representer.RoundTripRepresenter.add_representer( + type(None), my_represent_none + ) + yaml = YAML() + yaml.default_flow_style = False + yaml.indent = 4 + yaml.block_seq_indent = 2 + yaml.dump( generate_input_template(tool), sys.stdout, - default_flow_style=False, - indent=4, - block_seq_indent=2, ) diff --git a/cwltool/mpi.py b/cwltool/mpi.py index bb1b16ffd7..48d0d4b613 100644 --- a/cwltool/mpi.py +++ b/cwltool/mpi.py @@ -4,7 +4,7 @@ import re from typing import List, Mapping, MutableMapping, Optional, Type, TypeVar, Union -from ruamel import yaml +from ruamel.yaml.main import YAML MpiConfigT = TypeVar("MpiConfigT", bound="MpiConfig") @@ -53,7 +53,8 @@ def load(cls: Type[MpiConfigT], config_file_name: str) -> MpiConfigT: optional). """ with open(config_file_name) as cf: - data = yaml.main.round_trip_load(cf) + yaml = YAML() + data = yaml.load(cf) try: return cls(**data) except TypeError as e: diff --git a/requirements.txt b/requirements.txt index af498c9bb0..34217c8582 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ requests>=2.4.3 -ruamel.yaml>=0.12.4,<0.17.10 +ruamel.yaml>=0.15,<0.17.10 rdflib>=4.2.2,<5.1 shellescape>=3.4.1,<3.9 schema-salad>=7.1.20210518142926,<8 diff --git a/setup.py b/setup.py index 3dac1cd54a..813de0f9dc 100644 --- a/setup.py +++ b/setup.py @@ -108,7 +108,7 @@ "setuptools", "requests >= 2.6.1", # >= 2.6.1 to workaround # https://github.com/ionrock/cachecontrol/issues/137 - "ruamel.yaml >= 0.12.4, < 0.17.10", + "ruamel.yaml >= 0.15, < 0.17.10", "rdflib >= 4.2.2, < 5.1.0", "shellescape >= 3.4.1, < 3.9", # 7.1.20210518142926 or later required due to diff --git a/tests/test_dependencies.py b/tests/test_dependencies.py index 624fb7d82d..c02c1c40e3 100644 --- a/tests/test_dependencies.py +++ b/tests/test_dependencies.py @@ -7,13 +7,7 @@ import pytest - -from .util import ( - get_data, - get_main_output, - get_tool_env, - needs_docker, -) +from .util import get_data, get_main_output, get_tool_env, needs_docker deps = None # type: Optional[ModuleType] try: diff --git a/tests/test_mpi.py b/tests/test_mpi.py index 10e62e917d..400a8de4bd 100644 --- a/tests/test_mpi.py +++ b/tests/test_mpi.py @@ -8,8 +8,8 @@ import pkg_resources import pytest -from ruamel import yaml from ruamel.yaml.comments import CommentedMap, CommentedSeq +from ruamel.yaml.main import YAML from schema_salad.avro.schema import Names import cwltool.load_tool @@ -104,7 +104,8 @@ def run_many(self, n: int, args: List[str]): "env_pass": ["USER"], } plat_conf_file = mpitmp / "plat_mpi.yml" - plat_conf_file.write_text(yaml.main.round_trip_dump(plat_conf)) + yaml = YAML() + yaml.dump(plat_conf, plat_conf_file) yield str(plat_conf_file) diff --git a/tests/test_pack.py b/tests/test_pack.py index 45b3340926..44697b4bbb 100644 --- a/tests/test_pack.py +++ b/tests/test_pack.py @@ -8,7 +8,7 @@ from typing import Dict import pytest -from ruamel import yaml +from ruamel.yaml.main import YAML import cwltool.pack import cwltool.workflow @@ -23,9 +23,10 @@ def test_pack() -> None: loadingContext, workflowobj, uri = fetch_document(get_data("tests/wf/revsort.cwl")) + yaml = YAML(typ="safe", pure=True) with open(get_data("tests/wf/expect_packed.cwl")) as packed_file: - expect_packed = yaml.main.safe_load(packed_file) + expect_packed = yaml.load(packed_file) packed = cwltool.pack.pack(loadingContext, uri) adjustFileObjs( @@ -55,8 +56,9 @@ def test_pack_input_named_name() -> None: assert loader loader.resolve_ref(uri)[0] + yaml = YAML() with open(get_data("tests/wf/expect_trick_packed.cwl")) as packed_file: - expect_packed = yaml.main.round_trip_load(packed_file) + expect_packed = yaml.load(packed_file) packed = cwltool.pack.pack(loadingContext, uri) adjustFileObjs( @@ -91,8 +93,10 @@ def test_pack_single_tool() -> None: def test_pack_fragment() -> None: + yaml = YAML(typ="safe", pure=True) + with open(get_data("tests/wf/scatter2_subwf.cwl")) as packed_file: - expect_packed = yaml.main.safe_load(packed_file) + expect_packed = yaml.load(packed_file) loadingContext, workflowobj, uri = fetch_document(get_data("tests/wf/scatter2.cwl")) packed = cwltool.pack.pack(loadingContext, uri + "#scatterstep/mysub") diff --git a/tests/test_validate_js.py b/tests/test_validate_js.py index 8a8ce75618..45b708528c 100644 --- a/tests/test_validate_js.py +++ b/tests/test_validate_js.py @@ -1,6 +1,6 @@ from typing import Any -from ruamel import yaml +from ruamel.yaml.main import YAML from schema_salad.avro.schema import Names from cwltool import process, validate_js @@ -24,7 +24,8 @@ def test_get_expressions() -> None: - test_cwl_yaml = yaml.main.round_trip_load(TEST_CWL) + yaml = YAML() + test_cwl_yaml = yaml.load(TEST_CWL) schema = process.get_schema("v1.0")[1] assert isinstance(schema, Names) clt_schema = schema.names["CommandLineTool"] @@ -35,7 +36,8 @@ def test_get_expressions() -> None: def test_validate_js_expressions(mocker: Any) -> None: - test_cwl_yaml = yaml.main.round_trip_load(TEST_CWL) + yaml = YAML() + test_cwl_yaml = yaml.load(TEST_CWL) schema = process.get_schema("v1.0")[1] assert isinstance(schema, Names) clt_schema = schema.names["CommandLineTool"] From 83f5fb71a1b78213469036d83ac7a539ace38aa8 Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" Date: Wed, 23 Jun 2021 16:25:02 +0200 Subject: [PATCH 065/111] upgrade to Python 3.6+ syntax --- cwltool/builder.py | 4 ++-- cwltool/checker.py | 10 ++++------ cwltool/cwlrdf.py | 8 +++----- cwltool/expression.py | 9 +++------ cwltool/job.py | 6 +++--- cwltool/load_tool.py | 2 +- cwltool/provenance.py | 2 +- cwltool/update.py | 4 ++-- cwltool/utils.py | 2 +- tests/test_examples.py | 16 ++++++---------- 10 files changed, 26 insertions(+), 37 deletions(-) diff --git a/cwltool/builder.py b/cwltool/builder.py index d206e66352..6fe5396d44 100644 --- a/cwltool/builder.py +++ b/cwltool/builder.py @@ -116,7 +116,7 @@ def check_format( continue if "format" not in afile: raise ValidationException( - "File has no 'format' defined: {}".format(json_dumps(afile, indent=4)) + f"File has no 'format' defined: {json_dumps(afile, indent=4)}" ) for inpf in aslist(input_formats): if afile["format"] == inpf or formatSubclassOf( @@ -124,7 +124,7 @@ def check_format( ): return raise ValidationException( - "File has an incompatible format: {}".format(json_dumps(afile, indent=4)) + f"File has an incompatible format: {json_dumps(afile, indent=4)}" ) diff --git a/cwltool/checker.py b/cwltool/checker.py index ef30996016..4ccca2fefd 100644 --- a/cwltool/checker.py +++ b/cwltool/checker.py @@ -211,13 +211,11 @@ def static_checker( sink = warning.sink linkMerge = warning.linkMerge sinksf = sorted( - [ - p["pattern"] - for p in sink.get("secondaryFiles", []) - if p.get("required", True) - ] + p["pattern"] + for p in sink.get("secondaryFiles", []) + if p.get("required", True) ) - srcsf = sorted([p["pattern"] for p in src.get("secondaryFiles", [])]) + srcsf = sorted(p["pattern"] for p in src.get("secondaryFiles", [])) # Every secondaryFile required by the sink, should be declared # by the source missing = missing_subset(srcsf, sinksf) diff --git a/cwltool/cwlrdf.py b/cwltool/cwlrdf.py index 0c2f8357a1..f099f391fe 100644 --- a/cwltool/cwlrdf.py +++ b/cwltool/cwlrdf.py @@ -48,7 +48,7 @@ def dot_with_parameters(g: Graph, stdout: Union[TextIO, StreamWriter]) -> None: for step, run, _ in qres: stdout.write( '"%s" [label="%s"]\n' - % (lastpart(step), "{} ({})".format(lastpart(step), lastpart(run))) + % (lastpart(step), f"{lastpart(step)} ({lastpart(run)})") ) qres = g.query( @@ -149,9 +149,7 @@ def dot_without_parameters(g: Graph, stdout: Union[TextIO, StreamWriter]) -> Non if wf in subworkflows: if wf not in dotname: dotname[wf] = "cluster_" + lastpart(wf) - stdout.write( - 'subgraph "{}" {{ label="{}"\n'.format(dotname[wf], lastpart(wf)) - ) + stdout.write(f'subgraph "{dotname[wf]}" {{ label="{lastpart(wf)}"\n') currentwf = wf clusternode[wf] = step else: @@ -187,7 +185,7 @@ def dot_without_parameters(g: Graph, stdout: Union[TextIO, StreamWriter]) -> Non if sinkrun in clusternode: attr += ' lhead="%s"' % dotname[sinkrun] sink = clusternode[sinkrun] - stdout.write('"{}" -> "{}" [{}]\n'.format(dotname[src], dotname[sink], attr)) + stdout.write(f'"{dotname[src]}" -> "{dotname[sink]}" [{attr}]\n') def printdot( diff --git a/cwltool/expression.py b/cwltool/expression.py index 03aed7ccca..00e306fc97 100644 --- a/cwltool/expression.py +++ b/cwltool/expression.py @@ -30,10 +30,7 @@ def jshead(engine_config: List[str], rootvars: CWLObjectType) -> str: return "\n".join( engine_config - + [ - "var {} = {};".format(k, json_dumps(v, indent=4)) - for k, v in rootvars.items() - ] + + [f"var {k} = {json_dumps(v, indent=4)};" for k, v in rootvars.items()] ) @@ -258,7 +255,7 @@ def evaluator( def _convert_dumper(string: str) -> str: - return "{} + ".format(json.dumps(string)) + return f"{json.dumps(string)} + " def interpolate( @@ -291,7 +288,7 @@ def interpolate( w = scanner(scan) while w: if convert_to_expression: - parts.append('"{}" + '.format(scan[0 : w[0]])) + parts.append(f'"{scan[0 : w[0]]}" + ') else: parts.append(scan[0 : w[0]]) diff --git a/cwltool/job.py b/cwltool/job.py index b486a4017b..f515091ec2 100644 --- a/cwltool/job.py +++ b/cwltool/job.py @@ -496,9 +496,9 @@ def get_tree_mem_usage(memory_usage: MutableSequence[Optional[int]]) -> None: children = monitor.children() rss = monitor.memory_info().rss while len(children): - rss += sum([process.memory_info().rss for process in children]) + rss += sum(process.memory_info().rss for process in children) children = list( - itertools.chain(*[process.children() for process in children]) + itertools.chain(*(process.children() for process in children)) ) if memory_usage[0] is None or rss > memory_usage[0]: memory_usage[0] = rss @@ -887,7 +887,7 @@ def docker_monitor( return max_mem_percent = 0 # type: float mem_percent = 0 # type: float - with open(stats_file_name, mode="r") as stats: + with open(stats_file_name) as stats: while True: line = stats.readline() if not line: diff --git a/cwltool/load_tool.py b/cwltool/load_tool.py index 901cfedf90..e4842a02e1 100644 --- a/cwltool/load_tool.py +++ b/cwltool/load_tool.py @@ -314,7 +314,7 @@ def resolve_and_validate_document( if not isinstance(cwlVersion, str): with SourceLine(workflowobj, "cwlVersion", ValidationException): raise ValidationException( - "'cwlVersion' must be a string, got {}".format(type(cwlVersion)) + f"'cwlVersion' must be a string, got {type(cwlVersion)}" ) # strip out version cwlVersion = re.sub(r"^(?:cwl:|https://w3id.org/cwl/cwl#)", "", cwlVersion) diff --git a/cwltool/provenance.py b/cwltool/provenance.py index 3de641e8c3..3dc8e0395d 100644 --- a/cwltool/provenance.py +++ b/cwltool/provenance.py @@ -692,7 +692,7 @@ def _write_ro_manifest(self) -> None: manifest = OrderedDict( { "@context": [ - {"@base": "{}{}/".format(self.base_uri, posix_path(METADATA))}, + {"@base": f"{self.base_uri}{posix_path(METADATA)}/"}, "https://w3id.org/bundle/context", ], "id": "/", diff --git a/cwltool/update.py b/cwltool/update.py index 8a8434a2ee..3976265f21 100644 --- a/cwltool/update.py +++ b/cwltool/update.py @@ -72,7 +72,7 @@ def rewrite_requirements(t: CWLObjectType) -> None: r["class"] = rewrite[cls] else: raise ValidationException( - "hints entries must be dictionaries: {} {}.".format(type(r), r) + f"hints entries must be dictionaries: {type(r)} {r}." ) if "steps" in t: for s in cast(MutableSequence[CWLObjectType], t["steps"]): @@ -80,7 +80,7 @@ def rewrite_requirements(t: CWLObjectType) -> None: rewrite_requirements(s) else: raise ValidationException( - "steps entries must be dictionaries: {} {}.".format(type(s), s) + f"steps entries must be dictionaries: {type(s)} {s}." ) def update_secondaryFiles(t, top=False): diff --git a/cwltool/utils.py b/cwltool/utils.py index ef2c0d5d83..83257d1cf3 100644 --- a/cwltool/utils.py +++ b/cwltool/utils.py @@ -117,7 +117,7 @@ def versionstring() -> str: """Version of CWLtool used to execute the workflow.""" pkg = pkg_resources.require("cwltool") if pkg: - return "{} {}".format(sys.argv[0], pkg[0].version) + return f"{sys.argv[0]} {pkg[0].version}" return "{} {}".format(sys.argv[0], "unknown version") diff --git a/tests/test_examples.py b/tests/test_examples.py index 498bcaffff..bfab58594b 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -1008,16 +1008,12 @@ def test_print_dot() -> None: assert main(["--print-dot", cwl_path], stdout=stdout) == 0 computed_dot = pydot.graph_from_dot_data(stdout.getvalue())[0] computed_edges = sorted( - [ - (urlparse(source).fragment, urlparse(target).fragment) - for source, target in computed_dot.obj_dict["edges"] - ] + (urlparse(source).fragment, urlparse(target).fragment) + for source, target in computed_dot.obj_dict["edges"] ) expected_edges = sorted( - [ - (urlparse(source).fragment, urlparse(target).fragment) - for source, target in expected_dot.obj_dict["edges"] - ] + (urlparse(source).fragment, urlparse(target).fragment) + for source, target in expected_dot.obj_dict["edges"] ) assert computed_edges == expected_edges @@ -1130,7 +1126,7 @@ def test_cid_file_w_prefix(tmp_path: Path, factor: str) -> None: cidfiles_count = sum(1 for _ in tmp_path.glob("**/pytestcid*")) assert "completed success" in stderr assert error_code == 0 - assert cidfiles_count == 2, "{}/n{}".format(list(listing), stderr) + assert cidfiles_count == 2, f"{list(listing)}/n{stderr}" @needs_docker @@ -1391,7 +1387,7 @@ def test_scatter_output_filenames(tmp_path: Path) -> None: assert isinstance(result, dict) assert "output" in result - locations = sorted([element["location"] for element in result["output"]]) + locations = sorted(element["location"] for element in result["output"]) assert ( locations[0].endswith("output.txt") From 8ec1eb78e4c70638737656ecbcce02a4f0041c49 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 25 Jun 2021 07:12:32 +0000 Subject: [PATCH 066/111] Update ruamel-yaml requirement from <0.17.10,>=0.15 to >=0.15,<0.17.11 Updates the requirements on [ruamel-yaml](https://sourceforge.net/p/ruamel-yaml/code/ci/default/tree) to permit the latest version. --- updated-dependencies: - dependency-name: ruamel-yaml dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 34217c8582..770eceb467 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ requests>=2.4.3 -ruamel.yaml>=0.15,<0.17.10 +ruamel.yaml>=0.15,<0.17.11 rdflib>=4.2.2,<5.1 shellescape>=3.4.1,<3.9 schema-salad>=7.1.20210518142926,<8 diff --git a/setup.py b/setup.py index 813de0f9dc..f428ceed61 100644 --- a/setup.py +++ b/setup.py @@ -108,7 +108,7 @@ "setuptools", "requests >= 2.6.1", # >= 2.6.1 to workaround # https://github.com/ionrock/cachecontrol/issues/137 - "ruamel.yaml >= 0.15, < 0.17.10", + "ruamel.yaml >= 0.15, < 0.17.11", "rdflib >= 4.2.2, < 5.1.0", "shellescape >= 3.4.1, < 3.9", # 7.1.20210518142926 or later required due to From 56d5120900199d5dff70ff758b7dea92fb05d44a Mon Sep 17 00:00:00 2001 From: Peter Amstutz Date: Mon, 28 Jun 2021 10:45:41 -0400 Subject: [PATCH 067/111] Update schema-salad dependency for bug fix (#1463) * Update schema-salad dependency for bug fix Specifically https://github.com/common-workflow-language/schema_salad/pull/408 * Use vocabulary in schema salad validation * Fixes internal Avro type change from File to org.w3id.cwl.cwl.File Co-authored-by: Michael R. Crusoe <1330696+mr-c@users.noreply.github.com> Co-authored-by: Michael R. Crusoe --- cwltool/builder.py | 12 ++++-- cwltool/command_line_tool.py | 13 ++++++- cwltool/expression.py | 8 ++-- cwltool/process.py | 66 +++++++++++++++++++++----------- cwltool/validate_js.py | 3 +- requirements.txt | 2 +- setup.py | 2 +- tests/test_procgenerator.py | 1 + tests/test_validate_js.py | 4 +- tests/wf/generator/pytoolgen.cwl | 2 +- 10 files changed, 75 insertions(+), 38 deletions(-) diff --git a/cwltool/builder.py b/cwltool/builder.py index 6fe5396d44..d4b68581e4 100644 --- a/cwltool/builder.py +++ b/cwltool/builder.py @@ -46,6 +46,12 @@ from .pathmapper import PathMapper from .provenance_profile import ProvenanceProfile # pylint: disable=unused-import +INPUT_OBJ_VOCAB: Dict[str, str] = { + "Any": "https://w3id.org/cwl/salad#Any", + "File": "https://w3id.org/cwl/cwl#File", + "Directory": "https://w3id.org/cwl/cwl#Directory", +} + def content_limit_respected_read_bytes(f): # type: (IO[bytes]) -> bytes contents = f.read(CONTENT_LIMIT + 1) @@ -272,7 +278,7 @@ def bind_input( avsc = self.names.get_name(cast(str, t["name"]), None) if not avsc: avsc = make_avsc_object(convert_to_dict(t), self.names) - if validate(avsc, datum): + if validate(avsc, datum, vocab=INPUT_OBJ_VOCAB): schema = copy.deepcopy(schema) schema["type"] = t if not value_from_expression: @@ -376,7 +382,7 @@ def _capture_files(f: CWLObjectType) -> CWLObjectType: self.files.append(f) return f - if schema["type"] == "File": + if schema["type"] == "org.w3id.cwl.cwl.File": datum = cast(CWLObjectType, datum) self.files.append(datum) @@ -516,7 +522,7 @@ def addsf( _capture_files, ) - if schema["type"] == "Directory": + if schema["type"] == "org.w3id.cwl.cwl.Directory": datum = cast(CWLObjectType, datum) ll = schema.get("loadListing") or self.loadListing if ll and ll != "no_listing": diff --git a/cwltool/command_line_tool.py b/cwltool/command_line_tool.py index 03db249e24..054358f424 100644 --- a/cwltool/command_line_tool.py +++ b/cwltool/command_line_tool.py @@ -40,7 +40,12 @@ from schema_salad.validate import validate_ex from typing_extensions import TYPE_CHECKING, Type -from .builder import Builder, content_limit_respected_read_bytes, substitute +from .builder import ( + Builder, + content_limit_respected_read_bytes, + substitute, + INPUT_OBJ_VOCAB, +) from .context import LoadingContext, RuntimeContext, getdefault from .docker import DockerCommandLineJob from .errors import UnsupportedRequirement, WorkflowException @@ -1110,7 +1115,11 @@ def collect_output_ports( Schema, self.names.get_name("outputs_record_schema", None) ) validate_ex( - expected_schema, ret, strict=False, logger=_logger_validation_warnings + expected_schema, + ret, + strict=False, + logger=_logger_validation_warnings, + vocab=INPUT_OBJ_VOCAB, ) if ret is not None and builder.mutation_manager is not None: adjustFileObjs(ret, builder.mutation_manager.set_generation) diff --git a/cwltool/expression.py b/cwltool/expression.py index 00e306fc97..4e9c5126ac 100644 --- a/cwltool/expression.py +++ b/cwltool/expression.py @@ -134,7 +134,7 @@ def next_seg( m = segment_re.match(remaining_string) if not m: return current_value - next_segment_str = m.group(0) + next_segment_str = m.group(1) key = None # type: Optional[Union[str, int]] if next_segment_str[0] == ".": @@ -146,7 +146,7 @@ def next_seg( if ( isinstance(current_value, MutableSequence) and key == "length" - and not remaining_string[m.end(0) :] + and not remaining_string[m.end(1) :] ): return len(current_value) if not isinstance(current_value, MutableMapping): @@ -175,7 +175,7 @@ def next_seg( try: return next_seg( parsed_string + remaining_string, - remaining_string[m.end(0) :], + remaining_string[m.end(1) :], cast(CWLOutputType, current_value[cast(str, key)]), ) except KeyError: @@ -184,7 +184,7 @@ def next_seg( try: return next_seg( parsed_string + remaining_string, - remaining_string[m.end(0) :], + remaining_string[m.end(1) :], current_value[key], ) except KeyError: diff --git a/cwltool/process.py b/cwltool/process.py index e444d04ddc..354ecd2413 100644 --- a/cwltool/process.py +++ b/cwltool/process.py @@ -44,12 +44,12 @@ from schema_salad.ref_resolver import Loader, file_uri, uri_file_path from schema_salad.schema import load_schema, make_avro_schema, make_valid_avro from schema_salad.sourceline import SourceLine, strip_dup_lineno -from schema_salad.utils import convert_to_dict -from schema_salad.validate import validate_ex +from schema_salad.utils import ContextType, convert_to_dict +from schema_salad.validate import validate_ex, avro_type_name from typing_extensions import TYPE_CHECKING from . import expression -from .builder import Builder, HasReqsHints +from .builder import Builder, HasReqsHints, INPUT_OBJ_VOCAB from .context import LoadingContext, RuntimeContext, getdefault from .errors import UnsupportedRequirement, WorkflowException from .loghandler import _logger @@ -457,30 +457,34 @@ def fill_in_defaults( def avroize_type( - field_type: Union[ - CWLObjectType, MutableSequence[CWLOutputType], CWLOutputType, None - ], + field_type: Union[CWLObjectType, MutableSequence[Any], CWLOutputType, None], name_prefix: str = "", -) -> None: +) -> Union[CWLObjectType, MutableSequence[Any], CWLOutputType, None]: """Add missing information to a type so that CWL types are valid.""" if isinstance(field_type, MutableSequence): - for field in field_type: - avroize_type(field, name_prefix) + for i, field in enumerate(field_type): + field_type[i] = avroize_type(field, name_prefix) elif isinstance(field_type, MutableMapping): if field_type["type"] in ("enum", "record"): if "name" not in field_type: field_type["name"] = name_prefix + str(uuid.uuid4()) if field_type["type"] == "record": - avroize_type( + field_type["fields"] = avroize_type( cast(MutableSequence[CWLOutputType], field_type["fields"]), name_prefix ) - if field_type["type"] == "array": - avroize_type( + elif field_type["type"] == "array": + field_type["items"] = avroize_type( cast(MutableSequence[CWLOutputType], field_type["items"]), name_prefix ) - if isinstance(field_type["type"], MutableSequence): - for ctype in field_type["type"]: - avroize_type(cast(CWLOutputType, ctype), name_prefix) + else: + field_type["type"] = avroize_type( + cast(CWLOutputType, field_type["type"]), name_prefix + ) + elif field_type == "File": + return "org.w3id.cwl.cwl.File" + elif field_type == "Directory": + return "org.w3id.cwl.cwl.Directory" + return field_type def get_overrides( @@ -635,6 +639,7 @@ def __init__( sdtypes, {cast(str, t["name"]): cast(Dict[str, Any], t) for t in sdtypes}, set(), + vocab=INPUT_OBJ_VOCAB, ) for i in av: self.schemaDefs[i["name"]] = i # type: ignore @@ -669,7 +674,8 @@ def __init__( c["type"] = nullable else: c["type"] = c["type"] - avroize_type(c["type"], c["name"]) + + c["type"] = avroize_type(c["type"], c["name"]) if key == "inputs": cast( List[CWLObjectType], self.inputs_record_schema["fields"] @@ -709,9 +715,13 @@ def __init__( ) raise if self.doc_schema is not None: + classname = toolpath_object["class"] + avroname = classname + if self.doc_loader and classname in self.doc_loader.vocab: + avroname = avro_type_name(self.doc_loader.vocab[classname]) validate_js_expressions( toolpath_object, - self.doc_schema.names[toolpath_object["class"]], + self.doc_schema.names[avroname], validate_js_options, ) @@ -778,7 +788,13 @@ def _init_job( raise WorkflowException( "Missing input record schema: " "{}".format(self.names) ) - validate_ex(schema, job, strict=False, logger=_logger_validation_warnings) + validate_ex( + schema, + job, + strict=False, + logger=_logger_validation_warnings, + vocab=INPUT_OBJ_VOCAB, + ) if load_listing and load_listing != "no_listing": get_listing(fs_access, job, recursive=(load_listing == "deep_listing")) @@ -1015,13 +1031,16 @@ def evalResources( def validate_hints( self, avsc_names: Names, hints: List[CWLObjectType], strict: bool ) -> None: + if self.doc_loader is None: + return for i, r in enumerate(hints): sl = SourceLine(hints, i, ValidationException) with sl: - if ( - avsc_names.get_name(cast(str, r["class"]), None) is not None - and self.doc_loader is not None - ): + classname = cast(str, r["class"]) + avroname = classname + if classname in self.doc_loader.vocab: + avroname = avro_type_name(self.doc_loader.vocab[classname]) + if avsc_names.get_name(avroname, None) is not None: plain_hint = { key: r[key] for key in r @@ -1030,10 +1049,11 @@ def validate_hints( validate_ex( cast( Schema, - avsc_names.get_name(cast(str, plain_hint["class"]), None), + avsc_names.get_name(avroname, None), ), plain_hint, strict=strict, + vocab=self.doc_loader.vocab, ) elif r["class"] in ("NetworkAccess", "LoadListingRequirement"): pass diff --git a/cwltool/validate_js.py b/cwltool/validate_js.py index 14b9740227..226c0f686c 100644 --- a/cwltool/validate_js.py +++ b/cwltool/validate_js.py @@ -39,7 +39,7 @@ def is_expression(tool, schema): # type: (Any, Optional[Schema]) -> bool return ( isinstance(schema, EnumSchema) - and schema.name == "Expression" + and schema.name == "org.w3id.cwl.cwl.Expression" and isinstance(tool, str) ) @@ -76,6 +76,7 @@ def get_expressions( tool, raise_ex=False, logger=_logger_validation_warnings, + vocab={}, ): valid_schema = possible_schema diff --git a/requirements.txt b/requirements.txt index 770eceb467..cdd9e1ab64 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ requests>=2.4.3 ruamel.yaml>=0.15,<0.17.11 rdflib>=4.2.2,<5.1 shellescape>=3.4.1,<3.9 -schema-salad>=7.1.20210518142926,<8 +schema-salad>=8.1.20210627200047,<9 prov==1.5.1 bagit==1.8.1 mypy-extensions diff --git a/setup.py b/setup.py index f428ceed61..1a2828b986 100644 --- a/setup.py +++ b/setup.py @@ -113,7 +113,7 @@ "shellescape >= 3.4.1, < 3.9", # 7.1.20210518142926 or later required due to # https://github.com/common-workflow-language/schema_salad/issues/385 - "schema-salad >= 7.1.20210518142926, < 8", + "schema-salad >= 8.1.20210627200047, < 9", "mypy-extensions", "psutil >= 5.6.6", "prov == 1.5.1", diff --git a/tests/test_procgenerator.py b/tests/test_procgenerator.py index ad8c4d991a..39cd099146 100644 --- a/tests/test_procgenerator.py +++ b/tests/test_procgenerator.py @@ -20,6 +20,7 @@ def test_missing_enable_ext(monkeypatch: pytest.MonkeyPatch) -> None: assert ( main( [ + "--debug", "--enable-ext", "--enable-dev", get_data("tests/wf/generator/zing.cwl"), diff --git a/tests/test_validate_js.py b/tests/test_validate_js.py index 45b708528c..8d2ebd3d66 100644 --- a/tests/test_validate_js.py +++ b/tests/test_validate_js.py @@ -28,7 +28,7 @@ def test_get_expressions() -> None: test_cwl_yaml = yaml.load(TEST_CWL) schema = process.get_schema("v1.0")[1] assert isinstance(schema, Names) - clt_schema = schema.names["CommandLineTool"] + clt_schema = schema.names["org.w3id.cwl.cwl.CommandLineTool"] exprs = validate_js.get_expressions(test_cwl_yaml, clt_schema) @@ -40,7 +40,7 @@ def test_validate_js_expressions(mocker: Any) -> None: test_cwl_yaml = yaml.load(TEST_CWL) schema = process.get_schema("v1.0")[1] assert isinstance(schema, Names) - clt_schema = schema.names["CommandLineTool"] + clt_schema = schema.names["org.w3id.cwl.cwl.CommandLineTool"] mocker.patch("cwltool.validate_js._logger") # mocker.patch("cwltool.validate_js.print_js_hint_messages") diff --git a/tests/wf/generator/pytoolgen.cwl b/tests/wf/generator/pytoolgen.cwl index 35ccd145db..3d8ce78797 100644 --- a/tests/wf/generator/pytoolgen.cwl +++ b/tests/wf/generator/pytoolgen.cwl @@ -2,7 +2,7 @@ cwlVersion: v1.0 $namespaces: cwltool: "http://commonwl.org/cwltool#" -class: ProcessGenerator +class: cwltool:ProcessGenerator inputs: script: string dir: Directory From ed9dd4c3472e940a52dfe90049895f470bfd7329 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Mar=C3=ADa=20Fern=C3=A1ndez?= <2072150+jmfernandez@users.noreply.github.com> Date: Mon, 28 Jun 2021 18:32:08 +0200 Subject: [PATCH 068/111] Allow packing Operation records (#1467) * cwltool now packs `Operation` definitions CWL 1.2 introduced [Operation](https://www.commonwl.org/v1.2/Workflow.html#Operation) in order to model abstract workflow steps, needed to use CWL to describe a workflow which could not exist or be implemented in a different way than CWL. As `cwltool --pack` was ignoring records of class `Operation`, packed workflows were not correct (they did not validate with `cwltool --validate`). The issue was explained at https://github.com/common-workflow-language/cwltool/issues/1466. This commit fixes the issue which arises while packing workflows having `Operation` records. Co-authored-by: Michael R. Crusoe --- MANIFEST.in | 1 + cwltool/pack.py | 7 +- tests/test_pack.py | 72 ++++---- tests/wf/operation/abstract-cosifer.cwl | 25 +++ .../expect_operation-single_packed.cwl | 154 ++++++++++++++++++ tests/wf/operation/operation-single.cwl | 27 +++ 6 files changed, 241 insertions(+), 45 deletions(-) create mode 100644 tests/wf/operation/abstract-cosifer.cwl create mode 100644 tests/wf/operation/expect_operation-single_packed.cwl create mode 100644 tests/wf/operation/operation-single.cwl diff --git a/MANIFEST.in b/MANIFEST.in index bd42a86640..336c7f4c78 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -7,6 +7,7 @@ include tests/* include tests/tmp1/tmp2/tmp3/.gitkeep include tests/tmp4/alpha/* include tests/wf/* +include tests/wf/operation/* include tests/override/* include tests/checker_wf/* include tests/subgraph/* diff --git a/cwltool/pack.py b/cwltool/pack.py index 1de051055a..b0976a8aa7 100644 --- a/cwltool/pack.py +++ b/cwltool/pack.py @@ -251,7 +251,12 @@ def rewrite_id(r: str, mainuri: str) -> None: if "$schemas" in metadata: for s in metadata["$schemas"]: schemas.add(s) - if dcr.get("class") not in ("Workflow", "CommandLineTool", "ExpressionTool"): + if dcr.get("class") not in ( + "Workflow", + "CommandLineTool", + "ExpressionTool", + "Operation", + ): continue dc = cast(Dict[str, Any], copy.deepcopy(dcr)) v = rewrite[r] diff --git a/tests/test_pack.py b/tests/test_pack.py index 44697b4bbb..4918e79934 100644 --- a/tests/test_pack.py +++ b/tests/test_pack.py @@ -21,57 +21,41 @@ from .util import get_data, needs_docker -def test_pack() -> None: - loadingContext, workflowobj, uri = fetch_document(get_data("tests/wf/revsort.cwl")) - yaml = YAML(typ="safe", pure=True) - - with open(get_data("tests/wf/expect_packed.cwl")) as packed_file: - expect_packed = yaml.load(packed_file) - - packed = cwltool.pack.pack(loadingContext, uri) - adjustFileObjs( - packed, partial(make_relative, os.path.abspath(get_data("tests/wf"))) - ) - adjustDirObjs(packed, partial(make_relative, os.path.abspath(get_data("tests/wf")))) - - assert "$schemas" in packed - packed_schemas = packed["$schemas"] - assert isinstance(packed_schemas, Sized) - assert len(packed_schemas) == len(expect_packed["$schemas"]) - del packed["$schemas"] - del expect_packed["$schemas"] - - assert packed == expect_packed - - -def test_pack_input_named_name() -> None: - loadingContext, workflowobj, uri = fetch_document( - get_data("tests/wf/trick_revsort.cwl") - ) +@pytest.mark.parametrize( + "unpacked,expected", + [ + ("tests/wf/revsort.cwl", "tests/wf/expect_packed.cwl"), + ( + "tests/wf/operation/operation-single.cwl", + "tests/wf/operation/expect_operation-single_packed.cwl", + ), + ("tests/wf/trick_revsort.cwl", "tests/wf/expect_trick_packed.cwl"), + ], +) +def test_packing(unpacked: str, expected: str) -> None: + """Compare expected version reality with various workflows and --pack.""" + loadingContext, workflowobj, uri = fetch_document(get_data(unpacked)) loadingContext.do_update = False loadingContext, uri = resolve_and_validate_document( loadingContext, workflowobj, uri ) - loader = loadingContext.loader - assert loader - loader.resolve_ref(uri)[0] - - yaml = YAML() - with open(get_data("tests/wf/expect_trick_packed.cwl")) as packed_file: - expect_packed = yaml.load(packed_file) packed = cwltool.pack.pack(loadingContext, uri) - adjustFileObjs( - packed, partial(make_relative, os.path.abspath(get_data("tests/wf"))) - ) - adjustDirObjs(packed, partial(make_relative, os.path.abspath(get_data("tests/wf")))) + context_dir = os.path.abspath(os.path.dirname(get_data(unpacked))) + adjustFileObjs(packed, partial(make_relative, context_dir)) + adjustDirObjs(packed, partial(make_relative, context_dir)) - assert "$schemas" in packed - packed_schemas = packed["$schemas"] - assert isinstance(packed_schemas, Sized) - assert len(packed_schemas) == len(expect_packed["$schemas"]) - del packed["$schemas"] - del expect_packed["$schemas"] + yaml = YAML(typ="safe", pure=True) + with open(get_data(expected)) as packed_file: + expect_packed = yaml.load(packed_file) + + if "$schemas" in expect_packed: + assert "$schemas" in packed + packed_schemas = packed["$schemas"] + assert isinstance(packed_schemas, Sized) + assert len(packed_schemas) == len(expect_packed["$schemas"]) + del packed["$schemas"] + del expect_packed["$schemas"] assert packed == expect_packed diff --git a/tests/wf/operation/abstract-cosifer.cwl b/tests/wf/operation/abstract-cosifer.cwl new file mode 100644 index 0000000000..49f630e913 --- /dev/null +++ b/tests/wf/operation/abstract-cosifer.cwl @@ -0,0 +1,25 @@ +class: Operation +cwlVersion: v1.2 + +requirements: + DockerRequirement: + dockerPull: 'tsenit/cosifer:b4d5af45d2fc54b6bff2a9153a8e9054e560302e' + +inputs: + data_matrix: + type: File + separator: + type: string? + doc: The separator used in the data_matrix file + index_col: + type: int? + gmt_filepath: + type: File? + outdir: + type: string? + samples_on_rows: + type: boolean? + +outputs: + resdir: + type: Directory diff --git a/tests/wf/operation/expect_operation-single_packed.cwl b/tests/wf/operation/expect_operation-single_packed.cwl new file mode 100644 index 0000000000..7e01f69733 --- /dev/null +++ b/tests/wf/operation/expect_operation-single_packed.cwl @@ -0,0 +1,154 @@ +{ + "$graph": [ + { + "class": "Operation", + "requirements": [ + { + "dockerPull": "tsenit/cosifer:b4d5af45d2fc54b6bff2a9153a8e9054e560302e", + "class": "DockerRequirement" + } + ], + "inputs": [ + { + "type": "File", + "id": "#abstract-cosifer.cwl/data_matrix" + }, + { + "type": [ + "null", + "File" + ], + "id": "#abstract-cosifer.cwl/gmt_filepath" + }, + { + "type": [ + "null", + "int" + ], + "id": "#abstract-cosifer.cwl/index_col" + }, + { + "type": [ + "null", + "string" + ], + "id": "#abstract-cosifer.cwl/outdir" + }, + { + "type": [ + "null", + "boolean" + ], + "id": "#abstract-cosifer.cwl/samples_on_rows" + }, + { + "type": [ + "null", + "string" + ], + "doc": "The separator used in the data_matrix file", + "id": "#abstract-cosifer.cwl/separator" + } + ], + "outputs": [ + { + "type": "Directory", + "id": "#abstract-cosifer.cwl/resdir" + } + ], + "id": "#abstract-cosifer.cwl" + }, + { + "class": "Workflow", + "id": "#main", + "label": "abstract-cosifer-workflow", + "inputs": [ + { + "type": "File", + "doc": "Gene expression data matrix", + "id": "#data_matrix" + }, + { + "type": [ + "null", + "File" + ], + "doc": "Optional GMT file to perform inference on multiple gene sets", + "id": "#gmt_filepath" + }, + { + "type": [ + "null", + "int" + ], + "doc": "Column index in the data. Defaults to None, a.k.a., no index", + "id": "#index_col" + }, + { + "type": "string", + "doc": "Path to the output directory", + "id": "#outdir" + }, + { + "type": [ + "null", + "boolean" + ], + "doc": "Flag that indicates that data contain the samples on rows. Defaults to False.", + "id": "#samples_on_rows" + }, + { + "type": [ + "null", + "string" + ], + "doc": "Separator for the data. Defaults to .", + "id": "#separator" + } + ], + "outputs": [ + { + "type": "Directory", + "outputSource": "#/abstract_cosifer/resdir", + "id": "#resdir" + } + ], + "steps": [ + { + "run": "#abstract-cosifer.cwl", + "in": [ + { + "source": "#data_matrix", + "id": "#abstract_cosifer/data_matrix" + }, + { + "source": "#gmt_filepath", + "id": "#abstract_cosifer/gmt_filepath" + }, + { + "source": "#index_col", + "id": "#abstract_cosifer/index_col" + }, + { + "source": "#outdir", + "id": "#abstract_cosifer/outdir" + }, + { + "source": "#samples_on_rows", + "id": "#abstract_cosifer/samples_on_rows" + }, + { + "source": "#separator", + "id": "#abstract_cosifer/separator" + } + ], + "out": [ + "#/abstract_cosifer/resdir" + ], + "id": "#abstract_cosifer" + } + ] + } + ], + "cwlVersion": "v1.2" +} \ No newline at end of file diff --git a/tests/wf/operation/operation-single.cwl b/tests/wf/operation/operation-single.cwl new file mode 100644 index 0000000000..2119bf02e1 --- /dev/null +++ b/tests/wf/operation/operation-single.cwl @@ -0,0 +1,27 @@ +class: Workflow +cwlVersion: v1.2 +id: abstract_cosifer_workflow +label: abstract-cosifer-workflow + +inputs: + data_matrix: {type: File, doc: "Gene expression data matrix"} + gmt_filepath: {type: "File?", doc: "Optional GMT file to perform inference on multiple gene sets"} + index_col: {type: "int?", doc: "Column index in the data. Defaults to None, a.k.a., no index"} + outdir: {type: string, doc: "Path to the output directory"} + separator: {type: "string?", doc: "Separator for the data. Defaults to ."} + samples_on_rows: {type: "boolean?", doc: "Flag that indicates that data contain the samples on rows. Defaults to False."} + +outputs: + resdir: {type: Directory, outputSource: abstract_cosifer/resdir} + +steps: + abstract_cosifer: + run: abstract-cosifer.cwl + in: + data_matrix: data_matrix + separator: separator + index_col: index_col + gmt_filepath: gmt_filepath + outdir: outdir + samples_on_rows: samples_on_rows + out: [resdir] From 6f9b232bf19459c9b1bedbc84729b94ae6554bf7 Mon Sep 17 00:00:00 2001 From: Peter Amstutz Date: Fri, 9 Jul 2021 11:15:40 -0400 Subject: [PATCH 069/111] Pin networkx so we don't install numpy, scipy and pandas by default. --- requirements.txt | 1 + setup.py | 1 + 2 files changed, 2 insertions(+) diff --git a/requirements.txt b/requirements.txt index cdd9e1ab64..977cb6259d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,6 +4,7 @@ rdflib>=4.2.2,<5.1 shellescape>=3.4.1,<3.9 schema-salad>=8.1.20210627200047,<9 prov==1.5.1 +networkx==2.5.1 bagit==1.8.1 mypy-extensions psutil>=5.6.6 diff --git a/setup.py b/setup.py index 1a2828b986..6654fe45e2 100644 --- a/setup.py +++ b/setup.py @@ -117,6 +117,7 @@ "mypy-extensions", "psutil >= 5.6.6", "prov == 1.5.1", + "networkx < 2.6", "bagit >= 1.6.4", "typing-extensions", "coloredlogs", From 58238d323e1972c0d8948fe9955643e599685fdb Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" Date: Fri, 9 Jul 2021 17:28:19 +0200 Subject: [PATCH 070/111] ignore the snyk warning, does not apply to us --- .snyk | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 .snyk diff --git a/.snyk b/.snyk new file mode 100644 index 0000000000..2da43bac40 --- /dev/null +++ b/.snyk @@ -0,0 +1,10 @@ +# Snyk (https://snyk.io) policy file, patches or ignores known vulnerabilities. +version: v1.19.0 +# ignores vulnerabilities until expiry date; change duration by modifying expiry date +ignore: + SNYK-PYTHON-NETWORKX-1062709: + - '*': + reason: Prov does not use the affected code path + expires: 2022-08-08T15:27:21.289Z + created: 2021-07-09T15:27:21.300Z +patch: {} From 7094ede917c2d5b16d11f9231fe0c05260b51be6 Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" <1330696+mr-c@users.noreply.github.com> Date: Sat, 17 Jul 2021 11:14:31 +0200 Subject: [PATCH 071/111] Sort and expand the badge collection --- README.rst | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index a8792482fc..d04cd2d949 100644 --- a/README.rst +++ b/README.rst @@ -2,7 +2,13 @@ Common Workflow Language tool description reference implementation ================================================================== -|Linux Status| |Debian Stable package| |Debian Testing package| |Coverage Status| |Downloads| +|Linux Status| |Coverage Status| + +PyPI: |PyPI Version| |PyPI Downloads Month| |Total PyPI Downloads| + +Conda: |Conda Version| |Conda Installs| + +Debian: |Debian Testing package| |Debian Stable package| .. |Linux Status| image:: https://github.com/common-workflow-language/cwltool/actions/workflows/ci-tests.yml/badge.svg?branch=main :target: https://github.com/common-workflow-language/cwltool/actions/workflows/ci-tests.yml @@ -16,9 +22,21 @@ Common Workflow Language tool description reference implementation .. |Coverage Status| image:: https://img.shields.io/codecov/c/github/common-workflow-language/cwltool.svg :target: https://codecov.io/gh/common-workflow-language/cwltool -.. |Downloads| image:: https://pepy.tech/badge/cwltool/month +.. |PyPI Version| image:: https://badge.fury.io/py/cwltool.svg + :target: https://badge.fury.io/py/cwltool + +.. |PyPI Downloads Month| image:: https://pepy.tech/badge/cwltool/month :target: https://pepy.tech/project/cwltool +.. |Total PyPI Downloads| image:: https://static.pepy.tech/personalized-badge/cwltool?period=total&units=international_system&left_color=black&right_color=orange&left_text=Total%20PyPI%20Downloads + :target: https://pepy.tech/project/cwltool + +.. |Conda Version| image:: https://anaconda.org/conda-forge/cwltool/badges/version.svg + :target: https://anaconda.org/conda-forge/cwltool + +.. |Conda Installs| image:: https://anaconda.org/conda-forge/cwltool/badges/downloads.svg + :target: https://anaconda.org/conda-forge/cwltool + This is the reference implementation of the Common Workflow Language. It is intended to be feature complete and provide comprehensive validation of CWL files as well as provide other tools related to working with CWL. From 1fb45c85a7ef9ab20df53fa906f20468d069518e Mon Sep 17 00:00:00 2001 From: Peter Amstutz Date: Thu, 22 Jul 2021 10:18:29 -0400 Subject: [PATCH 072/111] Validates file with SchemaDefRequirement (#1475) refs bug #1473 --- cwltool/process.py | 2 +- tests/test_schemadef.py | 10 + tests/wf/schemadef-bug-1473.cwl | 1459 +++++++++++++++++++++++++++++++ 3 files changed, 1470 insertions(+), 1 deletion(-) create mode 100644 tests/test_schemadef.py create mode 100644 tests/wf/schemadef-bug-1473.cwl diff --git a/cwltool/process.py b/cwltool/process.py index 354ecd2413..39efb95e82 100644 --- a/cwltool/process.py +++ b/cwltool/process.py @@ -633,7 +633,7 @@ def __init__( sd, _ = self.get_requirement("SchemaDefRequirement") if sd is not None: - sdtypes = cast(MutableSequence[CWLObjectType], sd["types"]) + sdtypes = copy.deepcopy(cast(MutableSequence[CWLObjectType], sd["types"])) avroize_type(cast(MutableSequence[CWLOutputType], sdtypes)) av = make_valid_avro( sdtypes, diff --git a/tests/test_schemadef.py b/tests/test_schemadef.py new file mode 100644 index 0000000000..10c0e87b29 --- /dev/null +++ b/tests/test_schemadef.py @@ -0,0 +1,10 @@ +"""Tests related SchemaDefRequirement.""" + +from cwltool.main import main +from .util import get_data + + +def test_schemadef() -> None: + """Confirm bug 1473 is fixed by checking that the test case validates.""" + exit_code = main(["--validate", get_data("tests/wf/schemadef-bug-1473.cwl")]) + assert exit_code == 0 diff --git a/tests/wf/schemadef-bug-1473.cwl b/tests/wf/schemadef-bug-1473.cwl new file mode 100644 index 0000000000..47744032f4 --- /dev/null +++ b/tests/wf/schemadef-bug-1473.cwl @@ -0,0 +1,1459 @@ +{ + "$graph": [ + { + "class": "ExpressionTool", + "id": "#flatten-array-fastq-list__1.0.0.cwl", + "label": "flatten-array-fastq-list-schema v(1.0.0)", + "doc": "Documentation for flatten-array-fastq-list-schema v1.0.0\n", + "requirements": [ + { + "class": "InlineJavascriptRequirement" + }, + { + "types": [ + { + "type": "record", + "name": "#fastq-list-row__1.0.0.yaml/fastq-list-row", + "fields": [ + { + "label": "lane", + "doc": "The lane that the sample was run on\n", + "type": "int", + "name": "#fastq-list-row__1.0.0.yaml/fastq-list-row/lane" + }, + { + "label": "read 1", + "doc": "The path to R1 of a sample\n", + "type": "File", + "streamable": true, + "name": "#fastq-list-row__1.0.0.yaml/fastq-list-row/read_1" + }, + { + "label": "read 2", + "doc": "The path to R2 of a sample\n", + "type": [ + "null", + "File" + ], + "streamable": true, + "name": "#fastq-list-row__1.0.0.yaml/fastq-list-row/read_2" + }, + { + "label": "rgid", + "doc": "The read-group id of the sample.\nOften an index\n", + "type": "string", + "name": "#fastq-list-row__1.0.0.yaml/fastq-list-row/rgid" + }, + { + "label": "rglb", + "doc": "The read-group library of the sample.\n", + "type": "string", + "name": "#fastq-list-row__1.0.0.yaml/fastq-list-row/rglb" + }, + { + "label": "rgsm", + "doc": "The read-group sample name\n", + "type": "string", + "name": "#fastq-list-row__1.0.0.yaml/fastq-list-row/rgsm" + } + ], + "id": "#fastq-list-row__1.0.0.yaml" + } + ], + "class": "SchemaDefRequirement" + } + ], + "inputs": [ + { + "label": "two dim array", + "doc": "two dimensional array with fastq list row\n", + "type": { + "type": "array", + "items": { + "type": "array", + "items": "#fastq-list-row__1.0.0.yaml/fastq-list-row" + } + }, + "inputBinding": { + "loadContents": true + }, + "id": "#flatten-array-fastq-list__1.0.0.cwl/arrayTwoDim" + } + ], + "outputs": [ + { + "label": "one dim array", + "doc": "one dimensional array\n", + "type": { + "type": "array", + "items": "#fastq-list-row__1.0.0.yaml/fastq-list-row" + }, + "id": "#flatten-array-fastq-list__1.0.0.cwl/array1d" + } + ], + "expression": "${\n var newArray= [];\n for (var i = 0; i < inputs.arrayTwoDim.length; i++) {\n for (var k = 0; k < inputs.arrayTwoDim[i].length; k++) {\n newArray.push((inputs.arrayTwoDim[i])[k]);\n }\n }\n return { 'array1d' : newArray }\n}\n", + "https://schema.org/author": { + "class": "https://schema.org/Person", + "https://schema.org/name": "Sehrish Kanwal", + "https://schema.org/email": "sehrish.kanwal@umccr.org" + }, + "$namespaces": { + "s": "https://schema.org/" + } + }, + { + "class": "ExpressionTool", + "id": "#get-samplesheet-midfix-regex__1.0.0.cwl", + "label": "get-samplesheet-midfix-regex v(1.0.0)", + "doc": "Documentation for get-samplesheet-midfix-regex v1.0.0\n", + "requirements": [ + { + "expressionLib": [ + "var get_batch_name_from_samplesheet = function(samplesheet_basename) { /* Get everything between SampleSheet and csv https://regex101.com/r/KlF7LW/1 */ var samplesheet_regex = /SampleSheet\\.(\\S+)\\.csv/g; return samplesheet_regex.exec(samplesheet_basename)[1]; }", + "var get_batch_names = function(file_objs) { /* For each file object extract the midfix */\n/* Initialise batch names */ var batch_names = [];\nfor (var i = 0; i < file_objs.length; i++){ /* But of that basename, get the midfix */ batch_names.push(get_batch_name_from_samplesheet(file_objs[i].basename)); }\nreturn batch_names; }" + ], + "class": "InlineJavascriptRequirement" + } + ], + "inputs": [ + { + "label": "sample sheets", + "doc": "Input samplesheet to extract midfix from\n", + "type": { + "type": "array", + "items": "File" + }, + "id": "#get-samplesheet-midfix-regex__1.0.0.cwl/samplesheets" + } + ], + "outputs": [ + { + "label": "output batch names", + "doc": "List of output batch names\n", + "type": { + "type": "array", + "items": "string" + }, + "id": "#get-samplesheet-midfix-regex__1.0.0.cwl/batch_names" + } + ], + "expression": "${\n return {\"batch_names\": get_batch_names(inputs.samplesheets)};\n}", + "https://schema.org/author": { + "class": "https://schema.org/Person", + "https://schema.org/name": "Sehrish Kanwal", + "https://schema.org/email": "sehrish.kanwal@umccr.org" + } + }, + { + "class": "CommandLineTool", + "id": "#bclConvert__3.7.5.cwl", + "label": "bclConvert v(3.7.5)", + "doc": "Runs the BCL Convert application off standard architechture\n", + "hints": [ + { + "dockerPull": "umccr/bcl-convert:3.7.5", + "class": "DockerRequirement" + }, + { + "coresMin": 72, + "ramMin": 64000, + "class": "ResourceRequirement", + "http://platform.illumina.com/rdf/ica/resources": { + "type": "standardHiCpu", + "size": "large" + } + } + ], + "requirements": [ + { + "listing": [ + { + "entryname": "scripts/run_bclconvert.sh", + "entry": "#!/usr/bin/bash\n\n# Fail on non-zero exit code\nset -euo pipefail\n\n# Run bcl-convert with input parameters\neval bcl-convert '\"\\${@}\"'\n\n# Delete undetermined indices\nif [[ \"$(inputs.delete_undetermined_indices)\" == \"true\" ]]; then\n echo \"Deleting undetermined indices\" 1>&2\n find \"$(inputs.output_directory)\" -mindepth 1 -maxdepth 1 -name 'Undetermined_S0_*' -exec rm {} \\\\;\nfi\n" + } + ], + "class": "InitialWorkDirRequirement" + }, + { + "class": "InlineJavascriptRequirement" + }, + { + "types": [ + { + "$import": "#fastq-list-row__1.0.0.yaml" + } + ], + "class": "SchemaDefRequirement" + } + ], + "baseCommand": [ + "bash" + ], + "arguments": [ + { + "position": -1, + "valueFrom": "scripts/run_bclconvert.sh" + } + ], + "inputs": [ + { + "label": "bcl conversion threads", + "doc": "Specifies number of threads used for conversion per tile.\nMust be between 1 and available hardware threads,\ninclusive.\n", + "type": [ + "null", + "int" + ], + "inputBinding": { + "prefix": "--bcl-conversion-threads" + }, + "id": "#bclConvert__3.7.5.cwl/bcl_conversion_threads" + }, + { + "label": "bcl input directory", + "doc": "A main command-line option that indicates the path to the run\nfolder directory\n", + "type": "Directory", + "inputBinding": { + "prefix": "--bcl-input-directory" + }, + "id": "#bclConvert__3.7.5.cwl/bcl_input_directory" + }, + { + "label": "bcl num compression threads", + "doc": "Specifies number of CPU threads used for compression of\noutput FASTQ files. Must be between 1 and available\nhardware threads, inclusive.\n", + "type": [ + "null", + "int" + ], + "inputBinding": { + "prefix": "--bcl-num-compression-threads" + }, + "id": "#bclConvert__3.7.5.cwl/bcl_num_compression_threads" + }, + { + "label": "bcl num decompression threads", + "doc": "Specifies number of CPU threads used for decompression\nof input base call files. Must be between 1 and available\nhardware threads, inclusive.\n", + "type": [ + "null", + "int" + ], + "inputBinding": { + "prefix": "--bcl-num-decompression-threads" + }, + "id": "#bclConvert__3.7.5.cwl/bcl_num_decompression_threads" + }, + { + "label": "bcl num parallel tiles", + "doc": "Specifies number of tiles being converted to FASTQ files in\nparallel. Must be between 1 and available hardware threads,\ninclusive.\n", + "type": [ + "null", + "int" + ], + "inputBinding": { + "prefix": "--bcl-num-parallel-tiles" + }, + "id": "#bclConvert__3.7.5.cwl/bcl_num_parallel_tiles" + }, + { + "label": "convert only one lane", + "doc": "Convert only the specified lane number. The value must\nbe less than or equal to the number of lanes specified in the\nRunInfo.xml. Must be a single integer value.\n", + "type": [ + "null", + "int" + ], + "inputBinding": { + "prefix": "--bcl-only-lane" + }, + "id": "#bclConvert__3.7.5.cwl/bcl_only_lane" + }, + { + "label": "bcl sample project subdirectories", + "doc": "true \u2014 Allows creation of Sample_Project subdirectories\nas specified in the sample sheet. This option must be set to true for\nthe Sample_Project column in the data section to be used.\nDefault set to false.\n", + "type": [ + "null", + "boolean" + ], + "inputBinding": { + "prefix": "--bcl-sampleproject-subdirectories", + "valueFrom": "$(self.toString())" + }, + "id": "#bclConvert__3.7.5.cwl/bcl_sampleproject_subdirectories" + }, + { + "label": "delete undetermined indices", + "doc": "Delete undetermined indices on completion of the run\nDefault: false\n", + "type": [ + "null", + "boolean" + ], + "id": "#bclConvert__3.7.5.cwl/delete_undetermined_indices" + }, + { + "label": "first tile only", + "doc": "true \u2014 Only process the first tile of the first swath of the\ntop surface of each lane specified in the sample sheet.\nfalse \u2014 Process all tiles in each lane, as specified in the sample\nsheet. Default is false\n", + "type": [ + "null", + "boolean" + ], + "inputBinding": { + "prefix": "--first-tile-only", + "valueFrom": "$(self.toString())" + }, + "id": "#bclConvert__3.7.5.cwl/first_tile_only" + }, + { + "label": "force", + "doc": "Allow for the directory specified by the --output-directory\noption to already exist. Default is false\n", + "type": [ + "null", + "boolean" + ], + "inputBinding": { + "prefix": "--force" + }, + "id": "#bclConvert__3.7.5.cwl/force" + }, + { + "label": "output directory", + "doc": "A required command-line option that indicates the path to\ndemultuplexed fastq output. The directory must not exist, unless -f,\nforce is specified\n", + "type": "string", + "inputBinding": { + "prefix": "--output-directory" + }, + "id": "#bclConvert__3.7.5.cwl/output_directory" + }, + { + "label": "sample sheet", + "doc": "Indicates the path to the sample sheet to specify the\nsample sheet location and name, if different from the default.\n", + "type": [ + "null", + "File" + ], + "inputBinding": { + "prefix": "--sample-sheet" + }, + "id": "#bclConvert__3.7.5.cwl/samplesheet" + }, + { + "label": "shared thread odirect output", + "doc": "Uses experimental shared-thread file output code, which\nrequires O_DIRECT mode. Must be true or false.\nThis file output method is optimized for sample counts\ngreater than 100,000. It is not recommended for lower\nsample counts or using a distributed file system target such\nas GPFS or Lustre. Default is false\n", + "type": [ + "null", + "boolean" + ], + "inputBinding": { + "prefix": "--shared-thread-odirect-output" + }, + "id": "#bclConvert__3.7.5.cwl/shared_thread_odirect_output" + }, + { + "label": "strict mode", + "doc": "true \u2014 Abort the program if any filter, locs, bcl, or bci lane\nfiles are missing or corrupt.\nfalse \u2014 Continue processing if any filter, locs, bcl, or bci lane files\nare missing. Return a warning message for each missing or corrupt\nfile.\n", + "type": [ + "null", + "boolean" + ], + "inputBinding": { + "prefix": "--strict-mode", + "valueFrom": "$(self.toString())" + }, + "id": "#bclConvert__3.7.5.cwl/strict_mode" + } + ], + "outputs": [ + { + "label": "bcl convert directory output", + "doc": "Output directory containing the fastq files, reports and stats\n", + "type": "Directory", + "outputBinding": { + "glob": "$(inputs.output_directory)" + }, + "id": "#bclConvert__3.7.5.cwl/bcl_convert_directory_output" + }, + { + "label": "fastq list rows", + "doc": "This schema contains the following inputs:\n* rgid: The id of the sample\n* rgsm: The name of the sample\n* rglb: The library of the sample\n* lane: The lane of the sample\n* read_1: The read 1 File of the sample\n* read_2: The read 2 File of the sample (optional)\n", + "type": { + "type": "array", + "items": "#fastq-list-row__1.0.0.yaml/fastq-list-row" + }, + "outputBinding": { + "glob": "$(inputs.output_directory)/Reports/fastq_list.csv", + "loadContents": true, + "outputEval": "${\n /*\n Load inputs initialise output variables\n */\n var output_array = [];\n var lines = self[0].contents.split(\"\\n\")\n\n /*\n Generate output object by iterating through fastq_list csv\n */\n for (var i=0; i < lines.length - 1; i++){\n /*\n First line is a header row skip it\n */\n if (i === 0){\n continue;\n }\n\n /*\n Split row and collect corresponding file paths\n */\n var rgid = lines[i].split(\",\")[0];\n var rgsm = lines[i].split(\",\")[1];\n var rglb = lines[i].split(\",\")[2];\n var lane = parseInt(lines[i].split(\",\")[3]);\n var read_1_path = lines[i].split(\",\")[4];\n var read_2_path = lines[i].split(\",\")[5];\n\n /*\n Initialise the output row as a dict\n */\n var output_fastq_list_row = {\n \"rgid\": rgid,\n \"rglb\": rglb,\n \"rgsm\": rgsm,\n \"lane\": lane,\n \"read_1\": {\n \"class\": \"File\",\n \"path\": read_1_path\n },\n }\n\n\n if (read_2_path !== \"\"){\n /*\n read 2 path exists\n */\n output_fastq_list_row[\"read_2\"] = {\n \"class\": \"File\",\n \"path\": read_2_path\n }\n }\n\n /*\n Append object to output array\n */\n output_array.push(output_fastq_list_row);\n }\n return output_array;\n}\n" + }, + "id": "#bclConvert__3.7.5.cwl/fastq_list_rows" + } + ], + "successCodes": [ + 0 + ], + "https://schema.org/author": { + "class": "https://schema.org/Person", + "https://schema.org/name": "Sehrish Kanwal", + "https://schema.org/email": "sehrish.kanwal@umccr.org" + } + }, + { + "class": "CommandLineTool", + "id": "#custom-samplesheet-split-by-settings__1.0.0.cwl", + "label": "custom-samplesheet-split-by-settings v(1.0.0)", + "doc": "Use before running bcl-convert workflow to ensure that the bclConvert workflow can run in parallel.\nSamples will be split into separate samplesheets based on their cycles specification\n", + "hints": [ + { + "dockerPull": "umccr/alpine_pandas:latest-cwl", + "class": "DockerRequirement" + }, + { + "class": "ResourceRequirement", + "http://platform.illumina.com/rdf/ica/resources": { + "tier": "standard", + "type": "standard", + "size": "small", + "coresMin": 2, + "ramMin": 4000 + } + } + ], + "requirements": [ + { + "listing": [ + { + "entryname": "samplesheet-by-settings.py", + "entry": "#!/usr/bin/env python3\n\n\"\"\"\nTake in a samplesheet,\nLook through headers, rename as necessary\nLook through samples, update settings logic as necessary\nSplit samplesheet out into separate settings files\nWrite to separate files\nIf --samplesheet-format is set to v2 then:\n* rename Settings.Adapter to Settings.AdapterRead1\n* Reduce Data to the columns Lane, Sample_ID, index, index2, Sample_Project\n* Add FileFormatVersion=2 to Header\n* Convert Reads from list to dict with Read1Cycles and Read2Cycles as keys\n\"\"\"\n\n# Imports\nimport re\nimport os\nimport pandas as pd\nimport numpy as np\nimport logging\nimport argparse\nfrom pathlib import Path\nimport sys\nfrom copy import deepcopy\nimport json\n\n# Set logging level\nlogging.basicConfig(level=logging.DEBUG)\n\n# Globals\nSAMPLESHEET_HEADER_REGEX = r\"^\\[(\\S+)\\](,+)?\" # https://regex101.com/r/5nbe9I/1\nV2_SAMPLESHEET_HEADER_VALUES = {\"Data\": \"BCLConvert_Data\",\n \"Settings\": \"BCLConvert_Settings\"}\nV2_FILE_FORMAT_VERSION = \"2\"\nV2_DEFAULT_INSTRUMENT_TYPE = \"NovaSeq 6000\"\n\n\ndef get_args():\n \"\"\"\n Get arguments for the command\n \"\"\"\n parser = argparse.ArgumentParser(description=\"Create samplesheets based on settings inputs.\"\n \"Expects a v1 samplesheet as input and settings by samples\"\n \"as inputs through separated jsonised strings / arrays\")\n\n # Arguments\n parser.add_argument(\"--samplesheet-csv\", required=True,\n help=\"Path to v1 samplesheet csv\")\n parser.add_argument(\"--out-dir\", required=False,\n help=\"Output directory for samplesheets, set to cwd if not specified\")\n parser.add_argument(\"--settings-by-samples\", action=\"append\", nargs='*', required=False,\n default=[],\n help=\"Settings logic for each sample\")\n parser.add_argument(\"--ignore-missing-samples\", required=False,\n default=False, action=\"store_true\",\n help=\"If not set, error if samples in the samplesheet are not present in --settings-by-samples arg\")\n parser.add_argument(\"--samplesheet-format\", required=False,\n choices=[\"v1\", \"v2\"], default=\"v1\",\n help=\"Type of samplesheet we wish to output\")\n\n return parser.parse_args()\n\n\ndef set_args(args):\n \"\"\"\n Convert --settings-by-samples to dict\n :return:\n \"\"\"\n\n # Get user args\n samplesheet_csv_arg = getattr(args, \"samplesheet_csv\", None)\n outdir_arg = getattr(args, \"out_dir\", None)\n settings_by_samples_arg = getattr(args, \"settings_by_samples\", [])\n\n # Convert samplesheet csv to path\n samplesheet_csv_path = Path(samplesheet_csv_arg)\n # Check its a file\n if not samplesheet_csv_path.is_file():\n logging.error(\"Could not find file {}\".format(samplesheet_csv_path))\n sys.exit(1)\n # Set attribute as Path object\n setattr(args, \"samplesheet_csv\", samplesheet_csv_path)\n\n # Checking the output path\n if outdir_arg is None:\n outdir_arg = os.getcwd()\n outdir_path = Path(outdir_arg)\n if not outdir_path.parent.is_dir():\n logging.error(\"Could not create --out-dir, make sure parents exist. Exiting\")\n sys.exit(1)\n elif not outdir_path.is_dir():\n outdir_path.mkdir(parents=False)\n setattr(args, \"out_dir\", outdir_path)\n\n # Load json lists\n settings_by_samples_list = []\n for settings_by_samples in settings_by_samples_arg:\n settings_by_samples_list.append(json.loads(settings_by_samples[0]))\n\n # Set attr as dicts grouped by each batch_name\n settings_by_batch_names = {}\n for settings_by_samples in settings_by_samples_list:\n # Initialise batch name\n batch_name_key = settings_by_samples.get(\"batch_name\")\n settings_by_batch_names[batch_name_key] = {}\n # Add in sample ids\n settings_by_batch_names[batch_name_key][\"samples\"] = settings_by_samples.get(\"samples\")\n # Add in settings\n settings_by_batch_names[batch_name_key][\"settings\"] = settings_by_samples.get(\"settings\")\n\n # Write attributes back to args dict\n setattr(args, \"settings_by_batch_names\", settings_by_batch_names)\n\n # Return args\n return args\n\n\ndef read_samplesheet_csv(samplesheet_csv_path):\n \"\"\"\n Read the samplesheet like a dodgy INI parser\n :param samplesheet_csv_path:\n :return:\n \"\"\"\n with open(samplesheet_csv_path, \"r\") as samplesheet_csv_h:\n # Read samplesheet in\n sample_sheet_sections = {}\n current_section = None\n current_section_item_list = []\n header_match_regex = re.compile(SAMPLESHEET_HEADER_REGEX)\n\n for line in samplesheet_csv_h.readlines():\n # Check if blank line\n if line.strip().rstrip(\",\") == \"\":\n continue\n # Check if the current line is a header\n header_match_obj = header_match_regex.match(line.strip())\n if header_match_obj is not None and current_section is None:\n # First line, don't need to write out previous section to obj\n # Set current section to first group\n current_section = header_match_obj.group(1)\n current_section_item_list = []\n elif header_match_obj is not None and current_section is not None:\n # A header further down, write out previous section and then reset sections\n sample_sheet_sections[current_section] = current_section_item_list\n # Now reset sections\n current_section = header_match_obj.group(1)\n current_section_item_list = []\n # Make sure the first line is a section\n elif current_section is None and header_match_obj is None:\n logging.error(\"Top line of csv was not a section header. Exiting\")\n sys.exit(1)\n else: # We're in a section\n if not current_section == \"Data\":\n # Strip trailing slashes from line\n current_section_item_list.append(line.strip().rstrip(\",\"))\n else:\n # Don't strip trailing slashes from line\n current_section_item_list.append(line.strip())\n\n # Write out the last section\n sample_sheet_sections[current_section] = current_section_item_list\n\n return sample_sheet_sections\n\n\ndef configure_samplesheet_obj(sample_sheet_obj):\n \"\"\"\n Each section of the samplesheet obj is in a ',' delimiter ini format\n Except for [Reads] which is just a list\n And [Data] which is a dataframe\n :param sample_sheet_obj:\n :return:\n \"\"\"\n\n for section_name, section_str_list in sample_sheet_obj.items():\n if section_name == \"Data\":\n # Convert to dataframe\n sample_sheet_obj[section_name] = pd.DataFrame(columns=section_str_list[0].split(\",\"),\n data=[row.split(\",\") for row in\n section_str_list[1:]])\n elif section_name == \"Reads\":\n # Keep as a list\n continue\n else:\n # Convert to dict\n sample_sheet_obj[section_name] = {line.split(\",\", 1)[0]: line.split(\",\", 1)[-1]\n for line in section_str_list\n if not line.split(\",\", 1)[0] == \"\"}\n # Check all values are non empty\n for key, value in sample_sheet_obj[section_name].items():\n if value == \"\" or value.startswith(\",\"):\n logging.error(\"Could not parse key \\\"{}\\\" in section \\\"{}\\\"\".format(key, section_name))\n logging.error(\"Value retrieved was \\\"{}\\\"\".format(value))\n sys.exit(1)\n\n return sample_sheet_obj\n\n\ndef lower_under_score_to_camel_case(string):\n \"\"\"\n Quick script to update a string from \"this_type\" to \"ThisType\"\n Necessary for the bclconvert settings to be in camel case\n Parameters\n ----------\n string\n Returns\n -------\n \"\"\"\n\n camel_case_string_list = []\n words_list = string.split(\"_\")\n\n for word in words_list:\n camel_case_string_list.append(word.title())\n\n return \"\".join(map(str, camel_case_string_list))\n\n\ndef strip_ns_from_indexes(samplesheetobj_data_df):\n \"\"\"\n Strip Ns from the end of the index and index2 headers\n :param samplesheetobj_data_df:\n :return:\n \"\"\"\n\n samplesheetobj_data_df['index'] = samplesheetobj_data_df['index'].apply(lambda x: x.rstrip(\"N\"))\n if 'index2' in samplesheetobj_data_df.columns.tolist():\n samplesheetobj_data_df['index2'] = samplesheetobj_data_df['index2'].apply(lambda x: x.rstrip(\"N\"))\n samplesheetobj_data_df['index2'] = samplesheetobj_data_df['index2'].replace(\"\", np.nan)\n\n return samplesheetobj_data_df\n\n\ndef rename_settings_and_data_headers_v2(samplesheet_obj):\n \"\"\"\n :return:\n \"\"\"\n\n for v1_key, v2_key in V2_SAMPLESHEET_HEADER_VALUES.items():\n if v1_key in samplesheet_obj.keys():\n samplesheet_obj[v2_key] = samplesheet_obj.pop(v1_key)\n\n return samplesheet_obj\n\n\ndef add_file_format_version_v2(samplesheet_header):\n \"\"\"\n Add FileFormatVersion key pair to samplesheet header for v2 samplesheet\n :param samplesheet_header:\n :return:\n \"\"\"\n\n samplesheet_header['FileFormatVersion'] = V2_FILE_FORMAT_VERSION\n\n return samplesheet_header\n\n\ndef set_instrument_type(samplesheet_header):\n \"\"\"\n Fix InstrumentType if it's not specified\n :param samplesheet_header:\n :return:\n \"\"\"\n\n if \"InstrumentType\" not in samplesheet_header.keys():\n samplesheet_header[\"InstrumentType\"] = V2_DEFAULT_INSTRUMENT_TYPE\n\n return samplesheet_header\n\n\ndef update_settings_v2(samplesheet_settings):\n \"\"\"\n Convert Adapter To AdapterRead1 for v2 samplesheet\n :param samplesheet_settings:\n :return:\n \"\"\"\n\n # Rename Adapter to AdapterRead1\n if \"Adapter\" in samplesheet_settings.keys() and not \"AdapterRead1\" in samplesheet_settings.keys():\n samplesheet_settings[\"AdapterRead1\"] = samplesheet_settings.pop(\"Adapter\")\n elif \"Adapter\" in samplesheet_settings.keys() and \"AdapterRead1\" in samplesheet_settings.keys():\n _ = samplesheet_settings.pop(\"Adapter\")\n\n # Drop any settings where settings are \"\" - needed for \"AdapterRead2\"\n samplesheet_settings = {\n key: val\n for key, val in samplesheet_settings.items()\n if not val == \"\"\n }\n return samplesheet_settings\n\n\ndef truncate_data_columns_v2(samplesheet_data_df):\n \"\"\"\n Truncate data columns to v2 columns\n Lane,Sample_ID,index,index2,Sample_Project\n :param samplesheet_data_df:\n :return:\n \"\"\"\n\n v2_columns = [\"Lane\", \"Sample_ID\", \"index\", \"index2\", \"Sample_Project\"]\n samplesheet_data_df = samplesheet_data_df.filter(items=v2_columns)\n\n return samplesheet_data_df\n\n\ndef convert_reads_from_list_to_dict_v2(samplesheet_reads):\n \"\"\"\n Convert Reads from a list to a dict format\n :param samplesheet_reads:\n :return:\n \"\"\"\n\n samplesheet_reads = {\"Read{}Cycles\".format(i + 1): rnum for i, rnum in enumerate(samplesheet_reads)}\n\n return samplesheet_reads\n\n\ndef convert_samplesheet_to_v2(samplesheet_obj):\n \"\"\"\n Runs through necessary steps to convert object to v2 samplesheet\n :param samplesheet_obj:\n :return:\n \"\"\"\n samplesheet_obj[\"Header\"] = add_file_format_version_v2(samplesheet_obj[\"Header\"])\n samplesheet_obj[\"Header\"] = set_instrument_type(samplesheet_obj[\"Header\"])\n samplesheet_obj[\"Settings\"] = update_settings_v2(samplesheet_obj[\"Settings\"])\n samplesheet_obj[\"Data\"] = truncate_data_columns_v2(samplesheet_obj[\"Data\"])\n samplesheet_obj[\"Reads\"] = convert_reads_from_list_to_dict_v2(samplesheet_obj[\"Reads\"])\n samplesheet_obj = rename_settings_and_data_headers_v2(samplesheet_obj)\n\n return samplesheet_obj\n\n\ndef check_samples(samplesheet_obj, settings_by_samples, ignore_missing_samples=False):\n \"\"\"\n If settings_by_samples is defined, ensure that each sample is present\n \"\"\"\n all_samples_in_samplesheet = samplesheet_obj[\"Data\"][\"Sample_ID\"].tolist()\n popped_samples = []\n\n if len(settings_by_samples.keys()) == 0:\n # No problem as we're not splitting samples by sample sheet\n return\n\n for batch_name, batch_settings_and_samples_dict in settings_by_samples.items():\n samples = batch_settings_and_samples_dict.get(\"samples\")\n for sample in samples:\n if sample in popped_samples:\n logging.error(\"Sample \\\"{}\\\" registered multiple times\".format(sample))\n sys.exit(1)\n elif sample not in all_samples_in_samplesheet:\n logging.error(\"Could not find sample \\\"{}\\\"\".format(sample))\n sys.exit(1)\n else:\n popped_samples.append(all_samples_in_samplesheet.pop(all_samples_in_samplesheet.index(sample)))\n\n if ignore_missing_samples:\n # No issue\n return\n\n if not len(all_samples_in_samplesheet) == 0:\n logging.error(\"The following samples have no associated batch name: {}\".format(\n \", \".join(map(str, [\"\\\"{}\\\"\".format(sample) for sample in all_samples_in_samplesheet]))\n ))\n sys.exit(1)\n\n\ndef write_out_samplesheets(samplesheet_obj, out_dir, settings_by_samples,\n is_v2=False):\n \"\"\"\n Write out samplesheets to each csv file\n :return:\n \"\"\"\n\n if not len(list(settings_by_samples.keys())) == 0:\n for batch_name, batch_settings_and_samples_dict in settings_by_samples.items():\n # Get settings\n settings = batch_settings_and_samples_dict.get(\"settings\")\n samples = batch_settings_and_samples_dict.get(\"samples\")\n # Duplicate samplesheet_obj\n samplesheet_obj_by_settings_copy = deepcopy(samplesheet_obj)\n # Convert df to csv string\n samplesheet_obj_by_settings_copy[\"Data\"] = samplesheet_obj_by_settings_copy[\"Data\"].query(\"Sample_ID in @samples\")\n # Update settings\n for setting_key, setting_val in settings.items():\n \"\"\"\n Update settings\n \"\"\"\n if setting_val is None:\n # Don't add None var\n continue\n # Update setting value for boolean types\n if type(setting_val) == bool:\n setting_val = 1 if setting_val else 0\n # Then assign to settings dict\n samplesheet_obj_by_settings_copy[\"Settings\"][\n lower_under_score_to_camel_case(setting_key)] = setting_val\n\n # Write out config\n write_samplesheet(samplesheet_obj=samplesheet_obj_by_settings_copy,\n output_file=out_dir / \"SampleSheet.{}.csv\".format(batch_name),\n is_v2=is_v2)\n\n else: # No splitting required\n write_samplesheet(samplesheet_obj=samplesheet_obj,\n output_file=out_dir / \"SampleSheet.csv\",\n is_v2=is_v2)\n\n\ndef write_samplesheet(samplesheet_obj, output_file, is_v2):\n \"\"\"\n Write out the samplesheet object and a given file\n :param samplesheet_obj:\n :param output_file:\n :param is_v2\n :return:\n \"\"\"\n\n # Rename samplesheet at the last possible moment\n if is_v2:\n # Drop index2 if all are \"N/A\"\n if 'index2' in samplesheet_obj[\"Data\"].columns.tolist() and \\\n samplesheet_obj[\"Data\"][\"index2\"].isna().all():\n samplesheet_obj[\"Data\"] = samplesheet_obj[\"Data\"].drop(columns=\"index2\")\n\n samplesheet_obj = convert_samplesheet_to_v2(samplesheet_obj)\n\n # Write the output file\n with open(output_file, 'w') as samplesheet_h:\n for section, section_values in samplesheet_obj.items():\n # Write out the section header\n samplesheet_h.write(\"[{}]\\n\".format(section))\n # Write out values\n if type(section_values) == list: # [Reads] for v1 samplesheets\n # Write out each item in a new line\n samplesheet_h.write(\"\\n\".join(section_values))\n elif type(section_values) == dict:\n samplesheet_h.write(\"\\n\".join(map(str, [\"{},{}\".format(key, value)\n for key, value in section_values.items()])))\n elif type(section_values) == pd.DataFrame:\n section_values.to_csv(samplesheet_h, index=False, header=True, sep=\",\")\n # Add new line before the next section\n samplesheet_h.write(\"\\n\\n\")\n\n\ndef main():\n # Get args\n args = get_args()\n\n # Check / set args\n logging.info(\"Checking args\")\n args = set_args(args=args)\n\n # Read config\n logging.info(\"Reading samplesheet\")\n samplesheet_obj = read_samplesheet_csv(samplesheet_csv_path=args.samplesheet_csv)\n\n # Configure samplesheet\n logging.info(\"Configuring samplesheet\")\n samplesheet_obj = configure_samplesheet_obj(samplesheet_obj)\n\n # Check missing samples\n logging.info(\"Checking missing samples\")\n check_samples(samplesheet_obj=samplesheet_obj,\n settings_by_samples=getattr(args, \"settings_by_batch_names\", {}),\n ignore_missing_samples=args.ignore_missing_samples)\n\n # Strip Ns from samplesheet indexes\n logging.info(\"Stripping Ns from indexes\")\n samplesheet_obj[\"Data\"] = strip_ns_from_indexes(samplesheet_obj[\"Data\"])\n\n # Write out samplesheets\n logging.info(\"Writing out samplesheets\")\n write_out_samplesheets(samplesheet_obj=samplesheet_obj,\n out_dir=args.out_dir,\n settings_by_samples=getattr(args, \"settings_by_batch_names\", {}),\n is_v2=True if args.samplesheet_format == \"v2\" else False)\n\n# Run main script\nif __name__ == \"__main__\":\n main()\n" + } + ], + "class": "InitialWorkDirRequirement" + }, + { + "class": "InlineJavascriptRequirement" + }, + { + "types": [ + { + "type": "record", + "name": "#settings-by-samples__1.0.0.yaml/settings-by-samples", + "fields": [ + { + "label": "batch name", + "doc": "The name for this combination of settings and sample ids.\nWill be used as the midfix for the name of the sample sheet.\nWill be used as the output directory in the bclconvert workflow\n", + "type": "string", + "name": "#settings-by-samples__1.0.0.yaml/settings-by-samples/batch_name" + }, + { + "label": "samples", + "doc": "The list of Sample_IDs with these BClConvert settings\n", + "type": { + "type": "array", + "items": "string" + }, + "name": "#settings-by-samples__1.0.0.yaml/settings-by-samples/samples" + }, + { + "label": "settings by override cylces", + "doc": "Additional bcl convert settings\n", + "type": [ + "null", + { + "type": "record", + "name": "#settings-by-samples__1.0.0.yaml/settings-by-samples/settings/settings", + "fields": [ + { + "label": "adapter behavior", + "doc": "Defines whether the software\nmasks or trims Read 1 and/or\nRead 2 adapter sequence(s).\nWhen AdapterRead1 or\nAdapterRead2 is not specified, this\nsetting cannot be specified.\n\u2022 mask\u2014The software masks the\nidentified Read 1 and/or Read 2\nsequence(s) with N.\n\u2022 trim\u2014The software trims the\nidentified Read 1 and/or Read 2\nsequence(s)\n", + "type": [ + "null", + { + "type": "enum", + "symbols": [ + "#settings-by-samples__1.0.0.yaml/settings-by-samples/settings/settings/adapter_behavior/mask", + "#settings-by-samples__1.0.0.yaml/settings-by-samples/settings/settings/adapter_behavior/trim" + ] + } + ], + "name": "#settings-by-samples__1.0.0.yaml/settings-by-samples/settings/settings/adapter_behavior" + }, + { + "label": "adapter read 1", + "doc": "The sequence of the Read 1\nadapter to be masked or trimmed.\nTo trim multiple adapters, separate\nthe sequences with a plus sign (+)\nto indicate independent adapters\nthat must be independently\nassessed for masking or trimming\nfor each read.\nAllowed characters: A, T, C, G.\n", + "type": [ + "null", + "string" + ], + "name": "#settings-by-samples__1.0.0.yaml/settings-by-samples/settings/settings/adapter_read_1" + }, + { + "label": "adapter read 2", + "doc": "The sequence of the Read 2\nadapter to be masked or trimmed.\nTo trim multiple adapters, separate\nthe sequences with a plus sign (+)\nto indicate independent adapters\nthat must be independently\nassessed for masking or trimming\nfor each read.\nAllowed characters: A, T, C, G.\n", + "type": [ + "null", + "string" + ], + "name": "#settings-by-samples__1.0.0.yaml/settings-by-samples/settings/settings/adapter_read_2" + }, + { + "label": "adapter stringency", + "doc": "he minimum match rate that\ntriggers masking or trimming. This\nvalue is calculated as MatchCount\n/ (MatchCount+MismatchCount).\nAccepted values are 0.5\u20131. The\ndefault value of 0.9 indicates that\nonly reads with \u2265 90% sequence\nidentity with the adapter are\ntrimmed.\n", + "type": [ + "null", + "float" + ], + "name": "#settings-by-samples__1.0.0.yaml/settings-by-samples/settings/settings/adapter_stringency" + }, + { + "label": "barcode mismatches index 1", + "doc": "The number of mismatches\nallowed for index1. Accepted\nvalues are 0, 1, or 2.\n", + "type": [ + "null", + "int" + ], + "name": "#settings-by-samples__1.0.0.yaml/settings-by-samples/settings/settings/barcode_mismatches_index_1" + }, + { + "label": "barcode mismatches index 2", + "doc": "The number of mismatches\nallowed for index2. Accepted\nvalues are 0, 1, or 2.\n", + "type": [ + "null", + "int" + ], + "name": "#settings-by-samples__1.0.0.yaml/settings-by-samples/settings/settings/barcode_mismatches_index_2" + }, + { + "label": "create fastq for index reads", + "doc": "Specifies whether software will\noutput fastqs for index reads. If\nindex reads are defined as a\nUMI then fastqs for the UMI will\nbe output (if TrimUMI is also set\nto 0). At least 1 index read must\nbe specified in the sample\nsheet.\n\u2022 0\u2014Fastq files will not be output\nfor index reads.\n\u2022 1\u2014Fastq files will be output for\nfastq reads.\n", + "type": [ + "null", + "boolean" + ], + "name": "#settings-by-samples__1.0.0.yaml/settings-by-samples/settings/settings/create_fastq_for_index_reads" + }, + { + "label": "mask short reads", + "doc": "The minimum read length\ncontaining A, T, C, G values after\nadapter trimming. Reads with\nless than this number of bases\nbecome completely masked. If\nthis value is less than 22, the\ndefault becomes the\nMinimumTrimmedReadLength.\n", + "type": [ + "null", + "int" + ], + "name": "#settings-by-samples__1.0.0.yaml/settings-by-samples/settings/settings/mask_short_reads" + }, + { + "label": "minumum adapter overlap", + "doc": "Do not trim any bases unless the\nadapter matches are greater than\nor equal to the user specified\nnumber of bases. At least one\nAdapterRead1 or\nAdapterRead2 must be specified\nto use\nMinimumAdapterOverlap.\nAllowed characters: 1, 2, 3.\n", + "type": [ + "null", + "int" + ], + "name": "#settings-by-samples__1.0.0.yaml/settings-by-samples/settings/settings/minimum_adapter_overlap" + }, + { + "label": "minimum trimmed read length", + "doc": "The minimum read length after\nadapter trimming. The software\ntrims adapter sequences from\nreads to the value of this\nparameter. Bases below the\nspecified value are masked with\nN.\n", + "type": [ + "null", + "int" + ], + "name": "#settings-by-samples__1.0.0.yaml/settings-by-samples/settings/settings/minimum_trimmed_read_length" + }, + { + "label": "override cycles", + "doc": "Specifies the sequencing and\nindexing cycles that should be\nused when processing the data.\nThe following format must be\nused:\n* Must be same number of\nsemicolon delimited fields in\nstring as sequencing and\nindexing reads specified in\nRunInfo.xml\n* Indexing reads are specified\nwith an I.\n* Sequencing reads are specified\nwith a Y. UMI cycles are\nspecified with an U.\n* Trimmed reads are specified\nwith N.\n* The number of cycles specified\nfor each read must sum to the\nnumber of cycles specified for\nthat read in the RunInfo.xml.\n* Only one Y or I sequence can\nbe specified per read.\nExample: Y151;I8;I8;Y151\n", + "type": [ + "null", + "string" + ], + "name": "#settings-by-samples__1.0.0.yaml/settings-by-samples/settings/settings/override_cycles" + }, + { + "label": "trim umi", + "doc": "Specifies whether UMI cycles\nwill be excluded from fastq files.\nAt least one UMI is required to\nbe specified in the Sample\nSheet when this setting is\nprovided.\n\u2022 0\u2014UMI cycles will be output to\nfastq files.\n\u2022 1\u2014 UMI cycles will not be\noutput to fastq files.\n", + "type": [ + "null", + "boolean" + ], + "name": "#settings-by-samples__1.0.0.yaml/settings-by-samples/settings/settings/trim_umi" + } + ] + } + ], + "name": "#settings-by-samples__1.0.0.yaml/settings-by-samples/settings" + } + ], + "id": "#settings-by-samples__1.0.0.yaml" + } + ], + "class": "SchemaDefRequirement" + } + ], + "baseCommand": [ + "python3", + "samplesheet-by-settings.py" + ], + "inputs": [ + { + "label": "ignore missing samples", + "doc": "Don't raise an error if samples from the override cycles list are missing. Just remove them\n", + "type": [ + "null", + "boolean" + ], + "inputBinding": { + "prefix": "--ignore-missing-samples" + }, + "id": "#custom-samplesheet-split-by-settings__1.0.0.cwl/ignore_missing_samples" + }, + { + "label": "out dir", + "doc": "Where to place the output samplesheet csv files\n", + "type": [ + "null", + "string" + ], + "inputBinding": { + "prefix": "--out-dir" + }, + "default": "samplesheets-by-override-cycles", + "id": "#custom-samplesheet-split-by-settings__1.0.0.cwl/out_dir" + }, + { + "label": "samplesheet csv", + "doc": "The path to the original samplesheet csv file\n", + "type": "File", + "inputBinding": { + "prefix": "--samplesheet-csv" + }, + "id": "#custom-samplesheet-split-by-settings__1.0.0.cwl/samplesheet_csv" + }, + { + "label": "samplesheet format", + "type": [ + "null", + { + "type": "enum", + "symbols": [ + "#custom-samplesheet-split-by-settings__1.0.0.cwl/samplesheet_format/v1", + "#custom-samplesheet-split-by-settings__1.0.0.cwl/samplesheet_format/v2" + ] + } + ], + "doc": "Set samplesheet to be in v1 or v2 format\n", + "inputBinding": { + "prefix": "--samplesheet-format" + }, + "id": "#custom-samplesheet-split-by-settings__1.0.0.cwl/samplesheet_format" + }, + { + "label": "settings by samples", + "doc": "Takes in an object form of settings by samples. This is used to split samplesheets\n", + "type": [ + "null", + { + "type": "array", + "items": "#settings-by-samples__1.0.0.yaml/settings-by-samples", + "inputBinding": { + "prefix": "--settings-by-samples=", + "separate": false, + "valueFrom": "${\n /*\n Format is {\"batch_name\": \"WGS\", \"sample_ids\":[\"S1\", \"S2\", \"S3\"], \"settings\":{\"adapter_read_1\":\"foo\", \"setting_2\":\"bar\"}}\n Although BCLConvert settings are in camel-case, we have settings in lower case, with underscore separation instead\n Settings are translated to camel case in the workflow. adapter_read_1 becomes AdapterRead1\n */\n return JSON.stringify(self);\n}\n" + } + } + ], + "inputBinding": { + "position": 1 + }, + "id": "#custom-samplesheet-split-by-settings__1.0.0.cwl/settings_by_samples" + } + ], + "outputs": [ + { + "label": "samplesheets outdir", + "doc": "Directory of samplesheets\n", + "type": "Directory", + "outputBinding": { + "glob": "$(inputs.out_dir)" + }, + "id": "#custom-samplesheet-split-by-settings__1.0.0.cwl/samplesheet_outdir" + }, + { + "label": "output samplesheets", + "doc": "List of output samplesheets\n", + "type": { + "type": "array", + "items": "File" + }, + "outputBinding": { + "glob": "$(inputs.out_dir)/*.csv" + }, + "id": "#custom-samplesheet-split-by-settings__1.0.0.cwl/samplesheets" + } + ], + "successCodes": [ + 0 + ], + "https://schema.org/author": { + "class": "https://schema.org/Person", + "https://schema.org/name": "Sehrish Kanwal", + "https://schema.org/email": "sehrish.kanwal@umccr.org" + } + }, + { + "class": "CommandLineTool", + "id": "#custom-touch-file__1.0.0.cwl", + "label": "custom-create-dummy-file v(1.0.0)", + "doc": "Documentation for custom-create-dummy-file v1.0.0\n", + "hints": [ + { + "dockerPull": "alpine:latest", + "class": "DockerRequirement" + }, + { + "coresMin": 1, + "ramMin": 1000, + "class": "ResourceRequirement", + "http://platform.illumina.com/rdf/ica/resources": { + "tier": "standard", + "type": "standard", + "size": "small" + } + } + ], + "requirements": [ + { + "class": "InlineJavascriptRequirement" + } + ], + "baseCommand": [ + "touch" + ], + "inputs": [ + { + "label": "file name", + "doc": "Name of the file to create\n", + "type": "string", + "default": "dummy_file.txt", + "inputBinding": { + "position": 1 + }, + "id": "#custom-touch-file__1.0.0.cwl/file_name" + } + ], + "outputs": [ + { + "label": "dummy file", + "doc": "Output dummy file\n", + "type": "File", + "outputBinding": { + "glob": "$(inputs.file_name)" + }, + "id": "#custom-touch-file__1.0.0.cwl/dummy_file_output" + } + ], + "successCodes": [ + 0 + ], + "https://schema.org/author": { + "class": "https://schema.org/Person", + "https://schema.org/name": "Alexis Lucattini", + "https://schema.org/email": "Alexis.Lucattini@umccr.org", + "https://schema.org/identifier": "https://orcid.org/0000-0001-9754-647X" + } + }, + { + "class": "CommandLineTool", + "id": "#multiqc-interop__1.2.1.cwl", + "label": "multiqc-interop v(1.2.1)", + "doc": "Producing QC report using interop matrix\n", + "hints": [ + { + "dockerPull": "umccr/multiqc_dragen:1.2.1", + "class": "DockerRequirement" + }, + { + "coresMin": 1, + "ramMin": 4000, + "class": "ResourceRequirement", + "http://platform.illumina.com/rdf/ica/resources": { + "type": "standard", + "size": "medium" + } + } + ], + "requirements": [ + { + "listing": [ + { + "entryname": "generate_interop_files.sh", + "entry": "#!/usr/bin/env bash\n\n# Fail on non-zero exit of subshell\nset -euo pipefail\n\n# Generate interop files\ninterop_summary --csv=1 \"$(inputs.input_directory.path)\" > interop_summary.csv\ninterop_index-summary --csv=1 \"$(inputs.input_directory.path)\" > interop_index-summary.csv\n" + }, + { + "entryname": "run_multiqc_interop.sh", + "entry": "#!/usr/bin/env bash\n\n# Fail on non-zero exit of subshell\nset -euo pipefail\n\n# multiqc interop module needs to run a series of commands \n# ref: https://github.com/umccr-illumina/stratus/blob/806c76609af4755159b12cf5302d4e4e11cc614b/TES/multiqc.json\necho \"Generating interop files\" 1>&2\nbash generate_interop_files.sh\n\n# Now run multiqc\necho \"Running multiqc\" 1>&2\neval multiqc --module interop '\"\\${@}\"' interop_summary.csv interop_index-summary.csv\n" + } + ], + "class": "InitialWorkDirRequirement" + }, + { + "class": "InlineJavascriptRequirement" + } + ], + "baseCommand": [ + "bash", + "run_multiqc_interop.sh" + ], + "inputs": [ + { + "label": "dummy file", + "doc": "testing inputs stream logic\nIf used will set input mode to stream on ICA which\nsaves having to download the entire input folder\n", + "type": [ + "null", + "File" + ], + "streamable": true, + "id": "#multiqc-interop__1.2.1.cwl/dummy_file" + }, + { + "label": "input directory", + "doc": "The bcl directory\n", + "type": "Directory", + "inputBinding": { + "position": 100 + }, + "id": "#multiqc-interop__1.2.1.cwl/input_directory" + }, + { + "label": "output directory", + "doc": "The output directory, defaults to \"multiqc-outdir\"\n", + "type": [ + "null", + "string" + ], + "default": "multiqc-outdir", + "inputBinding": { + "prefix": "--outdir" + }, + "id": "#multiqc-interop__1.2.1.cwl/output_directory_name" + }, + { + "label": "output filename", + "doc": "Report filename in html format.\nDefaults to 'multiqc-report.html'\n", + "type": [ + "null", + "string" + ], + "default": "multiqc-report.html", + "inputBinding": { + "prefix": "--filename" + }, + "id": "#multiqc-interop__1.2.1.cwl/output_filename" + }, + { + "label": "title", + "doc": "Report title.\nPrinted as page header, used for filename if not otherwise specified.\n", + "type": "string", + "inputBinding": { + "prefix": "--title" + }, + "id": "#multiqc-interop__1.2.1.cwl/title" + } + ], + "outputs": [ + { + "label": "multiqc output", + "doc": "output dircetory with interop multiQC matrices\n", + "type": "Directory", + "outputBinding": { + "glob": "$(inputs.output_directory_name)" + }, + "id": "#multiqc-interop__1.2.1.cwl/interop_multi_qc_out" + } + ], + "successCodes": [ + 0 + ], + "https://schema.org/author": { + "class": "https://schema.org/Person", + "https://schema.org/name": "Sehrish Kanwal", + "https://schema.org/email": "sehrish.kanwal@umccr.org" + } + }, + { + "class": "CommandLineTool", + "id": "#multiqc__1.11.0.cwl", + "label": "multiqc v(1.11.0)", + "doc": "Documentation for multiqc v1.11.0\n", + "hints": [ + { + "dockerPull": "quay.io/biocontainers/multiqc:1.11--pyhdfd78af_0", + "class": "DockerRequirement" + }, + { + "coresMin": 2, + "ramMin": 4000, + "class": "ResourceRequirement", + "http://platform.illumina.com/rdf/ica/resources": { + "tier": "standard", + "type": "standard", + "size": "xlarge" + } + } + ], + "requirements": [ + { + "listing": [ + { + "entryname": "run_multiqc.sh", + "entry": "#!/usr/bin/env bash\n\n# Set up to fail\nset -euo pipefail\n\n# Create input dir\nmkdir \"$(get_input_dir())\"\n\n# Create an array of dirs\ninput_dir_path_array=( $(inputs.input_directories.map(function(a) {return '\"' + a.path + '\"';}).join(' ')) )\ninput_dir_basename_array=( $(inputs.input_directories.map(function(a) {return '\"' + a.basename + '\"';}).join(' ')) )\n\n# Iterate through input direcotires\nfor input_dir_path in \"\\${input_dir_path_array[@]}\"; do\n ln -s \"\\${input_dir_path}\" \"$(get_input_dir())/\"\ndone\n\n# Run multiqc\neval multiqc '\"\\${@}\"'\n\n# Unlink input directories - otherwise ICA tries to upload them onto gds (and fails)\nfor input_dir_basename in \"\\${input_dir_basename_array[@]}\"; do\n unlink \"$(get_input_dir())/\\${input_dir_basename}\"\ndone\n" + } + ], + "class": "InitialWorkDirRequirement" + }, + { + "expressionLib": [ + "var get_input_dir = function(){ /* Just returns the name of the input directory */ return \"multiqc_input_dir\"; }" + ], + "class": "InlineJavascriptRequirement" + } + ], + "baseCommand": [ + "bash", + "run_multiqc.sh" + ], + "arguments": [ + { + "position": 100, + "valueFrom": "$(get_input_dir())" + } + ], + "inputs": [ + { + "label": "cl config", + "doc": "Override config from the cli\n", + "type": [ + "null", + "string" + ], + "inputBinding": { + "prefix": "--cl_config" + }, + "id": "#multiqc__1.11.0.cwl/cl_config" + }, + { + "label": "comment", + "doc": "Custom comment, will be printed at the top of the report.\n", + "type": [ + "null", + "string" + ], + "inputBinding": { + "prefix": "--comment" + }, + "id": "#multiqc__1.11.0.cwl/comment" + }, + { + "label": "config", + "doc": "Configuration file for bclconvert\n", + "type": [ + "null", + "File" + ], + "streamable": true, + "inputBinding": { + "prefix": "--config" + }, + "id": "#multiqc__1.11.0.cwl/config" + }, + { + "label": "dummy file", + "doc": "testing inputs stream logic\nIf used will set input mode to stream on ICA which\nsaves having to download the entire input folder\n", + "type": [ + "null", + "File" + ], + "streamable": true, + "id": "#multiqc__1.11.0.cwl/dummy_file" + }, + { + "label": "input directories", + "doc": "The list of directories to place in the analysis\n", + "type": { + "type": "array", + "items": "Directory" + }, + "id": "#multiqc__1.11.0.cwl/input_directories" + }, + { + "label": "output directory", + "doc": "The output directory\n", + "type": "string", + "inputBinding": { + "prefix": "--outdir", + "valueFrom": "$(runtime.outdir)/$(self)" + }, + "id": "#multiqc__1.11.0.cwl/output_directory_name" + }, + { + "label": "output filename", + "doc": "Report filename in html format.\nDefaults to 'multiqc-report.html\"\n", + "type": "string", + "inputBinding": { + "prefix": "--filename" + }, + "id": "#multiqc__1.11.0.cwl/output_filename" + }, + { + "label": "title", + "doc": "Report title.\nPrinted as page header, used for filename if not otherwise specified.\n", + "type": "string", + "inputBinding": { + "prefix": "--title" + }, + "id": "#multiqc__1.11.0.cwl/title" + } + ], + "outputs": [ + { + "label": "output directory", + "doc": "Directory that contains all multiqc analysis data\n", + "type": "Directory", + "outputBinding": { + "glob": "$(inputs.output_directory_name)" + }, + "id": "#multiqc__1.11.0.cwl/output_directory" + }, + { + "label": "output file", + "doc": "Output html file\n", + "type": "File", + "outputBinding": { + "glob": "$(inputs.output_directory_name)/$(inputs.output_filename)" + }, + "id": "#multiqc__1.11.0.cwl/output_file" + } + ], + "successCodes": [ + 0 + ], + "https://schema.org/author": { + "class": "https://schema.org/Person", + "https://schema.org/name": "Alexis Lucattini", + "https://schema.org/email": "Alexis.Lucattini@umccr.org", + "https://schema.org/identifier": "https://orcid.org/0000-0001-9754-647X" + } + }, + { + "class": "Workflow", + "id": "#main", + "label": "bcl-conversion v(3.7.5)", + "doc": "Runs bcl-convert v3.7.5 with multiqc output of the bcl input directory\n", + "requirements": [ + { + "class": "InlineJavascriptRequirement" + }, + { + "class": "MultipleInputFeatureRequirement" + }, + { + "class": "ScatterFeatureRequirement" + }, + { + "types": [ + { + "$import": "#settings-by-samples__1.0.0.yaml" + }, + { + "$import": "#fastq-list-row__1.0.0.yaml" + } + ], + "class": "SchemaDefRequirement" + }, + { + "class": "StepInputExpressionRequirement" + } + ], + "inputs": [ + { + "label": "bcl input directory", + "doc": "Path to the bcl files\n", + "type": "Directory", + "id": "#bcl_input_directory" + }, + { + "label": "bcl only lane", + "doc": "Convert only the specified lane number. The value must\nbe less than or equal to the number of lanes specified in the\nRunInfo.xml. Must be a single integer value.\n", + "type": [ + "null", + "int" + ], + "id": "#bcl_only_lane_bcl_conversion" + }, + { + "label": "bcl sampleproject subdirectories", + "doc": "true \u2014 Allows creation of Sample_Project subdirectories\nas specified in the sample sheet. This option must be set to true for\nthe Sample_Project column in the data section to be used.\n", + "type": [ + "null", + "boolean" + ], + "id": "#bcl_sampleproject_subdirectories_bcl_conversion" + }, + { + "label": "delete undetermined indices", + "doc": "Delete undetermined indices on completion of the run\n", + "type": [ + "null", + "boolean" + ], + "default": true, + "id": "#delete_undetermined_indices_bcl_conversion" + }, + { + "label": "first tile only", + "doc": "true \u2014 Only process the first tile of the first swath of the\n top surface of each lane specified in the sample sheet.\nfalse \u2014 Process all tiles in each lane, as specified in the sample\n sheet.\n", + "type": [ + "null", + "boolean" + ], + "id": "#first_tile_only_bcl_conversion" + }, + { + "label": "ignore missing samples", + "doc": "Remove the samples not present in the override cycles record\n", + "type": [ + "null", + "boolean" + ], + "default": true, + "id": "#ignore_missing_samples" + }, + { + "label": "runfolder name", + "doc": "Required - used in naming run specific folder, reports and headings\n", + "type": "string", + "id": "#runfolder_name" + }, + { + "label": "sample sheet", + "doc": "The path to the full samplesheet\n", + "type": "File", + "id": "#samplesheet" + }, + { + "label": "samplesheet outdir", + "doc": "Output directory of the samplesheets split by settings\n", + "type": [ + "null", + "string" + ], + "id": "#samplesheet_outdir" + }, + { + "label": "set samplesheet output format", + "doc": "Convert headers to v2 samplesheet format\n", + "type": [ + "null", + { + "type": "enum", + "symbols": [ + "#/samplesheet_output_format/v1", + "#/samplesheet_output_format/v2" + ] + } + ], + "id": "#samplesheet_output_format" + }, + { + "label": "settings by samples", + "doc": "List of settings by samples\n", + "type": [ + "null", + { + "type": "array", + "items": "#settings-by-samples__1.0.0.yaml/settings-by-samples" + } + ], + "id": "#settings_by_samples" + }, + { + "label": "strict mode bcl conversion", + "doc": "true \u2014 Abort the program if any filter, locs, bcl, or bci lane\nfiles are missing or corrupt.\nfalse \u2014 Continue processing if any filter, locs, bcl, or bci lane files\nare missing. Return a warning message for each missing or corrupt\nfile.\n", + "type": [ + "null", + "boolean" + ], + "id": "#strict_mode_bcl_conversion" + } + ], + "steps": [ + { + "label": "bcl convert", + "doc": "BCLConvert is then scattered across each of the samplesheets.\n", + "scatter": [ + "#bcl_convert_step/samplesheet", + "#bcl_convert_step/output_directory" + ], + "scatterMethod": "dotproduct", + "in": [ + { + "source": "#bcl_input_directory", + "id": "#bcl_convert_step/bcl_input_directory" + }, + { + "source": "#bcl_only_lane_bcl_conversion", + "id": "#bcl_convert_step/bcl_only_lane" + }, + { + "source": "#bcl_sampleproject_subdirectories_bcl_conversion", + "id": "#bcl_convert_step/bcl_sampleproject_subdirectories" + }, + { + "source": "#delete_undetermined_indices_bcl_conversion", + "id": "#bcl_convert_step/delete_undetermined_indices" + }, + { + "source": "#first_tile_only_bcl_conversion", + "id": "#bcl_convert_step/first_tile_only" + }, + { + "source": "#get_batch_dirs/batch_names", + "id": "#bcl_convert_step/output_directory" + }, + { + "source": "#samplesheet_split_by_settings_step/samplesheets", + "id": "#bcl_convert_step/samplesheet" + }, + { + "source": "#strict_mode_bcl_conversion", + "id": "#bcl_convert_step/strict_mode" + } + ], + "out": [ + { + "id": "#bcl_convert_step/bcl_convert_directory_output" + }, + { + "id": "#bcl_convert_step/fastq_list_rows" + } + ], + "run": "#bclConvert__3.7.5.cwl", + "id": "#bcl_convert_step" + }, + { + "label": "bclconvert qc step", + "doc": "The bclconvert qc step - from scatter this takes in an array of dirs\n", + "in": [ + { + "valueFrom": "${\n return JSON.stringify({\"bclconvert\": { \"genome_size\": \"hg38_genome\" }});\n }\n", + "id": "#bclconvert_qc_step/cl_config" + }, + { + "source": "#create_dummy_file_step/dummy_file_output", + "id": "#bclconvert_qc_step/dummy_file" + }, + { + "source": "#bcl_convert_step/bcl_convert_directory_output", + "id": "#bclconvert_qc_step/input_directories" + }, + { + "source": "#runfolder_name", + "valueFrom": "$(self)_bclconvert_multiqc", + "id": "#bclconvert_qc_step/output_directory_name" + }, + { + "source": "#runfolder_name", + "valueFrom": "$(self)_bclconvert_multiqc.html", + "id": "#bclconvert_qc_step/output_filename" + }, + { + "source": "#runfolder_name", + "valueFrom": "UMCCR MultiQC BCLConvert report for $(self)", + "id": "#bclconvert_qc_step/title" + } + ], + "out": [ + { + "id": "#bclconvert_qc_step/output_directory" + } + ], + "run": "#multiqc__1.11.0.cwl", + "id": "#bclconvert_qc_step" + }, + { + "label": "create dummy file", + "doc": "Intermediate step for letting multiqc-interop be placed in stream mode\n", + "in": [], + "out": [ + { + "id": "#create_dummy_file_step/dummy_file_output" + } + ], + "run": "#custom-touch-file__1.0.0.cwl", + "id": "#create_dummy_file_step" + }, + { + "label": "flatten fastq list rows array", + "doc": "fastq list rows is an array and bcl convert is from a directory output.\nThis scatters the arrays to a single array\n", + "in": [ + { + "source": "#bcl_convert_step/fastq_list_rows", + "id": "#flatten_fastq_list_rows_array/arrayTwoDim" + } + ], + "out": [ + { + "id": "#flatten_fastq_list_rows_array/array1d" + } + ], + "run": "#flatten-array-fastq-list__1.0.0.cwl", + "id": "#flatten_fastq_list_rows_array" + }, + { + "label": "get batch directories", + "doc": "Get the directory names of each of the directories we wish to scatter over\n", + "in": [ + { + "source": "#samplesheet_split_by_settings_step/samplesheets", + "id": "#get_batch_dirs/samplesheets" + } + ], + "out": [ + { + "id": "#get_batch_dirs/batch_names" + } + ], + "run": "#get-samplesheet-midfix-regex__1.0.0.cwl", + "id": "#get_batch_dirs" + }, + { + "label": "interop qc step", + "doc": "Run the multiqc by first also generating the interop files for use\n", + "in": [ + { + "source": "#create_dummy_file_step/dummy_file_output", + "id": "#interop_qc_step/dummy_file" + }, + { + "source": "#bcl_input_directory", + "id": "#interop_qc_step/input_directory" + }, + { + "source": "#runfolder_name", + "valueFrom": "$(self)_interop_multiqc", + "id": "#interop_qc_step/output_directory_name" + }, + { + "source": "#runfolder_name", + "valueFrom": "$(self)_interop_multiqc.html", + "id": "#interop_qc_step/output_filename" + }, + { + "source": "#runfolder_name", + "valueFrom": "UMCCR MultiQC Interop report for $(self)", + "id": "#interop_qc_step/title" + } + ], + "out": [ + { + "id": "#interop_qc_step/interop_multi_qc_out" + } + ], + "run": "#multiqc-interop__1.2.1.cwl", + "id": "#interop_qc_step" + }, + { + "label": "Split samplesheet by settings step", + "doc": "Samplesheet is split by the different input types.\nThese are generally a difference in override cycles parameters or adapter trimming settings\nThis then scatters multiple bclconvert workflows split by sample id\n", + "in": [ + { + "source": "#ignore_missing_samples", + "id": "#samplesheet_split_by_settings_step/ignore_missing_samples" + }, + { + "source": "#samplesheet_outdir", + "id": "#samplesheet_split_by_settings_step/out_dir" + }, + { + "source": "#samplesheet", + "id": "#samplesheet_split_by_settings_step/samplesheet_csv" + }, + { + "source": "#samplesheet_output_format", + "id": "#samplesheet_split_by_settings_step/samplesheet_format" + }, + { + "source": "#settings_by_samples", + "id": "#samplesheet_split_by_settings_step/settings_by_samples" + } + ], + "out": [ + { + "id": "#samplesheet_split_by_settings_step/samplesheets" + }, + { + "id": "#samplesheet_split_by_settings_step/samplesheet_outdir" + } + ], + "run": "#custom-samplesheet-split-by-settings__1.0.0.cwl", + "id": "#samplesheet_split_by_settings_step" + } + ], + "outputs": [ + { + "label": "bclconvert multiqc", + "doc": "multiqc directory output that contains bclconvert multiqc data\n", + "type": "Directory", + "outputSource": "#bclconvert_qc_step/output_directory", + "id": "#bclconvert_multiqc_out" + }, + { + "label": "Output fastq directores", + "doc": "The outputs from the bclconvert-step\n", + "type": { + "type": "array", + "items": "Directory" + }, + "outputSource": "#bcl_convert_step/bcl_convert_directory_output", + "id": "#fastq_directories" + }, + { + "label": "rows of fastq list csv file", + "doc": "Contains the fastq list row schema for each of the output fastq files\n", + "type": { + "type": "array", + "items": "#fastq-list-row__1.0.0.yaml/fastq-list-row" + }, + "outputSource": "#flatten_fastq_list_rows_array/array1d", + "id": "#fastq_list_rows" + }, + { + "label": "interop multiqc", + "doc": "multiqc directory output that contains interop data\n", + "type": "Directory", + "outputSource": "#interop_qc_step/interop_multi_qc_out", + "id": "#interop_multiqc_out" + }, + { + "label": "split samplesheets", + "doc": "List of samplesheets split by override cycles\n", + "type": { + "type": "array", + "items": "File" + }, + "outputSource": "#samplesheet_split_by_settings_step/samplesheets", + "id": "#split_sheets" + }, + { + "label": "split sheets dir", + "doc": "The directory containing the samplesheets used for each bcl convert\n", + "type": "Directory", + "outputSource": "#samplesheet_split_by_settings_step/samplesheet_outdir", + "id": "#split_sheets_dir" + } + ], + "https://schema.org/author": { + "class": "https://schema.org/Person", + "https://schema.org/name": "Sehrish Kanwal", + "https://schema.org/email": "sehrish.kanwal@umccr.org" + } + } + ], + "cwlVersion": "v1.1", + "$schemas": [ + "https://schema.org/version/latest/schemaorg-current-http.rdf" + ] +} From cef7385f242273060a584ebdac8334a8cb8c5c59 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 26 Jul 2021 12:26:43 +0000 Subject: [PATCH 073/111] Update rdflib requirement from <5.1,>=4.2.2 to >=4.2.2,<6.1 (#1474) Updates the requirements on [rdflib](https://github.com/RDFLib/rdflib) to permit the latest version. - [Release notes](https://github.com/RDFLib/rdflib/releases) - [Changelog](https://github.com/RDFLib/rdflib/blob/master/CHANGELOG.md) - [Commits](https://github.com/RDFLib/rdflib/compare/4.2.2...6.0.0) --- updated-dependencies: - dependency-name: rdflib dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements.txt | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 977cb6259d..05b59996b3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ requests>=2.4.3 ruamel.yaml>=0.15,<0.17.11 -rdflib>=4.2.2,<5.1 +rdflib>=4.2.2,<6.1 shellescape>=3.4.1,<3.9 schema-salad>=8.1.20210627200047,<9 prov==1.5.1 diff --git a/setup.py b/setup.py index 6654fe45e2..294a1ccc81 100644 --- a/setup.py +++ b/setup.py @@ -109,7 +109,7 @@ "requests >= 2.6.1", # >= 2.6.1 to workaround # https://github.com/ionrock/cachecontrol/issues/137 "ruamel.yaml >= 0.15, < 0.17.11", - "rdflib >= 4.2.2, < 5.1.0", + "rdflib >= 4.2.2, < 6.1.0", "shellescape >= 3.4.1, < 3.9", # 7.1.20210518142926 or later required due to # https://github.com/common-workflow-language/schema_salad/issues/385 From 0e89055b41fc769af48782b57991039c84673a6f Mon Sep 17 00:00:00 2001 From: Popescu Mihai Date: Mon, 26 Jul 2021 16:20:56 +0300 Subject: [PATCH 074/111] Allow files to be named pipes (#1469) * allow files to be named pipes Co-authored-by: Michael R. Crusoe <1330696+mr-c@users.noreply.github.com> --- cwltool/context.py | 1 + cwltool/job.py | 20 ++++++-- tests/test_streaming.py | 108 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 125 insertions(+), 4 deletions(-) create mode 100644 tests/test_streaming.py diff --git a/cwltool/context.py b/cwltool/context.py index 83c098e12c..daf6aa8676 100644 --- a/cwltool/context.py +++ b/cwltool/context.py @@ -107,6 +107,7 @@ def __init__(self, kwargs: Optional[Dict[str, Any]] = None) -> None: self.pull_image = True # type: bool self.rm_container = True # type: bool self.move_outputs = "move" # type: str + self.streaming_allowed: bool = False self.singularity = False # type: bool self.debug = False # type: bool diff --git a/cwltool/job.py b/cwltool/job.py index f515091ec2..0f9eb131b4 100644 --- a/cwltool/job.py +++ b/cwltool/job.py @@ -3,6 +3,7 @@ import itertools import logging import os +import stat import re import shutil import subprocess # nosec @@ -175,13 +176,24 @@ def _setup(self, runtimeContext: RuntimeContext) -> None: if not os.path.exists(self.outdir): os.makedirs(self.outdir) + def is_streamable(file: str) -> bool: + if not runtimeContext.streaming_allowed: + return False + for inp in self.joborder.values(): + if isinstance(inp, dict) and inp.get("location", None) == file: + return inp.get("streamable", False) + return False + for knownfile in self.pathmapper.files(): p = self.pathmapper.mapper(knownfile) if p.type == "File" and not os.path.isfile(p[0]) and p.staged: - raise WorkflowException( - "Input file %s (at %s) not found or is not a regular " - "file." % (knownfile, self.pathmapper.mapper(knownfile)[0]) - ) + if not ( + is_streamable(knownfile) and stat.S_ISFIFO(os.stat(p[0]).st_mode) + ): + raise WorkflowException( + "Input file %s (at %s) not found or is not a regular " + "file." % (knownfile, self.pathmapper.mapper(knownfile)[0]) + ) if "listing" in self.generatefiles: runtimeContext = runtimeContext.copy() diff --git a/tests/test_streaming.py b/tests/test_streaming.py new file mode 100644 index 0000000000..e3d3ec759a --- /dev/null +++ b/tests/test_streaming.py @@ -0,0 +1,108 @@ +"""Test that files marked as 'streamable' when 'streaming_allowed' can be named pipes.""" +import os + +import pytest +from pathlib import Path +from typing import cast + +from ruamel.yaml.comments import CommentedMap +from schema_salad.sourceline import cmap + +from cwltool.command_line_tool import CommandLineTool +from cwltool.context import LoadingContext, RuntimeContext +from cwltool.errors import WorkflowException +from cwltool.job import JobBase +from cwltool.update import INTERNAL_VERSION +from cwltool.utils import CWLObjectType +from .util import get_data + +toolpath_object = cast( + CommentedMap, + cmap( + { + "cwlVersion": INTERNAL_VERSION, + "class": "CommandLineTool", + "inputs": [ + { + "type": "File", + "id": "inp", + "streamable": True, + } + ], + "outputs": [], + "requirements": [], + } + ), +) + +loading_context = LoadingContext( + { + "metadata": { + "cwlVersion": INTERNAL_VERSION, + "http://commonwl.org/cwltool#original_cwlVersion": INTERNAL_VERSION, + } + } +) + + +def test_regular_file() -> None: + """Test that regular files do not raise any exception when they are checked in job._setup.""" + clt = CommandLineTool( + toolpath_object, + loading_context, + ) + runtime_context = RuntimeContext() + + joborder: CWLObjectType = { + "inp": { + "class": "File", + "location": get_data("tests/wf/whale.txt"), + } + } + + job = next(clt.job(joborder, None, runtime_context)) + assert isinstance(job, JobBase) + + job._setup(runtime_context) + + +streaming = [ + (True, True, False), + (True, False, True), + (False, True, True), + (False, False, True), +] + + +@pytest.mark.parametrize("streamable,streaming_allowed,raise_exception", streaming) +def test_input_can_be_named_pipe( + tmp_path: Path, streamable: bool, streaming_allowed: bool, raise_exception: bool +) -> None: + """Test that input can be a named pipe.""" + clt = CommandLineTool( + toolpath_object, + loading_context, + ) + + runtime_context = RuntimeContext() + runtime_context.streaming_allowed = streaming_allowed + + path = tmp_path / "tmp" + os.mkfifo(path) + + joborder: CWLObjectType = { + "inp": { + "class": "File", + "location": str(path), + "streamable": streamable, + } + } + + job = next(clt.job(joborder, None, runtime_context)) + assert isinstance(job, JobBase) + + if raise_exception: + with pytest.raises(WorkflowException): + job._setup(runtime_context) + else: + job._setup(runtime_context) From 23acadf4b5553ca28d24b77e1116df28d99fc052 Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" <1330696+mr-c@users.noreply.github.com> Date: Mon, 26 Jul 2021 23:14:09 +0200 Subject: [PATCH 075/111] better handle symlinks in Directories (#1462) --- cwltool/process.py | 21 +++++++++++++++------ tests/symlinks.cwl | 14 ++++++++++++++ tests/test_relocate.py | 20 ++++++++++++++++---- 3 files changed, 45 insertions(+), 10 deletions(-) create mode 100644 tests/symlinks.cwl diff --git a/cwltool/process.py b/cwltool/process.py index 39efb95e82..4d0641af37 100644 --- a/cwltool/process.py +++ b/cwltool/process.py @@ -356,19 +356,28 @@ def _relocate(src: str, dst: str) -> None: return # If the source is not contained in source_directories we're not allowed to delete it - src = fs_access.realpath(src) + real_src = fs_access.realpath(src) src_can_deleted = any( - os.path.commonprefix([p, src]) == p for p in source_directories + os.path.commonprefix([p, real_src]) == p for p in source_directories ) _action = "move" if action == "move" and src_can_deleted else "copy" if _action == "move": _logger.debug("Moving %s to %s", src, dst) - if fs_access.isdir(src) and fs_access.isdir(dst): - # merge directories - for dir_entry in scandir(src): - _relocate(dir_entry.path, fs_access.join(dst, dir_entry.name)) + if fs_access.isdir(src): + if fs_access.isdir(dst): + if len(fs_access.listdir(dst)) > 0: + # merge directories + for dir_entry in scandir(src): + _relocate( + dir_entry.path, fs_access.join(dst, dir_entry.name) + ) + else: + os.rmdir(dst) + shutil.move(src, dst) + else: + shutil.move(src, dst) else: shutil.move(src, dst) diff --git a/tests/symlinks.cwl b/tests/symlinks.cwl new file mode 100644 index 0000000000..695092d3c6 --- /dev/null +++ b/tests/symlinks.cwl @@ -0,0 +1,14 @@ +#!/usr/bin/env cwl-runner +cwlVersion: v1.0 +class: CommandLineTool +inputs: [] +baseCommand: [ bash, -c ] +arguments: +# - "mkdir foo; echo 42 > foo/bar; ln -s bar foo/baz" + - "mkdir foo; echo 42 > foo/bar; ln -s $PWD/foo/bar foo/baz" +# - "mkdir foo; ln -s $PWD/foo/bar foo/baz ; echo 42 > foo/bar" +outputs: + result: + type: Directory + outputBinding: + glob: foo diff --git a/tests/test_relocate.py b/tests/test_relocate.py index 1d0ffb56cb..671a125b67 100644 --- a/tests/test_relocate.py +++ b/tests/test_relocate.py @@ -6,10 +6,7 @@ from .util import get_data, needs_docker -if sys.version_info[0] < 3: - from StringIO import StringIO -else: - from io import StringIO +from io import StringIO @needs_docker @@ -18,6 +15,21 @@ def test_for_910() -> None: assert main([get_data("tests/wf/910.cwl")]) == 0 +def test_symlinks_with_absolute_paths(tmp_path: Path) -> None: + """Confirm that absolute paths in Directory types don't cause problems.""" + assert ( + main( + [ + "--debug", + f"--outdir={tmp_path}/result", + f"--tmpdir-prefix={tmp_path}/tmp", + get_data("tests/symlinks.cwl"), + ] + ) + == 0 + ) + + @needs_docker def test_for_conflict_file_names(tmp_path: Path) -> None: stream = StringIO() From 0358a41c9529ab7f6ddd2eed4087e4bb91a3d596 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 28 Jul 2021 11:27:36 +0000 Subject: [PATCH 076/111] remove networkx pin (#1476) --- requirements.txt | 1 - setup.py | 1 - 2 files changed, 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 05b59996b3..ab94f47bb2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,6 @@ rdflib>=4.2.2,<6.1 shellescape>=3.4.1,<3.9 schema-salad>=8.1.20210627200047,<9 prov==1.5.1 -networkx==2.5.1 bagit==1.8.1 mypy-extensions psutil>=5.6.6 diff --git a/setup.py b/setup.py index 294a1ccc81..bf1398f97a 100644 --- a/setup.py +++ b/setup.py @@ -117,7 +117,6 @@ "mypy-extensions", "psutil >= 5.6.6", "prov == 1.5.1", - "networkx < 2.6", "bagit >= 1.6.4", "typing-extensions", "coloredlogs", From 58831093b9e3cb328f436e11ff33ac593b02dace Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" <1330696+mr-c@users.noreply.github.com> Date: Mon, 16 Aug 2021 05:27:30 -0500 Subject: [PATCH 077/111] Revert "better handle symlinks in Directories (#1462)" (#1483) This reverts commit 23acadf4b5553ca28d24b77e1116df28d99fc052. --- cwltool/process.py | 21 ++++++--------------- tests/symlinks.cwl | 14 -------------- tests/test_relocate.py | 20 ++++---------------- 3 files changed, 10 insertions(+), 45 deletions(-) delete mode 100644 tests/symlinks.cwl diff --git a/cwltool/process.py b/cwltool/process.py index 4d0641af37..39efb95e82 100644 --- a/cwltool/process.py +++ b/cwltool/process.py @@ -356,28 +356,19 @@ def _relocate(src: str, dst: str) -> None: return # If the source is not contained in source_directories we're not allowed to delete it - real_src = fs_access.realpath(src) + src = fs_access.realpath(src) src_can_deleted = any( - os.path.commonprefix([p, real_src]) == p for p in source_directories + os.path.commonprefix([p, src]) == p for p in source_directories ) _action = "move" if action == "move" and src_can_deleted else "copy" if _action == "move": _logger.debug("Moving %s to %s", src, dst) - if fs_access.isdir(src): - if fs_access.isdir(dst): - if len(fs_access.listdir(dst)) > 0: - # merge directories - for dir_entry in scandir(src): - _relocate( - dir_entry.path, fs_access.join(dst, dir_entry.name) - ) - else: - os.rmdir(dst) - shutil.move(src, dst) - else: - shutil.move(src, dst) + if fs_access.isdir(src) and fs_access.isdir(dst): + # merge directories + for dir_entry in scandir(src): + _relocate(dir_entry.path, fs_access.join(dst, dir_entry.name)) else: shutil.move(src, dst) diff --git a/tests/symlinks.cwl b/tests/symlinks.cwl deleted file mode 100644 index 695092d3c6..0000000000 --- a/tests/symlinks.cwl +++ /dev/null @@ -1,14 +0,0 @@ -#!/usr/bin/env cwl-runner -cwlVersion: v1.0 -class: CommandLineTool -inputs: [] -baseCommand: [ bash, -c ] -arguments: -# - "mkdir foo; echo 42 > foo/bar; ln -s bar foo/baz" - - "mkdir foo; echo 42 > foo/bar; ln -s $PWD/foo/bar foo/baz" -# - "mkdir foo; ln -s $PWD/foo/bar foo/baz ; echo 42 > foo/bar" -outputs: - result: - type: Directory - outputBinding: - glob: foo diff --git a/tests/test_relocate.py b/tests/test_relocate.py index 671a125b67..1d0ffb56cb 100644 --- a/tests/test_relocate.py +++ b/tests/test_relocate.py @@ -6,7 +6,10 @@ from .util import get_data, needs_docker -from io import StringIO +if sys.version_info[0] < 3: + from StringIO import StringIO +else: + from io import StringIO @needs_docker @@ -15,21 +18,6 @@ def test_for_910() -> None: assert main([get_data("tests/wf/910.cwl")]) == 0 -def test_symlinks_with_absolute_paths(tmp_path: Path) -> None: - """Confirm that absolute paths in Directory types don't cause problems.""" - assert ( - main( - [ - "--debug", - f"--outdir={tmp_path}/result", - f"--tmpdir-prefix={tmp_path}/tmp", - get_data("tests/symlinks.cwl"), - ] - ) - == 0 - ) - - @needs_docker def test_for_conflict_file_names(tmp_path: Path) -> None: stream = StringIO() From fa46c0a0c4c8ab0e5fa9358d959b53451c1a5c70 Mon Sep 17 00:00:00 2001 From: Peter Amstutz Date: Mon, 16 Aug 2021 07:24:28 -0400 Subject: [PATCH 078/111] Check python3 (#1478) --- Makefile | 18 +++++++++++------- release-test.sh | 3 ++- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/Makefile b/Makefile index 8d8438d4f8..24e790c403 100644 --- a/Makefile +++ b/Makefile @@ -66,7 +66,11 @@ dev: install-dep ## dist : create a module package for distribution dist: dist/${MODULE}-$(VERSION).tar.gz -dist/${MODULE}-$(VERSION).tar.gz: $(SOURCES) +check-python3: +# Check that the default python version is python 3 + python --version 2>&1 | grep "Python 3" + +dist/${MODULE}-$(VERSION).tar.gz: check-python3 $(SOURCES) python setup.py sdist bdist_wheel ## docs : make the docs @@ -74,7 +78,7 @@ docs: FORCE cd docs && $(MAKE) html ## clean : clean up all temporary / machine-generated files -clean: FORCE +clean: check-python3 FORCE rm -f ${MODILE}/*.pyc tests/*.pyc python setup.py clean --all || true rm -Rf .coverage @@ -142,11 +146,11 @@ diff-cover.html: coverage.xml diff-cover --compare-branch=main $^ --html-report $@ ## test : run the ${MODULE} test suite -test: $(PYSOURCES) +test: check-python3 $(PYSOURCES) python -m pytest ${PYTEST_EXTRA} ## testcov : run the ${MODULE} test suite and collect coverage -testcov: $(PYSOURCES) +testcov: check-python3 $(PYSOURCES) python -m pytest --cov --cov-config=.coveragerc --cov-report= ${PYTEST_EXTRA} sloccount.sc: $(PYSOURCES) Makefile @@ -162,10 +166,10 @@ list-author-emails: mypy3: mypy mypy: $(filter-out setup.py gittagger.py,$(PYSOURCES)) - if ! test -f $(shell python3 -c 'import ruamel.yaml; import os.path; print(os.path.dirname(ruamel.yaml.__file__))')/py.typed ; \ + if ! test -f $(shell python -c 'import ruamel.yaml; import os.path; print(os.path.dirname(ruamel.yaml.__file__))')/py.typed ; \ then \ rm -Rf typeshed/ruamel/yaml ; \ - ln -s $(shell python3 -c 'import ruamel.yaml; import os.path; print(os.path.dirname(ruamel.yaml.__file__))') \ + ln -s $(shell python -c 'import ruamel.yaml; import os.path; print(os.path.dirname(ruamel.yaml.__file__))') \ typeshed/ruamel/ ; \ fi # if minimally required ruamel.yaml version is 0.15.99 or greater, than the above can be removed MYPYPATH=$$MYPYPATH:typeshed mypy $^ @@ -181,7 +185,7 @@ shellcheck: FORCE pyupgrade: $(PYSOURCES) pyupgrade --exit-zero-even-if-changed --py36-plus $^ -release-test: FORCE +release-test: check-python3 FORCE git diff-index --quiet HEAD -- || ( echo You have uncommited changes, please commit them and try again; false ) ./release-test.sh diff --git a/release-test.sh b/release-test.sh index 72aa5cb6a3..62bbc99168 100755 --- a/release-test.sh +++ b/release-test.sh @@ -19,7 +19,7 @@ fi test_prefix="" run_tests() { local mod_loc - mod_loc=$(pip show ${package} | + mod_loc=$(pip show ${package} | grep ^Location | awk '{print $2}')/${module} "${test_prefix}"bin/py.test "--ignore=${mod_loc}/schemas/" \ --pyargs -x ${module} -n auto --dist=loadfile @@ -42,6 +42,7 @@ then && pip install --force-reinstall -U pip==${pipver} \ && pip install setuptools==${setuptoolsver} wheel pip install -rtest-requirements.txt + pip install -e . make test pip uninstall -y ${package} || true; pip uninstall -y ${package} || true; make install mkdir testenv1/not-${module} From 3cf62d85e746790b811ab333814228beffec4d1c Mon Sep 17 00:00:00 2001 From: John Didion Date: Mon, 16 Aug 2021 17:21:54 -0400 Subject: [PATCH 079/111] bug/1480: make writable workdir actually writable (#1481) Co-authored-by: Michael R. Crusoe --- cwltool/process.py | 2 +- cwltool/utils.py | 31 ++++++++------ tests/test_iwdr.py | 48 ++++++++++++++++++++++ tests/wf/iwdr_permutations_nocontainer.cwl | 43 +++++++++++++++++++ 4 files changed, 111 insertions(+), 13 deletions(-) create mode 100755 tests/wf/iwdr_permutations_nocontainer.cwl diff --git a/cwltool/process.py b/cwltool/process.py index 39efb95e82..88c23c08c0 100644 --- a/cwltool/process.py +++ b/cwltool/process.py @@ -308,7 +308,7 @@ def stage_files( os.makedirs(entry.target) else: shutil.copytree(entry.resolved, entry.target) - ensure_writable(entry.target) + ensure_writable(entry.target, include_root=True) elif entry.type == "CreateFile" or entry.type == "CreateWritableFile": with open(entry.target, "wb") as new: if secret_store is not None: diff --git a/cwltool/utils.py b/cwltool/utils.py index 83257d1cf3..aca19536e6 100644 --- a/cwltool/utils.py +++ b/cwltool/utils.py @@ -367,23 +367,30 @@ def downloadHttpFile(httpurl): return str(f.name) -def ensure_writable(path): # type: (str) -> None +def ensure_writable(path: str, include_root: bool = False) -> None: + """ + Ensure that 'path' is writable. + + If 'path' is a directory, then all files and directories under 'path' are + made writable, recursively. If 'path' is a file or if 'include_root' is + `True`, then 'path' itself is made writable. + """ + + def add_writable_flag(p: str) -> None: + st = os.stat(p) + mode = stat.S_IMODE(st.st_mode) + os.chmod(p, mode | stat.S_IWUSR) + if os.path.isdir(path): + if include_root: + add_writable_flag(path) for root, dirs, files in os.walk(path): for name in files: - j = os.path.join(root, name) - st = os.stat(j) - mode = stat.S_IMODE(st.st_mode) - os.chmod(j, mode | stat.S_IWUSR) + add_writable_flag(os.path.join(root, name)) for name in dirs: - j = os.path.join(root, name) - st = os.stat(j) - mode = stat.S_IMODE(st.st_mode) - os.chmod(j, mode | stat.S_IWUSR) + add_writable_flag(os.path.join(root, name)) else: - st = os.stat(path) - mode = stat.S_IMODE(st.st_mode) - os.chmod(path, mode | stat.S_IWUSR) + add_writable_flag(path) def ensure_non_writable(path): # type: (str) -> None diff --git a/tests/test_iwdr.py b/tests/test_iwdr.py index 22a0670c42..de43a86972 100644 --- a/tests/test_iwdr.py +++ b/tests/test_iwdr.py @@ -1,4 +1,5 @@ from pathlib import Path +from stat import S_IWRITE, S_IWGRP, S_IWOTH from typing import Any from cwltool.factory import Factory @@ -80,6 +81,53 @@ def test_iwdr_permutations(tmp_path_factory: Any) -> None: ) +def test_iwdr_permutations_readonly(tmp_path_factory: Any) -> None: + """Confirm that readonly input files are properly made writable.""" + misc = tmp_path_factory.mktemp("misc") + fifth = tmp_path_factory.mktemp("fifth") + fifth_file = fifth / "bar" + fifth_dir = fifth / "foo" + fifth_file.touch() + fifth_dir.mkdir() + sixth = tmp_path_factory.mktemp("sixth") + first = misc / "first" + first.touch() + second = misc / "second" + second.touch() + outdir = str(tmp_path_factory.mktemp("outdir")) + for entry in [first, second, fifth, sixth, fifth_file, fifth_dir]: + mode = entry.stat().st_mode + ro_mask = 0o777 ^ (S_IWRITE | S_IWGRP | S_IWOTH) + entry.chmod(mode & ro_mask) + assert ( + main( + [ + "--no-container", + "--debug", + "--leave-outputs", + "--outdir", + outdir, + get_data("tests/wf/iwdr_permutations_nocontainer.cwl"), + "--first", + str(first), + "--second", + str(second), + "--fifth", + str(fifth), + "--sixth", + str(sixth), + ] + ) + == 0 + ) + for entry in [first, second, fifth, sixth, fifth_file, fifth_dir]: + try: + mode = entry.stat().st_mode + entry.chmod(mode | S_IWRITE) + except PermissionError: + pass + + @needs_docker def test_iwdr_permutations_inplace(tmp_path_factory: Any) -> None: misc = tmp_path_factory.mktemp("misc") diff --git a/tests/wf/iwdr_permutations_nocontainer.cwl b/tests/wf/iwdr_permutations_nocontainer.cwl new file mode 100755 index 0000000000..0ef2e2646d --- /dev/null +++ b/tests/wf/iwdr_permutations_nocontainer.cwl @@ -0,0 +1,43 @@ +#!/usr/bin/env cwl-runner +class: CommandLineTool +cwlVersion: v1.2 +requirements: + InitialWorkDirRequirement: + listing: + - entry: $(inputs.first) + entryname: first_writable_file + writable: true + - entry: $(inputs.second) + entryname: second_read_only_file + writable: false + - entry: $(inputs.fifth) + entryname: fifth_writable_directory + writable: true + - entry: $(inputs.sixth) + entryname: sixth_read_only_directory + writable: false + - entry: $(inputs.ninth) + entryname: nineth_writable_directory_literal + writable: true +inputs: + first: File + second: File + fifth: Directory + sixth: Directory + ninth: + type: Directory + default: + class: Directory + basename: foo + listing: [] +outputs: + out: + type: Directory + outputBinding: + glob: . +baseCommand: [bash, -c] +arguments: + - | + find . + echo "a" > first_writable_file + touch fifth_writable_directory/c From a293c471d5d8dcb7a451618bddb51927d600f2c2 Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" Date: Wed, 18 Aug 2021 17:25:01 +0200 Subject: [PATCH 080/111] Modernize Dockerfile Use non-MUSL based container image so we can benefit from manylinux1 wheels --- build-cwl-docker.sh | 10 +++++----- cwltool.Dockerfile | 38 +++++++++++++++++++++++++++++--------- 2 files changed, 34 insertions(+), 14 deletions(-) diff --git a/build-cwl-docker.sh b/build-cwl-docker.sh index da4b8860ba..74f18671e7 100755 --- a/build-cwl-docker.sh +++ b/build-cwl-docker.sh @@ -1,10 +1,10 @@ #!/bin/bash -set -e -docker build --file=cwltool.Dockerfile --tag=commonworkflowlanguage/cwltool_module --target module . -docker build --file=cwltool.Dockerfile --tag=commonworkflowlanguage/cwltool . +set -ex +docker build --file=cwltool.Dockerfile --tag=quay.io/commonwl/cwltool_module --target module . +docker build --file=cwltool.Dockerfile --tag=quay.io/commonwl/cwltool . version=$(git describe --tags) if echo "$version" | grep -vq '\-' >& /dev/null ; then - docker tag commonworkflowlanguage/cwltool_module commonworkflowlanguage/cwltool_module:"$version" - docker tag commonworkflowlanguage/cwltool commonworkflowlanguage/cwltool:"$version" + docker tag quay.io/commonwl/cwltool_module quay.io/commonwl/cwltool_module:"$version" + docker tag quay.io/commonwl/cwltool quay.io/commonwl/cwltool:"$version" fi diff --git a/cwltool.Dockerfile b/cwltool.Dockerfile index 490efa0307..7f79fc7375 100644 --- a/cwltool.Dockerfile +++ b/cwltool.Dockerfile @@ -1,6 +1,14 @@ -FROM python:3.8-alpine as builder - -RUN apk add --no-cache git gcc python3-dev libxml2-dev libxslt-dev libc-dev linux-headers +FROM python:3.9-slim as builder + +RUN apt-get update && apt-get install -y \ + gcc \ + git \ + libc-dev \ + libxml2-dev \ + libxslt1-dev \ + python3-dev \ + && rm -rf /var/lib/apt/lists/* +# linux-headers \ WORKDIR /cwltool COPY . . @@ -9,16 +17,28 @@ RUN python setup.py bdist_wheel --dist-dir=/wheels RUN pip wheel -r requirements.txt --wheel-dir=/wheels RUN pip install --no-index --no-warn-script-location --root=/pythonroot/ /wheels/*.whl -FROM python:3.8-alpine as module -LABEL maintainer peter.amstutz@curoverse.com +FROM python:3.9-slim as module +LABEL maintainer peter.amstutz@curii.com -RUN apk add --no-cache docker nodejs graphviz libxml2 libxslt +RUN apt-get update && apt-get install -y \ + docker.io \ + graphviz \ + libxml2 \ + libxslt1.1 \ + nodejs \ + && rm -rf /var/lib/apt/lists/* COPY --from=builder /pythonroot/ / -FROM python:3.8-alpine -LABEL maintainer peter.amstutz@curoverse.com +FROM python:3.9-slim +LABEL maintainer peter.amstutz@curii.com -RUN apk add --no-cache docker nodejs graphviz libxml2 libxslt +RUN apt-get update && apt-get install -y \ + docker.io \ + graphviz \ + libxml2 \ + libxslt1.1 \ + nodejs \ + && rm -rf /var/lib/apt/lists/* COPY --from=builder /pythonroot/ / COPY cwltool-in-docker.sh /cwltool-in-docker.sh From 74994e57dacfdb1fabcfa887d6d8b828d2f58257 Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" Date: Wed, 18 Aug 2021 16:41:36 +0200 Subject: [PATCH 081/111] build and publish containers to quay.io --- .github/workflows/quay-publish.yml | 36 ++++++++++++++++++++++++++++++ README.rst | 5 +++++ 2 files changed, 41 insertions(+) create mode 100644 .github/workflows/quay-publish.yml diff --git a/.github/workflows/quay-publish.yml b/.github/workflows/quay-publish.yml new file mode 100644 index 0000000000..b67d1cbe4e --- /dev/null +++ b/.github/workflows/quay-publish.yml @@ -0,0 +1,36 @@ +name: publish-quay +on: + push: + branches: + - main + tags: + - '*' + workflow_dispatch: {} +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@master + - name: Get image tags + id: image_tags + run: | + echo -n ::set-output name=IMAGE_TAGS::$(git describe --tags) + - name: Build and publish cwltool_module image to Quay + uses: docker/build-push-action@v1 + with: + file: cwltool.Dockerfile + registry: ${{ secrets.REGISTRY_SERVER }} + repository: ${{ secrets.REGISTRY_NAMESPACE }}/cwltool_module + username: ${{ secrets.REGISTRY_USERNAME }} + password: ${{ secrets.REGISTRY_PASSWORD }} + tags: "${{ steps.image_tags.outputs.IMAGE_TAGS }}" + target: module + - name: Build and publish cwltool image to Quay + uses: docker/build-push-action@v1 + with: + file: cwltool.Dockerfile + registry: ${{ secrets.REGISTRY_SERVER }} + repository: ${{ secrets.REGISTRY_NAMESPACE }}/cwltool + username: ${{ secrets.REGISTRY_USERNAME }} + password: ${{ secrets.REGISTRY_PASSWORD }} + tags: "${{ steps.image_tags.outputs.IMAGE_TAGS }}" diff --git a/README.rst b/README.rst index d04cd2d949..b5a0cca5d2 100644 --- a/README.rst +++ b/README.rst @@ -10,6 +10,8 @@ Conda: |Conda Version| |Conda Installs| Debian: |Debian Testing package| |Debian Stable package| +Quay.io (Docker): |Quay.io Container| + .. |Linux Status| image:: https://github.com/common-workflow-language/cwltool/actions/workflows/ci-tests.yml/badge.svg?branch=main :target: https://github.com/common-workflow-language/cwltool/actions/workflows/ci-tests.yml @@ -37,6 +39,9 @@ Debian: |Debian Testing package| |Debian Stable package| .. |Conda Installs| image:: https://anaconda.org/conda-forge/cwltool/badges/downloads.svg :target: https://anaconda.org/conda-forge/cwltool +.. |Quay.io Container| image:: https://quay.io/repository/commonwl/cwltool/status + :target https://quay.io/repository/commonwl/cwltool + This is the reference implementation of the Common Workflow Language. It is intended to be feature complete and provide comprehensive validation of CWL files as well as provide other tools related to working with CWL. From 462377eb0fadad9b9dc2c69eefbb2732b2fbdb22 Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" Date: Wed, 18 Aug 2021 18:03:58 +0200 Subject: [PATCH 082/111] fix typo --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index b5a0cca5d2..18ab09c704 100644 --- a/README.rst +++ b/README.rst @@ -40,7 +40,7 @@ Quay.io (Docker): |Quay.io Container| :target: https://anaconda.org/conda-forge/cwltool .. |Quay.io Container| image:: https://quay.io/repository/commonwl/cwltool/status - :target https://quay.io/repository/commonwl/cwltool + :target: https://quay.io/repository/commonwl/cwltool This is the reference implementation of the Common Workflow Language. It is intended to be feature complete and provide comprehensive validation of CWL From abb5a7fb811be0c575fcf38d5c4b9d4b30b3b4d1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 19 Aug 2021 07:11:46 +0000 Subject: [PATCH 083/111] Update ruamel-yaml requirement from <0.17.11,>=0.15 to >=0.15,<0.17.12 Updates the requirements on [ruamel-yaml](https://sourceforge.net/p/ruamel-yaml/code/ci/default/tree) to permit the latest version. --- updated-dependencies: - dependency-name: ruamel-yaml dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index ab94f47bb2..58a12600e0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ requests>=2.4.3 -ruamel.yaml>=0.15,<0.17.11 +ruamel.yaml>=0.15,<0.17.12 rdflib>=4.2.2,<6.1 shellescape>=3.4.1,<3.9 schema-salad>=8.1.20210627200047,<9 diff --git a/setup.py b/setup.py index bf1398f97a..793fb37419 100644 --- a/setup.py +++ b/setup.py @@ -108,7 +108,7 @@ "setuptools", "requests >= 2.6.1", # >= 2.6.1 to workaround # https://github.com/ionrock/cachecontrol/issues/137 - "ruamel.yaml >= 0.15, < 0.17.11", + "ruamel.yaml >= 0.15, < 0.17.12", "rdflib >= 4.2.2, < 6.1.0", "shellescape >= 3.4.1, < 3.9", # 7.1.20210518142926 or later required due to From b7468c7a6843e3291821d4302522df6bdd04194b Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" <1330696+mr-c@users.noreply.github.com> Date: Thu, 19 Aug 2021 12:28:30 +0200 Subject: [PATCH 084/111] use matching version of the action for the syntax --- .github/workflows/quay-publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/quay-publish.yml b/.github/workflows/quay-publish.yml index b67d1cbe4e..698372d7c9 100644 --- a/.github/workflows/quay-publish.yml +++ b/.github/workflows/quay-publish.yml @@ -16,7 +16,7 @@ jobs: run: | echo -n ::set-output name=IMAGE_TAGS::$(git describe --tags) - name: Build and publish cwltool_module image to Quay - uses: docker/build-push-action@v1 + uses: docker/build-push-action@v2 with: file: cwltool.Dockerfile registry: ${{ secrets.REGISTRY_SERVER }} From 801a317d4b87f967c435b9b53409fae181fbf595 Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" Date: Thu, 19 Aug 2021 12:41:44 +0200 Subject: [PATCH 085/111] better use the @2 version --- .github/dependabot.yml | 5 +++++ .github/workflows/quay-publish.yml | 16 +++++++--------- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index cf7a39fb64..c0a96e68ee 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -9,3 +9,8 @@ updates: directory: "/" # Location of package manifests schedule: interval: "daily" + # Maintain dependencies for GitHub Actions + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "daily" diff --git a/.github/workflows/quay-publish.yml b/.github/workflows/quay-publish.yml index 698372d7c9..85916f2773 100644 --- a/.github/workflows/quay-publish.yml +++ b/.github/workflows/quay-publish.yml @@ -15,22 +15,20 @@ jobs: id: image_tags run: | echo -n ::set-output name=IMAGE_TAGS::$(git describe --tags) - - name: Build and publish cwltool_module image to Quay - uses: docker/build-push-action@v2 + - name: Login to Quay.io + uses: docker/login-action@v1 with: - file: cwltool.Dockerfile registry: ${{ secrets.REGISTRY_SERVER }} - repository: ${{ secrets.REGISTRY_NAMESPACE }}/cwltool_module username: ${{ secrets.REGISTRY_USERNAME }} password: ${{ secrets.REGISTRY_PASSWORD }} + - name: Build and publish cwltool_module image to Quay + uses: docker/build-push-action@v2 + with: + file: cwltool.Dockerfile tags: "${{ steps.image_tags.outputs.IMAGE_TAGS }}" target: module - name: Build and publish cwltool image to Quay - uses: docker/build-push-action@v1 + uses: docker/build-push-action@v2 with: file: cwltool.Dockerfile - registry: ${{ secrets.REGISTRY_SERVER }} - repository: ${{ secrets.REGISTRY_NAMESPACE }}/cwltool - username: ${{ secrets.REGISTRY_USERNAME }} - password: ${{ secrets.REGISTRY_PASSWORD }} tags: "${{ steps.image_tags.outputs.IMAGE_TAGS }}" From 8ef2512f7e6b23516ac144a50e47d5c7b400cb68 Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" Date: Thu, 19 Aug 2021 12:46:32 +0200 Subject: [PATCH 086/111] return to alpine for the Dockerfile --- cwltool.Dockerfile | 38 +++++++++----------------------------- 1 file changed, 9 insertions(+), 29 deletions(-) diff --git a/cwltool.Dockerfile b/cwltool.Dockerfile index 7f79fc7375..e58c5141c6 100644 --- a/cwltool.Dockerfile +++ b/cwltool.Dockerfile @@ -1,14 +1,6 @@ -FROM python:3.9-slim as builder - -RUN apt-get update && apt-get install -y \ - gcc \ - git \ - libc-dev \ - libxml2-dev \ - libxslt1-dev \ - python3-dev \ - && rm -rf /var/lib/apt/lists/* -# linux-headers \ +FROM python:3.9-alpine as builder + +RUN apk add --no-cache git gcc python3-dev libxml2-dev libxslt-dev libc-dev linux-headers WORKDIR /cwltool COPY . . @@ -17,28 +9,16 @@ RUN python setup.py bdist_wheel --dist-dir=/wheels RUN pip wheel -r requirements.txt --wheel-dir=/wheels RUN pip install --no-index --no-warn-script-location --root=/pythonroot/ /wheels/*.whl -FROM python:3.9-slim as module -LABEL maintainer peter.amstutz@curii.com +FROM python:3.9-alpine as module +LABEL maintainer peter.amstutz@curoverse.com -RUN apt-get update && apt-get install -y \ - docker.io \ - graphviz \ - libxml2 \ - libxslt1.1 \ - nodejs \ - && rm -rf /var/lib/apt/lists/* +RUN apk add --no-cache docker nodejs graphviz libxml2 libxslt COPY --from=builder /pythonroot/ / -FROM python:3.9-slim -LABEL maintainer peter.amstutz@curii.com +FROM python:3.9-alpine +LABEL maintainer peter.amstutz@curoverse.com -RUN apt-get update && apt-get install -y \ - docker.io \ - graphviz \ - libxml2 \ - libxslt1.1 \ - nodejs \ - && rm -rf /var/lib/apt/lists/* +RUN apk add --no-cache docker nodejs graphviz libxml2 libxslt COPY --from=builder /pythonroot/ / COPY cwltool-in-docker.sh /cwltool-in-docker.sh From f5e5700b052611e613b535c8c67dbe4d0558f001 Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" Date: Thu, 19 Aug 2021 12:56:13 +0200 Subject: [PATCH 087/111] only build containers on new tags --- .github/workflows/quay-publish.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/quay-publish.yml b/.github/workflows/quay-publish.yml index 85916f2773..223682f2aa 100644 --- a/.github/workflows/quay-publish.yml +++ b/.github/workflows/quay-publish.yml @@ -1,8 +1,6 @@ name: publish-quay on: push: - branches: - - main tags: - '*' workflow_dispatch: {} @@ -14,7 +12,7 @@ jobs: - name: Get image tags id: image_tags run: | - echo -n ::set-output name=IMAGE_TAGS::$(git describe --tags) + echo -n ::set-output name=IMAGE_TAGS::${GITHUB_REF#refs/*/} - name: Login to Quay.io uses: docker/login-action@v1 with: @@ -27,8 +25,10 @@ jobs: file: cwltool.Dockerfile tags: "${{ steps.image_tags.outputs.IMAGE_TAGS }}" target: module + push: true - name: Build and publish cwltool image to Quay uses: docker/build-push-action@v2 with: file: cwltool.Dockerfile tags: "${{ steps.image_tags.outputs.IMAGE_TAGS }}" + push: true From 45be161bc429ccba590380cda05b70215d5b8a6c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 19 Aug 2021 10:57:39 +0000 Subject: [PATCH 088/111] Bump codecov/codecov-action from 1 to 2.0.2 Bumps [codecov/codecov-action](https://github.com/codecov/codecov-action) from 1 to 2.0.2. - [Release notes](https://github.com/codecov/codecov-action/releases) - [Changelog](https://github.com/codecov/codecov-action/blob/master/CHANGELOG.md) - [Commits](https://github.com/codecov/codecov-action/compare/v1...v2.0.2) --- updated-dependencies: - dependency-name: codecov/codecov-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/ci-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index e2aafa2174..b16eb13bf8 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -70,7 +70,7 @@ jobs: - name: Upload coverage to Codecov if: ${{ matrix.step == 'unit' }} - uses: codecov/codecov-action@v1 + uses: codecov/codecov-action@v2.0.2 with: fail_ci_if_error: true From da64403c490d3e5e286a18b00937b9f9ea427693 Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" Date: Thu, 19 Aug 2021 13:14:07 +0200 Subject: [PATCH 089/111] correct image name --- .github/workflows/quay-publish.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/quay-publish.yml b/.github/workflows/quay-publish.yml index 223682f2aa..7536d8980e 100644 --- a/.github/workflows/quay-publish.yml +++ b/.github/workflows/quay-publish.yml @@ -23,12 +23,12 @@ jobs: uses: docker/build-push-action@v2 with: file: cwltool.Dockerfile - tags: "${{ steps.image_tags.outputs.IMAGE_TAGS }}" + tags: quay.io/commonwl/cwltool_module:"${{ steps.image_tags.outputs.IMAGE_TAGS }}" target: module push: true - name: Build and publish cwltool image to Quay uses: docker/build-push-action@v2 with: file: cwltool.Dockerfile - tags: "${{ steps.image_tags.outputs.IMAGE_TAGS }}" + tags: quay.io/commonwl/cwltool:"${{ steps.image_tags.outputs.IMAGE_TAGS }}" push: true From 5356a639e8ef8d72613e9f025c018573e1f80f20 Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" Date: Thu, 19 Aug 2021 13:20:12 +0200 Subject: [PATCH 090/111] codeql: follow best practices --- .github/workflows/codeql-analysis.yml | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 87598c18b0..121452711b 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -1,8 +1,8 @@ name: "Code scanning - action" on: - push: - pull_request: + push: [main] + pull_request: [main] schedule: - cron: '0 10 * * 2' @@ -11,19 +11,14 @@ jobs: runs-on: ubuntu-latest + permissions: + # required for all workflows + security-events: write + steps: - name: Checkout repository uses: actions/checkout@v2 - with: - # We must fetch at least the immediate parents so that if this is - # a pull request then we can checkout the head. - fetch-depth: 2 - # If this run was triggered by a pull request event, then checkout - # the head of the pull request instead of the merge commit. - - run: git checkout HEAD^2 - if: ${{ github.event_name == 'pull_request' }} - # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL uses: github/codeql-action/init@v1 From 330ad306b4d61550960ef241a3d5410c12e5f1ce Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" Date: Thu, 19 Aug 2021 13:20:12 +0200 Subject: [PATCH 091/111] codeql: fix syntax error --- .github/workflows/codeql-analysis.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 121452711b..bc4d680278 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -1,8 +1,10 @@ name: "Code scanning - action" on: - push: [main] - pull_request: [main] + push: + branches: [main] + pull_request: + branches: [main] schedule: - cron: '0 10 * * 2' From 9fc0f2c95d388f1fd768b3dc484347012a15e877 Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" Date: Thu, 19 Aug 2021 13:33:30 +0200 Subject: [PATCH 092/111] quay: remove extra quotes --- .github/workflows/quay-publish.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/quay-publish.yml b/.github/workflows/quay-publish.yml index 7536d8980e..7b9a3692e4 100644 --- a/.github/workflows/quay-publish.yml +++ b/.github/workflows/quay-publish.yml @@ -23,12 +23,12 @@ jobs: uses: docker/build-push-action@v2 with: file: cwltool.Dockerfile - tags: quay.io/commonwl/cwltool_module:"${{ steps.image_tags.outputs.IMAGE_TAGS }}" + tags: quay.io/commonwl/cwltool_module:${{ steps.image_tags.outputs.IMAGE_TAGS }} target: module push: true - name: Build and publish cwltool image to Quay uses: docker/build-push-action@v2 with: file: cwltool.Dockerfile - tags: quay.io/commonwl/cwltool:"${{ steps.image_tags.outputs.IMAGE_TAGS }}" + tags: quay.io/commonwl/cwltool:${{ steps.image_tags.outputs.IMAGE_TAGS }} push: true From b9fca03db868fb3ef433b8e4c601bc656623ad8f Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" Date: Thu, 19 Aug 2021 13:53:40 +0200 Subject: [PATCH 093/111] improve caching and build with mypyc --- .dockerignore | 22 +++++++++++++++++++ .github/workflows/quay-publish.yml | 6 +++++ Makefile | 2 +- ...d-cwl-docker.sh => build-cwltool-docker.sh | 6 +++++ conformance-test.sh | 8 +++---- cwltool.Dockerfile | 17 +++++++++----- 6 files changed, 51 insertions(+), 10 deletions(-) rename build-cwl-docker.sh => build-cwltool-docker.sh (55%) diff --git a/.dockerignore b/.dockerignore index 2d062154a0..fc71c15e3a 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,6 +1,28 @@ +.coverage +coverage.xml .tox/ .eggs/ .vscode/ build/ dist/ +.swp +.mypy_cache/ +.git/ +.pytest_cache/ +*.whl +env*/ +testenv*/ +*.img +*.sif +*.so +.github/ +cwltool/*.so *.Dockerfile +build-cwltool-docker.sh +__pycache__/ +*/__pycache__/ +**/__pycache__/ +*.egg-info/ +*.orig +.dockerignore +cache* diff --git a/.github/workflows/quay-publish.yml b/.github/workflows/quay-publish.yml index 7b9a3692e4..fd9fd1ba8c 100644 --- a/.github/workflows/quay-publish.yml +++ b/.github/workflows/quay-publish.yml @@ -13,6 +13,8 @@ jobs: id: image_tags run: | echo -n ::set-output name=IMAGE_TAGS::${GITHUB_REF#refs/*/} + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 - name: Login to Quay.io uses: docker/login-action@v1 with: @@ -26,9 +28,13 @@ jobs: tags: quay.io/commonwl/cwltool_module:${{ steps.image_tags.outputs.IMAGE_TAGS }} target: module push: true + cache-from: type=gha + cache-to: type=gha,mode=max - name: Build and publish cwltool image to Quay uses: docker/build-push-action@v2 with: file: cwltool.Dockerfile tags: quay.io/commonwl/cwltool:${{ steps.image_tags.outputs.IMAGE_TAGS }} push: true + cache-from: type=gha + cache-to: type=gha,mode=max diff --git a/Makefile b/Makefile index 24e790c403..51312c2d1e 100644 --- a/Makefile +++ b/Makefile @@ -179,7 +179,7 @@ mypyc: $(PYSOURCES) && pytest ${PYTEST_EXTRA} shellcheck: FORCE - shellcheck build-cwl-docker.sh cwl-docker.sh release-test.sh conformance-test.sh \ + shellcheck build-cwltool-docker.sh cwl-docker.sh release-test.sh conformance-test.sh \ cwltool-in-docker.sh pyupgrade: $(PYSOURCES) diff --git a/build-cwl-docker.sh b/build-cwltool-docker.sh similarity index 55% rename from build-cwl-docker.sh rename to build-cwltool-docker.sh index 74f18671e7..403c554903 100755 --- a/build-cwl-docker.sh +++ b/build-cwltool-docker.sh @@ -3,6 +3,12 @@ set -ex docker build --file=cwltool.Dockerfile --tag=quay.io/commonwl/cwltool_module --target module . docker build --file=cwltool.Dockerfile --tag=quay.io/commonwl/cwltool . +docker run -v -it /var/run/docker.sock:/var/run/docker.sock \ + -v /tmp:/tmp \ + -v $PWD:/tmp/cwltool \ + quay.io/commonwl/cwltool_module /bin/sh -c \ + "apk add gcc bash && pip install -r/tmp/cwltool/test-requirements.txt ; pytest -k 'not test_bioconda and not test_double_overwrite and not test_env_filtering' -n auto --dist=loadfile --pyargs cwltool" + version=$(git describe --tags) if echo "$version" | grep -vq '\-' >& /dev/null ; then docker tag quay.io/commonwl/cwltool_module quay.io/commonwl/cwltool_module:"$version" diff --git a/conformance-test.sh b/conformance-test.sh index e455055a2c..f46adeff4c 100755 --- a/conformance-test.sh +++ b/conformance-test.sh @@ -175,9 +175,9 @@ popd || exit deactivate # build new docker container -if [ "$GIT_BRANCH" = "origin/main" ] && [[ "$version" = "v1.0" ]] -then - ./build-cwl-docker.sh || true -fi +# if [ "$GIT_BRANCH" = "origin/main" ] && [[ "$version" = "v1.0" ]] +# then +# ./build-cwltool-docker.sh || true +# fi #docker rm -v $(docker ps -a -f status=exited | sed 's/ */ /g' | cut -d' ' -f1) exit ${CODE} diff --git a/cwltool.Dockerfile b/cwltool.Dockerfile index e58c5141c6..b1478a66a2 100644 --- a/cwltool.Dockerfile +++ b/cwltool.Dockerfile @@ -5,18 +5,25 @@ RUN apk add --no-cache git gcc python3-dev libxml2-dev libxslt-dev libc-dev linu WORKDIR /cwltool COPY . . -RUN python setup.py bdist_wheel --dist-dir=/wheels -RUN pip wheel -r requirements.txt --wheel-dir=/wheels -RUN pip install --no-index --no-warn-script-location --root=/pythonroot/ /wheels/*.whl +RUN pip install toml -rmypy_requirements.txt "$(grep ruamel requirements.txt)" \ + "$(grep schema.salad requirements.txt)" +# schema-salad is needed to be installed (this time as pure Python) for +# cwltool + mypyc +RUN CWLTOOL_USE_MYPYC=1 MYPYPATH=typeshed python setup.py bdist_wheel --dist-dir=/wheels +RUN pip wheel -r requirements.txt galaxy-tool-util --wheel-dir=/wheels +RUN rm /wheels/schema_salad* +RUN SCHEMA_SALAD_USE_MYPYC=1 MYPYPATH=typeshed pip wheel --no-binary :all: $(grep schema.salad requirements.txt) --wheel-dir=/wheels +RUN pip install --force-reinstall --no-index --no-warn-script-location --root=/pythonroot/ /wheels/*.whl +# --force-reinstall to install our new mypyc compiled schema-salad package FROM python:3.9-alpine as module -LABEL maintainer peter.amstutz@curoverse.com +LABEL maintainer peter.amstutz@curri.com RUN apk add --no-cache docker nodejs graphviz libxml2 libxslt COPY --from=builder /pythonroot/ / FROM python:3.9-alpine -LABEL maintainer peter.amstutz@curoverse.com +LABEL maintainer peter.amstutz@curri.com RUN apk add --no-cache docker nodejs graphviz libxml2 libxslt COPY --from=builder /pythonroot/ / From b1bc9cf32c13ef5c78b617a1a75b2264c4aaa308 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 23 Aug 2021 07:12:32 +0000 Subject: [PATCH 094/111] Update ruamel-yaml requirement from <0.17.12,>=0.15 to >=0.15,<0.17.14 Updates the requirements on [ruamel-yaml](https://sourceforge.net/p/ruamel-yaml/code/ci/default/tree) to permit the latest version. --- updated-dependencies: - dependency-name: ruamel-yaml dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 58a12600e0..079fdb9fa2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ requests>=2.4.3 -ruamel.yaml>=0.15,<0.17.12 +ruamel.yaml>=0.15,<0.17.14 rdflib>=4.2.2,<6.1 shellescape>=3.4.1,<3.9 schema-salad>=8.1.20210627200047,<9 diff --git a/setup.py b/setup.py index 793fb37419..f7767eab07 100644 --- a/setup.py +++ b/setup.py @@ -108,7 +108,7 @@ "setuptools", "requests >= 2.6.1", # >= 2.6.1 to workaround # https://github.com/ionrock/cachecontrol/issues/137 - "ruamel.yaml >= 0.15, < 0.17.12", + "ruamel.yaml >= 0.15, < 0.17.14", "rdflib >= 4.2.2, < 6.1.0", "shellescape >= 3.4.1, < 3.9", # 7.1.20210518142926 or later required due to From 3a419f13292168cc347a30a149e5bd23215b4b48 Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" Date: Mon, 23 Aug 2021 17:16:27 +0200 Subject: [PATCH 095/111] allow for spaces in the current working directory --- build-cwltool-docker.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build-cwltool-docker.sh b/build-cwltool-docker.sh index 403c554903..377a0281a5 100755 --- a/build-cwltool-docker.sh +++ b/build-cwltool-docker.sh @@ -5,7 +5,7 @@ docker build --file=cwltool.Dockerfile --tag=quay.io/commonwl/cwltool . docker run -v -it /var/run/docker.sock:/var/run/docker.sock \ -v /tmp:/tmp \ - -v $PWD:/tmp/cwltool \ + -v "$PWD":/tmp/cwltool \ quay.io/commonwl/cwltool_module /bin/sh -c \ "apk add gcc bash && pip install -r/tmp/cwltool/test-requirements.txt ; pytest -k 'not test_bioconda and not test_double_overwrite and not test_env_filtering' -n auto --dist=loadfile --pyargs cwltool" From 3163e8980f4f539c3d49c2d4eded7817eeb586cc Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" Date: Tue, 24 Aug 2021 10:10:07 +0200 Subject: [PATCH 096/111] update lingering reference to galaxy-lib --- cwltool/software_requirements.py | 6 +++--- tests/test_dependencies.py | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/cwltool/software_requirements.py b/cwltool/software_requirements.py index bdd216055a..1928a763c7 100644 --- a/cwltool/software_requirements.py +++ b/cwltool/software_requirements.py @@ -1,7 +1,7 @@ """This module handles resolution of SoftwareRequirement hints. -This is accomplished mainly by adapting cwltool internals to galaxy-lib's -concept of "dependencies". Despite the name, galaxy-lib is a light weight +This is accomplished mainly by adapting cwltool internals to galaxy-tool-util's +concept of "dependencies". Despite the name, galaxy-tool-util is a light weight library that can be used to map SoftwareRequirements in all sorts of ways - Homebrew, Conda, custom scripts, environment modules. We'd be happy to find ways to adapt new packages managers and such as well. @@ -159,5 +159,5 @@ def get_container_from_software_requirements( def ensure_galaxy_lib_available() -> None: if not SOFTWARE_REQUIREMENTS_ENABLED: raise Exception( - "Optional Python library galaxy-lib not available, it is required for this configuration." + "Optional Python library galaxy-tool-util not available, it is required for this configuration." ) diff --git a/tests/test_dependencies.py b/tests/test_dependencies.py index c02c1c40e3..db9366f7bc 100644 --- a/tests/test_dependencies.py +++ b/tests/test_dependencies.py @@ -17,7 +17,7 @@ @needs_docker -@pytest.mark.skipif(not deps, reason="galaxy-lib is not installed") +@pytest.mark.skipif(not deps, reason="galaxy-tool-util is not installed") def test_biocontainers() -> None: wflow = get_data("tests/seqtk_seq.cwl") job = get_data("tests/seqtk_seq_job.json") @@ -26,7 +26,7 @@ def test_biocontainers() -> None: assert error_code == 0 -@pytest.mark.skipif(not deps, reason="galaxy-lib is not installed") +@pytest.mark.skipif(not deps, reason="galaxy-tool-util is not installed") def test_bioconda() -> None: wflow = get_data("tests/seqtk_seq.cwl") job = get_data("tests/seqtk_seq_job.json") @@ -37,7 +37,7 @@ def test_bioconda() -> None: assert error_code == 0, stderr -@pytest.mark.skipif(not deps, reason="galaxy-lib is not installed") +@pytest.mark.skipif(not deps, reason="galaxy-tool-util is not installed") @pytest.mark.skipif(not which("modulecmd"), reason="modulecmd not installed") def test_modules(monkeypatch: pytest.MonkeyPatch) -> None: """Do a basic smoke test using environment modules to satisfy a SoftwareRequirement.""" @@ -59,7 +59,7 @@ def test_modules(monkeypatch: pytest.MonkeyPatch) -> None: assert error_code == 0, stderr -@pytest.mark.skipif(not deps, reason="galaxy-lib is not installed") +@pytest.mark.skipif(not deps, reason="galaxy-tool-util is not installed") @pytest.mark.skipif(not which("modulecmd"), reason="modulecmd not installed") def test_modules_environment(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: """ From 2b17b1493b6ad05803894b78c5fb0fa373c7c7c5 Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" Date: Tue, 24 Aug 2021 10:38:26 +0200 Subject: [PATCH 097/111] CommandLineTool.arguments: correctly set $(self) Fixes #781 --- cwltool/expression.py | 2 +- tests/test_examples.py | 13 +++++++++ tests/wf/paramref_arguments_self.cwl | 41 ++++++++++++++++++++++++++++ 3 files changed, 55 insertions(+), 1 deletion(-) create mode 100644 tests/wf/paramref_arguments_self.cwl diff --git a/cwltool/expression.py b/cwltool/expression.py index 4e9c5126ac..8057886b63 100644 --- a/cwltool/expression.py +++ b/cwltool/expression.py @@ -217,7 +217,7 @@ def evaluator( if first_symbol_end + 1 == len(ex) and first_symbol == "null": return None try: - if obj.get(first_symbol) is None: + if first_symbol not in obj: raise WorkflowException("%s is not defined" % first_symbol) return next_seg( diff --git a/tests/test_examples.py b/tests/test_examples.py index bfab58594b..1485aa93d3 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -1414,3 +1414,16 @@ def test_malformed_reqs() -> None: match=r".*wc-tool-bad-reqs\.cwl:6:1: If 'requirements' is\s*present\s*then\s*it\s*must\s*be\s*a\s*list.*", ): factory.make(get_data("tests/wc-tool-bad-reqs.cwl")) + + +def test_arguments_self() -> None: + """Confirm that $(self) works in the arguments list.""" + factory = cwltool.factory.Factory() + check = factory.make(get_data("tests/wf/paramref_arguments_self.cwl")) + outputs = cast(Dict[str, Any], check()) + assert "self_review" in outputs + assert len(outputs) == 1 + assert ( + outputs["self_review"]["checksum"] + == "sha1$724ba28f4a9a1b472057ff99511ed393a45552e1" + ) diff --git a/tests/wf/paramref_arguments_self.cwl b/tests/wf/paramref_arguments_self.cwl new file mode 100644 index 0000000000..9795f5dd05 --- /dev/null +++ b/tests/wf/paramref_arguments_self.cwl @@ -0,0 +1,41 @@ +#!/usr/bin/env cwl-runner +cwlVersion: v1.0 +class: Workflow +inputs: [] +outputs: + self_review: + type: File + outputSource: evaluate_self/self_review +steps: + dump_self: + run: + class: CommandLineTool + baseCommand: echo + inputs: [] + arguments: + - '{"self":$(self)}' + stdout: self.json + outputs: + self_json: stdout + in: [] + out: [self_json] + evaluate_self: + run: + class: CommandLineTool + hints: + DockerRequirement: + dockerPull: everpeace/curl-jq + inputs: + self: + type: File + inputBinding: + position: 2 + stdout: self_review.txt + outputs: + self_review: stdout + baseCommand: jq + arguments: + - valueFrom: '.self | type == "null"' + position: 1 + in: { self: dump_self/self_json } + out: [ self_review ] From f111cf1d522569f02fcc4e94a38c46fc6ccf7ab1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 25 Aug 2021 07:17:47 +0000 Subject: [PATCH 098/111] Bump codecov/codecov-action from 2.0.2 to 2.0.3 Bumps [codecov/codecov-action](https://github.com/codecov/codecov-action) from 2.0.2 to 2.0.3. - [Release notes](https://github.com/codecov/codecov-action/releases) - [Changelog](https://github.com/codecov/codecov-action/blob/master/CHANGELOG.md) - [Commits](https://github.com/codecov/codecov-action/compare/v2.0.2...v2.0.3) --- updated-dependencies: - dependency-name: codecov/codecov-action dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/ci-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index b16eb13bf8..b3945fb64e 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -70,7 +70,7 @@ jobs: - name: Upload coverage to Codecov if: ${{ matrix.step == 'unit' }} - uses: codecov/codecov-action@v2.0.2 + uses: codecov/codecov-action@v2.0.3 with: fail_ci_if_error: true From 7fe639994ac086988bba0af092d9e0706e2eda9c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 26 Aug 2021 07:11:41 +0000 Subject: [PATCH 099/111] Update ruamel-yaml requirement from <0.17.14,>=0.15 to >=0.15,<0.17.15 Updates the requirements on [ruamel-yaml](https://sourceforge.net/p/ruamel-yaml/code/ci/default/tree) to permit the latest version. --- updated-dependencies: - dependency-name: ruamel-yaml dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 079fdb9fa2..f1ccb62b80 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ requests>=2.4.3 -ruamel.yaml>=0.15,<0.17.14 +ruamel.yaml>=0.15,<0.17.15 rdflib>=4.2.2,<6.1 shellescape>=3.4.1,<3.9 schema-salad>=8.1.20210627200047,<9 diff --git a/setup.py b/setup.py index f7767eab07..a94602cfe2 100644 --- a/setup.py +++ b/setup.py @@ -108,7 +108,7 @@ "setuptools", "requests >= 2.6.1", # >= 2.6.1 to workaround # https://github.com/ionrock/cachecontrol/issues/137 - "ruamel.yaml >= 0.15, < 0.17.14", + "ruamel.yaml >= 0.15, < 0.17.15", "rdflib >= 4.2.2, < 6.1.0", "shellescape >= 3.4.1, < 3.9", # 7.1.20210518142926 or later required due to From 99463af2f72fbebe55b279426008bb97909b05b7 Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" Date: Sun, 29 Aug 2021 12:07:10 +0200 Subject: [PATCH 100/111] skip broken version of importlib_metadata --- tox.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/tox.ini b/tox.ini index f2f52fbb6b..1ced34b1b4 100644 --- a/tox.ini +++ b/tox.ini @@ -48,6 +48,7 @@ deps = py3{6,7,8,9}-lint: flake8-bugbear py3{6,7,8,9}-lint: black py3{6,7,8,9}-bandit: bandit + py3{6,7,8,9}-bandit: importlib_metadata != 4.8.0 py3{6,7,8,9}-mypy: -rmypy_requirements.txt py39-pydocstyle: pydocstyle py39-pydocstyle: diff-cover From 952de68622b2a27033af97638146fe662b70f454 Mon Sep 17 00:00:00 2001 From: vladislav doster Date: Sun, 29 Aug 2021 07:24:06 -0500 Subject: [PATCH 101/111] (docs): update README.rst (#1482) Updates include the following: - correct pronoun agreement - correct spelling - correct grammar - add & remove punctuation - remove verbiage - increase readability Signed-off-by: Vladislav Doster --- README.rst | 80 ++++++++++++++++++++++++++---------------------------- 1 file changed, 39 insertions(+), 41 deletions(-) diff --git a/README.rst b/README.rst index 18ab09c704..1973cf38ed 100644 --- a/README.rst +++ b/README.rst @@ -73,14 +73,14 @@ and similar Linux distribution try sudo apt-get install cwltool -If you are running MacOS X or other UNIXes and you want to use packages prepared by the conda-forge project, then +If you are running macOS X or other UNIXes and you want to use packages prepared by the conda-forge project, then please follow the install instructions for `conda-forge `_ (if you haven't already) and then .. code:: bash conda install -c conda-forge cwltool -All of the above methods of installing ``cwltool`` use packages which might contain bugs already fixed in newer versions, or be missing features that you desire. +All of the above methods of installing ``cwltool`` use packages that might contain bugs already fixed in newer versions or be missing desired features. If the packaged version of ``cwltool`` available to you is too old, then we recommend installing using ``pip`` and ``venv`` .. code:: bash @@ -122,7 +122,7 @@ Or you can skip the direct ``pip`` commands above and install the latest develop pip install .[deps] # Installs ``cwltool`` from source cwltool --version # Check if the installation works correctly -Remember, if co-installing multiple CWL implementations then you need to +Remember, if co-installing multiple CWL implementations, then you need to maintain which implementation ``cwl-runner`` points to via a symbolic file system link or `another facility `_. @@ -154,7 +154,7 @@ these will be inserted at the beginning of the command line:: Use with boot2docker on macOS ----------------------------- -boot2docker runs Docker inside a virtual machine and it only mounts ``Users`` +boot2docker runs Docker inside a virtual machine, and it only mounts ``Users`` on it. The default behavior of CWL is to create temporary directories under e.g. ``/Var`` which is not accessible to Docker containers. @@ -167,14 +167,14 @@ Using uDocker ------------- Some shared computing environments don't support Docker software containers for technical or policy reasons. -As a work around, the CWL reference runner supports using alternative ``docker`` implementations on Linux +As a workaround, the CWL reference runner supports using alternative ``docker`` implementations on Linux with the ``--user-space-docker-cmd`` option. -One such "user space" friendly docker replacement is ``udocker`` https://github.com/indigo-dc/udocker +One such "user space" friendly docker replacement is ``udocker`` https://github.com/indigo-dc/udocker. udocker installation: https://github.com/indigo-dc/udocker/blob/master/doc/installation_manual.md#22-install-from-udockertools-tarball -Run `cwltool` just as you normally would, but with the new option, e.g. from the conformance tests +Run `cwltool` just as you usually would, but with the new option, e.g., from the conformance tests .. code:: bash @@ -202,8 +202,8 @@ Running a tool or workflow from remote or local locations systems via its support for HTTP[S] URLs. Input job files and Workflow steps (via the `run` directive) can reference CWL -documents using absolute or relative local filesytem paths. If a relative path -is referenced and that document isn't found in the current directory then the +documents using absolute or relative local filesystem paths. If a relative path +is referenced and that document isn't found in the current directory, then the following locations will be searched: http://www.commonwl.org/v1.0/CommandLineTool.html#Discovering_CWL_documents_on_a_local_filesystem @@ -232,7 +232,7 @@ workflow step, or command line tool) to the process requirements that should be Overrides can be specified either on the command line, or as part of the job input document. Workflow steps are identified using the name of the workflow file followed by the step name as a document fragment identifier "#id". -Override identifiers are relative to the toplevel workflow document. +Override identifiers are relative to the top-level workflow document. .. code:: bash @@ -263,7 +263,7 @@ referenced by a workflow and builds a new CWL document with all Process objects (CommandLineTool and Workflow) in a list in the ``$graph`` field. Cross references (such as ``run:`` and ``source:`` fields) are updated to internal references within the new packed -document. The top level workflow is named ``#main``. +document. The top-level workflow is named ``#main``. .. code:: bash @@ -275,7 +275,7 @@ Running only part of a workflow You can run a partial workflow with the ``--target`` (``-t``) option. This takes the name of an output parameter, workflow step, or input -parameter in the top level workflow. You may provide multiple +parameter in the top-level workflow. You may provide multiple targets. .. code:: bash @@ -285,11 +285,11 @@ targets. If a target is an output parameter, it will only run only the steps that contribute to that output. If a target is a workflow step, it will run the workflow starting from that step. If a target is an -input parameter, it will only run only the steps that are connected to +input parameter, it will only run the steps connected to that input. Use ``--print-targets`` to get a listing of the targets of a workflow. -To see exactly which steps will run, use ``--print-subgraph`` with +To see which steps will run, use ``--print-subgraph`` with ``--target`` to get a printout of the workflow subgraph for the selected targets. @@ -323,7 +323,7 @@ Environment Variables in cwltool -------------------------------- This reference implementation supports several ways of setting -enviroment variables for tools, in addition to the standard +environment variables for tools, in addition to the standard ``EnvVarRequirement``. The sequence of steps applied to create the enviroment is: @@ -418,7 +418,7 @@ example does require an existing module setup so it is impossible to test this e "out of the box" with cwltool. For a more isolated test that demonstrates all the same concepts - the resolver plugin type ``galaxy_packages`` can be used. -"Galaxy packages" are a lighter weight alternative to Environment Modules that are +"Galaxy packages" are a lighter-weight alternative to Environment Modules that are really just defined by a way to lay out directories into packages and versions to find little scripts that are sourced to modify the environment. They have been used for years in Galaxy community to adapt Galaxy tools to cluster @@ -468,12 +468,12 @@ The resolvers configuration file in the above example was simply: It is possible that the ``SoftwareRequirement`` s in a given CWL tool will not match the module names for a given cluster. Such requirements can be re-mapped -to specific deployed packages and/or versions using another file specified using +to specific deployed packages or versions using another file specified using the resolver plugin parameter `mapping_files`. We will -demonstrate this using `galaxy_packages` but the concepts apply equally well -to Environment Modules or Conda packages (described below) for instance. +demonstrate this using `galaxy_packages,` but the concepts apply equally well +to Environment Modules or Conda packages (described below), for instance. -So consider the resolvers configuration file +So consider the resolvers configuration file. (`tests/test_deps_env_resolvers_conf_rewrite.yml`): .. code:: yaml @@ -510,18 +510,17 @@ support for Homebrew/Linuxbrew plugins is available, the most developed such plugin is for the `Conda `__ package manager. Conda has the nice properties of allowing multiple versions of a package to be installed simultaneously, not requiring evaluated permissions to install Conda itself or packages using -Conda, and being cross platform. For these reasons, cwltool may run as a normal +Conda, and being cross-platform. For these reasons, cwltool may run as a normal user, install its own Conda environment and manage multiple versions of Conda packages -on both Linux and Mac OS X. +on Linux and Mac OS X. The Conda plugin can be endlessly configured, but a sensible set of defaults that has proven a powerful stack for dependency management within the Galaxy tool development ecosystem can be enabled by simply passing cwltool the ``--beta-conda-dependencies`` flag. -With this we can use the seqtk example above without Docker and without -any externally managed services - cwltool should install everything it needs -and create an environment for the tool. Try it out with the follwing command:: +With this, we can use the seqtk example above without Docker or any externally managed services - cwltool should install everything it needs +and create an environment for the tool. Try it out with the following command:: cwltool --beta-conda-dependencies tests/seqtk_seq.cwl tests/seqtk_seq_job.json @@ -548,7 +547,7 @@ The example can be executed using the command:: cwltool --beta-conda-dependencies tests/seqtk_seq_wrong_name.cwl tests/seqtk_seq_job.json -The plugin framework for managing resolution of these software requirements +The plugin framework for managing the resolution of these software requirements as maintained as part of `galaxy-tool-util `__ - a small, portable subset of the Galaxy project. More information on configuration and implementation can be found at the following links: @@ -608,9 +607,9 @@ in an integer). For example:: Interaction with containers: the MPIRequirement currently prepends its commands to the front of the command line that is constructed. If you -wish to run a containerised application in parallel, for simple use -cases this does work with Singularity, depending upon the platform -setup. However this combination should be considered "alpha" -- please +wish to run a containerized application in parallel, for simple use +cases, this does work with Singularity, depending upon the platform +setup. However, this combination should be considered "alpha" -- please do report any issues you have! This does not work with Docker at the moment. (More precisely, you get `n` copies of the same single process image run at the same time that cannot communicate with each other.) @@ -638,8 +637,8 @@ given in the following table; all are optional. | | | | variables that should be | | | | | passed from the host | | | | | environment through to the | -| | | | tool (e.g. giving the | -| | | | nodelist as set by your | +| | | | tool (e.g., giving the | +| | | | node list as set by your | | | | | scheduler). | +----------------+------------------+----------+------------------------------+ | env_pass_regex | List[str] | [] | A list of python regular | @@ -672,8 +671,8 @@ To run the basic tests after installing `cwltool` execute the following: pip install -rtest-requirements.txt pytest -To run various tests in all supported Python environments we use `tox `_. To run the test suite in all supported Python environments -first downloading the complete code repository (see the ``git clone`` instructions above) and then run +To run various tests in all supported Python environments, we use `tox `_. To run the test suite in all supported Python environments +first clone the complete code repository (see the ``git clone`` instructions above) and then run the following in the terminal: ``pip install tox; tox -p`` @@ -690,7 +689,7 @@ The GitHub repository for the CWL specifications contains a script that tests a implementation against a wide array of valid CWL files using the `cwltest `_ program -Instructions for running these tests can be found in the Common Workflow Language Specification repository at https://github.com/common-workflow-language/common-workflow-language/blob/main/CONFORMANCE_TESTS.md +Instructions for running these tests can be found in the Common Workflow Language Specification repository at https://github.com/common-workflow-language/common-workflow-language/blob/main/CONFORMANCE_TESTS.md . Import as a module ------------------ @@ -726,7 +725,7 @@ Technical outline of how cwltool works internally, for maintainers. #. Fetches the document from file or URL #. Applies preprocessing (syntax/identifier expansion and normalization) #. Validates the document based on cwlVersion - #. If necessary, updates the document to latest spec + #. If necessary, updates the document to the latest spec #. Constructs a Process object using ``make_tool()``` callback. This yields a CommandLineTool, Workflow, or ExpressionTool. For workflows, this recursively constructs each workflow step. @@ -741,7 +740,7 @@ Technical outline of how cwltool works internally, for maintainers. there is currently no work ready to run) or end of iteration (indicating the process is complete.) #. Invoke the runnable item by calling ``run()``. This runs the tool and gets output. - #. Output of a process is reported by an output callback. + #. An output callback reports the output of a process. #. ``job()`` may be iterated over multiple times. It will yield all the work that is currently ready to run and then yield None. @@ -751,12 +750,11 @@ Technical outline of how cwltool works internally, for maintainers. inputs the step are ready. #. When a step is ready, it constructs an input object for that step and iterates on the ``job()`` method of the workflow job step. - #. Each runnable item is yielded back up to top level run loop + #. Each runnable item is yielded back up to top-level run loop #. When a step job completes and receives an output callback, the job outputs are assigned to the output of the workflow step. #. When all steps are complete, the intermediate files are moved to a final - workflow output, intermediate directories are deleted, and the output - callback for the workflow is called. + workflow output, intermediate directories are deleted, and the workflow's output callback is called. #. ``CommandLineTool`` job() objects yield a single runnable object. @@ -789,7 +787,7 @@ executor executor(tool, job_order_object, runtimeContext, logger) (Process, Dict[Text, Any], RuntimeContext) -> Tuple[Dict[Text, Any], Text] - An implementation of the toplevel workflow execution loop, should + An implementation of the top-level workflow execution loop should synchronously run a process object to completion and return the output object. @@ -826,7 +824,7 @@ resolver resolver(document_loader, document) (Loader, Union[Text, dict[Text, Any]]) -> Text - Resolve a relative document identifier to an absolute one which can be fetched. + Resolve a relative document identifier to an absolute one that can be fetched. The following functions can be set in RuntimeContext to override or augment the listed behaviors. From 05fc01bd00251217a219faaf63b9947d200eb849 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 30 Aug 2021 07:11:26 +0000 Subject: [PATCH 102/111] Update ruamel-yaml requirement from <0.17.15,>=0.15 to >=0.15,<0.17.17 Updates the requirements on [ruamel-yaml](https://sourceforge.net/p/ruamel-yaml/code/ci/default/tree) to permit the latest version. --- updated-dependencies: - dependency-name: ruamel-yaml dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index f1ccb62b80..2bdd01e1a3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ requests>=2.4.3 -ruamel.yaml>=0.15,<0.17.15 +ruamel.yaml>=0.15,<0.17.17 rdflib>=4.2.2,<6.1 shellescape>=3.4.1,<3.9 schema-salad>=8.1.20210627200047,<9 diff --git a/setup.py b/setup.py index a94602cfe2..46562ab956 100644 --- a/setup.py +++ b/setup.py @@ -108,7 +108,7 @@ "setuptools", "requests >= 2.6.1", # >= 2.6.1 to workaround # https://github.com/ionrock/cachecontrol/issues/137 - "ruamel.yaml >= 0.15, < 0.17.15", + "ruamel.yaml >= 0.15, < 0.17.17", "rdflib >= 4.2.2, < 6.1.0", "shellescape >= 3.4.1, < 3.9", # 7.1.20210518142926 or later required due to From 70179905cfbc6ca68a037a337410d005bb6af50f Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" Date: Wed, 1 Sep 2021 13:36:29 +0200 Subject: [PATCH 103/111] fix add_file_or_directory_volume for singularity Fixes #1441 --- conformance-test.sh | 5 --- cwltool/singularity.py | 85 ++++++++++++++++++++++++------------------ 2 files changed, 48 insertions(+), 42 deletions(-) diff --git a/conformance-test.sh b/conformance-test.sh index f46adeff4c..e383e06f66 100755 --- a/conformance-test.sh +++ b/conformance-test.sh @@ -109,11 +109,6 @@ if [[ "$container" = "singularity" ]]; then # See issue #1440 exclusions+=(stdin_shorcut) fi - - if [[ "${version}" = "v1.2" ]]; then - # See issue #1441 - exclusions+=(iwdr_dir_literal_real_file) - fi fi if [ -n "$EXTRA" ] diff --git a/cwltool/singularity.py b/cwltool/singularity.py index 7de8ee5658..3c02c5dcfd 100644 --- a/cwltool/singularity.py +++ b/cwltool/singularity.py @@ -74,6 +74,12 @@ def is_version_3_1_or_newer() -> bool: return int(version[0]) >= 4 or (int(version[0]) == 3 and int(version[1]) >= 1) +def is_version_3_4_or_newer() -> bool: + """Detect if Singularity v3.4+ is available.""" + version = get_version().split(".") + return int(version[0]) >= 4 or (int(version[0]) == 3 and int(version[1]) >= 4) + + def _normalize_image_id(string: str) -> str: return string.replace("/", "_") + ".img" @@ -285,18 +291,17 @@ def append_volume( def add_file_or_directory_volume( self, runtime: List[str], volume: MapperEnt, host_outdir_tgt: Optional[str] ) -> None: - if host_outdir_tgt is not None: - # workaround for lack of overlapping mounts in Singularity - # revert to daa923d5b0be3819b6ed0e6440e7193e65141052 - # once https://github.com/sylabs/singularity/issues/1607 - # is fixed - if volume.type == "File": - shutil.copy(volume.resolved, host_outdir_tgt) + if not volume.resolved.startswith("_:"): + if host_outdir_tgt is not None and not is_version_3_4_or_newer(): + # workaround for lack of overlapping mounts in Singularity <3.4 + if volume.type == "File": + os.makedirs(os.path.dirname(host_outdir_tgt), exist_ok=True) + shutil.copy(volume.resolved, host_outdir_tgt) + else: + shutil.copytree(volume.resolved, host_outdir_tgt) + ensure_non_writable(host_outdir_tgt) else: - shutil.copytree(volume.resolved, host_outdir_tgt) - ensure_non_writable(host_outdir_tgt) - elif not volume.resolved.startswith("_:"): - self.append_volume(runtime, volume.resolved, volume.target) + self.append_volume(runtime, volume.resolved, volume.target) def add_writable_file_volume( self, @@ -305,11 +310,8 @@ def add_writable_file_volume( host_outdir_tgt: Optional[str], tmpdir_prefix: str, ) -> None: - if host_outdir_tgt is not None: - # workaround for lack of overlapping mounts in Singularity - # revert to daa923d5b0be3819b6ed0e6440e7193e65141052 - # once https://github.com/sylabs/singularity/issues/1607 - # is fixed + if host_outdir_tgt is not None and not is_version_3_4_or_newer(): + # workaround for lack of overlapping mounts in Singularity <3.4 if self.inplace_update: try: os.link(os.path.realpath(volume.resolved), host_outdir_tgt) @@ -322,14 +324,21 @@ def add_writable_file_volume( self.append_volume(runtime, volume.resolved, volume.target, writable=True) ensure_writable(volume.resolved) else: - file_copy = os.path.join( - create_tmp_dir(tmpdir_prefix), - os.path.basename(volume.resolved), - ) - shutil.copy(volume.resolved, file_copy) - # volume.resolved = file_copy - self.append_volume(runtime, file_copy, volume.target, writable=True) - ensure_writable(file_copy) + if host_outdir_tgt: + # shortcut, just copy to the output directory + # which is already going to be mounted + if not os.path.exists(os.path.dirname(host_outdir_tgt)): + os.makedirs(os.path.dirname(host_outdir_tgt)) + shutil.copy(volume.resolved, host_outdir_tgt) + ensure_writable(host_outdir_tgt) + else: + file_copy = os.path.join( + create_tmp_dir(tmpdir_prefix), + os.path.basename(volume.resolved), + ) + shutil.copy(volume.resolved, file_copy) + self.append_volume(runtime, file_copy, volume.target, writable=True) + ensure_writable(file_copy) def add_writable_directory_volume( self, @@ -339,31 +348,33 @@ def add_writable_directory_volume( tmpdir_prefix: str, ) -> None: if volume.resolved.startswith("_:"): + # Synthetic directory that needs creating first if host_outdir_tgt is not None: new_dir = host_outdir_tgt else: new_dir = os.path.join( create_tmp_dir(tmpdir_prefix), - os.path.basename(volume.resolved), + os.path.basename(volume.target), ) + self.append_volume(runtime, new_dir, volume.target, writable=True) os.makedirs(new_dir) else: - if host_outdir_tgt is not None: - # workaround for lack of overlapping mounts in Singularity - # revert to daa923d5b0be3819b6ed0e6440e7193e65141052 - # once https://github.com/sylabs/singularity/issues/1607 - # is fixed + if host_outdir_tgt is not None and not is_version_3_4_or_newer(): + # workaround for lack of overlapping mounts in Singularity < 3.4 shutil.copytree(volume.resolved, host_outdir_tgt) ensure_writable(host_outdir_tgt) else: if not self.inplace_update: - dir_copy = os.path.join( - create_tmp_dir(tmpdir_prefix), - os.path.basename(volume.resolved), - ) - shutil.copytree(volume.resolved, dir_copy) - source = dir_copy - # volume.resolved = dir_copy + if not host_outdir_tgt: + tmpdir = create_tmp_dir(tmpdir_prefix) + new_dir = os.path.join( + tmpdir, os.path.basename(volume.resolved) + ) + shutil.copytree(volume.resolved, new_dir) + source = tmpdir + else: + shutil.copytree(volume.resolved, host_outdir_tgt) + source = host_outdir_tgt else: source = volume.resolved self.append_volume(runtime, source, volume.target, writable=True) From fbe4aa10d4b49dec430d34fa37d98fa5ffe7e546 Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" Date: Wed, 1 Sep 2021 19:09:40 +0200 Subject: [PATCH 104/111] tweaks for testing with Singularity 3.4.x --- tests/test_environment.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/tests/test_environment.py b/tests/test_environment.py index 397ab35eab..44b0f76a47 100644 --- a/tests/test_environment.py +++ b/tests/test_environment.py @@ -41,6 +41,7 @@ def assert_env_matches( """ e = dict(env) for k, check in checks.items(): + assert k in e v = e.pop(k) assert_envvar_matches(check, k, env) @@ -127,7 +128,6 @@ def PWD(v: str, env: Env) -> bool: "LANG": "C", "LD_LIBRARY_PATH": None, "PATH": None, - "PROMPT_COMMAND": None, "PS1": None, "PWD": PWD, "TMPDIR": "/tmp", @@ -140,12 +140,16 @@ def PWD(v: str, env: Env) -> bool: vminor = int(version[1]) sing_vars: EnvChecks = { "SINGULARITY_CONTAINER": None, - "SINGULARITY_ENVIRONMENT": None, "SINGULARITY_NAME": None, } - if vminor < 6: + if vminor < 5: + sing_vars["SINGULARITY_APPNAME"] = None + if vminor >= 5: + sing_vars["PROMPT_COMMAND"] = None + sing_vars["SINGULARITY_ENVIRONMENT"] = None + if vminor == 5: sing_vars["SINGULARITY_INIT"] = "1" - else: + elif vminor > 5: sing_vars["SINGULARITY_COMMAND"] = "exec" if vminor >= 7: From 62d1e116d3fd943cf3ccd4ba203ac922f3f0adf5 Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" Date: Wed, 1 Sep 2021 21:10:33 +0200 Subject: [PATCH 105/111] cope with datetime objects --- cwltool/main.py | 28 ++-- cwltool/mpi.py | 4 +- requirements.txt | 2 +- setup.py | 2 +- tests/test_mpi.py | 4 +- tests/test_pack.py | 18 ++- tests/test_validate_js.py | 6 +- tests/wf/expect_packed.cwl | 1 - tests/wf/expect_revsort_datetime_packed.cwl | 140 ++++++++++++++++++++ tests/wf/revsort_datetime.cwl | 80 +++++++++++ 10 files changed, 260 insertions(+), 25 deletions(-) create mode 100644 tests/wf/expect_revsort_datetime_packed.cwl create mode 100755 tests/wf/revsort_datetime.cwl diff --git a/cwltool/main.py b/cwltool/main.py index 235344dd88..5a5494ec4a 100755 --- a/cwltool/main.py +++ b/cwltool/main.py @@ -41,7 +41,7 @@ from schema_salad.exceptions import ValidationException from schema_salad.ref_resolver import Loader, file_uri, uri_file_path from schema_salad.sourceline import strip_dup_lineno -from schema_salad.utils import ContextType, FetcherCallableType, json_dumps +from schema_salad.utils import ContextType, FetcherCallableType, json_dumps, yaml_no_ts from . import CWL_CONTENT_TYPES, workflow from .argparser import arg_parser, generate_parser, get_default_args @@ -329,7 +329,7 @@ def load_job_order( if len(args.job_order) == 1 and args.job_order[0][0] != "-": job_order_file = args.job_order[0] elif len(args.job_order) == 1 and args.job_order[0] == "-": - yaml = YAML() + yaml = yaml_no_ts() job_order_object = yaml.load(stdin) job_order_object, _ = loader.resolve_all( job_order_object, file_uri(os.getcwd()) + "/" @@ -443,7 +443,7 @@ def init_job_order( if _logger.isEnabledFor(logging.DEBUG): _logger.debug( "Parsed job order from command line: %s", - json_dumps(job_order_object, indent=4), + json_dumps(job_order_object, indent=4, default=str), ) for inp in process.tool["inputs"]: @@ -546,7 +546,7 @@ def printdeps( elif relative_deps == "cwd": base = os.getcwd() visit_class(deps, ("File", "Directory"), functools.partial(make_relative, base)) - stdout.write(json_dumps(deps, indent=4)) + stdout.write(json_dumps(deps, indent=4, default=str)) def prov_deps( @@ -608,9 +608,9 @@ def print_pack( """Return a CWL serialization of the CWL document in JSON.""" packed = pack(loadingContext, uri) if len(cast(Sized, packed["$graph"])) > 1: - return json_dumps(packed, indent=4) + return json_dumps(packed, indent=4, default=str) return json_dumps( - cast(MutableSequence[CWLObjectType], packed["$graph"])[0], indent=4 + cast(MutableSequence[CWLObjectType], packed["$graph"])[0], indent=4, default=str ) @@ -1064,7 +1064,11 @@ def main( if args.print_pre: stdout.write( json_dumps( - processobj, indent=4, sort_keys=True, separators=(",", ": ") + processobj, + indent=4, + sort_keys=True, + separators=(",", ": "), + default=str, ) ) return 0 @@ -1125,7 +1129,11 @@ def main( del tool.tool["name"] stdout.write( json_dumps( - tool.tool, indent=4, sort_keys=True, separators=(",", ": ") + tool.tool, + indent=4, + sort_keys=True, + separators=(",", ": "), + default=str, ) ) return 0 @@ -1288,7 +1296,9 @@ def loc_to_path(obj: CWLObjectType) -> None: if isinstance(out, str): stdout.write(out) else: - stdout.write(json_dumps(out, indent=4, ensure_ascii=False)) + stdout.write( + json_dumps(out, indent=4, ensure_ascii=False, default=str) + ) stdout.write("\n") if hasattr(stdout, "flush"): stdout.flush() diff --git a/cwltool/mpi.py b/cwltool/mpi.py index 48d0d4b613..a660177962 100644 --- a/cwltool/mpi.py +++ b/cwltool/mpi.py @@ -4,7 +4,7 @@ import re from typing import List, Mapping, MutableMapping, Optional, Type, TypeVar, Union -from ruamel.yaml.main import YAML +from schema_salad.utils import yaml_no_ts MpiConfigT = TypeVar("MpiConfigT", bound="MpiConfig") @@ -53,7 +53,7 @@ def load(cls: Type[MpiConfigT], config_file_name: str) -> MpiConfigT: optional). """ with open(config_file_name) as cf: - yaml = YAML() + yaml = yaml_no_ts() data = yaml.load(cf) try: return cls(**data) diff --git a/requirements.txt b/requirements.txt index 2bdd01e1a3..9d66e7b2b9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ requests>=2.4.3 ruamel.yaml>=0.15,<0.17.17 rdflib>=4.2.2,<6.1 shellescape>=3.4.1,<3.9 -schema-salad>=8.1.20210627200047,<9 +schema-salad>=8.2,<9 prov==1.5.1 bagit==1.8.1 mypy-extensions diff --git a/setup.py b/setup.py index 46562ab956..6e1e098d60 100644 --- a/setup.py +++ b/setup.py @@ -113,7 +113,7 @@ "shellescape >= 3.4.1, < 3.9", # 7.1.20210518142926 or later required due to # https://github.com/common-workflow-language/schema_salad/issues/385 - "schema-salad >= 8.1.20210627200047, < 9", + "schema-salad >= 8.2, < 9", "mypy-extensions", "psutil >= 5.6.6", "prov == 1.5.1", diff --git a/tests/test_mpi.py b/tests/test_mpi.py index 400a8de4bd..2ca0823d72 100644 --- a/tests/test_mpi.py +++ b/tests/test_mpi.py @@ -9,8 +9,8 @@ import pkg_resources import pytest from ruamel.yaml.comments import CommentedMap, CommentedSeq -from ruamel.yaml.main import YAML from schema_salad.avro.schema import Names +from schema_salad.utils import yaml_no_ts import cwltool.load_tool import cwltool.singularity @@ -104,7 +104,7 @@ def run_many(self, n: int, args: List[str]): "env_pass": ["USER"], } plat_conf_file = mpitmp / "plat_mpi.yml" - yaml = YAML() + yaml = yaml_no_ts() yaml.dump(plat_conf, plat_conf_file) yield str(plat_conf_file) diff --git a/tests/test_pack.py b/tests/test_pack.py index 4918e79934..fea55aa9ad 100644 --- a/tests/test_pack.py +++ b/tests/test_pack.py @@ -5,10 +5,9 @@ from functools import partial from io import StringIO from pathlib import Path -from typing import Dict +from typing import Any, Dict, List, cast import pytest -from ruamel.yaml.main import YAML import cwltool.pack import cwltool.workflow @@ -18,6 +17,8 @@ from cwltool.resolver import tool_resolver from cwltool.utils import adjustDirObjs, adjustFileObjs +from schema_salad.utils import yaml_no_ts + from .util import get_data, needs_docker @@ -30,6 +31,10 @@ "tests/wf/operation/expect_operation-single_packed.cwl", ), ("tests/wf/trick_revsort.cwl", "tests/wf/expect_trick_packed.cwl"), + ( + "tests/wf/revsort_datetime.cwl", + "tests/wf/expect_revsort_datetime_packed.cwl", + ), ], ) def test_packing(unpacked: str, expected: str) -> None: @@ -40,14 +45,15 @@ def test_packing(unpacked: str, expected: str) -> None: loadingContext, workflowobj, uri ) - packed = cwltool.pack.pack(loadingContext, uri) + packed = json.loads(print_pack(loadingContext, uri)) + if len(cast(List[Any], packed["$graph"])) == 1: + packed = cast(List[Any], packed["$graph"])[0] context_dir = os.path.abspath(os.path.dirname(get_data(unpacked))) adjustFileObjs(packed, partial(make_relative, context_dir)) adjustDirObjs(packed, partial(make_relative, context_dir)) - yaml = YAML(typ="safe", pure=True) with open(get_data(expected)) as packed_file: - expect_packed = yaml.load(packed_file) + expect_packed = json.load(packed_file) if "$schemas" in expect_packed: assert "$schemas" in packed @@ -77,7 +83,7 @@ def test_pack_single_tool() -> None: def test_pack_fragment() -> None: - yaml = YAML(typ="safe", pure=True) + yaml = yaml_no_ts() with open(get_data("tests/wf/scatter2_subwf.cwl")) as packed_file: expect_packed = yaml.load(packed_file) diff --git a/tests/test_validate_js.py b/tests/test_validate_js.py index 8d2ebd3d66..34f68a6dc3 100644 --- a/tests/test_validate_js.py +++ b/tests/test_validate_js.py @@ -1,7 +1,7 @@ from typing import Any -from ruamel.yaml.main import YAML from schema_salad.avro.schema import Names +from schema_salad.utils import yaml_no_ts from cwltool import process, validate_js from cwltool.sandboxjs import code_fragment_to_js @@ -24,7 +24,7 @@ def test_get_expressions() -> None: - yaml = YAML() + yaml = yaml_no_ts() test_cwl_yaml = yaml.load(TEST_CWL) schema = process.get_schema("v1.0")[1] assert isinstance(schema, Names) @@ -36,7 +36,7 @@ def test_get_expressions() -> None: def test_validate_js_expressions(mocker: Any) -> None: - yaml = YAML() + yaml = yaml_no_ts() test_cwl_yaml = yaml.load(TEST_CWL) schema = process.get_schema("v1.0")[1] assert isinstance(schema, Names) diff --git a/tests/wf/expect_packed.cwl b/tests/wf/expect_packed.cwl index 9f0be2c862..dedc7c04bf 100644 --- a/tests/wf/expect_packed.cwl +++ b/tests/wf/expect_packed.cwl @@ -1,4 +1,3 @@ -#!/usr/bin/env cwl-runner { "$graph": [ { diff --git a/tests/wf/expect_revsort_datetime_packed.cwl b/tests/wf/expect_revsort_datetime_packed.cwl new file mode 100644 index 0000000000..0c3ed98e3d --- /dev/null +++ b/tests/wf/expect_revsort_datetime_packed.cwl @@ -0,0 +1,140 @@ +{ + "$graph": [ + { + "class": "Workflow", + "doc": "Reverse the lines in a document, then sort those lines.", + "hints": [ + { + "class": "DockerRequirement", + "dockerPull": "debian:8" + } + ], + "inputs": [ + { + "type": "boolean", + "default": true, + "doc": "If true, reverse (decending) sort", + "id": "#main/reverse_sort" + }, + { + "type": "File", + "doc": "The input file to be processed.", + "format": "https://www.iana.org/assignments/media-types/text/plain", + "default": { + "class": "File", + "location": "hello.txt" + }, + "id": "#main/workflow_input" + } + ], + "outputs": [ + { + "type": "File", + "outputSource": "#main/sorted/sorted_output", + "doc": "The output with the lines reversed and sorted.", + "id": "#main/sorted_output" + } + ], + "steps": [ + { + "in": [ + { + "source": "#main/workflow_input", + "id": "#main/rev/revtool_input" + } + ], + "out": [ + "#main/rev/revtool_output" + ], + "run": "#revtool.cwl", + "id": "#main/rev" + }, + { + "in": [ + { + "source": "#main/reverse_sort", + "id": "#main/sorted/reverse" + }, + { + "source": "#main/rev/revtool_output", + "id": "#main/sorted/sorted_input" + } + ], + "out": [ + "#main/sorted/sorted_output" + ], + "run": "#sorttool.cwl", + "id": "#main/sorted" + } + ], + "id": "#main", + "http://schema.org/dateCreated": "2020-10-08", + "$namespaces": { + "iana": "https://www.iana.org/assignments/media-types/", + "s": "http://schema.org/" + } + }, + { + "class": "CommandLineTool", + "doc": "Reverse each line using the `rev` command", + "inputs": [ + { + "type": "File", + "inputBinding": {}, + "id": "#revtool.cwl/revtool_input" + } + ], + "outputs": [ + { + "type": "File", + "outputBinding": { + "glob": "output.txt" + }, + "id": "#revtool.cwl/revtool_output" + } + ], + "baseCommand": "rev", + "stdout": "output.txt", + "id": "#revtool.cwl" + }, + { + "class": "CommandLineTool", + "doc": "Sort lines using the `sort` command", + "inputs": [ + { + "id": "#sorttool.cwl/reverse", + "type": "boolean", + "inputBinding": { + "position": 1, + "prefix": "--reverse" + } + }, + { + "id": "#sorttool.cwl/sorted_input", + "type": "File", + "inputBinding": { + "position": 2 + } + } + ], + "outputs": [ + { + "id": "#sorttool.cwl/sorted_output", + "type": "File", + "outputBinding": { + "glob": "output.txt" + } + } + ], + "baseCommand": "sort", + "stdout": "output.txt", + "id": "#sorttool.cwl" + } + ], + "cwlVersion": "v1.0", + "$schemas": [ + "empty2.ttl", + "https://schema.org/version/latest/schemaorg-current-https.rdf", + "empty.ttl" + ] +} \ No newline at end of file diff --git a/tests/wf/revsort_datetime.cwl b/tests/wf/revsort_datetime.cwl new file mode 100755 index 0000000000..812b384f5d --- /dev/null +++ b/tests/wf/revsort_datetime.cwl @@ -0,0 +1,80 @@ +#!/usr/bin/env cwl-runner +# +# This is a two-step workflow which uses "revtool" and "sorttool" defined above. +# +class: Workflow +doc: "Reverse the lines in a document, then sort those lines." +cwlVersion: v1.0 + +# Requirements & hints specify prerequisites and extensions to the workflow. +# In this example, DockerRequirement specifies a default Docker container +# in which the command line tools will execute. +hints: + - class: DockerRequirement + dockerPull: debian:8 + + +# The inputs array defines the structure of the input object that describes +# the inputs to the workflow. +# +# The "reverse_sort" input parameter demonstrates the "default" field. If the +# field "reverse_sort" is not provided in the input object, the default value will +# be used. +inputs: + workflow_input: + type: File + doc: "The input file to be processed." + format: iana:text/plain + default: + class: File + location: hello.txt + reverse_sort: + type: boolean + default: true + doc: "If true, reverse (decending) sort" + +# The "outputs" array defines the structure of the output object that describes +# the outputs of the workflow. +# +# Each output field must be connected to the output of one of the workflow +# steps using the "outputSource" field. Here, the parameter "sorted_output" of the +# workflow comes from the "sorted_output" output of the "sorted" step. +outputs: + sorted_output: + type: File + outputSource: sorted/sorted_output + doc: "The output with the lines reversed and sorted." + +# The "steps" array lists the executable steps that make up the workflow. +# The tool to execute each step is listed in the "run" field. +# +# In the first step, the "in" field of the step connects the upstream +# parameter "workflow_input" of the workflow to the input parameter of the tool +# "revtool_input" +# +# In the second step, the "in" field of the step connects the output +# parameter "revtool_output" from the first step to the input parameter of the +# tool "sorted_input". +steps: + rev: + in: + revtool_input: workflow_input + out: [revtool_output] + run: revtool.cwl + + sorted: + in: + sorted_input: rev/revtool_output + reverse: reverse_sort + out: [sorted_output] + run: sorttool.cwl + +$namespaces: + iana: https://www.iana.org/assignments/media-types/ + s: http://schema.org/ + +$schemas: + - https://schema.org/version/latest/schemaorg-current-https.rdf + - empty2.ttl + +s:dateCreated: 2020-10-08 From f4cc3d4872484e400055cf962b6ab99b0326b269 Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" Date: Thu, 2 Sep 2021 14:14:32 +0200 Subject: [PATCH 106/111] fix build-cwltool-docker.sh --- build-cwltool-docker.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build-cwltool-docker.sh b/build-cwltool-docker.sh index 377a0281a5..61f8cc031e 100755 --- a/build-cwltool-docker.sh +++ b/build-cwltool-docker.sh @@ -3,7 +3,7 @@ set -ex docker build --file=cwltool.Dockerfile --tag=quay.io/commonwl/cwltool_module --target module . docker build --file=cwltool.Dockerfile --tag=quay.io/commonwl/cwltool . -docker run -v -it /var/run/docker.sock:/var/run/docker.sock \ +docker run -it -v /var/run/docker.sock:/var/run/docker.sock \ -v /tmp:/tmp \ -v "$PWD":/tmp/cwltool \ quay.io/commonwl/cwltool_module /bin/sh -c \ From b67a5e0fc6b1e0dbd2277a112012bbe19a9dcc91 Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" Date: Mon, 30 Aug 2021 20:51:18 +0200 Subject: [PATCH 107/111] expression evaluation: better error messages --- Makefile | 3 +- cwltool/builder.py | 108 +++++++-- cwltool/command_line_tool.py | 376 ++++++++++++++++++++++++-------- cwltool/executors.py | 17 +- cwltool/job.py | 27 +-- cwltool/load_tool.py | 4 +- cwltool/pack.py | 10 +- cwltool/process.py | 50 ++--- cwltool/provenance_profile.py | 8 +- cwltool/singularity.py | 63 +++--- cwltool/update.py | 4 +- cwltool/validate_js.py | 12 +- cwltool/workflow.py | 29 ++- cwltool/workflow_job.py | 8 +- tests/echo-badposition-expr.cwl | 28 +++ tests/iwdr_bad_expr.cwl | 24 ++ tests/secondary-files-bad.cwl | 36 +++ tests/secondary-files.cwl | 4 +- tests/test_examples.py | 146 ++++++++++++- tests/test_iwdr.py | 281 +++++++++++++++--------- tests/test_pack.py | 3 +- tests/test_schemadef.py | 1 + tests/test_streaming.py | 8 +- tests/test_tmpdir.py | 6 +- tests/test_validate_js.py | 12 +- tests/wf/1496.cwl | 15 ++ tests/wf/bad-stderr-expr.cwl | 18 ++ tests/wf/bad-stdin-expr.cwl | 17 ++ tests/wf/bad-stdout-expr.cwl | 17 ++ tests/wf/bad_formattest.cwl | 25 +++ tests/wf/bad_networkaccess.cwl | 10 + tests/wf/bad_timelimit.cwl | 14 ++ tests/wf/iwdr_permutations.cwl | 18 +- 33 files changed, 1051 insertions(+), 351 deletions(-) create mode 100644 tests/echo-badposition-expr.cwl create mode 100644 tests/iwdr_bad_expr.cwl create mode 100644 tests/secondary-files-bad.cwl create mode 100644 tests/wf/1496.cwl create mode 100644 tests/wf/bad-stderr-expr.cwl create mode 100644 tests/wf/bad-stdin-expr.cwl create mode 100644 tests/wf/bad-stdout-expr.cwl create mode 100755 tests/wf/bad_formattest.cwl create mode 100644 tests/wf/bad_networkaccess.cwl create mode 100644 tests/wf/bad_timelimit.cwl diff --git a/Makefile b/Makefile index 51312c2d1e..0e07cff3e1 100644 --- a/Makefile +++ b/Makefile @@ -79,7 +79,8 @@ docs: FORCE ## clean : clean up all temporary / machine-generated files clean: check-python3 FORCE - rm -f ${MODILE}/*.pyc tests/*.pyc + rm -f ${MODULE}/*.pyc tests/*.pyc *.so ${MODULE}/*.so + rm -Rf ${MODULE}/__pycache__/ python setup.py clean --all || true rm -Rf .coverage rm -f diff-cover.html diff --git a/cwltool/builder.py b/cwltool/builder.py index d4b68581e4..f88ccc7591 100644 --- a/cwltool/builder.py +++ b/cwltool/builder.py @@ -229,6 +229,7 @@ def bind_input( lead_pos: Optional[Union[int, List[int]]] = None, tail_pos: Optional[Union[str, List[int]]] = None, ) -> List[MutableMapping[str, Union[str, List[int]]]]: + debug = _logger.isEnabledFor(logging.DEBUG) if tail_pos is None: tail_pos = [] @@ -250,8 +251,17 @@ def bind_input( position = binding["position"] if isinstance(position, str): # no need to test the CWL Version # the schema for v1.0 only allow ints - binding["position"] = self.do_eval(position, context=datum) - bp.append(binding["position"]) + result = self.do_eval(position, context=datum) + if not isinstance(result, int): + raise SourceLine( + schema["inputBinding"], "position", WorkflowException, debug + ).makeError( + "'position' expressions must evaluate to an int, " + f"not a {type(result)}. Expression {position} " + f"resulted in '{result}'." + ) + binding["position"] = result + bp.append(result) else: bp.extend(aslist(binding["position"])) else: @@ -396,7 +406,10 @@ def _capture_files(f: CWLObjectType) -> CWLObjectType: if loadContents_sourceline and loadContents_sourceline["loadContents"]: with SourceLine( - loadContents_sourceline, "loadContents", WorkflowException + loadContents_sourceline, + "loadContents", + WorkflowException, + debug, ): try: with self.fs_access.open( @@ -411,17 +424,42 @@ def _capture_files(f: CWLObjectType) -> CWLObjectType: if "secondaryFiles" in schema: if "secondaryFiles" not in datum: datum["secondaryFiles"] = [] - for sf in aslist(schema["secondaryFiles"]): - if "required" in sf: - sf_required = self.do_eval(sf["required"], context=datum) + for num, sf_entry in enumerate(aslist(schema["secondaryFiles"])): + if "required" in sf_entry and sf_entry["required"] is not None: + required_result = self.do_eval( + sf_entry["required"], context=datum + ) + if not ( + isinstance(required_result, bool) + or required_result is None + ): + if ( + aslist(schema["secondaryFiles"]) + == schema["secondaryFiles"] + ): + sf_item: Any = cast( + List[Any], schema["secondaryFiles"] + )[num] + else: + sf_item = schema["secondaryFiles"] + raise SourceLine( + sf_item, "required", WorkflowException, debug + ).makeError( + "The result of a expression in the field " + "'required' must " + f"be a bool or None, not a {type(required_result)}. " + f"Expression '{sf_entry['required']}' resulted " + f"in '{required_result}'." + ) + sf_required = required_result else: sf_required = True - if "$(" in sf["pattern"] or "${" in sf["pattern"]: - sfpath = self.do_eval(sf["pattern"], context=datum) + if "$(" in sf_entry["pattern"] or "${" in sf_entry["pattern"]: + sfpath = self.do_eval(sf_entry["pattern"], context=datum) else: sfpath = substitute( - cast(str, datum["basename"]), sf["pattern"] + cast(str, datum["basename"]), sf_entry["pattern"] ) for sfname in aslist(sfpath): @@ -443,9 +481,13 @@ def _capture_files(f: CWLObjectType) -> CWLObjectType: sf_location = sfname["location"] sfbasename = sfname["basename"] else: - raise WorkflowException( - "Expected secondaryFile expression to return type 'str' or 'MutableMapping', received '%s'" - % (type(sfname)) + raise SourceLine( + sf_entry, "pattern", WorkflowException, debug + ).makeError( + "Expected secondaryFile expression to " + "return type 'str', a 'File' or 'Directory' " + "dictionary, or a list of the same. Received " + f"'{type(sfname)} from '{sf_entry['pattern']}'." ) for d in cast( @@ -494,7 +536,12 @@ def addsf( }, ) elif sf_required: - raise WorkflowException( + raise SourceLine( + schema, + "secondaryFiles", + WorkflowException, + debug, + ).makeError( "Missing required secondary file '%s' from file object: %s" % (sfname, json_dumps(datum, indent=4)) ) @@ -504,10 +551,38 @@ def addsf( ) if "format" in schema: + eval_format: Any = self.do_eval(schema["format"]) + if isinstance(eval_format, str): + evaluated_format: Union[str, List[str]] = eval_format + elif isinstance(eval_format, MutableSequence): + for entry in eval_format: + if not isinstance(entry, str): + raise SourceLine( + schema, "format", WorkflowException, debug + ).makeError( + "An expression in the 'format' field must " + "evaluate to a string, or list of strings. " + "However a non-string item was received: " + f"'{entry}' of type '{type(entry)}'. " + f"The expression was '{schema['format']}' and " + f"its fully evaluated result is '{eval_format}'." + ) + evaluated_format = cast(List[str], eval_format) + else: + raise SourceLine( + schema, "format", WorkflowException, debug + ).makeError( + "An expression in the 'format' field must " + "evaluate to a string, or list of strings. " + "However the type of the expression result was " + f"{type(eval_format)}. " + f"The expression was '{schema['format']}' and " + f"its fully evaluated result is 'eval_format'." + ) try: check_format( datum, - cast(Union[List[str], str], self.do_eval(schema["format"])), + evaluated_format, self.formatgraph, ) except ValidationException as ve: @@ -561,12 +636,13 @@ def tostr(self, value: Union[MutableMapping[str, str], Any]) -> str: def generate_arg(self, binding: CWLObjectType) -> List[str]: value = binding.get("datum") + debug = _logger.isEnabledFor(logging.DEBUG) if "valueFrom" in binding: with SourceLine( binding, "valueFrom", WorkflowException, - _logger.isEnabledFor(logging.DEBUG), + debug, ): value = self.do_eval(cast(str, binding["valueFrom"]), context=value) @@ -577,7 +653,7 @@ def generate_arg(self, binding: CWLObjectType) -> List[str]: binding, "separate", WorkflowException, - _logger.isEnabledFor(logging.DEBUG), + debug, ): raise WorkflowException( "'separate' option can not be specified without prefix" diff --git a/cwltool/command_line_tool.py b/cwltool/command_line_tool.py index 054358f424..99f8b42189 100644 --- a/cwltool/command_line_tool.py +++ b/cwltool/command_line_tool.py @@ -41,10 +41,10 @@ from typing_extensions import TYPE_CHECKING, Type from .builder import ( + INPUT_OBJ_VOCAB, Builder, content_limit_respected_read_bytes, substitute, - INPUT_OBJ_VOCAB, ) from .context import LoadingContext, RuntimeContext, getdefault from .docker import DockerCommandLineJob @@ -65,7 +65,7 @@ from .singularity import SingularityCommandLineJob from .stdfsaccess import StdFsAccess from .udocker import UDockerCommandLineJob -from .update import ORDERED_VERSIONS +from .update import ORDERED_VERSIONS, ORIGINAL_CWLVERSION from .utils import ( CWLObjectType, CWLOutputType, @@ -325,9 +325,7 @@ def check_adjust( file_o["nameext"] = str(ne) if not accept_re.match(basename): raise WorkflowException( - "Invalid filename: '{}' contains illegal characters".format( - file_o["basename"] - ) + f"Invalid filename: '{file_o['basename']}' contains illegal characters" ) return file_o @@ -351,7 +349,7 @@ class ParameterOutputWorkflowException(WorkflowException): def __init__(self, msg: str, port: CWLObjectType, **kwargs: Any) -> None: """Exception for when there was an error collecting output for a parameter.""" super().__init__( - "Error collecting output for parameter '%s':\n%s" + "Error collecting output for parameter '%s': %s" % (shortname(cast(str, port["id"])), msg), kwargs, ) @@ -468,13 +466,87 @@ def _initialworkdir(self, j: JobBase, builder: Builder) -> None: initialWorkdir, _ = self.get_requirement("InitialWorkDirRequirement") if initialWorkdir is None: return + debug = _logger.isEnabledFor(logging.DEBUG) + cwl_version = cast(Optional[str], self.metadata.get(ORIGINAL_CWLVERSION, None)) + classic_dirent = cwl_version and ORDERED_VERSIONS.index( + cwl_version + ) < ORDERED_VERSIONS.index("v1.2.0-dev2") + classic_listing = cwl_version and ORDERED_VERSIONS.index( + cwl_version + ) < ORDERED_VERSIONS.index("v1.1.0-dev1") ls = [] # type: List[CWLObjectType] if isinstance(initialWorkdir["listing"], str): # "listing" is just a string (must be an expression) so # just evaluate it and use the result as if it was in # listing - ls = cast(List[CWLObjectType], builder.do_eval(initialWorkdir["listing"])) + ls_evaluated = builder.do_eval(initialWorkdir["listing"]) + fail: Any = False + fail_suffix: str = "" + if not isinstance(ls_evaluated, MutableSequence): + fail = ls_evaluated + else: + for entry in ls_evaluated: + if isinstance(entry, MutableSequence): + if classic_listing: + raise SourceLine( + initialWorkdir, "listing", WorkflowException, debug + ).makeError( + "InitialWorkDirRequirement.listing expressions " + "cannot return arrays of Files or Directories " + "before CWL v1.1. Please " + "considering using 'cwl-upgrader' to upgrade " + "your document to CWL v1.1' or later." + ) + else: + for entry2 in entry: + if not ( + isinstance(entry2, MutableMapping) + and ( + "class" in entry2 + and entry2["class"] == "File" + or "Directory" + ) + ): + fail = ( + "an array with an item ('{entry2}') that is " + "not a File nor a Directory object." + ) + elif not ( + ( + isinstance(entry, MutableMapping) + and ( + "class" in entry + and (entry["class"] == "File" or "Directory") + or "entry" in "entry" + ) + ) + ): + fail = entry + elif entry is None: + if classic_dirent: + fail = entry + fail_suffix = ( + " Dirent.entry cannot return 'null' before CWL " + "v1.2. Please consider using 'cwl-upgrader' to " + "upgrade your document to CWL version v1.2." + ) + if fail is not False: + message = ( + "Expression in a 'InitialWorkdirRequirement.listing' field " + "must return a list containing zero or more of: File or " + "Directory objects; Dirent objects" + ) + if classic_dirent: + message += ". " + else: + message += "; null; or arrays of File or Directory objects. " + message += f"Got '{fail}' among the results from " + message += f"'{initialWorkdir['listing'].strip()}'." + fail_suffix + raise SourceLine( + initialWorkdir, "listing", WorkflowException, debug + ).makeError(message) + ls = cast(List[CWLObjectType], ls_evaluated) else: # "listing" is an array of either expressions or Dirent so # evaluate each item @@ -484,16 +556,26 @@ def _initialworkdir(self, j: JobBase, builder: Builder) -> None: ): if isinstance(t, Mapping) and "entry" in t: # Dirent - entry = builder.do_eval( - cast(str, t["entry"]), strip_whitespace=False - ) + entry_field = cast(str, t["entry"]) + # the schema guarentees that 'entry' is a string, so the cast is safe + entry = builder.do_eval(entry_field, strip_whitespace=False) if entry is None: continue if isinstance(entry, MutableSequence): + if classic_listing: + raise SourceLine( + t, "entry", WorkflowException, debug + ).makeError( + "'entry' expressions are not allowed to evaluate " + "to an array of Files or Directories until CWL " + "v1.2. Consider using 'cwl-upgrader' to upgrade " + "your document to CWL version 1.2." + ) # Nested list. If it is a list of File or # Directory objects, add it to the - # file list, otherwise JSON serialize it. + # file list, otherwise JSON serialize it if CWL v1.2. + filelist = True for e in entry: if not isinstance(e, MutableMapping) or e.get( @@ -505,7 +587,7 @@ def _initialworkdir(self, j: JobBase, builder: Builder) -> None: if filelist: if "entryname" in t: raise SourceLine( - t, "entryname", WorkflowException + t, "entryname", WorkflowException, debug ).makeError( "'entryname' is invalid when 'entry' returns list of File or Directory" ) @@ -522,19 +604,39 @@ def _initialworkdir(self, j: JobBase, builder: Builder) -> None: ): et["entry"] = entry else: - et["entry"] = ( - entry - if isinstance(entry, str) - else json_dumps(entry, sort_keys=True) - ) + if isinstance(entry, str): + et["entry"] = entry + else: + if classic_dirent: + raise SourceLine( + t, "entry", WorkflowException, debug + ).makeError( + "'entry' expression resulted in " + "something other than number, object or " + "array besides a single File or Dirent object. " + "In CWL v1.2+ this would be serialized to a JSON object. " + "However this is a {cwl_version} document. " + "If that is the desired result then please " + "consider using 'cwl-upgrader' to upgrade " + "your document to CWL version 1.2. " + f"Result of '{entry_field}' was '{entry}'." + ) + et["entry"] = json_dumps(entry, sort_keys=True) if "entryname" in t: - en = builder.do_eval(cast(str, t["entryname"])) - if not isinstance(en, str): - raise SourceLine( - t, "entryname", WorkflowException - ).makeError("'entryname' must be a string") - et["entryname"] = en + entryname_field = cast(str, t["entryname"]) + if "${" in entryname_field or "$(" in entryname_field: + en = builder.do_eval(cast(str, t["entryname"])) + if not isinstance(en, str): + raise SourceLine( + t, "entryname", WorkflowException, debug + ).makeError( + "'entryname' expression must result a string. " + f"Got '{en}' from '{entryname_field}'" + ) + et["entryname"] = en + else: + et["entryname"] = entryname_field else: et["entryname"] = None et["writable"] = t.get("writable", False) @@ -553,7 +655,7 @@ def _initialworkdir(self, j: JobBase, builder: Builder) -> None: for i, t2 in enumerate(ls): if not isinstance(t2, Mapping): raise SourceLine( - initialWorkdir, "listing", WorkflowException + initialWorkdir, "listing", WorkflowException, debug ).makeError( "Entry at index %s of listing is not a record, was %s" % (i, type(t2)) @@ -566,7 +668,7 @@ def _initialworkdir(self, j: JobBase, builder: Builder) -> None: if isinstance(t2["entry"], str): if not t2["entryname"]: raise SourceLine( - initialWorkdir, "listing", WorkflowException + initialWorkdir, "listing", WorkflowException, debug ).makeError("Entry at index %s of listing missing entryname" % (i)) ls[i] = { "class": "File", @@ -578,7 +680,7 @@ def _initialworkdir(self, j: JobBase, builder: Builder) -> None: if not isinstance(t2["entry"], Mapping): raise SourceLine( - initialWorkdir, "listing", WorkflowException + initialWorkdir, "listing", WorkflowException, debug ).makeError( "Entry at index %s of listing is not a record, was %s" % (i, type(t2["entry"])) @@ -586,7 +688,7 @@ def _initialworkdir(self, j: JobBase, builder: Builder) -> None: if t2["entry"].get("class") not in ("File", "Directory"): raise SourceLine( - initialWorkdir, "listing", WorkflowException + initialWorkdir, "listing", WorkflowException, debug ).makeError( "Entry at index %s of listing is not a File or Directory object, was %s" % (i, t2) @@ -605,10 +707,10 @@ def _initialworkdir(self, j: JobBase, builder: Builder) -> None: if t3.get("class") not in ("File", "Directory"): # Check that every item is a File or Directory object now raise SourceLine( - initialWorkdir, "listing", WorkflowException + initialWorkdir, "listing", WorkflowException, debug ).makeError( - "Entry at index %s of listing is not a Dirent, File or Directory object, was %s" - % (i, t2) + f"Entry at index {i} of listing is not a Dirent, File or " + f"Directory object, was {t2}." ) if "basename" not in t3: continue @@ -616,36 +718,35 @@ def _initialworkdir(self, j: JobBase, builder: Builder) -> None: t3["basename"] = basename if basename.startswith("../"): raise SourceLine( - initialWorkdir, "listing", WorkflowException + initialWorkdir, "listing", WorkflowException, debug ).makeError( - "Name '%s' at index %s of listing is invalid, cannot start with '../'" - % (basename, i) + f"Name '{basename}' at index {i} of listing is invalid, " + "cannot start with '../'" ) if basename.startswith("/"): # only if DockerRequirement in requirements - cwl_version = self.metadata.get( - "http://commonwl.org/cwltool#original_cwlVersion", None - ) - if isinstance(cwl_version, str) and ORDERED_VERSIONS.index( + if cwl_version and ORDERED_VERSIONS.index( cwl_version ) < ORDERED_VERSIONS.index("v1.2.0-dev4"): raise SourceLine( - initialWorkdir, "listing", WorkflowException + initialWorkdir, "listing", WorkflowException, debug ).makeError( - "Name '%s' at index %s of listing is invalid, paths starting with '/' only permitted in CWL 1.2 and later" - % (basename, i) + f"Name '{basename}' at index {i} of listing is invalid, " + "paths starting with '/' only permitted in CWL 1.2 " + "and later." ) req, is_req = self.get_requirement("DockerRequirement") if is_req is not True: raise SourceLine( - initialWorkdir, "listing", WorkflowException + initialWorkdir, "listing", WorkflowException, debug ).makeError( - "Name '%s' at index %s of listing is invalid, name can only start with '/' when DockerRequirement is in 'requirements'" - % (basename, i) + f"Name '{basename}' at index {i} of listing is invalid, " + "name can only start with '/' when DockerRequirement " + "is in 'requirements'." ) - with SourceLine(initialWorkdir, "listing", WorkflowException): + with SourceLine(initialWorkdir, "listing", WorkflowException, debug): j.generatefiles["listing"] = ls for entry in ls: if "basename" in entry: @@ -902,13 +1003,25 @@ def update_status_output_callback( if self.tool.get("stdin"): with SourceLine(self.tool, "stdin", ValidationException, debug): - j.stdin = cast(str, builder.do_eval(self.tool["stdin"])) + stdin_eval = builder.do_eval(self.tool["stdin"]) + if not (isinstance(stdin_eval, str) or stdin_eval is None): + raise ValidationException( + f"'stdin' expression must return a string or null. Got '{stdin_eval}' " + f"for '{self.tool['stdin']}'." + ) + j.stdin = stdin_eval if j.stdin: reffiles.append({"class": "File", "path": j.stdin}) if self.tool.get("stderr"): with SourceLine(self.tool, "stderr", ValidationException, debug): - j.stderr = cast(str, builder.do_eval(self.tool["stderr"])) + stderr_eval = builder.do_eval(self.tool["stderr"]) + if not isinstance(stderr_eval, str): + raise ValidationException( + f"'stderr' expression must return a string. Got '{stderr_eval}' " + f"for '{self.tool['stderr']}'." + ) + j.stderr = stderr_eval if j.stderr: if os.path.isabs(j.stderr) or ".." in j.stderr: raise ValidationException( @@ -917,7 +1030,13 @@ def update_status_output_callback( if self.tool.get("stdout"): with SourceLine(self.tool, "stdout", ValidationException, debug): - j.stdout = cast(str, builder.do_eval(self.tool["stdout"])) + stdout_eval = builder.do_eval(self.tool["stdout"]) + if not isinstance(stdout_eval, str): + raise ValidationException( + f"'stdout' expression must return a string. Got '{stdout_eval}' " + f"for '{self.tool['stdout']}'." + ) + j.stdout = stdout_eval if j.stdout: if os.path.isabs(j.stdout) or ".." in j.stdout or not j.stdout: raise ValidationException( @@ -977,35 +1096,60 @@ def register_reader(f: CWLObjectType) -> None: timelimit, _ = self.get_requirement("ToolTimeLimit") if timelimit is not None: with SourceLine(timelimit, "timelimit", ValidationException, debug): - j.timelimit = cast( - Optional[int], - builder.do_eval(cast(Union[int, str], timelimit["timelimit"])), - ) - if not isinstance(j.timelimit, int) or j.timelimit < 0: + limit_field = cast(Dict[str, Union[str, int]], timelimit)["timelimit"] + if isinstance(limit_field, str): + timelimit_eval = builder.do_eval(limit_field) + if timelimit_eval and not isinstance(timelimit_eval, int): + raise WorkflowException( + "'timelimit' expression must evaluate to a long/int. Got " + f"'{timelimit_eval}' for expression '{limit_field}'." + ) + else: + timelimit_eval = limit_field + if not isinstance(timelimit_eval, int) or timelimit_eval < 0: raise WorkflowException( - "timelimit must be an integer >= 0, got: %s" % j.timelimit + f"timelimit must be an integer >= 0, got: {timelimit_eval}" ) + j.timelimit = timelimit_eval networkaccess, _ = self.get_requirement("NetworkAccess") if networkaccess is not None: with SourceLine(networkaccess, "networkAccess", ValidationException, debug): - j.networkaccess = cast( - bool, - builder.do_eval( - cast(Union[bool, str], networkaccess["networkAccess"]) - ), - ) - if not isinstance(j.networkaccess, bool): + networkaccess_field = networkaccess["networkAccess"] + if isinstance(networkaccess_field, str): + networkaccess_eval = builder.do_eval(networkaccess_field) + if not isinstance(networkaccess_eval, bool): + raise WorkflowException( + "'networkAccess' expression must evaluate to a bool. " + f"Got '{networkaccess_eval}' for expression '{networkaccess_field}'." + ) + else: + networkaccess_eval = networkaccess_field + if not isinstance(networkaccess_eval, bool): raise WorkflowException( - "networkAccess must be a boolean, got: %s" % j.networkaccess + "networkAccess must be a boolean, got: {networkaccess_eval}." ) + j.networkaccess = networkaccess_eval # Build a mapping to hold any EnvVarRequirement required_env = {} evr, _ = self.get_requirement("EnvVarRequirement") if evr is not None: - for t3 in cast(List[Dict[str, str]], evr["envDef"]): - required_env[t3["envName"]] = cast(str, builder.do_eval(t3["envValue"])) + for eindex, t3 in enumerate(cast(List[Dict[str, str]], evr["envDef"])): + env_value_field = t3["envValue"] + if "${" in env_value_field or "$(" in env_value_field: + env_value_eval = builder.do_eval(env_value_field) + if not isinstance(env_value_eval, str): + raise SourceLine( + evr["envDef"], eindex, WorkflowException, debug + ).makeError( + "'envValue expression must evaluate to a str. " + f"Got '{env_value_eval}' for expression '{env_value_field}'." + ) + env_value = env_value_eval + else: + env_value = env_value_field + required_env[t3["envName"]] = env_value # Construct the env j.prepare_environment(runtimeContext, required_env) @@ -1040,14 +1184,15 @@ def register_reader(f: CWLObjectType) -> None: mpi.get("processes", runtimeContext.mpi_config.default_nproc), ) if isinstance(np, str): - tmp = builder.do_eval(np) - if not isinstance(tmp, int): - raise TypeError( - "{} needs 'processes' to evaluate to an int, got {}".format( - MPIRequirementName, type(np) - ) + np_eval = builder.do_eval(np) + if not isinstance(np_eval, int): + raise SourceLine( + mpi, "processes", WorkflowException, debug + ).makeError( + f"{MPIRequirementName} needs 'processes' expression to " + f"evaluate to an int, got '{np_eval}' for expression '{np}'." ) - np = tmp + np = np_eval j.mpi_procs = np yield j @@ -1063,9 +1208,7 @@ def collect_output_ports( ) -> OutputPortsType: ret = {} # type: OutputPortsType debug = _logger.isEnabledFor(logging.DEBUG) - cwl_version = self.metadata.get( - "http://commonwl.org/cwltool#original_cwlVersion", None - ) + cwl_version = self.metadata.get(ORIGINAL_CWLVERSION, None) if cwl_version != "v1.0": builder.resources["exitCode"] = rcode try: @@ -1162,6 +1305,20 @@ def collect_output( for gb in aslist(binding["glob"]): gb = builder.do_eval(gb) if gb: + gb_eval_fail = False + if not isinstance(gb, str): + if isinstance(gb, list): + for entry in gb: + if not isinstance(entry, str): + gb_eval_fail = True + else: + gb_eval_fail = True + if gb_eval_fail: + raise WorkflowException( + "Resolved glob patterns must be strings " + f"or list of strings, not " + f"'{gb}' from '{binding['glob']}'" + ) globpatterns.extend(aslist(gb)) for gb in globpatterns: @@ -1261,22 +1418,20 @@ def collect_output( result = cast(CWLOutputType, r) if single: - if not result and not optional: - with SourceLine(binding, "glob", WorkflowException, debug): - raise WorkflowException( - "Did not find output file with glob pattern: '{}'".format( - globpatterns - ) - ) - elif not result and optional: - pass - elif isinstance(result, MutableSequence): - if len(result) > 1: + with SourceLine(binding, "glob", WorkflowException, debug): + if not result and not optional: raise WorkflowException( - "Multiple matches for output item that is a single file." + f"Did not find output file with glob pattern: '{globpatterns}'." ) - else: - result = cast(CWLOutputType, result[0]) + elif not result and optional: + pass + elif isinstance(result, MutableSequence): + if len(result) > 1: + raise WorkflowException( + "Multiple matches for output item that is a single file." + ) + else: + result = cast(CWLOutputType, result[0]) if "secondaryFiles" in schema: with SourceLine(schema, "secondaryFiles", WorkflowException, debug): @@ -1288,9 +1443,27 @@ def collect_output( ] for sf in aslist(schema["secondaryFiles"]): if "required" in sf: - sf_required = builder.do_eval( - sf["required"], context=primary - ) + with SourceLine( + schema["secondaryFiles"], + "required", + WorkflowException, + debug, + ): + sf_required_eval = builder.do_eval( + sf["required"], context=primary + ) + if not ( + isinstance(sf_required_eval, bool) + or sf_required_eval is None + ): + raise WorkflowException( + "Expressions in the field " + "'required' must evaluate to a " + "Boolean (true or false) or None. " + f"Got '{sf_required_eval}' for " + f"'{sf['required']}'." + ) + sf_required: bool = sf_required_eval or False else: sf_required = False @@ -1326,11 +1499,24 @@ def collect_output( primary["secondaryFiles"].append(sfitem) if "format" in schema: - for primary in aslist(result): - primary["format"] = builder.do_eval( - schema["format"], context=primary - ) - + format_field = cast(str, schema["format"]) + if "$(" in format_field or "${" in format_field: + for index, primary in enumerate(aslist(result)): + format_eval = builder.do_eval(format_field, context=primary) + if not isinstance(format_eval, str): + message = ( + f"'format' expression must evaluate to a string. " + f"Got '{format_eval}' from '{format_field}'." + ) + if isinstance(result, list): + message += f" 'self' had the value of the index {index} result: '{primary}'." + raise SourceLine( + schema, "format", WorkflowException, debug + ).makeError(message) + primary["format"] = format_eval + else: + for primary in aslist(result): + primary["format"] = format_field # Ensure files point to local references outside of the run environment adjustFileObjs(result, revmap) diff --git a/cwltool/executors.py b/cwltool/executors.py index 83817737e8..c04c78b0cb 100644 --- a/cwltool/executors.py +++ b/cwltool/executors.py @@ -32,6 +32,7 @@ from .process import Process, cleanIntermediate, relocateOutputs from .provenance_profile import ProvenanceProfile from .task_queue import TaskQueue +from .update import ORIGINAL_CWLVERSION from .utils import CWLObjectType, JobsType from .workflow import Workflow from .workflow_job import WorkflowJob, WorkflowJobStep @@ -88,9 +89,9 @@ def execute( def check_for_abstract_op(tool: CWLObjectType) -> None: if tool["class"] == "Operation": - raise SourceLine(tool, "class", WorkflowException).makeError( - "Workflow has unrunnable abstract Operation" - ) + raise SourceLine( + tool, "class", WorkflowException, runtime_context.debug + ).makeError("Workflow has unrunnable abstract Operation") process.visit(check_for_abstract_op) @@ -108,10 +109,7 @@ def check_for_abstract_op(tool: CWLObjectType) -> None: job_reqs = None # type: Optional[List[CWLObjectType]] if "https://w3id.org/cwl/cwl#requirements" in job_order_object: - if ( - process.metadata.get("http://commonwl.org/cwltool#original_cwlVersion") - == "v1.0" - ): + if process.metadata.get(ORIGINAL_CWLVERSION) == "v1.0": raise WorkflowException( "`cwl:requirements` in the input object is not part of CWL " "v1.0. You can adjust to use `cwltool:overrides` instead; or you " @@ -126,10 +124,7 @@ def check_for_abstract_op(tool: CWLObjectType) -> None: and "https://w3id.org/cwl/cwl#requirements" in cast(CWLObjectType, process.metadata["cwl:defaults"]) ): - if ( - process.metadata.get("http://commonwl.org/cwltool#original_cwlVersion") - == "v1.0" - ): + if process.metadata.get(ORIGINAL_CWLVERSION) == "v1.0": raise WorkflowException( "`cwl:requirements` in the input object is not part of CWL " "v1.0. You can adjust to use `cwltool:overrides` instead; or you " diff --git a/cwltool/job.py b/cwltool/job.py index 0f9eb131b4..fd74d937f6 100644 --- a/cwltool/job.py +++ b/cwltool/job.py @@ -3,9 +3,9 @@ import itertools import logging import os -import stat import re import shutil +import stat import subprocess # nosec import sys import tempfile @@ -729,6 +729,7 @@ def run( runtimeContext: RuntimeContext, tmpdir_lock: Optional[threading.Lock] = None, ) -> None: + debug = runtimeContext.debug if tmpdir_lock: with tmpdir_lock: if not os.path.exists(self.tmpdir): @@ -753,22 +754,18 @@ def run( try: subprocess.check_call(cmd, stdout=sys.stderr) # nosec except OSError: - raise WorkflowException( - SourceLine(docker_req).makeError( - "Either Docker container {} is not available with " - "user space docker implementation {} or {} is missing " - "or broken.".format( - img_id, user_space_docker_cmd, user_space_docker_cmd - ) - ) + raise SourceLine( + docker_req, None, WorkflowException, debug + ).makeError( + f"Either Docker container {img_id} is not available with " + f"user space docker implementation {user_space_docker_cmd} " + f" or {user_space_docker_cmd} is missing or broken." ) else: - raise WorkflowException( - SourceLine(docker_req).makeError( - "Docker image must be specified as 'dockerImageId' or " - "'dockerPull' when using user space implementations of " - "Docker" - ) + raise SourceLine(docker_req, None, WorkflowException, debug).makeError( + "Docker image must be specified as 'dockerImageId' or " + "'dockerPull' when using user space implementations of " + "Docker" ) else: try: diff --git a/cwltool/load_tool.py b/cwltool/load_tool.py index e4842a02e1..2361ef6d99 100644 --- a/cwltool/load_tool.py +++ b/cwltool/load_tool.py @@ -312,7 +312,9 @@ def resolve_and_validate_document( ) if not isinstance(cwlVersion, str): - with SourceLine(workflowobj, "cwlVersion", ValidationException): + with SourceLine( + workflowobj, "cwlVersion", ValidationException, loadingContext.debug + ): raise ValidationException( f"'cwlVersion' must be a string, got {type(cwlVersion)}" ) diff --git a/cwltool/pack.py b/cwltool/pack.py index b0976a8aa7..71ca53a6a9 100644 --- a/cwltool/pack.py +++ b/cwltool/pack.py @@ -21,7 +21,7 @@ from .context import LoadingContext from .load_tool import fetch_document, resolve_and_validate_document from .process import shortname, uniquename -from .update import ORDERED_VERSIONS, update +from .update import ORDERED_VERSIONS, ORIGINAL_CWLVERSION, update from .utils import CWLObjectType, CWLOutputType LoadRefType = Callable[[Optional[str], str], ResolveType] @@ -243,10 +243,10 @@ def rewrite_id(r: str, mainuri: str) -> None: update_to_version, ) - if "http://commonwl.org/cwltool#original_cwlVersion" in metadata: - del metadata["http://commonwl.org/cwltool#original_cwlVersion"] - if "http://commonwl.org/cwltool#original_cwlVersion" in dcr: - del dcr["http://commonwl.org/cwltool#original_cwlVersion"] + if ORIGINAL_CWLVERSION in metadata: + del metadata[ORIGINAL_CWLVERSION] + if ORIGINAL_CWLVERSION in dcr: + del dcr[ORIGINAL_CWLVERSION] if "$schemas" in metadata: for s in metadata["$schemas"]: diff --git a/cwltool/process.py b/cwltool/process.py index 88c23c08c0..9079e29b0c 100644 --- a/cwltool/process.py +++ b/cwltool/process.py @@ -44,12 +44,12 @@ from schema_salad.ref_resolver import Loader, file_uri, uri_file_path from schema_salad.schema import load_schema, make_avro_schema, make_valid_avro from schema_salad.sourceline import SourceLine, strip_dup_lineno -from schema_salad.utils import ContextType, convert_to_dict -from schema_salad.validate import validate_ex, avro_type_name +from schema_salad.utils import convert_to_dict +from schema_salad.validate import avro_type_name, validate_ex from typing_extensions import TYPE_CHECKING from . import expression -from .builder import Builder, HasReqsHints, INPUT_OBJ_VOCAB +from .builder import INPUT_OBJ_VOCAB, Builder, HasReqsHints from .context import LoadingContext, RuntimeContext, getdefault from .errors import UnsupportedRequirement, WorkflowException from .loghandler import _logger @@ -57,7 +57,7 @@ from .pathmapper import MapperEnt, PathMapper from .secrets import SecretStore from .stdfsaccess import StdFsAccess -from .update import INTERNAL_VERSION +from .update import INTERNAL_VERSION, ORIGINAL_CWLVERSION from .utils import ( CWLObjectType, CWLOutputAtomType, @@ -236,13 +236,14 @@ def checkRequirements( ) -> None: if isinstance(rec, MutableMapping): if "requirements" in rec: + debug = _logger.isEnabledFor(logging.DEBUG) for i, entry in enumerate( cast(MutableSequence[CWLObjectType], rec["requirements"]) ): - with SourceLine(rec["requirements"], i, UnsupportedRequirement): + with SourceLine(rec["requirements"], i, UnsupportedRequirement, debug): if cast(str, entry["class"]) not in supported_process_requirements: raise UnsupportedRequirement( - "Unsupported requirement {}".format(entry["class"]) + f"Unsupported requirement {entry['class']}." ) for key in rec: checkRequirements(rec[key], supported_process_requirements) @@ -438,10 +439,9 @@ def fill_in_defaults( job: CWLObjectType, fsaccess: StdFsAccess, ) -> None: + debug = _logger.isEnabledFor(logging.DEBUG) for e, inp in enumerate(inputs): - with SourceLine( - inputs, e, WorkflowException, _logger.isEnabledFor(logging.DEBUG) - ): + with SourceLine(inputs, e, WorkflowException, debug): fieldname = shortname(cast(str, inp["id"])) if job.get(fieldname) is not None: pass @@ -581,14 +581,15 @@ def __init__( self.names = make_avro_schema([SCHEMA_FILE, SCHEMA_DIR, SCHEMA_ANY], Loader({})) self.tool = toolpath_object + debug = loadingContext.debug self.requirements = copy.deepcopy(getdefault(loadingContext.requirements, [])) tool_requirements = self.tool.get("requirements", []) if tool_requirements is None: - raise ValidationException( - SourceLine(self.tool, "requirements").makeError( - "If 'requirements' is present then it must be a list " - "or map/dictionary, not empty." - ) + raise SourceLine( + self.tool, "requirements", ValidationException, debug + ).makeError( + "If 'requirements' is present then it must be a list " + "or map/dictionary, not empty." ) self.requirements.extend(tool_requirements) if "id" not in self.tool: @@ -604,11 +605,9 @@ def __init__( self.hints = copy.deepcopy(getdefault(loadingContext.hints, [])) tool_hints = self.tool.get("hints", []) if tool_hints is None: - raise ValidationException( - SourceLine(self.tool, "hints").makeError( - "If 'hints' is present then it must be a list " - "or map/dictionary, not empty." - ) + raise SourceLine(self.tool, "hints", ValidationException, debug).makeError( + "If 'hints' is present then it must be a list " + "or map/dictionary, not empty." ) self.hints.extend(tool_hints) # Versions of requirements and hints which aren't mutated. @@ -685,13 +684,13 @@ def __init__( List[CWLObjectType], self.outputs_record_schema["fields"] ).append(c) - with SourceLine(toolpath_object, "inputs", ValidationException): + with SourceLine(toolpath_object, "inputs", ValidationException, debug): self.inputs_record_schema = cast( CWLObjectType, make_valid_avro(self.inputs_record_schema, {}, set()), ) make_avsc_object(convert_to_dict(self.inputs_record_schema), self.names) - with SourceLine(toolpath_object, "outputs", ValidationException): + with SourceLine(toolpath_object, "outputs", ValidationException, debug): self.outputs_record_schema = cast( CWLObjectType, make_valid_avro(self.outputs_record_schema, {}, set()), @@ -879,7 +878,7 @@ def inc(d): # type: (List[int]) -> None cwl_version = cast( str, - self.metadata.get("http://commonwl.org/cwltool#original_cwlVersion", None), + self.metadata.get(ORIGINAL_CWLVERSION, None), ) builder = Builder( job, @@ -971,9 +970,7 @@ def evalResources( resourceReq, _ = self.get_requirement("ResourceRequirement") if resourceReq is None: resourceReq = {} - cwl_version = self.metadata.get( - "http://commonwl.org/cwltool#original_cwlVersion", None - ) + cwl_version = self.metadata.get(ORIGINAL_CWLVERSION, None) if cwl_version == "v1.0": ram = 1024 else: @@ -1033,8 +1030,9 @@ def validate_hints( ) -> None: if self.doc_loader is None: return + debug = _logger.isEnabledFor(logging.DEBUG) for i, r in enumerate(hints): - sl = SourceLine(hints, i, ValidationException) + sl = SourceLine(hints, i, ValidationException, debug) with sl: classname = cast(str, r["class"]) avroname = classname diff --git a/cwltool/provenance_profile.py b/cwltool/provenance_profile.py index 0a39e45d2d..4475e49be1 100644 --- a/cwltool/provenance_profile.py +++ b/cwltool/provenance_profile.py @@ -59,13 +59,9 @@ def copy_job_order( return job_order_object customised_job = {} # type: CWLObjectType # new job object for RO + debug = _logger.isEnabledFor(logging.DEBUG) for each, i in enumerate(job.tool["inputs"]): - with SourceLine( - job.tool["inputs"], - each, - WorkflowException, - _logger.isEnabledFor(logging.DEBUG), - ): + with SourceLine(job.tool["inputs"], each, WorkflowException, debug): iid = shortname(i["id"]) if iid in job_order_object: customised_job[iid] = copy.deepcopy(job_order_object[iid]) diff --git a/cwltool/singularity.py b/cwltool/singularity.py index 3c02c5dcfd..ea87acd144 100644 --- a/cwltool/singularity.py +++ b/cwltool/singularity.py @@ -1,5 +1,6 @@ """Support for executing Docker containers using the Singularity 2.x engine.""" +import logging import os import os.path import re @@ -52,12 +53,13 @@ def _singularity_supports_userns() -> bool: def get_version() -> str: global _SINGULARITY_VERSION # pylint: disable=global-statement - if not _SINGULARITY_VERSION: + if _SINGULARITY_VERSION == "": _SINGULARITY_VERSION = check_output( # nosec ["singularity", "--version"], universal_newlines=True - ) + ).strip() if _SINGULARITY_VERSION.startswith("singularity version "): _SINGULARITY_VERSION = _SINGULARITY_VERSION[20:] + _logger.debug(f"Singularity version: {_SINGULARITY_VERSION}.") return _SINGULARITY_VERSION @@ -120,6 +122,8 @@ def get_image( candidates = [] cache_folder = None + debug = _logger.isEnabledFor(logging.DEBUG) + if "CWL_SINGULARITY_CACHE" in os.environ: cache_folder = os.environ["CWL_SINGULARITY_CACHE"] elif is_version_2_6() and "SINGULARITY_PULLFOLDER" in os.environ: @@ -216,11 +220,11 @@ def get_image( found = True elif "dockerFile" in dockerRequirement: - raise WorkflowException( - SourceLine(dockerRequirement, "dockerFile").makeError( - "dockerFile is not currently supported when using the " - "Singularity runtime for Docker containers." - ) + raise SourceLine( + dockerRequirement, "dockerFile", WorkflowException, debug + ).makeError( + "dockerFile is not currently supported when using the " + "Singularity runtime for Docker containers." ) elif "dockerLoad" in dockerRequirement: if is_version_3_1_or_newer(): @@ -238,18 +242,18 @@ def get_image( check_call(cmd, stdout=sys.stderr) # nosec found = True dockerRequirement["dockerImageId"] = name - raise WorkflowException( - SourceLine(dockerRequirement, "dockerLoad").makeError( - "dockerLoad is not currently supported when using the " - "Singularity runtime (version less than 3.1) for Docker containers." - ) + raise SourceLine( + dockerRequirement, "dockerLoad", WorkflowException, debug + ).makeError( + "dockerLoad is not currently supported when using the " + "Singularity runtime (version less than 3.1) for Docker containers." ) elif "dockerImport" in dockerRequirement: - raise WorkflowException( - SourceLine(dockerRequirement, "dockerImport").makeError( - "dockerImport is not currently supported when using the " - "Singularity runtime for Docker containers." - ) + raise SourceLine( + dockerRequirement, "dockerImport", WorkflowException, debug + ).makeError( + "dockerImport is not currently supported when using the " + "Singularity runtime for Docker containers." ) return found @@ -349,36 +353,39 @@ def add_writable_directory_volume( ) -> None: if volume.resolved.startswith("_:"): # Synthetic directory that needs creating first - if host_outdir_tgt is not None: - new_dir = host_outdir_tgt - else: + if not host_outdir_tgt: new_dir = os.path.join( create_tmp_dir(tmpdir_prefix), os.path.basename(volume.target), ) self.append_volume(runtime, new_dir, volume.target, writable=True) - os.makedirs(new_dir) + os.makedirs(new_dir) + # ^^ Unlike Docker, Singularity won't create directories on demand + elif not os.path.exists(host_outdir_tgt): + os.makedirs(host_outdir_tgt) else: if host_outdir_tgt is not None and not is_version_3_4_or_newer(): # workaround for lack of overlapping mounts in Singularity < 3.4 shutil.copytree(volume.resolved, host_outdir_tgt) ensure_writable(host_outdir_tgt) else: - if not self.inplace_update: + if self.inplace_update: + self.append_volume( + runtime, volume.resolved, volume.target, writable=True + ) + else: if not host_outdir_tgt: tmpdir = create_tmp_dir(tmpdir_prefix) new_dir = os.path.join( tmpdir, os.path.basename(volume.resolved) ) shutil.copytree(volume.resolved, new_dir) - source = tmpdir + self.append_volume( + runtime, new_dir, volume.target, writable=True + ) else: shutil.copytree(volume.resolved, host_outdir_tgt) - source = host_outdir_tgt - else: - source = volume.resolved - self.append_volume(runtime, source, volume.target, writable=True) - ensure_writable(source) + ensure_writable(host_outdir_tgt or new_dir) def _required_env(self) -> Dict[str, str]: return { diff --git a/cwltool/update.py b/cwltool/update.py index 3976265f21..e60c696f9d 100644 --- a/cwltool/update.py +++ b/cwltool/update.py @@ -319,7 +319,7 @@ def update( cdoc["cwlVersion"] = version metadata["cwlVersion"] = version - metadata["http://commonwl.org/cwltool#original_cwlVersion"] = originalversion - cdoc["http://commonwl.org/cwltool#original_cwlVersion"] = originalversion + metadata[ORIGINAL_CWLVERSION] = originalversion + cdoc[ORIGINAL_CWLVERSION] = originalversion return cdoc diff --git a/cwltool/validate_js.py b/cwltool/validate_js.py index 226c0f686c..64e0df8972 100644 --- a/cwltool/validate_js.py +++ b/cwltool/validate_js.py @@ -63,6 +63,7 @@ def get_expressions( schema: Optional[Union[Schema, ArraySchema]], source_line: Optional[SourceLine] = None, ) -> List[Tuple[str, Optional[SourceLine]]]: + debug = _logger.isEnabledFor(logging.DEBUG) if is_expression(tool, schema): return [(cast(str, tool), source_line)] elif isinstance(schema, UnionSchema): @@ -90,7 +91,9 @@ def tmp_expr( ) -> List[Tuple[str, Optional[SourceLine]]]: # using a lambda for this broke mypyc v0.910 and before return get_expressions( - x[1], cast(ArraySchema, schema).items, SourceLine(tool, x[0]) + x[1], + cast(ArraySchema, schema).items, + SourceLine(tool, x[0], include_traceback=debug), ) return list( @@ -114,7 +117,7 @@ def tmp_expr( get_expressions( tool[schema_field.name], schema_field.type, - SourceLine(tool, schema_field.name), + SourceLine(tool, schema_field.name, include_traceback=debug), ) ) @@ -214,7 +217,7 @@ def validate_js_expressions( if tool.get("requirements") is None: return - + debug = _logger.isEnabledFor(logging.DEBUG) requirements = tool["requirements"] default_globals = ["self", "inputs", "runtime", "console"] @@ -234,7 +237,8 @@ def validate_js_expressions( ) js_globals.extend(expression_lib_line_globals) print_js_hint_messages( - expression_lib_line_errors, SourceLine(expression_lib, i) + expression_lib_line_errors, + SourceLine(expression_lib, i, include_traceback=debug), ) expressions = get_expressions(tool, schema) diff --git a/cwltool/workflow.py b/cwltool/workflow.py index 3b70ba5d38..1ed31ac7dd 100644 --- a/cwltool/workflow.py +++ b/cwltool/workflow.py @@ -198,6 +198,7 @@ def __init__( parentworkflowProv: Optional[ProvenanceProfile] = None, ) -> None: """Initialize this WorkflowStep.""" + debug = loadingContext.debug if "id" in toolpath_object: self.id = toolpath_object["id"] else: @@ -280,12 +281,18 @@ def __init__( else: step_entry_name = step_entry validation_errors.append( - SourceLine(self.tool["out"], index).makeError( + SourceLine( + self.tool["out"], index, include_traceback=debug + ).makeError( "Workflow step output '%s' does not correspond to" % shortname(step_entry_name) ) + "\n" - + SourceLine(self.embedded_tool.tool, "outputs").makeError( + + SourceLine( + self.embedded_tool.tool, + "outputs", + include_traceback=debug, + ).makeError( " tool output (expected '%s')" % ( "', '".join( @@ -313,7 +320,7 @@ def __init__( if missing_values: validation_errors.append( - SourceLine(self.tool, "in").makeError( + SourceLine(self.tool, "in", include_traceback=debug).makeError( "Step is missing required parameter%s '%s'" % ( "s" if len(missing_values) > 1 else "", @@ -356,14 +363,14 @@ def __init__( inp_map = {i["id"]: i for i in inputparms} for inp in scatter: if inp not in inp_map: - raise ValidationException( - SourceLine(self.tool, "scatter").makeError( - "Scatter parameter '%s' does not correspond to " - "an input parameter of this step, expecting '%s'" - % ( - shortname(inp), - "', '".join(shortname(k) for k in inp_map.keys()), - ) + SourceLine( + self.tool, "scatter", ValidationException, debug + ).makeError( + "Scatter parameter '%s' does not correspond to " + "an input parameter of this step, expecting '%s'" + % ( + shortname(inp), + "', '".join(shortname(k) for k in inp_map.keys()), ) ) diff --git a/cwltool/workflow_job.py b/cwltool/workflow_job.py index e67747caea..d49614cb28 100644 --- a/cwltool/workflow_job.py +++ b/cwltool/workflow_job.py @@ -800,14 +800,10 @@ def job( runtimeContext = runtimeContext.copy() runtimeContext.outdir = None + debug = runtimeContext.debug for index, inp in enumerate(self.tool["inputs"]): - with SourceLine( - self.tool["inputs"], - index, - WorkflowException, - _logger.isEnabledFor(logging.DEBUG), - ): + with SourceLine(self.tool["inputs"], index, WorkflowException, debug): inp_id = shortname(inp["id"]) if inp_id in joborder: self.state[inp["id"]] = WorkflowStateItem( diff --git a/tests/echo-badposition-expr.cwl b/tests/echo-badposition-expr.cwl new file mode 100644 index 0000000000..f7d94b4e20 --- /dev/null +++ b/tests/echo-badposition-expr.cwl @@ -0,0 +1,28 @@ +#!/usr/bin/env cwl-runner + +class: CommandLineTool +cwlVersion: v1.1 +requirements: + InlineJavascriptRequirement: {} +inputs: + one: + type: int + inputBinding: + position: $(self) + two: + type: int + inputBinding: + valueFrom: sensation! + position: ${return "two";} +arguments: + - position: ${return 2;} + valueFrom: singular +outputs: + out: + type: string + outputBinding: + glob: out.txt + loadContents: true + outputEval: $(self[0].contents) +baseCommand: echo +stdout: out.txt diff --git a/tests/iwdr_bad_expr.cwl b/tests/iwdr_bad_expr.cwl new file mode 100644 index 0000000000..8ba9eb0c02 --- /dev/null +++ b/tests/iwdr_bad_expr.cwl @@ -0,0 +1,24 @@ +#!/usr/bin/env cwl-runner +class: CommandLineTool +cwlVersion: v1.0 + +requirements: + DockerRequirement: + dockerPull: debian:stable-slim + InlineJavascriptRequirement: {} + InitialWorkDirRequirement: + listing: | + ${ + return [{"class": "Directory", + "basename": "subdir", + "listing": [ inputs.example ] + }], 42} + +inputs: + example: File + +outputs: + same: + type: File + outputBinding: + glob: subdir/$(inputs.example.basename) diff --git a/tests/secondary-files-bad.cwl b/tests/secondary-files-bad.cwl new file mode 100644 index 0000000000..8bf3157dce --- /dev/null +++ b/tests/secondary-files-bad.cwl @@ -0,0 +1,36 @@ +#!/usr/bin/env cwl-runner + +cwlVersion: v1.1 + +requirements: + - class: InlineJavascriptRequirement + - class: ShellCommandRequirement + +class: CommandLineTool + +inputs: + fasta_path: + type: File + secondaryFiles: + - pattern: ^.fastq + required: true + - pattern: .crai + required: "${ return 42 }" + - .bai? + - pattern: "${ return null }" + +outputs: + bai_list: + type: File + outputBinding: + glob: lsout + secondaryFiles: + - .bai? + - pattern: "${ return null }" + - pattern: .crai + required: false + - pattern: .idx + +baseCommand: ["ls"] + +stdout: lsout diff --git a/tests/secondary-files.cwl b/tests/secondary-files.cwl index 04849d5d95..3fb913790c 100644 --- a/tests/secondary-files.cwl +++ b/tests/secondary-files.cwl @@ -1,6 +1,6 @@ #!/usr/bin/env cwl-runner -cwlVersion: v1.1.0-dev1 +cwlVersion: v1.1 requirements: - class: InlineJavascriptRequirement @@ -33,4 +33,4 @@ outputs: baseCommand: ["ls"] -stdout: lsout \ No newline at end of file +stdout: lsout diff --git a/tests/test_examples.py b/tests/test_examples.py index 1485aa93d3..1deca04fb7 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -1,6 +1,7 @@ import json import logging import os +import re import stat import subprocess import sys @@ -887,6 +888,33 @@ def test_separate_without_prefix() -> None: factory.make(get_data("tests/wf/separate_without_prefix.cwl"))() +def test_glob_expr_error(tmp_path: Path) -> None: + """Better glob expression error.""" + error_code, _, stderr = get_main_output( + [get_data("tests/wf/1496.cwl"), "--index", str(tmp_path)] + ) + assert error_code != 0 + stderr = re.sub(r"\s\s+", " ", stderr) + assert "Resolved glob patterns must be strings" in stderr + + +def test_format_expr_error() -> None: + """Better format expression error.""" + error_code, _, stderr = get_main_output( + [ + get_data("tests/wf/bad_formattest.cwl"), + get_data("tests/wf/formattest-job.json"), + ] + ) + assert error_code != 0 + stderr = re.sub(r"\s\s+", " ", stderr) + assert ( + "An expression in the 'format' field must evaluate to a string, or list " + "of strings. However a non-string item was received: '42' of " + "type ''." in stderr + ) + + def test_static_checker() -> None: # check that the static checker raises exception when a source type # mismatches its sink type. @@ -1139,7 +1167,7 @@ def test_secondary_files_v1_1(factor: str) -> None: commands = factor.split() commands.extend( [ - "--enable-dev", + "--debug", get_data(os.path.join("tests", test_file)), get_data(os.path.join("tests", test_job_file)), ] @@ -1153,6 +1181,28 @@ def test_secondary_files_v1_1(factor: str) -> None: assert error_code == 0 +@needs_docker +@pytest.mark.parametrize("factor", test_factors) +def test_secondary_files_bad_v1_1(factor: str) -> None: + """Affirm the correct error message for a bad secondaryFiles expression.""" + test_file = "secondary-files-bad.cwl" + test_job_file = "secondary-files-job.yml" + commands = factor.split() + commands.extend( + [ + get_data(os.path.join("tests", test_file)), + get_data(os.path.join("tests", test_job_file)), + ] + ) + error_code, _, stderr = get_main_output(commands) + stderr = re.sub(r"\s\s+", " ", stderr) + assert ( + "The result of a expression in the field 'required' must be a bool " + "or None, not a ." in stderr + ), stderr + assert error_code == 1 + + @needs_docker @pytest.mark.parametrize("factor", test_factors) def test_secondary_files_v1_0(factor: str) -> None: @@ -1238,6 +1288,57 @@ def test_compute_checksum() -> None: assert result["checksum"] == "sha1$327fc7aedf4f6b69a42a7c8b808dc5a7aff61376" +def test_bad_stdin_expr_error() -> None: + """Confirm that a bad stdin expression gives a useful error.""" + error_code, _, stderr = get_main_output( + [ + get_data("tests/wf/bad-stdin-expr.cwl"), + "--file1", + get_data("tests/wf/whale.txt"), + ] + ) + assert error_code == 1 + stderr = re.sub(r"\s\s+", " ", stderr) + assert ( + "'stdin' expression must return a string or null. Got '1111' for '$(inputs.file1.size)'." + in stderr + ) + + +def test_bad_stderr_expr_error() -> None: + """Confirm that a bad stderr expression gives a useful error.""" + error_code, _, stderr = get_main_output( + [ + get_data("tests/wf/bad-stderr-expr.cwl"), + "--file1", + get_data("tests/wf/whale.txt"), + ] + ) + assert error_code == 1 + stderr = re.sub(r"\s\s+", " ", stderr) + assert ( + "'stderr' expression must return a string. Got '1111' for '$(inputs.file1.size)'." + in stderr + ) + + +def test_bad_stdout_expr_error() -> None: + """Confirm that a bad stdout expression gives a useful error.""" + error_code, _, stderr = get_main_output( + [ + get_data("tests/wf/bad-stdout-expr.cwl"), + "--file1", + get_data("tests/wf/whale.txt"), + ] + ) + assert error_code == 1 + stderr = re.sub(r"\s\s+", " ", stderr) + assert ( + "'stdout' expression must return a string. Got '1111' for '$(inputs.file1.size)'." + in stderr + ) + + @needs_docker @pytest.mark.parametrize("factor", test_factors) def test_no_compute_chcksum(tmp_path: Path, factor: str) -> None: @@ -1309,6 +1410,19 @@ def test_v1_0_position_expression(factor: str) -> None: assert error_code == 1 +@pytest.mark.parametrize("factor", test_factors) +def test_v1_1_position_badexpression(factor: str) -> None: + """Test for the correct error for a bad position expression.""" + test_file = "tests/echo-badposition-expr.cwl" + test_job = "tests/echo-position-expr-job.yml" + commands = factor.split() + commands.extend(["--debug", get_data(test_file), get_data(test_job)]) + error_code, _, stderr = get_main_output(commands) + stderr = re.sub(r"\s\s+", " ", stderr) + assert "expressions must evaluate to an int" in stderr, stderr + assert error_code == 1 + + @pytest.mark.parametrize("factor", test_factors) def test_optional_numeric_output_0(factor: str) -> None: test_file = "tests/wf/optional-numerical-output-0.cwl" @@ -1427,3 +1541,33 @@ def test_arguments_self() -> None: outputs["self_review"]["checksum"] == "sha1$724ba28f4a9a1b472057ff99511ed393a45552e1" ) + + +def test_bad_timelimit_expr() -> None: + """Confirm error message for bad timelimit expression.""" + err_code, _, stderr = get_main_output( + [ + get_data("tests/wf/bad_timelimit.cwl"), + ] + ) + stderr = re.sub(r"\s\s+", " ", stderr) + assert ( + "'timelimit' expression must evaluate to a long/int. " + "Got '42' for expression '${return \"42\";}" in stderr + ) + assert err_code == 1 + + +def test_bad_networkaccess_expr() -> None: + """Confirm error message for bad networkaccess expression.""" + err_code, _, stderr = get_main_output( + [ + get_data("tests/wf/bad_networkaccess.cwl"), + ] + ) + stderr = re.sub(r"\s\s+", " ", stderr) + assert ( + "'networkAccess' expression must evaluate to a bool. " + "Got '42' for expression '${return 42;}" in stderr + ) + assert err_code == 1 diff --git a/tests/test_iwdr.py b/tests/test_iwdr.py index de43a86972..48712da230 100644 --- a/tests/test_iwdr.py +++ b/tests/test_iwdr.py @@ -1,5 +1,8 @@ +"""InitialWorkDirRequirement related tests.""" +import json +import re from pathlib import Path -from stat import S_IWRITE, S_IWGRP, S_IWOTH +from stat import S_IWGRP, S_IWOTH, S_IWRITE from typing import Any from cwltool.factory import Factory @@ -36,13 +39,36 @@ def test_directory_literal_with_real_inputs_inside(tmp_path: Path) -> None: assert err_code == 0 +def test_bad_listing_expression(tmp_path: Path) -> None: + """Confirm better error message for bad listing expression.""" + err_code, _, stderr = get_main_output( + [ + "--out", + str(tmp_path), + get_data("tests/iwdr_bad_expr.cwl"), + "--example={}".format(get_data("tests/__init__.py")), + ] + ) + stderr = re.sub(r"\s\s+", " ", stderr) + assert ( + "Expression in a 'InitialWorkdirRequirement.listing' field must return " + "a list containing zero or more of: File or Directory objects; Dirent " + "objects. Got '42' among the results" in stderr + ) + assert err_code == 1 + + @needs_docker def test_iwdr_permutations(tmp_path_factory: Any) -> None: misc = tmp_path_factory.mktemp("misc") - fifth = str(tmp_path_factory.mktemp("fifth")) - sixth = str(tmp_path_factory.mktemp("sixth")) - seventh = str(tmp_path_factory.mktemp("seventh")) - eighth = str(tmp_path_factory.mktemp("eighth")) + fifth = misc / "fifth" + fifth.mkdir() + sixth = misc / "sixth" + sixth.mkdir() + seventh = misc / "seventh" + seventh.mkdir() + eighth = misc / "eighth" + eighth.mkdir() first = misc / "first" first.touch() second = misc / "second" @@ -51,40 +77,51 @@ def test_iwdr_permutations(tmp_path_factory: Any) -> None: third.touch() fourth = misc / "fourth" fourth.touch() + eleventh = misc / "eleventh" + eleventh.touch() + twelfth = misc / "twelfth" + twelfth.touch() outdir = str(tmp_path_factory.mktemp("outdir")) - assert ( - main( - [ - "--outdir", - outdir, - "--enable-dev", - get_data("tests/wf/iwdr_permutations.cwl"), - "--first", - str(first), - "--second", - str(second), - "--third", - str(third), - "--fourth", - str(fourth), - "--fifth", - fifth, - "--sixth", - sixth, - "--seventh", - seventh, - "--eighth", - eighth, - ] - ) - == 0 + err_code, stdout, _ = get_main_output( + [ + "--outdir", + outdir, + "--debug", + get_data("tests/wf/iwdr_permutations.cwl"), + "--first", + str(first), + "--second", + str(second), + "--third", + str(third), + "--fourth", + str(fourth), + "--fifth", + str(fifth), + "--sixth", + str(sixth), + "--seventh", + str(seventh), + "--eighth", + str(eighth), + "--eleventh", + str(eleventh), + "--eleventh", + str(twelfth), + ] ) + assert err_code == 0 + log = json.loads(stdout)["log"] + assert log["checksum"] == "sha1$bc51ebb3f65ca44282789dd1e6de9747d8abe75f", log def test_iwdr_permutations_readonly(tmp_path_factory: Any) -> None: """Confirm that readonly input files are properly made writable.""" misc = tmp_path_factory.mktemp("misc") - fifth = tmp_path_factory.mktemp("fifth") + fifth = misc / "fifth" + fifth.mkdir() + sixth = misc / "sixth" + sixth.mkdir() fifth_file = fifth / "bar" fifth_dir = fifth / "foo" fifth_file.touch() @@ -131,10 +168,14 @@ def test_iwdr_permutations_readonly(tmp_path_factory: Any) -> None: @needs_docker def test_iwdr_permutations_inplace(tmp_path_factory: Any) -> None: misc = tmp_path_factory.mktemp("misc") - fifth = str(tmp_path_factory.mktemp("fifth")) - sixth = str(tmp_path_factory.mktemp("sixth")) - seventh = str(tmp_path_factory.mktemp("seventh")) - eighth = str(tmp_path_factory.mktemp("eighth")) + fifth = misc / "fifth" + fifth.mkdir() + sixth = misc / "sixth" + sixth.mkdir() + seventh = misc / "seventh" + seventh.mkdir() + eighth = misc / "eighth" + eighth.mkdir() first = misc / "first" first.touch() second = misc / "second" @@ -143,46 +184,57 @@ def test_iwdr_permutations_inplace(tmp_path_factory: Any) -> None: third.touch() fourth = misc / "fourth" fourth.touch() + eleventh = misc / "eleventh" + eleventh.touch() + twelfth = misc / "twelfth" + twelfth.touch() outdir = str(tmp_path_factory.mktemp("outdir")) - assert ( - main( - [ - "--outdir", - outdir, - "--enable-ext", - "--enable-dev", - "--overrides", - get_data("tests/wf/iwdr_permutations_inplace.yml"), - get_data("tests/wf/iwdr_permutations.cwl"), - "--first", - str(first), - "--second", - str(second), - "--third", - str(third), - "--fourth", - str(fourth), - "--fifth", - fifth, - "--sixth", - sixth, - "--seventh", - seventh, - "--eighth", - eighth, - ] - ) - == 0 + err_code, stdout, _ = get_main_output( + [ + "--outdir", + outdir, + "--enable-ext", + "--overrides", + get_data("tests/wf/iwdr_permutations_inplace.yml"), + get_data("tests/wf/iwdr_permutations.cwl"), + "--first", + str(first), + "--second", + str(second), + "--third", + str(third), + "--fourth", + str(fourth), + "--fifth", + str(fifth), + "--sixth", + str(sixth), + "--seventh", + str(seventh), + "--eighth", + str(eighth), + "--eleventh", + str(eleventh), + "--eleventh", + str(twelfth), + ] ) + assert err_code == 0 + log = json.loads(stdout)["log"] + assert log["checksum"] == "sha1$bc51ebb3f65ca44282789dd1e6de9747d8abe75f", log @needs_singularity def test_iwdr_permutations_singularity(tmp_path_factory: Any) -> None: misc = tmp_path_factory.mktemp("misc") - fifth = str(tmp_path_factory.mktemp("fifth")) - sixth = str(tmp_path_factory.mktemp("sixth")) - seventh = str(tmp_path_factory.mktemp("seventh")) - eighth = str(tmp_path_factory.mktemp("eighth")) + fifth = misc / "fifth" + fifth.mkdir() + sixth = misc / "sixth" + sixth.mkdir() + seventh = misc / "seventh" + seventh.mkdir() + eighth = misc / "eighth" + eighth.mkdir() first = misc / "first" first.touch() second = misc / "second" @@ -191,44 +243,57 @@ def test_iwdr_permutations_singularity(tmp_path_factory: Any) -> None: third.touch() fourth = misc / "fourth" fourth.touch() + eleventh = misc / "eleventh" + eleventh.touch() + twelfth = misc / "twelfth" + twelfth.touch() outdir = str(tmp_path_factory.mktemp("outdir")) - assert ( - main( - [ - "--outdir", - outdir, - "--enable-dev", - "--singularity", - get_data("tests/wf/iwdr_permutations.cwl"), - "--first", - str(first), - "--second", - str(second), - "--third", - str(third), - "--fourth", - str(fourth), - "--fifth", - fifth, - "--sixth", - sixth, - "--seventh", - seventh, - "--eighth", - eighth, - ] - ) - == 0 + err_code, stdout, _ = get_main_output( + [ + "--outdir", + outdir, + "--debug", + "--singularity", + get_data("tests/wf/iwdr_permutations.cwl"), + "--first", + str(first), + "--second", + str(second), + "--third", + str(third), + "--fourth", + str(fourth), + "--fifth", + str(fifth), + "--sixth", + str(sixth), + "--seventh", + str(seventh), + "--eighth", + str(eighth), + "--eleventh", + str(eleventh), + "--eleventh", + str(twelfth), + ] ) + assert err_code == 0 + log = json.loads(stdout)["log"] + assert log["checksum"] == "sha1$bc51ebb3f65ca44282789dd1e6de9747d8abe75f", log @needs_singularity def test_iwdr_permutations_singularity_inplace(tmp_path_factory: Any) -> None: + """IWDR tests using --singularity and a forced InplaceUpdateRequirement.""" misc = tmp_path_factory.mktemp("misc") - fifth = str(tmp_path_factory.mktemp("fifth")) - sixth = str(tmp_path_factory.mktemp("sixth")) - seventh = str(tmp_path_factory.mktemp("seventh")) - eighth = str(tmp_path_factory.mktemp("eighth")) + fifth = misc / "fifth" + fifth.mkdir() + sixth = misc / "sixth" + sixth.mkdir() + seventh = misc / "seventh" + seventh.mkdir() + eighth = misc / "eighth" + eighth.mkdir() first = misc / "first" first.touch() second = misc / "second" @@ -237,6 +302,10 @@ def test_iwdr_permutations_singularity_inplace(tmp_path_factory: Any) -> None: third.touch() fourth = misc / "fourth" fourth.touch() + eleventh = misc / "eleventh" + eleventh.touch() + twelfth = misc / "twelfth" + twelfth.touch() outdir = str(tmp_path_factory.mktemp("outdir")) assert ( main( @@ -258,13 +327,17 @@ def test_iwdr_permutations_singularity_inplace(tmp_path_factory: Any) -> None: "--fourth", str(fourth), "--fifth", - fifth, + str(fifth), "--sixth", - sixth, + str(sixth), "--seventh", - seventh, + str(seventh), "--eighth", - eighth, + str(eighth), + "--eleventh", + str(eleventh), + "--eleventh", + str(twelfth), ] ) == 0 diff --git a/tests/test_pack.py b/tests/test_pack.py index fea55aa9ad..7319bf9933 100644 --- a/tests/test_pack.py +++ b/tests/test_pack.py @@ -8,6 +8,7 @@ from typing import Any, Dict, List, cast import pytest +from schema_salad.utils import yaml_no_ts import cwltool.pack import cwltool.workflow @@ -17,8 +18,6 @@ from cwltool.resolver import tool_resolver from cwltool.utils import adjustDirObjs, adjustFileObjs -from schema_salad.utils import yaml_no_ts - from .util import get_data, needs_docker diff --git a/tests/test_schemadef.py b/tests/test_schemadef.py index 10c0e87b29..224ebe89f4 100644 --- a/tests/test_schemadef.py +++ b/tests/test_schemadef.py @@ -1,6 +1,7 @@ """Tests related SchemaDefRequirement.""" from cwltool.main import main + from .util import get_data diff --git a/tests/test_streaming.py b/tests/test_streaming.py index e3d3ec759a..3c5526592d 100644 --- a/tests/test_streaming.py +++ b/tests/test_streaming.py @@ -1,10 +1,9 @@ """Test that files marked as 'streamable' when 'streaming_allowed' can be named pipes.""" import os - -import pytest from pathlib import Path from typing import cast +import pytest from ruamel.yaml.comments import CommentedMap from schema_salad.sourceline import cmap @@ -12,8 +11,9 @@ from cwltool.context import LoadingContext, RuntimeContext from cwltool.errors import WorkflowException from cwltool.job import JobBase -from cwltool.update import INTERNAL_VERSION +from cwltool.update import INTERNAL_VERSION, ORIGINAL_CWLVERSION from cwltool.utils import CWLObjectType + from .util import get_data toolpath_object = cast( @@ -39,7 +39,7 @@ { "metadata": { "cwlVersion": INTERNAL_VERSION, - "http://commonwl.org/cwltool#original_cwlVersion": INTERNAL_VERSION, + ORIGINAL_CWLVERSION: INTERNAL_VERSION, } } ) diff --git a/tests/test_tmpdir.py b/tests/test_tmpdir.py index ab35578425..90387b31ee 100644 --- a/tests/test_tmpdir.py +++ b/tests/test_tmpdir.py @@ -16,7 +16,7 @@ from cwltool.job import JobBase from cwltool.pathmapper import MapperEnt, PathMapper from cwltool.stdfsaccess import StdFsAccess -from cwltool.update import INTERNAL_VERSION +from cwltool.update import INTERNAL_VERSION, ORIGINAL_CWLVERSION from cwltool.utils import create_tmp_dir from .util import get_data, needs_docker @@ -28,7 +28,7 @@ def test_docker_commandLineTool_job_tmpdir_prefix(tmp_path: Path) -> None: { "metadata": { "cwlVersion": INTERNAL_VERSION, - "http://commonwl.org/cwltool#original_cwlVersion": INTERNAL_VERSION, + ORIGINAL_CWLVERSION: INTERNAL_VERSION, } } ) @@ -70,7 +70,7 @@ def test_commandLineTool_job_tmpdir_prefix(tmp_path: Path) -> None: { "metadata": { "cwlVersion": INTERNAL_VERSION, - "http://commonwl.org/cwltool#original_cwlVersion": INTERNAL_VERSION, + ORIGINAL_CWLVERSION: INTERNAL_VERSION, } } ) diff --git a/tests/test_validate_js.py b/tests/test_validate_js.py index 34f68a6dc3..4b965d4d27 100644 --- a/tests/test_validate_js.py +++ b/tests/test_validate_js.py @@ -1,5 +1,6 @@ from typing import Any +import pytest from schema_salad.avro.schema import Names from schema_salad.utils import yaml_no_ts @@ -35,18 +36,21 @@ def test_get_expressions() -> None: assert len(exprs) == 1 -def test_validate_js_expressions(mocker: Any) -> None: +def test_validate_js_expressions(caplog: pytest.LogCaptureFixture) -> None: + """Test invalid JS expression.""" yaml = yaml_no_ts() test_cwl_yaml = yaml.load(TEST_CWL) schema = process.get_schema("v1.0")[1] assert isinstance(schema, Names) clt_schema = schema.names["org.w3id.cwl.cwl.CommandLineTool"] - mocker.patch("cwltool.validate_js._logger") - # mocker.patch("cwltool.validate_js.print_js_hint_messages") validate_js.validate_js_expressions(test_cwl_yaml, clt_schema) - validate_js._logger.warning.assert_called_with(" JSHINT: (function(){return ((kjdbfkjd));})()\n JSHINT: ^\n JSHINT: W117: 'kjdbfkjd' is not defined.") # type: ignore + assert ( + " JSHINT: (function(){return ((kjdbfkjd));})()\n" + " JSHINT: ^\n" + " JSHINT: W117: 'kjdbfkjd' is not defined." + ) in caplog.text def test_js_hint_basic() -> None: diff --git a/tests/wf/1496.cwl b/tests/wf/1496.cwl new file mode 100644 index 0000000000..74f6ef49e3 --- /dev/null +++ b/tests/wf/1496.cwl @@ -0,0 +1,15 @@ +cwlVersion: v1.2 +class: CommandLineTool + +baseCommand: echo + +inputs: + index: + type: Directory + inputBinding: {} + +outputs: + salmon_index: + type: Directory + outputBinding: + glob: "$(inputs.index)" # not a valid glob, result needs to be a string, not a Directory object diff --git a/tests/wf/bad-stderr-expr.cwl b/tests/wf/bad-stderr-expr.cwl new file mode 100644 index 0000000000..cfb1f06d9b --- /dev/null +++ b/tests/wf/bad-stderr-expr.cwl @@ -0,0 +1,18 @@ +#!/usr/bin/env cwl-runner + +class: CommandLineTool +cwlVersion: v1.0 + +inputs: + file1: File + +outputs: + output: + type: File + outputBinding: { glob: output } + +baseCommand: [cat] + +stdin: $(inputs.file1.path) +stdout: output +stderr: $(inputs.file1.size) diff --git a/tests/wf/bad-stdin-expr.cwl b/tests/wf/bad-stdin-expr.cwl new file mode 100644 index 0000000000..b3638c68f5 --- /dev/null +++ b/tests/wf/bad-stdin-expr.cwl @@ -0,0 +1,17 @@ +#!/usr/bin/env cwl-runner + +class: CommandLineTool +cwlVersion: v1.0 + +inputs: + file1: File + +outputs: + output: + type: File + outputBinding: { glob: output } + +baseCommand: [cat] + +stdin: $(inputs.file1.size) +stdout: output diff --git a/tests/wf/bad-stdout-expr.cwl b/tests/wf/bad-stdout-expr.cwl new file mode 100644 index 0000000000..29c109675d --- /dev/null +++ b/tests/wf/bad-stdout-expr.cwl @@ -0,0 +1,17 @@ +#!/usr/bin/env cwl-runner + +class: CommandLineTool +cwlVersion: v1.0 + +inputs: + file1: File + +outputs: + output: + type: File + outputBinding: { glob: output } + +baseCommand: [cat] + +stdin: $(inputs.file1.path) +stdout: $(inputs.file1.size) diff --git a/tests/wf/bad_formattest.cwl b/tests/wf/bad_formattest.cwl new file mode 100755 index 0000000000..6d936f60d8 --- /dev/null +++ b/tests/wf/bad_formattest.cwl @@ -0,0 +1,25 @@ +#!/usr/bin/env cwl-runner +$namespaces: + edam: "http://edamontology.org/" +cwlVersion: v1.0 +class: CommandLineTool +requirements: + InlineJavascriptRequirement: {} +doc: "Reverse each line using the `rev` command" +inputs: + input: + type: File + inputBinding: {} + format: | + ${ return ["http://edamontology.org/format_2330", 42];} + +outputs: + output: + type: File + outputBinding: + glob: output.txt + format: | + ${return "http://edamontology.org/format_2330";} + +baseCommand: rev +stdout: output.txt diff --git a/tests/wf/bad_networkaccess.cwl b/tests/wf/bad_networkaccess.cwl new file mode 100644 index 0000000000..e30c81a5ac --- /dev/null +++ b/tests/wf/bad_networkaccess.cwl @@ -0,0 +1,10 @@ +#!/usr/bin/env cwl-runner +class: CommandLineTool +cwlVersion: v1.1 +requirements: + InlineJavascriptRequirement: {} + NetworkAccess: + networkAccess: '${return 42;}' +inputs: [] +outputs: [] +baseCommand: echo, Hello, World! diff --git a/tests/wf/bad_timelimit.cwl b/tests/wf/bad_timelimit.cwl new file mode 100644 index 0000000000..1dd0308bc5 --- /dev/null +++ b/tests/wf/bad_timelimit.cwl @@ -0,0 +1,14 @@ +#!/usr/bin/env cwl-runner +class: CommandLineTool +cwlVersion: v1.1 +inputs: + sleep_time: + type: int + default: 3 + inputBinding: {} +outputs: [] +requirements: + InlineJavascriptRequirement: {} + ToolTimeLimit: + timelimit: '${return "42";}' +baseCommand: sleep diff --git a/tests/wf/iwdr_permutations.cwl b/tests/wf/iwdr_permutations.cwl index b838cc2ade..bf154875ef 100755 --- a/tests/wf/iwdr_permutations.cwl +++ b/tests/wf/iwdr_permutations.cwl @@ -1,7 +1,10 @@ #!/usr/bin/env cwl-runner class: CommandLineTool -cwlVersion: v1.2.0-dev4 +cwlVersion: v1.2 requirements: + EnvVarRequirement: + envDef: + LC_ALL: C DockerRequirement: dockerPull: debian InitialWorkDirRequirement: @@ -36,6 +39,7 @@ requirements: - entry: $(inputs.tenth) entryname: /my_path/tenth_writable_directory_literal writable: true + - entry: $(inputs.eleventh) # array of Files - entry: baz entryname: /my_path/my_file_literal inputs: @@ -59,18 +63,24 @@ inputs: class: Directory basename: bar listing: [] + eleventh: File[] outputs: out: type: Directory outputBinding: glob: . + log: stdout +stdout: log.txt baseCommand: [bash, -c] arguments: - | - find . - find /my_path - find /my_other_path + find . | sort + find /my_path | sort + find /my_other_path | sort echo "a" > first_writable_file echo "b" > /my_path/third_writable_file touch fifth_writable_directory/c touch /my_path/seventh_writable_directory/d + find . | sort + find /my_path | sort + find /my_other_path | sort From 5a1488b5544c838e6e652205c3c4d9cb0c1a7041 Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" Date: Fri, 3 Sep 2021 12:22:03 +0200 Subject: [PATCH 108/111] subgraph: support long version of WorkflowStep[].out --- Makefile | 2 +- cwltool/subgraph.py | 2 + tests/subgraph/1432.cwl | 29 ++++++++++++++ tests/subgraph/extract_step2_1432.json | 35 +++++++++++++++++ tests/test_subgraph.py | 52 +++++++++++++++----------- tests/test_validate_js.py | 2 - 6 files changed, 97 insertions(+), 25 deletions(-) create mode 100644 tests/subgraph/1432.cwl create mode 100644 tests/subgraph/extract_step2_1432.json diff --git a/Makefile b/Makefile index 0e07cff3e1..dbe02adebc 100644 --- a/Makefile +++ b/Makefile @@ -177,7 +177,7 @@ mypy: $(filter-out setup.py gittagger.py,$(PYSOURCES)) mypyc: $(PYSOURCES) MYPYPATH=typeshed CWLTOOL_USE_MYPYC=1 pip install --verbose -e . \ - && pytest ${PYTEST_EXTRA} + && pytest -vv ${PYTEST_EXTRA} shellcheck: FORCE shellcheck build-cwltool-docker.sh cwl-docker.sh release-test.sh conformance-test.sh \ diff --git a/cwltool/subgraph.py b/cwltool/subgraph.py index 21da423116..ec4537fc20 100644 --- a/cwltool/subgraph.py +++ b/cwltool/subgraph.py @@ -92,6 +92,8 @@ def get_subgraph(roots: MutableSequence[str], tool: Workflow) -> CommentedMap: declare_node(nodes, src, None) nodes[src].down.append(st["id"]) for out in st["out"]: + if isinstance(out, Mapping) and "id" in out: + out = out["id"] # output is downstream from step step.down.append(out) # step is upstream from output diff --git a/tests/subgraph/1432.cwl b/tests/subgraph/1432.cwl new file mode 100644 index 0000000000..54d553875a --- /dev/null +++ b/tests/subgraph/1432.cwl @@ -0,0 +1,29 @@ +class: Workflow +cwlVersion: v1.2 +inputs: + - id: step1_in + type: string + - id: step2_in + type: string +outputs: + - id: step1_out + type: string + outputSource: step1/out + - id: step2_out + type: string + outputSource: step2/out +steps: + - id: step1 + in: + - id: inp + source: step1_in + out: + - id: out + run: ../echo.cwl + - id: step2 + in: + - id: inp + source: step2_in + out: + - id: out + run: ../echo.cwl diff --git a/tests/subgraph/extract_step2_1432.json b/tests/subgraph/extract_step2_1432.json new file mode 100644 index 0000000000..00bd87499a --- /dev/null +++ b/tests/subgraph/extract_step2_1432.json @@ -0,0 +1,35 @@ +{ + "class": "Workflow", + "cwlVersion": "v1.2", + "id": "subgraph/1432.cwl", + "inputs": [ + { + "id": "subgraph/1432.cwl#step2_in", + "type": "string" + } + ], + "outputs": [ + { + "id": "subgraph/1432.cwl#step2_out", + "outputSource": "subgraph/1432.cwl#step2/out", + "type": "string" + } + ], + "steps": [ + { + "id": "subgraph/1432.cwl#step2", + "in": [ + { + "id": "subgraph/1432.cwl#step2/inp", + "source": "subgraph/1432.cwl#step2_in" + } + ], + "out": [ + { + "id": "subgraph/1432.cwl#step2/out" + } + ], + "run": "echo.cwl" + } + ] +} diff --git a/tests/test_subgraph.py b/tests/test_subgraph.py index bae66b7ab4..ee4cb837d8 100644 --- a/tests/test_subgraph.py +++ b/tests/test_subgraph.py @@ -12,7 +12,20 @@ from .util import get_data +def clean(val: Any, path: str) -> Any: + """Remove the path prefix from an string values.""" + if isinstance(val, str): + if val.startswith(path): + return val[len(path) + 1 :] + if isinstance(val, dict): + return {k: clean(v, path) for k, v in val.items()} + if isinstance(val, list): + return [clean(v, path) for v in val] + return val + + def test_get_subgraph() -> None: + """Compare known correct subgraphs to generated subgraphs.""" loadingContext = LoadingContext({"construct_tool_object": default_make_tool}) wf = Path(get_data("tests/subgraph/count-lines1-wf.cwl")).as_uri() loadingContext.do_update = False @@ -20,16 +33,6 @@ def test_get_subgraph() -> None: sg = Path(get_data("tests/subgraph")).as_uri() - def clean(val: Any) -> Any: - if isinstance(val, str): - if val.startswith(sg): - return val[len(sg) + 1 :] - if isinstance(val, dict): - return {k: clean(v) for k, v in val.items()} - if isinstance(val, list): - return [clean(v) for v in val] - return val - for a in ( "file1", "file2", @@ -47,7 +50,22 @@ def clean(val: Any) -> Any: assert isinstance(tool, Workflow) extracted = get_subgraph([wf + "#" + a], tool) with open(get_data("tests/subgraph/extract_" + a + ".json")) as f: - assert json.load(f) == clean(convert_to_dict(extracted)) + assert json.load(f) == clean(convert_to_dict(extracted), sg) + + +def test_get_subgraph_long_out_form() -> None: + """Compare subgraphs generatation when 'out' is in the long form.""" + loadingContext = LoadingContext({"construct_tool_object": default_make_tool}) + wf = Path(get_data("tests/subgraph/1432.cwl")).as_uri() + loadingContext.do_update = False + tool = load_tool(wf, loadingContext) + + sg = Path(get_data("tests/")).as_uri() + + assert isinstance(tool, Workflow) + extracted = get_subgraph([wf + "#step2"], tool) + with open(get_data("tests/subgraph/extract_step2_1432.json")) as f: + assert json.load(f) == clean(convert_to_dict(extracted), sg) def test_get_step() -> None: @@ -59,16 +77,6 @@ def test_get_step() -> None: sg = Path(get_data("tests/subgraph")).as_uri() - def clean(val: Any) -> Any: - if isinstance(val, str): - if val.startswith(sg): - return val[len(sg) + 1 :] - if isinstance(val, dict): - return {k: clean(v) for k, v in val.items()} - if isinstance(val, list): - return [clean(v) for v in val] - return val - for a in ( "step1", "step2", @@ -78,4 +86,4 @@ def clean(val: Any) -> Any: ): extracted = get_step(tool, wf + "#" + a) with open(get_data("tests/subgraph/single_" + a + ".json")) as f: - assert json.load(f) == clean(convert_to_dict(extracted)) + assert json.load(f) == clean(convert_to_dict(extracted), sg) diff --git a/tests/test_validate_js.py b/tests/test_validate_js.py index 4b965d4d27..7047d1dc3f 100644 --- a/tests/test_validate_js.py +++ b/tests/test_validate_js.py @@ -1,5 +1,3 @@ -from typing import Any - import pytest from schema_salad.avro.schema import Names from schema_salad.utils import yaml_no_ts From cd9fde37005fd728fba755956ad3e26c922bb8cf Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" <1330696+mr-c@users.noreply.github.com> Date: Sat, 4 Sep 2021 17:52:17 +0200 Subject: [PATCH 109/111] pack: add $namespaces to the (efffective) root only (#1506) As per schema-salad v1.x, $namespaces are only valid in the root document: https://www.commonwl.org/v1.0/SchemaSalad.html#Explicit_context https://www.commonwl.org/v1.1/SchemaSalad.html#Explicit_context (same as v1.0) https://www.commonwl.org/v1.2/SchemaSalad.html#Explicit_context (same as v1.0, v1.1) Inspired by (and partial fix for) common-workflow-language/cwl-utils#71 (comment) --- cwltool/pack.py | 13 +++++++------ tests/test_pack.py | 7 ++++--- tests/wf/expect_packed.cwl | 10 +++++----- tests/wf/expect_revsort_datetime_packed.cwl | 14 +++++++------- tests/wf/expect_trick_packed.cwl | 12 ++++++------ tests/wf/scatter2_subwf.cwl | 5 ++++- 6 files changed, 33 insertions(+), 28 deletions(-) diff --git a/cwltool/pack.py b/cwltool/pack.py index 71ca53a6a9..93d0abb92f 100644 --- a/cwltool/pack.py +++ b/cwltool/pack.py @@ -268,6 +268,8 @@ def rewrite_id(r: str, mainuri: str) -> None: if schemas: packed["$schemas"] = list(schemas) + if namespaces: + packed["$namespaces"] = namespaces for r in list(rewrite.keys()): v = rewrite[r] @@ -276,14 +278,13 @@ def rewrite_id(r: str, mainuri: str) -> None: import_embed(packed, set()) if len(packed["$graph"]) == 1: - # duplicate 'cwlVersion' and $schemas inside $graph when there is only - # a single item because we will print the contents inside '$graph' - # rather than whole dict + # duplicate 'cwlVersion', '$schemas', and '$namespaces' inside '$graph' + # when there is only a single item because main.print_pack() will print + # the contents inside '$graph' rather than whole dict in this case packed["$graph"][0]["cwlVersion"] = packed["cwlVersion"] if schemas: packed["$graph"][0]["$schemas"] = list(schemas) - # always include $namespaces in the #main - if namespaces: - packed["$graph"][0]["$namespaces"] = namespaces + if namespaces: + packed["$graph"][0]["$namespaces"] = namespaces return packed diff --git a/tests/test_pack.py b/tests/test_pack.py index 7319bf9933..7fa6e796dc 100644 --- a/tests/test_pack.py +++ b/tests/test_pack.py @@ -94,9 +94,10 @@ def test_pack_fragment() -> None: ) adjustDirObjs(packed, partial(make_relative, os.path.abspath(get_data("tests/wf")))) - assert json.dumps(packed, sort_keys=True, indent=2) == json.dumps( - expect_packed, sort_keys=True, indent=2 - ) + packed_result = json.dumps(packed, sort_keys=True, indent=2) + expected = json.dumps(expect_packed, sort_keys=True, indent=2) + + assert packed_result == expected def test_pack_rewrites() -> None: diff --git a/tests/wf/expect_packed.cwl b/tests/wf/expect_packed.cwl index dedc7c04bf..a816e4d2cc 100644 --- a/tests/wf/expect_packed.cwl +++ b/tests/wf/expect_packed.cwl @@ -67,10 +67,7 @@ "id": "#main/sorted" } ], - "id": "#main", - "$namespaces": { - "iana": "https://www.iana.org/assignments/media-types/" - } + "id": "#main" }, { "class": "CommandLineTool", @@ -133,5 +130,8 @@ "$schemas": [ "empty.ttl", "empty2.ttl" - ] + ], + "$namespaces": { + "iana": "https://www.iana.org/assignments/media-types/" + } } diff --git a/tests/wf/expect_revsort_datetime_packed.cwl b/tests/wf/expect_revsort_datetime_packed.cwl index 0c3ed98e3d..172a030b04 100644 --- a/tests/wf/expect_revsort_datetime_packed.cwl +++ b/tests/wf/expect_revsort_datetime_packed.cwl @@ -68,11 +68,7 @@ } ], "id": "#main", - "http://schema.org/dateCreated": "2020-10-08", - "$namespaces": { - "iana": "https://www.iana.org/assignments/media-types/", - "s": "http://schema.org/" - } + "http://schema.org/dateCreated": "2020-10-08" }, { "class": "CommandLineTool", @@ -136,5 +132,9 @@ "empty2.ttl", "https://schema.org/version/latest/schemaorg-current-https.rdf", "empty.ttl" - ] -} \ No newline at end of file + ], + "$namespaces": { + "iana": "https://www.iana.org/assignments/media-types/", + "s": "http://schema.org/" + } +} diff --git a/tests/wf/expect_trick_packed.cwl b/tests/wf/expect_trick_packed.cwl index b946546390..1b2a0f937a 100644 --- a/tests/wf/expect_trick_packed.cwl +++ b/tests/wf/expect_trick_packed.cwl @@ -21,10 +21,7 @@ ], "baseCommand": "rev", "stdout": "output.txt", - "id": "#revtool.cwl", - "$namespaces": { - "iana": "https://www.iana.org/assignments/media-types/" - } + "id": "#revtool.cwl" }, { "class": "CommandLineTool", @@ -138,5 +135,8 @@ "$schemas": [ "empty2.ttl", "empty.ttl" - ] -} \ No newline at end of file + ], + "$namespaces": { + "iana": "https://www.iana.org/assignments/media-types/" + } +} diff --git a/tests/wf/scatter2_subwf.cwl b/tests/wf/scatter2_subwf.cwl index 7fb35d50fd..0e2b4b129d 100644 --- a/tests/wf/scatter2_subwf.cwl +++ b/tests/wf/scatter2_subwf.cwl @@ -72,5 +72,8 @@ ] } ], - "cwlVersion": "v1.0" + "cwlVersion": "v1.0", + "$namespaces": { + "arv": "http://arvados.org/cwl#" + } } From 48c9d0c84115aebc5765e2bc5a1a2bd7883185dd Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" Date: Sat, 4 Sep 2021 18:34:54 +0200 Subject: [PATCH 110/111] parallelize all tests Saves 30 seconds, at least --- tests/test_examples.py | 27 ++++++++++++--------------- tox.ini | 2 +- 2 files changed, 13 insertions(+), 16 deletions(-) diff --git a/tests/test_examples.py b/tests/test_examples.py index 1deca04fb7..060efecf95 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -1205,23 +1205,20 @@ def test_secondary_files_bad_v1_1(factor: str) -> None: @needs_docker @pytest.mark.parametrize("factor", test_factors) -def test_secondary_files_v1_0(factor: str) -> None: +def test_secondary_files_v1_0(tmp_path: Path, factor: str) -> None: + """Test plain strings under "secondaryFiles".""" test_file = "secondary-files-string-v1.cwl" test_job_file = "secondary-files-job.yml" - try: - old_umask = os.umask(stat.S_IWOTH) # test run with umask 002 - commands = factor.split() - commands.extend( - [ - get_data(os.path.join("tests", test_file)), - get_data(os.path.join("tests", test_job_file)), - ] - ) - error_code, _, stderr = get_main_output(commands) - finally: - # 664 in octal, '-rw-rw-r--' - assert stat.S_IMODE(os.stat("lsout").st_mode) == 436 - os.umask(old_umask) # revert back to original umask + commands = factor.split() + commands.extend( + [ + "--outdir", + str(tmp_path), + get_data(os.path.join("tests", test_file)), + get_data(os.path.join("tests", test_job_file)), + ] + ) + error_code, _, stderr = get_main_output(commands) assert "completed success" in stderr assert error_code == 0 diff --git a/tox.ini b/tox.ini index 1ced34b1b4..9bf36624ba 100644 --- a/tox.ini +++ b/tox.ini @@ -11,7 +11,7 @@ envlist = skip_missing_interpreters = True [pytest] -addopts=--ignore cwltool/schemas --basetemp ./tmp -n auto --dist=loadfile +addopts=--ignore cwltool/schemas --basetemp ./tmp -n auto testpaths = tests [gh-actions] From 0893cbeea9ab2c6dc659964d8ab849ebe753dc62 Mon Sep 17 00:00:00 2001 From: Fotis Georgatos Date: Sun, 5 Sep 2021 15:37:43 +0200 Subject: [PATCH 111/111] doc: explicit on node.js dependency during testing (#857) --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 1973cf38ed..9f7a9ecb7a 100644 --- a/README.rst +++ b/README.rst @@ -669,7 +669,7 @@ To run the basic tests after installing `cwltool` execute the following: .. code:: bash pip install -rtest-requirements.txt - pytest + pytest ## N.B. This requires node.js or docker to be available To run various tests in all supported Python environments, we use `tox `_. To run the test suite in all supported Python environments first clone the complete code repository (see the ``git clone`` instructions above) and then run