check the initial working directory before the cache attempt

cwltool/command_line_tool.py

@@ -492,7 +492,14 @@ def updatePathmap(self, outdir: str, pathmap: PathMapper, fn: CWLObjectType) ->
         for ls in cast(list[CWLObjectType], fn.get("listing", [])):
             self.updatePathmap(os.path.join(outdir, cast(str, fn["basename"])), pathmap, ls)
 
-    def _initialworkdir(self, j: JobBase, builder: Builder) -> None:
+    def _initialworkdir(self, j: Optional[JobBase], builder: Builder) -> None:
+        """
+        Test and initialize the working directory.
+
+        :param j: A :py:class:`~cwltool.job.CommandLineJob` or a
+                  specialized container-based job.
+                  If 'None', then only tests will be performed, no setup.
+        """
         initialWorkdir, _ = self.get_requirement("InitialWorkDirRequirement")
         if initialWorkdir is None:
             return
@@ -760,6 +767,9 @@ def _initialworkdir(self, j: JobBase, builder: Builder) -> None:
                         "is in 'requirements'."
                     )
 
+        if j is None:
+            return  # Only testing
+
         with SourceLine(initialWorkdir, "listing", WorkflowException, debug):
             j.generatefiles["listing"] = ls
             for entry in ls:
@@ -824,6 +834,7 @@ def job(
                 _check_adjust,
             )
             visit_class([cachebuilder.files, cachebuilder.bindings], ("File"), _checksum)
+            self._initialworkdir(None, cachebuilder)  # test the initial working directory
 
             cmdline = flatten(list(map(cachebuilder.generate_arg, cachebuilder.bindings)))
             docker_req, _ = self.get_requirement("DockerRequirement")

tests/test_caching.py

@@ -0,0 +1,192 @@
+import re
+from pathlib import Path
+
+import pytest
+
+from .util import get_data, get_main_output, needs_docker
+
+test_factors = [(""), ("--parallel"), ("--debug"), ("--parallel --debug")]
+
+
+@needs_docker
+@pytest.mark.parametrize("factor", test_factors)
+def test_cid_file_non_existing_dir(tmp_path: Path, factor: str) -> None:
+    """Test that --cachedir with a bad path should produce a specific error."""
+    test_file = "cache_test_workflow.cwl"
+    bad_cidfile_dir = tmp_path / "cidfile-dir-badpath"
+    commands = factor.split()
+    commands.extend(
+        [
+            "--record-container-id",
+            "--cidfile-dir",
+            str(bad_cidfile_dir),
+            get_data("tests/wf/" + test_file),
+        ]
+    )
+    error_code, _, stderr = get_main_output(commands)
+    stderr = re.sub(r"\s\s+", " ", stderr)
+    assert "directory doesn't exist, please create it first" in stderr, stderr
+    assert error_code == 2 or error_code == 1, stderr
+
+
+@needs_docker
+@pytest.mark.parametrize("factor", test_factors)
+def test_wf_without_container(tmp_path: Path, factor: str) -> None:
+    """Confirm that we can run a workflow without a container."""
+    test_file = "hello-workflow.cwl"
+    cache_dir = str(tmp_path / "cwltool_cache")
+    commands = factor.split()
+    commands.extend(
+        [
+            "--cachedir",
+            cache_dir,
+            "--outdir",
+            str(tmp_path / "outdir"),
+            get_data("tests/wf/" + test_file),
+            "--usermessage",
+            "hello",
+        ]
+    )
+    error_code, _, stderr = get_main_output(commands)
+
+    stderr = re.sub(r"\s\s+", " ", stderr)
+    assert "completed success" in stderr
+    assert error_code == 0
+
+
+@needs_docker
+@pytest.mark.parametrize("factor", test_factors)
+def test_issue_740_fixed(tmp_path: Path, factor: str) -> None:
+    """Confirm that re-running a particular workflow with caching succeeds."""
+    test_file = "cache_test_workflow.cwl"
+    cache_dir = str(tmp_path / "cwltool_cache")
+    commands = factor.split()
+    commands.extend(["--cachedir", cache_dir, get_data("tests/wf/" + test_file)])
+    error_code, _, stderr = get_main_output(commands)
+
+    stderr = re.sub(r"\s\s+", " ", stderr)
+    assert "completed success" in stderr
+    assert error_code == 0
+
+    commands = factor.split()
+    commands.extend(["--cachedir", cache_dir, get_data("tests/wf/" + test_file)])
+    error_code, _, stderr = get_main_output(commands)
+
+    stderr = re.sub(r"\s\s+", " ", stderr)
+    assert "Output of job will be cached in" not in stderr
+    assert error_code == 0, stderr
+
+
+@needs_docker
+@pytest.mark.parametrize("factor", test_factors)
+def test_cache_relative_paths(tmp_path: Path, factor: str) -> None:
+    """Confirm that re-running a particular workflow with caching succeeds."""
+    test_file = "secondary-files.cwl"
+    test_job_file = "secondary-files-job.yml"
+    cache_dir = str(tmp_path / "cwltool_cache")
+    commands = factor.split()
+    commands.extend(
+        [
+            "--out",
+            str(tmp_path / "out"),
+            "--cachedir",
+            cache_dir,
+            get_data(f"tests/{test_file}"),
+            get_data(f"tests/{test_job_file}"),
+        ]
+    )
+    error_code, _, stderr = get_main_output(commands)
+
+    stderr = re.sub(r"\s\s+", " ", stderr)
+    assert "completed success" in stderr
+    assert error_code == 0
+
+    commands = factor.split()
+    commands.extend(
+        [
+            "--out",
+            str(tmp_path / "out2"),
+            "--cachedir",
+            cache_dir,
+            get_data(f"tests/{test_file}"),
+            get_data(f"tests/{test_job_file}"),
+        ]
+    )
+    error_code, _, stderr = get_main_output(commands)
+
+    stderr = re.sub(r"\s\s+", " ", stderr)
+    assert "Output of job will be cached in" not in stderr
+    assert error_code == 0, stderr
+
+    assert (tmp_path / "cwltool_cache" / "27903451fc1ee10c148a0bdeb845b2cf").exists()
+
+
+@pytest.mark.parametrize("factor", test_factors)
+def test_cache_default_literal_file(tmp_path: Path, factor: str) -> None:
+    """Confirm that running a CLT with a default literal file with caching succeeds."""
+    test_file = "tests/wf/extract_region_specs.cwl"
+    cache_dir = str(tmp_path / "cwltool_cache")
+    commands = factor.split()
+    commands.extend(
+        [
+            "--out",
+            str(tmp_path / "out"),
+            "--cachedir",
+            cache_dir,
+            get_data(test_file),
+        ]
+    )
+    error_code, _, stderr = get_main_output(commands)
+
+    stderr = re.sub(r"\s\s+", " ", stderr)
+    assert "completed success" in stderr
+    assert error_code == 0
+
+
+@pytest.mark.parametrize("factor", test_factors)
+def test_cache_dockerreq_hint_instead_of_req(tmp_path: Path, factor: str) -> None:
+    """The cache must not be checked when there is an invalid use of an absolute path in iwdr.listing."""
+    cache_dir = str(tmp_path / "cwltool_cache")
+    test_job_file = "tests/wf/loadContents-input.yml"
+    # First, run the iwd-container-entryname1 conformance tests with caching turned on
+    test1_file = "tests/wf/iwd-container-entryname1.cwl"
+    commands1 = factor.split()
+    commands1.extend(
+        [
+            "--out",
+            str(tmp_path / "out1"),
+            "--cachedir",
+            cache_dir,
+            get_data(test1_file),
+            get_data(test_job_file),
+        ]
+    )
+    error_code1, _, stderr1 = get_main_output(commands1)
+
+    stderr1 = re.sub(r"\s\s+", " ", stderr1)
+    assert "completed success" in stderr1
+    assert error_code1 == 0
+    # Second, run the iwd-container-entryname3 test, which should fail
+    # even though it would be a cache hit, except that its DockerRequirement is
+    # in `hints` instead of `requirements` and one of the initial working directory
+    # items has an absolute path starting with `/`.
+    test2_file = "tests/wf/iwd-container-entryname3.cwl"
+    commands2 = factor.split()
+    commands2.extend(
+        [
+            "--out",
+            str(tmp_path / "out2"),
+            "--cachedir",
+            cache_dir,
+            get_data(test2_file),
+            get_data(test_job_file),
+        ]
+    )
+    error_code2, _, stderr2 = get_main_output(commands2)
+
+    stderr2 = re.sub(r"\s\s+", " ", stderr2)
+    assert (
+        "at index 0 of listing is invalid, name can only start with '/' "
+        "when DockerRequirement is in 'requirements" in stderr2
+    )
+    assert error_code2 == 1

tests/test_examples.py

@@ -1115,27 +1115,6 @@ def test_cid_file_dir_arg_is_file_instead_of_dir(tmp_path: Path, factor: str) ->
     assert error_code == 2 or error_code == 1, stderr
 
 
-@needs_docker
-@pytest.mark.parametrize("factor", test_factors)
-def test_cid_file_non_existing_dir(tmp_path: Path, factor: str) -> None:
-    """Test that --cachedir with a bad path should produce a specific error."""
-    test_file = "cache_test_workflow.cwl"
-    bad_cidfile_dir = tmp_path / "cidfile-dir-badpath"
-    commands = factor.split()
-    commands.extend(
-        [
-            "--record-container-id",
-            "--cidfile-dir",
-            str(bad_cidfile_dir),
-            get_data("tests/wf/" + test_file),
-        ]
-    )
-    error_code, _, stderr = get_main_output(commands)
-    stderr = re.sub(r"\s\s+", " ", stderr)
-    assert "directory doesn't exist, please create it first" in stderr, stderr
-    assert error_code == 2 or error_code == 1, stderr
-
-
 @needs_docker
 @pytest.mark.parametrize("factor", test_factors)
 def test_cid_file_w_prefix(tmp_path: Path, factor: str) -> None:
@@ -1233,120 +1212,6 @@ def test_secondary_files_v1_0(tmp_path: Path, factor: str) -> None:
     assert error_code == 0
 
 
-@needs_docker
-@pytest.mark.parametrize("factor", test_factors)
-def test_wf_without_container(tmp_path: Path, factor: str) -> None:
-    """Confirm that we can run a workflow without a container."""
-    test_file = "hello-workflow.cwl"
-    cache_dir = str(tmp_path / "cwltool_cache")
-    commands = factor.split()
-    commands.extend(
-        [
-            "--cachedir",
-            cache_dir,
-            "--outdir",
-            str(tmp_path / "outdir"),
-            get_data("tests/wf/" + test_file),
-            "--usermessage",
-            "hello",
-        ]
-    )
-    error_code, _, stderr = get_main_output(commands)
-
-    stderr = re.sub(r"\s\s+", " ", stderr)
-    assert "completed success" in stderr
-    assert error_code == 0
-
-
-@needs_docker
-@pytest.mark.parametrize("factor", test_factors)
-def test_issue_740_fixed(tmp_path: Path, factor: str) -> None:
-    """Confirm that re-running a particular workflow with caching succeeds."""
-    test_file = "cache_test_workflow.cwl"
-    cache_dir = str(tmp_path / "cwltool_cache")
-    commands = factor.split()
-    commands.extend(["--cachedir", cache_dir, get_data("tests/wf/" + test_file)])
-    error_code, _, stderr = get_main_output(commands)
-
-    stderr = re.sub(r"\s\s+", " ", stderr)
-    assert "completed success" in stderr
-    assert error_code == 0
-
-    commands = factor.split()
-    commands.extend(["--cachedir", cache_dir, get_data("tests/wf/" + test_file)])
-    error_code, _, stderr = get_main_output(commands)
-
-    stderr = re.sub(r"\s\s+", " ", stderr)
-    assert "Output of job will be cached in" not in stderr
-    assert error_code == 0, stderr
-
-
-@needs_docker
-@pytest.mark.parametrize("factor", test_factors)
-def test_cache_relative_paths(tmp_path: Path, factor: str) -> None:
-    """Confirm that re-running a particular workflow with caching succeeds."""
-    test_file = "secondary-files.cwl"
-    test_job_file = "secondary-files-job.yml"
-    cache_dir = str(tmp_path / "cwltool_cache")
-    commands = factor.split()
-    commands.extend(
-        [
-            "--out",
-            str(tmp_path / "out"),
-            "--cachedir",
-            cache_dir,
-            get_data(f"tests/{test_file}"),
-            get_data(f"tests/{test_job_file}"),
-        ]
-    )
-    error_code, _, stderr = get_main_output(commands)
-
-    stderr = re.sub(r"\s\s+", " ", stderr)
-    assert "completed success" in stderr
-    assert error_code == 0
-
-    commands = factor.split()
-    commands.extend(
-        [
-            "--out",
-            str(tmp_path / "out2"),
-            "--cachedir",
-            cache_dir,
-            get_data(f"tests/{test_file}"),
-            get_data(f"tests/{test_job_file}"),
-        ]
-    )
-    error_code, _, stderr = get_main_output(commands)
-
-    stderr = re.sub(r"\s\s+", " ", stderr)
-    assert "Output of job will be cached in" not in stderr
-    assert error_code == 0, stderr
-
-    assert (tmp_path / "cwltool_cache" / "27903451fc1ee10c148a0bdeb845b2cf").exists()
-
-
-@pytest.mark.parametrize("factor", test_factors)
-def test_cache_default_literal_file(tmp_path: Path, factor: str) -> None:
-    """Confirm that running a CLT with a default literal file with caching succeeds."""
-    test_file = "tests/wf/extract_region_specs.cwl"
-    cache_dir = str(tmp_path / "cwltool_cache")
-    commands = factor.split()
-    commands.extend(
-        [
-            "--out",
-            str(tmp_path / "out"),
-            "--cachedir",
-            cache_dir,
-            get_data(test_file),
-        ]
-    )
-    error_code, _, stderr = get_main_output(commands)
-
-    stderr = re.sub(r"\s\s+", " ", stderr)
-    assert "completed success" in stderr
-    assert error_code == 0
-
-
 def test_write_summary(tmp_path: Path) -> None:
     """Test --write-summary."""
     commands = [