Run CI tests on Julia v1.11 too (#193)

* Run CI tests on Julia v1.11 too

* Make junit reference tests v1.11 compatible

* Skip reference tests on unrelease julia versions

To try to make sure PkgEval failures on this package are real

* Update UndefVarError test for v1.11

* fixup! Skip reference tests on unrelease julia versions

* fixup! Update UndefVarError test for v1.11

* fixup! Update UndefVarError test for v1.11

.github/workflows/CI.yml

@@ -18,8 +18,8 @@ jobs:
       matrix:
         version:
           - '1.8'
-          - '1.9'
           - '1.10'
+          - '1.11'
         os:
           - ubuntu-latest
           - macOS-latest

test/internals.jl

@@ -320,7 +320,13 @@ end
     @test_throws "Test item \"x\" `skip` keyword must be a `Bool`, got `skip=2`" should_skip(ti)
 
     ti = @testitem("x", skip=:(x = 1; x + y), _run=false, begin end)
-    @test_throws UndefVarError(:y) should_skip(ti)
+    if v"1.11-" < VERSION < v"1.11.2"
+        # Testing for a specific UndefVarError was broken in v1.11.0 and v1.11.1, see:
+        # https://github.com/JuliaLang/julia/issues/54082
+        @test_throws UndefVarError should_skip(ti)
+    else
+        @test_throws UndefVarError(:y) should_skip(ti)
+    end
 end
 
 @testset "filtering.jl" begin

test/junit_xml.jl

@@ -12,6 +12,7 @@ using ReTestItems
 using DeepDiffs: deepdiff
 
 # remove things that vary on each run: timestamps, run times, line numbers, repo locations
+# or which depend on the Julia VERSION being used
 function remove_variables(str)
     return replace(str,
         # Replace timestamps and times with "0"
@@ -33,7 +34,11 @@ function remove_variables(str)
         r"`" => "",
         # Remove blank lines in error messages (these were add in v1.9+)
         # https://github.com/JuliaLang/julia/commit/ba1e568966f82f4732f9a906ab186b02741eccb0
-        r"\n\n" => "\n"
+        r"\n\n" => "\n",
+        # Remove the extra info added to `UndefVarError` messages in Julia v1.11
+        # e.g. "UndefVarError: `x` not defined in ..." => "UndefVarError: x not defined"
+        # see: https://github.com/JuliaLang/julia/commit/449c7a2504191a96cfd382e0dbc0b40bf922cd6d
+        r"UndefVarError: `(?<var>[^`]*)` not defined in (.+)" => s"UndefVarError: \g<var> not defined",
     )
 end
 

test/runtests.jl

@@ -20,7 +20,14 @@ end
         include("macros.jl")
         include("integrationtests.jl")
         include("log_capture.jl")
-        include("junit_xml.jl")
+        # junit_xml reference tests are very sensitive to changes in text output which is not
+        # stable across minor Julia versions, i.e. we expect these to fail on upcoming Julia
+        # releases so we may as well skip them (so PkgEval doesn't always fail for us).
+        if isempty(VERSION.prerelease)
+            include("junit_xml.jl")
+        else
+            @warn "Skipping JUnit XML reference tests on unrelease Julia version" VERSION
+        end
     end
 
     # After all tests have run, check we didn't leave Test printing disabled.