Throw on version mismatch (#49)

Author: LilithHafner

Project.toml

@@ -1,7 +1,7 @@
 name = "RegressionTests"
 uuid = "2cca1867-674a-4323-b559-fc77c0bbdb31"
 authors = ["Lilith Orion Hafner <[email protected]> and contributors"]
-version = "0.0.8"
+version = "0.0.9"
 
 [deps]
 Compat = "34da2185-b29b-5c13-b0c7-acf172513d20"

src/RegressionTests.jl

@@ -90,6 +90,7 @@ function try_runbenchmarks(;
         workers = 5,#Sys.CPU_THREADS,
         startup_file = Base.JLOptions().startupfile == 1 ? "yes" : "no",
         are_different = are_different,
+        allow_version_mismatch = false,
         )
 
     commands = Vector{Cmd}(undef, workers)
@@ -104,7 +105,8 @@ function try_runbenchmarks(;
         mkdir(projects[i])
         # bench_projectfile_exists && cp(bench_projectfile, joinpath(projects[i], "Project.toml"))
         cp(bench_project, projects[i], force=true)
-        script = "let; using RegressionTests; RegressionTests.FILTER[] = RegressionTests.deserialize($(repr(filter_path))); end; let; include($rfile); end; using RegressionTests; RegressionTests.serialize($(repr(channels[i])), (RegressionTests.STATIC_METADATA, RegressionTests.RUNTIME_METADATA, RegressionTests.DATA))"
+        verify_version = allow_version_mismatch ? "" : " RegressionTests.verify_source($(repr(dirname(@__DIR__))));"
+        script = "let; using RegressionTests;$verify_version RegressionTests.FILTER[] = RegressionTests.deserialize($(repr(filter_path))); end; let; include($rfile); end; using RegressionTests; RegressionTests.serialize($(repr(channels[i])), (RegressionTests.STATIC_METADATA, RegressionTests.RUNTIME_METADATA, RegressionTests.DATA))"
         commands[i] = if VERSION < v"1.10.0-alpha1"
             # --compiled-modules=no is a workaround for https://github.com/JuliaLang/julia/issues/52265
             `$julia_exe --compiled-modules=no --startup-file=$startup_file --project=$(projects[i]) -e $script`
@@ -688,6 +690,12 @@ end
 
 # Callie
 
+verify_source(path) = (path == dirname(@__DIR__)) || error("""
+    RegressionTests version mismatch
+    Outer process is at $path (version $(Pkg.TOML.parsefile(joinpath(path, "Project.toml"))["version"]))
+    Inner process is at $(dirname(@__DIR__)) (version $(Pkg.TOML.parsefile(joinpath(dirname(@__DIR__), "Project.toml"))["version"]))
+    Hint: make sure the bench project and the test project have the same version of RegressionTests.""")
+
 const FILTER = Ref{Union{Nothing, BitVector}}(nothing)
 const STATIC_METADATA = Tuple{Symbol, Int, String}[]
 const GROUP_ID = Ref(0)

test/runtests.jl

@@ -21,7 +21,7 @@ using Pkg
 
     # TODO: make this work when it comes after "Example usage" as well.
     @testset "Regression tests" begin
-        RegressionTests.test(skip_unsupported_platforms=true)
+        RegressionTests.test(skip_unsupported_platforms=true, allow_version_mismatch=true)
     end
 
     if RegressionTests.is_platform_supported()
@@ -48,7 +48,7 @@ using Pkg
                     old_src = read(src_file, String)
                     new_src = replace(old_src, "my_sum(x) = sum(x)" => "my_sum(x) = sum(Float64.(x))")
                     write(src_file, new_src)
-                    t = @elapsed changes = runbenchmarks(project = ".") # Fail
+                    t = @elapsed changes = runbenchmarks(project = ".", allow_version_mismatch=true) # Fail
                     println("Runtime for positive runbenchmarks: $t")
                     @test !isempty(changes)
 
@@ -61,7 +61,7 @@ using Pkg
                     println.(changes)
                     run(`git add $src_file`)
                     run(`git commit -m "Introduce regression"`)
-                    t = @elapsed @test isempty(runbenchmarks(project = ".")) # Pass
+                    t = @elapsed @test isempty(runbenchmarks(project = ".", allow_version_mismatch=true)) # Pass
                     println("Runtime for negative runbenchmarks: $t")
                     # TODO: handle this case well
                     # TODO: track the runtime of these runbenchmark calls... but we can't use RegressionTests.jl because that would be too slow.