Revert back to our hand-rolled test runner.

Author: maleadt

test/Project.toml

@@ -1,11 +1,14 @@
 [deps]
 Aqua = "4c88cf16-eb10-579e-8560-4a9242c79595"
+Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
+Distributed = "8ba89e20-285c-5b6f-9357-94700520ee1b"
 InteractiveUtils = "b77e0a4c-d291-57a0-90e8-8db25a27a240"
 LLVM = "929cbde3-209d-540e-8aea-75f648917ca0"
 Logging = "56ddb016-857b-54e1-b83d-db4d58db5568"
 PrecompileTools = "aea7be01-6a6a-4083-8856-8a6e6704d82a"
+Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7"
 REPL = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb"
-ReTestItems = "817f1d60-ba6b-4fd5-9520-3cf149f6a823"
+Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
 SPIRV_LLVM_Translator_unified_jll = "85f0d8ed-5b39-5caa-b1ae-7472de402361"
 SPIRV_Tools_jll = "6ac6d60f-d740-5983-97d7-a4482c0689f4"
 StaticArrays = "90137ffa-7385-5640-81b9-e52037218182"
@@ -14,3 +17,6 @@ demumble_jll = "1e29f10c-031c-5a83-9565-69cddfc27673"
 
 [compat]
 Aqua = "0.8"
+
+[extras]
+GPUCompiler = "61eb1bfa-7361-4325-ad38-22787b887f55"

test/aqua_tests.jl renamed to test/aqua.jl

@@ -1,7 +1,3 @@
-@testitem "Aqua" begin
-
 using Aqua
 
 Aqua.test_all(GPUCompiler)
-
-end

test/bpf_tests.jl renamed to test/bpf.jl

@@ -1,7 +1,3 @@
-@testitem "BPF" setup=[BPF, Helpers] begin
-
-############################################################################################
-
 @testset "No-op" begin
     kernel() = 0
 
@@ -40,5 +36,3 @@ end
         @test_throws GPUCompiler.InvalidIRError BPF.code_execution(kernel, (Int,))
     end
 end
-
-end

test/examples_tests.jl renamed to test/examples.jl

@@ -1,5 +1,3 @@
-@testitem "examples" begin
-
 function find_sources(path::String, sources=String[])
     if isdir(path)
         for entry in readdir(path)
@@ -23,5 +21,3 @@ cd(dir) do
         @test success(pipeline(`$cmd $example`, stderr=stderr))
     end
 end
-
-end

test/gcn_tests.jl renamed to test/gcn.jl

@@ -1,11 +1,3 @@
-@testitem "GCN" setup=[GCN, Helpers] begin
-
-@inline sink_gcn(i) = sink(i, Val(5))
-
-@test GCNCompilerTarget(dev_isa="gfx900") == GCNCompilerTarget("gfx900")
-
-############################################################################################
-
 @testset "IR" begin
 
 @testset "kernel calling convention" begin
@@ -222,7 +214,3 @@ end
 end
 
 end
-
-############################################################################################
-
-end

test/bpf_testsetup.jl renamed to test/helpers/bpf.jl

@@ -1,11 +1,8 @@
-@testsetup module BPF
+module BPF
 
-using GPUCompiler
+using ..GPUCompiler
+import ..TestRuntime
 
-
-# create a native test compiler, and generate reflection methods for it
-
-include("runtime.jl")
 struct CompilerParams <: AbstractCompilerParams end
 GPUCompiler.runtime_module(::CompilerJob{<:Any,CompilerParams}) = TestRuntime
 

test/gcn_testsetup.jl renamed to test/helpers/gcn.jl

@@ -1,11 +1,8 @@
-@testsetup module GCN
+module GCN
 
-using GPUCompiler
+using ..GPUCompiler
+import ..TestRuntime
 
-
-# create a GCN-based test compiler, and generate reflection methods for it
-
-include("runtime.jl")
 struct CompilerParams <: AbstractCompilerParams end
 GPUCompiler.runtime_module(::CompilerJob{<:Any,CompilerParams}) = TestRuntime
 

test/metal_testsetup.jl renamed to test/helpers/metal.jl

@@ -1,11 +1,8 @@
-@testsetup module Metal
+module Metal
 
-using GPUCompiler
+using ..GPUCompiler
+import ..TestRuntime
 
-
-# create a Metal test compiler, and generate reflection methods for it
-
-include("runtime.jl")
 struct CompilerParams <: AbstractCompilerParams end
 GPUCompiler.runtime_module(::CompilerJob{<:Any,CompilerParams}) = TestRuntime
 

test/native_testsetup.jl renamed to test/helpers/native.jl

@@ -1,11 +1,7 @@
-@testsetup module Native
+module Native
 
-using GPUCompiler
-
-
-# create a native test compiler, and generate reflection methods for it
-
-include("runtime.jl")
+using ..GPUCompiler
+import ..TestRuntime
 
 # local method table for device functions
 Base.Experimental.@MethodTable(test_method_table)

test/precompile_testsetup.jl renamed to test/helpers/precompile.jl

@@ -1,10 +1,3 @@
-@testsetup module Precompile
-
-using Test
-using ReTestItems
-
-export precompile_test_harness, check_presence, create_standalone
-
 function precompile_test_harness(@nospecialize(f), testset::String)
     @testset "$testset" begin
         precompile_test_harness(f, true)
@@ -50,19 +43,18 @@ function check_presence(mi, token)
 end
 
 function create_standalone(load_path, name::String, file)
-    cp(joinpath(@__DIR__, "runtime.jl"), joinpath(load_path, "runtime.jl"), force=true)
+    code = :(
+        module $(Symbol(name))
 
-    TS = include(file)
-    code = TS.code
-    if code.head == :begin
-        code.head = :block
-    end
-    @assert code.head == :block
-    code = Expr(:module, true, Symbol(name), code)
+        using GPUCompiler
+
+        include($(joinpath(@__DIR__, "runtime.jl")))
+        include($(joinpath(@__DIR__, file)))
+
+        end
+    )
 
     # Write out the test setup as a micro package
     write(joinpath(load_path, "$name.jl"), string(code))
     Base.compilecache(Base.PkgId(name))
 end
-
-end # testsetup

test/ptx_testsetup.jl renamed to test/helpers/ptx.jl

@@ -1,11 +1,8 @@
-@testsetup module PTX
+module PTX
 
-using GPUCompiler
+using ..GPUCompiler
+import ..TestRuntime
 
-
-# create a PTX-based test compiler, and generate reflection methods for it
-
-include("runtime.jl")
 struct CompilerParams <: AbstractCompilerParams end
 
 PTXCompilerJob = CompilerJob{PTXCompilerTarget,CompilerParams}

test/runtime.jl renamed to test/helpers/runtime.jl

@@ -1,11 +1,8 @@
-@testsetup module SPIRV
+module SPIRV
 
-using GPUCompiler
+using ..GPUCompiler
+import ..TestRuntime
 
-
-# create a SPIRV-based test compiler, and generate reflection methods for it
-
-include("runtime.jl")
 struct CompilerParams <: AbstractCompilerParams end
 GPUCompiler.runtime_module(::CompilerJob{<:Any,CompilerParams}) = TestRuntime
 

test/spirv_testsetup.jl renamed to test/helpers/spirv.jl

@@ -1,9 +1,3 @@
-@testsetup module Helpers
-
-using Test
-
-export @test_throws_message, sink
-
 # @test_throw, with additional testing for the exception message
 macro test_throws_message(f, typ, ex...)
     quote
@@ -41,5 +35,3 @@ end
                      ret i64 %value"""
     return :(Base.llvmcall($llvmcall_str, T, Tuple{T}, i))
 end
-
-end

test/helper_testsetup.jl renamed to test/helpers/test.jl

@@ -1,9 +1,3 @@
-@testitem "Metal" setup=[Metal, Helpers] begin
-
-using LLVM
-
-############################################################################################
-
 @testset "IR" begin
 
 @testset "kernel functions" begin
@@ -110,7 +104,6 @@ end
         return
     end
 
-
     ir = sprint(io->Metal.code_llvm(io, kernel1, Tuple{Core.LLVMPtr{Float32,1}}; validate=true))
     @test occursin("@metal_os_log", ir)
 
@@ -145,5 +138,3 @@ end
 end
 
 end
-
-end

test/metal_tests.jl renamed to test/metal.jl

@@ -1,9 +1,3 @@
-@testitem "native" setup=[Native, Helpers] begin
-
-using Test
-
-############################################################################################
-
 @testset "reflection" begin
     job, _ = Native.create_job(identity, (Int,))
 
@@ -233,7 +227,7 @@ end
     @test "We did not crash!" != ""
 end
 
-@testset "CUDAnative.jl#278" begin
+@testset "CUDAjl#278" begin
     # codegen idempotency
     # NOTE: this isn't fixed, but surfaces here due to bad inference of checked_sub
     # NOTE: with the fix to print_to_string this doesn't error anymore,
@@ -563,18 +557,17 @@ end
                ["jl_invoke", "apply_iterate",
                 "inttoptr", "apply_type"])
 end
-end # testitem
 
-@testitem "native precompile" setup=[Precompile,] begin
+############################################################################################
+
+@testset "precompile" begin
 
-using Test
 precompile_test_harness("Inference caching") do load_path
     # Write out the Native test setup as a micro package
-    create_standalone(load_path, "NativeCompiler", "native_testsetup.jl")
+    create_standalone(load_path, "TestCompiler", "native.jl")
 
     write(joinpath(load_path, "InferenceCaching.jl"), :(module InferenceCaching
-        import NativeCompiler
-        import GPUCompiler
+        import TestCompiler
         using PrecompileTools
 
         function kernel(A, x)
@@ -583,13 +576,13 @@ precompile_test_harness("Inference caching") do load_path
         end
 
         let
-            job, _ = NativeCompiler.create_job(kernel, (Vector{Int}, Int))
+            job, _ = TestCompiler.Native.create_job(kernel, (Vector{Int}, Int))
             precompile(job)
         end
 
         # identity is foreign
         @setup_workload begin
-            job, _ = NativeCompiler.create_job(identity, (Int,))
+            job, _ = TestCompiler.Native.create_job(identity, (Int,))
             @compile_workload begin
                 precompile(job)
             end
@@ -598,13 +591,13 @@ precompile_test_harness("Inference caching") do load_path
 
     Base.compilecache(Base.PkgId("InferenceCaching"))
     @eval let
-        import NativeCompiler
+        import TestCompiler
 
         # Check that no cached entry is present
         identity_mi = GPUCompiler.methodinstance(typeof(identity), Tuple{Int})
 
         token = let
-            job, _ = NativeCompiler.create_job(identity, (Int,))
+            job, _ = TestCompiler.Native.create_job(identity, (Int,))
             GPUCompiler.ci_cache_token(job)
         end
         @test !check_presence(identity_mi, token)
@@ -624,21 +617,22 @@ precompile_test_harness("Inference caching") do load_path
         GPUCompiler.enable_disk_cache!()
         @test GPUCompiler.disk_cache_enabled() == true
 
-        job, _ = NativeCompiler.create_job(InferenceCaching.kernel, (Vector{Int}, Int))
+        job, _ = TestCompiler.Native.create_job(InferenceCaching.kernel, (Vector{Int}, Int))
         @assert job.source == kernel_mi
         ci = GPUCompiler.ci_cache_lookup(GPUCompiler.ci_cache(job), job.source, job.world, job.world)
         @assert ci !== nothing
         @assert ci.inferred !== nothing
         path = GPUCompiler.cache_file(ci, job.config)
         @test path !== nothing
         @test !ispath(path)
-        NativeCompiler.cached_execution(InferenceCaching.kernel, (Vector{Int}, Int))
+        TestCompiler.Native.cached_execution(InferenceCaching.kernel, (Vector{Int}, Int))
         @test ispath(path)
         GPUCompiler.clear_disk_cache!()
         @test !ispath(path)
+
+        GPUCompiler.enable_disk_cache!(false)
+        @test GPUCompiler.disk_cache_enabled() == false
     end
 end
 
-############################################################################################
-
 end