Skip to content

Commit fb69e58

Browse files
majochamajocha
authored
TestConsole supporting parallel execution (#17993)
1 parent ec4abb3 commit fb69e58

29 files changed

+479
-481
lines changed

‎tests/FSharp.Build.UnitTests/FSharp.Build.UnitTests.fsproj

+3
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,9 @@
1111
</PropertyGroup>
1212

1313
<ItemGroup>
14+
<Compile Include="..\FSharp.Test.Utilities\XunitSetup.fs">
15+
<Link>XunitSetup.fs</Link>
16+
</Compile>
1417
<Compile Include="WriteCodeFragmentTests.fs" />
1518
<Compile Include="MapSourceRootsTests.fs" />
1619
</ItemGroup>

‎tests/FSharp.Compiler.ComponentTests/EmittedIL/TryCatch/TryCatch.fs

+2-2
Original file line numberDiff line numberDiff line change
@@ -56,9 +56,9 @@ let ``Stackoverflow reproduction`` compilation =
5656
| CompilationResult.Success ({OutputPath = Some dllFile} as s) ->
5757
let fsharpCoreFile = typeof<voption<_>>.Assembly.Location
5858
File.Copy(fsharpCoreFile, Path.Combine(Path.GetDirectoryName(dllFile), Path.GetFileName(fsharpCoreFile)), true)
59-
let _exitCode, _stdout, stderr, _exn = CompilerAssert.ExecuteAndReturnResult (dllFile, isFsx=false, deps = s.Dependencies, newProcess=true)
59+
let result = CompilerAssert.ExecuteAndReturnResult (dllFile, isFsx=false, deps = s.Dependencies, newProcess=true)
6060

61-
Assert.True(stderr.Contains "stack overflow" || stderr.Contains "StackOverflow")
61+
Assert.True(result.StdErr.Contains "stack overflow" || result.StdErr.Contains "StackOverflow")
6262

6363
| _ -> failwith (sprintf "%A" compilationResult)
6464

‎tests/FSharp.Compiler.ComponentTests/FSharp.Compiler.ComponentTests.fsproj

+3
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,9 @@
2828
</PropertyGroup>
2929

3030
<ItemGroup>
31+
<Compile Include="..\FSharp.Test.Utilities\XunitSetup.fs">
32+
<Link>XunitSetup.fs</Link>
33+
</Compile>
3134
<Compile Include="CompilerDirectives\Line.fs" />
3235
<Compile Include="CompilerDirectives\Ifdef.fs" />
3336
<Compile Include="CompilerDirectives\NonStringArgs.fs" />

‎tests/FSharp.Compiler.ComponentTests/Miscellaneous/FsharpSuiteMigrated.fs

+2-1
Original file line numberDiff line numberDiff line change
@@ -31,12 +31,13 @@ module ScriptRunner =
3131
let cu = cu |> withDefines defaultDefines
3232
match cu with
3333
| FS fsSource ->
34+
use capture = new TestConsole.ExecutionCapture()
3435
let engine = createEngine (fsSource.Options |> Array.ofList,version)
3536
let res = evalScriptFromDiskInSharedSession engine cu
3637
match res with
3738
| CompilationResult.Failure _ -> res
3839
| CompilationResult.Success s ->
39-
if engine.GetOutput().Contains "TEST PASSED OK" then
40+
if capture.OutText |> TestFramework.outputPassed then
4041
res
4142
else
4243
failwith $"Results looked correct, but 'TEST PASSED OK' was not printed. Result: %A{s}"

‎tests/FSharp.Compiler.ComponentTests/xunit.runner.json

+4-5
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
{
2-
"$schema": "https://xunit.net/schema/current/xunit.runner.schema.json",
3-
"appDomain": "ifAvailable",
4-
"shadowCopy": false,
5-
"parallelizeTestCollections": false,
6-
"maxParallelThreads": 1
2+
"$schema": "https://xunit.net/schema/current/xunit.runner.schema.json",
3+
"appDomain": "denied",
4+
"parallelizeTestCollections": false,
5+
"maxParallelThreads": 1
76
}

‎tests/FSharp.Compiler.Private.Scripting.UnitTests/DependencyManagerInteractiveTests.fs

+52-88
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ open FSharp.Compiler.DependencyManager
1313
open FSharp.Compiler.Diagnostics
1414
open FSharp.DependencyManager.Nuget
1515
open FSharp.Test.ScriptHelpers
16+
open FSharp.Test
1617
open FSharp.Test.Utilities
1718

1819
open Internal.Utilities
@@ -148,8 +149,7 @@ type DependencyManagerInteractiveTests() =
148149
Assert.Equal(0, result.Roots |> Seq.length)
149150
()
150151

151-
152-
[<Fact(Skip="failing on main")>]
152+
[<Fact>]
153153
member _.``Multiple Instances of DependencyProvider should be isolated``() =
154154

155155
let assemblyProbingPaths () = Seq.empty<string>
@@ -721,101 +721,65 @@ x |> Seq.iter(fun r ->
721721

722722
[<Fact>]
723723
member _.``Verify that #help produces help text for fsi + dependency manager``() =
724-
let expected = [|
725-
""" F# Interactive directives:"""
726-
""""""
727-
""" #r "file.dll";; // Reference (dynamically load) the given DLL"""
728-
""" #i "package source uri";; // Include package source uri when searching for packages"""
729-
""" #I "path";; // Add the given search path for referenced DLLs"""
730-
""" #load "file.fs" ...;; // Load the given file(s) as if compiled and referenced"""
731-
""" #time ["on"|"off"];; // Toggle timing on/off"""
732-
""" #clear;; // Clear screen"""
733-
""" #help;; // Display help"""
734-
""" #help "idn";; // Display documentation for an identifier, e.g. #help "List.map";;"""
735-
""" #quit;; // Exit"""
736-
""""""
737-
""" F# Interactive command line options:"""
738-
""""""
739-
740-
// this is the end of the line each different platform has a different mechanism for starting fsi
741-
// Actual output looks similar to: """ See 'testhost --help' for options"""
742-
"""--help' for options"""
743-
744-
""""""
745-
""""""
746-
|]
724+
let expected = """
725+
F# Interactive directives:
726+
727+
#r "file.dll";; // Reference (dynamically load) the given DLL
728+
#i "package source uri";; // Include package source uri when searching for packages
729+
#I "path";; // Add the given search path for referenced DLLs
730+
#load "file.fs" ...;; // Load the given file(s) as if compiled and referenced
731+
#time ["on"|"off"];; // Toggle timing on/off
732+
#help;; // Display help
733+
#help "idn";; // Display documentation for an identifier, e.g. #help "List.map";;
734+
#clear;; // Clear screen
735+
#quit;; // Exit
736+
737+
F# Interactive command line options:"""
747738

748-
let mutable found = 0
749-
let lines = System.Collections.Generic.List()
750-
use sawExpectedOutput = new ManualResetEvent(false)
751-
let verifyOutput (line: string) =
752-
let compareLine (s: string) =
753-
if s = "" then line = ""
754-
else line.EndsWith(s)
755-
lines.Add(line)
756-
match expected |> Array.tryFind(compareLine) with
757-
| None -> ()
758-
| Some t ->
759-
found <- found + 1
760-
if found = expected.Length then sawExpectedOutput.Set() |> ignore
761-
762-
let text = "#help"
763739
use script = new FSharpScript(quiet = false, langVersion = LangVersion.V47)
764-
let mutable found = 0
765-
script.OutputProduced.Add (fun line -> verifyOutput line)
766-
let opt = script.Eval(text) |> getValue
767-
Assert.True(sawExpectedOutput.WaitOne(TimeSpan.FromSeconds(5.0)), sprintf "Expected to see error sentinel value written\nexpected:%A\nactual:%A" expected lines)
768740

741+
use capture = new TestConsole.ExecutionCapture()
742+
let opt = script.Eval("#help") |> getValue
743+
744+
let output = capture.OutText
745+
746+
Assert.Contains(expected, output)
747+
748+
// this is the end of the line each different platform has a different mechanism for starting fsi
749+
// Actual output looks similar to: """ See 'testhost --help' for options"""
750+
Assert.EndsWith("--help' for options", output.Trim())
769751

770752
[<Fact>]
771753
member _.``Verify that #help produces help text for fsi + dependency manager language version preview``() =
772-
let expected = [|
773-
""" F# Interactive directives:"""
774-
""""""
775-
""" #r "file.dll";; // Reference (dynamically load) the given DLL"""
776-
""" #i "package source uri";; // Include package source uri when searching for packages"""
777-
""" #I "path";; // Add the given search path for referenced DLLs"""
778-
""" #load "file.fs" ...;; // Load the given file(s) as if compiled and referenced"""
779-
""" #time ["on"|"off"];; // Toggle timing on/off"""
780-
""" #help;; // Display help"""
781-
""" #help "idn";; // Display documentation for an identifier, e.g. #help "List.map";;"""
782-
""" #r "nuget:FSharp.Data, 3.1.2";; // Load Nuget Package 'FSharp.Data' version '3.1.2'"""
783-
""" #r "nuget:FSharp.Data";; // Load Nuget Package 'FSharp.Data' with the highest version"""
784-
""" #clear;; // Clear screen"""
785-
""" #quit;; // Exit"""
786-
""""""
787-
""" F# Interactive command line options:"""
788-
""""""
789-
790-
// this is the end of the line each different platform has a different mechanism for starting fsi
791-
// Actual output looks similar to: """ See 'testhost --help' for options"""
792-
"""--help' for options"""
793-
794-
""""""
795-
""""""
796-
|]
754+
let expected = """
755+
F# Interactive directives:
756+
757+
#r "file.dll";; // Reference (dynamically load) the given DLL
758+
#i "package source uri";; // Include package source uri when searching for packages
759+
#I "path";; // Add the given search path for referenced DLLs
760+
#load "file.fs" ...;; // Load the given file(s) as if compiled and referenced
761+
#time ["on"|"off"];; // Toggle timing on/off
762+
#help;; // Display help
763+
#help "idn";; // Display documentation for an identifier, e.g. #help "List.map";;
764+
#r "nuget:FSharp.Data, 3.1.2";; // Load Nuget Package 'FSharp.Data' version '3.1.2'
765+
#r "nuget:FSharp.Data";; // Load Nuget Package 'FSharp.Data' with the highest version
766+
#clear;; // Clear screen
767+
#quit;; // Exit
768+
769+
F# Interactive command line options:"""
797770

798-
let mutable found = 0
799-
let lines = System.Collections.Generic.List()
800-
use sawExpectedOutput = new ManualResetEvent(false)
801-
let verifyOutput (line: string) =
802-
let compareLine (s: string) =
803-
if s = "" then line = ""
804-
else line.EndsWith(s)
805-
lines.Add(line)
806-
match expected |> Array.tryFind(compareLine) with
807-
| None -> ()
808-
| Some t ->
809-
found <- found + 1
810-
if found = expected.Length then sawExpectedOutput.Set() |> ignore
811-
812-
let text = "#help"
813771
use script = new FSharpScript(quiet = false, langVersion = LangVersion.Preview)
814-
let mutable found = 0
815-
script.OutputProduced.Add (fun line -> verifyOutput line)
816-
let opt = script.Eval(text) |> getValue
817-
Assert.True(sawExpectedOutput.WaitOne(TimeSpan.FromSeconds(5.0)), sprintf "Expected to see error sentinel value written\nexpected:%A\nactual:%A" expected lines)
818772

773+
use capture = new TestConsole.ExecutionCapture()
774+
let opt = script.Eval("#help") |> getValue
775+
776+
let output = capture.OutText
777+
778+
Assert.Contains(expected, output)
779+
780+
// this is the end of the line each different platform has a different mechanism for starting fsi
781+
// Actual output looks similar to: """ See 'testhost --help' for options"""
782+
Assert.EndsWith("--help' for options", output.Trim())
819783

820784
[<Fact>]
821785
member _.``Verify that timeout --- times out and fails``() =

‎tests/FSharp.Compiler.Private.Scripting.UnitTests/FSharp.Compiler.Private.Scripting.UnitTests.fsproj

+3
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,9 @@
1212
</PropertyGroup>
1313

1414
<ItemGroup>
15+
<Compile Include="..\FSharp.Test.Utilities\XunitSetup.fs">
16+
<Link>XunitSetup.fs</Link>
17+
</Compile>
1518
<Compile Include="$(FSharpSourcesRoot)\Compiler\Interactive\ControlledExecution.fs" />
1619
<Compile Include="$(FSharpSourcesRoot)\Compiler\Utilities\RidHelpers.fs" />
1720
<Compile Include="DependencyManagerInteractiveTests.fs" />

‎tests/FSharp.Compiler.Private.Scripting.UnitTests/FSharpScriptTests.fs

+15-18
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
namespace FSharp.Compiler.Scripting.UnitTests
44

55
open System
6+
open System.Text
67
open System.Diagnostics
78
open System.IO
89
open System.Reflection
@@ -11,6 +12,7 @@ open System.Threading
1112
open System.Threading.Tasks
1213
open FSharp.Compiler.Interactive
1314
open FSharp.Compiler.Interactive.Shell
15+
open FSharp.Test
1416
open FSharp.Test.ScriptHelpers
1517

1618
open Xunit
@@ -87,22 +89,20 @@ x
8789

8890
[<Fact>]
8991
member _.``Capture console input``() =
90-
use script = new FSharpScript(input = "stdin:1234\r\n")
92+
use _ = new TestConsole.ProvideInput("stdin:1234\r\n")
93+
use script = new FSharpScript()
9194
let opt = script.Eval("System.Console.ReadLine()") |> getValue
9295
let value = opt.Value
9396
Assert.Equal(typeof<string>, value.ReflectionType)
9497
Assert.Equal("stdin:1234", downcast value.ReflectionValue)
9598

9699
[<Fact>]
97100
member _.``Capture console output/error``() =
101+
use capture = new TestConsole.ExecutionCapture()
98102
use script = new FSharpScript()
99-
use sawOutputSentinel = new ManualResetEvent(false)
100-
use sawErrorSentinel = new ManualResetEvent(false)
101-
script.OutputProduced.Add (fun line -> if line = "stdout:1234" then sawOutputSentinel.Set() |> ignore)
102-
script.ErrorProduced.Add (fun line -> if line = "stderr:5678" then sawErrorSentinel.Set() |> ignore)
103103
script.Eval("printfn \"stdout:1234\"; eprintfn \"stderr:5678\"") |> ignoreValue
104-
Assert.True(sawOutputSentinel.WaitOne(TimeSpan.FromSeconds(5.0)), "Expected to see output sentinel value written")
105-
Assert.True(sawErrorSentinel.WaitOne(TimeSpan.FromSeconds(5.0)), "Expected to see error sentinel value written")
104+
Assert.Contains("stdout:1234", capture.OutText)
105+
Assert.Contains("stderr:5678", capture.ErrorText)
106106

107107
[<Fact>]
108108
member _.``Maintain state between submissions``() =
@@ -306,30 +306,26 @@ printfn "{@"%A"}" result
306306

307307
[<Fact>]
308308
member _.``Eval script with invalid PackageName should fail immediately``() =
309+
use capture = new TestConsole.ExecutionCapture()
309310
use script = new FSharpScript(additionalArgs=[| |])
310-
let mutable found = 0
311-
let outp = System.Collections.Generic.List<string>()
312-
script.OutputProduced.Add(
313-
fun line ->
314-
if line.Contains("error NU1101:") && line.Contains("FSharp.Really.Not.A.Package") then
315-
found <- found + 1
316-
outp.Add(line))
317311
let result, errors = script.Eval("""#r "nuget:FSharp.Really.Not.A.Package" """)
318-
Assert.True( (found = 0), "Did not expect to see output contains 'error NU1101:' and 'FSharp.Really.Not.A.Package'")
312+
313+
let lines = capture.OutText.Split([| Environment.NewLine |], StringSplitOptions.None)
314+
let found = lines |> Seq.exists (fun line -> line.Contains("error NU1101:") && line.Contains("FSharp.Really.Not.A.Package"))
315+
Assert.False(found, "Did not expect to see output contains 'error NU1101:' and 'FSharp.Really.Not.A.Package'")
319316
Assert.True( errors |> Seq.exists (fun error -> error.Message.Contains("error NU1101:")), "Expect to error containing 'error NU1101:'")
320317
Assert.True( errors |> Seq.exists (fun error -> error.Message.Contains("FSharp.Really.Not.A.Package")), "Expect to error containing 'FSharp.Really.Not.A.Package'")
321318

322319
[<Fact>]
323320
member _.``Eval script with invalid PackageName should fail immediately and resolve one time only``() =
321+
use capture = new TestConsole.ExecutionCapture()
324322
use script = new FSharpScript(additionalArgs=[| |])
325-
let mutable foundResolve = 0
326-
script.OutputProduced.Add (fun line -> if line.Contains("error NU1101:") then foundResolve <- foundResolve + 1)
327323
let result, errors =
328324
script.Eval("""
329325
#r "nuget:FSharp.Really.Not.A.Package"
330326
#r "nuget:FSharp.Really.Not.Another.Package"
331327
""")
332-
Assert.True( (foundResolve = 0), (sprintf "Did not expected to see 'error NU1101:' in output" ))
328+
Assert.DoesNotContain("error NU1101:", capture.OutText)
333329
Assert.Equal(2, (errors |> Seq.filter (fun error -> error.Message.Contains("error NU1101:")) |> Seq.length))
334330
Assert.Equal(1, (errors |> Seq.filter (fun error -> error.Message.Contains("FSharp.Really.Not.A.Package")) |> Seq.length))
335331
Assert.Equal(1, (errors |> Seq.filter (fun error -> error.Message.Contains("FSharp.Really.Not.Another.Package")) |> Seq.length))
@@ -479,6 +475,7 @@ let x =
479475
script.Eval(code) |> ignoreValue
480476
Assert.False(foundInner)
481477

478+
482479
[<Fact>]
483480
member _.``Script with nuget package that yields out of order dependencies works correctly``() =
484481
// regression test for: https://github.com/dotnet/fsharp/issues/9217

‎tests/FSharp.Compiler.Private.Scripting.UnitTests/xunit.runner.json

+1-2
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
{
22
"$schema": "https://xunit.net/schema/current/xunit.runner.schema.json",
3-
"appDomain": "ifAvailable",
4-
"shadowCopy": false,
3+
"appDomain": "denied",
54
"parallelizeTestCollections": false,
65
"maxParallelThreads": 1
76
}

‎tests/FSharp.Compiler.Service.Tests/Common.fs

+1-1
Original file line numberDiff line numberDiff line change
@@ -476,7 +476,7 @@ let assertRange
476476
[<AutoOpen>]
477477
module TempDirUtils =
478478
let getTempPath dir =
479-
Path.Combine(TestFramework.tempDirectoryOfThisTestRun, dir)
479+
Path.Combine(tempDirectoryOfThisTestRun.Value.FullName, dir)
480480

481481
/// Returns the file name part of a temp file name created with tryCreateTemporaryFileName ()
482482
/// and an added process id and thread id to ensure uniqueness between threads.

‎tests/FSharp.Compiler.Service.Tests/FSharp.Compiler.Service.Tests.fsproj

+3
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,9 @@
2121
</Content>
2222
<Compile Include="AssemblyInfo.fs" />
2323
<Compile Include="SurfaceArea.fs" Condition="'$(TargetFrameworkIdentifier)' == '.NETCoreApp'" />
24+
<Compile Include="..\FSharp.Test.Utilities\XunitSetup.fs">
25+
<Link>XunitSetup.fs</Link>
26+
</Compile>
2427
<Compile Include="Common.fs" />
2528
<Compile Include="GeneratedCodeSymbolsTests.fs" />
2629
<Compile Include="AssemblyReaderShim.fs" />

‎tests/FSharp.Compiler.Service.Tests/ProjectAnalysisTests.fs

+1-1
Original file line numberDiff line numberDiff line change
@@ -4401,7 +4401,7 @@ let ``Test Project33 extension methods`` () =
44014401
("GetValue", ["member"; "extmem"])]
44024402

44034403
module internal Project34 =
4404-
let directoryPath = createTemporaryDirectory "Project34"
4404+
let directoryPath = createTemporaryDirectory().FullName
44054405
let sourceFileName = Path.Combine(directoryPath, "Program.fs")
44064406
let dllName = Path.ChangeExtension(sourceFileName, ".dll")
44074407
let projFileName = Path.ChangeExtension(sourceFileName, ".fsproj")

‎tests/FSharp.Compiler.Service.Tests/xunit.runner.json

+2-3
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,4 @@
11
{
2-
"$schema": "https://xunit.net/schema/current/xunit.runner.schema.json",
3-
"appDomain": "ifAvailable",
4-
"shadowCopy": false
2+
"$schema": "https://xunit.net/schema/current/xunit.runner.schema.json",
3+
"appDomain": "denied"
54
}

‎tests/FSharp.Core.UnitTests/FSharp.Core.UnitTests.fsproj

+3
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,9 @@
2626
</PropertyGroup>
2727

2828
<ItemGroup>
29+
<Compile Include="..\FSharp.Test.Utilities\XunitSetup.fs">
30+
<Link>XunitSetup.fs</Link>
31+
</Compile>
2932
<Compile Include="TestFrameworkHelpers.fs" />
3033
<Compile Include="LibraryTestFx.fs" />
3134
<Compile Include="FSharp.Core\PrimTypes.fs" />

‎tests/FSharp.Core.UnitTests/xunit.runner.json

+4-5
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
{
2-
"$schema": "https://xunit.net/schema/current/xunit.runner.schema.json",
3-
"appDomain": "ifAvailable",
4-
"shadowCopy": false,
5-
"parallelizeTestCollections": false,
6-
"maxParallelThreads": 1
2+
"$schema": "https://xunit.net/schema/current/xunit.runner.schema.json",
3+
"appDomain": "denied",
4+
"parallelizeTestCollections": false,
5+
"maxParallelThreads": 1
76
}

0 commit comments

Comments
 (0)