Skip to content

Commit 5f788a5

Browse files
youssef-backport-botYoussef1313
youssef-backport-bot
and
Youssef1313
authored
[rel/3.8] Fix TestFailedException outcome not propagating to TestResult outcome (#5244)
Co-authored-by: Youssef1313 <[email protected]>
1 parent e71432a commit 5f788a5

File tree

4 files changed

+245
-5
lines changed

4 files changed

+245
-5
lines changed

‎src/Adapter/MSTest.TestAdapter/Execution/TestClassInfo.cs

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -427,6 +427,7 @@ internal TestResult GetResultOrRunClassInitialize(ITestContext testContext, stri
427427
return new TestResult()
428428
{
429429
TestFailureException = new TestFailedException(UTFUnitTestOutcome.Error, ex.TryGetMessage(), ex.TryGetStackTraceInformation()),
430+
Outcome = UTFUnitTestOutcome.Error,
430431
};
431432
}
432433
}
@@ -468,11 +469,15 @@ TestResult DoRun()
468469
}
469470
catch (TestFailedException ex)
470471
{
471-
result = new TestResult() { TestFailureException = ex };
472+
result = new TestResult() { TestFailureException = ex, Outcome = ex.Outcome };
472473
}
473474
catch (Exception ex)
474475
{
475-
result = new TestResult() { TestFailureException = new TestFailedException(UTFUnitTestOutcome.Error, ex.TryGetMessage(), ex.TryGetStackTraceInformation()) };
476+
result = new TestResult()
477+
{
478+
TestFailureException = new TestFailedException(UTFUnitTestOutcome.Error, ex.TryGetMessage(), ex.TryGetStackTraceInformation()),
479+
Outcome = UTFUnitTestOutcome.Error,
480+
};
476481
}
477482
finally
478483
{

‎src/Adapter/MSTest.TestAdapter/Execution/UnitTestRunner.cs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -261,12 +261,12 @@ private static TestResult RunAssemblyInitializeIfNeeded(TestMethodInfo testMetho
261261
}
262262
catch (TestFailedException ex)
263263
{
264-
result = new TestResult() { TestFailureException = ex };
264+
result = new TestResult() { TestFailureException = ex, Outcome = ex.Outcome };
265265
}
266266
catch (Exception ex)
267267
{
268268
var testFailureException = new TestFailedException(UnitTestOutcome.Error, ex.TryGetMessage(), ex.TryGetStackTraceInformation());
269-
result = new TestResult() { TestFailureException = testFailureException };
269+
result = new TestResult() { TestFailureException = testFailureException, Outcome = UnitTestOutcome.Error };
270270
}
271271
finally
272272
{

‎test/IntegrationTests/MSTest.Acceptance.IntegrationTests/InconclusiveTests.cs

Lines changed: 235 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,235 @@
1+
// Copyright (c) Microsoft Corporation. All rights reserved.
2+
// Licensed under the MIT license. See LICENSE file in the project root for full license information.
3+
4+
using Microsoft.Testing.Platform.Acceptance.IntegrationTests;
5+
using Microsoft.Testing.Platform.Acceptance.IntegrationTests.Helpers;
6+
using Microsoft.Testing.Platform.Helpers;
7+
8+
namespace MSTest.Acceptance.IntegrationTests;
9+
10+
[TestClass]
11+
public sealed class InconclusiveTests : AcceptanceTestBase<InconclusiveTests.TestAssetFixture>
12+
{
13+
public enum Lifecycle
14+
{
15+
AssemblyInitialize,
16+
ClassInitialize,
17+
TestInitialize,
18+
TestMethod,
19+
TestCleanup,
20+
ClassCleanup,
21+
AssemblyCleanup,
22+
}
23+
24+
[TestMethod]
25+
[DataRow(Lifecycle.AssemblyInitialize)]
26+
[DataRow(Lifecycle.ClassInitialize)]
27+
[DataRow(Lifecycle.TestInitialize)]
28+
[DataRow(Lifecycle.TestMethod)]
29+
[DataRow(Lifecycle.TestCleanup)]
30+
[DataRow(Lifecycle.ClassCleanup)]
31+
[DataRow(Lifecycle.AssemblyCleanup)]
32+
public async Task TestOutcomeShouldBeRespectedCorrectly(Lifecycle inconclusiveStep)
33+
{
34+
var testHost = TestHost.LocateFrom(AssetFixture.ProjectPath, TestAssetFixture.ProjectName, TargetFrameworks.NetCurrent);
35+
TestHostResult testHostResult = await testHost.ExecuteAsync(
36+
"--settings my.runsettings",
37+
environmentVariables: new Dictionary<string, string?>()
38+
{
39+
[$"{inconclusiveStep}Inconclusive"] = "1",
40+
});
41+
42+
if (inconclusiveStep >= Lifecycle.ClassCleanup)
43+
{
44+
testHostResult.AssertExitCodeIs(ExitCodes.AtLeastOneTestFailed);
45+
testHostResult.AssertOutputContainsSummary(failed: 1, passed: 0, skipped: 0);
46+
}
47+
else
48+
{
49+
testHostResult.AssertExitCodeIs(ExitCodes.ZeroTests);
50+
testHostResult.AssertOutputContainsSummary(failed: 0, passed: 0, skipped: 1);
51+
}
52+
53+
testHostResult.AssertOutputContains("AssemblyInitialize called");
54+
55+
if (inconclusiveStep >= Lifecycle.ClassInitialize)
56+
{
57+
testHostResult.AssertOutputContains("ClassInitialize called");
58+
}
59+
else
60+
{
61+
testHostResult.AssertOutputDoesNotContain("ClassInitialize called");
62+
}
63+
64+
if (inconclusiveStep >= Lifecycle.TestInitialize)
65+
{
66+
testHostResult.AssertOutputContains("TestInitialize called");
67+
}
68+
else
69+
{
70+
testHostResult.AssertOutputDoesNotContain("TestInitialize called");
71+
}
72+
73+
if (inconclusiveStep >= Lifecycle.TestMethod)
74+
{
75+
testHostResult.AssertOutputContains("TestMethod called");
76+
}
77+
else
78+
{
79+
testHostResult.AssertOutputDoesNotContain("TestMethod called");
80+
}
81+
82+
if (inconclusiveStep >= Lifecycle.TestInitialize)
83+
{
84+
testHostResult.AssertOutputContains("TestCleanup called");
85+
}
86+
else
87+
{
88+
testHostResult.AssertOutputDoesNotContain("TestInitialize called");
89+
}
90+
91+
if (inconclusiveStep >= Lifecycle.ClassInitialize)
92+
{
93+
testHostResult.AssertOutputContains("ClassCleanup called");
94+
}
95+
else
96+
{
97+
testHostResult.AssertOutputDoesNotContain("ClassCleanup called");
98+
}
99+
100+
if (inconclusiveStep is Lifecycle.AssemblyCleanup or <= Lifecycle.TestCleanup)
101+
{
102+
testHostResult.AssertOutputContains("AssemblyCleanup called");
103+
}
104+
else
105+
{
106+
testHostResult.AssertOutputDoesNotContain("AssemblyCleanup called");
107+
}
108+
}
109+
110+
public sealed class TestAssetFixture() : TestAssetFixtureBase(AcceptanceFixture.NuGetGlobalPackagesFolder)
111+
{
112+
public const string ProjectName = "TestInconclusive";
113+
114+
public string ProjectPath => GetAssetPath(ProjectName);
115+
116+
public override IEnumerable<(string ID, string Name, string Code)> GetAssetsToGenerate()
117+
{
118+
yield return (ProjectName, ProjectName,
119+
SourceCode
120+
.PatchTargetFrameworks(TargetFrameworks.NetCurrent)
121+
.PatchCodeWithReplace("$MSTestVersion$", MSTestVersion));
122+
}
123+
124+
private const string SourceCode = """
125+
#file TestInconclusive.csproj
126+
<Project Sdk="Microsoft.NET.Sdk">
127+
128+
<PropertyGroup>
129+
<OutputType>Exe</OutputType>
130+
<EnableMSTestRunner>true</EnableMSTestRunner>
131+
<TargetFrameworks>$TargetFrameworks$</TargetFrameworks>
132+
</PropertyGroup>
133+
134+
<ItemGroup>
135+
<PackageReference Include="MSTest.TestAdapter" Version="$MSTestVersion$" />
136+
<PackageReference Include="MSTest.TestFramework" Version="$MSTestVersion$" />
137+
</ItemGroup>
138+
139+
<ItemGroup>
140+
<None Update="*.runsettings">
141+
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
142+
</None>
143+
</ItemGroup>
144+
145+
</Project>
146+
147+
#file my.runsettings
148+
<RunSettings>
149+
<MSTest>
150+
<CaptureTraceOutput>false</CaptureTraceOutput>
151+
</MSTest>
152+
</RunSettings>
153+
154+
#file UnitTest1.cs
155+
using System;
156+
using System.Collections.Generic;
157+
using System.Threading.Tasks;
158+
using Microsoft.VisualStudio.TestTools.UnitTesting;
159+
160+
[TestClass]
161+
public class UnitTest1
162+
{
163+
[AssemblyInitialize]
164+
public static void AsmInitialize(TestContext _)
165+
{
166+
Console.WriteLine("AssemblyInitialize called");
167+
if (Environment.GetEnvironmentVariable("AssemblyInitializeInconclusive") == "1")
168+
{
169+
Assert.Inconclusive();
170+
}
171+
}
172+
173+
[ClassInitialize]
174+
public static void ClassInit(TestContext _)
175+
{
176+
Console.WriteLine("ClassInitialize called");
177+
if (Environment.GetEnvironmentVariable("ClassInitializeInconclusive") == "1")
178+
{
179+
Assert.Inconclusive();
180+
}
181+
}
182+
183+
[TestInitialize]
184+
public void TestInit()
185+
{
186+
Console.WriteLine("TestInitialize called");
187+
if (Environment.GetEnvironmentVariable("TestInitializeInconclusive") == "1")
188+
{
189+
Assert.Inconclusive();
190+
}
191+
}
192+
193+
[TestMethod]
194+
public void TestMethod()
195+
{
196+
Console.WriteLine("TestMethod called");
197+
if (Environment.GetEnvironmentVariable("TestMethodInconclusive") == "1")
198+
{
199+
Assert.Inconclusive();
200+
}
201+
}
202+
203+
[TestCleanup]
204+
public void TestCleanup()
205+
{
206+
Console.WriteLine("TestCleanup called");
207+
if (Environment.GetEnvironmentVariable("TestCleanupInconclusive") == "1")
208+
{
209+
Assert.Inconclusive();
210+
}
211+
}
212+
213+
[ClassCleanup]
214+
public static void ClassCleanup(TestContext _)
215+
{
216+
Console.WriteLine("ClassCleanup called");
217+
if (Environment.GetEnvironmentVariable("ClassCleanupInconclusive") == "1")
218+
{
219+
Assert.Inconclusive();
220+
}
221+
}
222+
223+
[AssemblyCleanup]
224+
public static void AsmCleanup(TestContext _)
225+
{
226+
Console.WriteLine("AssemblyCleanup called");
227+
if (Environment.GetEnvironmentVariable("AssemblyCleanupInconclusive") == "1")
228+
{
229+
Assert.Inconclusive();
230+
}
231+
}
232+
}
233+
""";
234+
}
235+
}

‎test/UnitTests/MSTestAdapter.UnitTests/Execution/TestClassInfoTests.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -208,7 +208,7 @@ public void RunClassInitializeShouldThrowIfTestContextIsNull()
208208
TestResult result = GetResultOrRunClassInitialize(null);
209209
var exception = result.TestFailureException as TestFailedException;
210210
Verify(exception is not null);
211-
Verify(result.Outcome == UTF.UnitTestOutcome.Failed);
211+
Verify(result.Outcome == UTF.UnitTestOutcome.Error);
212212
Verify(exception.Message == "TestContext cannot be Null.");
213213
}
214214

0 commit comments

Comments
 (0)