diff --git a/TestFx.sln b/TestFx.sln index 9c851aee28..9a296bd667 100644 --- a/TestFx.sln +++ b/TestFx.sln @@ -184,6 +184,20 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "TimeoutTestProjectNetCore", EndProject Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "FSharpTestProject", "test\E2ETests\TestAssets\FSharpTestProject\FSharpTestProject.fsproj", "{E5E58613-82FC-44CD-B75F-4F1C7ED52D0D}" EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "DiscoveryAndExecutionTests", "test\E2ETests\DiscoveryAndExecutionTests\DiscoveryAndExecutionTests.csproj", "{EEE57613-6424-4A1C-9635-B73768F2146D}" + ProjectSection(ProjectDependencies) = postProject + {E5E58613-82FC-44CD-B75F-4F1C7ED52D0D} = {E5E58613-82FC-44CD-B75F-4F1C7ED52D0D} + {98BA6D2C-1F3D-4636-8E1D-D4932B7A253D} = {98BA6D2C-1F3D-4636-8E1D-D4932B7A253D} + {A7EA583B-A2B0-47DA-A058-458F247C7575} = {A7EA583B-A2B0-47DA-A058-458F247C7575} + {BBC99A6B-4490-49DD-9C12-AF2C1E95576E} = {BBC99A6B-4490-49DD-9C12-AF2C1E95576E} + {B0FCE474-14BC-449A-91EA-A433342C0D63} = {B0FCE474-14BC-449A-91EA-A433342C0D63} + {4004757A-0080-4410-B90A-6169B20F151B} = {4004757A-0080-4410-B90A-6169B20F151B} + {7FB80AAB-7123-4416-B6CD-8D3D69AA83F1} = {7FB80AAB-7123-4416-B6CD-8D3D69AA83F1} + {5A4967CD-B527-4D43-81C2-4CA90EE10222} = {5A4967CD-B527-4D43-81C2-4CA90EE10222} + {9C1219E0-E775-47F9-9236-63F03F774801} = {9C1219E0-E775-47F9-9236-63F03F774801} + {7252D9E3-267D-442C-96BC-C73AEF3241D6} = {7252D9E3-267D-442C-96BC-C73AEF3241D6} + EndProjectSection +EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Extension.WinUI", "src\TestFramework\Extension.WinUI\Extension.WinUI.csproj", "{23B9D9A2-4AEE-47E6-97B5-060DF21539FB}" EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "NetCore", "NetCore", "{D11C6664-1C4E-48F0-AA92-7F5BADC6F82C}" @@ -359,12 +373,12 @@ Global {B0FCE474-14BC-449A-91EA-A433342C0D63}.Debug|x86.Build.0 = Debug|Any CPU {B0FCE474-14BC-449A-91EA-A433342C0D63}.Release|Any CPU.ActiveCfg = Release|Any CPU {B0FCE474-14BC-449A-91EA-A433342C0D63}.Release|Any CPU.Build.0 = Release|Any CPU - {B0FCE474-14BC-449A-91EA-A433342C0D63}.Release|ARM.ActiveCfg = Debug|Any CPU - {B0FCE474-14BC-449A-91EA-A433342C0D63}.Release|ARM.Build.0 = Debug|Any CPU - {B0FCE474-14BC-449A-91EA-A433342C0D63}.Release|x64.ActiveCfg = Debug|Any CPU - {B0FCE474-14BC-449A-91EA-A433342C0D63}.Release|x64.Build.0 = Debug|Any CPU - {B0FCE474-14BC-449A-91EA-A433342C0D63}.Release|x86.ActiveCfg = Debug|Any CPU - {B0FCE474-14BC-449A-91EA-A433342C0D63}.Release|x86.Build.0 = Debug|Any CPU + {B0FCE474-14BC-449A-91EA-A433342C0D63}.Release|ARM.ActiveCfg = Release|Any CPU + {B0FCE474-14BC-449A-91EA-A433342C0D63}.Release|ARM.Build.0 = Release|Any CPU + {B0FCE474-14BC-449A-91EA-A433342C0D63}.Release|x64.ActiveCfg = Release|Any CPU + {B0FCE474-14BC-449A-91EA-A433342C0D63}.Release|x64.Build.0 = Release|Any CPU + {B0FCE474-14BC-449A-91EA-A433342C0D63}.Release|x86.ActiveCfg = Release|Any CPU + {B0FCE474-14BC-449A-91EA-A433342C0D63}.Release|x86.Build.0 = Release|Any CPU {A7EA583B-A2B0-47DA-A058-458F247C7575}.Code Analysis Debug|Any CPU.ActiveCfg = Debug|Any CPU {A7EA583B-A2B0-47DA-A058-458F247C7575}.Code Analysis Debug|Any CPU.Build.0 = Debug|Any CPU {A7EA583B-A2B0-47DA-A058-458F247C7575}.Code Analysis Debug|ARM.ActiveCfg = Debug|Any CPU @@ -1181,6 +1195,30 @@ Global {E5E58613-82FC-44CD-B75F-4F1C7ED52D0D}.Release|x64.Build.0 = Release|Any CPU {E5E58613-82FC-44CD-B75F-4F1C7ED52D0D}.Release|x86.ActiveCfg = Release|Any CPU {E5E58613-82FC-44CD-B75F-4F1C7ED52D0D}.Release|x86.Build.0 = Release|Any CPU + {EEE57613-6424-4A1C-9635-B73768F2146D}.Code Analysis Debug|Any CPU.ActiveCfg = Debug|Any CPU + {EEE57613-6424-4A1C-9635-B73768F2146D}.Code Analysis Debug|Any CPU.Build.0 = Debug|Any CPU + {EEE57613-6424-4A1C-9635-B73768F2146D}.Code Analysis Debug|ARM.ActiveCfg = Debug|Any CPU + {EEE57613-6424-4A1C-9635-B73768F2146D}.Code Analysis Debug|ARM.Build.0 = Debug|Any CPU + {EEE57613-6424-4A1C-9635-B73768F2146D}.Code Analysis Debug|x64.ActiveCfg = Debug|Any CPU + {EEE57613-6424-4A1C-9635-B73768F2146D}.Code Analysis Debug|x64.Build.0 = Debug|Any CPU + {EEE57613-6424-4A1C-9635-B73768F2146D}.Code Analysis Debug|x86.ActiveCfg = Debug|Any CPU + {EEE57613-6424-4A1C-9635-B73768F2146D}.Code Analysis Debug|x86.Build.0 = Debug|Any CPU + {EEE57613-6424-4A1C-9635-B73768F2146D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {EEE57613-6424-4A1C-9635-B73768F2146D}.Debug|Any CPU.Build.0 = Debug|Any CPU + {EEE57613-6424-4A1C-9635-B73768F2146D}.Debug|ARM.ActiveCfg = Debug|Any CPU + {EEE57613-6424-4A1C-9635-B73768F2146D}.Debug|ARM.Build.0 = Debug|Any CPU + {EEE57613-6424-4A1C-9635-B73768F2146D}.Debug|x64.ActiveCfg = Debug|Any CPU + {EEE57613-6424-4A1C-9635-B73768F2146D}.Debug|x64.Build.0 = Debug|Any CPU + {EEE57613-6424-4A1C-9635-B73768F2146D}.Debug|x86.ActiveCfg = Debug|Any CPU + {EEE57613-6424-4A1C-9635-B73768F2146D}.Debug|x86.Build.0 = Debug|Any CPU + {EEE57613-6424-4A1C-9635-B73768F2146D}.Release|Any CPU.ActiveCfg = Release|Any CPU + {EEE57613-6424-4A1C-9635-B73768F2146D}.Release|Any CPU.Build.0 = Release|Any CPU + {EEE57613-6424-4A1C-9635-B73768F2146D}.Release|ARM.ActiveCfg = Release|Any CPU + {EEE57613-6424-4A1C-9635-B73768F2146D}.Release|ARM.Build.0 = Release|Any CPU + {EEE57613-6424-4A1C-9635-B73768F2146D}.Release|x64.ActiveCfg = Release|Any CPU + {EEE57613-6424-4A1C-9635-B73768F2146D}.Release|x64.Build.0 = Release|Any CPU + {EEE57613-6424-4A1C-9635-B73768F2146D}.Release|x86.ActiveCfg = Release|Any CPU + {EEE57613-6424-4A1C-9635-B73768F2146D}.Release|x86.Build.0 = Release|Any CPU {23B9D9A2-4AEE-47E6-97B5-060DF21539FB}.Code Analysis Debug|Any CPU.ActiveCfg = Debug|Any CPU {23B9D9A2-4AEE-47E6-97B5-060DF21539FB}.Code Analysis Debug|Any CPU.Build.0 = Debug|Any CPU {23B9D9A2-4AEE-47E6-97B5-060DF21539FB}.Code Analysis Debug|ARM.ActiveCfg = Debug|Any CPU @@ -1293,6 +1331,7 @@ Global {26F0B8EF-92D4-4A23-ACB4-D1B662F0EEBE} = {D53BD452-F69F-4FB3-8B98-386EDA28A4C8} {ED27A844-6870-4FE6-8FEF-3ABDD1ED6564} = {D53BD452-F69F-4FB3-8B98-386EDA28A4C8} {E5E58613-82FC-44CD-B75F-4F1C7ED52D0D} = {D53BD452-F69F-4FB3-8B98-386EDA28A4C8} + {EEE57613-6424-4A1C-9635-B73768F2146D} = {F1A27537-78D1-4BBD-8E76-ADB31BC0C2B4} {23B9D9A2-4AEE-47E6-97B5-060DF21539FB} = {E48AC786-E150-4F41-9A16-32F02E4493D8} {D11C6664-1C4E-48F0-AA92-7F5BADC6F82C} = {CA01DAF5-8D9D-496E-9AD3-94BB7FBB2D34} {F4E2876F-6E42-4DCF-B629-041A9DF7C579} = {24088844-2107-4DB2-8F3F-CBCA94FC4B28} diff --git a/scripts/Build.ps1 b/scripts/Build.ps1 index fd060482f6..c616c86d24 100644 --- a/scripts/Build.ps1 +++ b/scripts/Build.ps1 @@ -325,7 +325,7 @@ function Sync-PackageVersions { Replace-InFile -File $_ -RegEx $packageRegex -ReplaceWith (' - 16.9.1 + 16.10.0-release-20210421-08 11.0 \ No newline at end of file diff --git a/scripts/test.ps1 b/scripts/test.ps1 index cdf2239303..7a153cc590 100644 --- a/scripts/test.ps1 +++ b/scripts/test.ps1 @@ -39,7 +39,7 @@ $CurrentScriptDir = (Get-Item (Split-Path $MyInvocation.MyCommand.Path)) $env:TF_ROOT_DIR = $CurrentScriptDir.Parent.FullName $env:TF_TOOLS_DIR = Join-Path $env:TF_ROOT_DIR "tools" $env:DOTNET_CLI_VERSION = "6.0.100-alpha.1.21067.8" -$env:TF_TESTS_OUTDIR_PATTERN = "*.Tests" +$env:TF_TESTS_OUTDIR_PATTERN = "*Tests" $env:TF_UNITTEST_FILES_PATTERN = "*.UnitTests*.dll" $env:TF_COMPONENTTEST_FILES_PATTERN = "*.ComponentTests*.dll" $env:TF_E2ETEST_FILES_PATTERN = "*.E2ETests*.dll" @@ -95,7 +95,7 @@ function Invoke-Test # Get test assemblies from these folders that match the pattern specified. foreach($container in $testFolders) { - $testContainer = Get-ChildItem $container\* -Recurse -Include $env:TF_UNITTEST_FILES_PATTERN, $env:TF_COMPONENTTEST_FILES_PATTERN, $env:TF_E2ETEST_FILES_PATTERN + $testContainer = Get-ChildItem $container\* -Recurse -Include $env:TF_UNITTEST_FILES_PATTERN, $env:TF_COMPONENTTEST_FILES_PATTERN, $env:TF_E2ETEST_FILES_PATTERN, "DiscoveryAndExecutionTests.dll" $testContainerName = $testContainer.Name $testContainerPath = $testContainer.FullName diff --git a/src/Adapter/MSTest.CoreAdapter/Constants.cs b/src/Adapter/MSTest.CoreAdapter/Constants.cs index 98a758d912..6d930643ae 100644 --- a/src/Adapter/MSTest.CoreAdapter/Constants.cs +++ b/src/Adapter/MSTest.CoreAdapter/Constants.cs @@ -89,6 +89,9 @@ internal static class Constants internal static readonly TestProperty TfsTeamProjectProperty = TestProperty.Register(TfsTeamProject, TfsTeamProject, typeof(string), TestPropertyAttributes.Hidden, typeof(TestCase)); + internal static readonly TestProperty TestDynamicDataTypeProperty = TestProperty.Register("MSTest.DynamicDataType", "DynamicDataType", typeof(int), TestPropertyAttributes.Hidden, typeof(TestCase)); + + internal static readonly TestProperty TestDynamicDataProperty = TestProperty.Register("MSTest.DynamicData", "DynamicData", typeof(string[]), TestPropertyAttributes.Hidden, typeof(TestCase)); #endregion #region Private Constants diff --git a/src/Adapter/MSTest.CoreAdapter/Discovery/AssemblyEnumerator.cs b/src/Adapter/MSTest.CoreAdapter/Discovery/AssemblyEnumerator.cs index 91744c05a1..d90563a5d6 100644 --- a/src/Adapter/MSTest.CoreAdapter/Discovery/AssemblyEnumerator.cs +++ b/src/Adapter/MSTest.CoreAdapter/Discovery/AssemblyEnumerator.cs @@ -6,21 +6,34 @@ namespace Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.Discovery using System; using System.Collections.Generic; using System.Diagnostics; - using System.Diagnostics.CodeAnalysis; using System.Globalization; using System.Linq; using System.Reflection; using System.Security; using System.Text; + using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.Execution; using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.Helpers; using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.ObjectModel; + using Microsoft.VisualStudio.TestPlatform.MSTestAdapter.PlatformServices.Interface; + + using UTF = Microsoft.VisualStudio.TestTools.UnitTesting; /// /// Enumerates through all types in the assembly in search of valid test methods. /// internal class AssemblyEnumerator : MarshalByRefObject { + /// + /// Helper for reflection API's. + /// + private static readonly ReflectHelper ReflectHelper = ReflectHelper.Instance; + + /// + /// Type cache + /// + private readonly TypeCache typeCache = new TypeCache(ReflectHelper); + /// /// Initializes a new instance of the class. /// @@ -40,6 +53,11 @@ public AssemblyEnumerator(MSTestSettings settings) MSTestSettings.PopulateSettings(settings); } + /// + /// Gets or sets the run settings to use for current discovery session. + /// + public string RunSettingsXml { get; set; } + /// /// Returns object to be used for controlling lifetime, null means infinite lifetime. /// @@ -62,6 +80,7 @@ internal ICollection EnumerateAssembly(string assemblyFileName, { Debug.Assert(!string.IsNullOrWhiteSpace(assemblyFileName), "Invalid assembly file name."); + var runSettingsXml = this.RunSettingsXml; var warningMessages = new List(); var tests = new List(); @@ -88,40 +107,8 @@ internal ICollection EnumerateAssembly(string assemblyFileName, continue; } - string typeFullName = null; - - try - { - typeFullName = type.FullName; - var unitTestCases = this.GetTypeEnumerator(type, assemblyFileName).Enumerate(out var warningsFromTypeEnumerator); - - if (warningsFromTypeEnumerator != null) - { - warningMessages.AddRange(warningsFromTypeEnumerator); - } - - if (unitTestCases != null) - { - tests.AddRange(unitTestCases); - } - } - catch (Exception exception) - { - // If we fail to discover type from a class, then don't abort the discovery - // Move to the next type. - string message = string.Format( - CultureInfo.CurrentCulture, - Resource.CouldNotInspectTypeDuringDiscovery, - typeFullName, - assemblyFileName, - exception.Message); - warningMessages.Add(message); - - PlatformServiceProvider.Instance.AdapterTraceLogger.LogInfo( - "AssemblyEnumerator: Exception occurred while enumerating type {0}. {1}", - typeFullName, - exception); - } + var testsInType = this.DiscoverTestsInType(assemblyFileName, runSettingsXml, assembly, type, warningMessages); + tests.AddRange(testsInType); } warnings = warningMessages; @@ -144,20 +131,13 @@ internal Type[] GetTypes(Assembly assembly, string assemblyFileName, ICollection } catch (ReflectionTypeLoadException ex) { - PlatformServiceProvider.Instance.AdapterTraceLogger.LogWarning( - "MSTestExecutor.TryGetTests: Failed to discover tests from {0}. Reason:{1}", - assemblyFileName, - ex); - PlatformServiceProvider.Instance.AdapterTraceLogger.LogWarning("Exceptions thrown from the Loader :"); + PlatformServiceProvider.Instance.AdapterTraceLogger.LogWarning($"MSTestExecutor.TryGetTests: {Resource.TestAssembly_AssemblyDiscoveryFailure}", assemblyFileName, ex); + PlatformServiceProvider.Instance.AdapterTraceLogger.LogWarning(Resource.ExceptionsThrown); if (ex.LoaderExceptions != null) { // If not able to load all type, log a warning and continue with loaded types. - var message = string.Format( - CultureInfo.CurrentCulture, - Resource.TypeLoadFailed, - assemblyFileName, - this.GetLoadExceptionDetails(ex)); + var message = string.Format(CultureInfo.CurrentCulture, Resource.TypeLoadFailed, assemblyFileName, this.GetLoadExceptionDetails(ex)); warningMessages?.Add(message); @@ -215,11 +195,184 @@ internal string GetLoadExceptionDetails(ReflectionTypeLoadException ex) /// a TypeEnumerator instance. internal virtual TypeEnumerator GetTypeEnumerator(Type type, string assemblyFileName) { - var reflectHelper = new ReflectHelper(); - var typevalidator = new TypeValidator(reflectHelper); - var testMethodValidator = new TestMethodValidator(reflectHelper); + var typeValidator = new TypeValidator(ReflectHelper); + var testMethodValidator = new TestMethodValidator(ReflectHelper); + + return new TypeEnumerator(type, assemblyFileName, ReflectHelper, typeValidator, testMethodValidator); + } + + private IEnumerable DiscoverTestsInType(string assemblyFileName, string runSettingsXml, Assembly assembly, Type type, List warningMessages) + { + var sourceLevelParameters = PlatformServiceProvider.Instance.SettingsProvider.GetProperties(assemblyFileName); + sourceLevelParameters = RunSettingsUtilities.GetTestRunParameters(runSettingsXml)?.ConcatWithOverwrites(sourceLevelParameters) + ?? sourceLevelParameters + ?? new Dictionary(); + + string typeFullName = null; + var tests = new List(); + + try + { + typeFullName = type.FullName; + var unitTestCases = this.GetTypeEnumerator(type, assemblyFileName).Enumerate(out var warningsFromTypeEnumerator); + var typeIgnored = ReflectHelper.IsAttributeDefined(type, typeof(UTF.IgnoreAttribute), false); + + if (warningsFromTypeEnumerator != null) + { + warningMessages.AddRange(warningsFromTypeEnumerator); + } + + if (unitTestCases != null) + { + foreach (var test in unitTestCases) + { + if (this.DynamicDataAttached(sourceLevelParameters, assembly, test, tests)) + { + continue; + } + + tests.Add(test); + } + } + } + catch (Exception exception) + { + // If we fail to discover type from a class, then don't abort the discovery + // Move to the next type. + string message = string.Format(CultureInfo.CurrentCulture, Resource.CouldNotInspectTypeDuringDiscovery, typeFullName, assemblyFileName, exception.Message); + PlatformServiceProvider.Instance.AdapterTraceLogger.LogInfo($"AssemblyEnumerator: {message}"); + warningMessages.Add(message); + } + + return tests; + } + + private bool DynamicDataAttached(IDictionary sourceLevelParameters, Assembly assembly, UnitTestElement test, List tests) + { + // It should always be `true`, but any part of the chain is obsolete; it might not contain those. Since we depend on those properties, if they don't exist, we bail out early. + if (!test.TestMethod.HasManagedMethodAndTypeProperties) + { + return false; + } + + using (var writer = new ThreadSafeStringWriter(CultureInfo.InvariantCulture)) + { + var testMethod = test.TestMethod; + var testContext = PlatformServiceProvider.Instance.GetTestContext(testMethod, writer, sourceLevelParameters); + var testMethodInfo = this.typeCache.GetTestMethodInfo(testMethod, testContext, MSTestSettings.CurrentSettings.CaptureDebugTraces); + if (testMethodInfo == null) + { + return false; + } + + return /* DataSourceAttribute discovery is disabled for now, since we cannot serialize DataRow values. + this.TryProcessDataSource(test, testMethodInfo, testContext, tests) || */ + this.TryProcessTestDataSourceTests(test, testMethodInfo, tests); + } + } + + private bool TryProcessDataSource(UnitTestElement test, TestMethodInfo testMethodInfo, ITestContext testContext, List tests) + { + var dataSourceAttributes = ReflectHelper.GetAttributes(testMethodInfo.MethodInfo, false); + if (dataSourceAttributes == null) + { + return false; + } + + if (dataSourceAttributes.Length > 1) + { + var message = string.Format(CultureInfo.CurrentCulture, Resource.CannotEnumerateDataSourceAttribute_MoreThenOneDefined, test.TestMethod.ManagedTypeName, test.TestMethod.ManagedMethodName, dataSourceAttributes.Length); + PlatformServiceProvider.Instance.AdapterTraceLogger.LogInfo($"DynamicDataEnumarator: {message}"); + throw new InvalidOperationException(message); + } + + // dataSourceAttributes.Length == 1 + try + { + return this.ProcessDataSourceTests(test, testMethodInfo, testContext, tests); + } + catch (Exception ex) + { + var message = string.Format(CultureInfo.CurrentCulture, Resource.CannotEnumerateDataSourceAttribute, test.TestMethod.ManagedTypeName, test.TestMethod.ManagedMethodName, ex); + PlatformServiceProvider.Instance.AdapterTraceLogger.LogInfo($"DynamicDataEnumarator: {message}"); + return false; + } + } + + private bool ProcessDataSourceTests(UnitTestElement test, TestMethodInfo testMethodInfo, ITestContext testContext, List tests) + { + var dataRows = PlatformServiceProvider.Instance.TestDataSource.GetData(testMethodInfo, testContext); + if (dataRows == null || !dataRows.Any()) + { + return false; + } + + try + { + int rowIndex = 0; + + foreach (var dataRow in dataRows) + { + // TODO: Test serialization + rowIndex++; + + var displayName = string.Format(CultureInfo.CurrentCulture, Resource.DataDrivenResultDisplayName, test.DisplayName, rowIndex); + var discoveredTest = test.Clone(); + discoveredTest.DisplayName = displayName; + discoveredTest.TestMethod.DataType = DynamicDataType.DataSourceAttribute; + discoveredTest.TestMethod.Data = new[] { (object)rowIndex }; + tests.Add(discoveredTest); + } + + return true; + } + finally + { + testContext.SetDataConnection(null); + testContext.SetDataRow(null); + } + } + + private bool TryProcessTestDataSourceTests(UnitTestElement test, TestMethodInfo testMethodInfo, List tests) + { + var methodInfo = testMethodInfo.MethodInfo; + var testDataSources = ReflectHelper.GetAttributes(methodInfo, false)?.Where(a => a is UTF.ITestDataSource).OfType().ToArray(); + if (testDataSources == null || testDataSources.Length == 0) + { + return false; + } + + try + { + return this.ProcessTestDataSourceTests(test, (MethodInfo)methodInfo, testDataSources, tests); + } + catch (Exception ex) + { + var message = string.Format(CultureInfo.CurrentCulture, Resource.CannotEnumerateIDataSourceAttribute, test.TestMethod.ManagedTypeName, test.TestMethod.ManagedMethodName, ex); + PlatformServiceProvider.Instance.AdapterTraceLogger.LogInfo($"DynamicDataEnumarator: {message}"); + return false; + } + } + + private bool ProcessTestDataSourceTests(UnitTestElement test, MethodInfo methodInfo, UTF.ITestDataSource[] testDataSources, List tests) + { + foreach (var dataSource in testDataSources) + { + var data = dataSource.GetData(methodInfo); + + foreach (var d in data) + { + var discoveredTest = test.Clone(); + discoveredTest.DisplayName = dataSource.GetDisplayName(methodInfo, d); + + discoveredTest.TestMethod.DataType = DynamicDataType.ITestDataSource; + discoveredTest.TestMethod.Data = d; + + tests.Add(discoveredTest); + } + } - return new TypeEnumerator(type, assemblyFileName, reflectHelper, typevalidator, testMethodValidator); + return true; } } } diff --git a/src/Adapter/MSTest.CoreAdapter/Discovery/AssemblyEnumeratorWrapper.cs b/src/Adapter/MSTest.CoreAdapter/Discovery/AssemblyEnumeratorWrapper.cs index 2a1d192445..42f2a7a76e 100644 --- a/src/Adapter/MSTest.CoreAdapter/Discovery/AssemblyEnumeratorWrapper.cs +++ b/src/Adapter/MSTest.CoreAdapter/Discovery/AssemblyEnumeratorWrapper.cs @@ -69,7 +69,7 @@ internal ICollection GetTests(string assemblyFileName, IRunSett { var message = string.Format(CultureInfo.CurrentCulture, Resource.TestAssembly_AssemblyDiscoveryFailure, fullFilePath, ex.Message); PlatformServiceProvider.Instance.AdapterTraceLogger.LogWarning($"{nameof(AssemblyEnumeratorWrapper)}.{nameof(this.GetTests)}: {Resource.TestAssembly_AssemblyDiscoveryFailure}", fullFilePath, ex); - PlatformServiceProvider.Instance.AdapterTraceLogger.LogWarning("Exceptions thrown from the loader: "); + PlatformServiceProvider.Instance.AdapterTraceLogger.LogWarning(Resource.ExceptionsThrown); warnings.Add(message); if (ex.LoaderExceptions != null) @@ -108,6 +108,16 @@ private ICollection GetTestsInIsolation(string fullFilePath, IR // Create an instance of a type defined in adapter so that adapter gets loaded in the child app domain var assemblyEnumerator = isolationHost.CreateInstanceForType(typeof(AssemblyEnumerator), new object[] { MSTestSettings.CurrentSettings }) as AssemblyEnumerator; + // This might not be supported if an older version of "PlatformServices" (Microsoft.VisualStudio.TestPlatform.MSTestAdapter.PlatformServices) assembly is already loaded into the App Domain. + try + { + assemblyEnumerator.RunSettingsXml = runSettings?.SettingsXml; + } + catch + { + PlatformServiceProvider.Instance.AdapterTraceLogger.LogWarning(Resource.OlderTFMVersionFound); + } + return assemblyEnumerator.EnumerateAssembly(fullFilePath, out warnings); } } diff --git a/src/Adapter/MSTest.CoreAdapter/Discovery/TypeEnumerator.cs b/src/Adapter/MSTest.CoreAdapter/Discovery/TypeEnumerator.cs index 5c585a579a..7695454844 100644 --- a/src/Adapter/MSTest.CoreAdapter/Discovery/TypeEnumerator.cs +++ b/src/Adapter/MSTest.CoreAdapter/Discovery/TypeEnumerator.cs @@ -198,11 +198,10 @@ internal UnitTestElement GetTestFromMethod(MethodInfo method, bool isDeclaredInT testElement.WorkItemIds = workItemAttributes.Select(x => x.Id.ToString()).ToArray(); } + testElement.Ignored = this.reflectHelper.IsAttributeDefined(method, typeof(IgnoreAttribute), false); + // Get Deployment items if any. - testElement.DeploymentItems = PlatformServiceProvider.Instance.TestDeployment.GetDeploymentItems( - method, - this.type, - warnings); + testElement.DeploymentItems = PlatformServiceProvider.Instance.TestDeployment.GetDeploymentItems(method, this.type, warnings); // get DisplayName from TestMethodAttribute var testMethodAttribute = this.reflectHelper.GetCustomAttribute(method, typeof(TestMethodAttribute)) as TestMethodAttribute; diff --git a/src/Adapter/MSTest.CoreAdapter/Execution/TestAssemblySettingsProvider.cs b/src/Adapter/MSTest.CoreAdapter/Execution/TestAssemblySettingsProvider.cs index a5d904d53c..b21eca5b22 100644 --- a/src/Adapter/MSTest.CoreAdapter/Execution/TestAssemblySettingsProvider.cs +++ b/src/Adapter/MSTest.CoreAdapter/Execution/TestAssemblySettingsProvider.cs @@ -5,6 +5,7 @@ namespace Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.Execution { using System; using System.Security; + using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.Helpers; using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.ObjectModel; @@ -13,7 +14,7 @@ internal class TestAssemblySettingsProvider : MarshalByRefObject private ReflectHelper reflectHelper; public TestAssemblySettingsProvider() - : this(new ReflectHelper()) + : this(ReflectHelper.Instance) { } diff --git a/src/Adapter/MSTest.CoreAdapter/Execution/TestMethodInfo.cs b/src/Adapter/MSTest.CoreAdapter/Execution/TestMethodInfo.cs index 949fe5acd7..6a3e6b9f71 100644 --- a/src/Adapter/MSTest.CoreAdapter/Execution/TestMethodInfo.cs +++ b/src/Adapter/MSTest.CoreAdapter/Execution/TestMethodInfo.cs @@ -11,10 +11,12 @@ namespace Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.Execution using System.Reflection; using System.Text; using System.Threading; + using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.Extensions; using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.Helpers; using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.ObjectModel; using Microsoft.VisualStudio.TestTools.UnitTesting; + using UnitTestOutcome = Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.ObjectModel.UnitTestOutcome; using UTF = Microsoft.VisualStudio.TestTools.UnitTesting; @@ -91,29 +93,7 @@ public Attribute[] GetAllAttributes(bool inherit) } public TAttributeType[] GetAttributes(bool inherit) - where TAttributeType : Attribute - { - Attribute[] attributeArray = ReflectHelper.GetCustomAttributes(this.TestMethod, typeof(TAttributeType), inherit); - - if (attributeArray is TAttributeType[] tAttributeArray) - { - return tAttributeArray; - } - - List tAttributeList = new List(); - if (attributeArray != null) - { - foreach (Attribute attribute in attributeArray) - { - if (attribute is TAttributeType tAttribute) - { - tAttributeList.Add(tAttribute); - } - } - } - - return tAttributeList.ToArray(); - } + where TAttributeType : Attribute => ReflectHelper.GetAttributes(this.TestMethod, inherit); /// /// Execute test method. Capture failures, handle async and return result. diff --git a/src/Adapter/MSTest.CoreAdapter/Execution/TestMethodRunner.cs b/src/Adapter/MSTest.CoreAdapter/Execution/TestMethodRunner.cs index b05ce4abef..45d5886935 100644 --- a/src/Adapter/MSTest.CoreAdapter/Execution/TestMethodRunner.cs +++ b/src/Adapter/MSTest.CoreAdapter/Execution/TestMethodRunner.cs @@ -69,7 +69,7 @@ public TestMethodRunner( TestMethod testMethod, ITestContext testContext, bool captureDebugTraces) - : this(testMethodInfo, testMethod, testContext, captureDebugTraces, new ReflectHelper()) + : this(testMethodInfo, testMethod, testContext, captureDebugTraces, ReflectHelper.Instance) { Debug.Assert(testMethodInfo != null, "testMethodInfo should not be null"); Debug.Assert(testMethod != null, "testMethod should not be null"); @@ -219,136 +219,30 @@ internal UnitTestResult[] RunTestMethod() List results = new List(); var isDataDriven = false; - // Parent result. Added in properties bag only when results are greater than 1. - var parentResultWatch = new Stopwatch(); - parentResultWatch.Start(); - var parentResult = new UTF.TestResult - { - Outcome = UTF.UnitTestOutcome.InProgress, - ExecutionId = Guid.NewGuid() - }; - if (this.testMethodInfo.TestMethodOptions.Executor != null) { - UTF.DataSourceAttribute[] dataSourceAttribute = this.testMethodInfo.GetAttributes(false); - if (dataSourceAttribute != null && dataSourceAttribute.Length == 1) + if (this.test.DataType == DynamicDataType.ITestDataSource) + { + var testResults = this.ExecuteTestWithDataSource(null, this.test.Data); + results.AddRange(testResults); + } + else if (this.ExecuteDataSourceBasedTests(results)) { isDataDriven = true; - Stopwatch watch = new Stopwatch(); - watch.Start(); - - try - { - IEnumerable dataRows = PlatformServiceProvider.Instance.TestDataSource.GetData(this.testMethodInfo, this.testContext); - - if (dataRows == null) - { - watch.Stop(); - var inconclusiveResult = new UTF.TestResult(); - inconclusiveResult.Outcome = UTF.UnitTestOutcome.Inconclusive; - inconclusiveResult.Duration = watch.Elapsed; - results.Add(inconclusiveResult); - } - else - { - try - { - int rowIndex = 0; - foreach (object dataRow in dataRows) - { - watch.Reset(); - watch.Start(); - - this.testContext.SetDataRow(dataRow); - UTF.TestResult[] testResults; - - try - { - testResults = this.testMethodInfo.TestMethodOptions.Executor.Execute(this.testMethodInfo); - } - catch (Exception ex) - { - testResults = new[] - { - new UTF.TestResult() { TestFailureException = new Exception(string.Format(CultureInfo.CurrentCulture, Resource.UTA_ExecuteThrewException, ex?.Message, ex?.StackTrace), ex) } - }; - } - - watch.Stop(); - foreach (var testResult in testResults) - { - testResult.DatarowIndex = rowIndex; - testResult.Duration = watch.Elapsed; - } - - rowIndex++; - - results.AddRange(testResults); - } - } - finally - { - this.testContext.SetDataConnection(null); - this.testContext.SetDataRow(null); - } - } - } - catch (Exception ex) - { - watch.Stop(); - var failedResult = new UTF.TestResult(); - failedResult.Outcome = UTF.UnitTestOutcome.Error; - failedResult.TestFailureException = ex; - failedResult.Duration = watch.Elapsed; - results.Add(failedResult); - } } else { - UTF.ITestDataSource[] testDataSources = this.testMethodInfo.GetAttributes(false)?.Where(a => a is UTF.ITestDataSource).OfType().ToArray(); + var testResults = this.ExecuteTest(); - if (testDataSources != null && testDataSources.Length > 0) - { - isDataDriven = true; - foreach (var testDataSource in testDataSources) - { - foreach (var data in testDataSource.GetData(this.testMethodInfo.MethodInfo)) - { - this.testMethodInfo.SetArguments(data); - UTF.TestResult[] testResults; - try - { - testResults = this.testMethodInfo.TestMethodOptions.Executor.Execute(this.testMethodInfo); - } - catch (Exception ex) - { - testResults = new[] - { - new UTF.TestResult() { TestFailureException = new Exception(string.Format(CultureInfo.CurrentCulture, Resource.UTA_ExecuteThrewException, ex?.Message, ex?.StackTrace), ex) } - }; - } - - foreach (var testResult in testResults) - { - testResult.DisplayName = testDataSource.GetDisplayName(this.testMethodInfo.MethodInfo, data); - } - - results.AddRange(testResults); - this.testMethodInfo.SetArguments(null); - } - } - } - else + foreach (var testResult in testResults) { - try - { - results.AddRange(this.testMethodInfo.TestMethodOptions.Executor.Execute(this.testMethodInfo)); - } - catch (Exception ex) + if (string.IsNullOrWhiteSpace(testResult.DisplayName)) { - results.Add(new UTF.TestResult() { TestFailureException = new Exception(string.Format(CultureInfo.CurrentCulture, Resource.UTA_ExecuteThrewException, ex?.Message, ex?.StackTrace), ex) }); + testResult.DisplayName = this.test.DisplayName; } } + + results.AddRange(testResults); } } else @@ -359,9 +253,6 @@ internal UnitTestResult[] RunTestMethod() this.testMethodInfo.TestMethodName); } - parentResultWatch.Stop(); - parentResult.Duration = parentResultWatch.Elapsed; - // Get aggregate outcome. var aggregateOutcome = this.GetAggregateOutcome(results); this.testContext.SetOutcome(aggregateOutcome); @@ -375,13 +266,163 @@ internal UnitTestResult[] RunTestMethod() // In case of data driven, set parent info in results. if (isDataDriven) { - parentResult.Outcome = aggregateOutcome; - results = this.UpdateResultsWithParentInfo(results, parentResult); + results = this.UpdateResultsWithParentInfo(results, Guid.NewGuid()); } return results.ToArray().ToUnitTestResults(); } + private bool ExecuteDataSourceBasedTests(List results) + { + var isDataDriven = false; + + UTF.DataSourceAttribute[] dataSourceAttribute = this.testMethodInfo.GetAttributes(false); + if (dataSourceAttribute != null && dataSourceAttribute.Length == 1) + { + isDataDriven = true; + Stopwatch watch = new Stopwatch(); + watch.Start(); + + try + { + IEnumerable dataRows = PlatformServiceProvider.Instance.TestDataSource.GetData(this.testMethodInfo, this.testContext); + + if (dataRows == null) + { + watch.Stop(); + var inconclusiveResult = new UTF.TestResult(); + inconclusiveResult.Outcome = UTF.UnitTestOutcome.Inconclusive; + inconclusiveResult.Duration = watch.Elapsed; + results.Add(inconclusiveResult); + } + else + { + try + { + int rowIndex = 0; + + foreach (object dataRow in dataRows) + { + UTF.TestResult[] testResults = this.ExecuteTestWithDataRow(dataRow, rowIndex++); + results.AddRange(testResults); + } + } + finally + { + this.testContext.SetDataConnection(null); + this.testContext.SetDataRow(null); + } + } + } + catch (Exception ex) + { + watch.Stop(); + var failedResult = new UTF.TestResult(); + failedResult.Outcome = UTF.UnitTestOutcome.Error; + failedResult.TestFailureException = ex; + failedResult.Duration = watch.Elapsed; + results.Add(failedResult); + } + } + else + { + UTF.ITestDataSource[] testDataSources = this.testMethodInfo.GetAttributes(false)?.Where(a => a is UTF.ITestDataSource).OfType().ToArray(); + + if (testDataSources != null && testDataSources.Length > 0) + { + isDataDriven = true; + foreach (var testDataSource in testDataSources) + { + foreach (var data in testDataSource.GetData(this.testMethodInfo.MethodInfo)) + { + try + { + var testResults = this.ExecuteTestWithDataSource(testDataSource, data); + + results.AddRange(testResults); + } + finally + { + this.testMethodInfo.SetArguments(null); + } + } + } + } + } + + return isDataDriven; + } + + private UTF.TestResult[] ExecuteTestWithDataSource(UTF.ITestDataSource testDataSource, object[] data) + { + var stopwatch = Stopwatch.StartNew(); + + this.testMethodInfo.SetArguments(data); + var testResults = this.ExecuteTest(); + stopwatch.Stop(); + + var hasDisplayName = !string.IsNullOrWhiteSpace(this.test.DisplayName); + foreach (var testResult in testResults) + { + if (testResult.Duration == TimeSpan.Zero) + { + testResult.Duration = stopwatch.Elapsed; + } + + var displayName = this.test.Name; + if (testDataSource != null) + { + displayName = testDataSource.GetDisplayName(this.testMethodInfo.MethodInfo, data); + } + else if (hasDisplayName) + { + displayName = this.test.DisplayName; + } + + testResult.DisplayName = displayName; + } + + return testResults; + } + + private UTF.TestResult[] ExecuteTestWithDataRow(object dataRow, int rowIndex) + { + var stopwatch = Stopwatch.StartNew(); + + this.testContext.SetDataRow(dataRow); + var testResults = this.ExecuteTest(); + stopwatch.Stop(); + + var displayName = string.Format(CultureInfo.CurrentCulture, Resource.DataDrivenResultDisplayName, this.test.DisplayName, rowIndex); + + foreach (var testResult in testResults) + { + testResult.DisplayName = displayName; + testResult.DatarowIndex = rowIndex; + testResult.Duration = stopwatch.Elapsed; + } + + return testResults; + } + + private UTF.TestResult[] ExecuteTest() + { + try + { + return this.testMethodInfo.TestMethodOptions.Executor.Execute(this.testMethodInfo); + } + catch (Exception ex) + { + return new[] + { + new UTF.TestResult() + { + TestFailureException = new Exception(string.Format(CultureInfo.CurrentCulture, Resource.UTA_ExecuteThrewException, ex?.Message, ex?.StackTrace), ex) + } + }; + } + } + /// /// Gets aggregate outcome. /// @@ -410,9 +451,9 @@ private UTF.UnitTestOutcome GetAggregateOutcome(List results) /// Add parent results as first result in updated result. /// /// Results. - /// Parent results. + /// Current execution id. /// Updated results which contains parent result as first result. All other results contains parent result info. - private List UpdateResultsWithParentInfo(List results, UTF.TestResult parentResult) + private List UpdateResultsWithParentInfo(List results, Guid executionId) { // Return results in case there are no results. if (!results.Any()) @@ -422,13 +463,11 @@ private UTF.UnitTestOutcome GetAggregateOutcome(List results) // UpdatedResults contain parent result at first position and remaining results has parent info updated. var updatedResults = new List(); - updatedResults.Add(parentResult); foreach (var result in results) { result.ExecutionId = Guid.NewGuid(); - result.ParentExecId = parentResult.ExecutionId; - parentResult.InnerResultsCount++; + result.ParentExecId = executionId; updatedResults.Add(result); } diff --git a/src/Adapter/MSTest.CoreAdapter/Execution/TypeCache.cs b/src/Adapter/MSTest.CoreAdapter/Execution/TypeCache.cs index f95be41da7..058f1a0573 100644 --- a/src/Adapter/MSTest.CoreAdapter/Execution/TypeCache.cs +++ b/src/Adapter/MSTest.CoreAdapter/Execution/TypeCache.cs @@ -54,7 +54,7 @@ internal class TypeCache : MarshalByRefObject /// Initializes a new instance of the class. /// internal TypeCache() - : this(new ReflectHelper()) + : this(ReflectHelper.Instance) { } diff --git a/src/Adapter/MSTest.CoreAdapter/Execution/UnitTestRunner.cs b/src/Adapter/MSTest.CoreAdapter/Execution/UnitTestRunner.cs index 597e2a6ca8..d419a98029 100644 --- a/src/Adapter/MSTest.CoreAdapter/Execution/UnitTestRunner.cs +++ b/src/Adapter/MSTest.CoreAdapter/Execution/UnitTestRunner.cs @@ -28,7 +28,7 @@ internal class UnitTestRunner : MarshalByRefObject /// /// Specifies adapter settings that need to be instantiated in the domain running these tests. public UnitTestRunner(MSTestSettings settings) - : this(settings, new ReflectHelper()) + : this(settings, ReflectHelper.Instance) { } diff --git a/src/Adapter/MSTest.CoreAdapter/Extensions/TestCaseExtensions.cs b/src/Adapter/MSTest.CoreAdapter/Extensions/TestCaseExtensions.cs index 2a11160ac4..2657bfc34a 100644 --- a/src/Adapter/MSTest.CoreAdapter/Extensions/TestCaseExtensions.cs +++ b/src/Adapter/MSTest.CoreAdapter/Extensions/TestCaseExtensions.cs @@ -3,8 +3,10 @@ namespace Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.Extensions { + using System.Collections.Generic; + using System.Linq; + using Microsoft.TestPlatform.AdapterUtilities; - using Microsoft.TestPlatform.AdapterUtilities.ManagedNameUtilities; using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.ObjectModel; using Microsoft.VisualStudio.TestPlatform.ObjectModel; @@ -16,8 +18,8 @@ namespace Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.Extensions internal static class TestCaseExtensions { internal static readonly TestProperty ManagedTypeProperty = TestProperty.Register( - id: Contants.ManagedTypePropertyId, - label: Contants.ManagedTypeLabel, + id: ManagedNameConstants.ManagedTypePropertyId, + label: ManagedNameConstants.ManagedTypeLabel, category: string.Empty, description: string.Empty, valueType: typeof(string), @@ -26,8 +28,8 @@ internal static class TestCaseExtensions owner: typeof(TestCase)); internal static readonly TestProperty ManagedMethodProperty = TestProperty.Register( - id: Contants.ManagedMethodPropertyId, - label: Contants.ManagedMethodLabel, + id: ManagedNameConstants.ManagedMethodPropertyId, + label: ManagedNameConstants.ManagedMethodLabel, category: string.Empty, description: string.Empty, valueType: typeof(string), @@ -35,6 +37,16 @@ internal static class TestCaseExtensions attributes: TestPropertyAttributes.Hidden, owner: typeof(TestCase)); + internal static readonly TestProperty HierarchyProperty = TestProperty.Register( + id: HierarchyConstants.HierarchyPropertyId, + label: HierarchyConstants.HierarchyLabel, + category: string.Empty, + description: string.Empty, + valueType: typeof(string[]), + validateValueCallback: null, + attributes: TestPropertyAttributes.Immutable, + owner: typeof(TestCase)); + /// /// The to unit test element. /// @@ -57,13 +69,24 @@ internal static UnitTestElement ToUnitTestElement(this TestCase testCase, string TestMethod testMethod; if (testCase.ContainsManagedMethodAndType()) { - testMethod = new TestMethod(testCase.GetManagedType(), testCase.GetManagedMethod(), name, testClassName, source, isAsync); + testMethod = new TestMethod(testCase.GetManagedType(), testCase.GetManagedMethod(), testCase.GetHierarchy(), name, testClassName, source, isAsync); } else { testMethod = new TestMethod(name, testClassName, source, isAsync); } + var dataType = (DynamicDataType)testCase.GetPropertyValue(Constants.TestDynamicDataTypeProperty, (int)DynamicDataType.None); + if (dataType != DynamicDataType.None) + { + var data = testCase.GetPropertyValue(Constants.TestDynamicDataProperty, null); + + testMethod.DataType = dataType; + testMethod.Data = Helpers.DataSerializationHelper.Deserialize(data); + } + + testMethod.DisplayName = testCase.DisplayName; + if (declaringClassName != null && declaringClassName != testClassName) { testMethod.DeclaringClassFullName = declaringClassName; @@ -77,6 +100,43 @@ internal static UnitTestElement ToUnitTestElement(this TestCase testCase, string DisplayName = testCase.DisplayName }; + if (testCase.Traits.Any()) + { + testElement.Traits = testCase.Traits.ToArray(); + } + + var cssIteration = testCase.GetPropertyValue(Constants.CssIterationProperty, null); + if (!string.IsNullOrWhiteSpace(cssIteration)) + { + testElement.CssIteration = cssIteration; + } + + var cssProjectStructure = testCase.GetPropertyValue(Constants.CssProjectStructureProperty, null); + if (!string.IsNullOrWhiteSpace(cssIteration)) + { + testElement.CssProjectStructure = cssProjectStructure; + } + + var description = testCase.GetPropertyValue(Constants.DescriptionProperty, null); + if (!string.IsNullOrWhiteSpace(description)) + { + testElement.Description = description; + } + + var workItemIds = testCase.GetPropertyValue(Constants.WorkItemIdsProperty, null); + if (workItemIds != null && workItemIds.Length > 0) + { + testElement.WorkItemIds = workItemIds; + } + + var deploymentItems = testCase.GetPropertyValue[]>(Constants.DeploymentItemsProperty, null); + if (deploymentItems != null && deploymentItems.Length > 0) + { + testElement.DeploymentItems = deploymentItems; + } + + testElement.DoNotParallelize = testCase.GetPropertyValue(Constants.DoNotParallelizeProperty, false); + return testElement; } @@ -89,5 +149,9 @@ internal static UnitTestElement ToUnitTestElement(this TestCase testCase, string internal static void SetManagedMethod(this TestCase testCase, string value) => testCase.SetPropertyValue(ManagedMethodProperty, value); internal static bool ContainsManagedMethodAndType(this TestCase testCase) => !string.IsNullOrWhiteSpace(testCase.GetManagedMethod()) && !string.IsNullOrWhiteSpace(testCase.GetManagedType()); + + internal static string[] GetHierarchy(this TestCase testCase) => testCase.GetPropertyValue(HierarchyProperty, null); + + internal static void SetHierarchy(this TestCase testCase, params string[] value) => testCase.SetPropertyValue(HierarchyProperty, value); } } diff --git a/src/Adapter/MSTest.CoreAdapter/Friends.cs b/src/Adapter/MSTest.CoreAdapter/Friends.cs index d4224b84e8..0b478fdb31 100644 --- a/src/Adapter/MSTest.CoreAdapter/Friends.cs +++ b/src/Adapter/MSTest.CoreAdapter/Friends.cs @@ -6,4 +6,5 @@ [assembly: InternalsVisibleTo(assemblyName: "Microsoft.VisualStudio.TestPlatform.MSTestAdapter.UnitTests, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] [assembly: InternalsVisibleTo(assemblyName: "DynamicProxyGenAssembly2, PublicKey=0024000004800000940000000602000000240000525341310004000001000100c547cac37abd99c8db225ef2f6c8a3602f3b3606cc9891605d02baa56104f4cfc0734aa39b93bf7852f7d9266654753cc297e7d2edfe0bac1cdcf9f717241550e0a7b191195b7667bb4f64bcb8e2121380fd1d9d46ad2d92d2d15605093924cceaf74c4861eff62abf69b9291ed0a340e113be11e6a7d3113e92484cf7045cc7")] -[assembly: InternalsVisibleTo(assemblyName: "MSTestAdapter.Smoke.E2ETests, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] \ No newline at end of file +[assembly: InternalsVisibleTo(assemblyName: "MSTestAdapter.Smoke.E2ETests, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] +[assembly: InternalsVisibleTo(assemblyName: "DiscoveryAndExecutionTests, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] diff --git a/src/Adapter/MSTest.CoreAdapter/Helpers/DataSerializationHelper.cs b/src/Adapter/MSTest.CoreAdapter/Helpers/DataSerializationHelper.cs new file mode 100644 index 0000000000..b1e30825c7 --- /dev/null +++ b/src/Adapter/MSTest.CoreAdapter/Helpers/DataSerializationHelper.cs @@ -0,0 +1,94 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.Helpers +{ + using System.IO; + using System.Runtime.Serialization.Json; + using System.Text; + + internal static class DataSerializationHelper + { + /// + /// Serializes the date in such a way that won't throw exceptions during deserialization in Test Platform. + /// The result can be deserialized using method. + /// + /// Data array to serialize. + /// Serialzed array. + public static string[] Serialize(object[] data) + { + if (data == null) + { + return null; + } + + var serializer = GetSerializer(); + var serializedData = new string[data.Length]; + + for (int i = 0; i < data.Length; i++) + { + if (data[i] == null) + { + serializedData[i] = null; + continue; + } + + using (var memoryStream = new MemoryStream()) + { + serializer.WriteObject(memoryStream, data[i]); + var serializerData = memoryStream.ToArray(); + + serializedData[i] = Encoding.UTF8.GetString(serializerData, 0, serializerData.Length); + } + } + + return serializedData; + } + + /// + /// Deserialzes the data serialzed by method. + /// + /// Serialized data array to deserialize. + /// Deserialized array. + public static object[] Deserialize(string[] serializedData) + { + if (serializedData == null) + { + return null; + } + + var serializer = GetSerializer(); + var data = new object[serializedData.Length]; + + for (int i = 0; i < serializedData.Length; i++) + { + if (serializedData[i] == null) + { + data[i] = null; + continue; + } + + var serialzedDataBytes = Encoding.UTF8.GetBytes(serializedData[i]); + using (var memoryStream = new MemoryStream(serialzedDataBytes)) + { + data[i] = serializer.ReadObject(memoryStream); + } + } + + return data; + } + + private static DataContractJsonSerializer GetSerializer() + { + var settings = new DataContractJsonSerializerSettings() + { + UseSimpleDictionaryFormat = true, + EmitTypeInformation = System.Runtime.Serialization.EmitTypeInformation.Always + }; + + var serializer = new DataContractJsonSerializer(typeof(object), settings); + + return serializer; + } + } +} diff --git a/src/Adapter/MSTest.CoreAdapter/Helpers/ReflectHelper.cs b/src/Adapter/MSTest.CoreAdapter/Helpers/ReflectHelper.cs index 22224a4058..7c7a681ca8 100644 --- a/src/Adapter/MSTest.CoreAdapter/Helpers/ReflectHelper.cs +++ b/src/Adapter/MSTest.CoreAdapter/Helpers/ReflectHelper.cs @@ -18,10 +18,18 @@ namespace Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.Helpers internal class ReflectHelper : MarshalByRefObject { + private static readonly Lazy InstanceValue = new Lazy(() => new ReflectHelper()); + /// /// Contains the memberInfo Vs the name/type of the attributes defined on that member. (FYI: - MemberInfo denotes properties, fields, methods, events) /// - private Dictionary> attributeCache = new Dictionary>(); + private readonly Dictionary> attributeCache = new Dictionary>(); + + internal ReflectHelper() + { + } + + public static ReflectHelper Instance => InstanceValue.Value; /// /// Checks to see if the parameter memberInfo contains the parameter attribute or not. @@ -113,7 +121,7 @@ public bool HasAttributeDerivedFrom(MemberInfo memberInfo, Type baseAttributeTyp Dictionary attributes = this.GetAttributes(memberInfo, inherit); if (attributes == null) { - PlatformServiceProvider.Instance.AdapterTraceLogger.LogWarning("ReflectHelper.HasAttributeDerivedFrom: Failed to get attribute cache. Ignoring attribute inheritance and falling into 'type defines Attribute model', so that we have some data."); + PlatformServiceProvider.Instance.AdapterTraceLogger.LogWarning($"{nameof(ReflectHelper)}.{nameof(GetAttributes)}: {Resource.FailedFetchAttributeCache}"); return this.IsAttributeDefined(memberInfo, baseAttributeType, inherit); } @@ -121,7 +129,7 @@ public bool HasAttributeDerivedFrom(MemberInfo memberInfo, Type baseAttributeTyp // Try to find the attribute that is derived from baseAttrType. foreach (object attribute in attributes.Values) { - Debug.Assert(attribute != null, "ReflectHeler.DefinesAttributeDerivedFrom: internal error: wrong value in the attrs dictionary."); + Debug.Assert(attribute != null, $"{nameof(ReflectHelper)}.{nameof(GetAttributes)}: internal error: wrong value in the attrs dictionary."); Type attributeType = attribute.GetType(); if (attributeType.GetTypeInfo().IsSubclassOf(baseAttributeType)) @@ -200,6 +208,23 @@ public override object InitializeLifetimeService() return null; } + internal static T[] GetAttributes(MethodBase methodBase, bool inherit) + where T : Attribute + { + Attribute[] attributeArray = GetCustomAttributes(methodBase, typeof(T), inherit); + if (attributeArray == null || attributeArray.Length == 0) + { + return null; + } + + if (attributeArray is T[] attributes) + { + return attributes; + } + + return attributeArray.Where(a => a is T).Cast().ToArray(); + } + /// /// Match return type of method. /// @@ -629,7 +654,6 @@ private IEnumerable GetTestPropertyAttributes(MemberInfo propertyAttr /// The member to inspect. /// Look at inheritance chain. /// attributes defined. - [System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Design", "CA1031:DoNotCatchGeneralExceptionTypes", Justification = "Requirement is to handle all kinds of user exceptions and message appropriately.")] private Dictionary GetAttributes(MemberInfo memberInfo, bool inherit) { // If the information is cached, then use it otherwise populate the cache using @@ -658,16 +682,10 @@ private Dictionary GetAttributes(MemberInfo memberInfo, bool inh } catch (Exception ex2) { - description = - ex.GetType().FullName + - ": (Failed to get exception description due to an exception of type " + - ex2.GetType().FullName + ')'; + description = string.Format(CultureInfo.CurrentCulture, Resource.ExceptionOccuredWhileGettingTheExceptionDescription, ex.GetType().FullName, ex2.GetType().FullName); // ex.GetType().FullName + } - PlatformServiceProvider.Instance.AdapterTraceLogger.LogWarning( - "Getting custom attributes for type {0} threw exception (will ignore and use the reflection way): {1}", - memberInfo.GetType().FullName, - description); + PlatformServiceProvider.Instance.AdapterTraceLogger.LogWarning(Resource.FailedToGetCustomAttribute, memberInfo.GetType().FullName, description); // Since we cannot check by attribute names, do it in reflection way. // Note 1: this will not work for different version of assembly but it is better than nothing. diff --git a/src/Adapter/MSTest.CoreAdapter/MSTest.CoreAdapter.csproj b/src/Adapter/MSTest.CoreAdapter/MSTest.CoreAdapter.csproj index f020861623..66c6afc6c0 100644 --- a/src/Adapter/MSTest.CoreAdapter/MSTest.CoreAdapter.csproj +++ b/src/Adapter/MSTest.CoreAdapter/MSTest.CoreAdapter.csproj @@ -38,6 +38,7 @@ + @@ -54,6 +55,7 @@ + diff --git a/src/Adapter/MSTest.CoreAdapter/ObjectModel/DynamicDataType.cs b/src/Adapter/MSTest.CoreAdapter/ObjectModel/DynamicDataType.cs new file mode 100644 index 0000000000..b1b82712c0 --- /dev/null +++ b/src/Adapter/MSTest.CoreAdapter/ObjectModel/DynamicDataType.cs @@ -0,0 +1,12 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.ObjectModel +{ + internal enum DynamicDataType : int + { + None = 0, + DataSourceAttribute = 1, + ITestDataSource = 2 + } +} diff --git a/src/Adapter/MSTest.CoreAdapter/ObjectModel/TestMethod.cs b/src/Adapter/MSTest.CoreAdapter/ObjectModel/TestMethod.cs index 1d0a4f80fc..7fcf6bfb43 100644 --- a/src/Adapter/MSTest.CoreAdapter/ObjectModel/TestMethod.cs +++ b/src/Adapter/MSTest.CoreAdapter/ObjectModel/TestMethod.cs @@ -4,9 +4,12 @@ namespace Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.ObjectModel { using System; + using System.Collections.Generic; + using System.Collections.ObjectModel; using System.Diagnostics; using System.Reflection; + using Microsoft.TestPlatform.AdapterUtilities; using Microsoft.TestPlatform.AdapterUtilities.ManagedNameUtilities; using MSTestAdapter.PlatformServices.Interface.ObjectModel; @@ -17,18 +20,15 @@ namespace Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.ObjectModel [Serializable] public sealed class TestMethod : ITestMethod { - #region Fields - /// - /// Member field for the property 'DeclaringClassFullName' + /// Number of elements in . /// - private string declaringClassFullName = null; + public const int TotalHierarchyLevels = HierarchyConstants.Levels.TotalLevelCount; - /// - /// Member field for the property 'DeclaringAssemblyName' - /// + #region Fields + private IReadOnlyCollection hierarchy; + private string declaringClassFullName = null; private string declaringAssemblyName = null; - #endregion public TestMethod(string name, string fullClassName, string assemblyName, bool isAsync) @@ -45,6 +45,12 @@ public TestMethod(string name, string fullClassName, string assemblyName, bool i this.FullClassName = fullClassName; this.AssemblyName = assemblyName; this.IsAsync = isAsync; + + var hierarchy = new string[HierarchyConstants.Levels.TotalLevelCount]; + hierarchy[HierarchyConstants.Levels.NamespaceIndex] = fullClassName; + hierarchy[HierarchyConstants.Levels.ClassIndex] = name; + + this.hierarchy = new ReadOnlyCollection(hierarchy); } internal TestMethod(MethodBase method, string name, string fullClassName, string assemblyName, bool isAsync) @@ -55,33 +61,25 @@ internal TestMethod(MethodBase method, string name, string fullClassName, string throw new ArgumentNullException(nameof(method)); } - ManagedNameHelper.GetManagedName(method, out var managedType, out var managedMethod); - - // ManagedNameHelpers currently does not support spaces in method names. - // If there are spaces in the method name, we'll use the legacy way to find the method. - if (!managedMethod.Contains(" ")) - { - this.ManagedTypeName = managedType; - this.ManagedMethodName = managedMethod; - } + ManagedNameHelper.GetManagedName(method, out var managedType, out var managedMethod, out var hierarchyValues); + this.ManagedTypeName = managedType; + this.ManagedMethodName = managedMethod; + this.hierarchy = new ReadOnlyCollection(hierarchyValues); } - internal TestMethod(string managedTypeName, string managedMethodName, string name, string fullClassName, string assemblyName, bool isAsync) + internal TestMethod(string managedTypeName, string managedMethodName, string[] hierarchyValues, string name, string fullClassName, string assemblyName, bool isAsync) : this(name, fullClassName, assemblyName, isAsync) { this.ManagedTypeName = managedTypeName; this.ManagedMethodName = managedMethodName; + this.hierarchy = new ReadOnlyCollection(hierarchyValues); } - /// - /// Gets the name of the test method - /// - public string Name { get; private set; } + /// + public string Name { get; } - /// - /// Gets the full classname of the test method - /// - public string FullClassName { get; private set; } + /// + public string FullClassName { get; } /// /// Gets or sets the declaring assembly full name. This will be used while getting navigation data. @@ -136,5 +134,30 @@ public string DeclaringClassFullName /// public bool HasManagedMethodAndTypeProperties => !string.IsNullOrWhiteSpace(this.ManagedTypeName) && !string.IsNullOrWhiteSpace(this.ManagedMethodName); + + /// + public IReadOnlyCollection Hierarchy => this.hierarchy; + + /// + /// Gets or sets type of dynamic data if any + /// + internal DynamicDataType DataType { get; set; } + + /// + /// Gets or sets indices of dynamic data + /// + internal object[] Data { get; set; } + + /// + /// Gets or sets the test group set during discovery + /// + internal string TestGroup { get; set; } + + /// + /// Gets or sets the display name set during discovery + /// + internal string DisplayName { get; set; } + + internal TestMethod Clone() => this.MemberwiseClone() as TestMethod; } } diff --git a/src/Adapter/MSTest.CoreAdapter/ObjectModel/UnitTestElement.cs b/src/Adapter/MSTest.CoreAdapter/ObjectModel/UnitTestElement.cs index b9b8d13bce..e947ca66aa 100644 --- a/src/Adapter/MSTest.CoreAdapter/ObjectModel/UnitTestElement.cs +++ b/src/Adapter/MSTest.CoreAdapter/ObjectModel/UnitTestElement.cs @@ -7,7 +7,10 @@ namespace Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.ObjectModel using System.Collections.Generic; using System.Diagnostics; using System.Globalization; + using System.IO; + using System.Linq; + using Microsoft.TestPlatform.AdapterUtilities; using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.Extensions; using Microsoft.VisualStudio.TestPlatform.ObjectModel; @@ -15,6 +18,7 @@ namespace Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.ObjectModel /// The unit test element. /// [Serializable] + [DebuggerDisplay("{GetDisplayName()} ({TestMethod.ManagedTypeName})")] internal class UnitTestElement { /// @@ -103,6 +107,17 @@ public UnitTestElement(TestMethod testMethod) /// internal string[] WorkItemIds { get; set; } + internal UnitTestElement Clone() + { + var clone = this.MemberwiseClone() as UnitTestElement; + if (this.TestMethod != null) + { + clone.TestMethod = this.TestMethod.Clone(); + } + + return clone; + } + /// /// Convert the UnitTestElement instance to an Object Model testCase instance. /// @@ -129,6 +144,12 @@ internal TestCase ToTestCase() testCase.SetPropertyValue(TestAdapter.Constants.TestClassNameProperty, this.TestMethod.FullClassName); } + var hierarchy = this.TestMethod.Hierarchy; + if (hierarchy != null && hierarchy.Count > 0) + { + testCase.SetHierarchy(hierarchy.ToArray()); + } + // Set declaring type if present so the correct method info can be retrieved if (this.TestMethod.DeclaringClassFullName != null) { @@ -190,6 +211,44 @@ internal TestCase ToTestCase() testCase.SetPropertyValue(TestAdapter.Constants.DoNotParallelizeProperty, this.DoNotParallelize); } + // Store resolved data if any + if (this.TestMethod.DataType != DynamicDataType.None) + { + var data = Helpers.DataSerializationHelper.Serialize(this.TestMethod.Data); + + testCase.SetPropertyValue(TestAdapter.Constants.TestDynamicDataTypeProperty, (int)this.TestMethod.DataType); + testCase.SetPropertyValue(TestAdapter.Constants.TestDynamicDataProperty, data); + } + + string fileName = testCase.Source; + try + { + fileName = Path.GetFileName(fileName); + } + catch + { + } + + var idProvider = new TestIdProvider(); + idProvider.AppendString(testCase.ExecutorUri?.ToString()); + idProvider.AppendString(fileName); + if (this.TestMethod.HasManagedMethodAndTypeProperties) + { + idProvider.AppendString(this.TestMethod.ManagedTypeName); + idProvider.AppendString(this.TestMethod.ManagedMethodName); + } + else + { + idProvider.AppendString(testCase.FullyQualifiedName); + } + + if (this.TestMethod.DataType != DynamicDataType.None) + { + idProvider.AppendString(testCase.DisplayName); + } + + testCase.Id = idProvider.GetId(); + return testCase; } diff --git a/src/Adapter/MSTest.CoreAdapter/Resources/Resource.Designer.cs b/src/Adapter/MSTest.CoreAdapter/Resources/Resource.Designer.cs index 6ee7b4f840..6d880c368a 100644 --- a/src/Adapter/MSTest.CoreAdapter/Resources/Resource.Designer.cs +++ b/src/Adapter/MSTest.CoreAdapter/Resources/Resource.Designer.cs @@ -70,6 +70,33 @@ internal static string AttachmentSetDisplayName { } } + /// + /// Looks up a localized string similar to Exception occurred while enumarating DataSourceAttribute on "{0}.{1}": {2}. + /// + internal static string CannotEnumerateDataSourceAttribute { + get { + return ResourceManager.GetString("CannotEnumerateDataSourceAttribute", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to A test method can only contain one DataSourceAttribute, but found {2} on "{0}.{1}".. + /// + internal static string CannotEnumerateDataSourceAttribute_MoreThenOneDefined { + get { + return ResourceManager.GetString("CannotEnumerateDataSourceAttribute_MoreThenOneDefined", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Exception occurred while enumarating IDataSource attribute on "{0}.{1}": {2}. + /// + internal static string CannotEnumerateIDataSourceAttribute { + get { + return ResourceManager.GetString("CannotEnumerateIDataSourceAttribute", resourceCulture); + } + } + /// /// Looks up a localized string similar to The parameter should not be null or empty.. /// @@ -142,6 +169,24 @@ internal static string EnumeratorLoadTypeErrorFormat { } } + /// + /// Looks up a localized string similar to "{0}": (Failed to get exception description due to an exception of type "{1}".. + /// + internal static string ExceptionOccuredWhileGettingTheExceptionDescription { + get { + return ResourceManager.GetString("ExceptionOccuredWhileGettingTheExceptionDescription", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Exceptions thrown:. + /// + internal static string ExceptionsThrown { + get { + return ResourceManager.GetString("ExceptionsThrown", resourceCulture); + } + } + /// /// Looks up a localized string similar to Test '{0}' execution has been aborted.. /// @@ -160,6 +205,24 @@ internal static string Execution_Test_Timeout { } } + /// + /// Looks up a localized string similar to Failed to get attribute cache. Ignoring attribute inheritance and falling into 'type defines Attribute model', so that we have some data.. + /// + internal static string FailedFetchAttributeCache { + get { + return ResourceManager.GetString("FailedFetchAttributeCache", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Getting custom attributes for type {0} threw exception (will ignore and use the reflection way): {1}. + /// + internal static string FailedToGetCustomAttribute { + get { + return ResourceManager.GetString("FailedToGetCustomAttribute", resourceCulture); + } + } + /// /// Looks up a localized string similar to Invalid value '{0}' specified for 'Scope'. Supported scopes are {1}.. /// @@ -205,6 +268,15 @@ internal static string LegacyScenariosNotSupportedWarning { } } + /// + /// Looks up a localized string similar to An older version of MSTestV2 package is loaded in assembly, test discovery might fail to discover all data tests if they depend on `.runsettings` file.. + /// + internal static string OlderTFMVersionFound { + get { + return ResourceManager.GetString("OlderTFMVersionFound", resourceCulture); + } + } + /// /// Looks up a localized string similar to Running tests in any of the provided sources is not supported for the selected platform. /// diff --git a/src/Adapter/MSTest.CoreAdapter/Resources/Resource.resx b/src/Adapter/MSTest.CoreAdapter/Resources/Resource.resx index cf72e6072d..872c6ab19d 100644 --- a/src/Adapter/MSTest.CoreAdapter/Resources/Resource.resx +++ b/src/Adapter/MSTest.CoreAdapter/Resources/Resource.resx @@ -320,6 +320,44 @@ Error: {1} Test '{0}' execution has been aborted. + + Exception occurred while enumarating DataSourceAttribute on "{0}.{1}": {2} + {0}: TypeName with namespace, +{1}: Method name, +{2}: Exception details + + + A test method can only contain one DataSourceAttribute, but found {2} on "{0}.{1}". + {0}: TypeName with namespace, +{1}: Method name, +{2}: Number of attributed defined. + + + Exception occurred while enumarating IDataSource attribute on "{0}.{1}": {2} + {0}: TypeName with namespace, +{1}: Method name, +{2}: Exception details + + + "{0}": (Failed to get exception description due to an exception of type "{1}". + {0}: Type of the original exception that we're trying to get the desciption of. +{1}: Thrown exception + + + Exceptions thrown: + This is usually preceeds by TestAssembly_AssemblyDiscoveryFailure message, and precceded by list of exceptions thrown in a test discovery session. + + + Failed to get attribute cache. Ignoring attribute inheritance and falling into 'type defines Attribute model', so that we have some data. + + + Getting custom attributes for type {0} threw exception (will ignore and use the reflection way): {1} + {0}: Attribute full type name. +{1}: Exception description + + + An older version of MSTestV2 package is loaded in assembly, test discovery might fail to discover all data tests if they depend on `.runsettings` file. + The called code threw an exception that was caught, but the exception value was null diff --git a/src/Adapter/PlatformServices.Desktop/packages.config b/src/Adapter/PlatformServices.Desktop/packages.config index fc97748e33..ae181db396 100644 --- a/src/Adapter/PlatformServices.Desktop/packages.config +++ b/src/Adapter/PlatformServices.Desktop/packages.config @@ -1,7 +1,7 @@  - + diff --git a/src/Adapter/PlatformServices.Interface/ObjectModel/ITestMethod.cs b/src/Adapter/PlatformServices.Interface/ObjectModel/ITestMethod.cs index abd883c7cc..aa7c734a8d 100644 --- a/src/Adapter/PlatformServices.Interface/ObjectModel/ITestMethod.cs +++ b/src/Adapter/PlatformServices.Interface/ObjectModel/ITestMethod.cs @@ -3,6 +3,8 @@ namespace Microsoft.VisualStudio.TestPlatform.MSTestAdapter.PlatformServices.Interface.ObjectModel { + using System.Collections.Generic; + /// /// TestMethod structure that is shared between adapter and platform services only. /// @@ -55,5 +57,13 @@ public interface ITestMethod /// Gets a value indicating whether both and are not null or whitespace. /// bool HasManagedMethodAndTypeProperties { get; } + + /// + /// Gets the test case hierarchy parsed by the adapter. + /// + /// + /// Contains four items in order: Namespace, class name, test group, display name. + /// + IReadOnlyCollection Hierarchy { get; } } } diff --git a/src/Adapter/PlatformServices.Interface/packages.config b/src/Adapter/PlatformServices.Interface/packages.config index 477f795ab9..46673e74c6 100644 --- a/src/Adapter/PlatformServices.Interface/packages.config +++ b/src/Adapter/PlatformServices.Interface/packages.config @@ -1,7 +1,7 @@  - + diff --git a/src/Adapter/PlatformServices.Portable/packages.config b/src/Adapter/PlatformServices.Portable/packages.config index 477f795ab9..46673e74c6 100644 --- a/src/Adapter/PlatformServices.Portable/packages.config +++ b/src/Adapter/PlatformServices.Portable/packages.config @@ -1,7 +1,7 @@  - + diff --git a/src/Adapter/PlatformServices.Shared/PlatformServices.Shared.projitems b/src/Adapter/PlatformServices.Shared/PlatformServices.Shared.projitems index 9baba9aead..9ce58de0a5 100644 --- a/src/Adapter/PlatformServices.Shared/PlatformServices.Shared.projitems +++ b/src/Adapter/PlatformServices.Shared/PlatformServices.Shared.projitems @@ -11,4 +11,4 @@ - \ No newline at end of file + diff --git a/src/Package/MSTest.TestFramework.enu.nuspec b/src/Package/MSTest.TestFramework.enu.nuspec index 5d52d6f4e3..f56c37ba44 100644 --- a/src/Package/MSTest.TestFramework.enu.nuspec +++ b/src/Package/MSTest.TestFramework.enu.nuspec @@ -18,8 +18,6 @@ - ASP.NET Core 1.0+ To discover and execute tests install MSTest.TestAdapter. - - To discover and execute tests for project.json based projects install dotnet-test-mstest. https://github.com/microsoft/testfx LICENSE diff --git a/src/Package/MSTest.TestFramework.nuspec b/src/Package/MSTest.TestFramework.nuspec index e485a1bb5a..8ff89f0a28 100644 --- a/src/Package/MSTest.TestFramework.nuspec +++ b/src/Package/MSTest.TestFramework.nuspec @@ -18,8 +18,6 @@ - ASP.NET Core 1.0+ To discover and execute tests install MSTest.TestAdapter. - - To discover and execute tests for project.json based projects install dotnet-test-mstest. https://github.com/microsoft/testfx LICENSE diff --git a/src/Package/MSTest.TestFramework.symbols.nuspec b/src/Package/MSTest.TestFramework.symbols.nuspec index 07606c4dd3..f06c5f98df 100644 --- a/src/Package/MSTest.TestFramework.symbols.nuspec +++ b/src/Package/MSTest.TestFramework.symbols.nuspec @@ -17,8 +17,6 @@ - ASP.NET Core 1.0+ To discover and execute tests install MSTest.TestAdapter. - - To discover and execute tests for project.json based projects install dotnet-test-mstest. https://github.com/microsoft/testfx LICENSE diff --git a/test/ComponentTests/PlatformServices.Desktop.Component.Tests/packages.config b/test/ComponentTests/PlatformServices.Desktop.Component.Tests/packages.config index e778835012..79fb921202 100644 --- a/test/ComponentTests/PlatformServices.Desktop.Component.Tests/packages.config +++ b/test/ComponentTests/PlatformServices.Desktop.Component.Tests/packages.config @@ -1,7 +1,7 @@  - + diff --git a/test/ComponentTests/TestAssets/Directory.Build.targets b/test/ComponentTests/TestAssets/Directory.Build.targets index 5a5c0f5db7..87043c6abb 100644 --- a/test/ComponentTests/TestAssets/Directory.Build.targets +++ b/test/ComponentTests/TestAssets/Directory.Build.targets @@ -9,4 +9,4 @@ - \ No newline at end of file + diff --git a/test/E2ETests/Automation.CLI/Automation.CLI.csproj b/test/E2ETests/Automation.CLI/Automation.CLI.csproj index 64ef23565a..83ee2a0787 100644 --- a/test/E2ETests/Automation.CLI/Automation.CLI.csproj +++ b/test/E2ETests/Automation.CLI/Automation.CLI.csproj @@ -88,10 +88,11 @@ + + - diff --git a/test/E2ETests/Automation.CLI/CLITestBase.common.cs b/test/E2ETests/Automation.CLI/CLITestBase.common.cs new file mode 100644 index 0000000000..5fa0221792 --- /dev/null +++ b/test/E2ETests/Automation.CLI/CLITestBase.common.cs @@ -0,0 +1,106 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Microsoft.MSTestV2.CLIAutomation +{ + using System; + using System.IO; + using System.Linq; + using System.Xml; + + using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client; + using Microsoft.VisualStudio.TestTools.UnitTesting; + + public partial class CLITestBase + { + private const string E2ETestsRelativePath = @"..\..\..\"; + private const string TestAssetsFolder = "TestAssets"; + private const string ArtifactsFolder = "artifacts"; + private const string PackagesFolder = "packages"; + + // This value is automatically updated by "build.ps1" script. + private const string TestPlatformCLIPackage = @"Microsoft.TestPlatform.16.10.0-release-20210421-08"; + private const string VstestConsoleRelativePath = @"tools\net451\Common7\IDE\Extensions\TestPlatform\vstest.console.exe"; + + /// + /// Gets the relative path of repository root from start-up directory. + /// + /// Relative path of the repository root + protected virtual string GetRelativeRepositoryRootPath() => E2ETestsRelativePath; + + /// + /// Gets the full path to a test asset. + /// + /// Name of the asset with extension. E.g. SimpleUnitTest.dll + /// Full path to the test asset. + /// + /// Test assets follow several conventions: + /// (a) They are built for provided build configuration. + /// (b) Name of the test asset matches the parent directory name. E.g. TestAssets\SimpleUnitTest\SimpleUnitTest.xproj must + /// produce TestAssets\SimpleUnitTest\bin\Debug\SimpleUnitTest.dll + /// (c) TestAssets are copied over to a central location i.e. "TestAssets\artifacts\*.*" + /// + protected string GetAssetFullPath(string assetName) + { + var assetPath = Path.Combine( + Environment.CurrentDirectory, + this.GetRelativeRepositoryRootPath(), + ArtifactsFolder, + TestAssetsFolder, + assetName); + + Assert.IsTrue(File.Exists(assetPath), "GetTestAsset: Path not found: {0}.", assetPath); + + return assetPath; + } + + protected string GetTestAdapterPath() + { + var testAdapterPath = Path.Combine( + Environment.CurrentDirectory, + this.GetRelativeRepositoryRootPath(), + ArtifactsFolder, + TestAssetsFolder); + + return testAdapterPath; + } + + /// + /// Gets the RunSettingXml having testadapterpath filled in specified by arguement. + /// Inserts testAdapterPath in existing runSetting if not present already, + /// or generates new runSettings with testAdapterPath if runSettings is Empty. + /// + /// RunSettings provided for discovery/execution + /// Full path to TestAdapter. + /// RunSettingXml as string + protected string GetRunSettingXml(string settingsXml, string testAdapterPath) + { + if (string.IsNullOrEmpty(settingsXml)) + { + settingsXml = XmlRunSettingsUtilities.CreateDefaultRunSettings(); + } + + XmlDocument doc = new XmlDocument(); + using (var xmlReader = XmlReader.Create(new StringReader(settingsXml), new XmlReaderSettings() { XmlResolver = null, CloseInput = true })) + { + doc.Load(xmlReader); + } + + XmlElement root = doc.DocumentElement; + RunConfiguration runConfiguration = new RunConfiguration(testAdapterPath); + XmlElement runConfigElement = runConfiguration.ToXml(); + if (root[runConfiguration.SettingsName] == null) + { + XmlNode newNode = doc.ImportNode(runConfigElement, true); + root.AppendChild(newNode); + } + else + { + XmlNode newNode = doc.ImportNode(runConfigElement.FirstChild, true); + root[runConfiguration.SettingsName].AppendChild(newNode); + } + + return doc.OuterXml; + } + } +} diff --git a/test/E2ETests/Automation.CLI/CLITestBase.cs b/test/E2ETests/Automation.CLI/CLITestBase.e2e.cs similarity index 77% rename from test/E2ETests/Automation.CLI/CLITestBase.cs rename to test/E2ETests/Automation.CLI/CLITestBase.e2e.cs index ec67f62064..3665779e21 100644 --- a/test/E2ETests/Automation.CLI/CLITestBase.cs +++ b/test/E2ETests/Automation.CLI/CLITestBase.e2e.cs @@ -12,17 +12,8 @@ namespace Microsoft.MSTestV2.CLIAutomation using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client; using Microsoft.VisualStudio.TestTools.UnitTesting; - public class CLITestBase + public partial class CLITestBase { - private const string E2ETestsRelativePath = @"..\..\..\"; - private const string TestAssetsFolder = "TestAssets"; - private const string ArtifactsFolder = "artifacts"; - private const string PackagesFolder = "packages"; - - // This value is automatically updated by "build.ps1" script. - private const string TestPlatformCLIPackage = "Microsoft.TestPlatform.16.9.1"; - private const string VstestConsoleRelativePath = @"tools\net451\Common7\IDE\Extensions\TestPlatform\vstest.console.exe"; - private static VsTestConsoleWrapper vsTestConsoleWrapper; private DiscoveryEventsHandler discoveryEventsHandler; private RunEventsHandler runEventsHandler; @@ -90,7 +81,7 @@ public void InvokeVsTestForExecution(string[] sources, string runSettings = "", /// Full path to vstest.console.exe public string GetConsoleRunnerPath() { - var packagesFolder = Path.Combine(Environment.CurrentDirectory, E2ETestsRelativePath, PackagesFolder); + var packagesFolder = Path.Combine(Environment.CurrentDirectory, this.GetRelativeRepositoryRootPath(), PackagesFolder); var vstestConsolePath = Path.Combine(packagesFolder, TestPlatformCLIPackage, VstestConsoleRelativePath); Assert.IsTrue(File.Exists(vstestConsolePath), "GetConsoleRunnerPath: Path not found: {0}", vstestConsolePath); @@ -258,81 +249,6 @@ public void ValidateTestRunTime(int thresholdTime) $"Test Run was expected to not exceed {thresholdTime} but it took {this.runEventsHandler.ElapsedTimeInRunningTests}"); } - /// - /// Gets the full path to a test asset. - /// - /// Name of the asset with extension. E.g. SimpleUnitTest.dll - /// Full path to the test asset. - /// - /// Test assets follow several conventions: - /// (a) They are built for provided build configuration. - /// (b) Name of the test asset matches the parent directory name. E.g. TestAssets\SimpleUnitTest\SimpleUnitTest.xproj must - /// produce TestAssets\SimpleUnitTest\bin\Debug\SimpleUnitTest.dll - /// (c) TestAssets are copied over to a central location i.e. "TestAssets\artifacts\*.*" - /// - protected string GetAssetFullPath(string assetName) - { - var assetPath = Path.Combine( - Environment.CurrentDirectory, - E2ETestsRelativePath, - ArtifactsFolder, - TestAssetsFolder, - assetName); - - Assert.IsTrue(File.Exists(assetPath), "GetTestAsset: Path not found: {0}.", assetPath); - - return assetPath; - } - - protected string GetTestAdapterPath() - { - var testAdapterPath = Path.Combine( - Environment.CurrentDirectory, - E2ETestsRelativePath, - ArtifactsFolder, - TestAssetsFolder); - - return testAdapterPath; - } - - /// - /// Gets the RunSettingXml having testadapterpath filled in specified by arguement. - /// Inserts testAdapterPath in existing runSetting if not present already, - /// or generates new runSettings with testAdapterPath if runSettings is Empty. - /// - /// RunSettings provided for discovery/execution - /// Full path to TestAdapter. - /// RunSettingXml as string - protected string GetRunSettingXml(string settingsXml, string testAdapterPath) - { - if (string.IsNullOrEmpty(settingsXml)) - { - settingsXml = XmlRunSettingsUtilities.CreateDefaultRunSettings(); - } - - XmlDocument doc = new XmlDocument(); - using (var xmlReader = XmlReader.Create(new StringReader(settingsXml), new XmlReaderSettings() { XmlResolver = null, CloseInput = true })) - { - doc.Load(xmlReader); - } - - XmlElement root = doc.DocumentElement; - RunConfiguration runConfiguration = new RunConfiguration(testAdapterPath); - XmlElement runConfigElement = runConfiguration.ToXml(); - if (root[runConfiguration.SettingsName] == null) - { - XmlNode newNode = doc.ImportNode(runConfigElement, true); - root.AppendChild(newNode); - } - else - { - XmlNode newNode = doc.ImportNode(runConfigElement.FirstChild, true); - root[runConfiguration.SettingsName].AppendChild(newNode); - } - - return doc.OuterXml; - } - /// /// Gets the test method name from full name. /// diff --git a/test/E2ETests/Automation.CLI/packages.config b/test/E2ETests/Automation.CLI/packages.config index cfe2887a1b..98803acf5a 100644 --- a/test/E2ETests/Automation.CLI/packages.config +++ b/test/E2ETests/Automation.CLI/packages.config @@ -1,9 +1,9 @@  - - - - + + + + diff --git a/test/E2ETests/DiscoveryAndExecutionTests/DataRowTests.cs b/test/E2ETests/DiscoveryAndExecutionTests/DataRowTests.cs new file mode 100644 index 0000000000..04bad79ef3 --- /dev/null +++ b/test/E2ETests/DiscoveryAndExecutionTests/DataRowTests.cs @@ -0,0 +1,127 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Microsoft.MSTestV2.Smoke.DiscoveryAndExecutionTests +{ + using Microsoft.MSTestV2.CLIAutomation; + using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.ObjectModel; + using Microsoft.VisualStudio.TestTools.UnitTesting; + + using System.IO; + + + [TestClass] + public class DataRowTests : CLITestBase + { + private const string TestAssembly = "DataRowTestProject.dll"; + + [TestMethod] + public void ExecuteOnlyDerivedClassDataRowsWhenBothBaseAndDerviedClassHasDataRows_SimpleDataRows() + { + // Arrange + var assemblyPath = Path.IsPathRooted(TestAssembly) ? TestAssembly : this.GetAssetFullPath(TestAssembly); + + // Act + var testCases = DiscoverTests(assemblyPath, "TestCategory~DataRowSimple"); + var testResults = RunTests(assemblyPath, testCases); + + // Assert + Assert.That.ContainsTestsPassed(testResults, + "DataRowTestMethod (BaseString1)", + "DataRowTestMethod (BaseString2)", + "DataRowTestMethod (BaseString3)", + "DataRowTestMethod (DerivedString1)", + "DataRowTestMethod (DerivedString2)" + ); + + // 4 tests of BaseClass.DataRowTestMethod - 3 data row results and 1 parent result + // 3 tests of DerivedClass.DataRowTestMethod - 2 data row results and 1 parent result + // Total 7 tests - Making sure that DerivedClass doesn't run BaseClass tests + Assert.That.PassedTestCount(testResults, 7 - 2); + } + + [TestMethod] + public void ExecuteOnlyDerivedClassDataRowsWhenItOverridesBaseClassDataRows_SimpleDataRows() + { + // Arrange + var assemblyPath = Path.IsPathRooted(TestAssembly) ? TestAssembly : this.GetAssetFullPath(TestAssembly); + + // Act + var testCases = DiscoverTests(assemblyPath, "FullyQualifiedName~DerivedClass&TestCategory~DataRowSimple"); + var testResults = RunTests(assemblyPath, testCases); + + // Assert + Assert.That.ContainsTestsPassed(testResults, + "DataRowTestMethod (DerivedString1)", + "DataRowTestMethod (DerivedString2)" + ); + + // 3 tests of DerivedClass.DataRowTestMethod - 2 datarow result and 1 parent result + Assert.That.PassedTestCount(testResults, 3 - 1); + } + + [TestMethod] + public void DataRowsExecuteWithRequiredAndOptionalParameters() + { + // Arrange + var assemblyPath = Path.IsPathRooted(TestAssembly) ? TestAssembly : this.GetAssetFullPath(TestAssembly); + + // Act + var testCases = DiscoverTests(assemblyPath, "TestCategory~DataRowSomeOptional"); + var testResults = RunTests(assemblyPath, testCases); + + // Assert + Assert.That.ContainsTestsPassed(testResults, + "DataRowTestMethodWithSomeOptionalParameters (123)", + "DataRowTestMethodWithSomeOptionalParameters (123,DerivedOptionalString1)", + "DataRowTestMethodWithSomeOptionalParameters (123,DerivedOptionalString2,DerivedOptionalString3)" + ); + + // 4 tests of DerivedClass.DataRowTestMethodWithSomeOptionalParameters - 3 datarow result and 1 parent result + Assert.That.PassedTestCount(testResults, 4 - 1); + } + + [TestMethod] + public void DataRowsExecuteWithParamsArrayParameter() + { + // Arrange + var assemblyPath = Path.IsPathRooted(TestAssembly) ? TestAssembly : this.GetAssetFullPath(TestAssembly); + + // Act + var testCases = DiscoverTests(assemblyPath, "TestCategory~DataRowParamsArgument"); + var testResults = RunTests(assemblyPath, testCases); + + // Assert + Assert.That.ContainsTestsPassed(testResults, + "DataRowTestMethodWithParamsParameters (2)", + "DataRowTestMethodWithParamsParameters (2,DerivedSingleParamsArg)", + "DataRowTestMethodWithParamsParameters (2,DerivedParamsArg1,DerivedParamsArg2)", + "DataRowTestMethodWithParamsParameters (2,DerivedParamsArg1,DerivedParamsArg2,DerivedParamsArg3)" + ); + + // 5 tests of DerivedClass.DataRowTestMethodWithParamsParameters - 4 datarow result and 1 parent result + Assert.That.PassedTestCount(testResults, 5 - 1); + } + + [TestMethod] + public void DataRowsFailWhenInvalidArgumentsProvided() + { + // Arrange + var assemblyPath = Path.IsPathRooted(TestAssembly) ? TestAssembly : this.GetAssetFullPath(TestAssembly); + + // Act + var testCases = DiscoverTests(assemblyPath, "TestCategory~DataRowOptionalInvalidArguments"); + var testResults = RunTests(assemblyPath, testCases); + + // Assert + Assert.That.ContainsTestsPassed(testResults, + "DataRowTestMethodFailsWithInvalidArguments ()", + "DataRowTestMethodFailsWithInvalidArguments (2)", + "DataRowTestMethodFailsWithInvalidArguments (2,DerivedRequiredArgument,DerivedOptionalArgument,DerivedExtraArgument)" + ); + + // 4 tests of DerivedClass.DataRowTestMethodFailsWithInvalidArguments - 3 datarow result and 1 parent result + Assert.That.PassedTestCount(testResults, 4 - 1); + } + } +} diff --git a/test/E2ETests/DiscoveryAndExecutionTests/DataSourceTests.cs b/test/E2ETests/DiscoveryAndExecutionTests/DataSourceTests.cs new file mode 100644 index 0000000000..5d8e4b11c8 --- /dev/null +++ b/test/E2ETests/DiscoveryAndExecutionTests/DataSourceTests.cs @@ -0,0 +1,40 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Microsoft.MSTestV2.Smoke.DiscoveryAndExecutionTests +{ + using Microsoft.MSTestV2.CLIAutomation; + using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.ObjectModel; + using Microsoft.VisualStudio.TestTools.UnitTesting; + + using System.IO; + using System.Linq; + + [TestClass] + public class DataSourceTests : CLITestBase + { + private const string TestAssembly = "DataSourceTestProject.dll"; + + [TestMethod] + public void ExecuteCsvTestDataSourceTests() + { + // Arrange + var assemblyPath = Path.IsPathRooted(TestAssembly) ? TestAssembly : this.GetAssetFullPath(TestAssembly); + + // Act + var testCases = DiscoverTests(assemblyPath, "CsvTestMethod"); + var testResults = RunTests(assemblyPath, testCases); + + // Assert + Assert.That.ContainsTestsPassed(testResults, + "CsvTestMethod (Data Row 0)", + "CsvTestMethod (Data Row 2)" + ); + + Assert.That.ContainsTestsFailed(testResults, + "CsvTestMethod (Data Row 1)", + "CsvTestMethod (Data Row 3)" + ); + } + } +} diff --git a/test/E2ETests/DiscoveryAndExecutionTests/DiscoveryAndExecutionTests.csproj b/test/E2ETests/DiscoveryAndExecutionTests/DiscoveryAndExecutionTests.csproj new file mode 100644 index 0000000000..c12fa7e130 --- /dev/null +++ b/test/E2ETests/DiscoveryAndExecutionTests/DiscoveryAndExecutionTests.csproj @@ -0,0 +1,58 @@ + + + ..\..\..\ + false + $(TestFxRoot)artifacts\$(Configuration)\ + + + + + net452 + v4.5.2 + false + + false + 1685 + true + false + + + + + + + + + + + + + + + + $(SourcePath)MSTest.CoreAdapter\Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.dll + + + + $(SourcePath)MSTest.CoreAdapter\System.Collections.Concurrent.dll + + + + $(SourcePath)PlatformServices.Desktop\Microsoft.VisualStudio.TestPlatform.MSTestAdapter.PlatformServices.dll + + + + $(SourcePath)PlatformServices.Interface\Microsoft.VisualStudio.TestPlatform.MSTestAdapter.PlatformServices.Interface.dll + + + + $(SourcePath)MSTest.Core\Microsoft.VisualStudio.TestPlatform.TestFramework.dll + + + + $(SourcePath)Extension.Desktop\Microsoft.VisualStudio.TestPlatform.TestFramework.Extensions.dll + + + + + \ No newline at end of file diff --git a/test/E2ETests/DiscoveryAndExecutionTests/Extensions/AssertionExtensions.cs b/test/E2ETests/DiscoveryAndExecutionTests/Extensions/AssertionExtensions.cs new file mode 100644 index 0000000000..7c51e42d53 --- /dev/null +++ b/test/E2ETests/DiscoveryAndExecutionTests/Extensions/AssertionExtensions.cs @@ -0,0 +1,118 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Microsoft.MSTestV2.Smoke.DiscoveryAndExecutionTests +{ + using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter; + using Microsoft.VisualStudio.TestPlatform.ObjectModel; + + using System.Collections.Generic; + using System.Linq; + + using Assert = Microsoft.VisualStudio.TestTools.UnitTesting.Assert; + + public static class AssertionExtensions + { + public static void PassedTestCount(this Assert _, IEnumerable actual, int expectedCount) + => AssertOutcomeCount(actual, TestOutcome.Passed, expectedCount); + + public static void FailedTestCount(this Assert _, IEnumerable actual, int expectedCount) + => AssertOutcomeCount(actual, TestOutcome.Failed, expectedCount); + + public static void TestsPassed(this Assert _, IEnumerable actual, IEnumerable testCases, IEnumerable expectedTests, MSTestSettings settings = null) + => ContainsExpectedTestsWithOutcome(actual, testCases, TestOutcome.Passed, expectedTests, true, settings); + + public static void TestsPassed(this Assert _, IEnumerable actual, params string[] expectedTests) + => ContainsExpectedTestsWithOutcome(actual, TestOutcome.Passed, expectedTests, true); + + public static void TestsFailed(this Assert _, IEnumerable actual, IEnumerable testCases, IEnumerable expectedTests, MSTestSettings settings = null) + => ContainsExpectedTestsWithOutcome(actual, testCases, TestOutcome.Failed, expectedTests, true, settings); + + public static void TestsFailed(this Assert _, IEnumerable actual, params string[] expectedTests) + => ContainsExpectedTestsWithOutcome(actual, TestOutcome.Failed, expectedTests, true); + + public static void ContainsTestsPassed(this Assert _, IEnumerable actual, IEnumerable testCases, IEnumerable expectedTests, MSTestSettings settings = null) + => ContainsExpectedTestsWithOutcome(actual, testCases, TestOutcome.Passed, expectedTests, false, settings); + + public static void ContainsTestsPassed(this Assert _, IEnumerable actual, params string[] expectedTests) + => ContainsExpectedTestsWithOutcome(actual, TestOutcome.Passed, expectedTests); + + public static void ContainsTestsFailed(this Assert _, IEnumerable actual, IEnumerable testCases, IEnumerable expectedTests, MSTestSettings settings = null) + => ContainsExpectedTestsWithOutcome(actual, testCases, TestOutcome.Failed, expectedTests, false, settings); + + public static void ContainsTestsFailed(this Assert _, IEnumerable actual, params string[] expectedTests) + => ContainsExpectedTestsWithOutcome(actual, TestOutcome.Failed, expectedTests); + + private static void ContainsExpectedTestsWithOutcome(IEnumerable outcomedTests, IEnumerable testCases, TestOutcome expectedOutcome, IEnumerable expectedTests, bool matchCount = false, MSTestSettings settings = null) + { + if (matchCount) + { + var expectedCount = expectedTests.Count(); + var outcomedCount = outcomedTests.Count(); + + AssertOutcomeCount(outcomedCount, expectedOutcome, expectedCount); + } + + foreach (var test in expectedTests) + { + var testFound = outcomedTests.Any( + p => test.Equals(p.TestCase?.FullyQualifiedName) + || test.Equals(p.DisplayName) + || test.Equals(p.TestCase.DisplayName)); + + Assert.IsTrue(testFound, GetOutcomeAssertString(test, expectedOutcome)); + } + } + + private static void ContainsExpectedTestsWithOutcome(IEnumerable outcomedTests, TestOutcome expectedOutcome, string[] expectedTests, bool matchCount = false) + { + if (matchCount) + { + var expectedCount = expectedTests.Count(); + AssertOutcomeCount(outcomedTests, expectedOutcome, expectedCount); + } + + foreach (var test in expectedTests) + { + var testFound = outcomedTests.Any(p => p.DisplayName == test); + + Assert.IsTrue(testFound, GetOutcomeAssertString(test, expectedOutcome)); + } + } + + private static string GetOutcomeAssertString(string testName, TestOutcome outcome) + { + switch (outcome) + { + case TestOutcome.None: + return $"\"{testName}\" does not have TestOutcome.None outcome."; + + case TestOutcome.Passed: + return $"\"{testName}\" does not appear in passed tests list."; + + case TestOutcome.Failed: + return $"\"{testName}\" does not appear in failed tests list."; + + case TestOutcome.Skipped: + return $"\"{testName}\" does not appear in skipped tests list."; + + case TestOutcome.NotFound: + return $"\"{testName}\" does not appear in not found tests list."; + } + + return string.Empty; + } + + private static void AssertOutcomeCount(IEnumerable actual, TestOutcome expectedOutcome, int expectedCount) + { + var outcomedTests = actual.Where(i => i.Outcome == expectedOutcome); + var actualCount = outcomedTests.Count(); + + AssertOutcomeCount(actualCount, expectedOutcome, expectedCount); + } + private static void AssertOutcomeCount(int actualCount, TestOutcome expectedOutcome, int expectedCount) + { + Assert.AreEqual(expectedCount, actualCount, $"Test run expected to contain {expectedCount} tests, but ran {actualCount}."); + } + } +} diff --git a/test/E2ETests/DiscoveryAndExecutionTests/FSharpTestProjectTests.cs b/test/E2ETests/DiscoveryAndExecutionTests/FSharpTestProjectTests.cs new file mode 100644 index 0000000000..eaf65dbb55 --- /dev/null +++ b/test/E2ETests/DiscoveryAndExecutionTests/FSharpTestProjectTests.cs @@ -0,0 +1,33 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Microsoft.MSTestV2.Smoke.DiscoveryAndExecutionTests +{ + using Microsoft.MSTestV2.CLIAutomation; + using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.ObjectModel; + using Microsoft.VisualStudio.TestTools.UnitTesting; + + using System.IO; + + [TestClass] + public class FSharpTestProjectTests : CLITestBase + { + private const string TestAssembly = "FSharpTestProject.dll"; + + [TestMethod] + public void ExecuteCsvTestDataSourceTests() + { + // Arrange + var assemblyPath = Path.IsPathRooted(TestAssembly) ? TestAssembly : this.GetAssetFullPath(TestAssembly); + + // Act + var testCases = DiscoverTests(assemblyPath); + var testResults = RunTests(assemblyPath, testCases); + + // Assert + Assert.That.TestsPassed(testResults, "Test method passing with a . in it"); + Assert.That.PassedTestCount(testResults, 1); + Assert.That.FailedTestCount(testResults, 0); + } + } +} diff --git a/test/E2ETests/DiscoveryAndExecutionTests/TestDataSourceExtensibilityTests.cs b/test/E2ETests/DiscoveryAndExecutionTests/TestDataSourceExtensibilityTests.cs new file mode 100644 index 0000000000..cb7896903f --- /dev/null +++ b/test/E2ETests/DiscoveryAndExecutionTests/TestDataSourceExtensibilityTests.cs @@ -0,0 +1,108 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Microsoft.MSTestV2.Smoke.DiscoveryAndExecutionTests +{ + using Microsoft.MSTestV2.CLIAutomation; + using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.ObjectModel; + using Microsoft.VisualStudio.TestTools.UnitTesting; + + using System.IO; + using System.Linq; + + [TestClass] + public class TestDataSourceExtensibilityTests : CLITestBase + { + private const string TestAssembly = "FxExtensibilityTestProject.dll"; + + /* + Add tests for: + - Ignored tests are discovered during discovery + - Ignored tests are not expanded (DataRow, DataSource, etc) + */ + + [TestMethod] + public void CustomTestDataSourceTests() + { + // Arrange + var assemblyPath = Path.IsPathRooted(TestAssembly) ? TestAssembly : this.GetAssetFullPath(TestAssembly); + + // Act + var testCases = DiscoverTests(assemblyPath, "CustomTestDataSourceTestMethod1"); + var testResults = RunTests(assemblyPath, testCases); + + // Assert + Assert.That.ContainsTestsPassed(testResults, "CustomTestDataSourceTestMethod1 (1,2,3)", "CustomTestDataSourceTestMethod1 (4,5,6)"); + } + + [TestMethod] + public void AssertExtensibilityTests() + { + // Arrange + var assemblyPath = Path.IsPathRooted(TestAssembly) ? TestAssembly : this.GetAssetFullPath(TestAssembly); + + // Act + var testCases = DiscoverTests(assemblyPath, "FxExtensibilityTestProject.AssertExTest"); + var testResults = RunTests(assemblyPath, testCases); + + // Assert + Assert.That.ContainsTestsPassed(testResults, "BasicAssertExtensionTest", "ChainedAssertExtensionTest"); + Assert.That.ContainsTestsFailed(testResults, "BasicFailingAssertExtensionTest", "ChainedFailingAssertExtensionTest"); + } + + [TestMethod] + public void ExecuteCustomTestExtensibilityTests() + { + // Arrange + var assemblyPath = Path.IsPathRooted(TestAssembly) ? TestAssembly : this.GetAssetFullPath(TestAssembly); + + // Act + var testCases = DiscoverTests(assemblyPath, "(Name~CustomTestMethod1)|(Name~CustomTestClass1)"); + var testResults = RunTests(assemblyPath, testCases); + + // Assert + Assert.That.ContainsTestsPassed(testResults, + "CustomTestMethod1 - Execution number 1", + "CustomTestMethod1 - Execution number 2", + "CustomTestMethod1 - Execution number 4", + "CustomTestMethod1 - Execution number 5", + "CustomTestClass1 - Execution number 1", + "CustomTestClass1 - Execution number 2", + "CustomTestClass1 - Execution number 4", + "CustomTestClass1 - Execution number 5" + ); + + Assert.That.ContainsTestsFailed(testResults, + "CustomTestMethod1 - Execution number 3", + "CustomTestClass1 - Execution number 3" + ); + } + + [TestMethod] + public void ExecuteCustomTestExtensibilityWithTestDataTests() + { + // Arrange + var assemblyPath = Path.IsPathRooted(TestAssembly) ? TestAssembly : this.GetAssetFullPath(TestAssembly); + + // Act + var testCases = DiscoverTests(assemblyPath, "Name~CustomTestMethod2"); + var testResults = RunTests(assemblyPath, testCases); + + // Assert + Assert.That.TestsPassed(testResults, + "CustomTestMethod2 (B)", + "CustomTestMethod2 (B)", + "CustomTestMethod2 (B)" + ); + + Assert.That.TestsFailed(testResults, + "CustomTestMethod2 (A)", + "CustomTestMethod2 (A)", + "CustomTestMethod2 (A)", + "CustomTestMethod2 (C)", + "CustomTestMethod2 (C)", + "CustomTestMethod2 (C)" + ); + } + } +} diff --git a/test/E2ETests/DiscoveryAndExecutionTests/Utilities/CLITestBase.discovery.cs b/test/E2ETests/DiscoveryAndExecutionTests/Utilities/CLITestBase.discovery.cs new file mode 100644 index 0000000000..b14d2ca58d Binary files /dev/null and b/test/E2ETests/DiscoveryAndExecutionTests/Utilities/CLITestBase.discovery.cs differ diff --git a/test/E2ETests/DiscoveryAndExecutionTests/Utilities/TestCaseFilterFactory.cs b/test/E2ETests/DiscoveryAndExecutionTests/Utilities/TestCaseFilterFactory.cs new file mode 100644 index 0000000000..97b041463a --- /dev/null +++ b/test/E2ETests/DiscoveryAndExecutionTests/Utilities/TestCaseFilterFactory.cs @@ -0,0 +1,358 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace DiscoveryAndExecutionTests.Utilities +{ + using Microsoft.VisualStudio.TestPlatform.ObjectModel; + using Microsoft.VisualStudio.TestPlatform.ObjectModel.Adapter; + using Microsoft.VisualStudio.TestPlatform.ObjectModel.Utilities; + + using System; + using System.Collections.Generic; + using System.Linq; + using System.Linq.Expressions; + using System.Reflection; + using System.Text; + using System.Text.RegularExpressions; + + internal static class TestCaseFilterFactory + { + private static readonly MethodInfo CachedGetMultiValueMethod; + private static readonly MethodInfo CachedEqualsComparerMethod; + private static readonly MethodInfo CachedContainsComparerMethod; + + static TestCaseFilterFactory() + { + CachedGetMultiValueMethod = typeof(TestCaseFilterFactory).GetMethod(nameof(GetMultiValue), BindingFlags.Static | BindingFlags.NonPublic); + CachedEqualsComparerMethod = typeof(TestCaseFilterFactory).GetMethod(nameof(EqualsComparer), BindingFlags.Static | BindingFlags.NonPublic); + CachedContainsComparerMethod = typeof(TestCaseFilterFactory).GetMethod(nameof(ContainsComparer), BindingFlags.Static | BindingFlags.NonPublic); + } + + public static ITestCaseFilterExpression ParseTestFilter(string filterString) + { + ValidateArg.NotNullOrEmpty(filterString, nameof(filterString)); + if (Regex.IsMatch(filterString, @"\(\s*\)")) + { + throw new FormatException($"Invalid filter, empty parenthesis: {filterString}"); + } + + var tokens = TokenizeFilter(filterString); + + var ops = new Stack(); + var exp = new Stack, bool>>>(); + + // simplified version of microsoft/vstest/src/Microsoft.TestPlatform.Common/Filtering/FilterExpression.cs + + // This is based on standard parsing of in order expression using two stacks (operand stack and operator stack) + // Precedence(And) > Precedence(Or) + foreach (var t in tokens) + { + var token = t.Trim(); + if (string.IsNullOrEmpty(token)) + { + continue; + } + + switch (token) + { + case "&": + case "|": + var op = token == "&" ? Operator.And : Operator.Or; + var top = ops.Count == 0 ? Operator.None : ops.Peek(); + if (ops.Count == 0 || top == Operator.OpenBrace || top < op) + { + ops.Push(op); + continue; + } + MergeExpression(exp, ops.Pop()); + continue; + + case "(": + ops.Push(Operator.OpenBrace); + continue; + + case ")": + if (ops.Count == 0) + { + throw new FormatException($"Invalid filter, missing parenthesis open: {filterString}"); + } + + while (ops.Peek() != Operator.OpenBrace) + { + MergeExpression(exp, ops.Pop()); + if (ops.Count == 0) + { + throw new FormatException($"Invalid filter, missing parenthesis open: {filterString}"); + } + } + ops.Pop(); + continue; + + default: + var e = ConditionExpresion(token); + exp.Push(e); + break; + } + } + + while (ops.Count != 0) + { + MergeExpression(exp, ops.Pop()); + } + + if (exp.Count != 1) + { + throw new FormatException($"Invalid filter, missing operator: {filterString}"); + } + + var lambda = exp.Pop().Compile(); + + return new TestFilterExpression(filterString, lambda); + } + + private class TestFilterExpression : ITestCaseFilterExpression + { + private readonly string filter; + private readonly Func, bool> expression; + + public TestFilterExpression(string filter, Func, bool> expression) + { + this.filter = filter; + this.expression = expression; + } + + public string TestCaseFilterValue => filter; + + public bool MatchTestCase(TestCase testCase, Func propertyValueProvider) => expression(propertyValueProvider); + } + + private static void MergeExpression(Stack, bool>>> exp, Operator op) + { + ValidateArg.NotNull(exp, nameof(exp)); + if (op != Operator.And && op != Operator.Or) + { + throw new ArgumentException($"Unexpected operator: {op}", nameof(op)); + } + if (exp.Count != 2) + { + throw new ArgumentException($"Unexpected expression tree: {exp.Count} elements, expected 2.", nameof(exp)); + } + + var parameter = Expression.Parameter(typeof(Func), "value"); + var right = Expression.Invoke(exp.Pop(), parameter); + var left = Expression.Invoke(exp.Pop(), parameter); + + Expression body = op == Operator.And ? Expression.And(left, right) : Expression.Or(left, right); + + var lambda = Expression.Lambda, bool>>(body, parameter); + + exp.Push(lambda); + } + + private static IEnumerable TokenizeFilter(string filterString) + { + var token = new StringBuilder(filterString.Length); + + var escaping = false; + for (int i = 0; i < filterString.Length; i++) + { + var c = filterString[i]; + + if (escaping) + { + token.Append(c); + escaping = false; + continue; + } + + switch (c) + { + case FilterHelper.EscapeCharacter: + escaping = true; + continue; + + case '&': + case '|': + case '(': + case ')': + if (token.Length != 0) + { + yield return token.ToString(); + token.Clear(); + } + yield return c.ToString(); + continue; + + default: + token.Append(c); + break; + } + } + + if (token.Length != 0) + { + yield return token.ToString(); + } + } + + private static IEnumerable TokenizeCondition(string conditionString) + { + ValidateArg.NotNullOrEmpty(conditionString, nameof(conditionString)); + var token = new StringBuilder(conditionString.Length); + + var escaped = false; + for (int i = 0; i < conditionString.Length; i++) + { + var c = conditionString[i]; + + if (escaped) + { + token.Append(c); + escaped = false; + continue; + } + + switch (c) + { + case '=': + case '~': + case '!': + if (token.Length > 0) + { + yield return token.ToString(); + token.Clear(); + } + + if (c == '!') + { + var op = conditionString[i + 1]; + + if (op == '~' || op == '=') + { + yield return token.ToString() + conditionString[++i]; + continue; + } + } + + yield return c.ToString(); + continue; + + default: + token.Append(c); + break; + } + } + + if (token.Length > 0) + { + yield return token.ToString(); + } + } + + private static string[] GetMultiValue(object value) + { + if (value is string[] i) + { + return i; + } + else if (value != null) + { + return new[] { value.ToString() }; + } + + return null; + } + + private static bool EqualsComparer(string[] values, string value) + { + foreach (var v in values) + { + if (v.Equals(value, StringComparison.OrdinalIgnoreCase)) + { + return true; + } + } + + return false; + } + + private static bool ContainsComparer(string[] values, string value) + { + foreach (var v in values) + { + if (v.IndexOf(value, StringComparison.OrdinalIgnoreCase) >= 0) + { + return true; + } + } + + return false; + } + + private static Expression, bool>> ConditionExpresion(string conditionString) + { + ValidateArg.NotNull(conditionString, nameof(conditionString)); + + var condition = TokenizeCondition(conditionString).ToArray(); + + Expression parameterName, expectedValue, parameterValueProvider, expression; + string op; + if (condition.Length == 1) + { + parameterName = Expression.Constant("FullyQualifiedName"); + expectedValue = Expression.Constant(conditionString.Trim()); + op = "~"; + } + else if (condition.Length == 3) + { + parameterName = Expression.Constant(condition[0]); + expectedValue = Expression.Constant(condition[2].Trim()); + op = condition[1]; + } + else + { + throw new FormatException("Invalid ConditionExpresion: " + conditionString); + } + + ParameterExpression parameter = Expression.Parameter(typeof(Func), "p"); + + parameterValueProvider = Expression.Call(CachedGetMultiValueMethod, Expression.Invoke(parameter, parameterName)); + + MethodInfo comparer; + switch (op.Last()) + { + case '=': + comparer = CachedEqualsComparerMethod; + break; + + case '~': + comparer = CachedContainsComparerMethod; + break; + + default: + throw new FormatException($"Invalid operator in {conditionString}: {condition[1]}"); + } + + expression = Expression.Call(comparer, parameterValueProvider, expectedValue); + + if (op[0] == '!') + { + expression = Expression.Not(expression); + } + + + var lambda = Expression.Lambda, bool>>(expression, parameter); + + return lambda; + } + + private enum Operator + { + None, + Or, + And, + OpenBrace, + CloseBrace, + } + } +} diff --git a/test/E2ETests/Smoke.E2E.Tests/CustomTestExecutionExtensibilityTests.cs b/test/E2ETests/Smoke.E2E.Tests/CustomTestExecutionExtensibilityTests.cs index 0358dc9835..d057a90ac0 100644 --- a/test/E2ETests/Smoke.E2E.Tests/CustomTestExecutionExtensibilityTests.cs +++ b/test/E2ETests/Smoke.E2E.Tests/CustomTestExecutionExtensibilityTests.cs @@ -15,6 +15,7 @@ public class CustomTestExecutionExtensibilityTests : CLITestBase public void ExecuteCustomTestExtensibilityTests() { this.InvokeVsTestForExecution(new string[] { TestAssembly }); + this.ValidatePassedTestsContain( "CustomTestMethod1 - Execution number 1", "CustomTestMethod1 - Execution number 2", @@ -35,13 +36,12 @@ public void ExecuteCustomTestExtensibilityTests() public void ExecuteCustomTestExtensibilityWithTestDataTests() { this.InvokeVsTestForExecution(new string[] { TestAssembly }, testCaseFilter: "FullyQualifiedName~CustomTestExTests.CustomTestMethod2"); + this.ValidatePassedTests( "CustomTestMethod2 (B)", "CustomTestMethod2 (B)", "CustomTestMethod2 (B)"); - - // Parent results should fail and thus failed count should be 7. - this.ValidateFailedTestsCount(7); + this.ValidateFailedTestsCount(6); this.ValidateFailedTestsContain( TestAssembly, true, diff --git a/test/E2ETests/Smoke.E2E.Tests/DataRowTests.cs b/test/E2ETests/Smoke.E2E.Tests/DataRowTests.cs index 280c4aeecd..c657993601 100644 --- a/test/E2ETests/Smoke.E2E.Tests/DataRowTests.cs +++ b/test/E2ETests/Smoke.E2E.Tests/DataRowTests.cs @@ -23,10 +23,10 @@ public void ExecuteOnlyDerivedClassDataRowsWhenBothBaseAndDerviedClassHasDataRow "DataRowTestMethod (DerivedString1)", "DataRowTestMethod (DerivedString2)"); - // 4 tests of BaseClass.DataRowTestMethod - 3 data row results and 1 parent result - // 3 tests of DerivedClass.DataRowTestMethod - 2 data row results and 1 parent result - // Total 7 tests - Making sure that DerivedClass doesn't run BaseClass tests - this.ValidatePassedTestsCount(7); + // 3 tests of BaseClass.DataRowTestMethod - 3 data row results and no parent result + // 2 tests of DerivedClass.DataRowTestMethod - 2 data row results and no parent result + // Total 5 tests - Making sure that DerivedClass doesn't run BaseClass tests + this.ValidatePassedTestsCount(5); } [TestMethod] @@ -38,8 +38,8 @@ public void ExecuteOnlyDerivedClassDataRowsWhenItOverridesBaseClassDataRows_Simp "DataRowTestMethod (DerivedString1)", "DataRowTestMethod (DerivedString2)"); - // 3 tests of DerivedClass.DataRowTestMethod - 2 datarow result and 1 parent result - this.ValidatePassedTestsCount(3); + // 2 tests of DerivedClass.DataRowTestMethod - 2 datarow result and no parent result + this.ValidatePassedTestsCount(2); } [TestMethod] @@ -52,8 +52,8 @@ public void DataRowsExecuteWithRequiredAndOptionalParameters() "DataRowTestMethodWithSomeOptionalParameters (123,DerivedOptionalString1)", "DataRowTestMethodWithSomeOptionalParameters (123,DerivedOptionalString2,DerivedOptionalString3)"); - // 4 tests of DerivedClass.DataRowTestMethodWithSomeOptionalParameters - 3 datarow result and 1 parent result - this.ValidatePassedTestsCount(4); + // 3 tests of DerivedClass.DataRowTestMethodWithSomeOptionalParameters - 3 datarow result and no parent result + this.ValidatePassedTestsCount(3); } [TestMethod] @@ -67,8 +67,8 @@ public void DataRowsExecuteWithAllOptionalParameters() "DataRowTestMethodWithAllOptionalParameters (123,DerivedOptionalString4)", "DataRowTestMethodWithAllOptionalParameters (123,DerivedOptionalString5,DerivedOptionalString6)"); - // 5 tests of DerivedClass.DataRowTestMethodWithAllOptionalParameters - 4 datarow result and 1 parent result - this.ValidatePassedTestsCount(5); + // 4 tests of DerivedClass.DataRowTestMethodWithAllOptionalParameters - 4 datarow result and no parent result + this.ValidatePassedTestsCount(4); } [TestMethod] @@ -82,8 +82,8 @@ public void DataRowsExecuteWithParamsArrayParameter() "DataRowTestMethodWithParamsParameters (2,DerivedParamsArg1,DerivedParamsArg2)", "DataRowTestMethodWithParamsParameters (2,DerivedParamsArg1,DerivedParamsArg2,DerivedParamsArg3)"); - // 5 tests of DerivedClass.DataRowTestMethodWithParamsParameters - 4 datarow result and 1 parent result - this.ValidatePassedTestsCount(5); + // 4 tests of DerivedClass.DataRowTestMethodWithParamsParameters - 4 datarow result and no parent result + this.ValidatePassedTestsCount(4); } [TestMethod] @@ -96,8 +96,8 @@ public void DataRowsFailWhenInvalidArgumentsProvided() "DataRowTestMethodFailsWithInvalidArguments (2)", "DataRowTestMethodFailsWithInvalidArguments (2,DerivedRequiredArgument,DerivedOptionalArgument,DerivedExtraArgument)"); - // 4 tests of DerivedClass.DataRowTestMethodFailsWithInvalidArguments - 3 datarow result and 1 parent result - this.ValidatePassedTestsCount(4); + // 3 tests of DerivedClass.DataRowTestMethodFailsWithInvalidArguments - 3 datarow result and no parent result + this.ValidatePassedTestsCount(3); } } } diff --git a/test/E2ETests/TestAssets/Directory.Build.targets b/test/E2ETests/TestAssets/Directory.Build.targets index 5a5c0f5db7..87043c6abb 100644 --- a/test/E2ETests/TestAssets/Directory.Build.targets +++ b/test/E2ETests/TestAssets/Directory.Build.targets @@ -9,4 +9,4 @@ - \ No newline at end of file + diff --git a/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/Discovery/AssemblyEnumeratorTests.cs b/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/Discovery/AssemblyEnumeratorTests.cs index 119edbe000..b4ca760266 100644 --- a/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/Discovery/AssemblyEnumeratorTests.cs +++ b/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/Discovery/AssemblyEnumeratorTests.cs @@ -19,6 +19,7 @@ namespace Microsoft.VisualStudio.TestPlatform.MSTestAdapter.UnitTests.Discovery using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.Helpers; using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.ObjectModel; using Microsoft.VisualStudio.TestPlatform.MSTestAdapter.UnitTests.TestableImplementations; + using Microsoft.VisualStudio.TestPlatform.ObjectModel.Adapter; using Moq; @@ -77,6 +78,7 @@ public void ConstructorShouldPopulateSettings() MSTestSettings adapterSettings = MSTestSettings.GetSettings(runSettingsXml, MSTestSettings.SettingsName); var assemblyEnumerator = new AssemblyEnumerator(adapterSettings); + assemblyEnumerator.RunSettingsXml = runSettingsXml; Assert.IsTrue(MSTestSettings.CurrentSettings.ForcedLegacyMode); Assert.AreEqual("DummyPath\\TestSettings1.testsettings", MSTestSettings.CurrentSettings.TestSettingsFile); diff --git a/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/Execution/TestMethodRunnerTests.cs b/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/Execution/TestMethodRunnerTests.cs index d800c29e48..ed26490906 100644 --- a/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/Execution/TestMethodRunnerTests.cs +++ b/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/Execution/TestMethodRunnerTests.cs @@ -555,11 +555,9 @@ public void RunTestMethodShouldSetDataRowIndexForDataDrivenTestsWhenDataIsProvid var results = testMethodRunner.RunTestMethod(); // check for datarowIndex - // 1st is parent result. - Assert.AreEqual(-1, results[0].DatarowIndex); - Assert.AreEqual(0, results[1].DatarowIndex); - Assert.AreEqual(1, results[2].DatarowIndex); - Assert.AreEqual(2, results[3].DatarowIndex); + Assert.AreEqual(0, results[0].DatarowIndex); + Assert.AreEqual(1, results[1].DatarowIndex); + Assert.AreEqual(2, results[2].DatarowIndex); } [TestMethodV1] @@ -584,11 +582,9 @@ public void RunTestMethodShoudlRunOnlyDataSourceTestsWhenBothDataSourceAndDataRo var results = testMethodRunner.RunTestMethod(); // check for datarowIndex as only DataSource Tests are Run - // 1st is parent result. - Assert.AreEqual(-1, results[0].DatarowIndex); - Assert.AreEqual(0, results[1].DatarowIndex); - Assert.AreEqual(1, results[2].DatarowIndex); - Assert.AreEqual(2, results[3].DatarowIndex); + Assert.AreEqual(0, results[0].DatarowIndex); + Assert.AreEqual(1, results[1].DatarowIndex); + Assert.AreEqual(2, results[2].DatarowIndex); } [TestMethodV1] @@ -612,9 +608,8 @@ public void RunTestMethodShouldFillInDisplayNameWithDataRowDisplayNameIfProvided var results = testMethodRunner.RunTestMethod(); - // 1st results should be parent result. - Assert.AreEqual(2, results.Length); - Assert.AreEqual("DataRowTestDisplayName", results[1].DisplayName); + Assert.AreEqual(1, results.Length); + Assert.AreEqual("DataRowTestDisplayName", results[0].DisplayName); } [TestMethodV1] @@ -637,9 +632,8 @@ public void RunTestMethodShouldFillInDisplayNameWithDataRowArgumentsIfNoDisplayN var results = testMethodRunner.RunTestMethod(); - // 1st results should be parent result. - Assert.AreEqual(2, results.Length); - Assert.AreEqual("DummyTestMethod (2,DummyString)", results[1].DisplayName); + Assert.AreEqual(1, results.Length); + Assert.AreEqual("DummyTestMethod (2,DummyString)", results[0].DisplayName); } [TestMethodV1] @@ -664,219 +658,8 @@ public void RunTestMethodShouldSetResultFilesIfPresentForDataDrivenTests() this.testablePlatformServiceProvider.MockReflectionOperations.Setup(rf => rf.GetCustomAttributes(this.methodInfo, It.IsAny(), It.IsAny())).Returns(attribs); var results = testMethodRunner.RunTestMethod(); + CollectionAssert.Contains(results[0].ResultFiles.ToList(), "C:\\temp.txt"); CollectionAssert.Contains(results[1].ResultFiles.ToList(), "C:\\temp.txt"); - CollectionAssert.Contains(results[2].ResultFiles.ToList(), "C:\\temp.txt"); - } - - [TestMethodV1] - public void RunTestMethodShouldReturnParentResultForDataSourceDataDrivenTests() - { - var testMethodInfo = new TestableTestmethodInfo(this.methodInfo, this.testClassInfo, this.testMethodOptions, () => new UTF.TestResult()); - var testMethodRunner = new TestMethodRunner(testMethodInfo, this.testMethod, this.testContextImplementation, false); - - UTF.DataSourceAttribute dataSourceAttribute = new UTF.DataSourceAttribute("DummyConnectionString", "DummyTableName"); - - var attribs = new Attribute[] { dataSourceAttribute }; - - // Setup mocks - this.testablePlatformServiceProvider.MockReflectionOperations.Setup(rf => rf.GetCustomAttributes(this.methodInfo, It.IsAny(), It.IsAny())).Returns(attribs); - this.testablePlatformServiceProvider.MockTestDataSource.Setup(tds => tds.GetData(testMethodInfo, this.testContextImplementation)).Returns(new object[] { 1, 2, 3 }); - - var results = testMethodRunner.RunTestMethod(); - - // check for parent result - Assert.AreEqual(4, results.Length); - Assert.AreEqual(results[0].ExecutionId, results[1].ParentExecId); - Assert.AreEqual(Guid.Empty, results[0].ParentExecId); - Assert.AreNotEqual(Guid.Empty, results[1].ParentExecId); - } - - [TestMethodV1] - public void RunTestMethodShouldReturnParentResultForDataSourceDataDrivenTestsContainingSingleTest() - { - var testMethodInfo = new TestableTestmethodInfo(this.methodInfo, this.testClassInfo, this.testMethodOptions, () => new UTF.TestResult()); - var testMethodRunner = new TestMethodRunner(testMethodInfo, this.testMethod, this.testContextImplementation, false); - - UTF.DataSourceAttribute dataSourceAttribute = new UTF.DataSourceAttribute("DummyConnectionString", "DummyTableName"); - - var attribs = new Attribute[] { dataSourceAttribute }; - - // Setup mocks - this.testablePlatformServiceProvider.MockReflectionOperations.Setup(rf => rf.GetCustomAttributes(this.methodInfo, It.IsAny(), It.IsAny())).Returns(attribs); - this.testablePlatformServiceProvider.MockTestDataSource.Setup(tds => tds.GetData(testMethodInfo, this.testContextImplementation)).Returns(new object[] { 1 }); - - var results = testMethodRunner.RunTestMethod(); - - // Parent result should exist. - Assert.AreEqual(2, results.Length); - Assert.AreEqual(results[0].ExecutionId, results[1].ParentExecId); - Assert.AreEqual(Guid.Empty, results[0].ParentExecId); - Assert.AreNotEqual(Guid.Empty, results[1].ParentExecId); - } - - [TestMethodV1] - public void RunTestMethodShouldReturnParentResultForDataRowDataDrivenTests() - { - UTF.TestResult testResult = new UTF.TestResult - { - ResultFiles = new List() { "C:\\temp.txt" } - }; - - var testMethodInfo = new TestableTestmethodInfo(this.methodInfo, this.testClassInfo, this.testMethodOptions, () => testResult); - var testMethodRunner = new TestMethodRunner(testMethodInfo, this.testMethod, this.testContextImplementation, false, this.mockReflectHelper.Object); - - int dummyIntData1 = 1; - int dummyIntData2 = 2; - UTF.DataRowAttribute dataRowAttribute1 = new UTF.DataRowAttribute(dummyIntData1); - UTF.DataRowAttribute dataRowAttribute2 = new UTF.DataRowAttribute(dummyIntData2); - - var attribs = new Attribute[] { dataRowAttribute1, dataRowAttribute2 }; - - // Setup mocks - this.testablePlatformServiceProvider.MockReflectionOperations.Setup(rf => rf.GetCustomAttributes(this.methodInfo, It.IsAny(), It.IsAny())).Returns(attribs); - - var results = testMethodRunner.RunTestMethod(); - CollectionAssert.Contains(results[1].ResultFiles.ToList(), "C:\\temp.txt"); - CollectionAssert.Contains(results[2].ResultFiles.ToList(), "C:\\temp.txt"); - - // Parent result should exist. - Assert.AreEqual(3, results.Length); - Assert.AreEqual(results[0].ExecutionId, results[1].ParentExecId); - Assert.AreEqual(results[0].ExecutionId, results[2].ParentExecId); - Assert.AreEqual(Guid.Empty, results[0].ParentExecId); - Assert.AreNotEqual(Guid.Empty, results[1].ParentExecId); - Assert.AreNotEqual(Guid.Empty, results[2].ParentExecId); - } - - [TestMethodV1] - public void RunTestMethodShouldReturnParentResultForDataRowDataDrivenTestsContainingSingleTest() - { - UTF.TestResult testResult = new UTF.TestResult - { - ResultFiles = new List() { "C:\\temp.txt" } - }; - - var testMethodInfo = new TestableTestmethodInfo(this.methodInfo, this.testClassInfo, this.testMethodOptions, () => testResult); - var testMethodRunner = new TestMethodRunner(testMethodInfo, this.testMethod, this.testContextImplementation, false, this.mockReflectHelper.Object); - - int dummyIntData1 = 1; - UTF.DataRowAttribute dataRowAttribute1 = new UTF.DataRowAttribute(dummyIntData1); - - var attribs = new Attribute[] { dataRowAttribute1 }; - - // Setup mocks - this.testablePlatformServiceProvider.MockReflectionOperations.Setup(rf => rf.GetCustomAttributes(this.methodInfo, It.IsAny(), It.IsAny())).Returns(attribs); - - var results = testMethodRunner.RunTestMethod(); - CollectionAssert.Contains(results[1].ResultFiles.ToList(), "C:\\temp.txt"); - - // Parent result should exist. - Assert.AreEqual(2, results.Length); - Assert.AreEqual(results[0].ExecutionId, results[1].ParentExecId); - Assert.AreEqual(Guid.Empty, results[0].ParentExecId); - Assert.AreNotEqual(Guid.Empty, results[1].ParentExecId); - } - - [TestMethodV1] - public void RunTestMethodShouldNotReturnParentResultForNonDataDrivenTests() - { - var testMethodAttributeMock = new Mock(); - testMethodAttributeMock.Setup(_ => _.Execute(It.IsAny())).Returns(new[] - { - new UTF.TestResult { Outcome = UTF.UnitTestOutcome.Passed }, - new UTF.TestResult { Outcome = UTF.UnitTestOutcome.Failed } - }); - - var localTestMethodOptions = new TestMethodOptions - { - Timeout = 200, - Executor = testMethodAttributeMock.Object, - TestContext = this.testContextImplementation, - ExpectedException = null - }; - - var testMethodInfo = new TestableTestmethodInfo(this.methodInfo, this.testClassInfo, localTestMethodOptions, null); - var testMethodRunner = new TestMethodRunner(testMethodInfo, this.testMethod, this.testContextImplementation, false); - - var results = testMethodRunner.Execute(); - Assert.AreEqual(2, results.Length); - - // Parent result should not exists as its not data driven test. - Assert.AreEqual(AdapterTestOutcome.Passed, results[0].Outcome); - Assert.AreEqual(AdapterTestOutcome.Failed, results[1].Outcome); - } - - [TestMethodV1] - public void RunTestMethodShouldSetParentResultOutcomeProperlyForDataSourceDataDrivenTests() - { - var testExecutedCount = 0; - var testMethodInfo = new TestableTestmethodInfo(this.methodInfo, this.testClassInfo, this.testMethodOptions, () => - { - return (testExecutedCount++ == 0) ? - new UTF.TestResult { Outcome = UTF.UnitTestOutcome.Failed } : - new UTF.TestResult { Outcome = UTF.UnitTestOutcome.Passed }; - }); - var testMethodRunner = new TestMethodRunner(testMethodInfo, this.testMethod, this.testContextImplementation, false); - - UTF.DataSourceAttribute dataSourceAttribute = new UTF.DataSourceAttribute("DummyConnectionString", "DummyTableName"); - - var attribs = new Attribute[] { dataSourceAttribute }; - - // Setup mocks - this.testablePlatformServiceProvider.MockReflectionOperations.Setup(rf => rf.GetCustomAttributes(this.methodInfo, It.IsAny(), It.IsAny())).Returns(attribs); - this.testablePlatformServiceProvider.MockTestDataSource.Setup(tds => tds.GetData(testMethodInfo, this.testContextImplementation)).Returns(new object[] { 1, 2, 3 }); - - var results = testMethodRunner.RunTestMethod(); - - // check for parent result - Assert.AreEqual(4, results.Length); - Assert.AreEqual(results[0].ExecutionId, results[1].ParentExecId); - Assert.AreEqual(Guid.Empty, results[0].ParentExecId); - Assert.AreNotEqual(Guid.Empty, results[1].ParentExecId); - - // Check for aggregate outcome. - Assert.AreEqual(AdapterTestOutcome.Failed, results[0].Outcome); - Assert.AreEqual(AdapterTestOutcome.Failed, results[1].Outcome); - Assert.AreEqual(AdapterTestOutcome.Passed, results[2].Outcome); - Assert.AreEqual(AdapterTestOutcome.Passed, results[3].Outcome); - } - - [TestMethodV1] - public void RunTestMethodShouldSetParentResultOutcomeProperlyForDataRowDataDrivenTests() - { - var testExecutedCount = 0; - var testMethodInfo = new TestableTestmethodInfo(this.methodInfo, this.testClassInfo, this.testMethodOptions, () => - { - return (testExecutedCount++ == 0) ? - new UTF.TestResult { Outcome = UTF.UnitTestOutcome.Failed } : - new UTF.TestResult { Outcome = UTF.UnitTestOutcome.Passed }; - }); - var testMethodRunner = new TestMethodRunner(testMethodInfo, this.testMethod, this.testContextImplementation, false, this.mockReflectHelper.Object); - - int dummyIntData1 = 1; - int dummyIntData2 = 2; - UTF.DataRowAttribute dataRowAttribute1 = new UTF.DataRowAttribute(dummyIntData1); - UTF.DataRowAttribute dataRowAttribute2 = new UTF.DataRowAttribute(dummyIntData2); - - var attribs = new Attribute[] { dataRowAttribute1, dataRowAttribute2 }; - - // Setup mocks - this.testablePlatformServiceProvider.MockReflectionOperations.Setup(rf => rf.GetCustomAttributes(this.methodInfo, It.IsAny(), It.IsAny())).Returns(attribs); - - var results = testMethodRunner.RunTestMethod(); - - // Parent result should exist. - Assert.AreEqual(3, results.Length); - Assert.AreEqual(results[0].ExecutionId, results[1].ParentExecId); - Assert.AreEqual(results[0].ExecutionId, results[2].ParentExecId); - Assert.AreEqual(Guid.Empty, results[0].ParentExecId); - Assert.AreNotEqual(Guid.Empty, results[1].ParentExecId); - Assert.AreNotEqual(Guid.Empty, results[2].ParentExecId); - - // Check for aggregate outcome. - Assert.AreEqual(AdapterTestOutcome.Failed, results[0].Outcome); - Assert.AreEqual(AdapterTestOutcome.Failed, results[1].Outcome); - Assert.AreEqual(AdapterTestOutcome.Passed, results[2].Outcome); } #region Test data diff --git a/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/packages.config b/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/packages.config index 73a75bf39f..f8236bbbc5 100644 --- a/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/packages.config +++ b/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/packages.config @@ -2,8 +2,8 @@ - - + + diff --git a/test/UnitTests/PlatformServices.Desktop.Unit.Tests/packages.config b/test/UnitTests/PlatformServices.Desktop.Unit.Tests/packages.config index 1438956c89..75ac30848c 100644 --- a/test/UnitTests/PlatformServices.Desktop.Unit.Tests/packages.config +++ b/test/UnitTests/PlatformServices.Desktop.Unit.Tests/packages.config @@ -1,7 +1,7 @@  - + diff --git a/test/UnitTests/PlatformServices.Portable.Unit.Tests/packages.config b/test/UnitTests/PlatformServices.Portable.Unit.Tests/packages.config index 637fb47db0..b5d1923828 100644 --- a/test/UnitTests/PlatformServices.Portable.Unit.Tests/packages.config +++ b/test/UnitTests/PlatformServices.Portable.Unit.Tests/packages.config @@ -1,7 +1,7 @@  - + diff --git a/test/UnitTests/PlatformServices.Universal.Unit.Tests/packages.config b/test/UnitTests/PlatformServices.Universal.Unit.Tests/packages.config index 311ae4c674..67cae056cf 100644 --- a/test/UnitTests/PlatformServices.Universal.Unit.Tests/packages.config +++ b/test/UnitTests/PlatformServices.Universal.Unit.Tests/packages.config @@ -1,8 +1,8 @@  - - + +