Skip to content

Commit 0b481d6

Browse files
author
Christine Zhou
committed
Testing commit for running regression tests on parametric testing feature
1 parent 5c77847 commit 0b481d6

File tree

5 files changed

+290
-5
lines changed

5 files changed

+290
-5
lines changed

‎Tst/UnitTests/Core/Constants.cs

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,8 @@ public static class Constants
4444

4545
public static string TestDirectory => Path.Combine(SolutionDirectory, TestDirectoryName);
4646

47+
public static string TestParamTesterDirectory => Path.Combine(TestDirectory, "TestParamTester");
48+
4749
public static readonly string ScratchParentDirectory = Path.Combine(TestDirectory, "temp_builds");
4850
}
49-
}
51+
}

‎Tst/UnitTests/Core/TestCaseFactory.cs

Lines changed: 41 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
using System.Collections.Generic;
33
using System.IO;
44
using System.Linq;
5+
using System.Text;
56
using Plang.Compiler;
67
using UnitTests.Runners;
78
using UnitTests.Validators;
@@ -58,7 +59,18 @@ public CompilerTestCase CreateTestCase(DirectoryInfo testDir, CompilerOutput out
5859
if (output.Equals(CompilerOutput.PChecker))
5960
{
6061
var nativeFiles = testDir.GetFiles("*.cs");
61-
runner = new PCheckerRunner(inputFiles, nativeFiles);
62+
63+
// Check if this is a parametric test
64+
bool isParametricTest = HasParametricTests(inputFiles);
65+
66+
if (isParametricTest)
67+
{
68+
runner = new ParametricPCheckerRunner(inputFiles, nativeFiles);
69+
}
70+
else
71+
{
72+
runner = new PCheckerRunner(inputFiles, nativeFiles);
73+
}
6274
}
6375
else
6476
{
@@ -95,5 +107,32 @@ public CompilerTestCase CreateTestCase(DirectoryInfo testDir)
95107
Directory.CreateDirectory(Path.Combine(testTempBaseDir.FullName, output.ToString(), testName));
96108
return new CompilerTestCase(tempDirName, runner, validator);
97109
}
110+
111+
/// <summary>
112+
/// Checks if any of the input files contain parametric test declarations
113+
/// </summary>
114+
/// <param name="inputFiles">Array of P source files to check</param>
115+
/// <returns>True if parametric tests are found, false otherwise</returns>
116+
private bool HasParametricTests(FileInfo[] inputFiles)
117+
{
118+
foreach (var file in inputFiles)
119+
{
120+
try
121+
{
122+
var content = File.ReadAllText(file.FullName);
123+
// Look for "test param" declarations which indicate parametric tests
124+
if (content.Contains("test param"))
125+
{
126+
return true;
127+
}
128+
}
129+
catch (Exception)
130+
{
131+
// If we can't read the file, assume it's not a parametric test
132+
continue;
133+
}
134+
}
135+
return false;
136+
}
98137
}
99-
}
138+
}

‎Tst/UnitTests/Core/TestCaseLoader.cs

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,8 @@ public static class TestCaseLoader
1818
Path.Combine("RegressionTests", "Feature3Exprs"),
1919
Path.Combine("RegressionTests", "Feature4DataTypes"),
2020
//Path.Combine("RegressionTests","Feature5ModuleSystem"),
21-
Path.Combine("RegressionTests", "Integration")
21+
Path.Combine("RegressionTests", "Integration"),
22+
"TestParamTester"
2223
};
2324

2425
public static IEnumerable<TestCaseData> FindTestCasesInDirectory(string directoryName, string[] testDirNames)
@@ -53,4 +54,4 @@ private static string GetCategory(DirectoryInfo dir, DirectoryInfo baseDirectory
5354
return category;
5455
}
5556
}
56-
}
57+
}

‎Tst/UnitTests/ParametricRegressionTests.cs

Lines changed: 113 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,113 @@
1+
using System;
2+
using System.IO;
3+
using System.Linq;
4+
using System.Reflection;
5+
using NUnit.Framework;
6+
using Plang;
7+
using Plang.Compiler;
8+
using Plang.Options;
9+
using PChecker;
10+
using PChecker.Testing;
11+
12+
namespace UnitTests
13+
{
14+
[TestFixture]
15+
public class ParametricRegressionTests
16+
{
17+
private string GetProjectRoot()
18+
{
19+
// Start from the current directory and walk up until we find the P.sln file
20+
var directory = new DirectoryInfo(Directory.GetCurrentDirectory());
21+
while (directory != null && !File.Exists(Path.Combine(directory.FullName, "P.sln")))
22+
{
23+
directory = directory.Parent;
24+
}
25+
26+
if (directory == null)
27+
{
28+
throw new Exception("Could not find P project root directory");
29+
}
30+
31+
return directory.FullName;
32+
}
33+
34+
[Test]
35+
public void TestClientServerParametric()
36+
{
37+
// Path to the project file
38+
string projectPath = Path.Combine(GetProjectRoot(), "Tst", "TestParamTester", "1_ClientServer", "ClientServer.pproj");
39+
40+
// First compile the project
41+
string[] compileArgs = new[] { "compile", "--pproj", projectPath };
42+
CommandLine.RunCompiler(compileArgs.Skip(1).ToArray());
43+
44+
// Then run different test cases
45+
string[] testCases = new[]
46+
{
47+
"tcSingleClient", // Basic test
48+
"aaaa1___nClients_2__g1_1__g2_4", // Parametric test
49+
"testTWise2___nClients_4__g1_1__g2_4__b1_True" // T-wise test
50+
};
51+
52+
foreach (var testCase in testCases)
53+
{
54+
var checkerOptions = new PCheckerOptions();
55+
var config = checkerOptions.Parse(new[] { "--testcase", testCase });
56+
Checker.Run(config);
57+
}
58+
}
59+
60+
[Test]
61+
public void TestClientServerAllParametricCombinations()
62+
{
63+
// Path to the project file
64+
string projectPath = Path.Combine(GetProjectRoot(), "Tst", "TestParamTester", "1_ClientServer", "ClientServer.pproj");
65+
66+
// First compile the project
67+
string[] compileArgs = new[] { "compile", "--pproj", projectPath };
68+
CommandLine.RunCompiler(compileArgs.Skip(1).ToArray());
69+
70+
// Run all parametric test combinations
71+
for (int nClients = 2; nClients <= 4; nClients++)
72+
{
73+
for (int g1 = 1; g1 <= 2; g1++)
74+
{
75+
for (int g2 = 4; g2 <= 5; g2++)
76+
{
77+
string testCase = $"aaaa1___nClients_{nClients}__g1_{g1}__g2_{g2}";
78+
var checkerOptions = new PCheckerOptions();
79+
var config = checkerOptions.Parse(new[] { "--testcase", testCase });
80+
Checker.Run(config);
81+
}
82+
}
83+
}
84+
}
85+
86+
[Test]
87+
public void TestClientServerTWise()
88+
{
89+
// Path to the project file
90+
string projectPath = Path.Combine(GetProjectRoot(), "Tst", "TestParamTester", "1_ClientServer", "ClientServer.pproj");
91+
92+
// First compile the project
93+
string[] compileArgs = new[] { "compile", "--pproj", projectPath };
94+
CommandLine.RunCompiler(compileArgs.Skip(1).ToArray());
95+
96+
// Run T-wise tests with different T values
97+
string[] tWiseTests = new[]
98+
{
99+
"testTWise1___nClients_4__g1_1__g2_4__b1_True", // 1-wise
100+
"testTWise2___nClients_4__g1_1__g2_4__b1_True", // 2-wise
101+
"testTWise3___nClients_4__g1_1__g2_4__b1_True", // 3-wise
102+
"testTWise4___nClients_4__g1_1__g2_4__b1_True" // 4-wise
103+
};
104+
105+
foreach (var testCase in tWiseTests)
106+
{
107+
var checkerOptions = new PCheckerOptions();
108+
var config = checkerOptions.Parse(new[] { "--testcase", testCase });
109+
Checker.Run(config);
110+
}
111+
}
112+
}
113+
}

‎Tst/UnitTests/Runners/ParametricPCheckerRunner.cs

Lines changed: 130 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,130 @@
1+
using System;
2+
using System.Collections.Generic;
3+
using System.IO;
4+
using System.Linq;
5+
using System.Threading;
6+
using Plang.Compiler;
7+
using UnitTests.Core;
8+
9+
namespace UnitTests.Runners
10+
{
11+
/// <summary>
12+
/// Runner for parametric tests that uses generated main.cs instead of creating a hardcoded one
13+
/// </summary>
14+
internal class ParametricPCheckerRunner : ICompilerTestRunner
15+
{
16+
private readonly FileInfo[] nativeSources;
17+
private readonly FileInfo[] sources;
18+
19+
public ParametricPCheckerRunner(FileInfo[] sources)
20+
{
21+
this.sources = sources;
22+
nativeSources = new FileInfo[] { };
23+
}
24+
25+
public ParametricPCheckerRunner(FileInfo[] sources, FileInfo[] nativeSources)
26+
{
27+
this.sources = sources;
28+
this.nativeSources = nativeSources;
29+
}
30+
31+
private void FileCopy(string src, string target, bool overwrite)
32+
{
33+
// during parallel testing we might get "The process cannot access the file because it is being used by another process."
34+
var retries = 5;
35+
while (retries-- > 0)
36+
{
37+
try
38+
{
39+
File.Copy(src, target, overwrite);
40+
return;
41+
}
42+
catch (IOException)
43+
{
44+
if (retries == 1)
45+
{
46+
throw;
47+
}
48+
Thread.Sleep(1000);
49+
}
50+
}
51+
}
52+
53+
public int? RunTest(DirectoryInfo scratchDirectory, out string stdout, out string stderr)
54+
{
55+
stdout = "";
56+
stderr = "";
57+
58+
// path to generated code
59+
DirectoryInfo scratchDirectoryGenerated = Directory.CreateDirectory(Path.Combine(scratchDirectory.FullName, "PChecker"));
60+
61+
// For parametric tests, we don't create our own main function - we use the generated one
62+
// Only create csproj file
63+
CreateCSProjFile(scratchDirectoryGenerated);
64+
65+
// copy the foreign code to the folder
66+
foreach (var nativeFile in nativeSources)
67+
{
68+
FileCopy(nativeFile.FullName, Path.Combine(scratchDirectoryGenerated.FullName, nativeFile.Name), true);
69+
}
70+
71+
var exitCode = DoCompile(scratchDirectory);
72+
73+
if (exitCode == 0)
74+
{
75+
exitCode = RunPChecker(scratchDirectoryGenerated.FullName,
76+
Path.Combine(scratchDirectoryGenerated.FullName, "./net8.0/Main.dll"), out var testStdout, out var testStderr);
77+
stdout += testStdout;
78+
stderr += testStderr;
79+
}
80+
else
81+
{
82+
// compilation returned unexpected error
83+
exitCode = 2;
84+
}
85+
86+
return exitCode;
87+
}
88+
89+
private void CreateCSProjFile(DirectoryInfo scratchDirectory)
90+
{
91+
const string csprojTemplate = @"
92+
<Project Sdk=""Microsoft.NET.Sdk"">
93+
<PropertyGroup>
94+
<TargetFramework>net8.0</TargetFramework>
95+
<ApplicationIcon />
96+
<OutputType>Exe</OutputType>
97+
<StartupObject />
98+
<LangVersion>latest</LangVersion>
99+
<OutputPath>.</OutputPath>
100+
</PropertyGroup>
101+
<ItemGroup>
102+
<ProjectReference Include=""$(PFolder)/Src/PChecker/CheckerCore/CheckerCore.csproj"" />
103+
</ItemGroup>
104+
</Project>";
105+
using var outputFile = new StreamWriter(Path.Combine(scratchDirectory.FullName, "Main.csproj"), false);
106+
outputFile.WriteLine(csprojTemplate);
107+
}
108+
109+
private int RunPChecker(string directory, string dllPath, out string stdout, out string stderr)
110+
{
111+
return ProcessHelper.RunWithOutput(directory, out stdout, out stderr, "dotnet", dllPath);
112+
}
113+
114+
public int DoCompile(DirectoryInfo scratchDirectory)
115+
{
116+
var compiler = new Compiler();
117+
var outputStream = new TestExecutionStream(scratchDirectory);
118+
var compilerConfiguration = new CompilerConfiguration(outputStream, scratchDirectory, new List<CompilerOutput>{CompilerOutput.PChecker}, sources.Select(x => x.FullName).ToList(), "Main", scratchDirectory);
119+
try
120+
{
121+
return compiler.Compile(compilerConfiguration);
122+
}
123+
catch (Exception ex)
124+
{
125+
compilerConfiguration.Output.WriteError($"<Internal Error>:\n {ex.Message}\n<Please report to the P team or create an issue on GitHub, Thanks!>");
126+
return 1;
127+
}
128+
}
129+
}
130+
}

0 commit comments

Comments
 (0)