Skip to content

test: Retry tests on known unrelated errors #1644

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
May 24, 2023
Merged

test: Retry tests on known unrelated errors #1644

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
c2e4b98
test: Retry tests on known unrelated errors
chynesNR May 19, 2023
e3af0e0
Retry on exceptions thrown by the test fixture
chynesNR May 19, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
44 changes: 33 additions & 11 deletions ...IntegrationTests/IntegrationTestHelpers/RemoteServiceFixtures/RemoteApplicationFixture.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net;
using System.Net.Http;
using System.Threading;
Expand All @@ -21,6 +22,12 @@ public abstract class RemoteApplicationFixture : IDisposable

private Action _setupConfiguration;
private Action _exerciseApplication;
private HashSet<uint> _errorsToRetryOn = new HashSet<uint>
{
0xC000_0005 // System.AccessViolationException. This is a .NET bug that
// is supposed to be fixed but we're still seeing
// https://github.com/dotnet/runtime/issues/62145
};

public Dictionary<string, string> EnvironmentVariables;

Expand Down Expand Up @@ -88,8 +95,8 @@ public bool KeepWorkingDirectory
set { RemoteApplication.KeepWorkingDirectory = value; }
}

// We think that the test retry loop may be masking real problems and/or actually causing them, so we've disabled it for now
protected virtual int MaxTries => 1;
// Tests are only retried if they return a known error not related to the test
protected virtual int MaxTries => 3;

public void DisableAsyncLocalCallStack()
{
Expand Down Expand Up @@ -190,11 +197,23 @@ public RemoteApplicationFixture SetAdditionalEnvironmentVariable(string key, str
return this;
}

public void AddErrorToRetryOn(uint error)
{
_errorsToRetryOn.Add(error);
}

private void ExerciseApplication()
{
_exerciseApplication?.Invoke();
}

private string FormatExitCode(int? exitCode)
{
if (exitCode == null) return "[null]";
if (Math.Abs(exitCode.Value) < 10) return exitCode.Value.ToString();
return exitCode.Value.ToString("X8");
}

public virtual void Initialize()
{
lock (_initializeLock)
Expand All @@ -214,16 +233,14 @@ public virtual void Initialize()
try
{
var retryTest = false;
var exceptionInExerciseApplication = false;
var retryMessage = "";
var applicationHadNonZeroExitCode = false;


do
{
TestLogger?.WriteLine("Test Home" + RemoteApplication.DestinationNewRelicHomeDirectoryPath);

// reset these for each loop iteration
exceptionInExerciseApplication = false;
applicationHadNonZeroExitCode = false;
retryTest = false;

Expand All @@ -245,8 +262,9 @@ public virtual void Initialize()
}
catch (Exception ex)
{
exceptionInExerciseApplication = true;
retryTest = true;
TestLogger?.WriteLine("Exception occurred in try number " + (numberOfTries + 1) + " : " + ex.ToString());
retryMessage = "Exception thrown.";
}
finally
{
Expand Down Expand Up @@ -278,15 +296,19 @@ public virtual void Initialize()
RemoteApplication.WaitForExit();

applicationHadNonZeroExitCode = RemoteApplication.ExitCode != 0;
var formattedExitCode = FormatExitCode(RemoteApplication.ExitCode);

TestLogger?.WriteLine($"Remote application exited with a {(applicationHadNonZeroExitCode ? "failure" : "success")} exit code of {RemoteApplication.ExitCode}.");
TestLogger?.WriteLine($"Remote application exited with a {(applicationHadNonZeroExitCode ? "failure" : "success")} exit code of {formattedExitCode}.");

retryTest = exceptionInExerciseApplication || applicationHadNonZeroExitCode;
if (applicationHadNonZeroExitCode && _errorsToRetryOn.Contains((uint)RemoteApplication.ExitCode.Value))
{
retryMessage = $"{formattedExitCode} is a known error.";
retryTest = true;
}

if (retryTest)
if (retryTest && (numberOfTries < MaxTries))
{
var message = $"Retrying test. Exception caught when exercising test app = {exceptionInExerciseApplication}, application had non-zero exit code = {applicationHadNonZeroExitCode}.";
TestLogger?.WriteLine(message);
TestLogger?.WriteLine(retryMessage + " Retrying test.");
Thread.Sleep(1000);
numberOfTries++;
}
Expand Down