More output from E2E

Author: stevejgordon

.github/workflows/e2e.yml

@@ -78,5 +78,5 @@ jobs:
         if: always()
         with:
           name: playwright-traces
-          path: .artifacts/playwright-traces/*-screenshot.jpeg
+          path: .artifacts/playwright-traces/*
           retention-days: 1

tests/Elastic.OpenTelemetry.EndToEndTests/DistributedFixture/ApmUIBrowserContext.cs

@@ -14,11 +14,16 @@ public class ApmUIBrowserContext : IAsyncLifetime
 {
 	private readonly IConfigurationRoot _configuration;
 	private readonly string _serviceName;
+	private readonly string _playwrightScreenshotsDir;
+	private readonly List<string> _output;
 
-	public ApmUIBrowserContext(IConfigurationRoot configuration, string serviceName)
+	public ApmUIBrowserContext(IConfigurationRoot configuration, string serviceName, string playwrightScreenshotsDir, List<string> output)
 	{
 		_configuration = configuration;
 		_serviceName = serviceName;
+		_playwrightScreenshotsDir = playwrightScreenshotsDir;
+		_output = output;
+
 		//"https://{instance}.apm.us-east-1.aws.elastic.cloud:443"
 		// https://{instance}.kb.us-east-1.aws.elastic.cloud/app/apm/services?comparisonEnabled=true&environment=ENVIRONMENT_ALL&rangeFrom=now-15m&rangeTo=now&offset=1d
 		var endpoint = configuration["E2E:Endpoint"]?.Trim() ?? string.Empty;
@@ -84,25 +89,35 @@ public async Task<IPage> OpenApmLandingPage(string testName)
 		return page;
 	}
 
+	private void Log(string message)
+	{
+		Console.WriteLine(message);
+		_output.Add(message);
+	}
+
 	public async Task WaitForServiceOnOverview(IPage page)
 	{
 		var timeout = (float)TimeSpan.FromSeconds(30).TotalMilliseconds;
 
 		var servicesHeader = page.GetByRole(AriaRole.Heading, new() { Name = "Services" });
 		await servicesHeader.WaitForAsync(new() { State = WaitForSelectorState.Visible, Timeout = timeout });
 
-		Console.WriteLine($"Search for service name: {_serviceName}");
+		await page.ScreenshotAsync(new() { Path = Path.Join(_playwrightScreenshotsDir, "services-loaded.jpeg"), FullPage = true });
+
+		Log($"Search for service name: {_serviceName}");
 
 		//service.name : dotnet-e2e-*
 		var queryBar = page.GetByRole(AriaRole.Searchbox, new() { Name = "Search services by name" });
 		await queryBar.WaitForAsync(new() { State = WaitForSelectorState.Visible, Timeout = timeout });
 		await queryBar.FillAsync(_serviceName);
 		await queryBar.PressAsync("Enter");
 
+		await page.ScreenshotAsync(new() { Path = Path.Join(_playwrightScreenshotsDir, "filter-services.jpeg"), FullPage = true });
+
 		Exception? observed = null;
 
 		var refreshTimeout = (float)TimeSpan.FromSeconds(5).TotalMilliseconds;
-		for (var i = 0; i < 10; i++)
+		for (var i = 0; i < 20; i++)
 		{
 			try
 			{
@@ -113,10 +128,7 @@ public async Task WaitForServiceOnOverview(IPage page)
 			}
 			catch (Exception e)
 			{
-				await page.ScreenshotAsync(new() { Path = $"squibble{i}.jpeg", FullPage = true });
-
 				observed ??= e;
-				await page.ReloadAsync();
 			}
 			finally
 			{
@@ -125,31 +137,32 @@ public async Task WaitForServiceOnOverview(IPage page)
 		}
 		if (observed != null)
 			throw observed; //TODO proper rethrow with stack
-
 	}
 
 	private int _unnamedTests;
 	public async Task StopTrace(IPage page, bool success, [CallerMemberName] string? testName = null)
 	{
 		testName ??= $"unknown_test_{_unnamedTests++}";
-		//only dump trace zip of test name is provided.
+
+		//only dump trace zip if tests failed
 		if (success)
+		{
 			await page.Context.Tracing.StopAsync(new());
+		}
 		else
 		{
 			var root = DotNetRunApplication.GetSolutionRoot();
 			var zip = Path.Combine(root.FullName, ".artifacts", "playwright-traces", $"{testName}.zip");
 			await page.Context.Tracing.StopAsync(new() { Path = zip });
 
-			using var archive = ZipFile.OpenRead(zip);
-			var entries = archive.Entries.Where(e => e.FullName.StartsWith("resources") && e.FullName.EndsWith(".jpeg")).ToList();
-			var lastScreenshot = entries.MaxBy(e => e.LastWriteTime);
-			lastScreenshot?.ExtractToFile(Path.Combine(root.FullName, ".artifacts", "playwright-traces", $"{testName}-screenshot.jpeg"));
+			//using var archive = ZipFile.OpenRead(zip);
+			//var entries = archive.Entries.Where(e => e.FullName.StartsWith("resources") && e.FullName.EndsWith(".jpeg")).ToList();
+			//var lastScreenshot = entries.MaxBy(e => e.LastWriteTime);
+			//lastScreenshot?.ExtractToFile(Path.Combine(root.FullName, ".artifacts", "playwright-traces", $"{testName}-screenshot.jpeg"));
 		}
 		await page.CloseAsync();
 	}
 
-
 	public async Task DisposeAsync()
 	{
 		await Browser.DisposeAsync();

tests/Elastic.OpenTelemetry.EndToEndTests/DistributedFixture/DistributedApplicationFixture.cs

@@ -6,7 +6,9 @@
 using System.Security.Cryptography;
 using System.Text;
 using Microsoft.Extensions.Configuration;
+using Microsoft.VisualStudio.TestPlatform.ObjectModel;
 using Nullean.Xunit.Partitions.Sdk;
+using Xunit.Abstractions;
 
 namespace Elastic.OpenTelemetry.EndToEndTests.DistributedFixture;
 
@@ -18,11 +20,14 @@ public class DistributedApplicationFixture : IPartitionLifetime
 
 	public bool Started => AspNetApplication.ProcessId.HasValue;
 
+	public string PlaywrightScreenshotsDirectory { get; } = Path.Combine(DotNetRunApplication.GetSolutionRoot().FullName, ".artifacts", "playwright-traces", "screenshots");
+
 	private readonly List<string> _output = [];
 
 	public int? MaxConcurrency => null;
 
 	private ApmUIBrowserContext? _apmUI;
+
 	public ApmUIBrowserContext ApmUI
 	{
 		get => _apmUI ??
@@ -42,11 +47,7 @@ public AspNetCoreExampleApplication AspNetApplication
 	private static string ShaForCurrentTicks()
 	{
 		var buffer = Encoding.UTF8.GetBytes(DateTime.UtcNow.Ticks.ToString(DateTimeFormatInfo.InvariantInfo));
-
-		return BitConverter.ToString(SHA1.Create().ComputeHash(buffer))
-			.Replace("-", "")
-			.ToLowerInvariant()
-			.Substring(0, 12);
+		return Convert.ToHexStringLower(SHA1.HashData(buffer)).Substring(0, 12);
 	}
 
 	public string FailureTestOutput()
@@ -66,6 +67,18 @@ public string FailureTestOutput()
 
 	}
 
+	public void WriteFailureTestOutput(ITestOutputHelper testOutputHelper)
+	{
+		foreach (var line in _output)
+			testOutputHelper.WriteLine(line);
+
+		DotNetRunApplication.IterateOverLog(s =>
+		{
+			Console.WriteLine(s);
+			testOutputHelper.WriteLine(s);
+		});
+	}
+
 	public async Task DisposeAsync()
 	{
 		_aspNetApplication?.Dispose();
@@ -91,7 +104,7 @@ public async Task InitializeAsync()
 
 		Log("Started ASP.NET application");
 
-		ApmUI = new ApmUIBrowserContext(configuration, ServiceName);
+		ApmUI = new ApmUIBrowserContext(configuration, ServiceName, PlaywrightScreenshotsDirectory, _output);
 
 		Log("Started UI Browser context");
 

tests/Elastic.OpenTelemetry.EndToEndTests/ServiceTests.cs

@@ -34,7 +34,6 @@ public async Task LatencyShowsAGraph()
 			.ToBeVisibleAsync(new() { Timeout = timeout });
 	}
 
-
 	public async Task InitializeAsync() => _page = await fixture.ApmUI.NewProfiledPage(_testName);
 
 	public async Task DisposeAsync()
@@ -45,6 +44,6 @@ public async Task DisposeAsync()
 		if (success)
 			return;
 
-		DotNetRunApplication.IterateOverLog(Output.WriteLine);
+		fixture.WriteFailureTestOutput(Output);
 	}
 }