test: Container test refactor (#2123)

Co-authored-by: Alex Hemsath <[email protected]>

Author: tippmar-nr

.github/workflows/all_solutions.yml

@@ -135,10 +135,10 @@ jobs:
           path: ${{ github.workspace }}\src\AwsLambda\AwsLambdaOpenTracer\bin\Release\netstandard2.0-ILRepacked
           if-no-files-found: error
 
-  run-linux-smoke-tests:
+  run-linux-container-tests:
     needs: build-fullagent-msi
-    name: Run Linux Smoke Tests
-    uses: ./.github/workflows/run_linux_smoke_tests.yml
+    name: Run Linux Container Tests
+    uses: ./.github/workflows/run_linux_container_tests.yml
     secrets: inherit
 
   build-integration-tests:

.github/workflows/run_linux_smoke_tests.yml renamed to .github/workflows/run_linux_container_tests.yml

@@ -1,4 +1,4 @@
-name: .NET Agent Linux Smoke Tests
+name: .NET Agent Linux Container Integration Tests
 
 on:
   workflow_call:
@@ -20,16 +20,16 @@ env:
 
 # only allow one instance of this workflow to be running per PR or branch, cancels any that are already running
 concurrency:
-  group: linux-smoke-tests-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  group: linux-container-tests-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
   cancel-in-progress: true
 
 permissions:
   contents: read
 
 jobs:
 
-  run-linux-smoke-tests:
-    name: Run Linux Smoke Tests
+  run-linux-container-tests:
+    name: Run Linux Container Integration Tests
     runs-on: ubuntu-latest
 
     env:
@@ -71,13 +71,12 @@ jobs:
           name: homefolders
           path: ${{ github.workspace }}/src/Agent
           repo: ${{ github.repository }}
-  
 
       - name: Set up secrets
         env:
           INTEGRATION_TEST_SECRETS: ${{ secrets.TEST_SECRETS }}
         run: |
           echo $INTEGRATION_TEST_SECRETS | dotnet user-secrets set --project ${{ env.integration_tests_shared_project }}
-
-      - name: Build & Run Linux Smoke Tests
+  
+      - name: Build & Run Linux Container Integration Tests
         run: dotnet test ./tests/Agent/IntegrationTests/ContainerIntegrationTests/ContainerIntegrationTests.csproj --framework net7.0

tests/Agent/IntegrationTests/ContainerApplications/KafkaTestApp/Program.cs

@@ -54,7 +54,7 @@ public static void SetupKafka(ILogger logger)
         {
             Thread.Sleep(15 * 1000); // Waiting for Kafka to get ready
 
-            var broker = Environment.GetEnvironmentVariable("NEW_RELIC_KAFKA_CONTAINER_NAME");
+            var broker = Environment.GetEnvironmentVariable("NEW_RELIC_KAFKA_BROKER_NAME");
             var kafkaConfig = new ConfigurationBuilder().AddInMemoryCollection().Build();
             kafkaConfig["bootstrap.servers"] = $"{broker}:9092";
             kafkaConfig["group.id"] = "kafka-dotnet-getting-started";

tests/Agent/IntegrationTests/ContainerApplications/SmokeTestApp/Dockerfile.fedora

@@ -0,0 +1,42 @@
+ARG DOTNET_VERSION
+ARG TARGET_ARCH
+# Uses a custom-built Fedora image with ASP.NET Core pre-installed, served from a private
+# container repository under the .NET Team Sandbox Azure subscription
+FROM --platform=${TARGET_ARCH} dotnetreg.azurecr.io/fedora-aspnet:${DOTNET_VERSION} AS base
+WORKDIR /app
+EXPOSE 80
+
+# build image is always amd64 (to match the runner architecture), even though the target architecture may be arm64
+FROM --platform=amd64 mcr.microsoft.com/dotnet/sdk:${DOTNET_VERSION} AS build 
+ARG TARGET_ARCH
+WORKDIR /src
+COPY ["SmokeTestApp/SmokeTestApp.csproj", "SmokeTestApp/"]
+RUN dotnet restore "SmokeTestApp/SmokeTestApp.csproj"  -a ${TARGET_ARCH}
+
+COPY . .
+WORKDIR "/src/SmokeTestApp"
+RUN dotnet build "SmokeTestApp.csproj" -c Release -o /app/build --os linux -a ${TARGET_ARCH}
+
+FROM build AS publish
+RUN dotnet publish "SmokeTestApp.csproj" -c Release -o /app/publish /p:UseAppHost=false --os linux -a ${TARGET_ARCH}
+
+FROM base AS final
+
+# Enable the agent
+ARG NEW_RELIC_HOST
+ARG NEW_RELIC_LICENSE_KEY
+ARG NEW_RELIC_APP_NAME
+
+ENV CORECLR_ENABLE_PROFILING=1 \
+CORECLR_PROFILER={36032161-FFC0-4B61-B559-F6C5D41BAE5A} \
+CORECLR_NEWRELIC_HOME=/usr/local/newrelic-dotnet-agent \
+CORECLR_PROFILER_PATH=/usr/local/newrelic-dotnet-agent/libNewRelicProfiler.so \
+NEW_RELIC_HOST=${NEW_RELIC_HOST} \
+NEW_RELIC_LICENSE_KEY=${NEW_RELIC_LICENSE_KEY} \
+NEW_RELIC_APP_NAME=${NEW_RELIC_APP_NAME} \
+NEWRELIC_LOG_DIRECTORY=/app/logs
+
+WORKDIR /app
+COPY --from=publish /app/publish .
+
+ENTRYPOINT ["dotnet", "SmokeTestApp.dll"]

tests/Agent/IntegrationTests/ContainerApplications/docker-compose-kafka.yml

@@ -11,6 +11,8 @@ version: "3"
 # PLATFORM        The platform that the service runs on -- linux/amd64 or linux/arm64/v8
 # DOTNET_VERSION  The dotnet version number to use (7.0, 8.0, etc)
 # TEST_DOCKERFILE The path and dockerfile to use for the service.
+#
+# NEW_RELIC_KAFKA_TOPIC
 # 
 # and the usual suspects:
 # NEW_RELIC_LICENSE_KEY
@@ -22,40 +24,46 @@ version: "3"
 # Alternatively, set COMPOSE_FILE environment variable to the path and omit the -f parameter
 
 services:
-    UbuntuX64Kafka1TestApp:
-        extends:
-            file: docker-compose-smoketestapp.yml
-            service: smoketestapp
-        depends_on:
-            - kafka-broker
-        environment:
-            - NEW_RELIC_KAFKA_TOPIC=${NEW_RELIC_KAFKA_TOPIC}
-            - NEW_RELIC_KAFKA_CONTAINER_NAME=${NEW_RELIC_KAFKA_CONTAINER_NAME}
-
-    UbuntuX64Kafka2TestApp:
-        extends:
-            file: docker-compose-smoketestapp.yml
-            service: smoketestapp
+    LinuxSmokeTestApp:
+        container_name: ${CONTAINER_NAME}
+        image: ${CONTAINER_NAME}
+        platform: ${PLATFORM}
+        build:
+            context: .
+            dockerfile: ${TEST_DOCKERFILE}
+            args:
+                DISTRO_TAG: ${DISTRO_TAG}
+                TARGET_ARCH: ${TARGET_ARCH}
+                NEW_RELIC_LICENSE_KEY: ${NEW_RELIC_LICENSE_KEY}
+                NEW_RELIC_APP_NAME: ${NEW_RELIC_APP_NAME}
+                NEW_RELIC_HOST: ${NEW_RELIC_HOST}
+                DOTNET_VERSION: ${DOTNET_VERSION}
+                APP_DOTNET_VERSION: ${APP_DOTNET_VERSION}
+        ports:
+          - "${PORT}:80"
+        volumes:
+          - ${AGENT_PATH}:/usr/local/newrelic-dotnet-agent # AGENT_PATH from .env, points to newrelichome_linux_x64
+          - ${LOG_PATH}:/app/logs # LOG_PATH from .env, should be a folder unique to this run of the smoketest app           
         depends_on:
             - kafka-broker
         environment:
             - NEW_RELIC_KAFKA_TOPIC=${NEW_RELIC_KAFKA_TOPIC}
-            - NEW_RELIC_KAFKA_CONTAINER_NAME=${NEW_RELIC_KAFKA_CONTAINER_NAME}
+            - NEW_RELIC_KAFKA_BROKER_NAME=kafka-broker
 
     kafka-broker:
         image: confluentinc/cp-kafka:7.5.0
         environment:
           KAFKA_BROKER_ID: 1
           KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
-          KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://${NEW_RELIC_KAFKA_CONTAINER_NAME}:29092,PLAINTEXT_HOST://${NEW_RELIC_KAFKA_CONTAINER_NAME}:9092
+          KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka-broker:29092,PLAINTEXT_HOST://kafka-broker:9092
           KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
           KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
           KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
           KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
           KAFKA_PROCESS_ROLES: broker,controller
           KAFKA_NODE_ID: 1
-          KAFKA_CONTROLLER_QUORUM_VOTERS: 1@${NEW_RELIC_KAFKA_CONTAINER_NAME}:29093
-          KAFKA_LISTENERS: PLAINTEXT://${NEW_RELIC_KAFKA_CONTAINER_NAME}:29092,CONTROLLER://${NEW_RELIC_KAFKA_CONTAINER_NAME}:29093,PLAINTEXT_HOST://:9092
+          KAFKA_CONTROLLER_QUORUM_VOTERS: 1@kafka-broker:29093
+          KAFKA_LISTENERS: PLAINTEXT://kafka-broker:29092,CONTROLLER://kafka-broker:29093,PLAINTEXT_HOST://:9092
           KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
           KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER
           KAFKA_LOG_DIRS: /tmp/kraft-combined-logs

tests/Agent/IntegrationTests/ContainerApplications/docker-compose.yml

@@ -10,7 +10,6 @@ version: "3"
 # CONTAINER_NAME  The name for the container
 # PLATFORM        The platform that the service runs on -- linux/amd64 or linux/arm64/v8
 # DOTNET_VERSION  The dotnet version number to use (7.0, 8.0, etc)
-# NETWORK_NAME    The network name to use for containers in this app. Should be unique among all running instances.
 # TEST_DOCKERFILE The path and dockerfile to use for the service.
 # 
 # and the usual suspects:
@@ -23,55 +22,26 @@ version: "3"
 # Alternatively, set COMPOSE_FILE environment variable to the path and omit the -f parameter
 
 services:
-    DebianX64SmokeTestApp:
-        extends:
-            file: docker-compose-smoketestapp.yml
-            service: smoketestapp
-    UbuntuX64SmokeTestApp:
-        extends:
-            file: docker-compose-smoketestapp.yml
-            service: smoketestapp
-    AlpineX64SmokeTestApp:
-        extends:
-            file: docker-compose-smoketestapp.yml
-            service: smoketestapp
-    DebianArm64SmokeTestApp:
-        extends:
-            file: docker-compose-smoketestapp.yml
-            service: smoketestapp
-    UbuntuArm64SmokeTestApp:
-        extends:
-            file: docker-compose-smoketestapp.yml
-            service: smoketestapp
-    CentosX64SmokeTestApp:
-        extends:
-            file: docker-compose-smoketestapp.yml
-            service: smoketestapp
+    LinuxSmokeTestApp:
+        container_name: ${CONTAINER_NAME}
+        image: ${CONTAINER_NAME}
+        platform: ${PLATFORM}
         build:
-            dockerfile: SmokeTestApp/Dockerfile.centos   
-    CentosArm64SmokeTestApp:
-        extends:
-            file: docker-compose-smoketestapp.yml
-            service: smoketestapp
-        build:
-            dockerfile: SmokeTestApp/Dockerfile.centos   
-    AmazonX64SmokeTestApp:
-        extends:
-            file: docker-compose-smoketestapp.yml
-            service: smoketestapp
-        build:
-            dockerfile: SmokeTestApp/Dockerfile.amazon
-    AmazonArm64SmokeTestApp:
-        extends:
-            file: docker-compose-smoketestapp.yml
-            service: smoketestapp
-        build:
-            dockerfile: SmokeTestApp/Dockerfile.amazon
-    LinuxUnicodeLogfileTestApp:
-        extends:
-            file: docker-compose-smoketestapp.yml
-            service: smoketestapp
-           
+            context: .
+            dockerfile: ${TEST_DOCKERFILE}
+            args:
+                DISTRO_TAG: ${DISTRO_TAG}
+                TARGET_ARCH: ${TARGET_ARCH}
+                NEW_RELIC_LICENSE_KEY: ${NEW_RELIC_LICENSE_KEY}
+                NEW_RELIC_APP_NAME: ${NEW_RELIC_APP_NAME}
+                NEW_RELIC_HOST: ${NEW_RELIC_HOST}
+                DOTNET_VERSION: ${DOTNET_VERSION}
+                APP_DOTNET_VERSION: ${APP_DOTNET_VERSION}
+        ports:
+          - "${PORT}:80"
+        volumes:
+          - ${AGENT_PATH}:/usr/local/newrelic-dotnet-agent # AGENT_PATH from .env, points to newrelichome_linux_x64
+          - ${LOG_PATH}:/app/logs # LOG_PATH from .env, should be a folder unique to this run of the smoketest app           
 networks:
     default:
         driver: bridge

tests/Agent/IntegrationTests/ContainerIntegrationTests.sln

@@ -18,7 +18,6 @@ EndProject
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "_docker", "_docker", "{FB10922F-3CC6-4497-AF53-DF6808380258}"
 	ProjectSection(SolutionItems) = preProject
 		ContainerApplications\docker-compose-kafka.yml = ContainerApplications\docker-compose-kafka.yml
-		ContainerApplications\docker-compose-smoketestapp.yml = ContainerApplications\docker-compose-smoketestapp.yml
 		ContainerApplications\docker-compose.yml = ContainerApplications\docker-compose.yml
 	EndProjectSection
 EndProject

tests/Agent/IntegrationTests/ContainerIntegrationTests/ContainerFixtures/ContainerApplication.cs renamed to tests/Agent/IntegrationTests/ContainerIntegrationTests/Applications/ContainerApplication.cs

@@ -11,7 +11,7 @@
 using NewRelic.Agent.IntegrationTests.Shared;
 using Xunit;
 
-namespace NewRelic.Agent.ContainerIntegrationTests.ContainerFixtures;
+namespace NewRelic.Agent.ContainerIntegrationTests.Applications;
 
 public class ContainerApplication : RemoteApplication
 {
@@ -22,12 +22,14 @@ public class ContainerApplication : RemoteApplication
     private readonly string _targetArch;
     private readonly string _agentArch;
     private readonly string _containerPlatform;
-    private readonly string _dockerComposeServiceName;
+
+    private static Random random = new Random();
+    private readonly long _randomId;
 
     // Used for handling dependent containers started automatically for services
     public readonly List<string> DockerDependencies;
 
-    protected override string ApplicationDirectoryName { get; }
+    protected override string ApplicationDirectoryName { get; } = "ContainerApplication";
 
     protected override string SourceApplicationDirectoryPath
     {
@@ -37,16 +39,16 @@ protected override string SourceApplicationDirectoryPath
         }
     }
 
-    public ContainerApplication(string applicationDirectoryName, string distroTag, Architecture containerArchitecture,
+    public ContainerApplication(string distroTag, Architecture containerArchitecture,
         string dotnetVersion, string dockerfile, string dockerComposeFile = "docker-compose.yml") : base(applicationType: ApplicationType.Container, isCoreApp: true)
     {
-        ApplicationDirectoryName = applicationDirectoryName;
-        _dockerComposeServiceName = applicationDirectoryName;
         _distroTag = distroTag;
         _dotnetVersion = dotnetVersion;
         _dockerfile = dockerfile;
         _dockerComposeFile = dockerComposeFile;
 
+        _randomId = random.NextInt64(); // a random id to help ensure container name uniqueness
+
         DockerDependencies = new List<string>();
 
         switch (containerArchitecture)
@@ -64,9 +66,9 @@ public ContainerApplication(string applicationDirectoryName, string distroTag, A
         }
     }
 
-    public override string AppName => $"{_dockerComposeServiceName}_{_dotnetVersion}-{_distroTag}_{_targetArch}";
+    public override string AppName => $"ContainerTestApp_{_dotnetVersion}-{_distroTag}_{_targetArch}_{_randomId}";
 
-    private string ContainerName => $"{_dockerComposeServiceName}_{_dotnetVersion}-{_distroTag}_{_targetArch}".ToLower(); // must be lowercase
+    private string ContainerName => AppName.ToLower().Replace(".", "_"); // must be lowercase, can't have any periods in it
 
     public override void CopyToRemote()
     {
@@ -79,7 +81,7 @@ public override void Start(string commandLineArguments, Dictionary<string, strin
     {
         CleanupContainer();
 
-        var arguments = $"compose -f {_dockerComposeFile} -p {_dockerComposeServiceName.ToLower()} up --abort-on-container-exit --remove-orphans --force-recreate {_dockerComposeServiceName}";
+        var arguments = $"compose -f {_dockerComposeFile} -p {ContainerName} up --abort-on-container-exit --remove-orphans --force-recreate LinuxSmokeTestApp";
 
         var newRelicHomeDirectoryPath = DestinationNewRelicHomeDirectoryPath;
         var profilerLogDirectoryPath = DefaultLogFileDirectoryPath;
@@ -107,6 +109,7 @@ public override void Start(string commandLineArguments, Dictionary<string, strin
         startInfo.EnvironmentVariables.Remove("NEWRELIC_LOG_DIRECTORY");
         startInfo.EnvironmentVariables.Remove("NEWRELIC_LOG_LEVEL");
         startInfo.EnvironmentVariables.Remove("NEWRELIC_LICENSEKEY");
+        startInfo.EnvironmentVariables.Remove("NEW_RELIC_APP_NAME");
         startInfo.EnvironmentVariables.Remove("NEW_RELIC_LICENSE_KEY");
         startInfo.EnvironmentVariables.Remove("NEW_RELIC_HOST");
         startInfo.EnvironmentVariables.Remove("NEWRELIC_INSTALL_PATH");
@@ -132,9 +135,6 @@ public override void Start(string commandLineArguments, Dictionary<string, strin
         startInfo.EnvironmentVariables.Add("AGENT_PATH", newRelicHomeDirectoryPath);
         startInfo.EnvironmentVariables.Add("LOG_PATH", profilerLogDirectoryPath);
         startInfo.EnvironmentVariables.Add("CONTAINER_NAME", ContainerName);
-        // generate a random network name to keep parallel test execution from failing
-        // network name length needs to be less than 15 characters to be compatible with linux containers
-        startInfo.EnvironmentVariables.Add("NETWORK_NAME", $"net-{System.Security.Cryptography.RandomNumberGenerator.GetInt32(1000000, 10000000)}");
 
         if (AdditionalEnvironmentVariables != null)
         {
@@ -183,7 +183,7 @@ public override void Shutdown()
 
         // stop and remove the container, no need to kill RemoteProcess, as it will die when this command runs
         // wait up to 5 seconds for the app to terminate gracefully before forcefully closing it
-        Process.Start("docker", $"compose -p {_dockerComposeServiceName.ToLower()} down --rmi local --remove-orphans");
+        Process.Start("docker", $"compose -p {ContainerName.ToLower()} down --rmi local --remove-orphans");
 
         Thread.Sleep(TimeSpan.FromSeconds(5)); // give things a chance to settle before destroying the container
     }
@@ -193,7 +193,7 @@ private void CleanupContainer()
         Console.WriteLine($"[{AppName} {DateTime.Now}] Cleaning up container and images related to {ContainerName} container.");
         TestLogger?.WriteLine($"[{AppName}] Cleaning up container and images related to {ContainerName} container.");
 
-        Process.Start("docker", $"compose -p {_dockerComposeServiceName.ToLower()} down --rmi local --remove-orphans");
+        Process.Start("docker", $"compose -p {ContainerName.ToLower()} down --rmi local --remove-orphans");
 
 
 #if DEBUG