Clean Azure agent in AZPs before running tests (#11330)

Author: im-konge

.azure/templates/steps/clean_up_workspace.yaml

@@ -0,0 +1,41 @@
+# Uninstall things that are not needed for running the test, so we clean up memory
+steps:
+  - bash: |
+      sudo apt-get purge --auto-remove dotnet-hostfxr-* || true
+      sudo apt-get purge --auto-remove dotnet-runtime-* || true
+      sudo apt-get purge --auto-remove dotnet-runtime-deps-* || true
+      sudo apt-get purge --auto-remove dotnet-sdk-* || true
+      sudo apt-get purge --auto-remove dotnet-targeting-pack-* || true
+      sudo apt-get purge --auto-remove gcc-* || true
+      sudo apt-get purge --auto-remove cpp-* || true
+      sudo apt-get purge --auto-remove google-chrome-stable || true
+      sudo apt-get purge --auto-remove microsoft-edge-stable || true
+      sudo apt-get purge --auto-remove mysql-server mysql-client mysql-common mysql-server-core-* mysql-client-core-* || true
+      sudo apt-get purge --auto-remove php* || true
+      sudo apt-get purge --auto-remove ruby* || true
+
+      sudo apt-get purge --auto-remove golang* || true
+      sudo apt-get purge --auto-remove perl* || true
+      sudo apt-get purge --auto-remove podman* || true
+      sudo apt-get purge --auto-remove postgresql* || true
+      sudo apt-get purge --auto-remove firefox* || true
+
+      sudo apt-get --purge remove r-base r-base-dev r-recommended r-cran-* || true
+
+      sudo rm -rf /usr/share/dotnet || true
+      sudo rm -rf /usr/local/lib/R || true
+      sudo rm -rf /usr/local/lib/android || true
+      sudo rm -rf /usr/local/lib/nodejs || true
+      sudo rm -rf /opt/ghc || true
+      sudo rm -rf /usr/local/go || true
+      sudo rm -rf /usr/local/aws-cli || true
+      sudo rm -rf /usr/local/aws-sam-cli || true
+      sudo rm -rf /usr/local/games || true
+      sudo rm -rf /usr/share/doc/ || true
+      sudo rm -rf /usr/share/man/ || true
+      sudo rm -rf /usr/share/locale/ || true
+      sudo rm -rf /var/cache/apt/archives || true
+    displayName: "Uninstall unnecessary dependencies and tools"
+  - bash: |
+      docker system prune -af
+    displayName: "Clear the Docker"

.azure/templates/steps/system_test_general.yaml

@@ -31,6 +31,9 @@ jobs:
     - template: 'default_variables.yaml'
   # Set timeout for jobs
   timeoutInMinutes: ${{ parameters.timeout }}
+  # Clean up whole workspace
+  workspace:
+    clean: all
   # Pipeline steps
   steps:
     - script: echo "##vso[task.setvariable variable=docker_tag]${{ parameters.releaseVersion }}"
@@ -47,6 +50,9 @@ jobs:
     # Log variables
     - template: "log_variables.yaml"
 
+    # Clean up unnecessary tools
+    - template: "clean_up_workspace.yaml"
+
     # Get cached Maven repository
     - template: "./maven_cache.yaml"
     - task: Cache@2