Enable PR validation for template changes (#1102)

This PR enables template tests to run as part of PR validation for modified templates.

Only the specific templates being modified are tested.

Author: weikanglim

eng/pipelines/repoman.yml

@@ -39,6 +39,9 @@ stages:
           name: azsdk-pool-mms-ubuntu-2004-general
           vmImage: MMSUbuntu20.04
 
+        variables:
+          RepoManResultsFile: $(Pipeline.Workspace)/repoman.md
+
         steps:
           - checkout: self
 
@@ -55,10 +58,23 @@ stages:
             parameters:
               TargetBranchName: pr/$(PRNumber)
               RemoteName: $(StagingRemoteName)
+              ResultsFileLocation: $(RepoManResultsFile)
+
+          - task: PowerShell@2
+            name: DetermineChanged
+            displayName: Set repoman changed templates
+            inputs:
+              pwsh: true
+              targetType: filePath
+              filePath: ./eng/scripts/Set-RepomanChangedRepositories.ps1
+              arguments: >-
+                -ResultsFile $(RepoManResultsFile)
+                -OutputTemplatesVariable "TemplateList"
+                -OutputTemplateBranchVariable "TemplateBranch"
 
           - pwsh: |
               $repomanContent = "No changes detected."
-              $repomanOutputFile = "$([System.IO.Path]::GetTempPath())/repoman.md"
+              $repomanOutputFile = "$(RepoManResultsFile)"
               if (Test-Path $repomanOutputFile) {
                 $repomanContent = Get-Content $repomanOutputFile -Raw
               }
@@ -89,6 +105,17 @@ stages:
             env:
               GH_TOKEN: $(azuresdk-github-pat)
 
+      - template: /eng/pipelines/templates/jobs/run-template-tests.yml
+        parameters:
+          TemplateList: $[ dependencies.Generate_Repos_For_PR.outputs['DetermineChanged.TemplateList'] ]
+          TemplateBranchName: $[ dependencies.Generate_Repos_For_PR.outputs['DetermineChanged.TemplateBranch'] ]
+          JobCondition: >-
+            and(
+              succeeded(),
+              ne(dependencies.Generate_Repos_For_PR.outputs['DetermineChanged.TemplateList'] , '')
+            )
+          JobDependsOn: Generate_Repos_For_PR
+
       - job: Publish_Staging
         condition: >-
           and(

eng/pipelines/templates/steps/azd-login.yml

@@ -25,5 +25,13 @@ steps:
 
       if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
 
+      # Export subscription ID
+      Write-Host "##vso[task.setvariable variable=SubscriptionId]$($subscriptionConfiguration.SubscriptionId)"
+
+      # Export service principal auth information for terraform testing
+      Write-Host "##vso[task.setvariable variable=arm-client-id;issecret=false]$($subscriptionConfiguration.TestApplicationId)"
+      Write-Host "##vso[task.setvariable variable=arm-client-secret;issecret=true]$($subscriptionConfiguration.TestApplicationSecret)"
+      Write-Host "##vso[task.setvariable variable=arm-tenant-id;issecret=false]$($subscriptionConfiguration.TenantId)"
+
     condition: and(succeeded(), ne(variables['Skip.LiveTest'], 'true'))
     displayName: Azure Dev Login

eng/pipelines/templates/steps/repoman-generate.yml

@@ -2,6 +2,7 @@ parameters:
   TargetBranchName: $(TargetBranchName)
   RemoteName: azure-samples-main
   GitHubPat: $(azuresdk-github-pat)
+  ResultsFileLocation: ""
 
 steps:
   - task: NodeTool@0
@@ -31,3 +32,4 @@ steps:
       arguments: >-
         -TargetBranchName "${{ parameters.TargetBranchName }}"
         -RemoteName "${{ parameters.RemoteName }}"
+        -ResultsFileLocation "${{ parameters.ResultsFileLocation }}"

eng/pipelines/templates/steps/template-test-run-job.yml

@@ -4,12 +4,12 @@
 # and additional variables specified by template-test-generate-jobs.yml
 
 steps:
-- template: /eng/pipelines/templates/steps/az-login.yml
-
 - template: /eng/pipelines/templates/steps/install-azd-live-sh.yml
   parameters:
     Version: $(AzdVersion)
 
+- template: /eng/pipelines/templates/steps/azd-login.yml
+
 # Java pre-requisites that isn't met
 # TODO: Use azd container as a form of validation that the container works
 - template: /eng/pipelines/templates/steps/install-ms-openjdk.yml
@@ -30,6 +30,12 @@ steps:
     Write-Host "##vso[task.setvariable variable=ResourceGroupName]$resourceGroupName"
     Write-Host "##vso[task.setvariable variable=EnvPrefixName]$envPrefixName"
 
+    $SynchronousCleanup = 'true'
+    if ('$(Build.Reason)' -eq 'PullRequest') {
+      $SynchronousCleanup = 'false'
+    }
+    Write-Host "##vso[task.setvariable variable=SynchronousCleanup]$SynchronousCleanup"
+
     $CleanupImmediate = 'true'
     if ($(CleanupHoursDelay) -ne '0') {
       $CleanupImmediate = 'false'
@@ -48,16 +54,19 @@ steps:
     targetType: filePath
     filePath: templates/tests/test-templates.sh
     arguments: >
+      -f '$(Pipeline.Workspace)'
       -e '$(EnvPrefixName)'
       -t '$(TemplateName)'
       -b '$(TemplateBranchName)'
       -s '$(SubscriptionId)'
       -u '$(Build.BuildId)'
       -l '$(AzureLocation)'
       -p 'list'
-      -c '$(CleanupImmediate)'
+      -c '$(SynchronousCleanup)'
     workingDirectory: templates/tests
 
+- template: /eng/pipelines/templates/steps/az-login.yml
+
 # First tag the resource group (if exists) so that it can get cleaned up
 # by the cleanup pipeline. Then attempt to delete the resource group 
 # directly. If the delete fails the cleanup pipeline will delete it.

eng/scripts/Invoke-RepomanGenerate.ps1

@@ -1,20 +1,24 @@
 param(
     [string] $TargetBranchName,
     [string] $RemoteName,
-    [string] $ResultsFileLocation = "$([System.IO.Path]::GetTempPath())/repoman.md",
+    [string] $ResultsFileLocation,
     [string] $RunnerTemp = [System.IO.Path]::GetTempPath(),
     [switch] $WhatIf
 )
 
 $projectsJson = repoman list --format json | Out-String
 $projects = ConvertFrom-Json $projectsJson
 
-foreach ($project in $projects) {
-    $additionalParameters = '--update'
-    if ($WhatIf) {
-        $additionalParameters = ''
-    }
+$additionalParameters = @()
+if (-not $WhatIf) {
+    $additionalParameters += '--update'
+}
 
+if ($ResultsFileLocation) {
+    $additionalParameters += "--resultsFile", $ResultsFileLocation
+}
+
+foreach ($project in $projects) {
     $projectPath = $project.projectPath
     $templatePath = $project.templatePath.Replace($projectPath, "")
 
@@ -27,19 +31,17 @@ repoman generate `
     --remote "$RemoteName" `
     --https `
     --fail-on-update-error `
-    --resultsFile $ResultsFileLocation `
-    $additionalParameters
+    $($additionalParameters -join ([System.Environment]::NewLine + "    "))
 "@
 
-    repoman generate `
+    & repoman generate `
         -s $projectPath `
         -o $RunnerTemp `
         -t $templatePath `
         --branch `"$TargetBranchName`" `
         --remote "$RemoteName" `
         --https `
         --fail-on-update-error `
-        --resultsFile $ResultsFileLocation `
         $additionalParameters
 
     if ($LASTEXITCODE) {

eng/scripts/Set-RepomanChangedRepositories.ps1

@@ -0,0 +1,48 @@
+<#
+.SYNOPSIS
+Parses a repoman generate results file, sets a variable that contains changed template repositories
+
+.PARAMETER ResultsFile
+Path to a repoman generate results file.
+
+.PARAMETER OutputTemplatesVariable
+The output variable to set that contains the list of comma-separated templates
+
+.PARAMETER OutputTemplateBranchVariable
+The output variable to set that contains the template branch
+
+#>
+param(
+    [string]$ResultsFile,
+    [string]$OutputTemplatesVariable,
+    [string]$OutputTemplateBranchVariable
+)
+
+if (-not (Test-Path -PathType Leaf $ResultsFile)) {
+    Write-Host "No templates were changed."
+    exit 0
+}
+
+$lines = Get-Content $ResultsFile
+$templates = @()
+
+foreach ($line in $lines) {
+    if ($line -match "azd init -t (.+?) -b (.+?)(?:$|\s)") {
+        $templateName = $Matches[1]
+        $branchName = $Matches[2]
+        $templates += $templateName
+    }
+}
+
+if ($templates.Length -eq 0) {
+    Write-Host "No templates were changed."
+    exit 0
+}
+
+Write-Host "Following templates were changed on $($branchName):"
+
+$templates | Format-List
+
+$templatesCsv = $templates -join ","
+Write-Host "##vso[task.setvariable variable=$OutputTemplatesVariable;isOutput=true]$templatesCsv"
+Write-Host "##vso[task.setvariable variable=$OutputTemplateBranchVariable;isOutput=true]$branchName"

eng/scripts/Set-TemplateTestMatrixVariable.ps1

@@ -67,7 +67,6 @@ if ($TemplateList -eq '(azd template list)') {
 }
 
 if ($TemplateListFilter -ne '.*') {
-} else {
     Write-Host "Filtering with TemplateListFilter regex: $TemplateListFilter"
 
     $templateNames = $templateNames -match $TemplateListFilter