Skip to content

Commit 7edea26

Browse files
raych1raych1
authored
New validation for REST API spec location before SDK release (#7451)
* Added script and pipeline for spec location validation SDK release pipeline would run this validation to ensure the spec comes from the main branch of Azure/azure-rest-api-specs repo * Update parameter * Use github rest api to validate commit * Added token parameter * Support more yaml cases and other languages * Removed the default setting in yaml template * Only validate in case of GA package * Follow APIView to retrieve package version for verification * Get github token from env variable * Removed obsolete parameter
1 parent 2936978 commit 7edea26

File tree

3 files changed

+325
-0
lines changed

3 files changed

+325
-0
lines changed

‎eng/common/pipelines/templates/steps/verify-restapi-spec-location.yml

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
parameters:
2+
- name: ServiceDirectory
3+
type: string
4+
default: ''
5+
- name: PackageName
6+
type: string
7+
default: ''
8+
- name: ArtifactLocation
9+
type: string
10+
default: ''
11+
12+
steps:
13+
- task: Powershell@2
14+
inputs:
15+
filePath: $(Build.SourcesDirectory)/eng/common/scripts/Verify-RestApiSpecLocation.ps1
16+
arguments: >
17+
-ServiceDirectory "${{ parameters.ServiceDirectory }}"
18+
-PackageName "${{ parameters.PackageName }}"
19+
-ArtifactLocation: "${{ parameters.ArtifactLocation }}"
20+
pwsh: true
21+
workingDirectory: $(Pipeline.Workspace)
22+
displayName: Verify REST API spec location for "${{ parameters.PackageName }}"
23+
env:
24+
GH_TOKEN: $(azuresdk-github-pat)

‎eng/common/scripts/Invoke-GitHubAPI.ps1

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -451,3 +451,23 @@ function Get-GithubReferenceCommitDate($commitUrl, $AuthToken) {
451451
}
452452
return $commitResponse.committer.date
453453
}
454+
455+
function Search-GitHubCommit {
456+
param (
457+
[ValidateNotNullOrEmpty()]
458+
$RepoOwner,
459+
[ValidateNotNullOrEmpty()]
460+
$RepoName,
461+
[ValidateNotNullOrEmpty()]
462+
$CommitHash,
463+
[ValidateNotNullOrEmpty()]
464+
$AuthToken
465+
)
466+
$uri = "https://api.github.com/search/commits?q=repo:$RepoOwner/$RepoName+hash:$CommitHash"
467+
468+
return Invoke-RestMethod `
469+
-Method GET `
470+
-Uri $uri `
471+
-Headers (Get-GitHubApiHeaders -token $AuthToken) `
472+
-MaximumRetryCount 3
473+
}

‎eng/common/scripts/Verify-RestApiSpecLocation.ps1

Lines changed: 281 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,281 @@
1+
<#
2+
.SYNOPSIS
3+
Verifies the location of the REST API specification.
4+
5+
.DESCRIPTION
6+
This script is used to verify the REST API specifications used to generate SDK package are from the main branch of Azure/azure-rest-api-specs repository.
7+
8+
.PARAMETER ServiceDirectory
9+
The directory path of the service.
10+
11+
.PARAMETER PackageName
12+
The name of the package.
13+
14+
.PARAMETER ArtifactLocation
15+
The location of the generated artifact for the package.
16+
17+
.PARAMETER GitHubPat
18+
The GitHub personal access token used for authentication. By default, it will use the environment variable GH_TOKEN.
19+
20+
.PARAMETER PackageInfoDirectory
21+
The directory path where the package information is stored.
22+
23+
.EXAMPLE
24+
Verify-RestApiSpecLocation -ServiceDirectory "/home/azure-sdk-for-net/sdk/serviceab" -PackageName "MyPackage" -ArtifactLocation "/home/ab/artifacts" -GitHubPat "xxxxxxxxxxxx" -PackageInfoDirectory "/home/ab/artifacts/PackageInfo"
25+
26+
#>
27+
[CmdletBinding()]
28+
param (
29+
[Parameter(Position = 0)]
30+
[ValidateNotNullOrEmpty()]
31+
[string] $ServiceDirectory,
32+
[Parameter(Position = 1)]
33+
[ValidateNotNullOrEmpty()]
34+
[string] $PackageName,
35+
[ValidateNotNullOrEmpty()]
36+
[string] $ArtifactLocation,
37+
[Parameter(Position = 2)]
38+
[string]$GitHubPat = $($env:GH_TOKEN),
39+
[string]$PackageInfoDirectory
40+
)
41+
42+
. (Join-Path $PSScriptRoot common.ps1)
43+
. (Join-Path $PSScriptRoot Helpers/PSModule-Helpers.ps1)
44+
45+
# Check if github token is set
46+
if(-not $GitHubPat) {
47+
LogError "GitHubPat is not set. Please set the environment variable GH_TOKEN or pass the GitHubPat parameter."
48+
exit 1
49+
}
50+
51+
Install-ModuleIfNotInstalled "powershell-yaml" "0.4.1" | Import-Module
52+
53+
# This function is used to verify the 'require' and 'input-file' settings in autorest.md point to the main branch of Azure/azure-rest-api-specs repository
54+
# input-file may be:
55+
# https://raw.githubusercontent.com/Azure/azure-rest-api-specs/main/specification/purview/data-plane/Azure.Analytics.Purview.MetadataPolicies/preview/2021-07-01-preview/purviewMetadataPolicy.json
56+
# or
57+
# https://github.com/Azure/azure-rest-api-specs/blob/0ebd4949e8e1cd9537ca5a07384c7661162cc7a6/specification/purview/data-plane/Azure.Analytics.Purview.Account/preview/2019-11-01-preview/account.json
58+
function Verify-Url([string]$fileUrl) {
59+
if($fileUrl -match "^https://(raw\.githubusercontent\.com|github\.com)/(?<repo>[^/]*/azure-rest-api-specs)(/(blob|tree))?/(?<commit>[^\/]+(\/[^\/]+)*|[0-9a-f]{40})/(?<path>specification/.*)") {
60+
$repo = $matches['repo']
61+
$commit = $matches['commit']
62+
if($repo -ne "Azure/azure-rest-api-specs") {
63+
LogError "ServiceDir:$ServiceDirectory, PackageName:$PackageName. Invalid repo in the file url: $fileUrl. Repo should be 'Azure/azure-rest-api-specs'."
64+
exit 1
65+
}
66+
# check the commit hash belongs to main branch
67+
Verify-CommitFromMainBranch $commit
68+
}
69+
else{
70+
LogError "ServiceDir:$ServiceDirectory, PackageName:$PackageName. Invalid file url: $fileUrl"
71+
exit 1
72+
}
73+
}
74+
75+
# This function is used to verify the 'repo' and 'commit' settings in tsp-location.yaml point to the main branch of Azure/azure-rest-api-specs repository
76+
function Verify-TspLocation([System.Object]$tspLocationObj) {
77+
$repo = $tspLocationObj["repo"]
78+
$commit = $tspLocationObj["commit"]
79+
if($repo -ne "Azure/azure-rest-api-specs") {
80+
LogError "Invalid repo setting in the tsp-location.yaml: $repo. Repo should be 'Azure/azure-rest-api-specs'. ServiceDir:$ServiceDirectory, PackageName:$PackageName"
81+
exit 1
82+
}
83+
84+
# check the commit hash belongs to main branch
85+
Verify-CommitFromMainBranch $commit
86+
}
87+
88+
# This function is used to verify the specific 'commit' belongs to the main branch of Azure/azure-rest-api-specs repository
89+
function Verify-CommitFromMainBranch([string]$commit) {
90+
if($commit -notmatch "^[0-9a-f]{40}$" -and $commit -ne "main") {
91+
LogError "Invalid commit hash or branch name: $commit. Branch name should be 'main' or the commit should be a 40-character SHA-1 hash. ServiceDir:$ServiceDirectory, PackageName:$PackageName"
92+
exit 1
93+
}
94+
if($commit -eq "main") {
95+
Write-Host "ServiceDir:$ServiceDirectory, PackageName:$PackageName. Branch is $commit branch of Azure/azure-rest-api-specs repository."
96+
return
97+
}
98+
try {
99+
$searchResult = Search-GitHubCommit -AuthToken $GitHubPat -CommitHash $commit -RepoOwner "Azure" -RepoName "azure-rest-api-specs"
100+
if ($searchResult.total_count -lt 1) {
101+
LogError "Commit $commit doesn't exist in 'main' branch of Azure/azure-rest-api-specs repository. ServiceDir:$ServiceDirectory, PackageName:$PackageName"
102+
exit 1
103+
}
104+
else{
105+
Write-Host "ServiceDir:$ServiceDirectory, PackageName:$PackageName. Commit $commit exists in 'main' branch of Azure/azure-rest-api-specs repository."
106+
}
107+
}
108+
catch {
109+
LogError "ServiceDir:$ServiceDirectory, PackageName:$PackageName. Failed to search commit $commit with exception:`n$_ "
110+
exit 1
111+
}
112+
}
113+
114+
function Verify-YamlContent([string]$markdownContent) {
115+
$splitString = '``` yaml|```yaml|```'
116+
$yamlContent = $markdownContent -split $splitString
117+
foreach($yamlSection in $yamlContent) {
118+
if ($yamlSection) {
119+
try {
120+
# remove the lines like: $(tag) == 'package-preview-2023-09'
121+
$yamlSection = $yamlSection -replace '^\s*\$\(.+\)\s*==.+', ''
122+
$yamlobj = ConvertFrom-Yaml -Yaml $yamlSection
123+
if($yamlobj) {
124+
$batchValue = $yamlobj["batch"]
125+
$requireValue = $yamlobj["require"]
126+
$inputFileValue = $yamlobj["input-file"]
127+
if ($requireValue) {
128+
LogDebug "ServiceDir:$ServiceDirectory, PackageName:$PackageName. 'require' is set as:$requireValue"
129+
Verify-Url $requireValue
130+
}
131+
elseif ($inputFileValue) {
132+
LogDebug "ServiceDir:$ServiceDirectory, PackageName:$PackageName. 'input-file' is set as:$inputFileValue"
133+
foreach($inputFile in $inputFileValue) {
134+
Verify-Url $inputFile
135+
}
136+
}
137+
elseif ($batchValue) {
138+
# there are some services which use batch mode for sdk generation, e.g. Azure.AI.Language.QuestionAnswering
139+
foreach($batch in $batchValue) {
140+
$requireValue = $batch["require"]
141+
$inputFileValue = $batch["input-file"]
142+
if ($requireValue) {
143+
LogDebug "ServiceDir:$ServiceDirectory, PackageName:$PackageName. 'require' is set as:$requireValue"
144+
Verify-Url $requireValue
145+
}
146+
elseif ($inputFileValue) {
147+
LogDebug "ServiceDir:$ServiceDirectory, PackageName:$PackageName. 'input-file' is set as:$inputFileValue"
148+
foreach($inputFile in $inputFileValue) {
149+
Verify-Url $inputFile
150+
}
151+
}
152+
}
153+
}
154+
}
155+
}
156+
catch {
157+
Write-Host "ServiceDir:$ServiceDirectory, PackageName:$PackageName. Failed to parse yaml section $yamlSection with exception:`n$_ "
158+
}
159+
}
160+
}
161+
}
162+
163+
function Verify-PackageVersion() {
164+
try {
165+
$packages = @{}
166+
if ($FindArtifactForApiReviewFn -and (Test-Path "Function:$FindArtifactForApiReviewFn"))
167+
{
168+
$packages = &$FindArtifactForApiReviewFn $ArtifactLocation $PackageName
169+
}
170+
else
171+
{
172+
LogError "The function for 'FindArtifactForApiReviewFn' was not found.`
173+
Make sure it is present in eng/scripts/Language-Settings.ps1 and referenced in eng/common/scripts/common.ps1.`
174+
See https://github.com/Azure/azure-sdk-tools/blob/main/doc/common/common_engsys.md#code-structure"
175+
exit 1
176+
}
177+
if (-not $PackageInfoDirectory)
178+
{
179+
$PackageInfoDirectory = Join-Path -Path $ArtifactLocation "PackageInfo"
180+
if (-not (Test-Path -Path $PackageInfoDirectory)) {
181+
# Call Save-Package-Properties.ps1 script to generate package info json files
182+
$savePropertiesScriptPath = Join-Path -Path $PSScriptRoot "Save-Package-Properties.ps1"
183+
& $savePropertiesScriptPath -serviceDirectory $ServiceDirectory -outDirectory $PackageInfoDirectory
184+
}
185+
}
186+
187+
$continueValidation = $false
188+
if ($packages)
189+
{
190+
foreach($pkgPath in $packages.Values)
191+
{
192+
$pkgPropPath = Join-Path -Path $PackageInfoDirectory "$PackageName.json"
193+
if (-Not (Test-Path $pkgPropPath))
194+
{
195+
Write-Host "ServiceDir:$ServiceDirectory, PackageName:$PackageName. Package property file path $($pkgPropPath) is invalid."
196+
continue
197+
}
198+
# Get package info from json file
199+
$pkgInfo = Get-Content $pkgPropPath | ConvertFrom-Json
200+
$version = [AzureEngSemanticVersion]::ParseVersionString($pkgInfo.Version)
201+
if ($null -eq $version)
202+
{
203+
LogError "ServiceDir:$ServiceDirectory, Version info is not available for package $PackageName, because version '$(pkgInfo.Version)' is invalid. Please check if the version follows Azure SDK package versioning guidelines."
204+
exit 1
205+
}
206+
207+
Write-Host "Version: $($version)"
208+
Write-Host "SDK Type: $($pkgInfo.SdkType)"
209+
Write-Host "Release Status: $($pkgInfo.ReleaseStatus)"
210+
211+
# Ignore the validation if the package is not GA version
212+
if ($version.IsPrerelease) {
213+
Write-Host "ServiceDir:$ServiceDirectory, Package $($parsedPackage.PackageId) is marked with version $($parsedPackage.PackageVersion), the version is a prerelease version and the validation of spec location is ignored."
214+
exit 0
215+
}
216+
$continueValidation = $true
217+
}
218+
}
219+
if($continueValidation -eq $false) {
220+
Write-Host "ServiceDir:$ServiceDirectory, no package info is found for package $PackageName, the validation of spec location is ignored."
221+
exit 0
222+
}
223+
}
224+
catch {
225+
LogError "ServiceDir:$ServiceDirectory, PackageName:$PackageName. Failed to retrieve package and validate package version with exception:`n$_ "
226+
exit 1
227+
}
228+
}
229+
230+
try{
231+
# Verify package version is not a prerelease version, only continue the validation if the package is GA version
232+
Verify-PackageVersion
233+
234+
$ServiceDir = Join-Path $RepoRoot 'sdk' $ServiceDirectory
235+
$PackageDirectory = Join-Path $ServiceDir $PackageName
236+
Push-Location $PackageDirectory
237+
238+
# Load tsp-location.yaml if existed
239+
$tspLocationYamlPath = Join-Path $PackageDirectory "tsp-location.yaml"
240+
$autorestMdPath = Join-Path $PackageDirectory "src/autorest.md"
241+
$swaggerReadmePath = Join-Path $PackageDirectory "swagger/README.md"
242+
$tspLocationYaml = @{}
243+
if (Test-Path -Path $tspLocationYamlPath) {
244+
# typespec scenario
245+
$tspLocationYaml = Get-Content -Path $tspLocationYamlPath -Raw | ConvertFrom-Yaml
246+
Verify-TspLocation $tspLocationYaml
247+
}
248+
elseif ($Language -eq "dotnet") {
249+
# only dotnet language sdk uses 'autorest.md' to configure the sdk generation
250+
if (Test-Path -Path $autorestMdPath) {
251+
try {
252+
$autorestMdContent = Get-Content -Path $autorestMdPath -Raw
253+
Verify-YamlContent $autorestMdContent
254+
}
255+
catch {
256+
Write-Host "ServiceDir:$ServiceDirectory, PackageName:$PackageName. Failed to parse autorest.md file with exception:`n$_ "
257+
}
258+
}
259+
}
260+
elseif ($Language -eq "java" -or $Language -eq "js" -or $Language -eq "python" -or $Language -eq "go") {
261+
# for these languages we ignore the validation because they always use the latest spec from main branch to release SDK
262+
# mgmt plane packages: azure-core-management|azure-resourcemanager|azure-resourcemanager-advisor (java), azure-mgmt-devcenter (python), arm-advisor (js), armaad (go)
263+
if($PackageName -match "^(arm|azure-mgmt|azure-resourcemanager|azure-core-management)[-a-z]*$") {
264+
Write-Host "ServiceDir:$ServiceDirectory, PackageName:$PackageName. Ignore the validation for $Language management plane package."
265+
exit 0
266+
}
267+
# for these languages they use 'swagger/readme.md' to configure the sdk generation for data plane scenarios
268+
if (Test-Path -Path $swaggerReadmePath) {
269+
try {
270+
$swaggerReadmeContent = Get-Content -Path $swaggerReadmePath -Raw
271+
Verify-YamlContent $swaggerReadmeContent
272+
}
273+
catch {
274+
Write-Host "ServiceDir:$ServiceDirectory, PackageName:$PackageName. Failed to parse swagger/readme.md file with exception:`n$_ "
275+
}
276+
}
277+
}
278+
}
279+
finally {
280+
Pop-Location
281+
}

0 commit comments

Comments
 (0)