From dbfaaf4fb74f8cef8bcf6cd40e57c37392e12526 Mon Sep 17 00:00:00 2001 From: DotNet-Bot Date: Thu, 13 Mar 2025 19:26:16 +0000 Subject: [PATCH 01/57] Update dependencies from https://dev.azure.com/dnceng/internal/_git/dotnet-runtime build 20250312.18 Microsoft.Bcl.AsyncInterfaces , Microsoft.Extensions.DependencyModel , Microsoft.Extensions.FileProviders.Abstractions , Microsoft.Extensions.FileSystemGlobbing , Microsoft.Extensions.Logging , Microsoft.Extensions.Logging.Abstractions , Microsoft.Extensions.Logging.Console , Microsoft.NET.HostModel , Microsoft.NET.ILLink.Tasks , Microsoft.NETCore.App.Host.win-x64 , Microsoft.NETCore.App.Ref , Microsoft.NETCore.App.Runtime.win-x64 , Microsoft.NETCore.Platforms , Microsoft.Win32.SystemEvents , System.CodeDom , System.Composition.AttributedModel , System.Composition.Convention , System.Composition.Hosting , System.Composition.Runtime , System.Composition.TypedParts , System.Configuration.ConfigurationManager , System.Formats.Asn1 , System.Reflection.MetadataLoadContext , System.Resources.Extensions , System.Security.Cryptography.Pkcs , System.Security.Cryptography.ProtectedData , System.Security.Cryptography.Xml , System.Security.Permissions , System.ServiceProcess.ServiceController , System.Text.Encoding.CodePages , System.Text.Json , System.Windows.Extensions , VS.Redist.Common.NetCore.SharedFramework.x64.9.0 , VS.Redist.Common.NetCore.TargetingPack.x64.9.0 , Microsoft.SourceBuild.Intermediate.runtime.linux-x64 From Version 9.0.3 -> To Version 9.0.4 Dependency coherency updates Microsoft.NET.Workload.Emscripten.Current.Manifest-9.0.100.Transport,Microsoft.NET.Workload.Emscripten.Current.Manifest-9.0.100,Microsoft.SourceBuild.Intermediate.emsdk From Version 9.0.3-servicing.25105.2 -> To Version 9.0.4-servicing.25157.2 (parent: Microsoft.NETCore.App.Runtime.win-x64 --- NuGet.config | 19 +++-- eng/Version.Details.xml | 152 ++++++++++++++++++++-------------------- eng/Versions.props | 72 +++++++++---------- 3 files changed, 125 insertions(+), 118 deletions(-) diff --git a/NuGet.config b/NuGet.config index a20104ba76b2..3b6f0ba86d93 100644 --- a/NuGet.config +++ b/NuGet.config @@ -24,10 +24,12 @@ - + - + + + @@ -35,14 +37,15 @@ - + - + + @@ -69,12 +72,16 @@ - + + + - + + + diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 4418387ff5ec..d3de7897551e 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -15,42 +15,42 @@ 693a4fd2526759d8408ec1388970143e666b03e2 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - 831d23e56149cd59c40fc00c7feb7c5334bd19c4 + 0c9b70db7b7bfcbc33e85bc40e215c3b138823e5 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - 831d23e56149cd59c40fc00c7feb7c5334bd19c4 + 0c9b70db7b7bfcbc33e85bc40e215c3b138823e5 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - 831d23e56149cd59c40fc00c7feb7c5334bd19c4 + 0c9b70db7b7bfcbc33e85bc40e215c3b138823e5 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - 831d23e56149cd59c40fc00c7feb7c5334bd19c4 + 0c9b70db7b7bfcbc33e85bc40e215c3b138823e5 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - 831d23e56149cd59c40fc00c7feb7c5334bd19c4 + 0c9b70db7b7bfcbc33e85bc40e215c3b138823e5 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - 831d23e56149cd59c40fc00c7feb7c5334bd19c4 + 0c9b70db7b7bfcbc33e85bc40e215c3b138823e5 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - 831d23e56149cd59c40fc00c7feb7c5334bd19c4 + 0c9b70db7b7bfcbc33e85bc40e215c3b138823e5 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - 831d23e56149cd59c40fc00c7feb7c5334bd19c4 + 0c9b70db7b7bfcbc33e85bc40e215c3b138823e5 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - 831d23e56149cd59c40fc00c7feb7c5334bd19c4 + 0c9b70db7b7bfcbc33e85bc40e215c3b138823e5 @@ -59,18 +59,18 @@ https://github.com/dotnet/core-setup 7d57652f33493fa022125b7f63aad0d70c52d810 - + https://github.com/dotnet/emsdk - dad5528e5bdf92a05a5a404c5f7939523390b96d + 78be8cdf4f0bfd93018fd7a87f8282a41d041298 - + https://github.com/dotnet/emsdk - dad5528e5bdf92a05a5a404c5f7939523390b96d + 78be8cdf4f0bfd93018fd7a87f8282a41d041298 - + https://github.com/dotnet/emsdk - dad5528e5bdf92a05a5a404c5f7939523390b96d + 78be8cdf4f0bfd93018fd7a87f8282a41d041298 @@ -230,29 +230,29 @@ bc9161306b23641b0364b8f93d546da4d48da1eb - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - 831d23e56149cd59c40fc00c7feb7c5334bd19c4 + 0c9b70db7b7bfcbc33e85bc40e215c3b138823e5 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - 831d23e56149cd59c40fc00c7feb7c5334bd19c4 + 0c9b70db7b7bfcbc33e85bc40e215c3b138823e5 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - 831d23e56149cd59c40fc00c7feb7c5334bd19c4 + 0c9b70db7b7bfcbc33e85bc40e215c3b138823e5 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - 831d23e56149cd59c40fc00c7feb7c5334bd19c4 + 0c9b70db7b7bfcbc33e85bc40e215c3b138823e5 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - 831d23e56149cd59c40fc00c7feb7c5334bd19c4 + 0c9b70db7b7bfcbc33e85bc40e215c3b138823e5 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - 831d23e56149cd59c40fc00c7feb7c5334bd19c4 + 0c9b70db7b7bfcbc33e85bc40e215c3b138823e5 https://dev.azure.com/dnceng/internal/_git/dotnet-windowsdesktop @@ -503,89 +503,89 @@ - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - 831d23e56149cd59c40fc00c7feb7c5334bd19c4 + 0c9b70db7b7bfcbc33e85bc40e215c3b138823e5 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - 831d23e56149cd59c40fc00c7feb7c5334bd19c4 + 0c9b70db7b7bfcbc33e85bc40e215c3b138823e5 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - 831d23e56149cd59c40fc00c7feb7c5334bd19c4 + 0c9b70db7b7bfcbc33e85bc40e215c3b138823e5 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - 831d23e56149cd59c40fc00c7feb7c5334bd19c4 + 0c9b70db7b7bfcbc33e85bc40e215c3b138823e5 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - 831d23e56149cd59c40fc00c7feb7c5334bd19c4 + 0c9b70db7b7bfcbc33e85bc40e215c3b138823e5 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - 831d23e56149cd59c40fc00c7feb7c5334bd19c4 + 0c9b70db7b7bfcbc33e85bc40e215c3b138823e5 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - 831d23e56149cd59c40fc00c7feb7c5334bd19c4 + 0c9b70db7b7bfcbc33e85bc40e215c3b138823e5 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - 831d23e56149cd59c40fc00c7feb7c5334bd19c4 + 0c9b70db7b7bfcbc33e85bc40e215c3b138823e5 https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore b96167fbfe8bd45d94e4dcda42c7d09eb5745459 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - 831d23e56149cd59c40fc00c7feb7c5334bd19c4 + 0c9b70db7b7bfcbc33e85bc40e215c3b138823e5 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - 831d23e56149cd59c40fc00c7feb7c5334bd19c4 + 0c9b70db7b7bfcbc33e85bc40e215c3b138823e5 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - 831d23e56149cd59c40fc00c7feb7c5334bd19c4 + 0c9b70db7b7bfcbc33e85bc40e215c3b138823e5 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - 831d23e56149cd59c40fc00c7feb7c5334bd19c4 + 0c9b70db7b7bfcbc33e85bc40e215c3b138823e5 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - 831d23e56149cd59c40fc00c7feb7c5334bd19c4 + 0c9b70db7b7bfcbc33e85bc40e215c3b138823e5 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - 831d23e56149cd59c40fc00c7feb7c5334bd19c4 + 0c9b70db7b7bfcbc33e85bc40e215c3b138823e5 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - 831d23e56149cd59c40fc00c7feb7c5334bd19c4 + 0c9b70db7b7bfcbc33e85bc40e215c3b138823e5 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - 831d23e56149cd59c40fc00c7feb7c5334bd19c4 + 0c9b70db7b7bfcbc33e85bc40e215c3b138823e5 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - 831d23e56149cd59c40fc00c7feb7c5334bd19c4 + 0c9b70db7b7bfcbc33e85bc40e215c3b138823e5 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - 831d23e56149cd59c40fc00c7feb7c5334bd19c4 + 0c9b70db7b7bfcbc33e85bc40e215c3b138823e5 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - 831d23e56149cd59c40fc00c7feb7c5334bd19c4 + 0c9b70db7b7bfcbc33e85bc40e215c3b138823e5 @@ -619,9 +619,9 @@ 5da211e1c42254cb35e7ef3d5a8428fb24853169 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - 831d23e56149cd59c40fc00c7feb7c5334bd19c4 + 0c9b70db7b7bfcbc33e85bc40e215c3b138823e5 https://github.com/dotnet/arcade-services diff --git a/eng/Versions.props b/eng/Versions.props index 6e415c5dd97a..87f60871abde 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -86,47 +86,47 @@ - 9.0.3 - 9.0.3-servicing.25111.13 - 9.0.3 - 9.0.3 - 9.0.3-servicing.25111.13 - 9.0.3 - 9.0.3 - 9.0.3 - 9.0.3 - 9.0.3 - 9.0.3 + 9.0.4 + 9.0.4-servicing.25162.18 + 9.0.4 + 9.0.4 + 9.0.4-servicing.25162.18 + 9.0.4 + 9.0.4 + 9.0.4 + 9.0.4 + 9.0.4 + 9.0.4 8.0.0-rc.1.23414.4 - 9.0.3-servicing.25111.13 - 9.0.3-servicing.25111.13 - 9.0.3 - 9.0.3 - 9.0.3 - 9.0.3 + 9.0.4-servicing.25162.18 + 9.0.4-servicing.25162.18 + 9.0.4 + 9.0.4 + 9.0.4 + 9.0.4 2.1.0 - 9.0.3 + 9.0.4 8.0.0 - 9.0.3 - 9.0.3 - 9.0.3 - 9.0.3 - 9.0.3 - 9.0.3 - 9.0.3 + 9.0.4 + 9.0.4 + 9.0.4 + 9.0.4 + 9.0.4 + 9.0.4 + 9.0.4 8.0.0 - 9.0.3 - 9.0.3 - 9.0.3 - 9.0.3 - 9.0.3 - 9.0.3 - 9.0.3 + 9.0.4 + 9.0.4 + 9.0.4 + 9.0.4 + 9.0.4 + 9.0.4 + 9.0.4 8.0.4 - 9.0.3 - 9.0.3 + 9.0.4 + 9.0.4 @@ -317,8 +317,8 @@ 15.0.9617 18.0.9617 - 9.0.3-servicing.25105.2 - 9.0.3 + 9.0.4-servicing.25157.2 + 9.0.4 $(MicrosoftNETWorkloadEmscriptenCurrentManifest90100PackageVersion) 9.0.100$([System.Text.RegularExpressions.Regex]::Match($(EmscriptenWorkloadManifestVersion), `-(?!rtm)[A-z]*[\.]*\d*`)) From aa76fd9123c8f9a948db4ee3c4729db5dd1a884a Mon Sep 17 00:00:00 2001 From: DotNet-Bot Date: Fri, 14 Mar 2025 02:59:17 +0000 Subject: [PATCH 02/57] Update dependencies from https://dev.azure.com/dnceng/internal/_git/dotnet-runtime build 20250313.5 Microsoft.Bcl.AsyncInterfaces , Microsoft.Extensions.DependencyModel , Microsoft.Extensions.FileProviders.Abstractions , Microsoft.Extensions.FileSystemGlobbing , Microsoft.Extensions.Logging , Microsoft.Extensions.Logging.Abstractions , Microsoft.Extensions.Logging.Console , Microsoft.NET.HostModel , Microsoft.NET.ILLink.Tasks , Microsoft.NETCore.App.Host.win-x64 , Microsoft.NETCore.App.Ref , Microsoft.NETCore.App.Runtime.win-x64 , Microsoft.NETCore.Platforms , Microsoft.Win32.SystemEvents , System.CodeDom , System.Composition.AttributedModel , System.Composition.Convention , System.Composition.Hosting , System.Composition.Runtime , System.Composition.TypedParts , System.Configuration.ConfigurationManager , System.Formats.Asn1 , System.Reflection.MetadataLoadContext , System.Resources.Extensions , System.Security.Cryptography.Pkcs , System.Security.Cryptography.ProtectedData , System.Security.Cryptography.Xml , System.Security.Permissions , System.ServiceProcess.ServiceController , System.Text.Encoding.CodePages , System.Text.Json , System.Windows.Extensions , VS.Redist.Common.NetCore.SharedFramework.x64.9.0 , VS.Redist.Common.NetCore.TargetingPack.x64.9.0 , Microsoft.SourceBuild.Intermediate.runtime.linux-x64 From Version 9.0.3 -> To Version 9.0.4 --- NuGet.config | 4 +-- eng/Version.Details.xml | 80 ++++++++++++++++++++--------------------- eng/Versions.props | 8 ++--- 3 files changed, 46 insertions(+), 46 deletions(-) diff --git a/NuGet.config b/NuGet.config index 3b6f0ba86d93..5cb464eb82f2 100644 --- a/NuGet.config +++ b/NuGet.config @@ -37,7 +37,7 @@ - + @@ -77,7 +77,7 @@ - + diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index d3de7897551e..cd58cfa39bcd 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -17,40 +17,40 @@ https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - 0c9b70db7b7bfcbc33e85bc40e215c3b138823e5 + f57e6dc747158ab7ade4e62a75a6750d16b771e8 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - 0c9b70db7b7bfcbc33e85bc40e215c3b138823e5 + f57e6dc747158ab7ade4e62a75a6750d16b771e8 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - 0c9b70db7b7bfcbc33e85bc40e215c3b138823e5 + f57e6dc747158ab7ade4e62a75a6750d16b771e8 https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - 0c9b70db7b7bfcbc33e85bc40e215c3b138823e5 + f57e6dc747158ab7ade4e62a75a6750d16b771e8 https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - 0c9b70db7b7bfcbc33e85bc40e215c3b138823e5 + f57e6dc747158ab7ade4e62a75a6750d16b771e8 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - 0c9b70db7b7bfcbc33e85bc40e215c3b138823e5 + f57e6dc747158ab7ade4e62a75a6750d16b771e8 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - 0c9b70db7b7bfcbc33e85bc40e215c3b138823e5 + f57e6dc747158ab7ade4e62a75a6750d16b771e8 https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - 0c9b70db7b7bfcbc33e85bc40e215c3b138823e5 + f57e6dc747158ab7ade4e62a75a6750d16b771e8 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - 0c9b70db7b7bfcbc33e85bc40e215c3b138823e5 + f57e6dc747158ab7ade4e62a75a6750d16b771e8 @@ -232,27 +232,27 @@ https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - 0c9b70db7b7bfcbc33e85bc40e215c3b138823e5 + f57e6dc747158ab7ade4e62a75a6750d16b771e8 https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - 0c9b70db7b7bfcbc33e85bc40e215c3b138823e5 + f57e6dc747158ab7ade4e62a75a6750d16b771e8 https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - 0c9b70db7b7bfcbc33e85bc40e215c3b138823e5 + f57e6dc747158ab7ade4e62a75a6750d16b771e8 https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - 0c9b70db7b7bfcbc33e85bc40e215c3b138823e5 + f57e6dc747158ab7ade4e62a75a6750d16b771e8 https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - 0c9b70db7b7bfcbc33e85bc40e215c3b138823e5 + f57e6dc747158ab7ade4e62a75a6750d16b771e8 https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - 0c9b70db7b7bfcbc33e85bc40e215c3b138823e5 + f57e6dc747158ab7ade4e62a75a6750d16b771e8 https://dev.azure.com/dnceng/internal/_git/dotnet-windowsdesktop @@ -505,39 +505,39 @@ https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - 0c9b70db7b7bfcbc33e85bc40e215c3b138823e5 + f57e6dc747158ab7ade4e62a75a6750d16b771e8 https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - 0c9b70db7b7bfcbc33e85bc40e215c3b138823e5 + f57e6dc747158ab7ade4e62a75a6750d16b771e8 https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - 0c9b70db7b7bfcbc33e85bc40e215c3b138823e5 + f57e6dc747158ab7ade4e62a75a6750d16b771e8 https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - 0c9b70db7b7bfcbc33e85bc40e215c3b138823e5 + f57e6dc747158ab7ade4e62a75a6750d16b771e8 https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - 0c9b70db7b7bfcbc33e85bc40e215c3b138823e5 + f57e6dc747158ab7ade4e62a75a6750d16b771e8 https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - 0c9b70db7b7bfcbc33e85bc40e215c3b138823e5 + f57e6dc747158ab7ade4e62a75a6750d16b771e8 https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - 0c9b70db7b7bfcbc33e85bc40e215c3b138823e5 + f57e6dc747158ab7ade4e62a75a6750d16b771e8 https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - 0c9b70db7b7bfcbc33e85bc40e215c3b138823e5 + f57e6dc747158ab7ade4e62a75a6750d16b771e8 https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore @@ -545,47 +545,47 @@ https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - 0c9b70db7b7bfcbc33e85bc40e215c3b138823e5 + f57e6dc747158ab7ade4e62a75a6750d16b771e8 https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - 0c9b70db7b7bfcbc33e85bc40e215c3b138823e5 + f57e6dc747158ab7ade4e62a75a6750d16b771e8 https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - 0c9b70db7b7bfcbc33e85bc40e215c3b138823e5 + f57e6dc747158ab7ade4e62a75a6750d16b771e8 https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - 0c9b70db7b7bfcbc33e85bc40e215c3b138823e5 + f57e6dc747158ab7ade4e62a75a6750d16b771e8 https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - 0c9b70db7b7bfcbc33e85bc40e215c3b138823e5 + f57e6dc747158ab7ade4e62a75a6750d16b771e8 https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - 0c9b70db7b7bfcbc33e85bc40e215c3b138823e5 + f57e6dc747158ab7ade4e62a75a6750d16b771e8 https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - 0c9b70db7b7bfcbc33e85bc40e215c3b138823e5 + f57e6dc747158ab7ade4e62a75a6750d16b771e8 https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - 0c9b70db7b7bfcbc33e85bc40e215c3b138823e5 + f57e6dc747158ab7ade4e62a75a6750d16b771e8 https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - 0c9b70db7b7bfcbc33e85bc40e215c3b138823e5 + f57e6dc747158ab7ade4e62a75a6750d16b771e8 https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - 0c9b70db7b7bfcbc33e85bc40e215c3b138823e5 + f57e6dc747158ab7ade4e62a75a6750d16b771e8 https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - 0c9b70db7b7bfcbc33e85bc40e215c3b138823e5 + f57e6dc747158ab7ade4e62a75a6750d16b771e8 @@ -621,7 +621,7 @@ https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - 0c9b70db7b7bfcbc33e85bc40e215c3b138823e5 + f57e6dc747158ab7ade4e62a75a6750d16b771e8 https://github.com/dotnet/arcade-services diff --git a/eng/Versions.props b/eng/Versions.props index 87f60871abde..64b24b170c1e 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -87,10 +87,10 @@ 9.0.4 - 9.0.4-servicing.25162.18 + 9.0.4-servicing.25163.5 9.0.4 9.0.4 - 9.0.4-servicing.25162.18 + 9.0.4-servicing.25163.5 9.0.4 9.0.4 9.0.4 @@ -98,8 +98,8 @@ 9.0.4 9.0.4 8.0.0-rc.1.23414.4 - 9.0.4-servicing.25162.18 - 9.0.4-servicing.25162.18 + 9.0.4-servicing.25163.5 + 9.0.4-servicing.25163.5 9.0.4 9.0.4 9.0.4 From bce2ffc4cf80c39cb479c81ce79e84ac4367d4eb Mon Sep 17 00:00:00 2001 From: DotNet-Bot Date: Fri, 14 Mar 2025 07:19:29 +0000 Subject: [PATCH 03/57] Update dependencies from https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore build 20250313.12 dotnet-dev-certs , dotnet-user-jwts , dotnet-user-secrets , Microsoft.AspNetCore.Analyzers , Microsoft.AspNetCore.App.Ref , Microsoft.AspNetCore.App.Ref.Internal , Microsoft.AspNetCore.App.Runtime.win-x64 , Microsoft.AspNetCore.Authorization , Microsoft.AspNetCore.Components.SdkAnalyzers , Microsoft.AspNetCore.Components.Web , Microsoft.AspNetCore.DeveloperCertificates.XPlat , Microsoft.AspNetCore.Mvc.Analyzers , Microsoft.AspNetCore.Mvc.Api.Analyzers , Microsoft.AspNetCore.TestHost , Microsoft.Extensions.FileProviders.Embedded , Microsoft.Extensions.ObjectPool , Microsoft.JSInterop , VS.Redist.Common.AspNetCore.SharedFramework.x64.9.0 , Microsoft.SourceBuild.Intermediate.aspnetcore From Version 9.0.3-servicing.25112.20 -> To Version 9.0.4-servicing.25163.12 --- NuGet.config | 8 ++--- eng/Version.Details.xml | 76 ++++++++++++++++++++--------------------- eng/Versions.props | 26 +++++++------- 3 files changed, 53 insertions(+), 57 deletions(-) diff --git a/NuGet.config b/NuGet.config index 5cb464eb82f2..02b04a8ac818 100644 --- a/NuGet.config +++ b/NuGet.config @@ -27,9 +27,7 @@ - - - + @@ -72,9 +70,7 @@ - - - + diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index cd58cfa39bcd..8405e2195db0 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -135,13 +135,13 @@ https://github.com/dotnet/roslyn 3f5cf9fbbd91f2047e988801a5142ca1cb6bab45 - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - b96167fbfe8bd45d94e4dcda42c7d09eb5745459 + 0e3201af63dfb753cb9219dc472ddfa0719bf7bf - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - b96167fbfe8bd45d94e4dcda42c7d09eb5745459 + 0e3201af63dfb753cb9219dc472ddfa0719bf7bf https://github.com/nuget/nuget.client @@ -275,54 +275,54 @@ https://dev.azure.com/dnceng/internal/_git/dotnet-wpf f58f709bc102a0d15aa6ffe4d9a7435bebd3079e - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - b96167fbfe8bd45d94e4dcda42c7d09eb5745459 + 0e3201af63dfb753cb9219dc472ddfa0719bf7bf - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - b96167fbfe8bd45d94e4dcda42c7d09eb5745459 + 0e3201af63dfb753cb9219dc472ddfa0719bf7bf - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - b96167fbfe8bd45d94e4dcda42c7d09eb5745459 + 0e3201af63dfb753cb9219dc472ddfa0719bf7bf - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - b96167fbfe8bd45d94e4dcda42c7d09eb5745459 + 0e3201af63dfb753cb9219dc472ddfa0719bf7bf - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - b96167fbfe8bd45d94e4dcda42c7d09eb5745459 + 0e3201af63dfb753cb9219dc472ddfa0719bf7bf - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - b96167fbfe8bd45d94e4dcda42c7d09eb5745459 + 0e3201af63dfb753cb9219dc472ddfa0719bf7bf - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - b96167fbfe8bd45d94e4dcda42c7d09eb5745459 + 0e3201af63dfb753cb9219dc472ddfa0719bf7bf - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - b96167fbfe8bd45d94e4dcda42c7d09eb5745459 + 0e3201af63dfb753cb9219dc472ddfa0719bf7bf - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - b96167fbfe8bd45d94e4dcda42c7d09eb5745459 + 0e3201af63dfb753cb9219dc472ddfa0719bf7bf - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - b96167fbfe8bd45d94e4dcda42c7d09eb5745459 + 0e3201af63dfb753cb9219dc472ddfa0719bf7bf - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - b96167fbfe8bd45d94e4dcda42c7d09eb5745459 + 0e3201af63dfb753cb9219dc472ddfa0719bf7bf - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - b96167fbfe8bd45d94e4dcda42c7d09eb5745459 + 0e3201af63dfb753cb9219dc472ddfa0719bf7bf @@ -343,21 +343,21 @@ d0fbb0748128955a5c5e5dd4289b3859d7d70600 - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - b96167fbfe8bd45d94e4dcda42c7d09eb5745459 + 0e3201af63dfb753cb9219dc472ddfa0719bf7bf - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - b96167fbfe8bd45d94e4dcda42c7d09eb5745459 + 0e3201af63dfb753cb9219dc472ddfa0719bf7bf - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - b96167fbfe8bd45d94e4dcda42c7d09eb5745459 + 0e3201af63dfb753cb9219dc472ddfa0719bf7bf - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - b96167fbfe8bd45d94e4dcda42c7d09eb5745459 + 0e3201af63dfb753cb9219dc472ddfa0719bf7bf https://github.com/dotnet/test-templates @@ -539,9 +539,9 @@ https://dev.azure.com/dnceng/internal/_git/dotnet-runtime f57e6dc747158ab7ade4e62a75a6750d16b771e8 - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - b96167fbfe8bd45d94e4dcda42c7d09eb5745459 + 0e3201af63dfb753cb9219dc472ddfa0719bf7bf https://dev.azure.com/dnceng/internal/_git/dotnet-runtime diff --git a/eng/Versions.props b/eng/Versions.props index 64b24b170c1e..49f50d0cc3a5 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -215,19 +215,19 @@ - 9.0.3 - 9.0.3-servicing.25112.20 - 9.0.3-servicing.25112.20 - 9.0.3-servicing.25112.20 - 9.0.3-servicing.25112.20 - 9.0.3-servicing.25112.20 - 9.0.3 - 9.0.3 - 9.0.3-servicing.25112.20 - 9.0.3-servicing.25112.20 - 9.0.3-servicing.25112.20 - 9.0.3-servicing.25112.20 - 9.0.3-servicing.25112.20 + 9.0.4 + 9.0.4-servicing.25163.12 + 9.0.4-servicing.25163.12 + 9.0.4-servicing.25163.12 + 9.0.4-servicing.25163.12 + 9.0.4-servicing.25163.12 + 9.0.4 + 9.0.4 + 9.0.4-servicing.25163.12 + 9.0.4-servicing.25163.12 + 9.0.4-servicing.25163.12 + 9.0.4-servicing.25163.12 + 9.0.4-servicing.25163.12 From 96eb75f20b3b325cf0469409bfe4bad4dae42c78 Mon Sep 17 00:00:00 2001 From: DotNet-Bot Date: Fri, 14 Mar 2025 19:09:12 +0000 Subject: [PATCH 04/57] Update dependencies from https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore build 20250314.3 dotnet-dev-certs , dotnet-user-jwts , dotnet-user-secrets , Microsoft.AspNetCore.Analyzers , Microsoft.AspNetCore.App.Ref , Microsoft.AspNetCore.App.Ref.Internal , Microsoft.AspNetCore.App.Runtime.win-x64 , Microsoft.AspNetCore.Authorization , Microsoft.AspNetCore.Components.SdkAnalyzers , Microsoft.AspNetCore.Components.Web , Microsoft.AspNetCore.DeveloperCertificates.XPlat , Microsoft.AspNetCore.Mvc.Analyzers , Microsoft.AspNetCore.Mvc.Api.Analyzers , Microsoft.AspNetCore.TestHost , Microsoft.Extensions.FileProviders.Embedded , Microsoft.Extensions.ObjectPool , Microsoft.JSInterop , VS.Redist.Common.AspNetCore.SharedFramework.x64.9.0 , Microsoft.SourceBuild.Intermediate.aspnetcore From Version 9.0.3-servicing.25112.20 -> To Version 9.0.4-servicing.25164.3 --- NuGet.config | 4 +-- eng/Version.Details.xml | 60 ++++++++++++++++++++--------------------- eng/Versions.props | 20 +++++++------- 3 files changed, 42 insertions(+), 42 deletions(-) diff --git a/NuGet.config b/NuGet.config index 02b04a8ac818..460c71b1e363 100644 --- a/NuGet.config +++ b/NuGet.config @@ -27,7 +27,7 @@ - + @@ -70,7 +70,7 @@ - + diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 8405e2195db0..daee693ca26b 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -135,13 +135,13 @@ https://github.com/dotnet/roslyn 3f5cf9fbbd91f2047e988801a5142ca1cb6bab45 - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - 0e3201af63dfb753cb9219dc472ddfa0719bf7bf + d5dc8a13cc618b9cbdc1e5744b4806c594d49553 https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - 0e3201af63dfb753cb9219dc472ddfa0719bf7bf + d5dc8a13cc618b9cbdc1e5744b4806c594d49553 https://github.com/nuget/nuget.client @@ -277,52 +277,52 @@ https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - 0e3201af63dfb753cb9219dc472ddfa0719bf7bf + d5dc8a13cc618b9cbdc1e5744b4806c594d49553 - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - 0e3201af63dfb753cb9219dc472ddfa0719bf7bf + d5dc8a13cc618b9cbdc1e5744b4806c594d49553 https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - 0e3201af63dfb753cb9219dc472ddfa0719bf7bf + d5dc8a13cc618b9cbdc1e5744b4806c594d49553 - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - 0e3201af63dfb753cb9219dc472ddfa0719bf7bf + d5dc8a13cc618b9cbdc1e5744b4806c594d49553 - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - 0e3201af63dfb753cb9219dc472ddfa0719bf7bf + d5dc8a13cc618b9cbdc1e5744b4806c594d49553 - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - 0e3201af63dfb753cb9219dc472ddfa0719bf7bf + d5dc8a13cc618b9cbdc1e5744b4806c594d49553 - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - 0e3201af63dfb753cb9219dc472ddfa0719bf7bf + d5dc8a13cc618b9cbdc1e5744b4806c594d49553 - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - 0e3201af63dfb753cb9219dc472ddfa0719bf7bf + d5dc8a13cc618b9cbdc1e5744b4806c594d49553 - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - 0e3201af63dfb753cb9219dc472ddfa0719bf7bf + d5dc8a13cc618b9cbdc1e5744b4806c594d49553 - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - 0e3201af63dfb753cb9219dc472ddfa0719bf7bf + d5dc8a13cc618b9cbdc1e5744b4806c594d49553 - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - 0e3201af63dfb753cb9219dc472ddfa0719bf7bf + d5dc8a13cc618b9cbdc1e5744b4806c594d49553 - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - 0e3201af63dfb753cb9219dc472ddfa0719bf7bf + d5dc8a13cc618b9cbdc1e5744b4806c594d49553 @@ -345,19 +345,19 @@ https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - 0e3201af63dfb753cb9219dc472ddfa0719bf7bf + d5dc8a13cc618b9cbdc1e5744b4806c594d49553 https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - 0e3201af63dfb753cb9219dc472ddfa0719bf7bf + d5dc8a13cc618b9cbdc1e5744b4806c594d49553 https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - 0e3201af63dfb753cb9219dc472ddfa0719bf7bf + d5dc8a13cc618b9cbdc1e5744b4806c594d49553 https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - 0e3201af63dfb753cb9219dc472ddfa0719bf7bf + d5dc8a13cc618b9cbdc1e5744b4806c594d49553 https://github.com/dotnet/test-templates @@ -541,7 +541,7 @@ https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - 0e3201af63dfb753cb9219dc472ddfa0719bf7bf + d5dc8a13cc618b9cbdc1e5744b4806c594d49553 https://dev.azure.com/dnceng/internal/_git/dotnet-runtime diff --git a/eng/Versions.props b/eng/Versions.props index 49f50d0cc3a5..97a5db4fd75a 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -216,18 +216,18 @@ 9.0.4 - 9.0.4-servicing.25163.12 - 9.0.4-servicing.25163.12 - 9.0.4-servicing.25163.12 - 9.0.4-servicing.25163.12 - 9.0.4-servicing.25163.12 + 9.0.4-servicing.25164.3 + 9.0.4-servicing.25164.3 + 9.0.4-servicing.25164.3 + 9.0.4-servicing.25164.3 + 9.0.4-servicing.25164.3 9.0.4 9.0.4 - 9.0.4-servicing.25163.12 - 9.0.4-servicing.25163.12 - 9.0.4-servicing.25163.12 - 9.0.4-servicing.25163.12 - 9.0.4-servicing.25163.12 + 9.0.4-servicing.25164.3 + 9.0.4-servicing.25164.3 + 9.0.4-servicing.25164.3 + 9.0.4-servicing.25164.3 + 9.0.4-servicing.25164.3 From d0f4c008cd0e0b78ad432ea27bf134544fe1ee6f Mon Sep 17 00:00:00 2001 From: DotNet-Bot Date: Fri, 14 Mar 2025 20:51:10 +0000 Subject: [PATCH 05/57] Update dependencies from https://dev.azure.com/dnceng/internal/_git/dotnet-windowsdesktop build 20250314.2 Microsoft.WindowsDesktop.App.Ref , Microsoft.WindowsDesktop.App.Runtime.win-x64 , VS.Redist.Common.WindowsDesktop.SharedFramework.x64.9.0 , VS.Redist.Common.WindowsDesktop.TargetingPack.x64.9.0 From Version 9.0.3 -> To Version 9.0.4 Dependency coherency updates Microsoft.NET.Sdk.WindowsDesktop,Microsoft.Dotnet.WinForms.ProjectTemplates,Microsoft.DotNet.Wpf.ProjectTemplates From Version 9.0.3-rtm.25113.4 -> To Version 9.0.4-rtm.25164.2 (parent: Microsoft.WindowsDesktop.App.Ref --- NuGet.config | 8 ++------ eng/Version.Details.xml | 28 ++++++++++++++-------------- eng/Versions.props | 14 +++++++------- 3 files changed, 23 insertions(+), 27 deletions(-) diff --git a/NuGet.config b/NuGet.config index 460c71b1e363..6b45b6bef950 100644 --- a/NuGet.config +++ b/NuGet.config @@ -41,9 +41,7 @@ - - - + @@ -76,9 +74,7 @@ - - - + diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index daee693ca26b..7e1c88a9776c 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -254,26 +254,26 @@ https://dev.azure.com/dnceng/internal/_git/dotnet-runtime f57e6dc747158ab7ade4e62a75a6750d16b771e8 - + https://dev.azure.com/dnceng/internal/_git/dotnet-windowsdesktop - 1c33258cdd6220239735d6a7a18ef6adc082038c + 6caea0d3a33c1eeaaa4270cfe448354e7cfa6b1e - + https://dev.azure.com/dnceng/internal/_git/dotnet-windowsdesktop - 1c33258cdd6220239735d6a7a18ef6adc082038c + 6caea0d3a33c1eeaaa4270cfe448354e7cfa6b1e - + https://dev.azure.com/dnceng/internal/_git/dotnet-windowsdesktop - 1c33258cdd6220239735d6a7a18ef6adc082038c + 6caea0d3a33c1eeaaa4270cfe448354e7cfa6b1e - + https://dev.azure.com/dnceng/internal/_git/dotnet-windowsdesktop - 1c33258cdd6220239735d6a7a18ef6adc082038c + 6caea0d3a33c1eeaaa4270cfe448354e7cfa6b1e - + https://dev.azure.com/dnceng/internal/_git/dotnet-wpf - f58f709bc102a0d15aa6ffe4d9a7435bebd3079e + 33f5371cc258de0a72569adf1e9fbcbf04256a17 https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore @@ -390,13 +390,13 @@ - + https://dev.azure.com/dnceng/internal/_git/dotnet-winforms - 5bf1bff0e58e41f748a8a6d7c666ed28e033e318 + ca42d9e58480ab4ebb845027c44c6b4f10e0539d - + https://dev.azure.com/dnceng/internal/_git/dotnet-wpf - f58f709bc102a0d15aa6ffe4d9a7435bebd3079e + 33f5371cc258de0a72569adf1e9fbcbf04256a17 https://github.com/dotnet/xdt diff --git a/eng/Versions.props b/eng/Versions.props index 97a5db4fd75a..a7ff3911f694 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -77,7 +77,7 @@ - 9.0.3-servicing.25112.2 + 9.0.4-servicing.25163.12 @@ -130,10 +130,10 @@ - 9.0.3-servicing.25113.2 - 9.0.3-servicing.25113.2 - 9.0.3 - 9.0.3 + 9.0.4-servicing.25164.2 + 9.0.4-servicing.25164.2 + 9.0.4 + 9.0.4 @@ -237,8 +237,8 @@ - 9.0.3-rtm.25113.4 - 9.0.3-rtm.25113.4 + 9.0.4-rtm.25164.2 + 9.0.4-rtm.25164.2 From 5b6ec2be8493f7cb1dd7a8bfddb68aff9d12e083 Mon Sep 17 00:00:00 2001 From: DotNet-Bot Date: Sat, 15 Mar 2025 06:04:48 +0000 Subject: [PATCH 06/57] Update dependencies from https://dev.azure.com/dnceng/internal/_git/dotnet-windowsdesktop build 20250314.6 Microsoft.WindowsDesktop.App.Ref , Microsoft.WindowsDesktop.App.Runtime.win-x64 , VS.Redist.Common.WindowsDesktop.SharedFramework.x64.9.0 , VS.Redist.Common.WindowsDesktop.TargetingPack.x64.9.0 From Version 9.0.4 -> To Version 9.0.4 --- NuGet.config | 4 ++-- eng/Version.Details.xml | 12 ++++++------ eng/Versions.props | 4 ++-- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/NuGet.config b/NuGet.config index 6b45b6bef950..efdee772486b 100644 --- a/NuGet.config +++ b/NuGet.config @@ -41,7 +41,7 @@ - + @@ -74,7 +74,7 @@ - + diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 7e1c88a9776c..a57442733e59 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -256,20 +256,20 @@ https://dev.azure.com/dnceng/internal/_git/dotnet-windowsdesktop - 6caea0d3a33c1eeaaa4270cfe448354e7cfa6b1e + ef0ddf89991072267608f432ca2163ac54fdecab - + https://dev.azure.com/dnceng/internal/_git/dotnet-windowsdesktop - 6caea0d3a33c1eeaaa4270cfe448354e7cfa6b1e + ef0ddf89991072267608f432ca2163ac54fdecab https://dev.azure.com/dnceng/internal/_git/dotnet-windowsdesktop - 6caea0d3a33c1eeaaa4270cfe448354e7cfa6b1e + ef0ddf89991072267608f432ca2163ac54fdecab - + https://dev.azure.com/dnceng/internal/_git/dotnet-windowsdesktop - 6caea0d3a33c1eeaaa4270cfe448354e7cfa6b1e + ef0ddf89991072267608f432ca2163ac54fdecab https://dev.azure.com/dnceng/internal/_git/dotnet-wpf diff --git a/eng/Versions.props b/eng/Versions.props index a7ff3911f694..ca5a8a4d07b4 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -130,8 +130,8 @@ - 9.0.4-servicing.25164.2 - 9.0.4-servicing.25164.2 + 9.0.4-servicing.25164.6 + 9.0.4-servicing.25164.6 9.0.4 9.0.4 From ad4c11477c521b9e0109fd7eb2fce0ada5ef82e4 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Tue, 15 Apr 2025 19:20:45 +0000 Subject: [PATCH 07/57] Update dependencies from https://github.com/dotnet/templating build 20250415.3 Microsoft.SourceBuild.Intermediate.templating , Microsoft.TemplateEngine.Abstractions , Microsoft.TemplateEngine.Mocks From Version 9.0.106-servicing.25209.3 -> To Version 9.0.106-servicing.25215.3 --- NuGet.config | 2 +- eng/Version.Details.xml | 10 +++++----- eng/Versions.props | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/NuGet.config b/NuGet.config index 207dc2aae546..3bea35ae691e 100644 --- a/NuGet.config +++ b/NuGet.config @@ -35,7 +35,7 @@ - + diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index b71d7bace6e9..b54b7b05eb58 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,16 +3,16 @@ https://github.com/dotnet/templating - 35eb08d048d4f4667db94b07f0e838ccd5fb5169 + 94bc1a8621167807199e338a3ef2a18162e3bc37 - + https://github.com/dotnet/templating - 35eb08d048d4f4667db94b07f0e838ccd5fb5169 + 94bc1a8621167807199e338a3ef2a18162e3bc37 - + https://github.com/dotnet/templating - 35eb08d048d4f4667db94b07f0e838ccd5fb5169 + 94bc1a8621167807199e338a3ef2a18162e3bc37 diff --git a/eng/Versions.props b/eng/Versions.props index f7619394fa7e..698a09ec9e8e 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -193,7 +193,7 @@ $(MicrosoftTemplateEngineAbstractionsPackageVersion) $(MicrosoftTemplateEngineAbstractionsPackageVersion) - 9.0.106-servicing.25209.3 + 9.0.106-servicing.25215.3 $(MicrosoftTemplateEngineMocksPackageVersion) $(MicrosoftTemplateEngineAbstractionsPackageVersion) $(MicrosoftTemplateEngineMocksPackageVersion) From 64cbf2a80623bcac93215d6a8800cf520953e03d Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Wed, 16 Apr 2025 05:02:06 +0000 Subject: [PATCH 08/57] Update dependencies from https://github.com/dotnet/templating build 20250415.2 Microsoft.SourceBuild.Intermediate.templating , Microsoft.TemplateEngine.Abstractions , Microsoft.TemplateEngine.Mocks From Version 8.0.409-servicing.25209.2 -> To Version 8.0.409-servicing.25215.2 --- NuGet.config | 2 +- eng/Version.Details.xml | 10 +++++----- eng/Versions.props | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/NuGet.config b/NuGet.config index b02836bc40ef..7fc6408c016b 100644 --- a/NuGet.config +++ b/NuGet.config @@ -18,7 +18,7 @@ - + diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 673c2fb15f12..a12e6a1e71de 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,15 +3,15 @@ https://github.com/dotnet/templating - 731d4697ceb43ca5ab88a4994b5a226e0cbf16c4 + 1e52e85918ccfe396647e8a96e293e6b4c0e17b7 - + https://github.com/dotnet/templating - 731d4697ceb43ca5ab88a4994b5a226e0cbf16c4 + 1e52e85918ccfe396647e8a96e293e6b4c0e17b7 - + https://github.com/dotnet/templating - 731d4697ceb43ca5ab88a4994b5a226e0cbf16c4 + 1e52e85918ccfe396647e8a96e293e6b4c0e17b7 diff --git a/eng/Versions.props b/eng/Versions.props index c217eab2fcd9..c6adc72fd19c 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -147,7 +147,7 @@ $(MicrosoftTemplateEngineAbstractionsPackageVersion) $(MicrosoftTemplateEngineAbstractionsPackageVersion) - 8.0.409-servicing.25209.2 + 8.0.409-servicing.25215.2 $(MicrosoftTemplateEngineMocksPackageVersion) $(MicrosoftTemplateEngineAbstractionsPackageVersion) $(MicrosoftTemplateEngineMocksPackageVersion) From ad312fc3980fd8c562fac172bc17318c447eeb75 Mon Sep 17 00:00:00 2001 From: Rainer Sigwald Date: Wed, 16 Apr 2025 16:00:37 +0000 Subject: [PATCH 09/57] [internal/release/9.0.1xx] MSBuild 17.12.32 MSBuild 17.12.32 via `darc update-dependencies --id 263902` --- NuGet.config | 21 ++++++++++++++++++++- eng/Version.Details.xml | 16 ++++++++-------- eng/Versions.props | 4 ++-- 3 files changed, 30 insertions(+), 11 deletions(-) diff --git a/NuGet.config b/NuGet.config index e4acfc9be4c6..a2aba39ff11d 100644 --- a/NuGet.config +++ b/NuGet.config @@ -25,23 +25,32 @@ + + + + - + + + + + + @@ -68,12 +77,22 @@ + + + + + + + + + + diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index b54b7b05eb58..21f84d605cc6 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -74,17 +74,17 @@ - https://github.com/dotnet/msbuild - ced361277f33b9eb9312d927869a77b6d9108876 + https://dev.azure.com/devdiv/DevDiv/_git/DotNet-msbuild-Trusted + 13c590f8d138d6a30b899a5172cc6ec494db930b - - https://github.com/dotnet/msbuild - ced361277f33b9eb9312d927869a77b6d9108876 + + https://dev.azure.com/devdiv/DevDiv/_git/DotNet-msbuild-Trusted + 13c590f8d138d6a30b899a5172cc6ec494db930b - - https://github.com/dotnet/msbuild - ced361277f33b9eb9312d927869a77b6d9108876 + + https://dev.azure.com/devdiv/DevDiv/_git/DotNet-msbuild-Trusted + 13c590f8d138d6a30b899a5172cc6ec494db930b diff --git a/eng/Versions.props b/eng/Versions.props index 698a09ec9e8e..ec284b2e3be9 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -181,8 +181,8 @@ Additionally, set the MinimumVSVersion for the installer UI that's required for targeting NetCurrent --> 17.12.35 - 17.12.35-preview-25214-08 - 17.11.4 + 17.12.35-preview-25210-18 + 17.11.31 17.12 From 0b967592f736b871f7cc915e13e5fb3ec4d85140 Mon Sep 17 00:00:00 2001 From: DotNet-Bot Date: Wed, 16 Apr 2025 21:05:47 +0000 Subject: [PATCH 10/57] Update dependencies from https://dev.azure.com/dnceng/internal/_git/dotnet-runtime build 20250415.9 Microsoft.Bcl.AsyncInterfaces , Microsoft.Extensions.DependencyModel , Microsoft.Extensions.FileProviders.Abstractions , Microsoft.Extensions.FileSystemGlobbing , Microsoft.Extensions.Logging , Microsoft.Extensions.Logging.Abstractions , Microsoft.Extensions.Logging.Console , Microsoft.NET.HostModel , Microsoft.NET.ILLink.Tasks , Microsoft.NETCore.App.Host.win-x64 , Microsoft.NETCore.App.Ref , Microsoft.NETCore.App.Runtime.win-x64 , Microsoft.NETCore.Platforms , Microsoft.Win32.SystemEvents , System.CodeDom , System.Composition.AttributedModel , System.Composition.Convention , System.Composition.Hosting , System.Composition.Runtime , System.Composition.TypedParts , System.Configuration.ConfigurationManager , System.Formats.Asn1 , System.Reflection.MetadataLoadContext , System.Resources.Extensions , System.Security.Cryptography.Pkcs , System.Security.Cryptography.ProtectedData , System.Security.Cryptography.Xml , System.Security.Permissions , System.ServiceProcess.ServiceController , System.Text.Encoding.CodePages , System.Text.Json , System.Windows.Extensions , VS.Redist.Common.NetCore.SharedFramework.x64.9.0 , VS.Redist.Common.NetCore.TargetingPack.x64.9.0 , Microsoft.SourceBuild.Intermediate.runtime.linux-x64 From Version 9.0.4 -> To Version 9.0.5 Dependency coherency updates Microsoft.NET.Workload.Emscripten.Current.Manifest-9.0.100.Transport,Microsoft.NET.Workload.Emscripten.Current.Manifest-9.0.100,Microsoft.SourceBuild.Intermediate.emsdk From Version 9.0.4-servicing.25157.2 -> To Version 9.0.5-servicing.25212.1 (parent: Microsoft.NETCore.App.Runtime.win-x64 --- NuGet.config | 26 +------ eng/Version.Details.xml | 152 ++++++++++++++++++++-------------------- eng/Versions.props | 72 +++++++++---------- 3 files changed, 115 insertions(+), 135 deletions(-) diff --git a/NuGet.config b/NuGet.config index a2aba39ff11d..c516984a145d 100644 --- a/NuGet.config +++ b/NuGet.config @@ -24,33 +24,22 @@ - - - + - - - - - - - + - - - @@ -78,22 +67,13 @@ - - - - - - - + - - - diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 21f84d605cc6..db1ff1a26a51 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -15,42 +15,42 @@ 94bc1a8621167807199e338a3ef2a18162e3bc37 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f57e6dc747158ab7ade4e62a75a6750d16b771e8 + e36e4d1a8f8dfb08d7e3a6041459c9791d732c01 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f57e6dc747158ab7ade4e62a75a6750d16b771e8 + e36e4d1a8f8dfb08d7e3a6041459c9791d732c01 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f57e6dc747158ab7ade4e62a75a6750d16b771e8 + e36e4d1a8f8dfb08d7e3a6041459c9791d732c01 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f57e6dc747158ab7ade4e62a75a6750d16b771e8 + e36e4d1a8f8dfb08d7e3a6041459c9791d732c01 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f57e6dc747158ab7ade4e62a75a6750d16b771e8 + e36e4d1a8f8dfb08d7e3a6041459c9791d732c01 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f57e6dc747158ab7ade4e62a75a6750d16b771e8 + e36e4d1a8f8dfb08d7e3a6041459c9791d732c01 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f57e6dc747158ab7ade4e62a75a6750d16b771e8 + e36e4d1a8f8dfb08d7e3a6041459c9791d732c01 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f57e6dc747158ab7ade4e62a75a6750d16b771e8 + e36e4d1a8f8dfb08d7e3a6041459c9791d732c01 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f57e6dc747158ab7ade4e62a75a6750d16b771e8 + e36e4d1a8f8dfb08d7e3a6041459c9791d732c01 @@ -59,18 +59,18 @@ https://github.com/dotnet/core-setup 7d57652f33493fa022125b7f63aad0d70c52d810 - + https://github.com/dotnet/emsdk - 78be8cdf4f0bfd93018fd7a87f8282a41d041298 + 78f6f07d38e8755e573039a8aa04e131d3e59b76 - + https://github.com/dotnet/emsdk - 78be8cdf4f0bfd93018fd7a87f8282a41d041298 + 78f6f07d38e8755e573039a8aa04e131d3e59b76 - + https://github.com/dotnet/emsdk - 78be8cdf4f0bfd93018fd7a87f8282a41d041298 + 78f6f07d38e8755e573039a8aa04e131d3e59b76 @@ -230,29 +230,29 @@ bc9161306b23641b0364b8f93d546da4d48da1eb - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f57e6dc747158ab7ade4e62a75a6750d16b771e8 + e36e4d1a8f8dfb08d7e3a6041459c9791d732c01 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f57e6dc747158ab7ade4e62a75a6750d16b771e8 + e36e4d1a8f8dfb08d7e3a6041459c9791d732c01 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f57e6dc747158ab7ade4e62a75a6750d16b771e8 + e36e4d1a8f8dfb08d7e3a6041459c9791d732c01 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f57e6dc747158ab7ade4e62a75a6750d16b771e8 + e36e4d1a8f8dfb08d7e3a6041459c9791d732c01 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f57e6dc747158ab7ade4e62a75a6750d16b771e8 + e36e4d1a8f8dfb08d7e3a6041459c9791d732c01 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f57e6dc747158ab7ade4e62a75a6750d16b771e8 + e36e4d1a8f8dfb08d7e3a6041459c9791d732c01 https://dev.azure.com/dnceng/internal/_git/dotnet-windowsdesktop @@ -503,89 +503,89 @@ - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f57e6dc747158ab7ade4e62a75a6750d16b771e8 + e36e4d1a8f8dfb08d7e3a6041459c9791d732c01 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f57e6dc747158ab7ade4e62a75a6750d16b771e8 + e36e4d1a8f8dfb08d7e3a6041459c9791d732c01 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f57e6dc747158ab7ade4e62a75a6750d16b771e8 + e36e4d1a8f8dfb08d7e3a6041459c9791d732c01 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f57e6dc747158ab7ade4e62a75a6750d16b771e8 + e36e4d1a8f8dfb08d7e3a6041459c9791d732c01 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f57e6dc747158ab7ade4e62a75a6750d16b771e8 + e36e4d1a8f8dfb08d7e3a6041459c9791d732c01 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f57e6dc747158ab7ade4e62a75a6750d16b771e8 + e36e4d1a8f8dfb08d7e3a6041459c9791d732c01 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f57e6dc747158ab7ade4e62a75a6750d16b771e8 + e36e4d1a8f8dfb08d7e3a6041459c9791d732c01 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f57e6dc747158ab7ade4e62a75a6750d16b771e8 + e36e4d1a8f8dfb08d7e3a6041459c9791d732c01 https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore d5dc8a13cc618b9cbdc1e5744b4806c594d49553 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f57e6dc747158ab7ade4e62a75a6750d16b771e8 + e36e4d1a8f8dfb08d7e3a6041459c9791d732c01 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f57e6dc747158ab7ade4e62a75a6750d16b771e8 + e36e4d1a8f8dfb08d7e3a6041459c9791d732c01 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f57e6dc747158ab7ade4e62a75a6750d16b771e8 + e36e4d1a8f8dfb08d7e3a6041459c9791d732c01 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f57e6dc747158ab7ade4e62a75a6750d16b771e8 + e36e4d1a8f8dfb08d7e3a6041459c9791d732c01 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f57e6dc747158ab7ade4e62a75a6750d16b771e8 + e36e4d1a8f8dfb08d7e3a6041459c9791d732c01 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f57e6dc747158ab7ade4e62a75a6750d16b771e8 + e36e4d1a8f8dfb08d7e3a6041459c9791d732c01 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f57e6dc747158ab7ade4e62a75a6750d16b771e8 + e36e4d1a8f8dfb08d7e3a6041459c9791d732c01 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f57e6dc747158ab7ade4e62a75a6750d16b771e8 + e36e4d1a8f8dfb08d7e3a6041459c9791d732c01 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f57e6dc747158ab7ade4e62a75a6750d16b771e8 + e36e4d1a8f8dfb08d7e3a6041459c9791d732c01 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f57e6dc747158ab7ade4e62a75a6750d16b771e8 + e36e4d1a8f8dfb08d7e3a6041459c9791d732c01 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f57e6dc747158ab7ade4e62a75a6750d16b771e8 + e36e4d1a8f8dfb08d7e3a6041459c9791d732c01 @@ -619,9 +619,9 @@ aa61e8c20a869bcc994f8b29eb07d927d2bec6f4 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f57e6dc747158ab7ade4e62a75a6750d16b771e8 + e36e4d1a8f8dfb08d7e3a6041459c9791d732c01 https://github.com/dotnet/arcade-services diff --git a/eng/Versions.props b/eng/Versions.props index ec284b2e3be9..6b6934d1d129 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -86,47 +86,47 @@ - 9.0.4 - 9.0.4-servicing.25163.5 - 9.0.4 - 9.0.4 - 9.0.4-servicing.25163.5 - 9.0.4 - 9.0.4 - 9.0.4 - 9.0.4 - 9.0.4 - 9.0.4 + 9.0.5 + 9.0.5-servicing.25215.9 + 9.0.5 + 9.0.5 + 9.0.5-servicing.25215.9 + 9.0.5 + 9.0.5 + 9.0.5 + 9.0.5 + 9.0.5 + 9.0.5 8.0.0-rc.1.23414.4 - 9.0.4-servicing.25163.5 - 9.0.4-servicing.25163.5 - 9.0.4 - 9.0.4 - 9.0.4 - 9.0.4 + 9.0.5-servicing.25215.9 + 9.0.5-servicing.25215.9 + 9.0.5 + 9.0.5 + 9.0.5 + 9.0.5 2.1.0 - 9.0.4 + 9.0.5 8.0.0 - 9.0.4 - 9.0.4 - 9.0.4 - 9.0.4 - 9.0.4 - 9.0.4 - 9.0.4 + 9.0.5 + 9.0.5 + 9.0.5 + 9.0.5 + 9.0.5 + 9.0.5 + 9.0.5 8.0.0 - 9.0.4 - 9.0.4 - 9.0.4 - 9.0.4 - 9.0.4 - 9.0.4 - 9.0.4 + 9.0.5 + 9.0.5 + 9.0.5 + 9.0.5 + 9.0.5 + 9.0.5 + 9.0.5 8.0.4 - 9.0.4 - 9.0.4 + 9.0.5 + 9.0.5 @@ -317,8 +317,8 @@ 15.0.9617 18.0.9617 - 9.0.4-servicing.25157.2 - 9.0.4 + 9.0.5-servicing.25212.1 + 9.0.5 $(MicrosoftNETWorkloadEmscriptenCurrentManifest90100PackageVersion) 9.0.100$([System.Text.RegularExpressions.Regex]::Match($(EmscriptenWorkloadManifestVersion), `-(?!rtm)[A-z]*[\.]*\d*`)) From 00a18ce812bdc9e835e882638a39a19253e5e274 Mon Sep 17 00:00:00 2001 From: DotNet-Bot Date: Wed, 16 Apr 2025 23:29:10 +0000 Subject: [PATCH 11/57] Update dependencies from https://dev.azure.com/dnceng/internal/_git/dotnet-windowsdesktop build 20250416.2 Microsoft.WindowsDesktop.App.Ref , Microsoft.WindowsDesktop.App.Runtime.win-x64 , VS.Redist.Common.WindowsDesktop.SharedFramework.x64.9.0 , VS.Redist.Common.WindowsDesktop.TargetingPack.x64.9.0 From Version 9.0.4 -> To Version 9.0.5 Dependency coherency updates Microsoft.NET.Sdk.WindowsDesktop,Microsoft.Dotnet.WinForms.ProjectTemplates,Microsoft.DotNet.Wpf.ProjectTemplates From Version 9.0.4-rtm.25164.2 -> To Version 9.0.5-rtm.25216.2 (parent: Microsoft.WindowsDesktop.App.Ref --- NuGet.config | 2 ++ eng/Version.Details.xml | 28 ++++++++++++++-------------- eng/Versions.props | 14 +++++++------- 3 files changed, 23 insertions(+), 21 deletions(-) diff --git a/NuGet.config b/NuGet.config index c516984a145d..aa54dcc9708d 100644 --- a/NuGet.config +++ b/NuGet.config @@ -40,6 +40,7 @@ + @@ -74,6 +75,7 @@ + diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index db1ff1a26a51..d777e6b72c3d 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -254,26 +254,26 @@ https://dev.azure.com/dnceng/internal/_git/dotnet-runtime e36e4d1a8f8dfb08d7e3a6041459c9791d732c01 - + https://dev.azure.com/dnceng/internal/_git/dotnet-windowsdesktop - ef0ddf89991072267608f432ca2163ac54fdecab + b141ff423da53a66cda0be31dad3059317be52f6 - + https://dev.azure.com/dnceng/internal/_git/dotnet-windowsdesktop - ef0ddf89991072267608f432ca2163ac54fdecab + b141ff423da53a66cda0be31dad3059317be52f6 - + https://dev.azure.com/dnceng/internal/_git/dotnet-windowsdesktop - ef0ddf89991072267608f432ca2163ac54fdecab + b141ff423da53a66cda0be31dad3059317be52f6 - + https://dev.azure.com/dnceng/internal/_git/dotnet-windowsdesktop - ef0ddf89991072267608f432ca2163ac54fdecab + b141ff423da53a66cda0be31dad3059317be52f6 - + https://dev.azure.com/dnceng/internal/_git/dotnet-wpf - 33f5371cc258de0a72569adf1e9fbcbf04256a17 + 6df813d45ebffd2739980b7f372767ac5fd68e2a https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore @@ -390,13 +390,13 @@ - + https://dev.azure.com/dnceng/internal/_git/dotnet-winforms - ca42d9e58480ab4ebb845027c44c6b4f10e0539d + 711ee9fe3398d64b89f2ae70b94c66d9d495cf72 - + https://dev.azure.com/dnceng/internal/_git/dotnet-wpf - 33f5371cc258de0a72569adf1e9fbcbf04256a17 + 6df813d45ebffd2739980b7f372767ac5fd68e2a https://github.com/dotnet/xdt diff --git a/eng/Versions.props b/eng/Versions.props index 6b6934d1d129..b3590847bd47 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -77,7 +77,7 @@ - 9.0.4-servicing.25163.12 + 9.0.5-servicing.25216.1 @@ -130,10 +130,10 @@ - 9.0.4-servicing.25164.6 - 9.0.4-servicing.25164.6 - 9.0.4 - 9.0.4 + 9.0.5-servicing.25216.2 + 9.0.5-servicing.25216.2 + 9.0.5 + 9.0.5 @@ -237,8 +237,8 @@ - 9.0.4-rtm.25164.2 - 9.0.4-rtm.25164.2 + 9.0.5-rtm.25216.2 + 9.0.5-rtm.25216.2 From c508a2c45dd628973754a766bdd34022f9d1ee56 Mon Sep 17 00:00:00 2001 From: DotNet-Bot Date: Thu, 17 Apr 2025 02:03:46 +0000 Subject: [PATCH 12/57] Update dependencies from https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore build 20250416.7 dotnet-dev-certs , dotnet-user-jwts , dotnet-user-secrets , Microsoft.AspNetCore.Analyzers , Microsoft.AspNetCore.App.Ref , Microsoft.AspNetCore.App.Ref.Internal , Microsoft.AspNetCore.App.Runtime.win-x64 , Microsoft.AspNetCore.Authorization , Microsoft.AspNetCore.Components.SdkAnalyzers , Microsoft.AspNetCore.Components.Web , Microsoft.AspNetCore.DeveloperCertificates.XPlat , Microsoft.AspNetCore.Mvc.Analyzers , Microsoft.AspNetCore.Mvc.Api.Analyzers , Microsoft.AspNetCore.TestHost , Microsoft.Extensions.FileProviders.Embedded , Microsoft.Extensions.ObjectPool , Microsoft.JSInterop , VS.Redist.Common.AspNetCore.SharedFramework.x64.9.0 , Microsoft.SourceBuild.Intermediate.aspnetcore From Version 9.0.4-servicing.25164.3 -> To Version 9.0.5-servicing.25216.7 --- NuGet.config | 2 ++ eng/Version.Details.xml | 76 ++++++++++++++++++++--------------------- eng/Versions.props | 26 +++++++------- 3 files changed, 53 insertions(+), 51 deletions(-) diff --git a/NuGet.config b/NuGet.config index aa54dcc9708d..f202d4006ad5 100644 --- a/NuGet.config +++ b/NuGet.config @@ -27,6 +27,7 @@ + @@ -67,6 +68,7 @@ + diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index d777e6b72c3d..0d75b5e85184 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -135,13 +135,13 @@ https://github.com/dotnet/roslyn 3f5cf9fbbd91f2047e988801a5142ca1cb6bab45 - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - d5dc8a13cc618b9cbdc1e5744b4806c594d49553 + 4b607610667282fef8960c479dfb7561b88d238d - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - d5dc8a13cc618b9cbdc1e5744b4806c594d49553 + 4b607610667282fef8960c479dfb7561b88d238d https://github.com/nuget/nuget.client @@ -275,54 +275,54 @@ https://dev.azure.com/dnceng/internal/_git/dotnet-wpf 6df813d45ebffd2739980b7f372767ac5fd68e2a - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - d5dc8a13cc618b9cbdc1e5744b4806c594d49553 + 4b607610667282fef8960c479dfb7561b88d238d - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - d5dc8a13cc618b9cbdc1e5744b4806c594d49553 + 4b607610667282fef8960c479dfb7561b88d238d - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - d5dc8a13cc618b9cbdc1e5744b4806c594d49553 + 4b607610667282fef8960c479dfb7561b88d238d - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - d5dc8a13cc618b9cbdc1e5744b4806c594d49553 + 4b607610667282fef8960c479dfb7561b88d238d - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - d5dc8a13cc618b9cbdc1e5744b4806c594d49553 + 4b607610667282fef8960c479dfb7561b88d238d - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - d5dc8a13cc618b9cbdc1e5744b4806c594d49553 + 4b607610667282fef8960c479dfb7561b88d238d - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - d5dc8a13cc618b9cbdc1e5744b4806c594d49553 + 4b607610667282fef8960c479dfb7561b88d238d - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - d5dc8a13cc618b9cbdc1e5744b4806c594d49553 + 4b607610667282fef8960c479dfb7561b88d238d - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - d5dc8a13cc618b9cbdc1e5744b4806c594d49553 + 4b607610667282fef8960c479dfb7561b88d238d - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - d5dc8a13cc618b9cbdc1e5744b4806c594d49553 + 4b607610667282fef8960c479dfb7561b88d238d - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - d5dc8a13cc618b9cbdc1e5744b4806c594d49553 + 4b607610667282fef8960c479dfb7561b88d238d - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - d5dc8a13cc618b9cbdc1e5744b4806c594d49553 + 4b607610667282fef8960c479dfb7561b88d238d @@ -343,21 +343,21 @@ 44568f7817cd7c6daf90c9340c64c3b69277b7f6 - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - d5dc8a13cc618b9cbdc1e5744b4806c594d49553 + 4b607610667282fef8960c479dfb7561b88d238d - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - d5dc8a13cc618b9cbdc1e5744b4806c594d49553 + 4b607610667282fef8960c479dfb7561b88d238d - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - d5dc8a13cc618b9cbdc1e5744b4806c594d49553 + 4b607610667282fef8960c479dfb7561b88d238d - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - d5dc8a13cc618b9cbdc1e5744b4806c594d49553 + 4b607610667282fef8960c479dfb7561b88d238d https://github.com/dotnet/test-templates @@ -539,9 +539,9 @@ https://dev.azure.com/dnceng/internal/_git/dotnet-runtime e36e4d1a8f8dfb08d7e3a6041459c9791d732c01 - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - d5dc8a13cc618b9cbdc1e5744b4806c594d49553 + 4b607610667282fef8960c479dfb7561b88d238d https://dev.azure.com/dnceng/internal/_git/dotnet-runtime diff --git a/eng/Versions.props b/eng/Versions.props index b3590847bd47..4df43e559ff4 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -215,19 +215,19 @@ - 9.0.4 - 9.0.4-servicing.25164.3 - 9.0.4-servicing.25164.3 - 9.0.4-servicing.25164.3 - 9.0.4-servicing.25164.3 - 9.0.4-servicing.25164.3 - 9.0.4 - 9.0.4 - 9.0.4-servicing.25164.3 - 9.0.4-servicing.25164.3 - 9.0.4-servicing.25164.3 - 9.0.4-servicing.25164.3 - 9.0.4-servicing.25164.3 + 9.0.5 + 9.0.5-servicing.25216.7 + 9.0.5-servicing.25216.7 + 9.0.5-servicing.25216.7 + 9.0.5-servicing.25216.7 + 9.0.5-servicing.25216.7 + 9.0.5 + 9.0.5 + 9.0.5-servicing.25216.7 + 9.0.5-servicing.25216.7 + 9.0.5-servicing.25216.7 + 9.0.5-servicing.25216.7 + 9.0.5-servicing.25216.7 From 91e5dcdae0c05f3e1d75a91a92ac6537ab96d74b Mon Sep 17 00:00:00 2001 From: "Sean Reeser (CSI Interfusion Inc)" Date: Thu, 17 Apr 2025 23:15:19 +0000 Subject: [PATCH 13/57] Updated NuGet.config - replace msbuild feed --- NuGet.config | 2 ++ 1 file changed, 2 insertions(+) diff --git a/NuGet.config b/NuGet.config index f202d4006ad5..35cf7709b385 100644 --- a/NuGet.config +++ b/NuGet.config @@ -30,6 +30,7 @@ + @@ -70,6 +71,7 @@ + From 3ee944537a58f61e4697a50502fbc23dd160b58a Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Fri, 18 Apr 2025 01:07:22 +0000 Subject: [PATCH 14/57] Update dependencies from https://github.com/dotnet/templating build 20250417.3 Microsoft.SourceBuild.Intermediate.templating , Microsoft.TemplateEngine.Abstractions , Microsoft.TemplateEngine.Mocks From Version 9.0.106-servicing.25215.3 -> To Version 9.0.106-servicing.25217.3 --- NuGet.config | 2 +- eng/Version.Details.xml | 10 +++++----- eng/Versions.props | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/NuGet.config b/NuGet.config index 3bea35ae691e..b286902fc015 100644 --- a/NuGet.config +++ b/NuGet.config @@ -35,7 +35,7 @@ - + diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index b54b7b05eb58..603468e33b5f 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,16 +3,16 @@ https://github.com/dotnet/templating - 94bc1a8621167807199e338a3ef2a18162e3bc37 + a0bc400b24c00478ed52d45fb72763c93f30d693 - + https://github.com/dotnet/templating - 94bc1a8621167807199e338a3ef2a18162e3bc37 + a0bc400b24c00478ed52d45fb72763c93f30d693 - + https://github.com/dotnet/templating - 94bc1a8621167807199e338a3ef2a18162e3bc37 + a0bc400b24c00478ed52d45fb72763c93f30d693 diff --git a/eng/Versions.props b/eng/Versions.props index 698a09ec9e8e..ce1989925058 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -193,7 +193,7 @@ $(MicrosoftTemplateEngineAbstractionsPackageVersion) $(MicrosoftTemplateEngineAbstractionsPackageVersion) - 9.0.106-servicing.25215.3 + 9.0.106-servicing.25217.3 $(MicrosoftTemplateEngineMocksPackageVersion) $(MicrosoftTemplateEngineAbstractionsPackageVersion) $(MicrosoftTemplateEngineMocksPackageVersion) From 0a17366c7e027f2e02bf5bb947e80e577ffae109 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Thu, 24 Apr 2025 05:01:39 +0000 Subject: [PATCH 15/57] Update dependencies from https://github.com/dotnet/source-build-reference-packages build 20250423.3 Microsoft.SourceBuild.Intermediate.source-build-reference-packages From Version 9.0.0-alpha.1.25209.1 -> To Version 9.0.0-alpha.1.25223.3 --- NuGet.config | 2 ++ eng/Version.Details.xml | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/NuGet.config b/NuGet.config index b286902fc015..f221c730c602 100644 --- a/NuGet.config +++ b/NuGet.config @@ -36,6 +36,8 @@ + + diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 603468e33b5f..b4466a6d2698 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -451,9 +451,9 @@ - + https://github.com/dotnet/source-build-reference-packages - 7dbf5deea5bdccf513df73cba179c4c0ad106010 + 19eb5ea4e5f9c4e5256843a92805c8c9e942207d From 80d02849972f7d085a33e8acfd54f82da6b52e71 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Thu, 24 Apr 2025 15:52:36 +0000 Subject: [PATCH 16/57] Update dependencies from https://github.com/dotnet/scenario-tests build 20250424.2 Microsoft.SourceBuild.Intermediate.scenario-tests , Microsoft.DotNet.ScenarioTests.SdkTemplateTests From Version 9.0.0-preview.25163.1 -> To Version 9.0.0-preview.25224.2 --- NuGet.config | 2 ++ eng/Version.Details.xml | 8 ++++---- eng/Versions.props | 2 +- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/NuGet.config b/NuGet.config index b286902fc015..f221c730c602 100644 --- a/NuGet.config +++ b/NuGet.config @@ -36,6 +36,8 @@ + + diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 603468e33b5f..738c23024753 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -631,14 +631,14 @@ https://github.com/dotnet/arcade-services 47e3672c762970073e4282bd563233da86bcca3e - + https://github.com/dotnet/scenario-tests - 75f784a5484a2d4e00905069160954f686dcc8e3 + a79c5c5153234603f72c43ac078cd613a2247eff - + https://github.com/dotnet/scenario-tests - 75f784a5484a2d4e00905069160954f686dcc8e3 + a79c5c5153234603f72c43ac078cd613a2247eff From 0e1448ca01c59dd5d1fd54d80df6227219b7c12a Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Sat, 26 Apr 2025 02:01:53 +0000 Subject: [PATCH 17/57] Update dependencies from https://github.com/dotnet/arcade build 20250425.5 Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.Helix.Sdk , Microsoft.DotNet.SignTool , Microsoft.DotNet.XUnitExtensions From Version 8.0.0-beta.25214.7 -> To Version 8.0.0-beta.25225.5 --- NuGet.config | 4 ++++ eng/Version.Details.xml | 16 ++++++++-------- eng/Versions.props | 4 ++-- .../templates-official/job/source-build.yml | 2 +- .../job/source-index-stage1.yml | 4 ++-- eng/common/templates/job/source-build.yml | 4 ++-- eng/common/templates/job/source-index-stage1.yml | 4 ++-- global.json | 4 ++-- 8 files changed, 23 insertions(+), 19 deletions(-) diff --git a/NuGet.config b/NuGet.config index b02836bc40ef..593c5da667d9 100644 --- a/NuGet.config +++ b/NuGet.config @@ -7,6 +7,8 @@ + + @@ -19,6 +21,8 @@ + + diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 673c2fb15f12..3c9dcb712a75 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -491,22 +491,22 @@ - + https://github.com/dotnet/arcade - c487e860d456cda2580600ad81fd425d3bba21f7 + d3c4c4644a5af855cd30944d9221886369ab375d - + https://github.com/dotnet/arcade - c487e860d456cda2580600ad81fd425d3bba21f7 + d3c4c4644a5af855cd30944d9221886369ab375d - + https://github.com/dotnet/arcade - c487e860d456cda2580600ad81fd425d3bba21f7 + d3c4c4644a5af855cd30944d9221886369ab375d - + https://github.com/dotnet/arcade - c487e860d456cda2580600ad81fd425d3bba21f7 + d3c4c4644a5af855cd30944d9221886369ab375d https://dev.azure.com/dnceng/internal/_git/dotnet-runtime diff --git a/eng/Versions.props b/eng/Versions.props index c217eab2fcd9..47fd542793a5 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -36,7 +36,7 @@ 8.0.0 4.0.0 8.0.0 - 8.0.0-beta.25214.7 + 8.0.0-beta.25225.5 7.0.0-preview.22423.2 8.0.0 4.3.0 @@ -213,7 +213,7 @@ 6.12.0 6.1.0 - 8.0.0-beta.25214.7 + 8.0.0-beta.25225.5 4.18.4 1.3.2 8.0.0-beta.23607.1 diff --git a/eng/common/templates-official/job/source-build.yml b/eng/common/templates-official/job/source-build.yml index f983033bb028..4217d6d8b148 100644 --- a/eng/common/templates-official/job/source-build.yml +++ b/eng/common/templates-official/job/source-build.yml @@ -54,7 +54,7 @@ jobs: pool: ${{ if eq(variables['System.TeamProject'], 'public') }}: name: $[replace(replace(eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), True, 'NetCore-Svc-Public' ), False, 'NetCore-Public')] - demands: ImageOverride -equals Build.Ubuntu.1804.Amd64.Open + demands: ImageOverride -equals Build.Ubuntu.2204.Amd64.Open ${{ if eq(variables['System.TeamProject'], 'internal') }}: name: $[replace(replace(eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), True, 'NetCore1ESPool-Svc-Internal'), False, 'NetCore1ESPool-Internal')] diff --git a/eng/common/templates-official/job/source-index-stage1.yml b/eng/common/templates-official/job/source-index-stage1.yml index 60dfb6b2d1c0..fb632b71a250 100644 --- a/eng/common/templates-official/job/source-index-stage1.yml +++ b/eng/common/templates-official/job/source-index-stage1.yml @@ -1,7 +1,7 @@ parameters: runAsPublic: false - sourceIndexUploadPackageVersion: 2.0.0-20240502.12 - sourceIndexProcessBinlogPackageVersion: 1.0.1-20240129.2 + sourceIndexUploadPackageVersion: 2.0.0-20250425.2 + sourceIndexProcessBinlogPackageVersion: 1.0.1-20250425.2 sourceIndexPackageSource: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json sourceIndexBuildCommand: powershell -NoLogo -NoProfile -ExecutionPolicy Bypass -Command "eng/common/build.ps1 -restore -build -binarylog -ci" preSteps: [] diff --git a/eng/common/templates/job/source-build.yml b/eng/common/templates/job/source-build.yml index c0ff472b697b..c48f95d93d91 100644 --- a/eng/common/templates/job/source-build.yml +++ b/eng/common/templates/job/source-build.yml @@ -54,11 +54,11 @@ jobs: pool: ${{ if eq(variables['System.TeamProject'], 'public') }}: name: $[replace(replace(eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), True, 'NetCore-Svc-Public' ), False, 'NetCore-Public')] - demands: ImageOverride -equals Build.Ubuntu.1804.Amd64.Open + demands: ImageOverride -equals Build.Ubuntu.2204.Amd64.Open ${{ if eq(variables['System.TeamProject'], 'internal') }}: name: $[replace(replace(eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), True, 'NetCore1ESPool-Svc-Internal'), False, 'NetCore1ESPool-Internal')] - demands: ImageOverride -equals Build.Ubuntu.1804.Amd64 + demands: ImageOverride -equals Build.Ubuntu.2204.Amd64 ${{ if ne(parameters.platform.pool, '') }}: pool: ${{ parameters.platform.pool }} diff --git a/eng/common/templates/job/source-index-stage1.yml b/eng/common/templates/job/source-index-stage1.yml index 0b6bb89dc78a..8538f44bab28 100644 --- a/eng/common/templates/job/source-index-stage1.yml +++ b/eng/common/templates/job/source-index-stage1.yml @@ -1,7 +1,7 @@ parameters: runAsPublic: false - sourceIndexUploadPackageVersion: 2.0.0-20240502.12 - sourceIndexProcessBinlogPackageVersion: 1.0.1-20240129.2 + sourceIndexUploadPackageVersion: 2.0.0-20250425.2 + sourceIndexProcessBinlogPackageVersion: 1.0.1-20250425.2 sourceIndexPackageSource: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json sourceIndexBuildCommand: powershell -NoLogo -NoProfile -ExecutionPolicy Bypass -Command "eng/common/build.ps1 -restore -build -binarylog -ci" preSteps: [] diff --git a/global.json b/global.json index 5208b63f114e..8395681dbeb6 100644 --- a/global.json +++ b/global.json @@ -14,7 +14,7 @@ } }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.25214.7", - "Microsoft.DotNet.Helix.Sdk": "8.0.0-beta.25214.7" + "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.25225.5", + "Microsoft.DotNet.Helix.Sdk": "8.0.0-beta.25225.5" } } From 3a466fe867af47925a91357844b4674bb2f85415 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Sun, 27 Apr 2025 02:02:16 +0000 Subject: [PATCH 18/57] Update dependencies from https://github.com/dotnet/arcade build 20250425.6 Microsoft.SourceBuild.Intermediate.arcade , Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.Build.Tasks.Installers , Microsoft.DotNet.Helix.Sdk , Microsoft.DotNet.SignTool , Microsoft.DotNet.XliffTasks , Microsoft.DotNet.XUnitExtensions From Version 9.0.0-beta.25208.6 -> To Version 9.0.0-beta.25225.6 --- NuGet.config | 2 ++ eng/Version.Details.xml | 28 +++++++++---------- eng/Versions.props | 8 +++--- .../core-templates/job/source-build.yml | 2 ++ .../job/source-index-stage1.yml | 4 +-- .../core-templates/steps/source-build.yml | 1 + eng/common/sdl/packages.config | 2 +- global.json | 4 +-- 8 files changed, 28 insertions(+), 23 deletions(-) diff --git a/NuGet.config b/NuGet.config index b286902fc015..f221c730c602 100644 --- a/NuGet.config +++ b/NuGet.config @@ -36,6 +36,8 @@ + + diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 603468e33b5f..b558534b2aa0 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -589,34 +589,34 @@ - + https://github.com/dotnet/arcade - aa61e8c20a869bcc994f8b29eb07d927d2bec6f4 + bfbc858ba868b60fffaf7b2150f1d2165b01e786 - + https://github.com/dotnet/arcade - aa61e8c20a869bcc994f8b29eb07d927d2bec6f4 + bfbc858ba868b60fffaf7b2150f1d2165b01e786 - + https://github.com/dotnet/arcade - aa61e8c20a869bcc994f8b29eb07d927d2bec6f4 + bfbc858ba868b60fffaf7b2150f1d2165b01e786 - + https://github.com/dotnet/arcade - aa61e8c20a869bcc994f8b29eb07d927d2bec6f4 + bfbc858ba868b60fffaf7b2150f1d2165b01e786 - + https://github.com/dotnet/arcade - aa61e8c20a869bcc994f8b29eb07d927d2bec6f4 + bfbc858ba868b60fffaf7b2150f1d2165b01e786 - + https://github.com/dotnet/arcade - aa61e8c20a869bcc994f8b29eb07d927d2bec6f4 + bfbc858ba868b60fffaf7b2150f1d2165b01e786 - + https://github.com/dotnet/arcade - aa61e8c20a869bcc994f8b29eb07d927d2bec6f4 + bfbc858ba868b60fffaf7b2150f1d2165b01e786 diff --git a/eng/Versions.props b/eng/Versions.props index ce1989925058..2e34dcec89fe 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -270,10 +270,10 @@ - 9.0.0-beta.25208.6 - 9.0.0-beta.25208.6 - 9.0.0-beta.25208.6 - 9.0.0-beta.25208.6 + 9.0.0-beta.25225.6 + 9.0.0-beta.25225.6 + 9.0.0-beta.25225.6 + 9.0.0-beta.25225.6 diff --git a/eng/common/core-templates/job/source-build.yml b/eng/common/core-templates/job/source-build.yml index c4713c8b6ede..d47f09d58fd9 100644 --- a/eng/common/core-templates/job/source-build.yml +++ b/eng/common/core-templates/job/source-build.yml @@ -26,6 +26,8 @@ parameters: # Specifies the build script to invoke to perform the build in the repo. The default # './build.sh' should work for typical Arcade repositories, but this is customizable for # difficult situations. + # buildArguments: '' + # Specifies additional build arguments to pass to the build script. # jobProperties: {} # A list of job properties to inject at the top level, for potential extensibility beyond # container and pool. diff --git a/eng/common/core-templates/job/source-index-stage1.yml b/eng/common/core-templates/job/source-index-stage1.yml index 205fb5b3a395..8b833332b3ee 100644 --- a/eng/common/core-templates/job/source-index-stage1.yml +++ b/eng/common/core-templates/job/source-index-stage1.yml @@ -1,7 +1,7 @@ parameters: runAsPublic: false - sourceIndexUploadPackageVersion: 2.0.0-20240522.1 - sourceIndexProcessBinlogPackageVersion: 1.0.1-20240522.1 + sourceIndexUploadPackageVersion: 2.0.0-20250425.2 + sourceIndexProcessBinlogPackageVersion: 1.0.1-20250425.2 sourceIndexPackageSource: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json sourceIndexBuildCommand: powershell -NoLogo -NoProfile -ExecutionPolicy Bypass -Command "eng/common/build.ps1 -restore -build -binarylog -ci" preSteps: [] diff --git a/eng/common/core-templates/steps/source-build.yml b/eng/common/core-templates/steps/source-build.yml index 2915d29bb7f6..37133b55b754 100644 --- a/eng/common/core-templates/steps/source-build.yml +++ b/eng/common/core-templates/steps/source-build.yml @@ -79,6 +79,7 @@ steps: ${{ coalesce(parameters.platform.buildScript, './build.sh') }} --ci \ --configuration $buildConfig \ --restore --build --pack $publishArgs -bl \ + ${{ parameters.platform.buildArguments }} \ $officialBuildArgs \ $internalRuntimeDownloadArgs \ $internalRestoreArgs \ diff --git a/eng/common/sdl/packages.config b/eng/common/sdl/packages.config index e5f543ea68c2..4585cfd6bba1 100644 --- a/eng/common/sdl/packages.config +++ b/eng/common/sdl/packages.config @@ -1,4 +1,4 @@ - + diff --git a/global.json b/global.json index aeb83564c306..c13f07c210e0 100644 --- a/global.json +++ b/global.json @@ -17,8 +17,8 @@ "cmake": "latest" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "9.0.0-beta.25208.6", - "Microsoft.DotNet.Helix.Sdk": "9.0.0-beta.25208.6", + "Microsoft.DotNet.Arcade.Sdk": "9.0.0-beta.25225.6", + "Microsoft.DotNet.Helix.Sdk": "9.0.0-beta.25225.6", "Microsoft.Build.NoTargets": "3.7.0", "Microsoft.DotNet.CMake.Sdk": "9.0.0-beta.24217.1" } From cb93b17433755fde4211d6b9c69ce67df93a8bd8 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Tue, 29 Apr 2025 02:02:35 +0000 Subject: [PATCH 19/57] Update dependencies from https://github.com/dotnet/arcade build 20250428.2 Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.Helix.Sdk , Microsoft.DotNet.SignTool , Microsoft.DotNet.XUnitExtensions From Version 8.0.0-beta.25214.7 -> To Version 8.0.0-beta.25228.2 --- eng/Version.Details.xml | 16 ++++++++-------- eng/Versions.props | 4 ++-- global.json | 4 ++-- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 3c9dcb712a75..6121953f242c 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -491,22 +491,22 @@ - + https://github.com/dotnet/arcade - d3c4c4644a5af855cd30944d9221886369ab375d + eaaefe0c7168235bf8fd93d62c6a053ee5c25ec0 - + https://github.com/dotnet/arcade - d3c4c4644a5af855cd30944d9221886369ab375d + eaaefe0c7168235bf8fd93d62c6a053ee5c25ec0 - + https://github.com/dotnet/arcade - d3c4c4644a5af855cd30944d9221886369ab375d + eaaefe0c7168235bf8fd93d62c6a053ee5c25ec0 - + https://github.com/dotnet/arcade - d3c4c4644a5af855cd30944d9221886369ab375d + eaaefe0c7168235bf8fd93d62c6a053ee5c25ec0 https://dev.azure.com/dnceng/internal/_git/dotnet-runtime diff --git a/eng/Versions.props b/eng/Versions.props index 47fd542793a5..5c85a8fdff9c 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -36,7 +36,7 @@ 8.0.0 4.0.0 8.0.0 - 8.0.0-beta.25225.5 + 8.0.0-beta.25228.2 7.0.0-preview.22423.2 8.0.0 4.3.0 @@ -213,7 +213,7 @@ 6.12.0 6.1.0 - 8.0.0-beta.25225.5 + 8.0.0-beta.25228.2 4.18.4 1.3.2 8.0.0-beta.23607.1 diff --git a/global.json b/global.json index 8395681dbeb6..e59cdf9b4461 100644 --- a/global.json +++ b/global.json @@ -14,7 +14,7 @@ } }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.25225.5", - "Microsoft.DotNet.Helix.Sdk": "8.0.0-beta.25225.5" + "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.25228.2", + "Microsoft.DotNet.Helix.Sdk": "8.0.0-beta.25228.2" } } From 861ae5aae6ac75570dabcef0f41118b9b875fb8d Mon Sep 17 00:00:00 2001 From: DotNet-Bot Date: Tue, 29 Apr 2025 22:15:30 +0000 Subject: [PATCH 20/57] Update dependencies from https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore build 20250429.4 dotnet-dev-certs , dotnet-user-jwts , dotnet-user-secrets , Microsoft.AspNetCore.Analyzers , Microsoft.AspNetCore.App.Ref , Microsoft.AspNetCore.App.Ref.Internal , Microsoft.AspNetCore.App.Runtime.win-x64 , Microsoft.AspNetCore.Authorization , Microsoft.AspNetCore.Components.SdkAnalyzers , Microsoft.AspNetCore.Components.Web , Microsoft.AspNetCore.DeveloperCertificates.XPlat , Microsoft.AspNetCore.Mvc.Analyzers , Microsoft.AspNetCore.Mvc.Api.Analyzers , Microsoft.AspNetCore.TestHost , Microsoft.Extensions.FileProviders.Embedded , Microsoft.Extensions.ObjectPool , Microsoft.JSInterop , VS.Redist.Common.AspNetCore.SharedFramework.x64.9.0 , Microsoft.SourceBuild.Intermediate.aspnetcore From Version 9.0.5-servicing.25216.7 -> To Version 9.0.5-servicing.25229.4 --- NuGet.config | 30 ++++++++++++++++++--- eng/Version.Details.xml | 60 ++++++++++++++++++++--------------------- eng/Versions.props | 20 +++++++------- 3 files changed, 66 insertions(+), 44 deletions(-) diff --git a/NuGet.config b/NuGet.config index b19776c18353..cf5f54b34784 100644 --- a/NuGet.config +++ b/NuGet.config @@ -25,24 +25,39 @@ + + + + - + - + + + + + + + + + + + + @@ -69,16 +84,23 @@ - + - + + + + + + + + diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index c74f9bbdae7c..ac622a151eb1 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -135,13 +135,13 @@ https://github.com/dotnet/roslyn 3f5cf9fbbd91f2047e988801a5142ca1cb6bab45 - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - 4b607610667282fef8960c479dfb7561b88d238d + ed74665e773dd1ebea3289c5662d71c590305932 https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - 4b607610667282fef8960c479dfb7561b88d238d + ed74665e773dd1ebea3289c5662d71c590305932 https://github.com/nuget/nuget.client @@ -277,52 +277,52 @@ https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - 4b607610667282fef8960c479dfb7561b88d238d + ed74665e773dd1ebea3289c5662d71c590305932 - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - 4b607610667282fef8960c479dfb7561b88d238d + ed74665e773dd1ebea3289c5662d71c590305932 https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - 4b607610667282fef8960c479dfb7561b88d238d + ed74665e773dd1ebea3289c5662d71c590305932 - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - 4b607610667282fef8960c479dfb7561b88d238d + ed74665e773dd1ebea3289c5662d71c590305932 - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - 4b607610667282fef8960c479dfb7561b88d238d + ed74665e773dd1ebea3289c5662d71c590305932 - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - 4b607610667282fef8960c479dfb7561b88d238d + ed74665e773dd1ebea3289c5662d71c590305932 - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - 4b607610667282fef8960c479dfb7561b88d238d + ed74665e773dd1ebea3289c5662d71c590305932 - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - 4b607610667282fef8960c479dfb7561b88d238d + ed74665e773dd1ebea3289c5662d71c590305932 - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - 4b607610667282fef8960c479dfb7561b88d238d + ed74665e773dd1ebea3289c5662d71c590305932 - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - 4b607610667282fef8960c479dfb7561b88d238d + ed74665e773dd1ebea3289c5662d71c590305932 - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - 4b607610667282fef8960c479dfb7561b88d238d + ed74665e773dd1ebea3289c5662d71c590305932 - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - 4b607610667282fef8960c479dfb7561b88d238d + ed74665e773dd1ebea3289c5662d71c590305932 @@ -345,19 +345,19 @@ https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - 4b607610667282fef8960c479dfb7561b88d238d + ed74665e773dd1ebea3289c5662d71c590305932 https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - 4b607610667282fef8960c479dfb7561b88d238d + ed74665e773dd1ebea3289c5662d71c590305932 https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - 4b607610667282fef8960c479dfb7561b88d238d + ed74665e773dd1ebea3289c5662d71c590305932 https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - 4b607610667282fef8960c479dfb7561b88d238d + ed74665e773dd1ebea3289c5662d71c590305932 https://github.com/dotnet/test-templates @@ -541,7 +541,7 @@ https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - 4b607610667282fef8960c479dfb7561b88d238d + ed74665e773dd1ebea3289c5662d71c590305932 https://dev.azure.com/dnceng/internal/_git/dotnet-runtime diff --git a/eng/Versions.props b/eng/Versions.props index c95f7b5c5130..b3cd95893e22 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -216,18 +216,18 @@ 9.0.5 - 9.0.5-servicing.25216.7 - 9.0.5-servicing.25216.7 - 9.0.5-servicing.25216.7 - 9.0.5-servicing.25216.7 - 9.0.5-servicing.25216.7 + 9.0.5-servicing.25229.4 + 9.0.5-servicing.25229.4 + 9.0.5-servicing.25229.4 + 9.0.5-servicing.25229.4 + 9.0.5-servicing.25229.4 9.0.5 9.0.5 - 9.0.5-servicing.25216.7 - 9.0.5-servicing.25216.7 - 9.0.5-servicing.25216.7 - 9.0.5-servicing.25216.7 - 9.0.5-servicing.25216.7 + 9.0.5-servicing.25229.4 + 9.0.5-servicing.25229.4 + 9.0.5-servicing.25229.4 + 9.0.5-servicing.25229.4 + 9.0.5-servicing.25229.4 From 492fc39eaf5c6657e7ea880aa326ab2a3ef043d8 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Wed, 30 Apr 2025 02:02:35 +0000 Subject: [PATCH 21/57] Update dependencies from https://github.com/dotnet/arcade build 20250429.5 Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.Helix.Sdk , Microsoft.DotNet.SignTool , Microsoft.DotNet.XUnitExtensions From Version 8.0.0-beta.25214.7 -> To Version 8.0.0-beta.25229.5 --- eng/Version.Details.xml | 16 ++++++++-------- eng/Versions.props | 4 ++-- .../templates-official/jobs/source-build.yml | 2 +- eng/common/templates/jobs/source-build.yml | 2 +- global.json | 4 ++-- 5 files changed, 14 insertions(+), 14 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 6121953f242c..1b85e3d5ce9b 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -491,22 +491,22 @@ - + https://github.com/dotnet/arcade - eaaefe0c7168235bf8fd93d62c6a053ee5c25ec0 + 6448619bfaaef14e8787f27d94b537d7ec73f36f - + https://github.com/dotnet/arcade - eaaefe0c7168235bf8fd93d62c6a053ee5c25ec0 + 6448619bfaaef14e8787f27d94b537d7ec73f36f - + https://github.com/dotnet/arcade - eaaefe0c7168235bf8fd93d62c6a053ee5c25ec0 + 6448619bfaaef14e8787f27d94b537d7ec73f36f - + https://github.com/dotnet/arcade - eaaefe0c7168235bf8fd93d62c6a053ee5c25ec0 + 6448619bfaaef14e8787f27d94b537d7ec73f36f https://dev.azure.com/dnceng/internal/_git/dotnet-runtime diff --git a/eng/Versions.props b/eng/Versions.props index 5c85a8fdff9c..bd08a6ffe5ac 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -36,7 +36,7 @@ 8.0.0 4.0.0 8.0.0 - 8.0.0-beta.25228.2 + 8.0.0-beta.25229.5 7.0.0-preview.22423.2 8.0.0 4.3.0 @@ -213,7 +213,7 @@ 6.12.0 6.1.0 - 8.0.0-beta.25228.2 + 8.0.0-beta.25229.5 4.18.4 1.3.2 8.0.0-beta.23607.1 diff --git a/eng/common/templates-official/jobs/source-build.yml b/eng/common/templates-official/jobs/source-build.yml index 5cf6a269c0b6..b9247be1547b 100644 --- a/eng/common/templates-official/jobs/source-build.yml +++ b/eng/common/templates-official/jobs/source-build.yml @@ -14,7 +14,7 @@ parameters: # This is the default platform provided by Arcade, intended for use by a managed-only repo. defaultManagedPlatform: name: 'Managed' - container: 'mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream8' + container: 'mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream-9-amd64' # Defines the platforms on which to run build jobs. One job is created for each platform, and the # object in this array is sent to the job template as 'platform'. If no platforms are specified, diff --git a/eng/common/templates/jobs/source-build.yml b/eng/common/templates/jobs/source-build.yml index 5f46bfa895c1..3ec997108107 100644 --- a/eng/common/templates/jobs/source-build.yml +++ b/eng/common/templates/jobs/source-build.yml @@ -14,7 +14,7 @@ parameters: # This is the default platform provided by Arcade, intended for use by a managed-only repo. defaultManagedPlatform: name: 'Managed' - container: 'mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream8' + container: 'mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream-9-amd64' # Defines the platforms on which to run build jobs. One job is created for each platform, and the # object in this array is sent to the job template as 'platform'. If no platforms are specified, diff --git a/global.json b/global.json index e59cdf9b4461..0d057253d827 100644 --- a/global.json +++ b/global.json @@ -14,7 +14,7 @@ } }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.25228.2", - "Microsoft.DotNet.Helix.Sdk": "8.0.0-beta.25228.2" + "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.25229.5", + "Microsoft.DotNet.Helix.Sdk": "8.0.0-beta.25229.5" } } From c3bbc864264eea1555769c37ba99a6832cb51b9a Mon Sep 17 00:00:00 2001 From: "Sean Reeser (CSI Interfusion Inc)" Date: Wed, 30 Apr 2025 02:49:45 +0000 Subject: [PATCH 22/57] Updated NuGet.config - replace msbuild feed --- NuGet.config | 1 + 1 file changed, 1 insertion(+) diff --git a/NuGet.config b/NuGet.config index cf5f54b34784..a05f2119f167 100644 --- a/NuGet.config +++ b/NuGet.config @@ -86,6 +86,7 @@ + From 96da6c3d0e86fe416a6aed637d46b8af5062c3d2 Mon Sep 17 00:00:00 2001 From: "Sean Reeser (CSI Interfusion Inc)" Date: Wed, 30 Apr 2025 02:51:41 +0000 Subject: [PATCH 23/57] Updated NuGet.config replace msbuild --- NuGet.config | 1 + 1 file changed, 1 insertion(+) diff --git a/NuGet.config b/NuGet.config index a05f2119f167..fd2c5f1f2511 100644 --- a/NuGet.config +++ b/NuGet.config @@ -34,6 +34,7 @@ + From 570ce8f629deaaca9a8fa4fbf381ca0b7f694e77 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Thu, 1 May 2025 02:02:30 +0000 Subject: [PATCH 24/57] Update dependencies from https://github.com/dotnet/arcade build 20250430.1 Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.Helix.Sdk , Microsoft.DotNet.SignTool , Microsoft.DotNet.XUnitExtensions From Version 8.0.0-beta.25214.7 -> To Version 8.0.0-beta.25230.1 --- NuGet.config | 6 ++++++ eng/Version.Details.xml | 16 ++++++++-------- eng/Versions.props | 4 ++-- global.json | 4 ++-- 4 files changed, 18 insertions(+), 12 deletions(-) diff --git a/NuGet.config b/NuGet.config index 593c5da667d9..de1573a96114 100644 --- a/NuGet.config +++ b/NuGet.config @@ -7,6 +7,9 @@ + + + @@ -21,6 +24,9 @@ + + + diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 1b85e3d5ce9b..75c89e8910de 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -491,22 +491,22 @@ - + https://github.com/dotnet/arcade - 6448619bfaaef14e8787f27d94b537d7ec73f36f + 4246a31e5de9de87f760218c4f588cebf4661f45 - + https://github.com/dotnet/arcade - 6448619bfaaef14e8787f27d94b537d7ec73f36f + 4246a31e5de9de87f760218c4f588cebf4661f45 - + https://github.com/dotnet/arcade - 6448619bfaaef14e8787f27d94b537d7ec73f36f + 4246a31e5de9de87f760218c4f588cebf4661f45 - + https://github.com/dotnet/arcade - 6448619bfaaef14e8787f27d94b537d7ec73f36f + 4246a31e5de9de87f760218c4f588cebf4661f45 https://dev.azure.com/dnceng/internal/_git/dotnet-runtime diff --git a/eng/Versions.props b/eng/Versions.props index bd08a6ffe5ac..25b15aeee62b 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -36,7 +36,7 @@ 8.0.0 4.0.0 8.0.0 - 8.0.0-beta.25229.5 + 8.0.0-beta.25230.1 7.0.0-preview.22423.2 8.0.0 4.3.0 @@ -213,7 +213,7 @@ 6.12.0 6.1.0 - 8.0.0-beta.25229.5 + 8.0.0-beta.25230.1 4.18.4 1.3.2 8.0.0-beta.23607.1 diff --git a/global.json b/global.json index 0d057253d827..5fa65d6275d1 100644 --- a/global.json +++ b/global.json @@ -14,7 +14,7 @@ } }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.25229.5", - "Microsoft.DotNet.Helix.Sdk": "8.0.0-beta.25229.5" + "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.25230.1", + "Microsoft.DotNet.Helix.Sdk": "8.0.0-beta.25230.1" } } From 4ce62e08ba5abf3eae5b0216a4fa3cfa35033c5d Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Mon, 5 May 2025 16:55:15 +0000 Subject: [PATCH 25/57] Update dependencies from https://github.com/dotnet/scenario-tests build 20250505.2 Microsoft.SourceBuild.Intermediate.scenario-tests , Microsoft.DotNet.ScenarioTests.SdkTemplateTests From Version 9.0.0-preview.25163.1 -> To Version 9.0.0-preview.25255.2 --- NuGet.config | 6 ++++++ eng/Version.Details.xml | 8 ++++---- eng/Versions.props | 2 +- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/NuGet.config b/NuGet.config index f221c730c602..85b075de314b 100644 --- a/NuGet.config +++ b/NuGet.config @@ -36,6 +36,12 @@ + + + + + + diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 738c23024753..09aa0d23aea2 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -631,14 +631,14 @@ https://github.com/dotnet/arcade-services 47e3672c762970073e4282bd563233da86bcca3e - + https://github.com/dotnet/scenario-tests - a79c5c5153234603f72c43ac078cd613a2247eff + 9e64f5176da63e40337f958e656f24a865538d6b - + https://github.com/dotnet/scenario-tests - a79c5c5153234603f72c43ac078cd613a2247eff + 9e64f5176da63e40337f958e656f24a865538d6b From 2ad1df06f0a153f57acdfec34e3bf7c210127fba Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Tue, 6 May 2025 02:02:24 +0000 Subject: [PATCH 26/57] Update dependencies from https://github.com/dotnet/arcade build 20250505.3 Microsoft.SourceBuild.Intermediate.arcade , Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.Build.Tasks.Installers , Microsoft.DotNet.Helix.Sdk , Microsoft.DotNet.SignTool , Microsoft.DotNet.XliffTasks , Microsoft.DotNet.XUnitExtensions From Version 9.0.0-beta.25208.6 -> To Version 9.0.0-beta.25255.3 --- NuGet.config | 6 ++++++ eng/Version.Details.xml | 28 ++++++++++++++-------------- eng/Versions.props | 8 ++++---- global.json | 4 ++-- 4 files changed, 26 insertions(+), 20 deletions(-) diff --git a/NuGet.config b/NuGet.config index f221c730c602..85b075de314b 100644 --- a/NuGet.config +++ b/NuGet.config @@ -36,6 +36,12 @@ + + + + + + diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index b558534b2aa0..07e89c165f31 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -589,34 +589,34 @@ - + https://github.com/dotnet/arcade - bfbc858ba868b60fffaf7b2150f1d2165b01e786 + dba26274ff5a0906a1880443fc3c0466d1a298ae - + https://github.com/dotnet/arcade - bfbc858ba868b60fffaf7b2150f1d2165b01e786 + dba26274ff5a0906a1880443fc3c0466d1a298ae - + https://github.com/dotnet/arcade - bfbc858ba868b60fffaf7b2150f1d2165b01e786 + dba26274ff5a0906a1880443fc3c0466d1a298ae - + https://github.com/dotnet/arcade - bfbc858ba868b60fffaf7b2150f1d2165b01e786 + dba26274ff5a0906a1880443fc3c0466d1a298ae - + https://github.com/dotnet/arcade - bfbc858ba868b60fffaf7b2150f1d2165b01e786 + dba26274ff5a0906a1880443fc3c0466d1a298ae - + https://github.com/dotnet/arcade - bfbc858ba868b60fffaf7b2150f1d2165b01e786 + dba26274ff5a0906a1880443fc3c0466d1a298ae - + https://github.com/dotnet/arcade - bfbc858ba868b60fffaf7b2150f1d2165b01e786 + dba26274ff5a0906a1880443fc3c0466d1a298ae diff --git a/eng/Versions.props b/eng/Versions.props index 2e34dcec89fe..7c4028744d93 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -270,10 +270,10 @@ - 9.0.0-beta.25225.6 - 9.0.0-beta.25225.6 - 9.0.0-beta.25225.6 - 9.0.0-beta.25225.6 + 9.0.0-beta.25255.3 + 9.0.0-beta.25255.3 + 9.0.0-beta.25255.3 + 9.0.0-beta.25255.3 diff --git a/global.json b/global.json index c13f07c210e0..940cef75c5cb 100644 --- a/global.json +++ b/global.json @@ -17,8 +17,8 @@ "cmake": "latest" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "9.0.0-beta.25225.6", - "Microsoft.DotNet.Helix.Sdk": "9.0.0-beta.25225.6", + "Microsoft.DotNet.Arcade.Sdk": "9.0.0-beta.25255.3", + "Microsoft.DotNet.Helix.Sdk": "9.0.0-beta.25255.3", "Microsoft.Build.NoTargets": "3.7.0", "Microsoft.DotNet.CMake.Sdk": "9.0.0-beta.24217.1" } From 5c7cb343797acfa5df6c0e00d4c84a05c214f9d1 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Tue, 6 May 2025 15:17:08 +0000 Subject: [PATCH 27/57] Update dependencies from https://github.com/dotnet/scenario-tests build 20250506.2 Microsoft.SourceBuild.Intermediate.scenario-tests , Microsoft.DotNet.ScenarioTests.SdkTemplateTests From Version 9.0.0-preview.25163.1 -> To Version 9.0.0-preview.25256.2 --- eng/Version.Details.xml | 8 ++++---- eng/Versions.props | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 09aa0d23aea2..ec16993a7ba1 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -631,14 +631,14 @@ https://github.com/dotnet/arcade-services 47e3672c762970073e4282bd563233da86bcca3e - + https://github.com/dotnet/scenario-tests - 9e64f5176da63e40337f958e656f24a865538d6b + 67b8d8889af0f0b48e1e50d7e46a19d3f86a6b98 - + https://github.com/dotnet/scenario-tests - 9e64f5176da63e40337f958e656f24a865538d6b + 67b8d8889af0f0b48e1e50d7e46a19d3f86a6b98 From 923e6017416661eed450e604444854075683a47a Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Tue, 6 May 2025 23:42:14 +0000 Subject: [PATCH 28/57] Update dependencies from https://github.com/dotnet/razor build 20250506.3 Microsoft.SourceBuild.Intermediate.razor , Microsoft.AspNetCore.Mvc.Razor.Extensions.Tooling.Internal , Microsoft.CodeAnalysis.Razor.Tooling.Internal , Microsoft.NET.Sdk.Razor.SourceGenerators.Transport From Version 9.0.0-preview.25175.6 -> To Version 9.0.0-preview.25256.3 --- NuGet.config | 8 ++++++++ eng/Version.Details.xml | 8 ++++---- eng/Versions.props | 6 +++--- 3 files changed, 15 insertions(+), 7 deletions(-) diff --git a/NuGet.config b/NuGet.config index b286902fc015..85b075de314b 100644 --- a/NuGet.config +++ b/NuGet.config @@ -36,6 +36,14 @@ + + + + + + + + diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 603468e33b5f..e4c58e3fd937 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -325,20 +325,20 @@ d5dc8a13cc618b9cbdc1e5744b4806c594d49553 - + https://github.com/dotnet/razor 44568f7817cd7c6daf90c9340c64c3b69277b7f6 - + https://github.com/dotnet/razor 44568f7817cd7c6daf90c9340c64c3b69277b7f6 - + https://github.com/dotnet/razor 44568f7817cd7c6daf90c9340c64c3b69277b7f6 - + https://github.com/dotnet/razor 44568f7817cd7c6daf90c9340c64c3b69277b7f6 diff --git a/eng/Versions.props b/eng/Versions.props index ce1989925058..aa976458257c 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -231,9 +231,9 @@ - 9.0.0-preview.25175.6 - 9.0.0-preview.25175.6 - 9.0.0-preview.25175.6 + 9.0.0-preview.25256.3 + 9.0.0-preview.25256.3 + 9.0.0-preview.25256.3 From 7e4759768b19a3e9df2eb39e7d4fb5ced616201a Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Wed, 7 May 2025 02:01:48 +0000 Subject: [PATCH 29/57] Update dependencies from https://github.com/dotnet/roslyn build 20250506.6 Microsoft.SourceBuild.Intermediate.roslyn , Microsoft.CodeAnalysis , Microsoft.CodeAnalysis.CSharp , Microsoft.CodeAnalysis.CSharp.CodeStyle , Microsoft.CodeAnalysis.CSharp.Features , Microsoft.CodeAnalysis.CSharp.Workspaces , Microsoft.CodeAnalysis.Workspaces.MSBuild , Microsoft.Net.Compilers.Toolset , Microsoft.Net.Compilers.Toolset.Framework From Version 4.12.0-3.25173.5 -> To Version 4.12.0-3.25256.6 --- NuGet.config | 8 ++++++++ eng/Version.Details.xml | 18 +++++++++--------- eng/Versions.props | 16 ++++++++-------- 3 files changed, 25 insertions(+), 17 deletions(-) diff --git a/NuGet.config b/NuGet.config index b286902fc015..85b075de314b 100644 --- a/NuGet.config +++ b/NuGet.config @@ -36,6 +36,14 @@ + + + + + + + + diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 603468e33b5f..349df9f76f51 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -97,41 +97,41 @@ 47d4e3f91e4e5414b6dafbf14288b9c5a798ef99 - + https://github.com/dotnet/roslyn 3f5cf9fbbd91f2047e988801a5142ca1cb6bab45 - + https://github.com/dotnet/roslyn 3f5cf9fbbd91f2047e988801a5142ca1cb6bab45 - + https://github.com/dotnet/roslyn 3f5cf9fbbd91f2047e988801a5142ca1cb6bab45 - + https://github.com/dotnet/roslyn 3f5cf9fbbd91f2047e988801a5142ca1cb6bab45 - + https://github.com/dotnet/roslyn 3f5cf9fbbd91f2047e988801a5142ca1cb6bab45 - + https://github.com/dotnet/roslyn 3f5cf9fbbd91f2047e988801a5142ca1cb6bab45 - + https://github.com/dotnet/roslyn 3f5cf9fbbd91f2047e988801a5142ca1cb6bab45 - + https://github.com/dotnet/roslyn 3f5cf9fbbd91f2047e988801a5142ca1cb6bab45 - + https://github.com/dotnet/roslyn 3f5cf9fbbd91f2047e988801a5142ca1cb6bab45 diff --git a/eng/Versions.props b/eng/Versions.props index ce1989925058..8e90c4e3bf48 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -204,14 +204,14 @@ - 4.12.0-3.25173.5 - 4.12.0-3.25173.5 - 4.12.0-3.25173.5 - 4.12.0-3.25173.5 - 4.12.0-3.25173.5 - 4.12.0-3.25173.5 - 4.12.0-3.25173.5 - 4.12.0-3.25173.5 + 4.12.0-3.25256.6 + 4.12.0-3.25256.6 + 4.12.0-3.25256.6 + 4.12.0-3.25256.6 + 4.12.0-3.25256.6 + 4.12.0-3.25256.6 + 4.12.0-3.25256.6 + 4.12.0-3.25256.6 From 4fa52d9ee36d5680cd0f317749a457a1aa363ee8 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Wed, 7 May 2025 02:02:07 +0000 Subject: [PATCH 30/57] Update dependencies from https://github.com/dotnet/arcade build 20250505.5 Microsoft.SourceBuild.Intermediate.arcade , Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.Build.Tasks.Installers , Microsoft.DotNet.Helix.Sdk , Microsoft.DotNet.SignTool , Microsoft.DotNet.XliffTasks , Microsoft.DotNet.XUnitExtensions From Version 9.0.0-beta.25208.6 -> To Version 9.0.0-beta.25255.5 --- eng/Version.Details.xml | 28 ++++++++++++++-------------- eng/Versions.props | 8 ++++---- global.json | 4 ++-- 3 files changed, 20 insertions(+), 20 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 07e89c165f31..269dfac814ea 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -589,34 +589,34 @@ - + https://github.com/dotnet/arcade - dba26274ff5a0906a1880443fc3c0466d1a298ae + 1cfa39f82d00b3659a3d367bc344241946e10681 - + https://github.com/dotnet/arcade - dba26274ff5a0906a1880443fc3c0466d1a298ae + 1cfa39f82d00b3659a3d367bc344241946e10681 - + https://github.com/dotnet/arcade - dba26274ff5a0906a1880443fc3c0466d1a298ae + 1cfa39f82d00b3659a3d367bc344241946e10681 - + https://github.com/dotnet/arcade - dba26274ff5a0906a1880443fc3c0466d1a298ae + 1cfa39f82d00b3659a3d367bc344241946e10681 - + https://github.com/dotnet/arcade - dba26274ff5a0906a1880443fc3c0466d1a298ae + 1cfa39f82d00b3659a3d367bc344241946e10681 - + https://github.com/dotnet/arcade - dba26274ff5a0906a1880443fc3c0466d1a298ae + 1cfa39f82d00b3659a3d367bc344241946e10681 - + https://github.com/dotnet/arcade - dba26274ff5a0906a1880443fc3c0466d1a298ae + 1cfa39f82d00b3659a3d367bc344241946e10681 diff --git a/eng/Versions.props b/eng/Versions.props index 7c4028744d93..2acf8fb99749 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -270,10 +270,10 @@ - 9.0.0-beta.25255.3 - 9.0.0-beta.25255.3 - 9.0.0-beta.25255.3 - 9.0.0-beta.25255.3 + 9.0.0-beta.25255.5 + 9.0.0-beta.25255.5 + 9.0.0-beta.25255.5 + 9.0.0-beta.25255.5 diff --git a/global.json b/global.json index 940cef75c5cb..fe3e74049ae6 100644 --- a/global.json +++ b/global.json @@ -17,8 +17,8 @@ "cmake": "latest" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "9.0.0-beta.25255.3", - "Microsoft.DotNet.Helix.Sdk": "9.0.0-beta.25255.3", + "Microsoft.DotNet.Arcade.Sdk": "9.0.0-beta.25255.5", + "Microsoft.DotNet.Helix.Sdk": "9.0.0-beta.25255.5", "Microsoft.Build.NoTargets": "3.7.0", "Microsoft.DotNet.CMake.Sdk": "9.0.0-beta.24217.1" } From e66c2c0b58bf1996f27ef9bfcda65368647c585d Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Wed, 7 May 2025 11:31:11 +0000 Subject: [PATCH 31/57] Update dependencies from https://github.com/dotnet/scenario-tests build 20250507.1 Microsoft.SourceBuild.Intermediate.scenario-tests , Microsoft.DotNet.ScenarioTests.SdkTemplateTests From Version 9.0.0-preview.25163.1 -> To Version 9.0.0-preview.25257.1 --- eng/Version.Details.xml | 8 ++++---- eng/Versions.props | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index ec16993a7ba1..5655d7250613 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -631,14 +631,14 @@ https://github.com/dotnet/arcade-services 47e3672c762970073e4282bd563233da86bcca3e - + https://github.com/dotnet/scenario-tests - 67b8d8889af0f0b48e1e50d7e46a19d3f86a6b98 + db1f9548f1e32819b3d546faf1ba7d3fa6880ae7 - + https://github.com/dotnet/scenario-tests - 67b8d8889af0f0b48e1e50d7e46a19d3f86a6b98 + db1f9548f1e32819b3d546faf1ba7d3fa6880ae7 From a17f88bceae8c89f1228e06137e7bad4a051071d Mon Sep 17 00:00:00 2001 From: Sean Reeser Date: Wed, 7 May 2025 14:48:31 -0700 Subject: [PATCH 32/57] Update branding to 8.0.410 --- eng/Versions.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/Versions.props b/eng/Versions.props index c217eab2fcd9..be0d24968df4 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -11,7 +11,7 @@ - 8.0.409 + 8.0.410 8.0.400 true From 9fecbc59e994b9b45d4527310f4141c4d5f3b70c Mon Sep 17 00:00:00 2001 From: Sean Reeser Date: Wed, 7 May 2025 14:48:49 -0700 Subject: [PATCH 33/57] Update branding to 9.0.107 --- eng/Versions.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/Versions.props b/eng/Versions.props index ce1989925058..d0688f11b7a8 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -6,7 +6,7 @@ 9 0 1 - 06 + 07 From e38501148827224a6665413567166069eab274b6 Mon Sep 17 00:00:00 2001 From: Jason Zhai Date: Wed, 7 May 2025 18:57:13 -0700 Subject: [PATCH 34/57] Update queue to ubuntu 2204 --- eng/common/templates-official/job/source-build.yml | 2 +- eng/common/templates/job/source-build.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/eng/common/templates-official/job/source-build.yml b/eng/common/templates-official/job/source-build.yml index f983033bb028..4217d6d8b148 100644 --- a/eng/common/templates-official/job/source-build.yml +++ b/eng/common/templates-official/job/source-build.yml @@ -54,7 +54,7 @@ jobs: pool: ${{ if eq(variables['System.TeamProject'], 'public') }}: name: $[replace(replace(eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), True, 'NetCore-Svc-Public' ), False, 'NetCore-Public')] - demands: ImageOverride -equals Build.Ubuntu.1804.Amd64.Open + demands: ImageOverride -equals Build.Ubuntu.2204.Amd64.Open ${{ if eq(variables['System.TeamProject'], 'internal') }}: name: $[replace(replace(eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), True, 'NetCore1ESPool-Svc-Internal'), False, 'NetCore1ESPool-Internal')] diff --git a/eng/common/templates/job/source-build.yml b/eng/common/templates/job/source-build.yml index c0ff472b697b..c48f95d93d91 100644 --- a/eng/common/templates/job/source-build.yml +++ b/eng/common/templates/job/source-build.yml @@ -54,11 +54,11 @@ jobs: pool: ${{ if eq(variables['System.TeamProject'], 'public') }}: name: $[replace(replace(eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), True, 'NetCore-Svc-Public' ), False, 'NetCore-Public')] - demands: ImageOverride -equals Build.Ubuntu.1804.Amd64.Open + demands: ImageOverride -equals Build.Ubuntu.2204.Amd64.Open ${{ if eq(variables['System.TeamProject'], 'internal') }}: name: $[replace(replace(eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), True, 'NetCore1ESPool-Svc-Internal'), False, 'NetCore1ESPool-Internal')] - demands: ImageOverride -equals Build.Ubuntu.1804.Amd64 + demands: ImageOverride -equals Build.Ubuntu.2204.Amd64 ${{ if ne(parameters.platform.pool, '') }}: pool: ${{ parameters.platform.pool }} From 3ced77ef520bf1ae81099c102d589fb4ccc863d3 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Thu, 8 May 2025 03:01:53 +0000 Subject: [PATCH 35/57] Update dependencies from https://github.com/dotnet/templating build 20250507.2 Microsoft.SourceBuild.Intermediate.templating , Microsoft.TemplateEngine.Abstractions , Microsoft.TemplateEngine.Mocks From Version 9.0.106-servicing.25217.3 -> To Version 9.0.107-servicing.25257.2 --- NuGet.config | 2 +- eng/Version.Details.xml | 12 ++++++------ eng/Versions.props | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/NuGet.config b/NuGet.config index b286902fc015..228ceb916a9c 100644 --- a/NuGet.config +++ b/NuGet.config @@ -35,7 +35,7 @@ - + diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 603468e33b5f..96811fb325be 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -1,18 +1,18 @@ - + https://github.com/dotnet/templating - a0bc400b24c00478ed52d45fb72763c93f30d693 + 981f61d44d9e782efe65fc6642f188be5b838d09 - + https://github.com/dotnet/templating - a0bc400b24c00478ed52d45fb72763c93f30d693 + 981f61d44d9e782efe65fc6642f188be5b838d09 - + https://github.com/dotnet/templating - a0bc400b24c00478ed52d45fb72763c93f30d693 + 981f61d44d9e782efe65fc6642f188be5b838d09 diff --git a/eng/Versions.props b/eng/Versions.props index d0688f11b7a8..9f4809201406 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -187,13 +187,13 @@ - 9.0.106 + 9.0.107 $(MicrosoftTemplateEngineAbstractionsPackageVersion) $(MicrosoftTemplateEngineAbstractionsPackageVersion) $(MicrosoftTemplateEngineAbstractionsPackageVersion) $(MicrosoftTemplateEngineAbstractionsPackageVersion) - 9.0.106-servicing.25217.3 + 9.0.107-servicing.25257.2 $(MicrosoftTemplateEngineMocksPackageVersion) $(MicrosoftTemplateEngineAbstractionsPackageVersion) $(MicrosoftTemplateEngineMocksPackageVersion) From e5ccf9b55f9fa71b6af66a140875b62d07e56f1b Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Thu, 8 May 2025 04:16:04 +0000 Subject: [PATCH 36/57] Update dependencies from https://github.com/dotnet/templating build 20250507.8 Microsoft.SourceBuild.Intermediate.templating , Microsoft.TemplateEngine.Abstractions , Microsoft.TemplateEngine.Mocks From Version 9.0.106-servicing.25217.3 -> To Version 9.0.107-servicing.25257.8 --- NuGet.config | 2 +- eng/Version.Details.xml | 10 +++++----- eng/Versions.props | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/NuGet.config b/NuGet.config index 228ceb916a9c..40572c4172e0 100644 --- a/NuGet.config +++ b/NuGet.config @@ -35,7 +35,7 @@ - + diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 96811fb325be..60db258b8dbc 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,16 +3,16 @@ https://github.com/dotnet/templating - 981f61d44d9e782efe65fc6642f188be5b838d09 + 32962ed33b98f8949abe238b3443f50b1f512f20 - + https://github.com/dotnet/templating - 981f61d44d9e782efe65fc6642f188be5b838d09 + 32962ed33b98f8949abe238b3443f50b1f512f20 - + https://github.com/dotnet/templating - 981f61d44d9e782efe65fc6642f188be5b838d09 + 32962ed33b98f8949abe238b3443f50b1f512f20 diff --git a/eng/Versions.props b/eng/Versions.props index 9f4809201406..fbc01e78cb9a 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -193,7 +193,7 @@ $(MicrosoftTemplateEngineAbstractionsPackageVersion) $(MicrosoftTemplateEngineAbstractionsPackageVersion) - 9.0.107-servicing.25257.2 + 9.0.107-servicing.25257.8 $(MicrosoftTemplateEngineMocksPackageVersion) $(MicrosoftTemplateEngineAbstractionsPackageVersion) $(MicrosoftTemplateEngineMocksPackageVersion) From 383448daf66786036c11013d78df5abdcabac398 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Thu, 8 May 2025 10:14:30 +0000 Subject: [PATCH 37/57] Update dependencies from https://github.com/dotnet/templating build 20250508.2 Microsoft.SourceBuild.Intermediate.templating , Microsoft.TemplateEngine.Abstractions , Microsoft.TemplateEngine.Mocks From Version 9.0.106-servicing.25217.3 -> To Version 9.0.107-servicing.25258.2 --- NuGet.config | 11 +---------- eng/Version.Details.xml | 10 +++++----- eng/Versions.props | 2 +- 3 files changed, 7 insertions(+), 16 deletions(-) diff --git a/NuGet.config b/NuGet.config index 0dc5dd3f073e..e6c5a0ccc913 100644 --- a/NuGet.config +++ b/NuGet.config @@ -35,16 +35,7 @@ - - - - - - - - - - + diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index c094d451ada0..7859584ef7b4 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,16 +3,16 @@ https://github.com/dotnet/templating - 32962ed33b98f8949abe238b3443f50b1f512f20 + 0c96b36ccec4428bbfdeafae392bda49c1667361 - + https://github.com/dotnet/templating - 32962ed33b98f8949abe238b3443f50b1f512f20 + 0c96b36ccec4428bbfdeafae392bda49c1667361 - + https://github.com/dotnet/templating - 32962ed33b98f8949abe238b3443f50b1f512f20 + 0c96b36ccec4428bbfdeafae392bda49c1667361 diff --git a/eng/Versions.props b/eng/Versions.props index 5b4df109b092..65ddee647e7a 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -193,7 +193,7 @@ $(MicrosoftTemplateEngineAbstractionsPackageVersion) $(MicrosoftTemplateEngineAbstractionsPackageVersion) - 9.0.107-servicing.25257.8 + 9.0.107-servicing.25258.2 $(MicrosoftTemplateEngineMocksPackageVersion) $(MicrosoftTemplateEngineAbstractionsPackageVersion) $(MicrosoftTemplateEngineMocksPackageVersion) From 55822654c97a93636744cd2394bf7f5536e383ef Mon Sep 17 00:00:00 2001 From: Daniel Plaisted Date: Fri, 21 Mar 2025 10:57:50 -0400 Subject: [PATCH 38/57] Fix path to tool settings file in build --- .../targets/Microsoft.NET.PackTool.targets | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Tasks/Microsoft.NET.Build.Tasks/targets/Microsoft.NET.PackTool.targets b/src/Tasks/Microsoft.NET.Build.Tasks/targets/Microsoft.NET.PackTool.targets index cedf7e510ece..dbc4c9ee1bf0 100644 --- a/src/Tasks/Microsoft.NET.Build.Tasks/targets/Microsoft.NET.PackTool.targets +++ b/src/Tasks/Microsoft.NET.Build.Tasks/targets/Microsoft.NET.PackTool.targets @@ -27,7 +27,7 @@ Copyright (c) .NET Foundation. All rights reserved. there is no dependency in nuspec. And add Microsoft.NETCore.Platforms, which is used to select correct RID assets. --> - <_ToolsSettingsFilePath>$(BaseIntermediateOutputPath)DotnetToolSettings.xml + <_ToolsSettingsFilePath>$(IntermediateOutputPath)DotnetToolSettings.xml true <_PackToolPublishDependency Condition=" ('$(GeneratePackageOnBuild)' != 'true' and '$(NoBuild)' != 'true') and $(IsPublishable) == 'true' ">_PublishBuildAlternative <_PackToolPublishDependency Condition=" ('$(GeneratePackageOnBuild)' == 'true' or '$(NoBuild)' == 'true') and $(IsPublishable) == 'true' ">$(_PublishNoBuildAlternativeDependsOn) From 44549e49ccab7fd63de34a37f0965f4b9036efd0 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Fri, 9 May 2025 02:01:47 +0000 Subject: [PATCH 39/57] Update dependencies from https://github.com/dotnet/templating build 20250507.11 Microsoft.SourceBuild.Intermediate.templating , Microsoft.TemplateEngine.Abstractions , Microsoft.TemplateEngine.Mocks From Version 8.0.409-servicing.25215.2 -> To Version 8.0.410-servicing.25257.11 --- NuGet.config | 8 +------- eng/Version.Details.xml | 12 ++++++------ eng/Versions.props | 4 ++-- 3 files changed, 9 insertions(+), 15 deletions(-) diff --git a/NuGet.config b/NuGet.config index 10209b7d7d2b..0990751f9def 100644 --- a/NuGet.config +++ b/NuGet.config @@ -23,13 +23,7 @@ - - - - - - - + diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 3723e87e2399..597be0b3f0c4 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -1,17 +1,17 @@ - + https://github.com/dotnet/templating - 1e52e85918ccfe396647e8a96e293e6b4c0e17b7 + e3bb940a424c2ec7d25f32a6fa872776f78b711c - + https://github.com/dotnet/templating - 1e52e85918ccfe396647e8a96e293e6b4c0e17b7 + e3bb940a424c2ec7d25f32a6fa872776f78b711c - + https://github.com/dotnet/templating - 1e52e85918ccfe396647e8a96e293e6b4c0e17b7 + e3bb940a424c2ec7d25f32a6fa872776f78b711c diff --git a/eng/Versions.props b/eng/Versions.props index 8b9cb13d2108..cfc2822781df 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -141,13 +141,13 @@ - 8.0.409 + 8.0.410 $(MicrosoftTemplateEngineAbstractionsPackageVersion) $(MicrosoftTemplateEngineAbstractionsPackageVersion) $(MicrosoftTemplateEngineAbstractionsPackageVersion) $(MicrosoftTemplateEngineAbstractionsPackageVersion) - 8.0.409-servicing.25215.2 + 8.0.410-servicing.25257.11 $(MicrosoftTemplateEngineMocksPackageVersion) $(MicrosoftTemplateEngineAbstractionsPackageVersion) $(MicrosoftTemplateEngineMocksPackageVersion) From 8147a93ce2a62861521be64be7fa59d36dc55deb Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Mon, 12 May 2025 11:01:46 +0000 Subject: [PATCH 40/57] Update dependencies from https://github.com/dotnet/test-templates build 20250512.1 Microsoft.SourceBuild.Intermediate.test-templates , Microsoft.DotNet.Test.ProjectTemplates.9.0 From Version 1.1.0-rtm.25168.1 -> To Version 1.1.0-rtm.25262.1 --- eng/Version.Details.xml | 8 ++++---- eng/Versions.props | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 91bfa27d8a3b..8f4d4413483f 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -379,14 +379,14 @@ https://github.com/dotnet/test-templates 49c9ad01f057b3c6352bbec12b117acc2224493c - + https://github.com/dotnet/test-templates - e66f2336162ebe9f1d781a8dae591e30d9ab08b4 + 47c90e140b027225b799ca8413af10ee3d5f1126 - + https://github.com/dotnet/test-templates - e66f2336162ebe9f1d781a8dae591e30d9ab08b4 + 47c90e140b027225b799ca8413af10ee3d5f1126 diff --git a/eng/Versions.props b/eng/Versions.props index ca0c7e69af35..f5e089f931bf 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -82,7 +82,7 @@ - 1.1.0-rtm.25168.1 + 1.1.0-rtm.25262.1 From 60b9b3ef2ab459b1459ade8cad41ae346fc30685 Mon Sep 17 00:00:00 2001 From: Eduardo Villalpando Mello Date: Tue, 13 May 2025 15:00:27 -0500 Subject: [PATCH 41/57] macOS Installer: Update relese notes link (#48722) --- .../packaging/osx/resources/cs.lproj/conclusion.html | 2 +- .../packaging/osx/resources/de.lproj/conclusion.html | 2 +- .../packaging/osx/resources/en.lproj/conclusion.html | 2 +- .../packaging/osx/resources/es.lproj/conclusion.html | 2 +- .../packaging/osx/resources/fr.lproj/conclusion.html | 2 +- .../packaging/osx/resources/it.lproj/conclusion.html | 2 +- .../packaging/osx/resources/ja.lproj/conclusion.html | 2 +- .../packaging/osx/resources/ko.lproj/conclusion.html | 2 +- .../packaging/osx/resources/pl.lproj/conclusion.html | 2 +- .../packaging/osx/resources/pt-br.lproj/conclusion.html | 2 +- .../packaging/osx/resources/ru.lproj/conclusion.html | 2 +- .../packaging/osx/resources/tr.lproj/conclusion.html | 2 +- .../packaging/osx/resources/zh-hans.lproj/conclusion.html | 2 +- .../packaging/osx/resources/zh-hant.lproj/conclusion.html | 2 +- src/Installer/redist-installer/targets/GeneratePKG.targets | 3 +++ 15 files changed, 17 insertions(+), 14 deletions(-) diff --git a/src/Installer/redist-installer/packaging/osx/resources/cs.lproj/conclusion.html b/src/Installer/redist-installer/packaging/osx/resources/cs.lproj/conclusion.html index 79216d0de8cb..35d26a7d3fd5 100644 --- a/src/Installer/redist-installer/packaging/osx/resources/cs.lproj/conclusion.html +++ b/src/Installer/redist-installer/packaging/osx/resources/cs.lproj/conclusion.html @@ -25,7 +25,7 @@

Prostředky

diff --git a/src/Installer/redist-installer/packaging/osx/resources/de.lproj/conclusion.html b/src/Installer/redist-installer/packaging/osx/resources/de.lproj/conclusion.html index 88d0fcbfe8b3..27fac17c56d5 100644 --- a/src/Installer/redist-installer/packaging/osx/resources/de.lproj/conclusion.html +++ b/src/Installer/redist-installer/packaging/osx/resources/de.lproj/conclusion.html @@ -25,7 +25,7 @@

Ressourcen

diff --git a/src/Installer/redist-installer/packaging/osx/resources/en.lproj/conclusion.html b/src/Installer/redist-installer/packaging/osx/resources/en.lproj/conclusion.html index b95c78cfbb2a..6a4bb42db3f2 100644 --- a/src/Installer/redist-installer/packaging/osx/resources/en.lproj/conclusion.html +++ b/src/Installer/redist-installer/packaging/osx/resources/en.lproj/conclusion.html @@ -25,7 +25,7 @@

Resources

diff --git a/src/Installer/redist-installer/packaging/osx/resources/es.lproj/conclusion.html b/src/Installer/redist-installer/packaging/osx/resources/es.lproj/conclusion.html index 08e4bc6f6732..126fd038a670 100644 --- a/src/Installer/redist-installer/packaging/osx/resources/es.lproj/conclusion.html +++ b/src/Installer/redist-installer/packaging/osx/resources/es.lproj/conclusion.html @@ -25,7 +25,7 @@

Recursos

diff --git a/src/Installer/redist-installer/packaging/osx/resources/fr.lproj/conclusion.html b/src/Installer/redist-installer/packaging/osx/resources/fr.lproj/conclusion.html index 52a3457c9e95..2d01d8901f1e 100644 --- a/src/Installer/redist-installer/packaging/osx/resources/fr.lproj/conclusion.html +++ b/src/Installer/redist-installer/packaging/osx/resources/fr.lproj/conclusion.html @@ -25,7 +25,7 @@

Ressources

diff --git a/src/Installer/redist-installer/packaging/osx/resources/it.lproj/conclusion.html b/src/Installer/redist-installer/packaging/osx/resources/it.lproj/conclusion.html index 729ff570201e..f28c92121102 100644 --- a/src/Installer/redist-installer/packaging/osx/resources/it.lproj/conclusion.html +++ b/src/Installer/redist-installer/packaging/osx/resources/it.lproj/conclusion.html @@ -25,7 +25,7 @@

Risorse

diff --git a/src/Installer/redist-installer/packaging/osx/resources/ja.lproj/conclusion.html b/src/Installer/redist-installer/packaging/osx/resources/ja.lproj/conclusion.html index 1ab4a40277ee..2825b9d476f2 100644 --- a/src/Installer/redist-installer/packaging/osx/resources/ja.lproj/conclusion.html +++ b/src/Installer/redist-installer/packaging/osx/resources/ja.lproj/conclusion.html @@ -25,7 +25,7 @@

リソース

diff --git a/src/Installer/redist-installer/packaging/osx/resources/ko.lproj/conclusion.html b/src/Installer/redist-installer/packaging/osx/resources/ko.lproj/conclusion.html index af7f152f0b5a..eb219d0e0cad 100644 --- a/src/Installer/redist-installer/packaging/osx/resources/ko.lproj/conclusion.html +++ b/src/Installer/redist-installer/packaging/osx/resources/ko.lproj/conclusion.html @@ -25,7 +25,7 @@

리소스

diff --git a/src/Installer/redist-installer/packaging/osx/resources/pl.lproj/conclusion.html b/src/Installer/redist-installer/packaging/osx/resources/pl.lproj/conclusion.html index 41d3e4bd3362..f2ac9170802f 100644 --- a/src/Installer/redist-installer/packaging/osx/resources/pl.lproj/conclusion.html +++ b/src/Installer/redist-installer/packaging/osx/resources/pl.lproj/conclusion.html @@ -25,7 +25,7 @@

Zasoby

diff --git a/src/Installer/redist-installer/packaging/osx/resources/pt-br.lproj/conclusion.html b/src/Installer/redist-installer/packaging/osx/resources/pt-br.lproj/conclusion.html index bf8eef4579be..ad36ba47f4f4 100644 --- a/src/Installer/redist-installer/packaging/osx/resources/pt-br.lproj/conclusion.html +++ b/src/Installer/redist-installer/packaging/osx/resources/pt-br.lproj/conclusion.html @@ -25,7 +25,7 @@

Recursos

diff --git a/src/Installer/redist-installer/packaging/osx/resources/ru.lproj/conclusion.html b/src/Installer/redist-installer/packaging/osx/resources/ru.lproj/conclusion.html index 8aab312fb96f..232c9a5c3e21 100644 --- a/src/Installer/redist-installer/packaging/osx/resources/ru.lproj/conclusion.html +++ b/src/Installer/redist-installer/packaging/osx/resources/ru.lproj/conclusion.html @@ -25,7 +25,7 @@

Ресурсы

diff --git a/src/Installer/redist-installer/packaging/osx/resources/tr.lproj/conclusion.html b/src/Installer/redist-installer/packaging/osx/resources/tr.lproj/conclusion.html index a15637eb2bdc..d9eb096279f5 100644 --- a/src/Installer/redist-installer/packaging/osx/resources/tr.lproj/conclusion.html +++ b/src/Installer/redist-installer/packaging/osx/resources/tr.lproj/conclusion.html @@ -25,7 +25,7 @@

Kaynaklar

diff --git a/src/Installer/redist-installer/packaging/osx/resources/zh-hans.lproj/conclusion.html b/src/Installer/redist-installer/packaging/osx/resources/zh-hans.lproj/conclusion.html index 0dde9c060ea6..8f5818c314c5 100644 --- a/src/Installer/redist-installer/packaging/osx/resources/zh-hans.lproj/conclusion.html +++ b/src/Installer/redist-installer/packaging/osx/resources/zh-hans.lproj/conclusion.html @@ -25,7 +25,7 @@

资源

diff --git a/src/Installer/redist-installer/packaging/osx/resources/zh-hant.lproj/conclusion.html b/src/Installer/redist-installer/packaging/osx/resources/zh-hant.lproj/conclusion.html index ff397625bd1a..e162c27fe0f8 100644 --- a/src/Installer/redist-installer/packaging/osx/resources/zh-hant.lproj/conclusion.html +++ b/src/Installer/redist-installer/packaging/osx/resources/zh-hant.lproj/conclusion.html @@ -25,7 +25,7 @@

資源

diff --git a/src/Installer/redist-installer/targets/GeneratePKG.targets b/src/Installer/redist-installer/targets/GeneratePKG.targets index 937f7c857262..2c309ec8c769 100644 --- a/src/Installer/redist-installer/targets/GeneratePKG.targets +++ b/src/Installer/redist-installer/targets/GeneratePKG.targets @@ -110,6 +110,9 @@ $(Version) + + $(VersionMajor) + $(MicrosoftNETCoreAppRuntimePackageVersion) From 9ca68729086eba18118f5ec70106572b498b0080 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Wed, 14 May 2025 02:02:40 +0000 Subject: [PATCH 42/57] Update dependencies from https://github.com/dotnet/arcade build 20250513.1 Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.Helix.Sdk , Microsoft.DotNet.SignTool , Microsoft.DotNet.XUnitExtensions From Version 8.0.0-beta.25230.1 -> To Version 8.0.0-beta.25263.1 --- eng/Version.Details.xml | 16 +- eng/Versions.props | 4 +- global.json | 4 +- .../build-configuration.json | 4 + .../eng/common/PSScriptAnalyzerSettings.psd1 | 11 + src/arcade/eng/common/README.md | 28 ++ .../eng/common/cross/arm/tizen/tizen.patch | 9 + .../eng/common/cross/armel/tizen/tizen.patch | 9 + .../eng/common/cross/x64/tizen/tizen.patch | 9 + .../eng/common/cross/x86/tizen/tizen.patch | 9 + src/arcade/eng/common/dotnet-install.cmd | 2 + src/arcade/eng/common/dotnet-install.ps1 | 28 ++ .../common/enable-cross-org-publishing.ps1 | 13 + src/arcade/eng/common/generate-locproject.ps1 | 189 +++++++++++++ src/arcade/eng/common/generate-sbom-prep.ps1 | 29 ++ src/arcade/eng/common/init-tools-native.cmd | 3 + src/arcade/eng/common/init-tools-native.ps1 | 203 ++++++++++++++ src/arcade/eng/common/init-tools-native.sh | 238 ++++++++++++++++ .../eng/common/internal-feed-operations.ps1 | 132 +++++++++ .../eng/common/internal-feed-operations.sh | 141 ++++++++++ src/arcade/eng/common/internal/NuGet.config | 7 + .../common/loc/P22DotNetHtmlLocalization.lss | 29 ++ src/arcade/eng/common/msbuild.ps1 | 28 ++ src/arcade/eng/common/msbuild.sh | 58 ++++ .../eng/common/native/common-library.sh | 172 ++++++++++++ .../eng/common/native/install-cmake-test.sh | 117 ++++++++ src/arcade/eng/common/native/install-cmake.sh | 117 ++++++++ src/arcade/eng/common/native/install-tool.ps1 | 132 +++++++++ .../eng/common/pipeline-logging-functions.ps1 | 260 ++++++++++++++++++ .../eng/common/pipeline-logging-functions.sh | 206 ++++++++++++++ src/arcade/eng/common/retain-build.ps1 | 45 +++ .../eng/common/sdl/configure-sdl-tool.ps1 | 130 +++++++++ .../common/sdl/extract-artifact-archives.ps1 | 63 +++++ .../common/sdl/extract-artifact-packages.ps1 | 82 ++++++ src/arcade/eng/common/sdl/packages.config | 4 + src/arcade/eng/common/sdl/run-sdl.ps1 | 49 ++++ .../variables/pool-providers.yml | 45 +++ .../variables/sdl-variables.yml | 7 + 38 files changed, 2620 insertions(+), 12 deletions(-) create mode 100644 src/arcade/eng/common/BuildConfiguration/build-configuration.json create mode 100644 src/arcade/eng/common/PSScriptAnalyzerSettings.psd1 create mode 100644 src/arcade/eng/common/README.md create mode 100644 src/arcade/eng/common/cross/arm/tizen/tizen.patch create mode 100644 src/arcade/eng/common/cross/armel/tizen/tizen.patch create mode 100644 src/arcade/eng/common/cross/x64/tizen/tizen.patch create mode 100644 src/arcade/eng/common/cross/x86/tizen/tizen.patch create mode 100644 src/arcade/eng/common/dotnet-install.cmd create mode 100644 src/arcade/eng/common/dotnet-install.ps1 create mode 100644 src/arcade/eng/common/enable-cross-org-publishing.ps1 create mode 100644 src/arcade/eng/common/generate-locproject.ps1 create mode 100644 src/arcade/eng/common/generate-sbom-prep.ps1 create mode 100644 src/arcade/eng/common/init-tools-native.cmd create mode 100644 src/arcade/eng/common/init-tools-native.ps1 create mode 100644 src/arcade/eng/common/init-tools-native.sh create mode 100644 src/arcade/eng/common/internal-feed-operations.ps1 create mode 100644 src/arcade/eng/common/internal-feed-operations.sh create mode 100644 src/arcade/eng/common/internal/NuGet.config create mode 100644 src/arcade/eng/common/loc/P22DotNetHtmlLocalization.lss create mode 100644 src/arcade/eng/common/msbuild.ps1 create mode 100644 src/arcade/eng/common/msbuild.sh create mode 100644 src/arcade/eng/common/native/common-library.sh create mode 100644 src/arcade/eng/common/native/install-cmake-test.sh create mode 100644 src/arcade/eng/common/native/install-cmake.sh create mode 100644 src/arcade/eng/common/native/install-tool.ps1 create mode 100644 src/arcade/eng/common/pipeline-logging-functions.ps1 create mode 100644 src/arcade/eng/common/pipeline-logging-functions.sh create mode 100644 src/arcade/eng/common/retain-build.ps1 create mode 100644 src/arcade/eng/common/sdl/configure-sdl-tool.ps1 create mode 100644 src/arcade/eng/common/sdl/extract-artifact-archives.ps1 create mode 100644 src/arcade/eng/common/sdl/extract-artifact-packages.ps1 create mode 100644 src/arcade/eng/common/sdl/packages.config create mode 100644 src/arcade/eng/common/sdl/run-sdl.ps1 create mode 100644 src/arcade/eng/common/templates-official/variables/pool-providers.yml create mode 100644 src/arcade/eng/common/templates-official/variables/sdl-variables.yml diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 597be0b3f0c4..0a91764b9e59 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -491,22 +491,22 @@
- + https://github.com/dotnet/arcade - 4246a31e5de9de87f760218c4f588cebf4661f45 + abb15166a8da9284d6ed177924cfd9b6f74c9f08 - + https://github.com/dotnet/arcade - 4246a31e5de9de87f760218c4f588cebf4661f45 + abb15166a8da9284d6ed177924cfd9b6f74c9f08 - + https://github.com/dotnet/arcade - 4246a31e5de9de87f760218c4f588cebf4661f45 + abb15166a8da9284d6ed177924cfd9b6f74c9f08 - + https://github.com/dotnet/arcade - 4246a31e5de9de87f760218c4f588cebf4661f45 + abb15166a8da9284d6ed177924cfd9b6f74c9f08 https://dev.azure.com/dnceng/internal/_git/dotnet-runtime diff --git a/eng/Versions.props b/eng/Versions.props index cfc2822781df..0a5a4d49a534 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -36,7 +36,7 @@ 8.0.0 4.0.0 8.0.0 - 8.0.0-beta.25230.1 + 8.0.0-beta.25263.1 7.0.0-preview.22423.2 8.0.0 4.3.0 @@ -213,7 +213,7 @@ 6.12.0 6.1.0 - 8.0.0-beta.25230.1 + 8.0.0-beta.25263.1 4.18.4 1.3.2 8.0.0-beta.23607.1 diff --git a/global.json b/global.json index 5fa65d6275d1..60b1b1cde1ca 100644 --- a/global.json +++ b/global.json @@ -14,7 +14,7 @@ } }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.25230.1", - "Microsoft.DotNet.Helix.Sdk": "8.0.0-beta.25230.1" + "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.25263.1", + "Microsoft.DotNet.Helix.Sdk": "8.0.0-beta.25263.1" } } diff --git a/src/arcade/eng/common/BuildConfiguration/build-configuration.json b/src/arcade/eng/common/BuildConfiguration/build-configuration.json new file mode 100644 index 000000000000..3d1cc89894c9 --- /dev/null +++ b/src/arcade/eng/common/BuildConfiguration/build-configuration.json @@ -0,0 +1,4 @@ +{ + "RetryCountLimit": 1, + "RetryByAnyError": false +} diff --git a/src/arcade/eng/common/PSScriptAnalyzerSettings.psd1 b/src/arcade/eng/common/PSScriptAnalyzerSettings.psd1 new file mode 100644 index 000000000000..4c1ea7c98ea4 --- /dev/null +++ b/src/arcade/eng/common/PSScriptAnalyzerSettings.psd1 @@ -0,0 +1,11 @@ +@{ + IncludeRules=@('PSAvoidUsingCmdletAliases', + 'PSAvoidUsingWMICmdlet', + 'PSAvoidUsingPositionalParameters', + 'PSAvoidUsingInvokeExpression', + 'PSUseDeclaredVarsMoreThanAssignments', + 'PSUseCmdletCorrectly', + 'PSStandardDSCFunctionsInResource', + 'PSUseIdenticalMandatoryParametersForDSC', + 'PSUseIdenticalParametersForDSC') +} \ No newline at end of file diff --git a/src/arcade/eng/common/README.md b/src/arcade/eng/common/README.md new file mode 100644 index 000000000000..ff49c371527a --- /dev/null +++ b/src/arcade/eng/common/README.md @@ -0,0 +1,28 @@ +# Don't touch this folder + + uuuuuuuuuuuuuuuuuuuu + u" uuuuuuuuuuuuuuuuuu "u + u" u$$$$$$$$$$$$$$$$$$$$u "u + u" u$$$$$$$$$$$$$$$$$$$$$$$$u "u + u" u$$$$$$$$$$$$$$$$$$$$$$$$$$$$u "u + u" u$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$u "u + u" u$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$u "u + $ $$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$ $ + $ $$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$ $ + $ $$$" ... "$... ...$" ... "$$$ ... "$$$ $ + $ $$$u `"$$$$$$$ $$$ $$$$$ $$ $$$ $$$ $ + $ $$$$$$uu "$$$$ $$$ $$$$$ $$ """ u$$$ $ + $ $$$""$$$ $$$$ $$$u "$$$" u$$ $$$$$$$$ $ + $ $$$$....,$$$$$..$$$$$....,$$$$..$$$$$$$$ $ + $ $$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$ $ + "u "$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$" u" + "u "$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$" u" + "u "$$$$$$$$$$$$$$$$$$$$$$$$$$$$" u" + "u "$$$$$$$$$$$$$$$$$$$$$$$$" u" + "u "$$$$$$$$$$$$$$$$$$$$" u" + "u """""""""""""""""" u" + """""""""""""""""""" + +!!! Changes made in this directory are subject to being overwritten by automation !!! + +The files in this directory are shared by all Arcade repos and managed by automation. If you need to make changes to these files, open an issue or submit a pull request to https://github.com/dotnet/arcade first. diff --git a/src/arcade/eng/common/cross/arm/tizen/tizen.patch b/src/arcade/eng/common/cross/arm/tizen/tizen.patch new file mode 100644 index 000000000000..fb12ade7250a --- /dev/null +++ b/src/arcade/eng/common/cross/arm/tizen/tizen.patch @@ -0,0 +1,9 @@ +diff -u -r a/usr/lib/libc.so b/usr/lib/libc.so +--- a/usr/lib/libc.so 2016-12-30 23:00:08.284951863 +0900 ++++ b/usr/lib/libc.so 2016-12-30 23:00:32.140951815 +0900 +@@ -2,4 +2,4 @@ + Use the shared library, but some functions are only in + the static library, so try that secondarily. */ + OUTPUT_FORMAT(elf32-littlearm) +-GROUP ( /lib/libc.so.6 /usr/lib/libc_nonshared.a AS_NEEDED ( /lib/ld-linux-armhf.so.3 ) ) ++GROUP ( libc.so.6 libc_nonshared.a AS_NEEDED ( ld-linux-armhf.so.3 ) ) diff --git a/src/arcade/eng/common/cross/armel/tizen/tizen.patch b/src/arcade/eng/common/cross/armel/tizen/tizen.patch new file mode 100644 index 000000000000..ca7c7c1ff751 --- /dev/null +++ b/src/arcade/eng/common/cross/armel/tizen/tizen.patch @@ -0,0 +1,9 @@ +diff -u -r a/usr/lib/libc.so b/usr/lib/libc.so +--- a/usr/lib/libc.so 2016-12-30 23:00:08.284951863 +0900 ++++ b/usr/lib/libc.so 2016-12-30 23:00:32.140951815 +0900 +@@ -2,4 +2,4 @@ + Use the shared library, but some functions are only in + the static library, so try that secondarily. */ + OUTPUT_FORMAT(elf32-littlearm) +-GROUP ( /lib/libc.so.6 /usr/lib/libc_nonshared.a AS_NEEDED ( /lib/ld-linux.so.3 ) ) ++GROUP ( libc.so.6 libc_nonshared.a AS_NEEDED ( ld-linux.so.3 ) ) diff --git a/src/arcade/eng/common/cross/x64/tizen/tizen.patch b/src/arcade/eng/common/cross/x64/tizen/tizen.patch new file mode 100644 index 000000000000..56fbc881095b --- /dev/null +++ b/src/arcade/eng/common/cross/x64/tizen/tizen.patch @@ -0,0 +1,9 @@ +diff -u -r a/usr/lib64/libc.so b/usr/lib64/libc.so +--- a/usr/lib64/libc.so 2016-12-30 23:00:08.284951863 +0900 ++++ b/usr/lib64/libc.so 2016-12-30 23:00:32.140951815 +0900 +@@ -2,4 +2,4 @@ + Use the shared library, but some functions are only in + the static library, so try that secondarily. */ + OUTPUT_FORMAT(elf64-x86-64) +-GROUP ( /lib64/libc.so.6 /usr/lib64/libc_nonshared.a AS_NEEDED ( /lib64/ld-linux-x86-64.so.2 ) ) ++GROUP ( libc.so.6 libc_nonshared.a AS_NEEDED ( ld-linux-x86-64.so.2 ) ) diff --git a/src/arcade/eng/common/cross/x86/tizen/tizen.patch b/src/arcade/eng/common/cross/x86/tizen/tizen.patch new file mode 100644 index 000000000000..f4fe8838ad66 --- /dev/null +++ b/src/arcade/eng/common/cross/x86/tizen/tizen.patch @@ -0,0 +1,9 @@ +diff -u -r a/usr/lib/libc.so b/usr/lib/libc.so +--- a/usr/lib/libc.so 2016-12-30 23:00:08.284951863 +0900 ++++ b/usr/lib/libc.so 2016-12-30 23:00:32.140951815 +0900 +@@ -2,4 +2,4 @@ + Use the shared library, but some functions are only in + the static library, so try that secondarily. */ + OUTPUT_FORMAT(elf32-i386) +-GROUP ( /lib/libc.so.6 /usr/lib/libc_nonshared.a AS_NEEDED ( /lib/ld-linux.so.2 ) ) ++GROUP ( libc.so.6 libc_nonshared.a AS_NEEDED ( ld-linux.so.2 ) ) diff --git a/src/arcade/eng/common/dotnet-install.cmd b/src/arcade/eng/common/dotnet-install.cmd new file mode 100644 index 000000000000..b1c2642e76f7 --- /dev/null +++ b/src/arcade/eng/common/dotnet-install.cmd @@ -0,0 +1,2 @@ +@echo off +powershell -ExecutionPolicy ByPass -NoProfile -command "& """%~dp0dotnet-install.ps1""" %*" \ No newline at end of file diff --git a/src/arcade/eng/common/dotnet-install.ps1 b/src/arcade/eng/common/dotnet-install.ps1 new file mode 100644 index 000000000000..811f0f717f73 --- /dev/null +++ b/src/arcade/eng/common/dotnet-install.ps1 @@ -0,0 +1,28 @@ +[CmdletBinding(PositionalBinding=$false)] +Param( + [string] $verbosity = 'minimal', + [string] $architecture = '', + [string] $version = 'Latest', + [string] $runtime = 'dotnet', + [string] $RuntimeSourceFeed = '', + [string] $RuntimeSourceFeedKey = '' +) + +. $PSScriptRoot\tools.ps1 + +$dotnetRoot = Join-Path $RepoRoot '.dotnet' + +$installdir = $dotnetRoot +try { + if ($architecture -and $architecture.Trim() -eq 'x86') { + $installdir = Join-Path $installdir 'x86' + } + InstallDotNet $installdir $version $architecture $runtime $true -RuntimeSourceFeed $RuntimeSourceFeed -RuntimeSourceFeedKey $RuntimeSourceFeedKey +} +catch { + Write-Host $_.ScriptStackTrace + Write-PipelineTelemetryError -Category 'InitializeToolset' -Message $_ + ExitWithExitCode 1 +} + +ExitWithExitCode 0 diff --git a/src/arcade/eng/common/enable-cross-org-publishing.ps1 b/src/arcade/eng/common/enable-cross-org-publishing.ps1 new file mode 100644 index 000000000000..da09da4f1fc4 --- /dev/null +++ b/src/arcade/eng/common/enable-cross-org-publishing.ps1 @@ -0,0 +1,13 @@ +param( + [string] $token +) + + +. $PSScriptRoot\pipeline-logging-functions.ps1 + +# Write-PipelineSetVariable will no-op if a variable named $ci is not defined +# Since this script is only ever called in AzDO builds, just universally set it +$ci = $true + +Write-PipelineSetVariable -Name 'VSS_NUGET_ACCESSTOKEN' -Value $token -IsMultiJobVariable $false +Write-PipelineSetVariable -Name 'VSS_NUGET_URI_PREFIXES' -Value 'https://dnceng.pkgs.visualstudio.com/;https://pkgs.dev.azure.com/dnceng/;https://devdiv.pkgs.visualstudio.com/;https://pkgs.dev.azure.com/devdiv/' -IsMultiJobVariable $false diff --git a/src/arcade/eng/common/generate-locproject.ps1 b/src/arcade/eng/common/generate-locproject.ps1 new file mode 100644 index 000000000000..524aaa57f2b7 --- /dev/null +++ b/src/arcade/eng/common/generate-locproject.ps1 @@ -0,0 +1,189 @@ +Param( + [Parameter(Mandatory=$true)][string] $SourcesDirectory, # Directory where source files live; if using a Localize directory it should live in here + [string] $LanguageSet = 'VS_Main_Languages', # Language set to be used in the LocProject.json + [switch] $UseCheckedInLocProjectJson, # When set, generates a LocProject.json and compares it to one that already exists in the repo; otherwise just generates one + [switch] $CreateNeutralXlfs # Creates neutral xlf files. Only set to false when running locally +) + +# Generates LocProject.json files for the OneLocBuild task. OneLocBuildTask is described here: +# https://ceapex.visualstudio.com/CEINTL/_wiki/wikis/CEINTL.wiki/107/Localization-with-OneLocBuild-Task + +Set-StrictMode -Version 2.0 +$ErrorActionPreference = "Stop" +. $PSScriptRoot\pipeline-logging-functions.ps1 + +$exclusionsFilePath = "$SourcesDirectory\eng\Localize\LocExclusions.json" +$exclusions = @{ Exclusions = @() } +if (Test-Path -Path $exclusionsFilePath) +{ + $exclusions = Get-Content "$exclusionsFilePath" | ConvertFrom-Json +} + +Push-Location "$SourcesDirectory" # push location for Resolve-Path -Relative to work + +# Template files +$jsonFiles = @() +$jsonTemplateFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "\.template\.config\\localize\\.+\.en\.json" } # .NET templating pattern +$jsonTemplateFiles | ForEach-Object { + $null = $_.Name -Match "(.+)\.[\w-]+\.json" # matches '[filename].[langcode].json + + $destinationFile = "$($_.Directory.FullName)\$($Matches.1).json" + $jsonFiles += Copy-Item "$($_.FullName)" -Destination $destinationFile -PassThru +} + +$jsonWinformsTemplateFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "en\\strings\.json" } # current winforms pattern + +$wxlFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "\\.+\.wxl" -And -Not( $_.Directory.Name -Match "\d{4}" ) } # localized files live in four digit lang ID directories; this excludes them +if (-not $wxlFiles) { + $wxlEnFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "\\1033\\.+\.wxl" } # pick up en files (1033 = en) specifically so we can copy them to use as the neutral xlf files + if ($wxlEnFiles) { + $wxlFiles = @() + $wxlEnFiles | ForEach-Object { + $destinationFile = "$($_.Directory.Parent.FullName)\$($_.Name)" + $wxlFiles += Copy-Item "$($_.FullName)" -Destination $destinationFile -PassThru + } + } +} + +$macosHtmlEnFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "en\.lproj\\.+\.html$" } # add installer HTML files +$macosHtmlFiles = @() +if ($macosHtmlEnFiles) { + $macosHtmlEnFiles | ForEach-Object { + $destinationFile = "$($_.Directory.Parent.FullName)\$($_.Name)" + $macosHtmlFiles += Copy-Item "$($_.FullName)" -Destination $destinationFile -PassThru + } +} + +$xlfFiles = @() + +$allXlfFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory\*\*.xlf" +$langXlfFiles = @() +if ($allXlfFiles) { + $null = $allXlfFiles[0].FullName -Match "\.([\w-]+)\.xlf" # matches '[langcode].xlf' + $firstLangCode = $Matches.1 + $langXlfFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory\*\*.$firstLangCode.xlf" +} +$langXlfFiles | ForEach-Object { + $null = $_.Name -Match "(.+)\.[\w-]+\.xlf" # matches '[filename].[langcode].xlf + + $destinationFile = "$($_.Directory.FullName)\$($Matches.1).xlf" + $xlfFiles += Copy-Item "$($_.FullName)" -Destination $destinationFile -PassThru +} + +$locFiles = $jsonFiles + $jsonWinformsTemplateFiles + $xlfFiles + +$locJson = @{ + Projects = @( + @{ + LanguageSet = $LanguageSet + LocItems = @( + $locFiles | ForEach-Object { + $outputPath = "$(($_.DirectoryName | Resolve-Path -Relative) + "\")" + $continue = $true + foreach ($exclusion in $exclusions.Exclusions) { + if ($_.FullName.Contains($exclusion)) + { + $continue = $false + } + } + $sourceFile = ($_.FullName | Resolve-Path -Relative) + if (!$CreateNeutralXlfs -and $_.Extension -eq '.xlf') { + Remove-Item -Path $sourceFile + } + if ($continue) + { + if ($_.Directory.Name -eq 'en' -and $_.Extension -eq '.json') { + return @{ + SourceFile = $sourceFile + CopyOption = "LangIDOnPath" + OutputPath = "$($_.Directory.Parent.FullName | Resolve-Path -Relative)\" + } + } else { + return @{ + SourceFile = $sourceFile + CopyOption = "LangIDOnName" + OutputPath = $outputPath + } + } + } + } + ) + }, + @{ + LanguageSet = $LanguageSet + CloneLanguageSet = "WiX_CloneLanguages" + LssFiles = @( "wxl_loc.lss" ) + LocItems = @( + $wxlFiles | ForEach-Object { + $outputPath = "$($_.Directory.FullName | Resolve-Path -Relative)\" + $continue = $true + foreach ($exclusion in $exclusions.Exclusions) { + if ($_.FullName.Contains($exclusion)) { + $continue = $false + } + } + $sourceFile = ($_.FullName | Resolve-Path -Relative) + if ($continue) + { + return @{ + SourceFile = $sourceFile + CopyOption = "LangIDOnPath" + OutputPath = $outputPath + } + } + } + ) + }, + @{ + LanguageSet = $LanguageSet + CloneLanguageSet = "VS_macOS_CloneLanguages" + LssFiles = @( ".\eng\common\loc\P22DotNetHtmlLocalization.lss" ) + LocItems = @( + $macosHtmlFiles | ForEach-Object { + $outputPath = "$($_.Directory.FullName | Resolve-Path -Relative)\" + $continue = $true + foreach ($exclusion in $exclusions.Exclusions) { + if ($_.FullName.Contains($exclusion)) { + $continue = $false + } + } + $sourceFile = ($_.FullName | Resolve-Path -Relative) + $lciFile = $sourceFile + ".lci" + if ($continue) { + $result = @{ + SourceFile = $sourceFile + CopyOption = "LangIDOnPath" + OutputPath = $outputPath + } + if (Test-Path $lciFile -PathType Leaf) { + $result["LciFile"] = $lciFile + } + return $result + } + } + ) + } + ) +} + +$json = ConvertTo-Json $locJson -Depth 5 +Write-Host "LocProject.json generated:`n`n$json`n`n" +Pop-Location + +if (!$UseCheckedInLocProjectJson) { + New-Item "$SourcesDirectory\eng\Localize\LocProject.json" -Force # Need this to make sure the Localize directory is created + Set-Content "$SourcesDirectory\eng\Localize\LocProject.json" $json +} +else { + New-Item "$SourcesDirectory\eng\Localize\LocProject-generated.json" -Force # Need this to make sure the Localize directory is created + Set-Content "$SourcesDirectory\eng\Localize\LocProject-generated.json" $json + + if ((Get-FileHash "$SourcesDirectory\eng\Localize\LocProject-generated.json").Hash -ne (Get-FileHash "$SourcesDirectory\eng\Localize\LocProject.json").Hash) { + Write-PipelineTelemetryError -Category "OneLocBuild" -Message "Existing LocProject.json differs from generated LocProject.json. Download LocProject-generated.json and compare them." + + exit 1 + } + else { + Write-Host "Generated LocProject.json and current LocProject.json are identical." + } +} diff --git a/src/arcade/eng/common/generate-sbom-prep.ps1 b/src/arcade/eng/common/generate-sbom-prep.ps1 new file mode 100644 index 000000000000..a0c7d792a76f --- /dev/null +++ b/src/arcade/eng/common/generate-sbom-prep.ps1 @@ -0,0 +1,29 @@ +Param( + [Parameter(Mandatory=$true)][string] $ManifestDirPath # Manifest directory where sbom will be placed +) + +. $PSScriptRoot\pipeline-logging-functions.ps1 + +# Normally - we'd listen to the manifest path given, but 1ES templates will overwrite if this level gets uploaded directly +# with their own overwriting ours. So we create it as a sub directory of the requested manifest path. +$ArtifactName = "${env:SYSTEM_STAGENAME}_${env:AGENT_JOBNAME}_SBOM" +$SafeArtifactName = $ArtifactName -replace '["/:<>\\|?@*"() ]', '_' +$SbomGenerationDir = Join-Path $ManifestDirPath $SafeArtifactName + +Write-Host "Artifact name before : $ArtifactName" +Write-Host "Artifact name after : $SafeArtifactName" + +Write-Host "Creating dir $ManifestDirPath" + +# create directory for sbom manifest to be placed +if (!(Test-Path -path $SbomGenerationDir)) +{ + New-Item -ItemType Directory -path $SbomGenerationDir + Write-Host "Successfully created directory $SbomGenerationDir" +} +else{ + Write-PipelineTelemetryError -category 'Build' "Unable to create sbom folder." +} + +Write-Host "Updating artifact name" +Write-Host "##vso[task.setvariable variable=ARTIFACT_NAME]$SafeArtifactName" diff --git a/src/arcade/eng/common/init-tools-native.cmd b/src/arcade/eng/common/init-tools-native.cmd new file mode 100644 index 000000000000..438cd548c452 --- /dev/null +++ b/src/arcade/eng/common/init-tools-native.cmd @@ -0,0 +1,3 @@ +@echo off +powershell -NoProfile -NoLogo -ExecutionPolicy ByPass -command "& """%~dp0init-tools-native.ps1""" %*" +exit /b %ErrorLevel% \ No newline at end of file diff --git a/src/arcade/eng/common/init-tools-native.ps1 b/src/arcade/eng/common/init-tools-native.ps1 new file mode 100644 index 000000000000..27ccdb9ecc95 --- /dev/null +++ b/src/arcade/eng/common/init-tools-native.ps1 @@ -0,0 +1,203 @@ +<# +.SYNOPSIS +Entry point script for installing native tools + +.DESCRIPTION +Reads $RepoRoot\global.json file to determine native assets to install +and executes installers for those tools + +.PARAMETER BaseUri +Base file directory or Url from which to acquire tool archives + +.PARAMETER InstallDirectory +Directory to install native toolset. This is a command-line override for the default +Install directory precedence order: +- InstallDirectory command-line override +- NETCOREENG_INSTALL_DIRECTORY environment variable +- (default) %USERPROFILE%/.netcoreeng/native + +.PARAMETER Clean +Switch specifying to not install anything, but cleanup native asset folders + +.PARAMETER Force +Clean and then install tools + +.PARAMETER DownloadRetries +Total number of retry attempts + +.PARAMETER RetryWaitTimeInSeconds +Wait time between retry attempts in seconds + +.PARAMETER GlobalJsonFile +File path to global.json file + +.PARAMETER PathPromotion +Optional switch to enable either promote native tools specified in the global.json to the path (in Azure Pipelines) +or break the build if a native tool is not found on the path (on a local dev machine) + +.NOTES +#> +[CmdletBinding(PositionalBinding=$false)] +Param ( + [string] $BaseUri = 'https://netcorenativeassets.blob.core.windows.net/resource-packages/external', + [string] $InstallDirectory, + [switch] $Clean = $False, + [switch] $Force = $False, + [int] $DownloadRetries = 5, + [int] $RetryWaitTimeInSeconds = 30, + [string] $GlobalJsonFile, + [switch] $PathPromotion +) + +if (!$GlobalJsonFile) { + $GlobalJsonFile = Join-Path (Get-Item $PSScriptRoot).Parent.Parent.FullName 'global.json' +} + +Set-StrictMode -version 2.0 +$ErrorActionPreference='Stop' + +. $PSScriptRoot\pipeline-logging-functions.ps1 +Import-Module -Name (Join-Path $PSScriptRoot 'native\CommonLibrary.psm1') + +try { + # Define verbose switch if undefined + $Verbose = $VerbosePreference -Eq 'Continue' + + $EngCommonBaseDir = Join-Path $PSScriptRoot 'native\' + $NativeBaseDir = $InstallDirectory + if (!$NativeBaseDir) { + $NativeBaseDir = CommonLibrary\Get-NativeInstallDirectory + } + $Env:CommonLibrary_NativeInstallDir = $NativeBaseDir + $InstallBin = Join-Path $NativeBaseDir 'bin' + $InstallerPath = Join-Path $EngCommonBaseDir 'install-tool.ps1' + + # Process tools list + Write-Host "Processing $GlobalJsonFile" + If (-Not (Test-Path $GlobalJsonFile)) { + Write-Host "Unable to find '$GlobalJsonFile'" + exit 0 + } + $NativeTools = Get-Content($GlobalJsonFile) -Raw | + ConvertFrom-Json | + Select-Object -Expand 'native-tools' -ErrorAction SilentlyContinue + if ($NativeTools) { + if ($PathPromotion -eq $True) { + $ArcadeToolsDirectory = "$env:SYSTEMDRIVE\arcade-tools" + if (Test-Path $ArcadeToolsDirectory) { # if this directory exists, we should use native tools on machine + $NativeTools.PSObject.Properties | ForEach-Object { + $ToolName = $_.Name + $ToolVersion = $_.Value + $InstalledTools = @{} + + if ((Get-Command "$ToolName" -ErrorAction SilentlyContinue) -eq $null) { + if ($ToolVersion -eq "latest") { + $ToolVersion = "" + } + $ToolDirectories = (Get-ChildItem -Path "$ArcadeToolsDirectory" -Filter "$ToolName-$ToolVersion*" | Sort-Object -Descending) + if ($ToolDirectories -eq $null) { + Write-Error "Unable to find directory for $ToolName $ToolVersion; please make sure the tool is installed on this image." + exit 1 + } + $ToolDirectory = $ToolDirectories[0] + $BinPathFile = "$($ToolDirectory.FullName)\binpath.txt" + if (-not (Test-Path -Path "$BinPathFile")) { + Write-Error "Unable to find binpath.txt in '$($ToolDirectory.FullName)' ($ToolName $ToolVersion); artifact is either installed incorrectly or is not a bootstrappable tool." + exit 1 + } + $BinPath = Get-Content "$BinPathFile" + $ToolPath = Convert-Path -Path $BinPath + Write-Host "Adding $ToolName to the path ($ToolPath)..." + Write-Host "##vso[task.prependpath]$ToolPath" + $env:PATH = "$ToolPath;$env:PATH" + $InstalledTools += @{ $ToolName = $ToolDirectory.FullName } + } + } + return $InstalledTools + } else { + $NativeTools.PSObject.Properties | ForEach-Object { + $ToolName = $_.Name + $ToolVersion = $_.Value + + if ((Get-Command "$ToolName" -ErrorAction SilentlyContinue) -eq $null) { + Write-PipelineTelemetryError -Category 'NativeToolsBootstrap' -Message "$ToolName not found on path. Please install $ToolName $ToolVersion before proceeding." + Write-PipelineTelemetryError -Category 'NativeToolsBootstrap' -Message "If this is running on a build machine, the arcade-tools directory was not found, which means there's an error with the image." + } + } + exit 0 + } + } else { + $NativeTools.PSObject.Properties | ForEach-Object { + $ToolName = $_.Name + $ToolVersion = $_.Value + $LocalInstallerArguments = @{ ToolName = "$ToolName" } + $LocalInstallerArguments += @{ InstallPath = "$InstallBin" } + $LocalInstallerArguments += @{ BaseUri = "$BaseUri" } + $LocalInstallerArguments += @{ CommonLibraryDirectory = "$EngCommonBaseDir" } + $LocalInstallerArguments += @{ Version = "$ToolVersion" } + + if ($Verbose) { + $LocalInstallerArguments += @{ Verbose = $True } + } + if (Get-Variable 'Force' -ErrorAction 'SilentlyContinue') { + if($Force) { + $LocalInstallerArguments += @{ Force = $True } + } + } + if ($Clean) { + $LocalInstallerArguments += @{ Clean = $True } + } + + Write-Verbose "Installing $ToolName version $ToolVersion" + Write-Verbose "Executing '$InstallerPath $($LocalInstallerArguments.Keys.ForEach({"-$_ '$($LocalInstallerArguments.$_)'"}) -join ' ')'" + & $InstallerPath @LocalInstallerArguments + if ($LASTEXITCODE -Ne "0") { + $errMsg = "$ToolName installation failed" + if ((Get-Variable 'DoNotAbortNativeToolsInstallationOnFailure' -ErrorAction 'SilentlyContinue') -and $DoNotAbortNativeToolsInstallationOnFailure) { + $showNativeToolsWarning = $true + if ((Get-Variable 'DoNotDisplayNativeToolsInstallationWarnings' -ErrorAction 'SilentlyContinue') -and $DoNotDisplayNativeToolsInstallationWarnings) { + $showNativeToolsWarning = $false + } + if ($showNativeToolsWarning) { + Write-Warning $errMsg + } + $toolInstallationFailure = $true + } else { + # We cannot change this to Write-PipelineTelemetryError because of https://github.com/dotnet/arcade/issues/4482 + Write-Host $errMsg + exit 1 + } + } + } + + if ((Get-Variable 'toolInstallationFailure' -ErrorAction 'SilentlyContinue') -and $toolInstallationFailure) { + # We cannot change this to Write-PipelineTelemetryError because of https://github.com/dotnet/arcade/issues/4482 + Write-Host 'Native tools bootstrap failed' + exit 1 + } + } + } + else { + Write-Host 'No native tools defined in global.json' + exit 0 + } + + if ($Clean) { + exit 0 + } + if (Test-Path $InstallBin) { + Write-Host 'Native tools are available from ' (Convert-Path -Path $InstallBin) + Write-Host "##vso[task.prependpath]$(Convert-Path -Path $InstallBin)" + return $InstallBin + } + elseif (-not ($PathPromotion)) { + Write-PipelineTelemetryError -Category 'NativeToolsBootstrap' -Message 'Native tools install directory does not exist, installation failed' + exit 1 + } + exit 0 +} +catch { + Write-Host $_.ScriptStackTrace + Write-PipelineTelemetryError -Category 'NativeToolsBootstrap' -Message $_ + ExitWithExitCode 1 +} diff --git a/src/arcade/eng/common/init-tools-native.sh b/src/arcade/eng/common/init-tools-native.sh new file mode 100644 index 000000000000..3e6a8d6acf2f --- /dev/null +++ b/src/arcade/eng/common/init-tools-native.sh @@ -0,0 +1,238 @@ +#!/usr/bin/env bash + +source="${BASH_SOURCE[0]}" +scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" + +base_uri='https://netcorenativeassets.blob.core.windows.net/resource-packages/external' +install_directory='' +clean=false +force=false +download_retries=5 +retry_wait_time_seconds=30 +global_json_file="$(dirname "$(dirname "${scriptroot}")")/global.json" +declare -a native_assets + +. $scriptroot/pipeline-logging-functions.sh +. $scriptroot/native/common-library.sh + +while (($# > 0)); do + lowerI="$(echo $1 | tr "[:upper:]" "[:lower:]")" + case $lowerI in + --baseuri) + base_uri=$2 + shift 2 + ;; + --installdirectory) + install_directory=$2 + shift 2 + ;; + --clean) + clean=true + shift 1 + ;; + --force) + force=true + shift 1 + ;; + --donotabortonfailure) + donotabortonfailure=true + shift 1 + ;; + --donotdisplaywarnings) + donotdisplaywarnings=true + shift 1 + ;; + --downloadretries) + download_retries=$2 + shift 2 + ;; + --retrywaittimeseconds) + retry_wait_time_seconds=$2 + shift 2 + ;; + --help) + echo "Common settings:" + echo " --installdirectory Directory to install native toolset." + echo " This is a command-line override for the default" + echo " Install directory precedence order:" + echo " - InstallDirectory command-line override" + echo " - NETCOREENG_INSTALL_DIRECTORY environment variable" + echo " - (default) %USERPROFILE%/.netcoreeng/native" + echo "" + echo " --clean Switch specifying not to install anything, but cleanup native asset folders" + echo " --donotabortonfailure Switch specifiying whether to abort native tools installation on failure" + echo " --donotdisplaywarnings Switch specifiying whether to display warnings during native tools installation on failure" + echo " --force Clean and then install tools" + echo " --help Print help and exit" + echo "" + echo "Advanced settings:" + echo " --baseuri Base URI for where to download native tools from" + echo " --downloadretries Number of times a download should be attempted" + echo " --retrywaittimeseconds Wait time between download attempts" + echo "" + exit 0 + ;; + esac +done + +function ReadGlobalJsonNativeTools { + # happy path: we have a proper JSON parsing tool `jq(1)` in PATH! + if command -v jq &> /dev/null; then + + # jq: read each key/value pair under "native-tools" entry and emit: + # KEY="" VALUE="" + # followed by a null byte. + # + # bash: read line with null byte delimeter and push to array (for later `eval`uation). + + while IFS= read -rd '' line; do + native_assets+=("$line") + done < <(jq -r '. | + select(has("native-tools")) | + ."native-tools" | + keys[] as $k | + @sh "KEY=\($k) VALUE=\(.[$k])\u0000"' "$global_json_file") + + return + fi + + # Warning: falling back to manually parsing JSON, which is not recommended. + + # Following routine matches the output and escaping logic of jq(1)'s @sh formatter used above. + # It has been tested with several weird strings with escaped characters in entries (key and value) + # and results were compared with the output of jq(1) in binary representation using xxd(1); + # just before the assignment to 'native_assets' array (above and below). + + # try to capture the section under "native-tools". + if [[ ! "$(cat "$global_json_file")" =~ \"native-tools\"[[:space:]\:\{]*([^\}]+) ]]; then + return + fi + + section="${BASH_REMATCH[1]}" + + parseStarted=0 + possibleEnd=0 + escaping=0 + escaped=0 + isKey=1 + + for (( i=0; i<${#section}; i++ )); do + char="${section:$i:1}" + if ! ((parseStarted)) && [[ "$char" =~ [[:space:],:] ]]; then continue; fi + + if ! ((escaping)) && [[ "$char" == "\\" ]]; then + escaping=1 + elif ((escaping)) && ! ((escaped)); then + escaped=1 + fi + + if ! ((parseStarted)) && [[ "$char" == "\"" ]]; then + parseStarted=1 + possibleEnd=0 + elif [[ "$char" == "'" ]]; then + token="$token'\\\''" + possibleEnd=0 + elif ((escaping)) || [[ "$char" != "\"" ]]; then + token="$token$char" + possibleEnd=1 + fi + + if ((possibleEnd)) && ! ((escaping)) && [[ "$char" == "\"" ]]; then + # Use printf to unescape token to match jq(1)'s @sh formatting rules. + # do not use 'token="$(printf "$token")"' syntax, as $() eats the trailing linefeed. + printf -v token "'$token'" + + if ((isKey)); then + KEY="$token" + isKey=0 + else + line="KEY=$KEY VALUE=$token" + native_assets+=("$line") + isKey=1 + fi + + # reset for next token + parseStarted=0 + token= + elif ((escaping)) && ((escaped)); then + escaping=0 + escaped=0 + fi + done +} + +native_base_dir=$install_directory +if [[ -z $install_directory ]]; then + native_base_dir=$(GetNativeInstallDirectory) +fi + +install_bin="${native_base_dir}/bin" +installed_any=false + +ReadGlobalJsonNativeTools + +if [[ ${#native_assets[@]} -eq 0 ]]; then + echo "No native tools defined in global.json" + exit 0; +else + native_installer_dir="$scriptroot/native" + for index in "${!native_assets[@]}"; do + eval "${native_assets["$index"]}" + + installer_path="$native_installer_dir/install-$KEY.sh" + installer_command="$installer_path" + installer_command+=" --baseuri $base_uri" + installer_command+=" --installpath $install_bin" + installer_command+=" --version $VALUE" + echo $installer_command + + if [[ $force = true ]]; then + installer_command+=" --force" + fi + + if [[ $clean = true ]]; then + installer_command+=" --clean" + fi + + if [[ -a $installer_path ]]; then + $installer_command + if [[ $? != 0 ]]; then + if [[ $donotabortonfailure = true ]]; then + if [[ $donotdisplaywarnings != true ]]; then + Write-PipelineTelemetryError -category 'NativeToolsBootstrap' "Execution Failed" + fi + else + Write-PipelineTelemetryError -category 'NativeToolsBootstrap' "Execution Failed" + exit 1 + fi + else + $installed_any = true + fi + else + if [[ $donotabortonfailure == true ]]; then + if [[ $donotdisplaywarnings != true ]]; then + Write-PipelineTelemetryError -category 'NativeToolsBootstrap' "Execution Failed: no install script" + fi + else + Write-PipelineTelemetryError -category 'NativeToolsBootstrap' "Execution Failed: no install script" + exit 1 + fi + fi + done +fi + +if [[ $clean = true ]]; then + exit 0 +fi + +if [[ -d $install_bin ]]; then + echo "Native tools are available from $install_bin" + echo "##vso[task.prependpath]$install_bin" +else + if [[ $installed_any = true ]]; then + Write-PipelineTelemetryError -category 'NativeToolsBootstrap' "Native tools install directory does not exist, installation failed" + exit 1 + fi +fi + +exit 0 diff --git a/src/arcade/eng/common/internal-feed-operations.ps1 b/src/arcade/eng/common/internal-feed-operations.ps1 new file mode 100644 index 000000000000..92b77347d990 --- /dev/null +++ b/src/arcade/eng/common/internal-feed-operations.ps1 @@ -0,0 +1,132 @@ +param( + [Parameter(Mandatory=$true)][string] $Operation, + [string] $AuthToken, + [string] $CommitSha, + [string] $RepoName, + [switch] $IsFeedPrivate +) + +$ErrorActionPreference = 'Stop' +Set-StrictMode -Version 2.0 +. $PSScriptRoot\tools.ps1 + +# Sets VSS_NUGET_EXTERNAL_FEED_ENDPOINTS based on the "darc-int-*" feeds defined in NuGet.config. This is needed +# in build agents by CredProvider to authenticate the restore requests to internal feeds as specified in +# https://github.com/microsoft/artifacts-credprovider/blob/0f53327cd12fd893d8627d7b08a2171bf5852a41/README.md#environment-variables. This should ONLY be called from identified +# internal builds +function SetupCredProvider { + param( + [string] $AuthToken + ) + + # Install the Cred Provider NuGet plugin + Write-Host 'Setting up Cred Provider NuGet plugin in the agent...' + Write-Host "Getting 'installcredprovider.ps1' from 'https://github.com/microsoft/artifacts-credprovider'..." + + $url = 'https://raw.githubusercontent.com/microsoft/artifacts-credprovider/master/helpers/installcredprovider.ps1' + + Write-Host "Writing the contents of 'installcredprovider.ps1' locally..." + Invoke-WebRequest $url -OutFile installcredprovider.ps1 + + Write-Host 'Installing plugin...' + .\installcredprovider.ps1 -Force + + Write-Host "Deleting local copy of 'installcredprovider.ps1'..." + Remove-Item .\installcredprovider.ps1 + + if (-Not("$env:USERPROFILE\.nuget\plugins\netcore")) { + Write-PipelineTelemetryError -Category 'Arcade' -Message 'CredProvider plugin was not installed correctly!' + ExitWithExitCode 1 + } + else { + Write-Host 'CredProvider plugin was installed correctly!' + } + + # Then, we set the 'VSS_NUGET_EXTERNAL_FEED_ENDPOINTS' environment variable to restore from the stable + # feeds successfully + + $nugetConfigPath = Join-Path $RepoRoot "NuGet.config" + + if (-Not (Test-Path -Path $nugetConfigPath)) { + Write-PipelineTelemetryError -Category 'Build' -Message 'NuGet.config file not found in repo root!' + ExitWithExitCode 1 + } + + $endpoints = New-Object System.Collections.ArrayList + $nugetConfigPackageSources = Select-Xml -Path $nugetConfigPath -XPath "//packageSources/add[contains(@key, 'darc-int-')]/@value" | foreach{$_.Node.Value} + + if (($nugetConfigPackageSources | Measure-Object).Count -gt 0 ) { + foreach ($stableRestoreResource in $nugetConfigPackageSources) { + $trimmedResource = ([string]$stableRestoreResource).Trim() + [void]$endpoints.Add(@{endpoint="$trimmedResource"; password="$AuthToken"}) + } + } + + if (($endpoints | Measure-Object).Count -gt 0) { + $endpointCredentials = @{endpointCredentials=$endpoints} | ConvertTo-Json -Compress + + # Create the environment variables the AzDo way + Write-LoggingCommand -Area 'task' -Event 'setvariable' -Data $endpointCredentials -Properties @{ + 'variable' = 'VSS_NUGET_EXTERNAL_FEED_ENDPOINTS' + 'issecret' = 'false' + } + + # We don't want sessions cached since we will be updating the endpoints quite frequently + Write-LoggingCommand -Area 'task' -Event 'setvariable' -Data 'False' -Properties @{ + 'variable' = 'NUGET_CREDENTIALPROVIDER_SESSIONTOKENCACHE_ENABLED' + 'issecret' = 'false' + } + } + else + { + Write-Host 'No internal endpoints found in NuGet.config' + } +} + +#Workaround for https://github.com/microsoft/msbuild/issues/4430 +function InstallDotNetSdkAndRestoreArcade { + $dotnetTempDir = Join-Path $RepoRoot "dotnet" + $dotnetSdkVersion="2.1.507" # After experimentation we know this version works when restoring the SDK (compared to 3.0.*) + $dotnet = "$dotnetTempDir\dotnet.exe" + $restoreProjPath = "$PSScriptRoot\restore.proj" + + Write-Host "Installing dotnet SDK version $dotnetSdkVersion to restore Arcade SDK..." + InstallDotNetSdk "$dotnetTempDir" "$dotnetSdkVersion" + + '' | Out-File "$restoreProjPath" + + & $dotnet restore $restoreProjPath + + Write-Host 'Arcade SDK restored!' + + if (Test-Path -Path $restoreProjPath) { + Remove-Item $restoreProjPath + } + + if (Test-Path -Path $dotnetTempDir) { + Remove-Item $dotnetTempDir -Recurse + } +} + +try { + Push-Location $PSScriptRoot + + if ($Operation -like 'setup') { + SetupCredProvider $AuthToken + } + elseif ($Operation -like 'install-restore') { + InstallDotNetSdkAndRestoreArcade + } + else { + Write-PipelineTelemetryError -Category 'Arcade' -Message "Unknown operation '$Operation'!" + ExitWithExitCode 1 + } +} +catch { + Write-Host $_.ScriptStackTrace + Write-PipelineTelemetryError -Category 'Arcade' -Message $_ + ExitWithExitCode 1 +} +finally { + Pop-Location +} diff --git a/src/arcade/eng/common/internal-feed-operations.sh b/src/arcade/eng/common/internal-feed-operations.sh new file mode 100644 index 000000000000..9378223ba095 --- /dev/null +++ b/src/arcade/eng/common/internal-feed-operations.sh @@ -0,0 +1,141 @@ +#!/usr/bin/env bash + +set -e + +# Sets VSS_NUGET_EXTERNAL_FEED_ENDPOINTS based on the "darc-int-*" feeds defined in NuGet.config. This is needed +# in build agents by CredProvider to authenticate the restore requests to internal feeds as specified in +# https://github.com/microsoft/artifacts-credprovider/blob/0f53327cd12fd893d8627d7b08a2171bf5852a41/README.md#environment-variables. +# This should ONLY be called from identified internal builds +function SetupCredProvider { + local authToken=$1 + + # Install the Cred Provider NuGet plugin + echo "Setting up Cred Provider NuGet plugin in the agent..."... + echo "Getting 'installcredprovider.ps1' from 'https://github.com/microsoft/artifacts-credprovider'..." + + local url="https://raw.githubusercontent.com/microsoft/artifacts-credprovider/master/helpers/installcredprovider.sh" + + echo "Writing the contents of 'installcredprovider.ps1' locally..." + local installcredproviderPath="installcredprovider.sh" + if command -v curl > /dev/null; then + curl $url > "$installcredproviderPath" + else + wget -q -O "$installcredproviderPath" "$url" + fi + + echo "Installing plugin..." + . "$installcredproviderPath" + + echo "Deleting local copy of 'installcredprovider.sh'..." + rm installcredprovider.sh + + if [ ! -d "$HOME/.nuget/plugins" ]; then + Write-PipelineTelemetryError -category 'Build' 'CredProvider plugin was not installed correctly!' + ExitWithExitCode 1 + else + echo "CredProvider plugin was installed correctly!" + fi + + # Then, we set the 'VSS_NUGET_EXTERNAL_FEED_ENDPOINTS' environment variable to restore from the stable + # feeds successfully + + local nugetConfigPath="{$repo_root}NuGet.config" + + if [ ! "$nugetConfigPath" ]; then + Write-PipelineTelemetryError -category 'Build' "NuGet.config file not found in repo's root!" + ExitWithExitCode 1 + fi + + local endpoints='[' + local nugetConfigPackageValues=`cat "$nugetConfigPath" | grep "key=\"darc-int-"` + local pattern="value=\"(.*)\"" + + for value in $nugetConfigPackageValues + do + if [[ $value =~ $pattern ]]; then + local endpoint="${BASH_REMATCH[1]}" + endpoints+="{\"endpoint\": \"$endpoint\", \"password\": \"$authToken\"}," + fi + done + + endpoints=${endpoints%?} + endpoints+=']' + + if [ ${#endpoints} -gt 2 ]; then + local endpointCredentials="{\"endpointCredentials\": "$endpoints"}" + + echo "##vso[task.setvariable variable=VSS_NUGET_EXTERNAL_FEED_ENDPOINTS]$endpointCredentials" + echo "##vso[task.setvariable variable=NUGET_CREDENTIALPROVIDER_SESSIONTOKENCACHE_ENABLED]False" + else + echo "No internal endpoints found in NuGet.config" + fi +} + +# Workaround for https://github.com/microsoft/msbuild/issues/4430 +function InstallDotNetSdkAndRestoreArcade { + local dotnetTempDir="$repo_root/dotnet" + local dotnetSdkVersion="2.1.507" # After experimentation we know this version works when restoring the SDK (compared to 3.0.*) + local restoreProjPath="$repo_root/eng/common/restore.proj" + + echo "Installing dotnet SDK version $dotnetSdkVersion to restore Arcade SDK..." + echo "" > "$restoreProjPath" + + InstallDotNetSdk "$dotnetTempDir" "$dotnetSdkVersion" + + local res=`$dotnetTempDir/dotnet restore $restoreProjPath` + echo "Arcade SDK restored!" + + # Cleanup + if [ "$restoreProjPath" ]; then + rm "$restoreProjPath" + fi + + if [ "$dotnetTempDir" ]; then + rm -r $dotnetTempDir + fi +} + +source="${BASH_SOURCE[0]}" +operation='' +authToken='' +repoName='' + +while [[ $# > 0 ]]; do + opt="$(echo "$1" | tr "[:upper:]" "[:lower:]")" + case "$opt" in + --operation) + operation=$2 + shift + ;; + --authtoken) + authToken=$2 + shift + ;; + *) + echo "Invalid argument: $1" + usage + exit 1 + ;; + esac + + shift +done + +while [[ -h "$source" ]]; do + scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" + source="$(readlink "$source")" + # if $source was a relative symlink, we need to resolve it relative to the path where the + # symlink file was located + [[ $source != /* ]] && source="$scriptroot/$source" +done +scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" + +. "$scriptroot/tools.sh" + +if [ "$operation" = "setup" ]; then + SetupCredProvider $authToken +elif [ "$operation" = "install-restore" ]; then + InstallDotNetSdkAndRestoreArcade +else + echo "Unknown operation '$operation'!" +fi diff --git a/src/arcade/eng/common/internal/NuGet.config b/src/arcade/eng/common/internal/NuGet.config new file mode 100644 index 000000000000..19d3d311b166 --- /dev/null +++ b/src/arcade/eng/common/internal/NuGet.config @@ -0,0 +1,7 @@ + + + + + + + diff --git a/src/arcade/eng/common/loc/P22DotNetHtmlLocalization.lss b/src/arcade/eng/common/loc/P22DotNetHtmlLocalization.lss new file mode 100644 index 000000000000..5d892d619398 --- /dev/null +++ b/src/arcade/eng/common/loc/P22DotNetHtmlLocalization.lss @@ -0,0 +1,29 @@ + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/src/arcade/eng/common/msbuild.ps1 b/src/arcade/eng/common/msbuild.ps1 new file mode 100644 index 000000000000..f041e5ddd958 --- /dev/null +++ b/src/arcade/eng/common/msbuild.ps1 @@ -0,0 +1,28 @@ +[CmdletBinding(PositionalBinding=$false)] +Param( + [string] $verbosity = 'minimal', + [bool] $warnAsError = $true, + [bool] $nodeReuse = $true, + [switch] $ci, + [switch] $prepareMachine, + [switch] $excludePrereleaseVS, + [string] $msbuildEngine = $null, + [Parameter(ValueFromRemainingArguments=$true)][String[]]$extraArgs +) + +. $PSScriptRoot\tools.ps1 + +try { + if ($ci) { + $nodeReuse = $false + } + + MSBuild @extraArgs +} +catch { + Write-Host $_.ScriptStackTrace + Write-PipelineTelemetryError -Category 'Build' -Message $_ + ExitWithExitCode 1 +} + +ExitWithExitCode 0 \ No newline at end of file diff --git a/src/arcade/eng/common/msbuild.sh b/src/arcade/eng/common/msbuild.sh new file mode 100644 index 000000000000..20d3dad54352 --- /dev/null +++ b/src/arcade/eng/common/msbuild.sh @@ -0,0 +1,58 @@ +#!/usr/bin/env bash + +source="${BASH_SOURCE[0]}" + +# resolve $source until the file is no longer a symlink +while [[ -h "$source" ]]; do + scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" + source="$(readlink "$source")" + # if $source was a relative symlink, we need to resolve it relative to the path where the + # symlink file was located + [[ $source != /* ]] && source="$scriptroot/$source" +done +scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" + +verbosity='minimal' +warn_as_error=true +node_reuse=true +prepare_machine=false +extra_args='' + +while (($# > 0)); do + lowerI="$(echo $1 | tr "[:upper:]" "[:lower:]")" + case $lowerI in + --verbosity) + verbosity=$2 + shift 2 + ;; + --warnaserror) + warn_as_error=$2 + shift 2 + ;; + --nodereuse) + node_reuse=$2 + shift 2 + ;; + --ci) + ci=true + shift 1 + ;; + --preparemachine) + prepare_machine=true + shift 1 + ;; + *) + extra_args="$extra_args $1" + shift 1 + ;; + esac +done + +. "$scriptroot/tools.sh" + +if [[ "$ci" == true ]]; then + node_reuse=false +fi + +MSBuild $extra_args +ExitWithExitCode 0 diff --git a/src/arcade/eng/common/native/common-library.sh b/src/arcade/eng/common/native/common-library.sh new file mode 100644 index 000000000000..080c2c283ae4 --- /dev/null +++ b/src/arcade/eng/common/native/common-library.sh @@ -0,0 +1,172 @@ +#!/usr/bin/env bash + +function GetNativeInstallDirectory { + local install_dir + + if [[ -z $NETCOREENG_INSTALL_DIRECTORY ]]; then + install_dir=$HOME/.netcoreeng/native/ + else + install_dir=$NETCOREENG_INSTALL_DIRECTORY + fi + + echo $install_dir + return 0 +} + +function GetTempDirectory { + + echo $(GetNativeInstallDirectory)temp/ + return 0 +} + +function ExpandZip { + local zip_path=$1 + local output_directory=$2 + local force=${3:-false} + + echo "Extracting $zip_path to $output_directory" + if [[ -d $output_directory ]] && [[ $force = false ]]; then + echo "Directory '$output_directory' already exists, skipping extract" + return 0 + fi + + if [[ -d $output_directory ]]; then + echo "'Force flag enabled, but '$output_directory' exists. Removing directory" + rm -rf $output_directory + if [[ $? != 0 ]]; then + Write-PipelineTelemetryError -category 'NativeToolsBootstrap' "Unable to remove '$output_directory'" + return 1 + fi + fi + + echo "Creating directory: '$output_directory'" + mkdir -p $output_directory + + echo "Extracting archive" + tar -xf $zip_path -C $output_directory + if [[ $? != 0 ]]; then + Write-PipelineTelemetryError -category 'NativeToolsBootstrap' "Unable to extract '$zip_path'" + return 1 + fi + + return 0 +} + +function GetCurrentOS { + local unameOut="$(uname -s)" + case $unameOut in + Linux*) echo "Linux";; + Darwin*) echo "MacOS";; + esac + return 0 +} + +function GetFile { + local uri=$1 + local path=$2 + local force=${3:-false} + local download_retries=${4:-5} + local retry_wait_time_seconds=${5:-30} + + if [[ -f $path ]]; then + if [[ $force = false ]]; then + echo "File '$path' already exists. Skipping download" + return 0 + else + rm -rf $path + fi + fi + + if [[ -f $uri ]]; then + echo "'$uri' is a file path, copying file to '$path'" + cp $uri $path + return $? + fi + + echo "Downloading $uri" + # Use curl if available, otherwise use wget + if command -v curl > /dev/null; then + curl "$uri" -sSL --retry $download_retries --retry-delay $retry_wait_time_seconds --create-dirs -o "$path" --fail + else + wget -q -O "$path" "$uri" --tries="$download_retries" + fi + + return $? +} + +function GetTempPathFileName { + local path=$1 + + local temp_dir=$(GetTempDirectory) + local temp_file_name=$(basename $path) + echo $temp_dir$temp_file_name + return 0 +} + +function DownloadAndExtract { + local uri=$1 + local installDir=$2 + local force=${3:-false} + local download_retries=${4:-5} + local retry_wait_time_seconds=${5:-30} + + local temp_tool_path=$(GetTempPathFileName $uri) + + echo "downloading to: $temp_tool_path" + + # Download file + GetFile "$uri" "$temp_tool_path" $force $download_retries $retry_wait_time_seconds + if [[ $? != 0 ]]; then + Write-PipelineTelemetryError -category 'NativeToolsBootstrap' "Failed to download '$uri' to '$temp_tool_path'." + return 1 + fi + + # Extract File + echo "extracting from $temp_tool_path to $installDir" + ExpandZip "$temp_tool_path" "$installDir" $force $download_retries $retry_wait_time_seconds + if [[ $? != 0 ]]; then + Write-PipelineTelemetryError -category 'NativeToolsBootstrap' "Failed to extract '$temp_tool_path' to '$installDir'." + return 1 + fi + + return 0 +} + +function NewScriptShim { + local shimpath=$1 + local tool_file_path=$2 + local force=${3:-false} + + echo "Generating '$shimpath' shim" + if [[ -f $shimpath ]]; then + if [[ $force = false ]]; then + echo "File '$shimpath' already exists." >&2 + return 1 + else + rm -rf $shimpath + fi + fi + + if [[ ! -f $tool_file_path ]]; then + # try to see if the path is lower cased + tool_file_path="$(echo $tool_file_path | tr "[:upper:]" "[:lower:]")" + if [[ ! -f $tool_file_path ]]; then + Write-PipelineTelemetryError -category 'NativeToolsBootstrap' "Specified tool file path:'$tool_file_path' does not exist" + return 1 + fi + fi + + local shim_contents=$'#!/usr/bin/env bash\n' + shim_contents+="SHIMARGS="$'$1\n' + shim_contents+="$tool_file_path"$' $SHIMARGS\n' + + # Write shim file + echo "$shim_contents" > $shimpath + + chmod +x $shimpath + + echo "Finished generating shim '$shimpath'" + + return $? +} + diff --git a/src/arcade/eng/common/native/install-cmake-test.sh b/src/arcade/eng/common/native/install-cmake-test.sh new file mode 100644 index 000000000000..8a5e7cf0db5a --- /dev/null +++ b/src/arcade/eng/common/native/install-cmake-test.sh @@ -0,0 +1,117 @@ +#!/usr/bin/env bash + +source="${BASH_SOURCE[0]}" +scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" + +. $scriptroot/common-library.sh + +base_uri= +install_path= +version= +clean=false +force=false +download_retries=5 +retry_wait_time_seconds=30 + +while (($# > 0)); do + lowerI="$(echo $1 | tr "[:upper:]" "[:lower:]")" + case $lowerI in + --baseuri) + base_uri=$2 + shift 2 + ;; + --installpath) + install_path=$2 + shift 2 + ;; + --version) + version=$2 + shift 2 + ;; + --clean) + clean=true + shift 1 + ;; + --force) + force=true + shift 1 + ;; + --downloadretries) + download_retries=$2 + shift 2 + ;; + --retrywaittimeseconds) + retry_wait_time_seconds=$2 + shift 2 + ;; + --help) + echo "Common settings:" + echo " --baseuri Base file directory or Url wrom which to acquire tool archives" + echo " --installpath Base directory to install native tool to" + echo " --clean Don't install the tool, just clean up the current install of the tool" + echo " --force Force install of tools even if they previously exist" + echo " --help Print help and exit" + echo "" + echo "Advanced settings:" + echo " --downloadretries Total number of retry attempts" + echo " --retrywaittimeseconds Wait time between retry attempts in seconds" + echo "" + exit 0 + ;; + esac +done + +tool_name="cmake-test" +tool_os=$(GetCurrentOS) +tool_folder="$(echo $tool_os | tr "[:upper:]" "[:lower:]")" +tool_arch="x86_64" +tool_name_moniker="$tool_name-$version-$tool_os-$tool_arch" +tool_install_directory="$install_path/$tool_name/$version" +tool_file_path="$tool_install_directory/$tool_name_moniker/bin/$tool_name" +shim_path="$install_path/$tool_name.sh" +uri="${base_uri}/$tool_folder/$tool_name/$tool_name_moniker.tar.gz" + +# Clean up tool and installers +if [[ $clean = true ]]; then + echo "Cleaning $tool_install_directory" + if [[ -d $tool_install_directory ]]; then + rm -rf $tool_install_directory + fi + + echo "Cleaning $shim_path" + if [[ -f $shim_path ]]; then + rm -rf $shim_path + fi + + tool_temp_path=$(GetTempPathFileName $uri) + echo "Cleaning $tool_temp_path" + if [[ -f $tool_temp_path ]]; then + rm -rf $tool_temp_path + fi + + exit 0 +fi + +# Install tool +if [[ -f $tool_file_path ]] && [[ $force = false ]]; then + echo "$tool_name ($version) already exists, skipping install" + exit 0 +fi + +DownloadAndExtract $uri $tool_install_directory $force $download_retries $retry_wait_time_seconds + +if [[ $? != 0 ]]; then + Write-PipelineTelemetryError -category 'NativeToolsBootstrap' 'Installation failed' + exit 1 +fi + +# Generate Shim +# Always rewrite shims so that we are referencing the expected version +NewScriptShim $shim_path $tool_file_path true + +if [[ $? != 0 ]]; then + Write-PipelineTelemetryError -category 'NativeToolsBootstrap' 'Shim generation failed' + exit 1 +fi + +exit 0 diff --git a/src/arcade/eng/common/native/install-cmake.sh b/src/arcade/eng/common/native/install-cmake.sh new file mode 100644 index 000000000000..de496beebc5a --- /dev/null +++ b/src/arcade/eng/common/native/install-cmake.sh @@ -0,0 +1,117 @@ +#!/usr/bin/env bash + +source="${BASH_SOURCE[0]}" +scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" + +. $scriptroot/common-library.sh + +base_uri= +install_path= +version= +clean=false +force=false +download_retries=5 +retry_wait_time_seconds=30 + +while (($# > 0)); do + lowerI="$(echo $1 | tr "[:upper:]" "[:lower:]")" + case $lowerI in + --baseuri) + base_uri=$2 + shift 2 + ;; + --installpath) + install_path=$2 + shift 2 + ;; + --version) + version=$2 + shift 2 + ;; + --clean) + clean=true + shift 1 + ;; + --force) + force=true + shift 1 + ;; + --downloadretries) + download_retries=$2 + shift 2 + ;; + --retrywaittimeseconds) + retry_wait_time_seconds=$2 + shift 2 + ;; + --help) + echo "Common settings:" + echo " --baseuri Base file directory or Url wrom which to acquire tool archives" + echo " --installpath Base directory to install native tool to" + echo " --clean Don't install the tool, just clean up the current install of the tool" + echo " --force Force install of tools even if they previously exist" + echo " --help Print help and exit" + echo "" + echo "Advanced settings:" + echo " --downloadretries Total number of retry attempts" + echo " --retrywaittimeseconds Wait time between retry attempts in seconds" + echo "" + exit 0 + ;; + esac +done + +tool_name="cmake" +tool_os=$(GetCurrentOS) +tool_folder="$(echo $tool_os | tr "[:upper:]" "[:lower:]")" +tool_arch="x86_64" +tool_name_moniker="$tool_name-$version-$tool_os-$tool_arch" +tool_install_directory="$install_path/$tool_name/$version" +tool_file_path="$tool_install_directory/$tool_name_moniker/bin/$tool_name" +shim_path="$install_path/$tool_name.sh" +uri="${base_uri}/$tool_folder/$tool_name/$tool_name_moniker.tar.gz" + +# Clean up tool and installers +if [[ $clean = true ]]; then + echo "Cleaning $tool_install_directory" + if [[ -d $tool_install_directory ]]; then + rm -rf $tool_install_directory + fi + + echo "Cleaning $shim_path" + if [[ -f $shim_path ]]; then + rm -rf $shim_path + fi + + tool_temp_path=$(GetTempPathFileName $uri) + echo "Cleaning $tool_temp_path" + if [[ -f $tool_temp_path ]]; then + rm -rf $tool_temp_path + fi + + exit 0 +fi + +# Install tool +if [[ -f $tool_file_path ]] && [[ $force = false ]]; then + echo "$tool_name ($version) already exists, skipping install" + exit 0 +fi + +DownloadAndExtract $uri $tool_install_directory $force $download_retries $retry_wait_time_seconds + +if [[ $? != 0 ]]; then + Write-PipelineTelemetryError -category 'NativeToolsBootstrap' 'Installation failed' + exit 1 +fi + +# Generate Shim +# Always rewrite shims so that we are referencing the expected version +NewScriptShim $shim_path $tool_file_path true + +if [[ $? != 0 ]]; then + Write-PipelineTelemetryError -category 'NativeToolsBootstrap' 'Shim generation failed' + exit 1 +fi + +exit 0 diff --git a/src/arcade/eng/common/native/install-tool.ps1 b/src/arcade/eng/common/native/install-tool.ps1 new file mode 100644 index 000000000000..78f2d84a4e4b --- /dev/null +++ b/src/arcade/eng/common/native/install-tool.ps1 @@ -0,0 +1,132 @@ +<# +.SYNOPSIS +Install native tool + +.DESCRIPTION +Install cmake native tool from Azure blob storage + +.PARAMETER InstallPath +Base directory to install native tool to + +.PARAMETER BaseUri +Base file directory or Url from which to acquire tool archives + +.PARAMETER CommonLibraryDirectory +Path to folder containing common library modules + +.PARAMETER Force +Force install of tools even if they previously exist + +.PARAMETER Clean +Don't install the tool, just clean up the current install of the tool + +.PARAMETER DownloadRetries +Total number of retry attempts + +.PARAMETER RetryWaitTimeInSeconds +Wait time between retry attempts in seconds + +.NOTES +Returns 0 if install succeeds, 1 otherwise +#> +[CmdletBinding(PositionalBinding=$false)] +Param ( + [Parameter(Mandatory=$True)] + [string] $ToolName, + [Parameter(Mandatory=$True)] + [string] $InstallPath, + [Parameter(Mandatory=$True)] + [string] $BaseUri, + [Parameter(Mandatory=$True)] + [string] $Version, + [string] $CommonLibraryDirectory = $PSScriptRoot, + [switch] $Force = $False, + [switch] $Clean = $False, + [int] $DownloadRetries = 5, + [int] $RetryWaitTimeInSeconds = 30 +) + +. $PSScriptRoot\..\pipeline-logging-functions.ps1 + +# Import common library modules +Import-Module -Name (Join-Path $CommonLibraryDirectory "CommonLibrary.psm1") + +try { + # Define verbose switch if undefined + $Verbose = $VerbosePreference -Eq "Continue" + + $Arch = CommonLibrary\Get-MachineArchitecture + $ToolOs = "win64" + if($Arch -Eq "x32") { + $ToolOs = "win32" + } + $ToolNameMoniker = "$ToolName-$Version-$ToolOs-$Arch" + $ToolInstallDirectory = Join-Path $InstallPath "$ToolName\$Version\" + $Uri = "$BaseUri/windows/$ToolName/$ToolNameMoniker.zip" + $ShimPath = Join-Path $InstallPath "$ToolName.exe" + + if ($Clean) { + Write-Host "Cleaning $ToolInstallDirectory" + if (Test-Path $ToolInstallDirectory) { + Remove-Item $ToolInstallDirectory -Force -Recurse + } + Write-Host "Cleaning $ShimPath" + if (Test-Path $ShimPath) { + Remove-Item $ShimPath -Force + } + $ToolTempPath = CommonLibrary\Get-TempPathFilename -Path $Uri + Write-Host "Cleaning $ToolTempPath" + if (Test-Path $ToolTempPath) { + Remove-Item $ToolTempPath -Force + } + exit 0 + } + + # Install tool + if ((Test-Path $ToolInstallDirectory) -And (-Not $Force)) { + Write-Verbose "$ToolName ($Version) already exists, skipping install" + } + else { + $InstallStatus = CommonLibrary\DownloadAndExtract -Uri $Uri ` + -InstallDirectory $ToolInstallDirectory ` + -Force:$Force ` + -DownloadRetries $DownloadRetries ` + -RetryWaitTimeInSeconds $RetryWaitTimeInSeconds ` + -Verbose:$Verbose + + if ($InstallStatus -Eq $False) { + Write-PipelineTelemetryError "Installation failed" -Category "NativeToolsetBootstrapping" + exit 1 + } + } + + $ToolFilePath = Get-ChildItem $ToolInstallDirectory -Recurse -Filter "$ToolName.exe" | % { $_.FullName } + if (@($ToolFilePath).Length -Gt 1) { + Write-Error "There are multiple copies of $ToolName in $($ToolInstallDirectory): `n$(@($ToolFilePath | out-string))" + exit 1 + } elseif (@($ToolFilePath).Length -Lt 1) { + Write-Host "$ToolName was not found in $ToolInstallDirectory." + exit 1 + } + + # Generate shim + # Always rewrite shims so that we are referencing the expected version + $GenerateShimStatus = CommonLibrary\New-ScriptShim -ShimName $ToolName ` + -ShimDirectory $InstallPath ` + -ToolFilePath "$ToolFilePath" ` + -BaseUri $BaseUri ` + -Force:$Force ` + -Verbose:$Verbose + + if ($GenerateShimStatus -Eq $False) { + Write-PipelineTelemetryError "Generate shim failed" -Category "NativeToolsetBootstrapping" + return 1 + } + + exit 0 +} +catch { + Write-Host $_.ScriptStackTrace + Write-PipelineTelemetryError -Category "NativeToolsetBootstrapping" -Message $_ + exit 1 +} diff --git a/src/arcade/eng/common/pipeline-logging-functions.ps1 b/src/arcade/eng/common/pipeline-logging-functions.ps1 new file mode 100644 index 000000000000..8e422c561e4b --- /dev/null +++ b/src/arcade/eng/common/pipeline-logging-functions.ps1 @@ -0,0 +1,260 @@ +# Source for this file was taken from https://github.com/microsoft/azure-pipelines-task-lib/blob/11c9439d4af17e6475d9fe058e6b2e03914d17e6/powershell/VstsTaskSdk/LoggingCommandFunctions.ps1 and modified. + +# NOTE: You should not be calling these method directly as they are likely to change. Instead you should be calling the Write-Pipeline* functions defined in tools.ps1 + +$script:loggingCommandPrefix = '##vso[' +$script:loggingCommandEscapeMappings = @( # TODO: WHAT ABOUT "="? WHAT ABOUT "%"? + New-Object psobject -Property @{ Token = ';' ; Replacement = '%3B' } + New-Object psobject -Property @{ Token = "`r" ; Replacement = '%0D' } + New-Object psobject -Property @{ Token = "`n" ; Replacement = '%0A' } + New-Object psobject -Property @{ Token = "]" ; Replacement = '%5D' } +) +# TODO: BUG: Escape % ??? +# TODO: Add test to verify don't need to escape "=". + +# Specify "-Force" to force pipeline formatted output even if "$ci" is false or not set +function Write-PipelineTelemetryError { + [CmdletBinding()] + param( + [Parameter(Mandatory = $true)] + [string]$Category, + [Parameter(Mandatory = $true)] + [string]$Message, + [Parameter(Mandatory = $false)] + [string]$Type = 'error', + [string]$ErrCode, + [string]$SourcePath, + [string]$LineNumber, + [string]$ColumnNumber, + [switch]$AsOutput, + [switch]$Force) + + $PSBoundParameters.Remove('Category') | Out-Null + + if ($Force -Or ((Test-Path variable:ci) -And $ci)) { + $Message = "(NETCORE_ENGINEERING_TELEMETRY=$Category) $Message" + } + $PSBoundParameters.Remove('Message') | Out-Null + $PSBoundParameters.Add('Message', $Message) + Write-PipelineTaskError @PSBoundParameters +} + +# Specify "-Force" to force pipeline formatted output even if "$ci" is false or not set +function Write-PipelineTaskError { + [CmdletBinding()] + param( + [Parameter(Mandatory = $true)] + [string]$Message, + [Parameter(Mandatory = $false)] + [string]$Type = 'error', + [string]$ErrCode, + [string]$SourcePath, + [string]$LineNumber, + [string]$ColumnNumber, + [switch]$AsOutput, + [switch]$Force + ) + + if (!$Force -And (-Not (Test-Path variable:ci) -Or !$ci)) { + if ($Type -eq 'error') { + Write-Host $Message -ForegroundColor Red + return + } + elseif ($Type -eq 'warning') { + Write-Host $Message -ForegroundColor Yellow + return + } + } + + if (($Type -ne 'error') -and ($Type -ne 'warning')) { + Write-Host $Message + return + } + $PSBoundParameters.Remove('Force') | Out-Null + if (-not $PSBoundParameters.ContainsKey('Type')) { + $PSBoundParameters.Add('Type', 'error') + } + Write-LogIssue @PSBoundParameters +} + +function Write-PipelineSetVariable { + [CmdletBinding()] + param( + [Parameter(Mandatory = $true)] + [string]$Name, + [string]$Value, + [switch]$Secret, + [switch]$AsOutput, + [bool]$IsMultiJobVariable = $true) + + if ((Test-Path variable:ci) -And $ci) { + Write-LoggingCommand -Area 'task' -Event 'setvariable' -Data $Value -Properties @{ + 'variable' = $Name + 'isSecret' = $Secret + 'isOutput' = $IsMultiJobVariable + } -AsOutput:$AsOutput + } +} + +function Write-PipelinePrependPath { + [CmdletBinding()] + param( + [Parameter(Mandatory = $true)] + [string]$Path, + [switch]$AsOutput) + + if ((Test-Path variable:ci) -And $ci) { + Write-LoggingCommand -Area 'task' -Event 'prependpath' -Data $Path -AsOutput:$AsOutput + } +} + +function Write-PipelineSetResult { + [CmdletBinding()] + param( + [ValidateSet("Succeeded", "SucceededWithIssues", "Failed", "Cancelled", "Skipped")] + [Parameter(Mandatory = $true)] + [string]$Result, + [string]$Message) + if ((Test-Path variable:ci) -And $ci) { + Write-LoggingCommand -Area 'task' -Event 'complete' -Data $Message -Properties @{ + 'result' = $Result + } + } +} + +<######################################## +# Private functions. +########################################> +function Format-LoggingCommandData { + [CmdletBinding()] + param([string]$Value, [switch]$Reverse) + + if (!$Value) { + return '' + } + + if (!$Reverse) { + foreach ($mapping in $script:loggingCommandEscapeMappings) { + $Value = $Value.Replace($mapping.Token, $mapping.Replacement) + } + } + else { + for ($i = $script:loggingCommandEscapeMappings.Length - 1 ; $i -ge 0 ; $i--) { + $mapping = $script:loggingCommandEscapeMappings[$i] + $Value = $Value.Replace($mapping.Replacement, $mapping.Token) + } + } + + return $Value +} + +function Format-LoggingCommand { + [CmdletBinding()] + param( + [Parameter(Mandatory = $true)] + [string]$Area, + [Parameter(Mandatory = $true)] + [string]$Event, + [string]$Data, + [hashtable]$Properties) + + # Append the preamble. + [System.Text.StringBuilder]$sb = New-Object -TypeName System.Text.StringBuilder + $null = $sb.Append($script:loggingCommandPrefix).Append($Area).Append('.').Append($Event) + + # Append the properties. + if ($Properties) { + $first = $true + foreach ($key in $Properties.Keys) { + [string]$value = Format-LoggingCommandData $Properties[$key] + if ($value) { + if ($first) { + $null = $sb.Append(' ') + $first = $false + } + else { + $null = $sb.Append(';') + } + + $null = $sb.Append("$key=$value") + } + } + } + + # Append the tail and output the value. + $Data = Format-LoggingCommandData $Data + $sb.Append(']').Append($Data).ToString() +} + +function Write-LoggingCommand { + [CmdletBinding(DefaultParameterSetName = 'Parameters')] + param( + [Parameter(Mandatory = $true, ParameterSetName = 'Parameters')] + [string]$Area, + [Parameter(Mandatory = $true, ParameterSetName = 'Parameters')] + [string]$Event, + [Parameter(ParameterSetName = 'Parameters')] + [string]$Data, + [Parameter(ParameterSetName = 'Parameters')] + [hashtable]$Properties, + [Parameter(Mandatory = $true, ParameterSetName = 'Object')] + $Command, + [switch]$AsOutput) + + if ($PSCmdlet.ParameterSetName -eq 'Object') { + Write-LoggingCommand -Area $Command.Area -Event $Command.Event -Data $Command.Data -Properties $Command.Properties -AsOutput:$AsOutput + return + } + + $command = Format-LoggingCommand -Area $Area -Event $Event -Data $Data -Properties $Properties + if ($AsOutput) { + $command + } + else { + Write-Host $command + } +} + +function Write-LogIssue { + [CmdletBinding()] + param( + [ValidateSet('warning', 'error')] + [Parameter(Mandatory = $true)] + [string]$Type, + [string]$Message, + [string]$ErrCode, + [string]$SourcePath, + [string]$LineNumber, + [string]$ColumnNumber, + [switch]$AsOutput) + + $command = Format-LoggingCommand -Area 'task' -Event 'logissue' -Data $Message -Properties @{ + 'type' = $Type + 'code' = $ErrCode + 'sourcepath' = $SourcePath + 'linenumber' = $LineNumber + 'columnnumber' = $ColumnNumber + } + if ($AsOutput) { + return $command + } + + if ($Type -eq 'error') { + $foregroundColor = $host.PrivateData.ErrorForegroundColor + $backgroundColor = $host.PrivateData.ErrorBackgroundColor + if ($foregroundColor -isnot [System.ConsoleColor] -or $backgroundColor -isnot [System.ConsoleColor]) { + $foregroundColor = [System.ConsoleColor]::Red + $backgroundColor = [System.ConsoleColor]::Black + } + } + else { + $foregroundColor = $host.PrivateData.WarningForegroundColor + $backgroundColor = $host.PrivateData.WarningBackgroundColor + if ($foregroundColor -isnot [System.ConsoleColor] -or $backgroundColor -isnot [System.ConsoleColor]) { + $foregroundColor = [System.ConsoleColor]::Yellow + $backgroundColor = [System.ConsoleColor]::Black + } + } + + Write-Host $command -ForegroundColor $foregroundColor -BackgroundColor $backgroundColor +} diff --git a/src/arcade/eng/common/pipeline-logging-functions.sh b/src/arcade/eng/common/pipeline-logging-functions.sh new file mode 100644 index 000000000000..6a0b2255e911 --- /dev/null +++ b/src/arcade/eng/common/pipeline-logging-functions.sh @@ -0,0 +1,206 @@ +#!/usr/bin/env bash + +function Write-PipelineTelemetryError { + local telemetry_category='' + local force=false + local function_args=() + local message='' + while [[ $# -gt 0 ]]; do + opt="$(echo "${1/#--/-}" | tr "[:upper:]" "[:lower:]")" + case "$opt" in + -category|-c) + telemetry_category=$2 + shift + ;; + -force|-f) + force=true + ;; + -*) + function_args+=("$1 $2") + shift + ;; + *) + message=$* + ;; + esac + shift + done + + if [[ $force != true ]] && [[ "$ci" != true ]]; then + echo "$message" >&2 + return + fi + + if [[ $force == true ]]; then + function_args+=("-force") + fi + message="(NETCORE_ENGINEERING_TELEMETRY=$telemetry_category) $message" + function_args+=("$message") + Write-PipelineTaskError ${function_args[@]} +} + +function Write-PipelineTaskError { + local message_type="error" + local sourcepath='' + local linenumber='' + local columnnumber='' + local error_code='' + local force=false + + while [[ $# -gt 0 ]]; do + opt="$(echo "${1/#--/-}" | tr "[:upper:]" "[:lower:]")" + case "$opt" in + -type|-t) + message_type=$2 + shift + ;; + -sourcepath|-s) + sourcepath=$2 + shift + ;; + -linenumber|-ln) + linenumber=$2 + shift + ;; + -columnnumber|-cn) + columnnumber=$2 + shift + ;; + -errcode|-e) + error_code=$2 + shift + ;; + -force|-f) + force=true + ;; + *) + break + ;; + esac + + shift + done + + if [[ $force != true ]] && [[ "$ci" != true ]]; then + echo "$@" >&2 + return + fi + + local message="##vso[task.logissue" + + message="$message type=$message_type" + + if [ -n "$sourcepath" ]; then + message="$message;sourcepath=$sourcepath" + fi + + if [ -n "$linenumber" ]; then + message="$message;linenumber=$linenumber" + fi + + if [ -n "$columnnumber" ]; then + message="$message;columnnumber=$columnnumber" + fi + + if [ -n "$error_code" ]; then + message="$message;code=$error_code" + fi + + message="$message]$*" + echo "$message" +} + +function Write-PipelineSetVariable { + if [[ "$ci" != true ]]; then + return + fi + + local name='' + local value='' + local secret=false + local as_output=false + local is_multi_job_variable=true + + while [[ $# -gt 0 ]]; do + opt="$(echo "${1/#--/-}" | tr "[:upper:]" "[:lower:]")" + case "$opt" in + -name|-n) + name=$2 + shift + ;; + -value|-v) + value=$2 + shift + ;; + -secret|-s) + secret=true + ;; + -as_output|-a) + as_output=true + ;; + -is_multi_job_variable|-i) + is_multi_job_variable=$2 + shift + ;; + esac + shift + done + + value=${value/;/%3B} + value=${value/\\r/%0D} + value=${value/\\n/%0A} + value=${value/]/%5D} + + local message="##vso[task.setvariable variable=$name;isSecret=$secret;isOutput=$is_multi_job_variable]$value" + + if [[ "$as_output" == true ]]; then + $message + else + echo "$message" + fi +} + +function Write-PipelinePrependPath { + local prepend_path='' + + while [[ $# -gt 0 ]]; do + opt="$(echo "${1/#--/-}" | tr "[:upper:]" "[:lower:]")" + case "$opt" in + -path|-p) + prepend_path=$2 + shift + ;; + esac + shift + done + + export PATH="$prepend_path:$PATH" + + if [[ "$ci" == true ]]; then + echo "##vso[task.prependpath]$prepend_path" + fi +} + +function Write-PipelineSetResult { + local result='' + local message='' + + while [[ $# -gt 0 ]]; do + opt="$(echo "${1/#--/-}" | tr "[:upper:]" "[:lower:]")" + case "$opt" in + -result|-r) + result=$2 + shift + ;; + -message|-m) + message=$2 + shift + ;; + esac + shift + done + + if [[ "$ci" == true ]]; then + echo "##vso[task.complete result=$result;]$message" + fi +} diff --git a/src/arcade/eng/common/retain-build.ps1 b/src/arcade/eng/common/retain-build.ps1 new file mode 100644 index 000000000000..e7ba975adeb6 --- /dev/null +++ b/src/arcade/eng/common/retain-build.ps1 @@ -0,0 +1,45 @@ + +Param( +[Parameter(Mandatory=$true)][int] $buildId, +[Parameter(Mandatory=$true)][string] $azdoOrgUri, +[Parameter(Mandatory=$true)][string] $azdoProject, +[Parameter(Mandatory=$true)][string] $token +) + +$ErrorActionPreference = 'Stop' +Set-StrictMode -Version 2.0 + +function Get-AzDOHeaders( + [string] $token) +{ + $base64AuthInfo = [Convert]::ToBase64String([Text.Encoding]::ASCII.GetBytes(":${token}")) + $headers = @{"Authorization"="Basic $base64AuthInfo"} + return $headers +} + +function Update-BuildRetention( + [string] $azdoOrgUri, + [string] $azdoProject, + [int] $buildId, + [string] $token) +{ + $headers = Get-AzDOHeaders -token $token + $requestBody = "{ + `"keepForever`": `"true`" + }" + + $requestUri = "${azdoOrgUri}/${azdoProject}/_apis/build/builds/${buildId}?api-version=6.0" + write-Host "Attempting to retain build using the following URI: ${requestUri} ..." + + try { + Invoke-RestMethod -Uri $requestUri -Method Patch -Body $requestBody -Header $headers -contentType "application/json" + Write-Host "Updated retention settings for build ${buildId}." + } + catch { + Write-Error "Failed to update retention settings for build: $_.Exception.Response.StatusDescription" + exit 1 + } +} + +Update-BuildRetention -azdoOrgUri $azdoOrgUri -azdoProject $azdoProject -buildId $buildId -token $token +exit 0 diff --git a/src/arcade/eng/common/sdl/configure-sdl-tool.ps1 b/src/arcade/eng/common/sdl/configure-sdl-tool.ps1 new file mode 100644 index 000000000000..27f5a4115fc7 --- /dev/null +++ b/src/arcade/eng/common/sdl/configure-sdl-tool.ps1 @@ -0,0 +1,130 @@ +Param( + [string] $GuardianCliLocation, + [string] $WorkingDirectory, + [string] $TargetDirectory, + [string] $GdnFolder, + # The list of Guardian tools to configure. For each object in the array: + # - If the item is a [hashtable], it must contain these entries: + # - Name = The tool name as Guardian knows it. + # - Scenario = (Optional) Scenario-specific name for this configuration entry. It must be unique + # among all tool entries with the same Name. + # - Args = (Optional) Array of Guardian tool configuration args, like '@("Target > C:\temp")' + # - If the item is a [string] $v, it is treated as '@{ Name="$v" }' + [object[]] $ToolsList, + [string] $GuardianLoggerLevel='Standard', + # Optional: Additional params to add to any tool using CredScan. + [string[]] $CrScanAdditionalRunConfigParams, + # Optional: Additional params to add to any tool using PoliCheck. + [string[]] $PoliCheckAdditionalRunConfigParams, + # Optional: Additional params to add to any tool using CodeQL/Semmle. + [string[]] $CodeQLAdditionalRunConfigParams, + # Optional: Additional params to add to any tool using Binskim. + [string[]] $BinskimAdditionalRunConfigParams +) + +$ErrorActionPreference = 'Stop' +Set-StrictMode -Version 2.0 +$disableConfigureToolsetImport = $true +$global:LASTEXITCODE = 0 + +try { + # `tools.ps1` checks $ci to perform some actions. Since the SDL + # scripts don't necessarily execute in the same agent that run the + # build.ps1/sh script this variable isn't automatically set. + $ci = $true + . $PSScriptRoot\..\tools.ps1 + + # Normalize tools list: all in [hashtable] form with defined values for each key. + $ToolsList = $ToolsList | + ForEach-Object { + if ($_ -is [string]) { + $_ = @{ Name = $_ } + } + + if (-not ($_['Scenario'])) { $_.Scenario = "" } + if (-not ($_['Args'])) { $_.Args = @() } + $_ + } + + Write-Host "List of tools to configure:" + $ToolsList | ForEach-Object { $_ | Out-String | Write-Host } + + # We store config files in the r directory of .gdn + $gdnConfigPath = Join-Path $GdnFolder 'r' + $ValidPath = Test-Path $GuardianCliLocation + + if ($ValidPath -eq $False) + { + Write-PipelineTelemetryError -Force -Category 'Sdl' -Message "Invalid Guardian CLI Location." + ExitWithExitCode 1 + } + + foreach ($tool in $ToolsList) { + # Put together the name and scenario to make a unique key. + $toolConfigName = $tool.Name + if ($tool.Scenario) { + $toolConfigName += "_" + $tool.Scenario + } + + Write-Host "=== Configuring $toolConfigName..." + + $gdnConfigFile = Join-Path $gdnConfigPath "$toolConfigName-configure.gdnconfig" + + # For some tools, add default and automatic args. + switch -Exact ($tool.Name) { + 'credscan' { + if ($targetDirectory) { + $tool.Args += "`"TargetDirectory < $TargetDirectory`"" + } + $tool.Args += "`"OutputType < pre`"" + $tool.Args += $CrScanAdditionalRunConfigParams + } + 'policheck' { + if ($targetDirectory) { + $tool.Args += "`"Target < $TargetDirectory`"" + } + $tool.Args += $PoliCheckAdditionalRunConfigParams + } + {$_ -in 'semmle', 'codeql'} { + if ($targetDirectory) { + $tool.Args += "`"SourceCodeDirectory < $TargetDirectory`"" + } + $tool.Args += $CodeQLAdditionalRunConfigParams + } + 'binskim' { + if ($targetDirectory) { + # Binskim crashes due to specific PDBs. GitHub issue: https://github.com/microsoft/binskim/issues/924. + # We are excluding all `_.pdb` files from the scan. + $tool.Args += "`"Target < $TargetDirectory\**;-:file|$TargetDirectory\**\_.pdb`"" + } + $tool.Args += $BinskimAdditionalRunConfigParams + } + } + + # Create variable pointing to the args array directly so we can use splat syntax later. + $toolArgs = $tool.Args + + # Configure the tool. If args array is provided or the current tool has some default arguments + # defined, add "--args" and splat each element on the end. Arg format is "{Arg id} < {Value}", + # one per parameter. Doc page for "guardian configure": + # https://dev.azure.com/securitytools/SecurityIntegration/_wiki/wikis/Guardian/1395/configure + Exec-BlockVerbosely { + & $GuardianCliLocation configure ` + --working-directory $WorkingDirectory ` + --tool $tool.Name ` + --output-path $gdnConfigFile ` + --logger-level $GuardianLoggerLevel ` + --noninteractive ` + --force ` + $(if ($toolArgs) { "--args" }) @toolArgs + Exit-IfNZEC "Sdl" + } + + Write-Host "Created '$toolConfigName' configuration file: $gdnConfigFile" + } +} +catch { + Write-Host $_.ScriptStackTrace + Write-PipelineTelemetryError -Force -Category 'Sdl' -Message $_ + ExitWithExitCode 1 +} diff --git a/src/arcade/eng/common/sdl/extract-artifact-archives.ps1 b/src/arcade/eng/common/sdl/extract-artifact-archives.ps1 new file mode 100644 index 000000000000..68da4fbf2571 --- /dev/null +++ b/src/arcade/eng/common/sdl/extract-artifact-archives.ps1 @@ -0,0 +1,63 @@ +# This script looks for each archive file in a directory and extracts it into the target directory. +# For example, the file "$InputPath/bin.tar.gz" extracts to "$ExtractPath/bin.tar.gz.extracted/**". +# Uses the "tar" utility added to Windows 10 / Windows 2019 that supports tar.gz and zip. +param( + # Full path to directory where archives are stored. + [Parameter(Mandatory=$true)][string] $InputPath, + # Full path to directory to extract archives into. May be the same as $InputPath. + [Parameter(Mandatory=$true)][string] $ExtractPath +) + +$ErrorActionPreference = 'Stop' +Set-StrictMode -Version 2.0 + +$disableConfigureToolsetImport = $true + +try { + # `tools.ps1` checks $ci to perform some actions. Since the SDL + # scripts don't necessarily execute in the same agent that run the + # build.ps1/sh script this variable isn't automatically set. + $ci = $true + . $PSScriptRoot\..\tools.ps1 + + Measure-Command { + $jobs = @() + + # Find archive files for non-Windows and Windows builds. + $archiveFiles = @( + Get-ChildItem (Join-Path $InputPath "*.tar.gz") + Get-ChildItem (Join-Path $InputPath "*.zip") + ) + + foreach ($targzFile in $archiveFiles) { + $jobs += Start-Job -ScriptBlock { + $file = $using:targzFile + $fileName = [System.IO.Path]::GetFileName($file) + $extractDir = Join-Path $using:ExtractPath "$fileName.extracted" + + New-Item $extractDir -ItemType Directory -Force | Out-Null + + Write-Host "Extracting '$file' to '$extractDir'..." + + # Pipe errors to stdout to prevent PowerShell detecting them and quitting the job early. + # This type of quit skips the catch, so we wouldn't be able to tell which file triggered the + # error. Save output so it can be stored in the exception string along with context. + $output = tar -xf $file -C $extractDir 2>&1 + # Handle NZEC manually rather than using Exit-IfNZEC: we are in a background job, so we + # don't have access to the outer scope. + if ($LASTEXITCODE -ne 0) { + throw "Error extracting '$file': non-zero exit code ($LASTEXITCODE). Output: '$output'" + } + + Write-Host "Extracted to $extractDir" + } + } + + Receive-Job $jobs -Wait + } +} +catch { + Write-Host $_ + Write-PipelineTelemetryError -Force -Category 'Sdl' -Message $_ + ExitWithExitCode 1 +} diff --git a/src/arcade/eng/common/sdl/extract-artifact-packages.ps1 b/src/arcade/eng/common/sdl/extract-artifact-packages.ps1 new file mode 100644 index 000000000000..f031ed5b25e9 --- /dev/null +++ b/src/arcade/eng/common/sdl/extract-artifact-packages.ps1 @@ -0,0 +1,82 @@ +param( + [Parameter(Mandatory=$true)][string] $InputPath, # Full path to directory where artifact packages are stored + [Parameter(Mandatory=$true)][string] $ExtractPath # Full path to directory where the packages will be extracted +) + +$ErrorActionPreference = 'Stop' +Set-StrictMode -Version 2.0 + +$disableConfigureToolsetImport = $true + +function ExtractArtifacts { + if (!(Test-Path $InputPath)) { + Write-Host "Input Path does not exist: $InputPath" + ExitWithExitCode 0 + } + $Jobs = @() + Get-ChildItem "$InputPath\*.nupkg" | + ForEach-Object { + $Jobs += Start-Job -ScriptBlock $ExtractPackage -ArgumentList $_.FullName + } + + foreach ($Job in $Jobs) { + Wait-Job -Id $Job.Id | Receive-Job + } +} + +try { + # `tools.ps1` checks $ci to perform some actions. Since the SDL + # scripts don't necessarily execute in the same agent that run the + # build.ps1/sh script this variable isn't automatically set. + $ci = $true + . $PSScriptRoot\..\tools.ps1 + + $ExtractPackage = { + param( + [string] $PackagePath # Full path to a NuGet package + ) + + if (!(Test-Path $PackagePath)) { + Write-PipelineTelemetryError -Category 'Build' -Message "Input file does not exist: $PackagePath" + ExitWithExitCode 1 + } + + $RelevantExtensions = @('.dll', '.exe', '.pdb') + Write-Host -NoNewLine 'Extracting ' ([System.IO.Path]::GetFileName($PackagePath)) '...' + + $PackageId = [System.IO.Path]::GetFileNameWithoutExtension($PackagePath) + $ExtractPath = Join-Path -Path $using:ExtractPath -ChildPath $PackageId + + Add-Type -AssemblyName System.IO.Compression.FileSystem + + [System.IO.Directory]::CreateDirectory($ExtractPath); + + try { + $zip = [System.IO.Compression.ZipFile]::OpenRead($PackagePath) + + $zip.Entries | + Where-Object {$RelevantExtensions -contains [System.IO.Path]::GetExtension($_.Name)} | + ForEach-Object { + $TargetPath = Join-Path -Path $ExtractPath -ChildPath (Split-Path -Path $_.FullName) + [System.IO.Directory]::CreateDirectory($TargetPath); + + $TargetFile = Join-Path -Path $ExtractPath -ChildPath $_.FullName + [System.IO.Compression.ZipFileExtensions]::ExtractToFile($_, $TargetFile) + } + } + catch { + Write-Host $_ + Write-PipelineTelemetryError -Force -Category 'Sdl' -Message $_ + ExitWithExitCode 1 + } + finally { + $zip.Dispose() + } + } + Measure-Command { ExtractArtifacts } +} +catch { + Write-Host $_ + Write-PipelineTelemetryError -Force -Category 'Sdl' -Message $_ + ExitWithExitCode 1 +} diff --git a/src/arcade/eng/common/sdl/packages.config b/src/arcade/eng/common/sdl/packages.config new file mode 100644 index 000000000000..e5f543ea68c2 --- /dev/null +++ b/src/arcade/eng/common/sdl/packages.config @@ -0,0 +1,4 @@ + + + + diff --git a/src/arcade/eng/common/sdl/run-sdl.ps1 b/src/arcade/eng/common/sdl/run-sdl.ps1 new file mode 100644 index 000000000000..2eac8c78f103 --- /dev/null +++ b/src/arcade/eng/common/sdl/run-sdl.ps1 @@ -0,0 +1,49 @@ +Param( + [string] $GuardianCliLocation, + [string] $WorkingDirectory, + [string] $GdnFolder, + [string] $UpdateBaseline, + [string] $GuardianLoggerLevel='Standard' +) + +$ErrorActionPreference = 'Stop' +Set-StrictMode -Version 2.0 +$disableConfigureToolsetImport = $true +$global:LASTEXITCODE = 0 + +try { + # `tools.ps1` checks $ci to perform some actions. Since the SDL + # scripts don't necessarily execute in the same agent that run the + # build.ps1/sh script this variable isn't automatically set. + $ci = $true + . $PSScriptRoot\..\tools.ps1 + + # We store config files in the r directory of .gdn + $gdnConfigPath = Join-Path $GdnFolder 'r' + $ValidPath = Test-Path $GuardianCliLocation + + if ($ValidPath -eq $False) + { + Write-PipelineTelemetryError -Force -Category 'Sdl' -Message "Invalid Guardian CLI Location." + ExitWithExitCode 1 + } + + $gdnConfigFiles = Get-ChildItem $gdnConfigPath -Recurse -Include '*.gdnconfig' + Write-Host "Discovered Guardian config files:" + $gdnConfigFiles | Out-String | Write-Host + + Exec-BlockVerbosely { + & $GuardianCliLocation run ` + --working-directory $WorkingDirectory ` + --baseline mainbaseline ` + --update-baseline $UpdateBaseline ` + --logger-level $GuardianLoggerLevel ` + --config @gdnConfigFiles + Exit-IfNZEC "Sdl" + } +} +catch { + Write-Host $_.ScriptStackTrace + Write-PipelineTelemetryError -Force -Category 'Sdl' -Message $_ + ExitWithExitCode 1 +} diff --git a/src/arcade/eng/common/templates-official/variables/pool-providers.yml b/src/arcade/eng/common/templates-official/variables/pool-providers.yml new file mode 100644 index 000000000000..1f308b24efc4 --- /dev/null +++ b/src/arcade/eng/common/templates-official/variables/pool-providers.yml @@ -0,0 +1,45 @@ +# Select a pool provider based off branch name. Anything with branch name containing 'release' must go into an -Svc pool, +# otherwise it should go into the "normal" pools. This separates out the queueing and billing of released branches. + +# Motivation: +# Once a given branch of a repository's output has been officially "shipped" once, it is then considered to be COGS +# (Cost of goods sold) and should be moved to a servicing pool provider. This allows both separation of queueing +# (allowing release builds and main PR builds to not intefere with each other) and billing (required for COGS. +# Additionally, the pool provider name itself may be subject to change when the .NET Core Engineering Services +# team needs to move resources around and create new and potentially differently-named pools. Using this template +# file from an Arcade-ified repo helps guard against both having to update one's release/* branches and renaming. + +# How to use: +# This yaml assumes your shipped product branches use the naming convention "release/..." (which many do). +# If we find alternate naming conventions in broad usage it can be added to the condition below. +# +# First, import the template in an arcade-ified repo to pick up the variables, e.g.: +# +# variables: +# - template: /eng/common/templates-official/variables/pool-providers.yml +# +# ... then anywhere specifying the pool provider use the runtime variables, +# $(DncEngInternalBuildPool) +# +# pool: +# name: $(DncEngInternalBuildPool) +# image: 1es-windows-2022 + +variables: + # Coalesce the target and source branches so we know when a PR targets a release branch + # If these variables are somehow missing, fall back to main (tends to have more capacity) + + # Any new -Svc alternative pools should have variables added here to allow for splitting work + + - name: DncEngInternalBuildPool + value: $[ + replace( + replace( + eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), + True, + 'NetCore1ESPool-Svc-Internal' + ), + False, + 'NetCore1ESPool-Internal' + ) + ] \ No newline at end of file diff --git a/src/arcade/eng/common/templates-official/variables/sdl-variables.yml b/src/arcade/eng/common/templates-official/variables/sdl-variables.yml new file mode 100644 index 000000000000..dbdd66d4a4b3 --- /dev/null +++ b/src/arcade/eng/common/templates-official/variables/sdl-variables.yml @@ -0,0 +1,7 @@ +variables: +# The Guardian version specified in 'eng/common/sdl/packages.config'. This value must be kept in +# sync with the packages.config file. +- name: DefaultGuardianVersion + value: 0.109.0 +- name: GuardianPackagesConfigFile + value: $(Build.SourcesDirectory)\eng\common\sdl\packages.config \ No newline at end of file From 5c414b76dd665d0b040cce4614a48a5f7120b1e9 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Wed, 14 May 2025 02:02:49 +0000 Subject: [PATCH 43/57] Update dependencies from https://github.com/dotnet/arcade build 20250513.2 Microsoft.SourceBuild.Intermediate.arcade , Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.Build.Tasks.Installers , Microsoft.DotNet.Helix.Sdk , Microsoft.DotNet.SignTool , Microsoft.DotNet.XliffTasks , Microsoft.DotNet.XUnitExtensions From Version 9.0.0-beta.25255.5 -> To Version 9.0.0-beta.25263.2 --- eng/Version.Details.xml | 28 +- eng/Versions.props | 8 +- global.json | 4 +- .../build-configuration.json | 4 + .../eng/common/PSScriptAnalyzerSettings.psd1 | 11 + src/arcade/eng/common/README.md | 28 ++ src/arcade/eng/common/SetupNugetSources.ps1 | 171 ++++++++ src/arcade/eng/common/SetupNugetSources.sh | 167 ++++++++ src/arcade/eng/common/build.cmd | 3 + .../common/core-templates/job/onelocbuild.yml | 121 ++++++ .../core-templates/jobs/source-build.yml | 58 +++ .../post-build/common-variables.yml | 22 + .../post-build/setup-maestro-vars.yml | 74 ++++ .../steps/component-governance.yml | 16 + .../steps/enable-internal-runtimes.yml | 32 ++ .../steps/enable-internal-sources.yml | 47 ++ .../steps/get-federated-access-token.yml | 42 ++ .../steps/publish-build-artifacts.yml | 20 + .../steps/publish-pipeline-artifacts.yml | 20 + .../core-templates/steps/retain-build.yml | 28 ++ .../core-templates/steps/send-to-helix.yml | 93 ++++ .../variables/pool-providers.yml | 8 + .../eng/common/cross/arm/tizen/tizen.patch | 9 + .../eng/common/cross/armel/tizen/tizen.patch | 9 + .../common/cross/riscv64/tizen/tizen.patch | 9 + .../eng/common/cross/tizen-build-rootfs.sh | 82 ++++ .../eng/common/cross/x64/tizen/tizen.patch | 9 + .../eng/common/cross/x86/tizen/tizen.patch | 9 + src/arcade/eng/common/darc-init.ps1 | 47 ++ src/arcade/eng/common/darc-init.sh | 82 ++++ src/arcade/eng/common/dotnet-install.cmd | 2 + src/arcade/eng/common/dotnet-install.ps1 | 28 ++ src/arcade/eng/common/dotnet-install.sh | 94 ++++ .../common/enable-cross-org-publishing.ps1 | 13 + src/arcade/eng/common/generate-locproject.ps1 | 189 +++++++++ src/arcade/eng/common/generate-sbom-prep.ps1 | 29 ++ src/arcade/eng/common/generate-sbom-prep.sh | 39 ++ src/arcade/eng/common/helixpublish.proj | 27 ++ src/arcade/eng/common/init-tools-native.ps1 | 203 +++++++++ src/arcade/eng/common/init-tools-native.sh | 238 +++++++++++ .../eng/common/internal-feed-operations.ps1 | 132 ++++++ .../eng/common/internal-feed-operations.sh | 141 ++++++ .../eng/common/internal/Directory.Build.props | 11 + src/arcade/eng/common/internal/NuGet.config | 7 + src/arcade/eng/common/internal/Tools.csproj | 22 + .../common/loc/P22DotNetHtmlLocalization.lss | 29 ++ src/arcade/eng/common/msbuild.ps1 | 28 ++ src/arcade/eng/common/msbuild.sh | 58 +++ .../eng/common/native/CommonLibrary.psm1 | 401 ++++++++++++++++++ .../eng/common/native/common-library.sh | 172 ++++++++ src/arcade/eng/common/native/init-compiler.sh | 146 +++++++ .../eng/common/native/init-distro-rid.sh | 110 +++++ .../eng/common/native/init-os-and-arch.sh | 85 ++++ .../eng/common/native/install-cmake-test.sh | 117 +++++ src/arcade/eng/common/native/install-cmake.sh | 117 +++++ src/arcade/eng/common/native/install-tool.ps1 | 132 ++++++ .../eng/common/pipeline-logging-functions.ps1 | 260 ++++++++++++ .../eng/common/pipeline-logging-functions.sh | 206 +++++++++ .../post-build/check-channel-consistency.ps1 | 48 +++ .../common/post-build/nuget-validation.ps1 | 22 + .../common/post-build/nuget-verification.ps1 | 121 ++++++ .../eng/common/post-build/redact-logs.ps1 | 89 ++++ .../post-build/sourcelink-validation.ps1 | 327 ++++++++++++++ .../common/post-build/symbols-validation.ps1 | 337 +++++++++++++++ src/arcade/eng/common/retain-build.ps1 | 45 ++ src/arcade/eng/common/sdl/NuGet.config | 18 + .../eng/common/sdl/configure-sdl-tool.ps1 | 130 ++++++ .../eng/common/sdl/execute-all-sdl-tools.ps1 | 167 ++++++++ .../common/sdl/extract-artifact-archives.ps1 | 63 +++ .../common/sdl/extract-artifact-packages.ps1 | 82 ++++ src/arcade/eng/common/sdl/init-sdl.ps1 | 55 +++ src/arcade/eng/common/sdl/run-sdl.ps1 | 49 +++ src/arcade/eng/common/sdl/sdl.ps1 | 38 ++ .../eng/common/sdl/trim-assets-version.ps1 | 75 ++++ src/arcade/eng/common/template-guidance.md | 133 ++++++ .../templates-official/job/onelocbuild.yml | 7 + .../job/publish-build-assets.yml | 7 + .../templates-official/job/source-build.yml | 7 + .../job/source-index-stage1.yml | 7 + .../templates-official/jobs/codeql-build.yml | 7 + .../common/templates-official/jobs/jobs.yml | 7 + .../templates-official/jobs/source-build.yml | 7 + .../post-build/common-variables.yml | 8 + .../post-build/post-build.yml | 8 + .../post-build/setup-maestro-vars.yml | 8 + .../steps/component-governance.yml | 7 + .../steps/enable-internal-runtimes.yml | 9 + .../steps/enable-internal-sources.yml | 7 + .../steps/generate-sbom.yml | 7 + .../steps/get-delegation-sas.yml | 7 + .../steps/get-federated-access-token.yml | 7 + .../templates-official/steps/publish-logs.yml | 7 + .../steps/publish-pipeline-artifacts.yml | 28 ++ .../templates-official/steps/retain-build.yml | 7 + .../steps/send-to-helix.yml | 7 + .../templates-official/steps/source-build.yml | 7 + .../variables/pool-providers.yml | 45 ++ .../variables/sdl-variables.yml | 7 + .../eng/common/templates/job/onelocbuild.yml | 7 + .../templates/job/publish-build-assets.yml | 7 + .../eng/common/templates/job/source-build.yml | 7 + .../templates/job/source-index-stage1.yml | 7 + .../common/templates/jobs/codeql-build.yml | 7 + src/arcade/eng/common/templates/jobs/jobs.yml | 7 + .../common/templates/jobs/source-build.yml | 7 + .../templates/post-build/common-variables.yml | 8 + .../templates/post-build/post-build.yml | 8 + .../post-build/setup-maestro-vars.yml | 8 + .../templates/steps/component-governance.yml | 7 + .../steps/enable-internal-runtimes.yml | 10 + .../steps/enable-internal-sources.yml | 7 + .../common/templates/steps/generate-sbom.yml | 7 + .../templates/steps/get-delegation-sas.yml | 7 + .../steps/get-federated-access-token.yml | 7 + .../common/templates/steps/publish-logs.yml | 7 + .../steps/publish-pipeline-artifacts.yml | 34 ++ .../common/templates/steps/retain-build.yml | 7 + .../common/templates/steps/send-to-helix.yml | 7 + .../common/templates/steps/source-build.yml | 7 + .../templates/variables/pool-providers.yml | 59 +++ 120 files changed, 6540 insertions(+), 20 deletions(-) create mode 100644 src/arcade/eng/common/BuildConfiguration/build-configuration.json create mode 100644 src/arcade/eng/common/PSScriptAnalyzerSettings.psd1 create mode 100644 src/arcade/eng/common/README.md create mode 100644 src/arcade/eng/common/SetupNugetSources.ps1 create mode 100644 src/arcade/eng/common/SetupNugetSources.sh create mode 100644 src/arcade/eng/common/build.cmd create mode 100644 src/arcade/eng/common/core-templates/job/onelocbuild.yml create mode 100644 src/arcade/eng/common/core-templates/jobs/source-build.yml create mode 100644 src/arcade/eng/common/core-templates/post-build/common-variables.yml create mode 100644 src/arcade/eng/common/core-templates/post-build/setup-maestro-vars.yml create mode 100644 src/arcade/eng/common/core-templates/steps/component-governance.yml create mode 100644 src/arcade/eng/common/core-templates/steps/enable-internal-runtimes.yml create mode 100644 src/arcade/eng/common/core-templates/steps/enable-internal-sources.yml create mode 100644 src/arcade/eng/common/core-templates/steps/get-federated-access-token.yml create mode 100644 src/arcade/eng/common/core-templates/steps/publish-build-artifacts.yml create mode 100644 src/arcade/eng/common/core-templates/steps/publish-pipeline-artifacts.yml create mode 100644 src/arcade/eng/common/core-templates/steps/retain-build.yml create mode 100644 src/arcade/eng/common/core-templates/steps/send-to-helix.yml create mode 100644 src/arcade/eng/common/core-templates/variables/pool-providers.yml create mode 100644 src/arcade/eng/common/cross/arm/tizen/tizen.patch create mode 100644 src/arcade/eng/common/cross/armel/tizen/tizen.patch create mode 100644 src/arcade/eng/common/cross/riscv64/tizen/tizen.patch create mode 100644 src/arcade/eng/common/cross/tizen-build-rootfs.sh create mode 100644 src/arcade/eng/common/cross/x64/tizen/tizen.patch create mode 100644 src/arcade/eng/common/cross/x86/tizen/tizen.patch create mode 100644 src/arcade/eng/common/darc-init.ps1 create mode 100644 src/arcade/eng/common/darc-init.sh create mode 100644 src/arcade/eng/common/dotnet-install.cmd create mode 100644 src/arcade/eng/common/dotnet-install.ps1 create mode 100644 src/arcade/eng/common/dotnet-install.sh create mode 100644 src/arcade/eng/common/enable-cross-org-publishing.ps1 create mode 100644 src/arcade/eng/common/generate-locproject.ps1 create mode 100644 src/arcade/eng/common/generate-sbom-prep.ps1 create mode 100644 src/arcade/eng/common/generate-sbom-prep.sh create mode 100644 src/arcade/eng/common/helixpublish.proj create mode 100644 src/arcade/eng/common/init-tools-native.ps1 create mode 100644 src/arcade/eng/common/init-tools-native.sh create mode 100644 src/arcade/eng/common/internal-feed-operations.ps1 create mode 100644 src/arcade/eng/common/internal-feed-operations.sh create mode 100644 src/arcade/eng/common/internal/Directory.Build.props create mode 100644 src/arcade/eng/common/internal/NuGet.config create mode 100644 src/arcade/eng/common/internal/Tools.csproj create mode 100644 src/arcade/eng/common/loc/P22DotNetHtmlLocalization.lss create mode 100644 src/arcade/eng/common/msbuild.ps1 create mode 100644 src/arcade/eng/common/msbuild.sh create mode 100644 src/arcade/eng/common/native/CommonLibrary.psm1 create mode 100644 src/arcade/eng/common/native/common-library.sh create mode 100644 src/arcade/eng/common/native/init-compiler.sh create mode 100644 src/arcade/eng/common/native/init-distro-rid.sh create mode 100644 src/arcade/eng/common/native/init-os-and-arch.sh create mode 100644 src/arcade/eng/common/native/install-cmake-test.sh create mode 100644 src/arcade/eng/common/native/install-cmake.sh create mode 100644 src/arcade/eng/common/native/install-tool.ps1 create mode 100644 src/arcade/eng/common/pipeline-logging-functions.ps1 create mode 100644 src/arcade/eng/common/pipeline-logging-functions.sh create mode 100644 src/arcade/eng/common/post-build/check-channel-consistency.ps1 create mode 100644 src/arcade/eng/common/post-build/nuget-validation.ps1 create mode 100644 src/arcade/eng/common/post-build/nuget-verification.ps1 create mode 100644 src/arcade/eng/common/post-build/redact-logs.ps1 create mode 100644 src/arcade/eng/common/post-build/sourcelink-validation.ps1 create mode 100644 src/arcade/eng/common/post-build/symbols-validation.ps1 create mode 100644 src/arcade/eng/common/retain-build.ps1 create mode 100644 src/arcade/eng/common/sdl/NuGet.config create mode 100644 src/arcade/eng/common/sdl/configure-sdl-tool.ps1 create mode 100644 src/arcade/eng/common/sdl/execute-all-sdl-tools.ps1 create mode 100644 src/arcade/eng/common/sdl/extract-artifact-archives.ps1 create mode 100644 src/arcade/eng/common/sdl/extract-artifact-packages.ps1 create mode 100644 src/arcade/eng/common/sdl/init-sdl.ps1 create mode 100644 src/arcade/eng/common/sdl/run-sdl.ps1 create mode 100644 src/arcade/eng/common/sdl/sdl.ps1 create mode 100644 src/arcade/eng/common/sdl/trim-assets-version.ps1 create mode 100644 src/arcade/eng/common/template-guidance.md create mode 100644 src/arcade/eng/common/templates-official/job/onelocbuild.yml create mode 100644 src/arcade/eng/common/templates-official/job/publish-build-assets.yml create mode 100644 src/arcade/eng/common/templates-official/job/source-build.yml create mode 100644 src/arcade/eng/common/templates-official/job/source-index-stage1.yml create mode 100644 src/arcade/eng/common/templates-official/jobs/codeql-build.yml create mode 100644 src/arcade/eng/common/templates-official/jobs/jobs.yml create mode 100644 src/arcade/eng/common/templates-official/jobs/source-build.yml create mode 100644 src/arcade/eng/common/templates-official/post-build/common-variables.yml create mode 100644 src/arcade/eng/common/templates-official/post-build/post-build.yml create mode 100644 src/arcade/eng/common/templates-official/post-build/setup-maestro-vars.yml create mode 100644 src/arcade/eng/common/templates-official/steps/component-governance.yml create mode 100644 src/arcade/eng/common/templates-official/steps/enable-internal-runtimes.yml create mode 100644 src/arcade/eng/common/templates-official/steps/enable-internal-sources.yml create mode 100644 src/arcade/eng/common/templates-official/steps/generate-sbom.yml create mode 100644 src/arcade/eng/common/templates-official/steps/get-delegation-sas.yml create mode 100644 src/arcade/eng/common/templates-official/steps/get-federated-access-token.yml create mode 100644 src/arcade/eng/common/templates-official/steps/publish-logs.yml create mode 100644 src/arcade/eng/common/templates-official/steps/publish-pipeline-artifacts.yml create mode 100644 src/arcade/eng/common/templates-official/steps/retain-build.yml create mode 100644 src/arcade/eng/common/templates-official/steps/send-to-helix.yml create mode 100644 src/arcade/eng/common/templates-official/steps/source-build.yml create mode 100644 src/arcade/eng/common/templates-official/variables/pool-providers.yml create mode 100644 src/arcade/eng/common/templates-official/variables/sdl-variables.yml create mode 100644 src/arcade/eng/common/templates/job/onelocbuild.yml create mode 100644 src/arcade/eng/common/templates/job/publish-build-assets.yml create mode 100644 src/arcade/eng/common/templates/job/source-build.yml create mode 100644 src/arcade/eng/common/templates/job/source-index-stage1.yml create mode 100644 src/arcade/eng/common/templates/jobs/codeql-build.yml create mode 100644 src/arcade/eng/common/templates/jobs/jobs.yml create mode 100644 src/arcade/eng/common/templates/jobs/source-build.yml create mode 100644 src/arcade/eng/common/templates/post-build/common-variables.yml create mode 100644 src/arcade/eng/common/templates/post-build/post-build.yml create mode 100644 src/arcade/eng/common/templates/post-build/setup-maestro-vars.yml create mode 100644 src/arcade/eng/common/templates/steps/component-governance.yml create mode 100644 src/arcade/eng/common/templates/steps/enable-internal-runtimes.yml create mode 100644 src/arcade/eng/common/templates/steps/enable-internal-sources.yml create mode 100644 src/arcade/eng/common/templates/steps/generate-sbom.yml create mode 100644 src/arcade/eng/common/templates/steps/get-delegation-sas.yml create mode 100644 src/arcade/eng/common/templates/steps/get-federated-access-token.yml create mode 100644 src/arcade/eng/common/templates/steps/publish-logs.yml create mode 100644 src/arcade/eng/common/templates/steps/publish-pipeline-artifacts.yml create mode 100644 src/arcade/eng/common/templates/steps/retain-build.yml create mode 100644 src/arcade/eng/common/templates/steps/send-to-helix.yml create mode 100644 src/arcade/eng/common/templates/steps/source-build.yml create mode 100644 src/arcade/eng/common/templates/variables/pool-providers.yml diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 8f4d4413483f..e8370ec1547b 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -589,34 +589,34 @@
- + https://github.com/dotnet/arcade - 1cfa39f82d00b3659a3d367bc344241946e10681 + ac63dcf2791c32bf2102fd2b5ebb46479bd48a83 - + https://github.com/dotnet/arcade - 1cfa39f82d00b3659a3d367bc344241946e10681 + ac63dcf2791c32bf2102fd2b5ebb46479bd48a83 - + https://github.com/dotnet/arcade - 1cfa39f82d00b3659a3d367bc344241946e10681 + ac63dcf2791c32bf2102fd2b5ebb46479bd48a83 - + https://github.com/dotnet/arcade - 1cfa39f82d00b3659a3d367bc344241946e10681 + ac63dcf2791c32bf2102fd2b5ebb46479bd48a83 - + https://github.com/dotnet/arcade - 1cfa39f82d00b3659a3d367bc344241946e10681 + ac63dcf2791c32bf2102fd2b5ebb46479bd48a83 - + https://github.com/dotnet/arcade - 1cfa39f82d00b3659a3d367bc344241946e10681 + ac63dcf2791c32bf2102fd2b5ebb46479bd48a83 - + https://github.com/dotnet/arcade - 1cfa39f82d00b3659a3d367bc344241946e10681 + ac63dcf2791c32bf2102fd2b5ebb46479bd48a83 diff --git a/eng/Versions.props b/eng/Versions.props index f5e089f931bf..55cd98608777 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -270,10 +270,10 @@
- 9.0.0-beta.25255.5 - 9.0.0-beta.25255.5 - 9.0.0-beta.25255.5 - 9.0.0-beta.25255.5 + 9.0.0-beta.25263.2 + 9.0.0-beta.25263.2 + 9.0.0-beta.25263.2 + 9.0.0-beta.25263.2 diff --git a/global.json b/global.json index fe3e74049ae6..0ed73aeaa54e 100644 --- a/global.json +++ b/global.json @@ -17,8 +17,8 @@ "cmake": "latest" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "9.0.0-beta.25255.5", - "Microsoft.DotNet.Helix.Sdk": "9.0.0-beta.25255.5", + "Microsoft.DotNet.Arcade.Sdk": "9.0.0-beta.25263.2", + "Microsoft.DotNet.Helix.Sdk": "9.0.0-beta.25263.2", "Microsoft.Build.NoTargets": "3.7.0", "Microsoft.DotNet.CMake.Sdk": "9.0.0-beta.24217.1" } diff --git a/src/arcade/eng/common/BuildConfiguration/build-configuration.json b/src/arcade/eng/common/BuildConfiguration/build-configuration.json new file mode 100644 index 000000000000..3d1cc89894c9 --- /dev/null +++ b/src/arcade/eng/common/BuildConfiguration/build-configuration.json @@ -0,0 +1,4 @@ +{ + "RetryCountLimit": 1, + "RetryByAnyError": false +} diff --git a/src/arcade/eng/common/PSScriptAnalyzerSettings.psd1 b/src/arcade/eng/common/PSScriptAnalyzerSettings.psd1 new file mode 100644 index 000000000000..4c1ea7c98ea4 --- /dev/null +++ b/src/arcade/eng/common/PSScriptAnalyzerSettings.psd1 @@ -0,0 +1,11 @@ +@{ + IncludeRules=@('PSAvoidUsingCmdletAliases', + 'PSAvoidUsingWMICmdlet', + 'PSAvoidUsingPositionalParameters', + 'PSAvoidUsingInvokeExpression', + 'PSUseDeclaredVarsMoreThanAssignments', + 'PSUseCmdletCorrectly', + 'PSStandardDSCFunctionsInResource', + 'PSUseIdenticalMandatoryParametersForDSC', + 'PSUseIdenticalParametersForDSC') +} \ No newline at end of file diff --git a/src/arcade/eng/common/README.md b/src/arcade/eng/common/README.md new file mode 100644 index 000000000000..ff49c371527a --- /dev/null +++ b/src/arcade/eng/common/README.md @@ -0,0 +1,28 @@ +# Don't touch this folder + + uuuuuuuuuuuuuuuuuuuu + u" uuuuuuuuuuuuuuuuuu "u + u" u$$$$$$$$$$$$$$$$$$$$u "u + u" u$$$$$$$$$$$$$$$$$$$$$$$$u "u + u" u$$$$$$$$$$$$$$$$$$$$$$$$$$$$u "u + u" u$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$u "u + u" u$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$u "u + $ $$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$ $ + $ $$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$ $ + $ $$$" ... "$... ...$" ... "$$$ ... "$$$ $ + $ $$$u `"$$$$$$$ $$$ $$$$$ $$ $$$ $$$ $ + $ $$$$$$uu "$$$$ $$$ $$$$$ $$ """ u$$$ $ + $ $$$""$$$ $$$$ $$$u "$$$" u$$ $$$$$$$$ $ + $ $$$$....,$$$$$..$$$$$....,$$$$..$$$$$$$$ $ + $ $$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$ $ + "u "$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$" u" + "u "$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$" u" + "u "$$$$$$$$$$$$$$$$$$$$$$$$$$$$" u" + "u "$$$$$$$$$$$$$$$$$$$$$$$$" u" + "u "$$$$$$$$$$$$$$$$$$$$" u" + "u """""""""""""""""" u" + """""""""""""""""""" + +!!! Changes made in this directory are subject to being overwritten by automation !!! + +The files in this directory are shared by all Arcade repos and managed by automation. If you need to make changes to these files, open an issue or submit a pull request to https://github.com/dotnet/arcade first. diff --git a/src/arcade/eng/common/SetupNugetSources.ps1 b/src/arcade/eng/common/SetupNugetSources.ps1 new file mode 100644 index 000000000000..5db4ad71ee2f --- /dev/null +++ b/src/arcade/eng/common/SetupNugetSources.ps1 @@ -0,0 +1,171 @@ +# This script adds internal feeds required to build commits that depend on internal package sources. For instance, +# dotnet6-internal would be added automatically if dotnet6 was found in the nuget.config file. In addition also enables +# disabled internal Maestro (darc-int*) feeds. +# +# Optionally, this script also adds a credential entry for each of the internal feeds if supplied. +# +# See example call for this script below. +# +# - task: PowerShell@2 +# displayName: Setup Private Feeds Credentials +# condition: eq(variables['Agent.OS'], 'Windows_NT') +# inputs: +# filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.ps1 +# arguments: -ConfigFile $(Build.SourcesDirectory)/NuGet.config -Password $Env:Token +# env: +# Token: $(dn-bot-dnceng-artifact-feeds-rw) +# +# Note that the NuGetAuthenticate task should be called after SetupNugetSources. +# This ensures that: +# - Appropriate creds are set for the added internal feeds (if not supplied to the scrupt) +# - The credential provider is installed. +# +# This logic is also abstracted into enable-internal-sources.yml. + +[CmdletBinding()] +param ( + [Parameter(Mandatory = $true)][string]$ConfigFile, + $Password +) + +$ErrorActionPreference = "Stop" +Set-StrictMode -Version 2.0 +[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 + +. $PSScriptRoot\tools.ps1 + +# Add source entry to PackageSources +function AddPackageSource($sources, $SourceName, $SourceEndPoint, $creds, $Username, $pwd) { + $packageSource = $sources.SelectSingleNode("add[@key='$SourceName']") + + if ($packageSource -eq $null) + { + $packageSource = $doc.CreateElement("add") + $packageSource.SetAttribute("key", $SourceName) + $packageSource.SetAttribute("value", $SourceEndPoint) + $sources.AppendChild($packageSource) | Out-Null + } + else { + Write-Host "Package source $SourceName already present." + } + + AddCredential -Creds $creds -Source $SourceName -Username $Username -pwd $pwd +} + +# Add a credential node for the specified source +function AddCredential($creds, $source, $username, $pwd) { + # If no cred supplied, don't do anything. + if (!$pwd) { + return; + } + + # Looks for credential configuration for the given SourceName. Create it if none is found. + $sourceElement = $creds.SelectSingleNode($Source) + if ($sourceElement -eq $null) + { + $sourceElement = $doc.CreateElement($Source) + $creds.AppendChild($sourceElement) | Out-Null + } + + # Add the node to the credential if none is found. + $usernameElement = $sourceElement.SelectSingleNode("add[@key='Username']") + if ($usernameElement -eq $null) + { + $usernameElement = $doc.CreateElement("add") + $usernameElement.SetAttribute("key", "Username") + $sourceElement.AppendChild($usernameElement) | Out-Null + } + $usernameElement.SetAttribute("value", $Username) + + # Add the to the credential if none is found. + # Add it as a clear text because there is no support for encrypted ones in non-windows .Net SDKs. + # -> https://github.com/NuGet/Home/issues/5526 + $passwordElement = $sourceElement.SelectSingleNode("add[@key='ClearTextPassword']") + if ($passwordElement -eq $null) + { + $passwordElement = $doc.CreateElement("add") + $passwordElement.SetAttribute("key", "ClearTextPassword") + $sourceElement.AppendChild($passwordElement) | Out-Null + } + + $passwordElement.SetAttribute("value", $pwd) +} + +function InsertMaestroPrivateFeedCredentials($Sources, $Creds, $Username, $pwd) { + $maestroPrivateSources = $Sources.SelectNodes("add[contains(@key,'darc-int')]") + + Write-Host "Inserting credentials for $($maestroPrivateSources.Count) Maestro's private feeds." + + ForEach ($PackageSource in $maestroPrivateSources) { + Write-Host "`tInserting credential for Maestro's feed:" $PackageSource.Key + AddCredential -Creds $creds -Source $PackageSource.Key -Username $Username -pwd $pwd + } +} + +function EnablePrivatePackageSources($DisabledPackageSources) { + $maestroPrivateSources = $DisabledPackageSources.SelectNodes("add[contains(@key,'darc-int')]") + ForEach ($DisabledPackageSource in $maestroPrivateSources) { + Write-Host "`tEnsuring private source '$($DisabledPackageSource.key)' is enabled by deleting it from disabledPackageSource" + # Due to https://github.com/NuGet/Home/issues/10291, we must actually remove the disabled entries + $DisabledPackageSources.RemoveChild($DisabledPackageSource) + } +} + +if (!(Test-Path $ConfigFile -PathType Leaf)) { + Write-PipelineTelemetryError -Category 'Build' -Message "Eng/common/SetupNugetSources.ps1 returned a non-zero exit code. Couldn't find the NuGet config file: $ConfigFile" + ExitWithExitCode 1 +} + +# Load NuGet.config +$doc = New-Object System.Xml.XmlDocument +$filename = (Get-Item $ConfigFile).FullName +$doc.Load($filename) + +# Get reference to or create one if none exist already +$sources = $doc.DocumentElement.SelectSingleNode("packageSources") +if ($sources -eq $null) { + $sources = $doc.CreateElement("packageSources") + $doc.DocumentElement.AppendChild($sources) | Out-Null +} + +$creds = $null +if ($Password) { + # Looks for a node. Create it if none is found. + $creds = $doc.DocumentElement.SelectSingleNode("packageSourceCredentials") + if ($creds -eq $null) { + $creds = $doc.CreateElement("packageSourceCredentials") + $doc.DocumentElement.AppendChild($creds) | Out-Null + } +} + +# Check for disabledPackageSources; we'll enable any darc-int ones we find there +$disabledSources = $doc.DocumentElement.SelectSingleNode("disabledPackageSources") +if ($disabledSources -ne $null) { + Write-Host "Checking for any darc-int disabled package sources in the disabledPackageSources node" + EnablePrivatePackageSources -DisabledPackageSources $disabledSources +} + +$userName = "dn-bot" + +# Insert credential nodes for Maestro's private feeds +InsertMaestroPrivateFeedCredentials -Sources $sources -Creds $creds -Username $userName -pwd $Password + +# 3.1 uses a different feed url format so it's handled differently here +$dotnet31Source = $sources.SelectSingleNode("add[@key='dotnet3.1']") +if ($dotnet31Source -ne $null) { + AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal/nuget/v2" -Creds $creds -Username $userName -pwd $Password + AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-transport/nuget/v2" -Creds $creds -Username $userName -pwd $Password +} + +$dotnetVersions = @('5','6','7','8','9') + +foreach ($dotnetVersion in $dotnetVersions) { + $feedPrefix = "dotnet" + $dotnetVersion; + $dotnetSource = $sources.SelectSingleNode("add[@key='$feedPrefix']") + if ($dotnetSource -ne $null) { + AddPackageSource -Sources $sources -SourceName "$feedPrefix-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/$feedPrefix-internal/nuget/v2" -Creds $creds -Username $userName -pwd $Password + AddPackageSource -Sources $sources -SourceName "$feedPrefix-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/$feedPrefix-internal-transport/nuget/v2" -Creds $creds -Username $userName -pwd $Password + } +} + +$doc.Save($filename) diff --git a/src/arcade/eng/common/SetupNugetSources.sh b/src/arcade/eng/common/SetupNugetSources.sh new file mode 100644 index 000000000000..4604b61b0323 --- /dev/null +++ b/src/arcade/eng/common/SetupNugetSources.sh @@ -0,0 +1,167 @@ +#!/usr/bin/env bash + +# This script adds internal feeds required to build commits that depend on internal package sources. For instance, +# dotnet6-internal would be added automatically if dotnet6 was found in the nuget.config file. In addition also enables +# disabled internal Maestro (darc-int*) feeds. +# +# Optionally, this script also adds a credential entry for each of the internal feeds if supplied. +# +# See example call for this script below. +# +# - task: Bash@3 +# displayName: Setup Internal Feeds +# inputs: +# filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh +# arguments: $(Build.SourcesDirectory)/NuGet.config +# condition: ne(variables['Agent.OS'], 'Windows_NT') +# - task: NuGetAuthenticate@1 +# +# Note that the NuGetAuthenticate task should be called after SetupNugetSources. +# This ensures that: +# - Appropriate creds are set for the added internal feeds (if not supplied to the scrupt) +# - The credential provider is installed. +# +# This logic is also abstracted into enable-internal-sources.yml. + +ConfigFile=$1 +CredToken=$2 +NL='\n' +TB=' ' + +source="${BASH_SOURCE[0]}" + +# resolve $source until the file is no longer a symlink +while [[ -h "$source" ]]; do + scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" + source="$(readlink "$source")" + # if $source was a relative symlink, we need to resolve it relative to the path where the + # symlink file was located + [[ $source != /* ]] && source="$scriptroot/$source" +done +scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" + +. "$scriptroot/tools.sh" + +if [ ! -f "$ConfigFile" ]; then + Write-PipelineTelemetryError -Category 'Build' "Error: Eng/common/SetupNugetSources.sh returned a non-zero exit code. Couldn't find the NuGet config file: $ConfigFile" + ExitWithExitCode 1 +fi + +if [[ `uname -s` == "Darwin" ]]; then + NL=$'\\\n' + TB='' +fi + +# Ensure there is a ... section. +grep -i "" $ConfigFile +if [ "$?" != "0" ]; then + echo "Adding ... section." + ConfigNodeHeader="" + PackageSourcesTemplate="${TB}${NL}${TB}" + + sed -i.bak "s|$ConfigNodeHeader|$ConfigNodeHeader${NL}$PackageSourcesTemplate|" $ConfigFile +fi + +# Ensure there is a ... section. +grep -i "" $ConfigFile +if [ "$?" != "0" ]; then + echo "Adding ... section." + + PackageSourcesNodeFooter="" + PackageSourceCredentialsTemplate="${TB}${NL}${TB}" + + sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourcesNodeFooter${NL}$PackageSourceCredentialsTemplate|" $ConfigFile +fi + +PackageSources=() + +# Ensure dotnet3.1-internal and dotnet3.1-internal-transport are in the packageSources if the public dotnet3.1 feeds are present +grep -i "" + + sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourceTemplate${NL}$PackageSourcesNodeFooter|" $ConfigFile + fi + PackageSources+=('dotnet3.1-internal') + + grep -i "" $ConfigFile + if [ "$?" != "0" ]; then + echo "Adding dotnet3.1-internal-transport to the packageSources." + PackageSourcesNodeFooter="" + PackageSourceTemplate="${TB}" + + sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourceTemplate${NL}$PackageSourcesNodeFooter|" $ConfigFile + fi + PackageSources+=('dotnet3.1-internal-transport') +fi + +DotNetVersions=('5' '6' '7' '8' '9') + +for DotNetVersion in ${DotNetVersions[@]} ; do + FeedPrefix="dotnet${DotNetVersion}"; + grep -i "" + + sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourceTemplate${NL}$PackageSourcesNodeFooter|" $ConfigFile + fi + PackageSources+=("$FeedPrefix-internal") + + grep -i "" $ConfigFile + if [ "$?" != "0" ]; then + echo "Adding $FeedPrefix-internal-transport to the packageSources." + PackageSourcesNodeFooter="" + PackageSourceTemplate="${TB}" + + sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourceTemplate${NL}$PackageSourcesNodeFooter|" $ConfigFile + fi + PackageSources+=("$FeedPrefix-internal-transport") + fi +done + +# I want things split line by line +PrevIFS=$IFS +IFS=$'\n' +PackageSources+="$IFS" +PackageSources+=$(grep -oh '"darc-int-[^"]*"' $ConfigFile | tr -d '"') +IFS=$PrevIFS + +if [ "$CredToken" ]; then + for FeedName in ${PackageSources[@]} ; do + # Check if there is no existing credential for this FeedName + grep -i "<$FeedName>" $ConfigFile + if [ "$?" != "0" ]; then + echo "Adding credentials for $FeedName." + + PackageSourceCredentialsNodeFooter="" + NewCredential="${TB}${TB}<$FeedName>${NL}${NL}${NL}" + + sed -i.bak "s|$PackageSourceCredentialsNodeFooter|$NewCredential${NL}$PackageSourceCredentialsNodeFooter|" $ConfigFile + fi + done +fi + +# Re-enable any entries in disabledPackageSources where the feed name contains darc-int +grep -i "" $ConfigFile +if [ "$?" == "0" ]; then + DisabledDarcIntSources=() + echo "Re-enabling any disabled \"darc-int\" package sources in $ConfigFile" + DisabledDarcIntSources+=$(grep -oh '"darc-int-[^"]*" value="true"' $ConfigFile | tr -d '"') + for DisabledSourceName in ${DisabledDarcIntSources[@]} ; do + if [[ $DisabledSourceName == darc-int* ]] + then + OldDisableValue="" + NewDisableValue="" + sed -i.bak "s|$OldDisableValue|$NewDisableValue|" $ConfigFile + echo "Neutralized disablePackageSources entry for '$DisabledSourceName'" + fi + done +fi diff --git a/src/arcade/eng/common/build.cmd b/src/arcade/eng/common/build.cmd new file mode 100644 index 000000000000..99daf368abae --- /dev/null +++ b/src/arcade/eng/common/build.cmd @@ -0,0 +1,3 @@ +@echo off +powershell -ExecutionPolicy ByPass -NoProfile -command "& """%~dp0build.ps1""" %*" +exit /b %ErrorLevel% diff --git a/src/arcade/eng/common/core-templates/job/onelocbuild.yml b/src/arcade/eng/common/core-templates/job/onelocbuild.yml new file mode 100644 index 000000000000..00feec8ebbc3 --- /dev/null +++ b/src/arcade/eng/common/core-templates/job/onelocbuild.yml @@ -0,0 +1,121 @@ +parameters: + # Optional: dependencies of the job + dependsOn: '' + + # Optional: A defined YAML pool - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#pool + pool: '' + + CeapexPat: $(dn-bot-ceapex-package-r) # PAT for the loc AzDO instance https://dev.azure.com/ceapex + GithubPat: $(BotAccount-dotnet-bot-repo-PAT) + + SourcesDirectory: $(Build.SourcesDirectory) + CreatePr: true + AutoCompletePr: false + ReusePr: true + UseLfLineEndings: true + UseCheckedInLocProjectJson: false + SkipLocProjectJsonGeneration: false + LanguageSet: VS_Main_Languages + LclSource: lclFilesInRepo + LclPackageId: '' + RepoType: gitHub + GitHubOrg: dotnet + MirrorRepo: '' + MirrorBranch: main + condition: '' + JobNameSuffix: '' + is1ESPipeline: '' +jobs: +- job: OneLocBuild${{ parameters.JobNameSuffix }} + + dependsOn: ${{ parameters.dependsOn }} + + displayName: OneLocBuild${{ parameters.JobNameSuffix }} + + variables: + - group: OneLocBuildVariables # Contains the CeapexPat and GithubPat + - name: _GenerateLocProjectArguments + value: -SourcesDirectory ${{ parameters.SourcesDirectory }} + -LanguageSet "${{ parameters.LanguageSet }}" + -CreateNeutralXlfs + - ${{ if eq(parameters.UseCheckedInLocProjectJson, 'true') }}: + - name: _GenerateLocProjectArguments + value: ${{ variables._GenerateLocProjectArguments }} -UseCheckedInLocProjectJson + - template: /eng/common/core-templates/variables/pool-providers.yml + parameters: + is1ESPipeline: ${{ parameters.is1ESPipeline }} + + ${{ if ne(parameters.pool, '') }}: + pool: ${{ parameters.pool }} + ${{ if eq(parameters.pool, '') }}: + pool: + # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) + ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: + name: AzurePipelines-EO + image: 1ESPT-Windows2022 + demands: Cmd + os: windows + # If it's not devdiv, it's dnceng + ${{ if ne(variables['System.TeamProject'], 'DevDiv') }}: + name: $(DncEngInternalBuildPool) + image: 1es-windows-2022 + os: windows + + steps: + - ${{ if eq(parameters.is1ESPipeline, '') }}: + - 'Illegal entry point, is1ESPipeline is not defined. Repository yaml should not directly reference templates in core-templates folder.': error + + - ${{ if ne(parameters.SkipLocProjectJsonGeneration, 'true') }}: + - task: Powershell@2 + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/generate-locproject.ps1 + arguments: $(_GenerateLocProjectArguments) + displayName: Generate LocProject.json + condition: ${{ parameters.condition }} + + - task: OneLocBuild@2 + displayName: OneLocBuild + env: + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + inputs: + locProj: eng/Localize/LocProject.json + outDir: $(Build.ArtifactStagingDirectory) + lclSource: ${{ parameters.LclSource }} + lclPackageId: ${{ parameters.LclPackageId }} + isCreatePrSelected: ${{ parameters.CreatePr }} + isAutoCompletePrSelected: ${{ parameters.AutoCompletePr }} + ${{ if eq(parameters.CreatePr, true) }}: + isUseLfLineEndingsSelected: ${{ parameters.UseLfLineEndings }} + ${{ if eq(parameters.RepoType, 'gitHub') }}: + isShouldReusePrSelected: ${{ parameters.ReusePr }} + packageSourceAuth: patAuth + patVariable: ${{ parameters.CeapexPat }} + ${{ if eq(parameters.RepoType, 'gitHub') }}: + repoType: ${{ parameters.RepoType }} + gitHubPatVariable: "${{ parameters.GithubPat }}" + ${{ if ne(parameters.MirrorRepo, '') }}: + isMirrorRepoSelected: true + gitHubOrganization: ${{ parameters.GitHubOrg }} + mirrorRepo: ${{ parameters.MirrorRepo }} + mirrorBranch: ${{ parameters.MirrorBranch }} + condition: ${{ parameters.condition }} + + - template: /eng/common/core-templates/steps/publish-build-artifacts.yml + parameters: + is1ESPipeline: ${{ parameters.is1ESPipeline }} + args: + displayName: Publish Localization Files + pathToPublish: '$(Build.ArtifactStagingDirectory)/loc' + publishLocation: Container + artifactName: Loc + condition: ${{ parameters.condition }} + + - template: /eng/common/core-templates/steps/publish-build-artifacts.yml + parameters: + is1ESPipeline: ${{ parameters.is1ESPipeline }} + args: + displayName: Publish LocProject.json + pathToPublish: '$(Build.SourcesDirectory)/eng/Localize/' + publishLocation: Container + artifactName: Loc + condition: ${{ parameters.condition }} \ No newline at end of file diff --git a/src/arcade/eng/common/core-templates/jobs/source-build.yml b/src/arcade/eng/common/core-templates/jobs/source-build.yml new file mode 100644 index 000000000000..a10ccfbee6de --- /dev/null +++ b/src/arcade/eng/common/core-templates/jobs/source-build.yml @@ -0,0 +1,58 @@ +parameters: + # This template adds arcade-powered source-build to CI. A job is created for each platform, as + # well as an optional server job that completes when all platform jobs complete. + + # The name of the "join" job for all source-build platforms. If set to empty string, the job is + # not included. Existing repo pipelines can use this job depend on all source-build jobs + # completing without maintaining a separate list of every single job ID: just depend on this one + # server job. By default, not included. Recommended name if used: 'Source_Build_Complete'. + allCompletedJobId: '' + + # See /eng/common/core-templates/job/source-build.yml + jobNamePrefix: 'Source_Build' + + # This is the default platform provided by Arcade, intended for use by a managed-only repo. + defaultManagedPlatform: + name: 'Managed' + container: 'mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream9' + + # Defines the platforms on which to run build jobs. One job is created for each platform, and the + # object in this array is sent to the job template as 'platform'. If no platforms are specified, + # one job runs on 'defaultManagedPlatform'. + platforms: [] + + is1ESPipeline: '' + + # If set to true and running on a non-public project, + # Internal nuget and blob storage locations will be enabled. + # This is not enabled by default because many repositories do not need internal sources + # and do not need to have the required service connections approved in the pipeline. + enableInternalSources: false + +jobs: + +- ${{ if ne(parameters.allCompletedJobId, '') }}: + - job: ${{ parameters.allCompletedJobId }} + displayName: Source-Build Complete + pool: server + dependsOn: + - ${{ each platform in parameters.platforms }}: + - ${{ parameters.jobNamePrefix }}_${{ platform.name }} + - ${{ if eq(length(parameters.platforms), 0) }}: + - ${{ parameters.jobNamePrefix }}_${{ parameters.defaultManagedPlatform.name }} + +- ${{ each platform in parameters.platforms }}: + - template: /eng/common/core-templates/job/source-build.yml + parameters: + is1ESPipeline: ${{ parameters.is1ESPipeline }} + jobNamePrefix: ${{ parameters.jobNamePrefix }} + platform: ${{ platform }} + enableInternalSources: ${{ parameters.enableInternalSources }} + +- ${{ if eq(length(parameters.platforms), 0) }}: + - template: /eng/common/core-templates/job/source-build.yml + parameters: + is1ESPipeline: ${{ parameters.is1ESPipeline }} + jobNamePrefix: ${{ parameters.jobNamePrefix }} + platform: ${{ parameters.defaultManagedPlatform }} + enableInternalSources: ${{ parameters.enableInternalSources }} diff --git a/src/arcade/eng/common/core-templates/post-build/common-variables.yml b/src/arcade/eng/common/core-templates/post-build/common-variables.yml new file mode 100644 index 000000000000..d5627a994ae5 --- /dev/null +++ b/src/arcade/eng/common/core-templates/post-build/common-variables.yml @@ -0,0 +1,22 @@ +variables: + - group: Publish-Build-Assets + + # Whether the build is internal or not + - name: IsInternalBuild + value: ${{ and(ne(variables['System.TeamProject'], 'public'), contains(variables['Build.SourceBranch'], 'internal')) }} + + # Default Maestro++ API Endpoint and API Version + - name: MaestroApiEndPoint + value: "https://maestro.dot.net" + - name: MaestroApiVersion + value: "2020-02-20" + + - name: SourceLinkCLIVersion + value: 3.0.0 + - name: SymbolToolVersion + value: 1.0.1 + - name: BinlogToolVersion + value: 1.0.11 + + - name: runCodesignValidationInjection + value: false diff --git a/src/arcade/eng/common/core-templates/post-build/setup-maestro-vars.yml b/src/arcade/eng/common/core-templates/post-build/setup-maestro-vars.yml new file mode 100644 index 000000000000..f7602980dbe7 --- /dev/null +++ b/src/arcade/eng/common/core-templates/post-build/setup-maestro-vars.yml @@ -0,0 +1,74 @@ +parameters: + BARBuildId: '' + PromoteToChannelIds: '' + is1ESPipeline: '' + +steps: + - ${{ if eq(parameters.is1ESPipeline, '') }}: + - 'Illegal entry point, is1ESPipeline is not defined. Repository yaml should not directly reference templates in core-templates folder.': error + + - ${{ if eq(coalesce(parameters.PromoteToChannelIds, 0), 0) }}: + - task: DownloadBuildArtifacts@0 + displayName: Download Release Configs + inputs: + buildType: current + artifactName: ReleaseConfigs + checkDownloadedFiles: true + + - task: AzureCLI@2 + name: setReleaseVars + displayName: Set Release Configs Vars + inputs: + azureSubscription: "Darc: Maestro Production" + scriptType: pscore + scriptLocation: inlineScript + inlineScript: | + try { + if (!$Env:PromoteToMaestroChannels -or $Env:PromoteToMaestroChannels.Trim() -eq '') { + $Content = Get-Content $(Build.StagingDirectory)/ReleaseConfigs/ReleaseConfigs.txt + + $BarId = $Content | Select -Index 0 + $Channels = $Content | Select -Index 1 + $IsStableBuild = $Content | Select -Index 2 + + $AzureDevOpsProject = $Env:System_TeamProject + $AzureDevOpsBuildDefinitionId = $Env:System_DefinitionId + $AzureDevOpsBuildId = $Env:Build_BuildId + } + else { + . $(Build.SourcesDirectory)\eng\common\tools.ps1 + $darc = Get-Darc + $buildInfo = & $darc get-build ` + --id ${{ parameters.BARBuildId }} ` + --extended ` + --output-format json ` + --ci ` + | convertFrom-Json + + $BarId = ${{ parameters.BARBuildId }} + $Channels = $Env:PromoteToMaestroChannels -split "," + $Channels = $Channels -join "][" + $Channels = "[$Channels]" + + $IsStableBuild = $buildInfo.stable + $AzureDevOpsProject = $buildInfo.azureDevOpsProject + $AzureDevOpsBuildDefinitionId = $buildInfo.azureDevOpsBuildDefinitionId + $AzureDevOpsBuildId = $buildInfo.azureDevOpsBuildId + } + + Write-Host "##vso[task.setvariable variable=BARBuildId]$BarId" + Write-Host "##vso[task.setvariable variable=TargetChannels]$Channels" + Write-Host "##vso[task.setvariable variable=IsStableBuild]$IsStableBuild" + + Write-Host "##vso[task.setvariable variable=AzDOProjectName]$AzureDevOpsProject" + Write-Host "##vso[task.setvariable variable=AzDOPipelineId]$AzureDevOpsBuildDefinitionId" + Write-Host "##vso[task.setvariable variable=AzDOBuildId]$AzureDevOpsBuildId" + } + catch { + Write-Host $_ + Write-Host $_.Exception + Write-Host $_.ScriptStackTrace + exit 1 + } + env: + PromoteToMaestroChannels: ${{ parameters.PromoteToChannelIds }} diff --git a/src/arcade/eng/common/core-templates/steps/component-governance.yml b/src/arcade/eng/common/core-templates/steps/component-governance.yml new file mode 100644 index 000000000000..cf0649aa9565 --- /dev/null +++ b/src/arcade/eng/common/core-templates/steps/component-governance.yml @@ -0,0 +1,16 @@ +parameters: + disableComponentGovernance: false + componentGovernanceIgnoreDirectories: '' + is1ESPipeline: false + displayName: 'Component Detection' + +steps: +- ${{ if eq(parameters.disableComponentGovernance, 'true') }}: + - script: echo "##vso[task.setvariable variable=skipComponentGovernanceDetection]true" + displayName: Set skipComponentGovernanceDetection variable +- ${{ if ne(parameters.disableComponentGovernance, 'true') }}: + - task: ComponentGovernanceComponentDetection@0 + continueOnError: true + displayName: ${{ parameters.displayName }} + inputs: + ignoreDirectories: ${{ parameters.componentGovernanceIgnoreDirectories }} diff --git a/src/arcade/eng/common/core-templates/steps/enable-internal-runtimes.yml b/src/arcade/eng/common/core-templates/steps/enable-internal-runtimes.yml new file mode 100644 index 000000000000..6bdbf62ac500 --- /dev/null +++ b/src/arcade/eng/common/core-templates/steps/enable-internal-runtimes.yml @@ -0,0 +1,32 @@ +# Obtains internal runtime download credentials and populates the 'dotnetbuilds-internal-container-read-token-base64' +# variable with the base64-encoded SAS token, by default + +parameters: +- name: federatedServiceConnection + type: string + default: 'dotnetbuilds-internal-read' +- name: outputVariableName + type: string + default: 'dotnetbuilds-internal-container-read-token-base64' +- name: expiryInHours + type: number + default: 1 +- name: base64Encode + type: boolean + default: true +- name: is1ESPipeline + type: boolean + default: false + +steps: +- ${{ if ne(variables['System.TeamProject'], 'public') }}: + - template: /eng/common/core-templates/steps/get-delegation-sas.yml + parameters: + federatedServiceConnection: ${{ parameters.federatedServiceConnection }} + outputVariableName: ${{ parameters.outputVariableName }} + expiryInHours: ${{ parameters.expiryInHours }} + base64Encode: ${{ parameters.base64Encode }} + storageAccount: dotnetbuilds + container: internal + permissions: rl + is1ESPipeline: ${{ parameters.is1ESPipeline }} \ No newline at end of file diff --git a/src/arcade/eng/common/core-templates/steps/enable-internal-sources.yml b/src/arcade/eng/common/core-templates/steps/enable-internal-sources.yml new file mode 100644 index 000000000000..64f881bffc3c --- /dev/null +++ b/src/arcade/eng/common/core-templates/steps/enable-internal-sources.yml @@ -0,0 +1,47 @@ +parameters: +# This is the Azure federated service connection that we log into to get an access token. +- name: nugetFederatedServiceConnection + type: string + default: 'dnceng-artifacts-feeds-read' +- name: is1ESPipeline + type: boolean + default: false +# Legacy parameters to allow for PAT usage +- name: legacyCredential + type: string + default: '' + +steps: +- ${{ if ne(variables['System.TeamProject'], 'public') }}: + - ${{ if ne(parameters.legacyCredential, '') }}: + - task: PowerShell@2 + displayName: Setup Internal Feeds + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.ps1 + arguments: -ConfigFile $(Build.SourcesDirectory)/NuGet.config -Password $Env:Token + env: + Token: ${{ parameters.legacyCredential }} + # If running on dnceng (internal project), just use the default behavior for NuGetAuthenticate. + # If running on DevDiv, NuGetAuthenticate is not really an option. It's scoped to a single feed, and we have many feeds that + # may be added. Instead, we'll use the traditional approach (add cred to nuget.config), but use an account token. + - ${{ else }}: + - ${{ if eq(variables['System.TeamProject'], 'internal') }}: + - task: PowerShell@2 + displayName: Setup Internal Feeds + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.ps1 + arguments: -ConfigFile $(Build.SourcesDirectory)/NuGet.config + - ${{ else }}: + - template: /eng/common/templates/steps/get-federated-access-token.yml + parameters: + federatedServiceConnection: ${{ parameters.nugetFederatedServiceConnection }} + outputVariableName: 'dnceng-artifacts-feeds-read-access-token' + - task: PowerShell@2 + displayName: Setup Internal Feeds + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.ps1 + arguments: -ConfigFile $(Build.SourcesDirectory)/NuGet.config -Password $(dnceng-artifacts-feeds-read-access-token) + # This is required in certain scenarios to install the ADO credential provider. + # It installed by default in some msbuild invocations (e.g. VS msbuild), but needs to be installed for others + # (e.g. dotnet msbuild). + - task: NuGetAuthenticate@1 diff --git a/src/arcade/eng/common/core-templates/steps/get-federated-access-token.yml b/src/arcade/eng/common/core-templates/steps/get-federated-access-token.yml new file mode 100644 index 000000000000..3a4d4410c482 --- /dev/null +++ b/src/arcade/eng/common/core-templates/steps/get-federated-access-token.yml @@ -0,0 +1,42 @@ +parameters: +- name: federatedServiceConnection + type: string +- name: outputVariableName + type: string +- name: is1ESPipeline + type: boolean +- name: stepName + type: string + default: 'getFederatedAccessToken' +- name: condition + type: string + default: '' +# Resource to get a token for. Common values include: +# - '499b84ac-1321-427f-aa17-267ca6975798' for Azure DevOps +# - 'https://storage.azure.com/' for storage +# Defaults to Azure DevOps +- name: resource + type: string + default: '499b84ac-1321-427f-aa17-267ca6975798' +- name: isStepOutputVariable + type: boolean + default: false + +steps: +- task: AzureCLI@2 + displayName: 'Getting federated access token for feeds' + name: ${{ parameters.stepName }} + ${{ if ne(parameters.condition, '') }}: + condition: ${{ parameters.condition }} + inputs: + azureSubscription: ${{ parameters.federatedServiceConnection }} + scriptType: 'pscore' + scriptLocation: 'inlineScript' + inlineScript: | + $accessToken = az account get-access-token --query accessToken --resource ${{ parameters.resource }} --output tsv + if ($LASTEXITCODE -ne 0) { + Write-Error "Failed to get access token for resource '${{ parameters.resource }}'" + exit 1 + } + Write-Host "Setting '${{ parameters.outputVariableName }}' with the access token value" + Write-Host "##vso[task.setvariable variable=${{ parameters.outputVariableName }};issecret=true;isOutput=${{ parameters.isStepOutputVariable }}]$accessToken" \ No newline at end of file diff --git a/src/arcade/eng/common/core-templates/steps/publish-build-artifacts.yml b/src/arcade/eng/common/core-templates/steps/publish-build-artifacts.yml new file mode 100644 index 000000000000..f24ce346684e --- /dev/null +++ b/src/arcade/eng/common/core-templates/steps/publish-build-artifacts.yml @@ -0,0 +1,20 @@ +parameters: +- name: is1ESPipeline + type: boolean + default: false +- name: args + type: object + default: {} +steps: +- ${{ if ne(parameters.is1ESPipeline, true) }}: + - template: /eng/common/templates/steps/publish-build-artifacts.yml + parameters: + is1ESPipeline: ${{ parameters.is1ESPipeline }} + ${{ each parameter in parameters.args }}: + ${{ parameter.key }}: ${{ parameter.value }} +- ${{ else }}: + - template: /eng/common/templates-official/steps/publish-build-artifacts.yml + parameters: + is1ESPipeline: ${{ parameters.is1ESPipeline }} + ${{ each parameter in parameters.args }}: + ${{ parameter.key }}: ${{ parameter.value }} \ No newline at end of file diff --git a/src/arcade/eng/common/core-templates/steps/publish-pipeline-artifacts.yml b/src/arcade/eng/common/core-templates/steps/publish-pipeline-artifacts.yml new file mode 100644 index 000000000000..2efec04dc2c1 --- /dev/null +++ b/src/arcade/eng/common/core-templates/steps/publish-pipeline-artifacts.yml @@ -0,0 +1,20 @@ +parameters: +- name: is1ESPipeline + type: boolean + default: false + +- name: args + type: object + default: {} + +steps: +- ${{ if ne(parameters.is1ESPipeline, true) }}: + - template: /eng/common/templates/steps/publish-pipeline-artifacts.yml + parameters: + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} +- ${{ else }}: + - template: /eng/common/templates-official/steps/publish-pipeline-artifacts.yml + parameters: + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} diff --git a/src/arcade/eng/common/core-templates/steps/retain-build.yml b/src/arcade/eng/common/core-templates/steps/retain-build.yml new file mode 100644 index 000000000000..83d97a26a01f --- /dev/null +++ b/src/arcade/eng/common/core-templates/steps/retain-build.yml @@ -0,0 +1,28 @@ +parameters: + # Optional azure devops PAT with build execute permissions for the build's organization, + # only needed if the build that should be retained ran on a different organization than + # the pipeline where this template is executing from + Token: '' + # Optional BuildId to retain, defaults to the current running build + BuildId: '' + # Azure devops Organization URI for the build in the https://dev.azure.com/ format. + # Defaults to the organization the current pipeline is running on + AzdoOrgUri: '$(System.CollectionUri)' + # Azure devops project for the build. Defaults to the project the current pipeline is running on + AzdoProject: '$(System.TeamProject)' + +steps: + - task: powershell@2 + inputs: + targetType: 'filePath' + filePath: eng/common/retain-build.ps1 + pwsh: true + arguments: > + -AzdoOrgUri: ${{parameters.AzdoOrgUri}} + -AzdoProject ${{parameters.AzdoProject}} + -Token ${{coalesce(parameters.Token, '$env:SYSTEM_ACCESSTOKEN') }} + -BuildId ${{coalesce(parameters.BuildId, '$env:BUILD_ID')}} + displayName: Enable permanent build retention + env: + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + BUILD_ID: $(Build.BuildId) \ No newline at end of file diff --git a/src/arcade/eng/common/core-templates/steps/send-to-helix.yml b/src/arcade/eng/common/core-templates/steps/send-to-helix.yml new file mode 100644 index 000000000000..68fa739c4ab2 --- /dev/null +++ b/src/arcade/eng/common/core-templates/steps/send-to-helix.yml @@ -0,0 +1,93 @@ +# Please remember to update the documentation if you make changes to these parameters! +parameters: + HelixSource: 'pr/default' # required -- sources must start with pr/, official/, prodcon/, or agent/ + HelixType: 'tests/default/' # required -- Helix telemetry which identifies what type of data this is; should include "test" for clarity and must end in '/' + HelixBuild: $(Build.BuildNumber) # required -- the build number Helix will use to identify this -- automatically set to the AzDO build number + HelixTargetQueues: '' # required -- semicolon-delimited list of Helix queues to test on; see https://helix.dot.net/ for a list of queues + HelixAccessToken: '' # required -- access token to make Helix API requests; should be provided by the appropriate variable group + HelixProjectPath: 'eng/common/helixpublish.proj' # optional -- path to the project file to build relative to BUILD_SOURCESDIRECTORY + HelixProjectArguments: '' # optional -- arguments passed to the build command + HelixConfiguration: '' # optional -- additional property attached to a job + HelixPreCommands: '' # optional -- commands to run before Helix work item execution + HelixPostCommands: '' # optional -- commands to run after Helix work item execution + WorkItemDirectory: '' # optional -- a payload directory to zip up and send to Helix; requires WorkItemCommand; incompatible with XUnitProjects + WorkItemCommand: '' # optional -- a command to execute on the payload; requires WorkItemDirectory; incompatible with XUnitProjects + WorkItemTimeout: '' # optional -- a timeout in TimeSpan.Parse-ready value (e.g. 00:02:00) for the work item command; requires WorkItemDirectory; incompatible with XUnitProjects + CorrelationPayloadDirectory: '' # optional -- a directory to zip up and send to Helix as a correlation payload + XUnitProjects: '' # optional -- semicolon-delimited list of XUnitProjects to parse and send to Helix; requires XUnitRuntimeTargetFramework, XUnitPublishTargetFramework, XUnitRunnerVersion, and IncludeDotNetCli=true + XUnitWorkItemTimeout: '' # optional -- the workitem timeout in seconds for all workitems created from the xUnit projects specified by XUnitProjects + XUnitPublishTargetFramework: '' # optional -- framework to use to publish your xUnit projects + XUnitRuntimeTargetFramework: '' # optional -- framework to use for the xUnit console runner + XUnitRunnerVersion: '' # optional -- version of the xUnit nuget package you wish to use on Helix; required for XUnitProjects + IncludeDotNetCli: false # optional -- true will download a version of the .NET CLI onto the Helix machine as a correlation payload; requires DotNetCliPackageType and DotNetCliVersion + DotNetCliPackageType: '' # optional -- either 'sdk', 'runtime' or 'aspnetcore-runtime'; determines whether the sdk or runtime will be sent to Helix; see https://raw.githubusercontent.com/dotnet/core/main/release-notes/releases-index.json + DotNetCliVersion: '' # optional -- version of the CLI to send to Helix; based on this: https://raw.githubusercontent.com/dotnet/core/main/release-notes/releases-index.json + WaitForWorkItemCompletion: true # optional -- true will make the task wait until work items have been completed and fail the build if work items fail. False is "fire and forget." + IsExternal: false # [DEPRECATED] -- doesn't do anything, jobs are external if HelixAccessToken is empty and Creator is set + HelixBaseUri: 'https://helix.dot.net/' # optional -- sets the Helix API base URI (allows targeting https://helix.int-dot.net ) + Creator: '' # optional -- if the build is external, use this to specify who is sending the job + DisplayNamePrefix: 'Run Tests' # optional -- rename the beginning of the displayName of the steps in AzDO + condition: succeeded() # optional -- condition for step to execute; defaults to succeeded() + continueOnError: false # optional -- determines whether to continue the build if the step errors; defaults to false + +steps: + - powershell: 'powershell "$env:BUILD_SOURCESDIRECTORY\eng\common\msbuild.ps1 $env:BUILD_SOURCESDIRECTORY/${{ parameters.HelixProjectPath }} /restore /p:TreatWarningsAsErrors=false ${{ parameters.HelixProjectArguments }} /t:Test /bl:$env:BUILD_SOURCESDIRECTORY\artifacts\log\$env:BuildConfig\SendToHelix.binlog"' + displayName: ${{ parameters.DisplayNamePrefix }} (Windows) + env: + BuildConfig: $(_BuildConfig) + HelixSource: ${{ parameters.HelixSource }} + HelixType: ${{ parameters.HelixType }} + HelixBuild: ${{ parameters.HelixBuild }} + HelixConfiguration: ${{ parameters.HelixConfiguration }} + HelixTargetQueues: ${{ parameters.HelixTargetQueues }} + HelixAccessToken: ${{ parameters.HelixAccessToken }} + HelixPreCommands: ${{ parameters.HelixPreCommands }} + HelixPostCommands: ${{ parameters.HelixPostCommands }} + WorkItemDirectory: ${{ parameters.WorkItemDirectory }} + WorkItemCommand: ${{ parameters.WorkItemCommand }} + WorkItemTimeout: ${{ parameters.WorkItemTimeout }} + CorrelationPayloadDirectory: ${{ parameters.CorrelationPayloadDirectory }} + XUnitProjects: ${{ parameters.XUnitProjects }} + XUnitWorkItemTimeout: ${{ parameters.XUnitWorkItemTimeout }} + XUnitPublishTargetFramework: ${{ parameters.XUnitPublishTargetFramework }} + XUnitRuntimeTargetFramework: ${{ parameters.XUnitRuntimeTargetFramework }} + XUnitRunnerVersion: ${{ parameters.XUnitRunnerVersion }} + IncludeDotNetCli: ${{ parameters.IncludeDotNetCli }} + DotNetCliPackageType: ${{ parameters.DotNetCliPackageType }} + DotNetCliVersion: ${{ parameters.DotNetCliVersion }} + WaitForWorkItemCompletion: ${{ parameters.WaitForWorkItemCompletion }} + HelixBaseUri: ${{ parameters.HelixBaseUri }} + Creator: ${{ parameters.Creator }} + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + condition: and(${{ parameters.condition }}, eq(variables['Agent.Os'], 'Windows_NT')) + continueOnError: ${{ parameters.continueOnError }} + - script: $BUILD_SOURCESDIRECTORY/eng/common/msbuild.sh $BUILD_SOURCESDIRECTORY/${{ parameters.HelixProjectPath }} /restore /p:TreatWarningsAsErrors=false ${{ parameters.HelixProjectArguments }} /t:Test /bl:$BUILD_SOURCESDIRECTORY/artifacts/log/$BuildConfig/SendToHelix.binlog + displayName: ${{ parameters.DisplayNamePrefix }} (Unix) + env: + BuildConfig: $(_BuildConfig) + HelixSource: ${{ parameters.HelixSource }} + HelixType: ${{ parameters.HelixType }} + HelixBuild: ${{ parameters.HelixBuild }} + HelixConfiguration: ${{ parameters.HelixConfiguration }} + HelixTargetQueues: ${{ parameters.HelixTargetQueues }} + HelixAccessToken: ${{ parameters.HelixAccessToken }} + HelixPreCommands: ${{ parameters.HelixPreCommands }} + HelixPostCommands: ${{ parameters.HelixPostCommands }} + WorkItemDirectory: ${{ parameters.WorkItemDirectory }} + WorkItemCommand: ${{ parameters.WorkItemCommand }} + WorkItemTimeout: ${{ parameters.WorkItemTimeout }} + CorrelationPayloadDirectory: ${{ parameters.CorrelationPayloadDirectory }} + XUnitProjects: ${{ parameters.XUnitProjects }} + XUnitWorkItemTimeout: ${{ parameters.XUnitWorkItemTimeout }} + XUnitPublishTargetFramework: ${{ parameters.XUnitPublishTargetFramework }} + XUnitRuntimeTargetFramework: ${{ parameters.XUnitRuntimeTargetFramework }} + XUnitRunnerVersion: ${{ parameters.XUnitRunnerVersion }} + IncludeDotNetCli: ${{ parameters.IncludeDotNetCli }} + DotNetCliPackageType: ${{ parameters.DotNetCliPackageType }} + DotNetCliVersion: ${{ parameters.DotNetCliVersion }} + WaitForWorkItemCompletion: ${{ parameters.WaitForWorkItemCompletion }} + HelixBaseUri: ${{ parameters.HelixBaseUri }} + Creator: ${{ parameters.Creator }} + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + condition: and(${{ parameters.condition }}, ne(variables['Agent.Os'], 'Windows_NT')) + continueOnError: ${{ parameters.continueOnError }} diff --git a/src/arcade/eng/common/core-templates/variables/pool-providers.yml b/src/arcade/eng/common/core-templates/variables/pool-providers.yml new file mode 100644 index 000000000000..41053d382a2e --- /dev/null +++ b/src/arcade/eng/common/core-templates/variables/pool-providers.yml @@ -0,0 +1,8 @@ +parameters: + is1ESPipeline: false + +variables: + - ${{ if eq(parameters.is1ESPipeline, 'true') }}: + - template: /eng/common/templates-official/variables/pool-providers.yml + - ${{ else }}: + - template: /eng/common/templates/variables/pool-providers.yml \ No newline at end of file diff --git a/src/arcade/eng/common/cross/arm/tizen/tizen.patch b/src/arcade/eng/common/cross/arm/tizen/tizen.patch new file mode 100644 index 000000000000..fb12ade7250a --- /dev/null +++ b/src/arcade/eng/common/cross/arm/tizen/tizen.patch @@ -0,0 +1,9 @@ +diff -u -r a/usr/lib/libc.so b/usr/lib/libc.so +--- a/usr/lib/libc.so 2016-12-30 23:00:08.284951863 +0900 ++++ b/usr/lib/libc.so 2016-12-30 23:00:32.140951815 +0900 +@@ -2,4 +2,4 @@ + Use the shared library, but some functions are only in + the static library, so try that secondarily. */ + OUTPUT_FORMAT(elf32-littlearm) +-GROUP ( /lib/libc.so.6 /usr/lib/libc_nonshared.a AS_NEEDED ( /lib/ld-linux-armhf.so.3 ) ) ++GROUP ( libc.so.6 libc_nonshared.a AS_NEEDED ( ld-linux-armhf.so.3 ) ) diff --git a/src/arcade/eng/common/cross/armel/tizen/tizen.patch b/src/arcade/eng/common/cross/armel/tizen/tizen.patch new file mode 100644 index 000000000000..ca7c7c1ff751 --- /dev/null +++ b/src/arcade/eng/common/cross/armel/tizen/tizen.patch @@ -0,0 +1,9 @@ +diff -u -r a/usr/lib/libc.so b/usr/lib/libc.so +--- a/usr/lib/libc.so 2016-12-30 23:00:08.284951863 +0900 ++++ b/usr/lib/libc.so 2016-12-30 23:00:32.140951815 +0900 +@@ -2,4 +2,4 @@ + Use the shared library, but some functions are only in + the static library, so try that secondarily. */ + OUTPUT_FORMAT(elf32-littlearm) +-GROUP ( /lib/libc.so.6 /usr/lib/libc_nonshared.a AS_NEEDED ( /lib/ld-linux.so.3 ) ) ++GROUP ( libc.so.6 libc_nonshared.a AS_NEEDED ( ld-linux.so.3 ) ) diff --git a/src/arcade/eng/common/cross/riscv64/tizen/tizen.patch b/src/arcade/eng/common/cross/riscv64/tizen/tizen.patch new file mode 100644 index 000000000000..eb6d1c07470b --- /dev/null +++ b/src/arcade/eng/common/cross/riscv64/tizen/tizen.patch @@ -0,0 +1,9 @@ +diff -u -r a/usr/lib/libc.so b/usr/lib/libc.so +--- a/usr/lib64/libc.so 2016-12-30 23:00:08.284951863 +0900 ++++ b/usr/lib64/libc.so 2016-12-30 23:00:32.140951815 +0900 +@@ -2,4 +2,4 @@ + Use the shared library, but some functions are only in + the static library, so try that secondarily. */ + OUTPUT_FORMAT(elf64-littleriscv) +-GROUP ( /lib64/libc.so.6 /usr/lib64/libc_nonshared.a AS_NEEDED ( /lib64/ld-linux-riscv64-lp64d.so.1 ) ) ++GROUP ( libc.so.6 libc_nonshared.a AS_NEEDED ( ld-linux-riscv64-lp64d.so.1 ) ) diff --git a/src/arcade/eng/common/cross/tizen-build-rootfs.sh b/src/arcade/eng/common/cross/tizen-build-rootfs.sh new file mode 100644 index 000000000000..ba31c93285f6 --- /dev/null +++ b/src/arcade/eng/common/cross/tizen-build-rootfs.sh @@ -0,0 +1,82 @@ +#!/usr/bin/env bash +set -e + +ARCH=$1 +LINK_ARCH=$ARCH + +case "$ARCH" in + arm) + TIZEN_ARCH="armv7hl" + ;; + armel) + TIZEN_ARCH="armv7l" + LINK_ARCH="arm" + ;; + arm64) + TIZEN_ARCH="aarch64" + ;; + x86) + TIZEN_ARCH="i686" + ;; + x64) + TIZEN_ARCH="x86_64" + LINK_ARCH="x86" + ;; + riscv64) + TIZEN_ARCH="riscv64" + LINK_ARCH="riscv" + ;; + *) + echo "Unsupported architecture for tizen: $ARCH" + exit 1 +esac + +__CrossDir=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) +__TIZEN_CROSSDIR="$__CrossDir/${ARCH}/tizen" + +if [[ -z "$ROOTFS_DIR" ]]; then + echo "ROOTFS_DIR is not defined." + exit 1; +fi + +TIZEN_TMP_DIR=$ROOTFS_DIR/tizen_tmp +mkdir -p $TIZEN_TMP_DIR + +# Download files +echo ">>Start downloading files" +VERBOSE=1 $__CrossDir/tizen-fetch.sh $TIZEN_TMP_DIR $TIZEN_ARCH +echo "<>Start constructing Tizen rootfs" +TIZEN_RPM_FILES=`ls $TIZEN_TMP_DIR/*.rpm` +cd $ROOTFS_DIR +for f in $TIZEN_RPM_FILES; do + rpm2cpio $f | cpio -idm --quiet +done +echo "<>Start configuring Tizen rootfs" +ln -sfn asm-${LINK_ARCH} ./usr/include/asm +patch -p1 < $__TIZEN_CROSSDIR/tizen.patch +if [[ "$TIZEN_ARCH" == "riscv64" ]]; then + echo "Fixing broken symlinks in $PWD" + rm ./usr/lib64/libresolv.so + ln -s ../../lib64/libresolv.so.2 ./usr/lib64/libresolv.so + rm ./usr/lib64/libpthread.so + ln -s ../../lib64/libpthread.so.0 ./usr/lib64/libpthread.so + rm ./usr/lib64/libdl.so + ln -s ../../lib64/libdl.so.2 ./usr/lib64/libdl.so + rm ./usr/lib64/libutil.so + ln -s ../../lib64/libutil.so.1 ./usr/lib64/libutil.so + rm ./usr/lib64/libm.so + ln -s ../../lib64/libm.so.6 ./usr/lib64/libm.so + rm ./usr/lib64/librt.so + ln -s ../../lib64/librt.so.1 ./usr/lib64/librt.so + rm ./lib/ld-linux-riscv64-lp64d.so.1 + ln -s ../lib64/ld-linux-riscv64-lp64d.so.1 ./lib/ld-linux-riscv64-lp64d.so.1 +fi +echo "< 0 ]]; do + opt="$(echo "$1" | tr "[:upper:]" "[:lower:]")" + case "$opt" in + --darcversion) + darcVersion=$2 + shift + ;; + --versionendpoint) + versionEndpoint=$2 + shift + ;; + --verbosity) + verbosity=$2 + shift + ;; + --toolpath) + toolpath=$2 + shift + ;; + *) + echo "Invalid argument: $1" + usage + exit 1 + ;; + esac + + shift +done + +# resolve $source until the file is no longer a symlink +while [[ -h "$source" ]]; do + scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" + source="$(readlink "$source")" + # if $source was a relative symlink, we need to resolve it relative to the path where the + # symlink file was located + [[ $source != /* ]] && source="$scriptroot/$source" +done +scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" + +. "$scriptroot/tools.sh" + +if [ -z "$darcVersion" ]; then + darcVersion=$(curl -X GET "$versionEndpoint" -H "accept: text/plain") +fi + +function InstallDarcCli { + local darc_cli_package_name="microsoft.dotnet.darc" + + InitializeDotNetCli true + local dotnet_root=$_InitializeDotNetCli + + if [ -z "$toolpath" ]; then + local tool_list=$($dotnet_root/dotnet tool list -g) + if [[ $tool_list = *$darc_cli_package_name* ]]; then + echo $($dotnet_root/dotnet tool uninstall $darc_cli_package_name -g) + fi + else + local tool_list=$($dotnet_root/dotnet tool list --tool-path "$toolpath") + if [[ $tool_list = *$darc_cli_package_name* ]]; then + echo $($dotnet_root/dotnet tool uninstall $darc_cli_package_name --tool-path "$toolpath") + fi + fi + + local arcadeServicesSource="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json" + + echo "Installing Darc CLI version $darcVersion..." + echo "You may need to restart your command shell if this is the first dotnet tool you have installed." + if [ -z "$toolpath" ]; then + echo $($dotnet_root/dotnet tool install $darc_cli_package_name --version $darcVersion --add-source "$arcadeServicesSource" -v $verbosity -g) + else + echo $($dotnet_root/dotnet tool install $darc_cli_package_name --version $darcVersion --add-source "$arcadeServicesSource" -v $verbosity --tool-path "$toolpath") + fi +} + +InstallDarcCli diff --git a/src/arcade/eng/common/dotnet-install.cmd b/src/arcade/eng/common/dotnet-install.cmd new file mode 100644 index 000000000000..b1c2642e76f7 --- /dev/null +++ b/src/arcade/eng/common/dotnet-install.cmd @@ -0,0 +1,2 @@ +@echo off +powershell -ExecutionPolicy ByPass -NoProfile -command "& """%~dp0dotnet-install.ps1""" %*" \ No newline at end of file diff --git a/src/arcade/eng/common/dotnet-install.ps1 b/src/arcade/eng/common/dotnet-install.ps1 new file mode 100644 index 000000000000..811f0f717f73 --- /dev/null +++ b/src/arcade/eng/common/dotnet-install.ps1 @@ -0,0 +1,28 @@ +[CmdletBinding(PositionalBinding=$false)] +Param( + [string] $verbosity = 'minimal', + [string] $architecture = '', + [string] $version = 'Latest', + [string] $runtime = 'dotnet', + [string] $RuntimeSourceFeed = '', + [string] $RuntimeSourceFeedKey = '' +) + +. $PSScriptRoot\tools.ps1 + +$dotnetRoot = Join-Path $RepoRoot '.dotnet' + +$installdir = $dotnetRoot +try { + if ($architecture -and $architecture.Trim() -eq 'x86') { + $installdir = Join-Path $installdir 'x86' + } + InstallDotNet $installdir $version $architecture $runtime $true -RuntimeSourceFeed $RuntimeSourceFeed -RuntimeSourceFeedKey $RuntimeSourceFeedKey +} +catch { + Write-Host $_.ScriptStackTrace + Write-PipelineTelemetryError -Category 'InitializeToolset' -Message $_ + ExitWithExitCode 1 +} + +ExitWithExitCode 0 diff --git a/src/arcade/eng/common/dotnet-install.sh b/src/arcade/eng/common/dotnet-install.sh new file mode 100644 index 000000000000..7b9d97e3bd4d --- /dev/null +++ b/src/arcade/eng/common/dotnet-install.sh @@ -0,0 +1,94 @@ +#!/usr/bin/env bash + +source="${BASH_SOURCE[0]}" +# resolve $source until the file is no longer a symlink +while [[ -h "$source" ]]; do + scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" + source="$(readlink "$source")" + # if $source was a relative symlink, we need to resolve it relative to the path where the + # symlink file was located + [[ $source != /* ]] && source="$scriptroot/$source" +done +scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" + +. "$scriptroot/tools.sh" + +version='Latest' +architecture='' +runtime='dotnet' +runtimeSourceFeed='' +runtimeSourceFeedKey='' +while [[ $# > 0 ]]; do + opt="$(echo "$1" | tr "[:upper:]" "[:lower:]")" + case "$opt" in + -version|-v) + shift + version="$1" + ;; + -architecture|-a) + shift + architecture="$1" + ;; + -runtime|-r) + shift + runtime="$1" + ;; + -runtimesourcefeed) + shift + runtimeSourceFeed="$1" + ;; + -runtimesourcefeedkey) + shift + runtimeSourceFeedKey="$1" + ;; + *) + Write-PipelineTelemetryError -Category 'Build' -Message "Invalid argument: $1" + exit 1 + ;; + esac + shift +done + +# Use uname to determine what the CPU is, see https://en.wikipedia.org/wiki/Uname#Examples +cpuname=$(uname -m) +case $cpuname in + arm64|aarch64) + buildarch=arm64 + if [ "$(getconf LONG_BIT)" -lt 64 ]; then + # This is 32-bit OS running on 64-bit CPU (for example Raspberry Pi OS) + buildarch=arm + fi + ;; + loongarch64) + buildarch=loongarch64 + ;; + amd64|x86_64) + buildarch=x64 + ;; + armv*l) + buildarch=arm + ;; + i[3-6]86) + buildarch=x86 + ;; + riscv64) + buildarch=riscv64 + ;; + *) + echo "Unknown CPU $cpuname detected, treating it as x64" + buildarch=x64 + ;; +esac + +dotnetRoot="${repo_root}.dotnet" +if [[ $architecture != "" ]] && [[ $architecture != $buildarch ]]; then + dotnetRoot="$dotnetRoot/$architecture" +fi + +InstallDotNet "$dotnetRoot" $version "$architecture" $runtime true $runtimeSourceFeed $runtimeSourceFeedKey || { + local exit_code=$? + Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "dotnet-install.sh failed (exit code '$exit_code')." >&2 + ExitWithExitCode $exit_code +} + +ExitWithExitCode 0 diff --git a/src/arcade/eng/common/enable-cross-org-publishing.ps1 b/src/arcade/eng/common/enable-cross-org-publishing.ps1 new file mode 100644 index 000000000000..da09da4f1fc4 --- /dev/null +++ b/src/arcade/eng/common/enable-cross-org-publishing.ps1 @@ -0,0 +1,13 @@ +param( + [string] $token +) + + +. $PSScriptRoot\pipeline-logging-functions.ps1 + +# Write-PipelineSetVariable will no-op if a variable named $ci is not defined +# Since this script is only ever called in AzDO builds, just universally set it +$ci = $true + +Write-PipelineSetVariable -Name 'VSS_NUGET_ACCESSTOKEN' -Value $token -IsMultiJobVariable $false +Write-PipelineSetVariable -Name 'VSS_NUGET_URI_PREFIXES' -Value 'https://dnceng.pkgs.visualstudio.com/;https://pkgs.dev.azure.com/dnceng/;https://devdiv.pkgs.visualstudio.com/;https://pkgs.dev.azure.com/devdiv/' -IsMultiJobVariable $false diff --git a/src/arcade/eng/common/generate-locproject.ps1 b/src/arcade/eng/common/generate-locproject.ps1 new file mode 100644 index 000000000000..524aaa57f2b7 --- /dev/null +++ b/src/arcade/eng/common/generate-locproject.ps1 @@ -0,0 +1,189 @@ +Param( + [Parameter(Mandatory=$true)][string] $SourcesDirectory, # Directory where source files live; if using a Localize directory it should live in here + [string] $LanguageSet = 'VS_Main_Languages', # Language set to be used in the LocProject.json + [switch] $UseCheckedInLocProjectJson, # When set, generates a LocProject.json and compares it to one that already exists in the repo; otherwise just generates one + [switch] $CreateNeutralXlfs # Creates neutral xlf files. Only set to false when running locally +) + +# Generates LocProject.json files for the OneLocBuild task. OneLocBuildTask is described here: +# https://ceapex.visualstudio.com/CEINTL/_wiki/wikis/CEINTL.wiki/107/Localization-with-OneLocBuild-Task + +Set-StrictMode -Version 2.0 +$ErrorActionPreference = "Stop" +. $PSScriptRoot\pipeline-logging-functions.ps1 + +$exclusionsFilePath = "$SourcesDirectory\eng\Localize\LocExclusions.json" +$exclusions = @{ Exclusions = @() } +if (Test-Path -Path $exclusionsFilePath) +{ + $exclusions = Get-Content "$exclusionsFilePath" | ConvertFrom-Json +} + +Push-Location "$SourcesDirectory" # push location for Resolve-Path -Relative to work + +# Template files +$jsonFiles = @() +$jsonTemplateFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "\.template\.config\\localize\\.+\.en\.json" } # .NET templating pattern +$jsonTemplateFiles | ForEach-Object { + $null = $_.Name -Match "(.+)\.[\w-]+\.json" # matches '[filename].[langcode].json + + $destinationFile = "$($_.Directory.FullName)\$($Matches.1).json" + $jsonFiles += Copy-Item "$($_.FullName)" -Destination $destinationFile -PassThru +} + +$jsonWinformsTemplateFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "en\\strings\.json" } # current winforms pattern + +$wxlFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "\\.+\.wxl" -And -Not( $_.Directory.Name -Match "\d{4}" ) } # localized files live in four digit lang ID directories; this excludes them +if (-not $wxlFiles) { + $wxlEnFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "\\1033\\.+\.wxl" } # pick up en files (1033 = en) specifically so we can copy them to use as the neutral xlf files + if ($wxlEnFiles) { + $wxlFiles = @() + $wxlEnFiles | ForEach-Object { + $destinationFile = "$($_.Directory.Parent.FullName)\$($_.Name)" + $wxlFiles += Copy-Item "$($_.FullName)" -Destination $destinationFile -PassThru + } + } +} + +$macosHtmlEnFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "en\.lproj\\.+\.html$" } # add installer HTML files +$macosHtmlFiles = @() +if ($macosHtmlEnFiles) { + $macosHtmlEnFiles | ForEach-Object { + $destinationFile = "$($_.Directory.Parent.FullName)\$($_.Name)" + $macosHtmlFiles += Copy-Item "$($_.FullName)" -Destination $destinationFile -PassThru + } +} + +$xlfFiles = @() + +$allXlfFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory\*\*.xlf" +$langXlfFiles = @() +if ($allXlfFiles) { + $null = $allXlfFiles[0].FullName -Match "\.([\w-]+)\.xlf" # matches '[langcode].xlf' + $firstLangCode = $Matches.1 + $langXlfFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory\*\*.$firstLangCode.xlf" +} +$langXlfFiles | ForEach-Object { + $null = $_.Name -Match "(.+)\.[\w-]+\.xlf" # matches '[filename].[langcode].xlf + + $destinationFile = "$($_.Directory.FullName)\$($Matches.1).xlf" + $xlfFiles += Copy-Item "$($_.FullName)" -Destination $destinationFile -PassThru +} + +$locFiles = $jsonFiles + $jsonWinformsTemplateFiles + $xlfFiles + +$locJson = @{ + Projects = @( + @{ + LanguageSet = $LanguageSet + LocItems = @( + $locFiles | ForEach-Object { + $outputPath = "$(($_.DirectoryName | Resolve-Path -Relative) + "\")" + $continue = $true + foreach ($exclusion in $exclusions.Exclusions) { + if ($_.FullName.Contains($exclusion)) + { + $continue = $false + } + } + $sourceFile = ($_.FullName | Resolve-Path -Relative) + if (!$CreateNeutralXlfs -and $_.Extension -eq '.xlf') { + Remove-Item -Path $sourceFile + } + if ($continue) + { + if ($_.Directory.Name -eq 'en' -and $_.Extension -eq '.json') { + return @{ + SourceFile = $sourceFile + CopyOption = "LangIDOnPath" + OutputPath = "$($_.Directory.Parent.FullName | Resolve-Path -Relative)\" + } + } else { + return @{ + SourceFile = $sourceFile + CopyOption = "LangIDOnName" + OutputPath = $outputPath + } + } + } + } + ) + }, + @{ + LanguageSet = $LanguageSet + CloneLanguageSet = "WiX_CloneLanguages" + LssFiles = @( "wxl_loc.lss" ) + LocItems = @( + $wxlFiles | ForEach-Object { + $outputPath = "$($_.Directory.FullName | Resolve-Path -Relative)\" + $continue = $true + foreach ($exclusion in $exclusions.Exclusions) { + if ($_.FullName.Contains($exclusion)) { + $continue = $false + } + } + $sourceFile = ($_.FullName | Resolve-Path -Relative) + if ($continue) + { + return @{ + SourceFile = $sourceFile + CopyOption = "LangIDOnPath" + OutputPath = $outputPath + } + } + } + ) + }, + @{ + LanguageSet = $LanguageSet + CloneLanguageSet = "VS_macOS_CloneLanguages" + LssFiles = @( ".\eng\common\loc\P22DotNetHtmlLocalization.lss" ) + LocItems = @( + $macosHtmlFiles | ForEach-Object { + $outputPath = "$($_.Directory.FullName | Resolve-Path -Relative)\" + $continue = $true + foreach ($exclusion in $exclusions.Exclusions) { + if ($_.FullName.Contains($exclusion)) { + $continue = $false + } + } + $sourceFile = ($_.FullName | Resolve-Path -Relative) + $lciFile = $sourceFile + ".lci" + if ($continue) { + $result = @{ + SourceFile = $sourceFile + CopyOption = "LangIDOnPath" + OutputPath = $outputPath + } + if (Test-Path $lciFile -PathType Leaf) { + $result["LciFile"] = $lciFile + } + return $result + } + } + ) + } + ) +} + +$json = ConvertTo-Json $locJson -Depth 5 +Write-Host "LocProject.json generated:`n`n$json`n`n" +Pop-Location + +if (!$UseCheckedInLocProjectJson) { + New-Item "$SourcesDirectory\eng\Localize\LocProject.json" -Force # Need this to make sure the Localize directory is created + Set-Content "$SourcesDirectory\eng\Localize\LocProject.json" $json +} +else { + New-Item "$SourcesDirectory\eng\Localize\LocProject-generated.json" -Force # Need this to make sure the Localize directory is created + Set-Content "$SourcesDirectory\eng\Localize\LocProject-generated.json" $json + + if ((Get-FileHash "$SourcesDirectory\eng\Localize\LocProject-generated.json").Hash -ne (Get-FileHash "$SourcesDirectory\eng\Localize\LocProject.json").Hash) { + Write-PipelineTelemetryError -Category "OneLocBuild" -Message "Existing LocProject.json differs from generated LocProject.json. Download LocProject-generated.json and compare them." + + exit 1 + } + else { + Write-Host "Generated LocProject.json and current LocProject.json are identical." + } +} diff --git a/src/arcade/eng/common/generate-sbom-prep.ps1 b/src/arcade/eng/common/generate-sbom-prep.ps1 new file mode 100644 index 000000000000..a0c7d792a76f --- /dev/null +++ b/src/arcade/eng/common/generate-sbom-prep.ps1 @@ -0,0 +1,29 @@ +Param( + [Parameter(Mandatory=$true)][string] $ManifestDirPath # Manifest directory where sbom will be placed +) + +. $PSScriptRoot\pipeline-logging-functions.ps1 + +# Normally - we'd listen to the manifest path given, but 1ES templates will overwrite if this level gets uploaded directly +# with their own overwriting ours. So we create it as a sub directory of the requested manifest path. +$ArtifactName = "${env:SYSTEM_STAGENAME}_${env:AGENT_JOBNAME}_SBOM" +$SafeArtifactName = $ArtifactName -replace '["/:<>\\|?@*"() ]', '_' +$SbomGenerationDir = Join-Path $ManifestDirPath $SafeArtifactName + +Write-Host "Artifact name before : $ArtifactName" +Write-Host "Artifact name after : $SafeArtifactName" + +Write-Host "Creating dir $ManifestDirPath" + +# create directory for sbom manifest to be placed +if (!(Test-Path -path $SbomGenerationDir)) +{ + New-Item -ItemType Directory -path $SbomGenerationDir + Write-Host "Successfully created directory $SbomGenerationDir" +} +else{ + Write-PipelineTelemetryError -category 'Build' "Unable to create sbom folder." +} + +Write-Host "Updating artifact name" +Write-Host "##vso[task.setvariable variable=ARTIFACT_NAME]$SafeArtifactName" diff --git a/src/arcade/eng/common/generate-sbom-prep.sh b/src/arcade/eng/common/generate-sbom-prep.sh new file mode 100644 index 000000000000..b8ecca72bbf5 --- /dev/null +++ b/src/arcade/eng/common/generate-sbom-prep.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash + +source="${BASH_SOURCE[0]}" + +# resolve $SOURCE until the file is no longer a symlink +while [[ -h $source ]]; do + scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" + source="$(readlink "$source")" + + # if $source was a relative symlink, we need to resolve it relative to the path where the + # symlink file was located + [[ $source != /* ]] && source="$scriptroot/$source" +done +scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" +. $scriptroot/pipeline-logging-functions.sh + + +# replace all special characters with _, some builds use special characters like : in Agent.Jobname, that is not a permissible name while uploading artifacts. +artifact_name=$SYSTEM_STAGENAME"_"$AGENT_JOBNAME"_SBOM" +safe_artifact_name="${artifact_name//["/:<>\\|?@*$" ]/_}" +manifest_dir=$1 + +# Normally - we'd listen to the manifest path given, but 1ES templates will overwrite if this level gets uploaded directly +# with their own overwriting ours. So we create it as a sub directory of the requested manifest path. +sbom_generation_dir="$manifest_dir/$safe_artifact_name" + +if [ ! -d "$sbom_generation_dir" ] ; then + mkdir -p "$sbom_generation_dir" + echo "Sbom directory created." $sbom_generation_dir +else + Write-PipelineTelemetryError -category 'Build' "Unable to create sbom folder." +fi + +echo "Artifact name before : "$artifact_name +echo "Artifact name after : "$safe_artifact_name +export ARTIFACT_NAME=$safe_artifact_name +echo "##vso[task.setvariable variable=ARTIFACT_NAME]$safe_artifact_name" + +exit 0 diff --git a/src/arcade/eng/common/helixpublish.proj b/src/arcade/eng/common/helixpublish.proj new file mode 100644 index 000000000000..c1323bf41210 --- /dev/null +++ b/src/arcade/eng/common/helixpublish.proj @@ -0,0 +1,27 @@ + + + + + msbuild + + + + + %(Identity) + + + + + + $(WorkItemDirectory) + $(WorkItemCommand) + $(WorkItemTimeout) + + + + + + + + + diff --git a/src/arcade/eng/common/init-tools-native.ps1 b/src/arcade/eng/common/init-tools-native.ps1 new file mode 100644 index 000000000000..27ccdb9ecc95 --- /dev/null +++ b/src/arcade/eng/common/init-tools-native.ps1 @@ -0,0 +1,203 @@ +<# +.SYNOPSIS +Entry point script for installing native tools + +.DESCRIPTION +Reads $RepoRoot\global.json file to determine native assets to install +and executes installers for those tools + +.PARAMETER BaseUri +Base file directory or Url from which to acquire tool archives + +.PARAMETER InstallDirectory +Directory to install native toolset. This is a command-line override for the default +Install directory precedence order: +- InstallDirectory command-line override +- NETCOREENG_INSTALL_DIRECTORY environment variable +- (default) %USERPROFILE%/.netcoreeng/native + +.PARAMETER Clean +Switch specifying to not install anything, but cleanup native asset folders + +.PARAMETER Force +Clean and then install tools + +.PARAMETER DownloadRetries +Total number of retry attempts + +.PARAMETER RetryWaitTimeInSeconds +Wait time between retry attempts in seconds + +.PARAMETER GlobalJsonFile +File path to global.json file + +.PARAMETER PathPromotion +Optional switch to enable either promote native tools specified in the global.json to the path (in Azure Pipelines) +or break the build if a native tool is not found on the path (on a local dev machine) + +.NOTES +#> +[CmdletBinding(PositionalBinding=$false)] +Param ( + [string] $BaseUri = 'https://netcorenativeassets.blob.core.windows.net/resource-packages/external', + [string] $InstallDirectory, + [switch] $Clean = $False, + [switch] $Force = $False, + [int] $DownloadRetries = 5, + [int] $RetryWaitTimeInSeconds = 30, + [string] $GlobalJsonFile, + [switch] $PathPromotion +) + +if (!$GlobalJsonFile) { + $GlobalJsonFile = Join-Path (Get-Item $PSScriptRoot).Parent.Parent.FullName 'global.json' +} + +Set-StrictMode -version 2.0 +$ErrorActionPreference='Stop' + +. $PSScriptRoot\pipeline-logging-functions.ps1 +Import-Module -Name (Join-Path $PSScriptRoot 'native\CommonLibrary.psm1') + +try { + # Define verbose switch if undefined + $Verbose = $VerbosePreference -Eq 'Continue' + + $EngCommonBaseDir = Join-Path $PSScriptRoot 'native\' + $NativeBaseDir = $InstallDirectory + if (!$NativeBaseDir) { + $NativeBaseDir = CommonLibrary\Get-NativeInstallDirectory + } + $Env:CommonLibrary_NativeInstallDir = $NativeBaseDir + $InstallBin = Join-Path $NativeBaseDir 'bin' + $InstallerPath = Join-Path $EngCommonBaseDir 'install-tool.ps1' + + # Process tools list + Write-Host "Processing $GlobalJsonFile" + If (-Not (Test-Path $GlobalJsonFile)) { + Write-Host "Unable to find '$GlobalJsonFile'" + exit 0 + } + $NativeTools = Get-Content($GlobalJsonFile) -Raw | + ConvertFrom-Json | + Select-Object -Expand 'native-tools' -ErrorAction SilentlyContinue + if ($NativeTools) { + if ($PathPromotion -eq $True) { + $ArcadeToolsDirectory = "$env:SYSTEMDRIVE\arcade-tools" + if (Test-Path $ArcadeToolsDirectory) { # if this directory exists, we should use native tools on machine + $NativeTools.PSObject.Properties | ForEach-Object { + $ToolName = $_.Name + $ToolVersion = $_.Value + $InstalledTools = @{} + + if ((Get-Command "$ToolName" -ErrorAction SilentlyContinue) -eq $null) { + if ($ToolVersion -eq "latest") { + $ToolVersion = "" + } + $ToolDirectories = (Get-ChildItem -Path "$ArcadeToolsDirectory" -Filter "$ToolName-$ToolVersion*" | Sort-Object -Descending) + if ($ToolDirectories -eq $null) { + Write-Error "Unable to find directory for $ToolName $ToolVersion; please make sure the tool is installed on this image." + exit 1 + } + $ToolDirectory = $ToolDirectories[0] + $BinPathFile = "$($ToolDirectory.FullName)\binpath.txt" + if (-not (Test-Path -Path "$BinPathFile")) { + Write-Error "Unable to find binpath.txt in '$($ToolDirectory.FullName)' ($ToolName $ToolVersion); artifact is either installed incorrectly or is not a bootstrappable tool." + exit 1 + } + $BinPath = Get-Content "$BinPathFile" + $ToolPath = Convert-Path -Path $BinPath + Write-Host "Adding $ToolName to the path ($ToolPath)..." + Write-Host "##vso[task.prependpath]$ToolPath" + $env:PATH = "$ToolPath;$env:PATH" + $InstalledTools += @{ $ToolName = $ToolDirectory.FullName } + } + } + return $InstalledTools + } else { + $NativeTools.PSObject.Properties | ForEach-Object { + $ToolName = $_.Name + $ToolVersion = $_.Value + + if ((Get-Command "$ToolName" -ErrorAction SilentlyContinue) -eq $null) { + Write-PipelineTelemetryError -Category 'NativeToolsBootstrap' -Message "$ToolName not found on path. Please install $ToolName $ToolVersion before proceeding." + Write-PipelineTelemetryError -Category 'NativeToolsBootstrap' -Message "If this is running on a build machine, the arcade-tools directory was not found, which means there's an error with the image." + } + } + exit 0 + } + } else { + $NativeTools.PSObject.Properties | ForEach-Object { + $ToolName = $_.Name + $ToolVersion = $_.Value + $LocalInstallerArguments = @{ ToolName = "$ToolName" } + $LocalInstallerArguments += @{ InstallPath = "$InstallBin" } + $LocalInstallerArguments += @{ BaseUri = "$BaseUri" } + $LocalInstallerArguments += @{ CommonLibraryDirectory = "$EngCommonBaseDir" } + $LocalInstallerArguments += @{ Version = "$ToolVersion" } + + if ($Verbose) { + $LocalInstallerArguments += @{ Verbose = $True } + } + if (Get-Variable 'Force' -ErrorAction 'SilentlyContinue') { + if($Force) { + $LocalInstallerArguments += @{ Force = $True } + } + } + if ($Clean) { + $LocalInstallerArguments += @{ Clean = $True } + } + + Write-Verbose "Installing $ToolName version $ToolVersion" + Write-Verbose "Executing '$InstallerPath $($LocalInstallerArguments.Keys.ForEach({"-$_ '$($LocalInstallerArguments.$_)'"}) -join ' ')'" + & $InstallerPath @LocalInstallerArguments + if ($LASTEXITCODE -Ne "0") { + $errMsg = "$ToolName installation failed" + if ((Get-Variable 'DoNotAbortNativeToolsInstallationOnFailure' -ErrorAction 'SilentlyContinue') -and $DoNotAbortNativeToolsInstallationOnFailure) { + $showNativeToolsWarning = $true + if ((Get-Variable 'DoNotDisplayNativeToolsInstallationWarnings' -ErrorAction 'SilentlyContinue') -and $DoNotDisplayNativeToolsInstallationWarnings) { + $showNativeToolsWarning = $false + } + if ($showNativeToolsWarning) { + Write-Warning $errMsg + } + $toolInstallationFailure = $true + } else { + # We cannot change this to Write-PipelineTelemetryError because of https://github.com/dotnet/arcade/issues/4482 + Write-Host $errMsg + exit 1 + } + } + } + + if ((Get-Variable 'toolInstallationFailure' -ErrorAction 'SilentlyContinue') -and $toolInstallationFailure) { + # We cannot change this to Write-PipelineTelemetryError because of https://github.com/dotnet/arcade/issues/4482 + Write-Host 'Native tools bootstrap failed' + exit 1 + } + } + } + else { + Write-Host 'No native tools defined in global.json' + exit 0 + } + + if ($Clean) { + exit 0 + } + if (Test-Path $InstallBin) { + Write-Host 'Native tools are available from ' (Convert-Path -Path $InstallBin) + Write-Host "##vso[task.prependpath]$(Convert-Path -Path $InstallBin)" + return $InstallBin + } + elseif (-not ($PathPromotion)) { + Write-PipelineTelemetryError -Category 'NativeToolsBootstrap' -Message 'Native tools install directory does not exist, installation failed' + exit 1 + } + exit 0 +} +catch { + Write-Host $_.ScriptStackTrace + Write-PipelineTelemetryError -Category 'NativeToolsBootstrap' -Message $_ + ExitWithExitCode 1 +} diff --git a/src/arcade/eng/common/init-tools-native.sh b/src/arcade/eng/common/init-tools-native.sh new file mode 100644 index 000000000000..3e6a8d6acf2f --- /dev/null +++ b/src/arcade/eng/common/init-tools-native.sh @@ -0,0 +1,238 @@ +#!/usr/bin/env bash + +source="${BASH_SOURCE[0]}" +scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" + +base_uri='https://netcorenativeassets.blob.core.windows.net/resource-packages/external' +install_directory='' +clean=false +force=false +download_retries=5 +retry_wait_time_seconds=30 +global_json_file="$(dirname "$(dirname "${scriptroot}")")/global.json" +declare -a native_assets + +. $scriptroot/pipeline-logging-functions.sh +. $scriptroot/native/common-library.sh + +while (($# > 0)); do + lowerI="$(echo $1 | tr "[:upper:]" "[:lower:]")" + case $lowerI in + --baseuri) + base_uri=$2 + shift 2 + ;; + --installdirectory) + install_directory=$2 + shift 2 + ;; + --clean) + clean=true + shift 1 + ;; + --force) + force=true + shift 1 + ;; + --donotabortonfailure) + donotabortonfailure=true + shift 1 + ;; + --donotdisplaywarnings) + donotdisplaywarnings=true + shift 1 + ;; + --downloadretries) + download_retries=$2 + shift 2 + ;; + --retrywaittimeseconds) + retry_wait_time_seconds=$2 + shift 2 + ;; + --help) + echo "Common settings:" + echo " --installdirectory Directory to install native toolset." + echo " This is a command-line override for the default" + echo " Install directory precedence order:" + echo " - InstallDirectory command-line override" + echo " - NETCOREENG_INSTALL_DIRECTORY environment variable" + echo " - (default) %USERPROFILE%/.netcoreeng/native" + echo "" + echo " --clean Switch specifying not to install anything, but cleanup native asset folders" + echo " --donotabortonfailure Switch specifiying whether to abort native tools installation on failure" + echo " --donotdisplaywarnings Switch specifiying whether to display warnings during native tools installation on failure" + echo " --force Clean and then install tools" + echo " --help Print help and exit" + echo "" + echo "Advanced settings:" + echo " --baseuri Base URI for where to download native tools from" + echo " --downloadretries Number of times a download should be attempted" + echo " --retrywaittimeseconds Wait time between download attempts" + echo "" + exit 0 + ;; + esac +done + +function ReadGlobalJsonNativeTools { + # happy path: we have a proper JSON parsing tool `jq(1)` in PATH! + if command -v jq &> /dev/null; then + + # jq: read each key/value pair under "native-tools" entry and emit: + # KEY="" VALUE="" + # followed by a null byte. + # + # bash: read line with null byte delimeter and push to array (for later `eval`uation). + + while IFS= read -rd '' line; do + native_assets+=("$line") + done < <(jq -r '. | + select(has("native-tools")) | + ."native-tools" | + keys[] as $k | + @sh "KEY=\($k) VALUE=\(.[$k])\u0000"' "$global_json_file") + + return + fi + + # Warning: falling back to manually parsing JSON, which is not recommended. + + # Following routine matches the output and escaping logic of jq(1)'s @sh formatter used above. + # It has been tested with several weird strings with escaped characters in entries (key and value) + # and results were compared with the output of jq(1) in binary representation using xxd(1); + # just before the assignment to 'native_assets' array (above and below). + + # try to capture the section under "native-tools". + if [[ ! "$(cat "$global_json_file")" =~ \"native-tools\"[[:space:]\:\{]*([^\}]+) ]]; then + return + fi + + section="${BASH_REMATCH[1]}" + + parseStarted=0 + possibleEnd=0 + escaping=0 + escaped=0 + isKey=1 + + for (( i=0; i<${#section}; i++ )); do + char="${section:$i:1}" + if ! ((parseStarted)) && [[ "$char" =~ [[:space:],:] ]]; then continue; fi + + if ! ((escaping)) && [[ "$char" == "\\" ]]; then + escaping=1 + elif ((escaping)) && ! ((escaped)); then + escaped=1 + fi + + if ! ((parseStarted)) && [[ "$char" == "\"" ]]; then + parseStarted=1 + possibleEnd=0 + elif [[ "$char" == "'" ]]; then + token="$token'\\\''" + possibleEnd=0 + elif ((escaping)) || [[ "$char" != "\"" ]]; then + token="$token$char" + possibleEnd=1 + fi + + if ((possibleEnd)) && ! ((escaping)) && [[ "$char" == "\"" ]]; then + # Use printf to unescape token to match jq(1)'s @sh formatting rules. + # do not use 'token="$(printf "$token")"' syntax, as $() eats the trailing linefeed. + printf -v token "'$token'" + + if ((isKey)); then + KEY="$token" + isKey=0 + else + line="KEY=$KEY VALUE=$token" + native_assets+=("$line") + isKey=1 + fi + + # reset for next token + parseStarted=0 + token= + elif ((escaping)) && ((escaped)); then + escaping=0 + escaped=0 + fi + done +} + +native_base_dir=$install_directory +if [[ -z $install_directory ]]; then + native_base_dir=$(GetNativeInstallDirectory) +fi + +install_bin="${native_base_dir}/bin" +installed_any=false + +ReadGlobalJsonNativeTools + +if [[ ${#native_assets[@]} -eq 0 ]]; then + echo "No native tools defined in global.json" + exit 0; +else + native_installer_dir="$scriptroot/native" + for index in "${!native_assets[@]}"; do + eval "${native_assets["$index"]}" + + installer_path="$native_installer_dir/install-$KEY.sh" + installer_command="$installer_path" + installer_command+=" --baseuri $base_uri" + installer_command+=" --installpath $install_bin" + installer_command+=" --version $VALUE" + echo $installer_command + + if [[ $force = true ]]; then + installer_command+=" --force" + fi + + if [[ $clean = true ]]; then + installer_command+=" --clean" + fi + + if [[ -a $installer_path ]]; then + $installer_command + if [[ $? != 0 ]]; then + if [[ $donotabortonfailure = true ]]; then + if [[ $donotdisplaywarnings != true ]]; then + Write-PipelineTelemetryError -category 'NativeToolsBootstrap' "Execution Failed" + fi + else + Write-PipelineTelemetryError -category 'NativeToolsBootstrap' "Execution Failed" + exit 1 + fi + else + $installed_any = true + fi + else + if [[ $donotabortonfailure == true ]]; then + if [[ $donotdisplaywarnings != true ]]; then + Write-PipelineTelemetryError -category 'NativeToolsBootstrap' "Execution Failed: no install script" + fi + else + Write-PipelineTelemetryError -category 'NativeToolsBootstrap' "Execution Failed: no install script" + exit 1 + fi + fi + done +fi + +if [[ $clean = true ]]; then + exit 0 +fi + +if [[ -d $install_bin ]]; then + echo "Native tools are available from $install_bin" + echo "##vso[task.prependpath]$install_bin" +else + if [[ $installed_any = true ]]; then + Write-PipelineTelemetryError -category 'NativeToolsBootstrap' "Native tools install directory does not exist, installation failed" + exit 1 + fi +fi + +exit 0 diff --git a/src/arcade/eng/common/internal-feed-operations.ps1 b/src/arcade/eng/common/internal-feed-operations.ps1 new file mode 100644 index 000000000000..92b77347d990 --- /dev/null +++ b/src/arcade/eng/common/internal-feed-operations.ps1 @@ -0,0 +1,132 @@ +param( + [Parameter(Mandatory=$true)][string] $Operation, + [string] $AuthToken, + [string] $CommitSha, + [string] $RepoName, + [switch] $IsFeedPrivate +) + +$ErrorActionPreference = 'Stop' +Set-StrictMode -Version 2.0 +. $PSScriptRoot\tools.ps1 + +# Sets VSS_NUGET_EXTERNAL_FEED_ENDPOINTS based on the "darc-int-*" feeds defined in NuGet.config. This is needed +# in build agents by CredProvider to authenticate the restore requests to internal feeds as specified in +# https://github.com/microsoft/artifacts-credprovider/blob/0f53327cd12fd893d8627d7b08a2171bf5852a41/README.md#environment-variables. This should ONLY be called from identified +# internal builds +function SetupCredProvider { + param( + [string] $AuthToken + ) + + # Install the Cred Provider NuGet plugin + Write-Host 'Setting up Cred Provider NuGet plugin in the agent...' + Write-Host "Getting 'installcredprovider.ps1' from 'https://github.com/microsoft/artifacts-credprovider'..." + + $url = 'https://raw.githubusercontent.com/microsoft/artifacts-credprovider/master/helpers/installcredprovider.ps1' + + Write-Host "Writing the contents of 'installcredprovider.ps1' locally..." + Invoke-WebRequest $url -OutFile installcredprovider.ps1 + + Write-Host 'Installing plugin...' + .\installcredprovider.ps1 -Force + + Write-Host "Deleting local copy of 'installcredprovider.ps1'..." + Remove-Item .\installcredprovider.ps1 + + if (-Not("$env:USERPROFILE\.nuget\plugins\netcore")) { + Write-PipelineTelemetryError -Category 'Arcade' -Message 'CredProvider plugin was not installed correctly!' + ExitWithExitCode 1 + } + else { + Write-Host 'CredProvider plugin was installed correctly!' + } + + # Then, we set the 'VSS_NUGET_EXTERNAL_FEED_ENDPOINTS' environment variable to restore from the stable + # feeds successfully + + $nugetConfigPath = Join-Path $RepoRoot "NuGet.config" + + if (-Not (Test-Path -Path $nugetConfigPath)) { + Write-PipelineTelemetryError -Category 'Build' -Message 'NuGet.config file not found in repo root!' + ExitWithExitCode 1 + } + + $endpoints = New-Object System.Collections.ArrayList + $nugetConfigPackageSources = Select-Xml -Path $nugetConfigPath -XPath "//packageSources/add[contains(@key, 'darc-int-')]/@value" | foreach{$_.Node.Value} + + if (($nugetConfigPackageSources | Measure-Object).Count -gt 0 ) { + foreach ($stableRestoreResource in $nugetConfigPackageSources) { + $trimmedResource = ([string]$stableRestoreResource).Trim() + [void]$endpoints.Add(@{endpoint="$trimmedResource"; password="$AuthToken"}) + } + } + + if (($endpoints | Measure-Object).Count -gt 0) { + $endpointCredentials = @{endpointCredentials=$endpoints} | ConvertTo-Json -Compress + + # Create the environment variables the AzDo way + Write-LoggingCommand -Area 'task' -Event 'setvariable' -Data $endpointCredentials -Properties @{ + 'variable' = 'VSS_NUGET_EXTERNAL_FEED_ENDPOINTS' + 'issecret' = 'false' + } + + # We don't want sessions cached since we will be updating the endpoints quite frequently + Write-LoggingCommand -Area 'task' -Event 'setvariable' -Data 'False' -Properties @{ + 'variable' = 'NUGET_CREDENTIALPROVIDER_SESSIONTOKENCACHE_ENABLED' + 'issecret' = 'false' + } + } + else + { + Write-Host 'No internal endpoints found in NuGet.config' + } +} + +#Workaround for https://github.com/microsoft/msbuild/issues/4430 +function InstallDotNetSdkAndRestoreArcade { + $dotnetTempDir = Join-Path $RepoRoot "dotnet" + $dotnetSdkVersion="2.1.507" # After experimentation we know this version works when restoring the SDK (compared to 3.0.*) + $dotnet = "$dotnetTempDir\dotnet.exe" + $restoreProjPath = "$PSScriptRoot\restore.proj" + + Write-Host "Installing dotnet SDK version $dotnetSdkVersion to restore Arcade SDK..." + InstallDotNetSdk "$dotnetTempDir" "$dotnetSdkVersion" + + '' | Out-File "$restoreProjPath" + + & $dotnet restore $restoreProjPath + + Write-Host 'Arcade SDK restored!' + + if (Test-Path -Path $restoreProjPath) { + Remove-Item $restoreProjPath + } + + if (Test-Path -Path $dotnetTempDir) { + Remove-Item $dotnetTempDir -Recurse + } +} + +try { + Push-Location $PSScriptRoot + + if ($Operation -like 'setup') { + SetupCredProvider $AuthToken + } + elseif ($Operation -like 'install-restore') { + InstallDotNetSdkAndRestoreArcade + } + else { + Write-PipelineTelemetryError -Category 'Arcade' -Message "Unknown operation '$Operation'!" + ExitWithExitCode 1 + } +} +catch { + Write-Host $_.ScriptStackTrace + Write-PipelineTelemetryError -Category 'Arcade' -Message $_ + ExitWithExitCode 1 +} +finally { + Pop-Location +} diff --git a/src/arcade/eng/common/internal-feed-operations.sh b/src/arcade/eng/common/internal-feed-operations.sh new file mode 100644 index 000000000000..9378223ba095 --- /dev/null +++ b/src/arcade/eng/common/internal-feed-operations.sh @@ -0,0 +1,141 @@ +#!/usr/bin/env bash + +set -e + +# Sets VSS_NUGET_EXTERNAL_FEED_ENDPOINTS based on the "darc-int-*" feeds defined in NuGet.config. This is needed +# in build agents by CredProvider to authenticate the restore requests to internal feeds as specified in +# https://github.com/microsoft/artifacts-credprovider/blob/0f53327cd12fd893d8627d7b08a2171bf5852a41/README.md#environment-variables. +# This should ONLY be called from identified internal builds +function SetupCredProvider { + local authToken=$1 + + # Install the Cred Provider NuGet plugin + echo "Setting up Cred Provider NuGet plugin in the agent..."... + echo "Getting 'installcredprovider.ps1' from 'https://github.com/microsoft/artifacts-credprovider'..." + + local url="https://raw.githubusercontent.com/microsoft/artifacts-credprovider/master/helpers/installcredprovider.sh" + + echo "Writing the contents of 'installcredprovider.ps1' locally..." + local installcredproviderPath="installcredprovider.sh" + if command -v curl > /dev/null; then + curl $url > "$installcredproviderPath" + else + wget -q -O "$installcredproviderPath" "$url" + fi + + echo "Installing plugin..." + . "$installcredproviderPath" + + echo "Deleting local copy of 'installcredprovider.sh'..." + rm installcredprovider.sh + + if [ ! -d "$HOME/.nuget/plugins" ]; then + Write-PipelineTelemetryError -category 'Build' 'CredProvider plugin was not installed correctly!' + ExitWithExitCode 1 + else + echo "CredProvider plugin was installed correctly!" + fi + + # Then, we set the 'VSS_NUGET_EXTERNAL_FEED_ENDPOINTS' environment variable to restore from the stable + # feeds successfully + + local nugetConfigPath="{$repo_root}NuGet.config" + + if [ ! "$nugetConfigPath" ]; then + Write-PipelineTelemetryError -category 'Build' "NuGet.config file not found in repo's root!" + ExitWithExitCode 1 + fi + + local endpoints='[' + local nugetConfigPackageValues=`cat "$nugetConfigPath" | grep "key=\"darc-int-"` + local pattern="value=\"(.*)\"" + + for value in $nugetConfigPackageValues + do + if [[ $value =~ $pattern ]]; then + local endpoint="${BASH_REMATCH[1]}" + endpoints+="{\"endpoint\": \"$endpoint\", \"password\": \"$authToken\"}," + fi + done + + endpoints=${endpoints%?} + endpoints+=']' + + if [ ${#endpoints} -gt 2 ]; then + local endpointCredentials="{\"endpointCredentials\": "$endpoints"}" + + echo "##vso[task.setvariable variable=VSS_NUGET_EXTERNAL_FEED_ENDPOINTS]$endpointCredentials" + echo "##vso[task.setvariable variable=NUGET_CREDENTIALPROVIDER_SESSIONTOKENCACHE_ENABLED]False" + else + echo "No internal endpoints found in NuGet.config" + fi +} + +# Workaround for https://github.com/microsoft/msbuild/issues/4430 +function InstallDotNetSdkAndRestoreArcade { + local dotnetTempDir="$repo_root/dotnet" + local dotnetSdkVersion="2.1.507" # After experimentation we know this version works when restoring the SDK (compared to 3.0.*) + local restoreProjPath="$repo_root/eng/common/restore.proj" + + echo "Installing dotnet SDK version $dotnetSdkVersion to restore Arcade SDK..." + echo "" > "$restoreProjPath" + + InstallDotNetSdk "$dotnetTempDir" "$dotnetSdkVersion" + + local res=`$dotnetTempDir/dotnet restore $restoreProjPath` + echo "Arcade SDK restored!" + + # Cleanup + if [ "$restoreProjPath" ]; then + rm "$restoreProjPath" + fi + + if [ "$dotnetTempDir" ]; then + rm -r $dotnetTempDir + fi +} + +source="${BASH_SOURCE[0]}" +operation='' +authToken='' +repoName='' + +while [[ $# > 0 ]]; do + opt="$(echo "$1" | tr "[:upper:]" "[:lower:]")" + case "$opt" in + --operation) + operation=$2 + shift + ;; + --authtoken) + authToken=$2 + shift + ;; + *) + echo "Invalid argument: $1" + usage + exit 1 + ;; + esac + + shift +done + +while [[ -h "$source" ]]; do + scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" + source="$(readlink "$source")" + # if $source was a relative symlink, we need to resolve it relative to the path where the + # symlink file was located + [[ $source != /* ]] && source="$scriptroot/$source" +done +scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" + +. "$scriptroot/tools.sh" + +if [ "$operation" = "setup" ]; then + SetupCredProvider $authToken +elif [ "$operation" = "install-restore" ]; then + InstallDotNetSdkAndRestoreArcade +else + echo "Unknown operation '$operation'!" +fi diff --git a/src/arcade/eng/common/internal/Directory.Build.props b/src/arcade/eng/common/internal/Directory.Build.props new file mode 100644 index 000000000000..f1d041c33da5 --- /dev/null +++ b/src/arcade/eng/common/internal/Directory.Build.props @@ -0,0 +1,11 @@ + + + + + false + false + + + + + diff --git a/src/arcade/eng/common/internal/NuGet.config b/src/arcade/eng/common/internal/NuGet.config new file mode 100644 index 000000000000..19d3d311b166 --- /dev/null +++ b/src/arcade/eng/common/internal/NuGet.config @@ -0,0 +1,7 @@ + + + + + + + diff --git a/src/arcade/eng/common/internal/Tools.csproj b/src/arcade/eng/common/internal/Tools.csproj new file mode 100644 index 000000000000..feaa6d20812d --- /dev/null +++ b/src/arcade/eng/common/internal/Tools.csproj @@ -0,0 +1,22 @@ + + + + + net472 + false + false + + + + + + + + + + + + + + + diff --git a/src/arcade/eng/common/loc/P22DotNetHtmlLocalization.lss b/src/arcade/eng/common/loc/P22DotNetHtmlLocalization.lss new file mode 100644 index 000000000000..5d892d619398 --- /dev/null +++ b/src/arcade/eng/common/loc/P22DotNetHtmlLocalization.lss @@ -0,0 +1,29 @@ + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/src/arcade/eng/common/msbuild.ps1 b/src/arcade/eng/common/msbuild.ps1 new file mode 100644 index 000000000000..f041e5ddd958 --- /dev/null +++ b/src/arcade/eng/common/msbuild.ps1 @@ -0,0 +1,28 @@ +[CmdletBinding(PositionalBinding=$false)] +Param( + [string] $verbosity = 'minimal', + [bool] $warnAsError = $true, + [bool] $nodeReuse = $true, + [switch] $ci, + [switch] $prepareMachine, + [switch] $excludePrereleaseVS, + [string] $msbuildEngine = $null, + [Parameter(ValueFromRemainingArguments=$true)][String[]]$extraArgs +) + +. $PSScriptRoot\tools.ps1 + +try { + if ($ci) { + $nodeReuse = $false + } + + MSBuild @extraArgs +} +catch { + Write-Host $_.ScriptStackTrace + Write-PipelineTelemetryError -Category 'Build' -Message $_ + ExitWithExitCode 1 +} + +ExitWithExitCode 0 \ No newline at end of file diff --git a/src/arcade/eng/common/msbuild.sh b/src/arcade/eng/common/msbuild.sh new file mode 100644 index 000000000000..20d3dad54352 --- /dev/null +++ b/src/arcade/eng/common/msbuild.sh @@ -0,0 +1,58 @@ +#!/usr/bin/env bash + +source="${BASH_SOURCE[0]}" + +# resolve $source until the file is no longer a symlink +while [[ -h "$source" ]]; do + scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" + source="$(readlink "$source")" + # if $source was a relative symlink, we need to resolve it relative to the path where the + # symlink file was located + [[ $source != /* ]] && source="$scriptroot/$source" +done +scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" + +verbosity='minimal' +warn_as_error=true +node_reuse=true +prepare_machine=false +extra_args='' + +while (($# > 0)); do + lowerI="$(echo $1 | tr "[:upper:]" "[:lower:]")" + case $lowerI in + --verbosity) + verbosity=$2 + shift 2 + ;; + --warnaserror) + warn_as_error=$2 + shift 2 + ;; + --nodereuse) + node_reuse=$2 + shift 2 + ;; + --ci) + ci=true + shift 1 + ;; + --preparemachine) + prepare_machine=true + shift 1 + ;; + *) + extra_args="$extra_args $1" + shift 1 + ;; + esac +done + +. "$scriptroot/tools.sh" + +if [[ "$ci" == true ]]; then + node_reuse=false +fi + +MSBuild $extra_args +ExitWithExitCode 0 diff --git a/src/arcade/eng/common/native/CommonLibrary.psm1 b/src/arcade/eng/common/native/CommonLibrary.psm1 new file mode 100644 index 000000000000..f71f6af6cdbc --- /dev/null +++ b/src/arcade/eng/common/native/CommonLibrary.psm1 @@ -0,0 +1,401 @@ +<# +.SYNOPSIS +Helper module to install an archive to a directory + +.DESCRIPTION +Helper module to download and extract an archive to a specified directory + +.PARAMETER Uri +Uri of artifact to download + +.PARAMETER InstallDirectory +Directory to extract artifact contents to + +.PARAMETER Force +Force download / extraction if file or contents already exist. Default = False + +.PARAMETER DownloadRetries +Total number of retry attempts. Default = 5 + +.PARAMETER RetryWaitTimeInSeconds +Wait time between retry attempts in seconds. Default = 30 + +.NOTES +Returns False if download or extraction fail, True otherwise +#> +function DownloadAndExtract { + [CmdletBinding(PositionalBinding=$false)] + Param ( + [Parameter(Mandatory=$True)] + [string] $Uri, + [Parameter(Mandatory=$True)] + [string] $InstallDirectory, + [switch] $Force = $False, + [int] $DownloadRetries = 5, + [int] $RetryWaitTimeInSeconds = 30 + ) + # Define verbose switch if undefined + $Verbose = $VerbosePreference -Eq "Continue" + + $TempToolPath = CommonLibrary\Get-TempPathFilename -Path $Uri + + # Download native tool + $DownloadStatus = CommonLibrary\Get-File -Uri $Uri ` + -Path $TempToolPath ` + -DownloadRetries $DownloadRetries ` + -RetryWaitTimeInSeconds $RetryWaitTimeInSeconds ` + -Force:$Force ` + -Verbose:$Verbose + + if ($DownloadStatus -Eq $False) { + Write-Error "Download failed from $Uri" + return $False + } + + # Extract native tool + $UnzipStatus = CommonLibrary\Expand-Zip -ZipPath $TempToolPath ` + -OutputDirectory $InstallDirectory ` + -Force:$Force ` + -Verbose:$Verbose + + if ($UnzipStatus -Eq $False) { + # Retry Download one more time with Force=true + $DownloadRetryStatus = CommonLibrary\Get-File -Uri $Uri ` + -Path $TempToolPath ` + -DownloadRetries 1 ` + -RetryWaitTimeInSeconds $RetryWaitTimeInSeconds ` + -Force:$True ` + -Verbose:$Verbose + + if ($DownloadRetryStatus -Eq $False) { + Write-Error "Last attempt of download failed as well" + return $False + } + + # Retry unzip again one more time with Force=true + $UnzipRetryStatus = CommonLibrary\Expand-Zip -ZipPath $TempToolPath ` + -OutputDirectory $InstallDirectory ` + -Force:$True ` + -Verbose:$Verbose + if ($UnzipRetryStatus -Eq $False) + { + Write-Error "Last attempt of unzip failed as well" + # Clean up partial zips and extracts + if (Test-Path $TempToolPath) { + Remove-Item $TempToolPath -Force + } + if (Test-Path $InstallDirectory) { + Remove-Item $InstallDirectory -Force -Recurse + } + return $False + } + } + + return $True +} + +<# +.SYNOPSIS +Download a file, retry on failure + +.DESCRIPTION +Download specified file and retry if attempt fails + +.PARAMETER Uri +Uri of file to download. If Uri is a local path, the file will be copied instead of downloaded + +.PARAMETER Path +Path to download or copy uri file to + +.PARAMETER Force +Overwrite existing file if present. Default = False + +.PARAMETER DownloadRetries +Total number of retry attempts. Default = 5 + +.PARAMETER RetryWaitTimeInSeconds +Wait time between retry attempts in seconds Default = 30 + +#> +function Get-File { + [CmdletBinding(PositionalBinding=$false)] + Param ( + [Parameter(Mandatory=$True)] + [string] $Uri, + [Parameter(Mandatory=$True)] + [string] $Path, + [int] $DownloadRetries = 5, + [int] $RetryWaitTimeInSeconds = 30, + [switch] $Force = $False + ) + $Attempt = 0 + + if ($Force) { + if (Test-Path $Path) { + Remove-Item $Path -Force + } + } + if (Test-Path $Path) { + Write-Host "File '$Path' already exists, skipping download" + return $True + } + + $DownloadDirectory = Split-Path -ErrorAction Ignore -Path "$Path" -Parent + if (-Not (Test-Path $DownloadDirectory)) { + New-Item -path $DownloadDirectory -force -itemType "Directory" | Out-Null + } + + $TempPath = "$Path.tmp" + if (Test-Path -IsValid -Path $Uri) { + Write-Verbose "'$Uri' is a file path, copying temporarily to '$TempPath'" + Copy-Item -Path $Uri -Destination $TempPath + Write-Verbose "Moving temporary file to '$Path'" + Move-Item -Path $TempPath -Destination $Path + return $? + } + else { + Write-Verbose "Downloading $Uri" + # Don't display the console progress UI - it's a huge perf hit + $ProgressPreference = 'SilentlyContinue' + while($Attempt -Lt $DownloadRetries) + { + try { + Invoke-WebRequest -UseBasicParsing -Uri $Uri -OutFile $TempPath + Write-Verbose "Downloaded to temporary location '$TempPath'" + Move-Item -Path $TempPath -Destination $Path + Write-Verbose "Moved temporary file to '$Path'" + return $True + } + catch { + $Attempt++ + if ($Attempt -Lt $DownloadRetries) { + $AttemptsLeft = $DownloadRetries - $Attempt + Write-Warning "Download failed, $AttemptsLeft attempts remaining, will retry in $RetryWaitTimeInSeconds seconds" + Start-Sleep -Seconds $RetryWaitTimeInSeconds + } + else { + Write-Error $_ + Write-Error $_.Exception + } + } + } + } + + return $False +} + +<# +.SYNOPSIS +Generate a shim for a native tool + +.DESCRIPTION +Creates a wrapper script (shim) that passes arguments forward to native tool assembly + +.PARAMETER ShimName +The name of the shim + +.PARAMETER ShimDirectory +The directory where shims are stored + +.PARAMETER ToolFilePath +Path to file that shim forwards to + +.PARAMETER Force +Replace shim if already present. Default = False + +.NOTES +Returns $True if generating shim succeeds, $False otherwise +#> +function New-ScriptShim { + [CmdletBinding(PositionalBinding=$false)] + Param ( + [Parameter(Mandatory=$True)] + [string] $ShimName, + [Parameter(Mandatory=$True)] + [string] $ShimDirectory, + [Parameter(Mandatory=$True)] + [string] $ToolFilePath, + [Parameter(Mandatory=$True)] + [string] $BaseUri, + [switch] $Force + ) + try { + Write-Verbose "Generating '$ShimName' shim" + + if (-Not (Test-Path $ToolFilePath)){ + Write-Error "Specified tool file path '$ToolFilePath' does not exist" + return $False + } + + # WinShimmer is a small .NET Framework program that creates .exe shims to bootstrapped programs + # Many of the checks for installed programs expect a .exe extension for Windows tools, rather + # than a .bat or .cmd file. + # Source: https://github.com/dotnet/arcade/tree/master/src/WinShimmer + if (-Not (Test-Path "$ShimDirectory\WinShimmer\winshimmer.exe")) { + $InstallStatus = DownloadAndExtract -Uri "$BaseUri/windows/winshimmer/WinShimmer.zip" ` + -InstallDirectory $ShimDirectory\WinShimmer ` + -Force:$Force ` + -DownloadRetries 2 ` + -RetryWaitTimeInSeconds 5 ` + -Verbose:$Verbose + } + + if ((Test-Path (Join-Path $ShimDirectory "$ShimName.exe"))) { + Write-Host "$ShimName.exe already exists; replacing..." + Remove-Item (Join-Path $ShimDirectory "$ShimName.exe") + } + + & "$ShimDirectory\WinShimmer\winshimmer.exe" $ShimName $ToolFilePath $ShimDirectory + return $True + } + catch { + Write-Host $_ + Write-Host $_.Exception + return $False + } +} + +<# +.SYNOPSIS +Returns the machine architecture of the host machine + +.NOTES +Returns 'x64' on 64 bit machines + Returns 'x86' on 32 bit machines +#> +function Get-MachineArchitecture { + $ProcessorArchitecture = $Env:PROCESSOR_ARCHITECTURE + $ProcessorArchitectureW6432 = $Env:PROCESSOR_ARCHITEW6432 + if($ProcessorArchitecture -Eq "X86") + { + if(($ProcessorArchitectureW6432 -Eq "") -Or + ($ProcessorArchitectureW6432 -Eq "X86")) { + return "x86" + } + $ProcessorArchitecture = $ProcessorArchitectureW6432 + } + if (($ProcessorArchitecture -Eq "AMD64") -Or + ($ProcessorArchitecture -Eq "IA64") -Or + ($ProcessorArchitecture -Eq "ARM64") -Or + ($ProcessorArchitecture -Eq "LOONGARCH64") -Or + ($ProcessorArchitecture -Eq "RISCV64")) { + return "x64" + } + return "x86" +} + +<# +.SYNOPSIS +Get the name of a temporary folder under the native install directory +#> +function Get-TempDirectory { + return Join-Path (Get-NativeInstallDirectory) "temp/" +} + +function Get-TempPathFilename { + [CmdletBinding(PositionalBinding=$false)] + Param ( + [Parameter(Mandatory=$True)] + [string] $Path + ) + $TempDir = CommonLibrary\Get-TempDirectory + $TempFilename = Split-Path $Path -leaf + $TempPath = Join-Path $TempDir $TempFilename + return $TempPath +} + +<# +.SYNOPSIS +Returns the base directory to use for native tool installation + +.NOTES +Returns the value of the NETCOREENG_INSTALL_DIRECTORY if that environment variable +is set, or otherwise returns an install directory under the %USERPROFILE% +#> +function Get-NativeInstallDirectory { + $InstallDir = $Env:NETCOREENG_INSTALL_DIRECTORY + if (!$InstallDir) { + $InstallDir = Join-Path $Env:USERPROFILE ".netcoreeng/native/" + } + return $InstallDir +} + +<# +.SYNOPSIS +Unzip an archive + +.DESCRIPTION +Powershell module to unzip an archive to a specified directory + +.PARAMETER ZipPath (Required) +Path to archive to unzip + +.PARAMETER OutputDirectory (Required) +Output directory for archive contents + +.PARAMETER Force +Overwrite output directory contents if they already exist + +.NOTES +- Returns True and does not perform an extraction if output directory already exists but Overwrite is not True. +- Returns True if unzip operation is successful +- Returns False if Overwrite is True and it is unable to remove contents of OutputDirectory +- Returns False if unable to extract zip archive +#> +function Expand-Zip { + [CmdletBinding(PositionalBinding=$false)] + Param ( + [Parameter(Mandatory=$True)] + [string] $ZipPath, + [Parameter(Mandatory=$True)] + [string] $OutputDirectory, + [switch] $Force + ) + + Write-Verbose "Extracting '$ZipPath' to '$OutputDirectory'" + try { + if ((Test-Path $OutputDirectory) -And (-Not $Force)) { + Write-Host "Directory '$OutputDirectory' already exists, skipping extract" + return $True + } + if (Test-Path $OutputDirectory) { + Write-Verbose "'Force' is 'True', but '$OutputDirectory' exists, removing directory" + Remove-Item $OutputDirectory -Force -Recurse + if ($? -Eq $False) { + Write-Error "Unable to remove '$OutputDirectory'" + return $False + } + } + + $TempOutputDirectory = Join-Path "$(Split-Path -Parent $OutputDirectory)" "$(Split-Path -Leaf $OutputDirectory).tmp" + if (Test-Path $TempOutputDirectory) { + Remove-Item $TempOutputDirectory -Force -Recurse + } + New-Item -Path $TempOutputDirectory -Force -ItemType "Directory" | Out-Null + + Add-Type -assembly "system.io.compression.filesystem" + [io.compression.zipfile]::ExtractToDirectory("$ZipPath", "$TempOutputDirectory") + if ($? -Eq $False) { + Write-Error "Unable to extract '$ZipPath'" + return $False + } + + Move-Item -Path $TempOutputDirectory -Destination $OutputDirectory + } + catch { + Write-Host $_ + Write-Host $_.Exception + + return $False + } + return $True +} + +export-modulemember -function DownloadAndExtract +export-modulemember -function Expand-Zip +export-modulemember -function Get-File +export-modulemember -function Get-MachineArchitecture +export-modulemember -function Get-NativeInstallDirectory +export-modulemember -function Get-TempDirectory +export-modulemember -function Get-TempPathFilename +export-modulemember -function New-ScriptShim diff --git a/src/arcade/eng/common/native/common-library.sh b/src/arcade/eng/common/native/common-library.sh new file mode 100644 index 000000000000..080c2c283ae4 --- /dev/null +++ b/src/arcade/eng/common/native/common-library.sh @@ -0,0 +1,172 @@ +#!/usr/bin/env bash + +function GetNativeInstallDirectory { + local install_dir + + if [[ -z $NETCOREENG_INSTALL_DIRECTORY ]]; then + install_dir=$HOME/.netcoreeng/native/ + else + install_dir=$NETCOREENG_INSTALL_DIRECTORY + fi + + echo $install_dir + return 0 +} + +function GetTempDirectory { + + echo $(GetNativeInstallDirectory)temp/ + return 0 +} + +function ExpandZip { + local zip_path=$1 + local output_directory=$2 + local force=${3:-false} + + echo "Extracting $zip_path to $output_directory" + if [[ -d $output_directory ]] && [[ $force = false ]]; then + echo "Directory '$output_directory' already exists, skipping extract" + return 0 + fi + + if [[ -d $output_directory ]]; then + echo "'Force flag enabled, but '$output_directory' exists. Removing directory" + rm -rf $output_directory + if [[ $? != 0 ]]; then + Write-PipelineTelemetryError -category 'NativeToolsBootstrap' "Unable to remove '$output_directory'" + return 1 + fi + fi + + echo "Creating directory: '$output_directory'" + mkdir -p $output_directory + + echo "Extracting archive" + tar -xf $zip_path -C $output_directory + if [[ $? != 0 ]]; then + Write-PipelineTelemetryError -category 'NativeToolsBootstrap' "Unable to extract '$zip_path'" + return 1 + fi + + return 0 +} + +function GetCurrentOS { + local unameOut="$(uname -s)" + case $unameOut in + Linux*) echo "Linux";; + Darwin*) echo "MacOS";; + esac + return 0 +} + +function GetFile { + local uri=$1 + local path=$2 + local force=${3:-false} + local download_retries=${4:-5} + local retry_wait_time_seconds=${5:-30} + + if [[ -f $path ]]; then + if [[ $force = false ]]; then + echo "File '$path' already exists. Skipping download" + return 0 + else + rm -rf $path + fi + fi + + if [[ -f $uri ]]; then + echo "'$uri' is a file path, copying file to '$path'" + cp $uri $path + return $? + fi + + echo "Downloading $uri" + # Use curl if available, otherwise use wget + if command -v curl > /dev/null; then + curl "$uri" -sSL --retry $download_retries --retry-delay $retry_wait_time_seconds --create-dirs -o "$path" --fail + else + wget -q -O "$path" "$uri" --tries="$download_retries" + fi + + return $? +} + +function GetTempPathFileName { + local path=$1 + + local temp_dir=$(GetTempDirectory) + local temp_file_name=$(basename $path) + echo $temp_dir$temp_file_name + return 0 +} + +function DownloadAndExtract { + local uri=$1 + local installDir=$2 + local force=${3:-false} + local download_retries=${4:-5} + local retry_wait_time_seconds=${5:-30} + + local temp_tool_path=$(GetTempPathFileName $uri) + + echo "downloading to: $temp_tool_path" + + # Download file + GetFile "$uri" "$temp_tool_path" $force $download_retries $retry_wait_time_seconds + if [[ $? != 0 ]]; then + Write-PipelineTelemetryError -category 'NativeToolsBootstrap' "Failed to download '$uri' to '$temp_tool_path'." + return 1 + fi + + # Extract File + echo "extracting from $temp_tool_path to $installDir" + ExpandZip "$temp_tool_path" "$installDir" $force $download_retries $retry_wait_time_seconds + if [[ $? != 0 ]]; then + Write-PipelineTelemetryError -category 'NativeToolsBootstrap' "Failed to extract '$temp_tool_path' to '$installDir'." + return 1 + fi + + return 0 +} + +function NewScriptShim { + local shimpath=$1 + local tool_file_path=$2 + local force=${3:-false} + + echo "Generating '$shimpath' shim" + if [[ -f $shimpath ]]; then + if [[ $force = false ]]; then + echo "File '$shimpath' already exists." >&2 + return 1 + else + rm -rf $shimpath + fi + fi + + if [[ ! -f $tool_file_path ]]; then + # try to see if the path is lower cased + tool_file_path="$(echo $tool_file_path | tr "[:upper:]" "[:lower:]")" + if [[ ! -f $tool_file_path ]]; then + Write-PipelineTelemetryError -category 'NativeToolsBootstrap' "Specified tool file path:'$tool_file_path' does not exist" + return 1 + fi + fi + + local shim_contents=$'#!/usr/bin/env bash\n' + shim_contents+="SHIMARGS="$'$1\n' + shim_contents+="$tool_file_path"$' $SHIMARGS\n' + + # Write shim file + echo "$shim_contents" > $shimpath + + chmod +x $shimpath + + echo "Finished generating shim '$shimpath'" + + return $? +} + diff --git a/src/arcade/eng/common/native/init-compiler.sh b/src/arcade/eng/common/native/init-compiler.sh new file mode 100644 index 000000000000..9a0e1f2b4567 --- /dev/null +++ b/src/arcade/eng/common/native/init-compiler.sh @@ -0,0 +1,146 @@ +#!/bin/sh +# +# This file detects the C/C++ compiler and exports it to the CC/CXX environment variables +# +# NOTE: some scripts source this file and rely on stdout being empty, make sure +# to not output *anything* here, unless it is an error message that fails the +# build. + +if [ -z "$build_arch" ] || [ -z "$compiler" ]; then + echo "Usage..." + echo "build_arch= compiler= init-compiler.sh" + echo "Specify the target architecture." + echo "Specify the name of compiler (clang or gcc)." + exit 1 +fi + +case "$compiler" in + clang*|-clang*|--clang*) + # clangx.y or clang-x.y + version="$(echo "$compiler" | tr -d '[:alpha:]-=')" + majorVersion="${version%%.*}" + + # LLVM based on v18 released in early 2024, with two releases per year + maxVersion="$((18 + ((($(date +%Y) - 2024) * 12 + $(date +%-m) - 3) / 6)))" + compiler=clang + ;; + + gcc*|-gcc*|--gcc*) + # gccx.y or gcc-x.y + version="$(echo "$compiler" | tr -d '[:alpha:]-=')" + majorVersion="${version%%.*}" + + # GCC based on v14 released in early 2024, with one release per year + maxVersion="$((14 + ((($(date +%Y) - 2024) * 12 + $(date +%-m) - 3) / 12)))" + compiler=gcc + ;; +esac + +cxxCompiler="$compiler++" + +# clear the existing CC and CXX from environment +CC= +CXX= +LDFLAGS= + +if [ "$compiler" = "gcc" ]; then cxxCompiler="g++"; fi + +check_version_exists() { + desired_version=-1 + + # Set up the environment to be used for building with the desired compiler. + if command -v "$compiler-$1" > /dev/null; then + desired_version="-$1" + elif command -v "$compiler$1" > /dev/null; then + desired_version="$1" + fi + + echo "$desired_version" +} + +__baseOS="$(uname)" +set_compiler_version_from_CC() { + if [ "$__baseOS" = "Darwin" ]; then + # On Darwin, the versions from -version/-dumpversion refer to Xcode + # versions, not llvm versions, so we can't rely on them. + return + fi + + version="$("$CC" -dumpversion)" + if [ -z "$version" ]; then + echo "Error: $CC -dumpversion didn't provide a version" + exit 1 + fi + + # gcc and clang often display 3 part versions. However, gcc can show only 1 part in some environments. + IFS=. read -r majorVersion _ < /dev/null; then + echo "Error: No compatible version of $compiler was found within the range of $minVersion to $maxVersion. Please upgrade your toolchain or specify the compiler explicitly using CLR_CC and CLR_CXX environment variables." + exit 1 + fi + + CC="$(command -v "$compiler" 2> /dev/null)" + CXX="$(command -v "$cxxCompiler" 2> /dev/null)" + set_compiler_version_from_CC + fi + else + desired_version="$(check_version_exists "$majorVersion")" + if [ "$desired_version" = "-1" ]; then + echo "Error: Could not find specific version of $compiler: $majorVersion." + exit 1 + fi + fi + + if [ -z "$CC" ]; then + CC="$(command -v "$compiler$desired_version" 2> /dev/null)" + CXX="$(command -v "$cxxCompiler$desired_version" 2> /dev/null)" + if [ -z "$CXX" ]; then CXX="$(command -v "$cxxCompiler" 2> /dev/null)"; fi + set_compiler_version_from_CC + fi +else + if [ ! -f "$CLR_CC" ]; then + echo "Error: CLR_CC is set but path '$CLR_CC' does not exist" + exit 1 + fi + CC="$CLR_CC" + CXX="$CLR_CXX" + set_compiler_version_from_CC +fi + +if [ -z "$CC" ]; then + echo "Error: Unable to find $compiler." + exit 1 +fi + +if [ "$__baseOS" != "Darwin" ]; then + # On Darwin, we always want to use the Apple linker. + + # Only lld version >= 9 can be considered stable. lld supports s390x starting from 18.0. + if [ "$compiler" = "clang" ] && [ -n "$majorVersion" ] && [ "$majorVersion" -ge 9 ] && { [ "$build_arch" != "s390x" ] || [ "$majorVersion" -ge 18 ]; }; then + if "$CC" -fuse-ld=lld -Wl,--version >/dev/null 2>&1; then + LDFLAGS="-fuse-ld=lld" + fi + fi +fi + +SCAN_BUILD_COMMAND="$(command -v "scan-build$desired_version" 2> /dev/null)" + +export CC CXX LDFLAGS SCAN_BUILD_COMMAND diff --git a/src/arcade/eng/common/native/init-distro-rid.sh b/src/arcade/eng/common/native/init-distro-rid.sh new file mode 100644 index 000000000000..83ea7aab0e08 --- /dev/null +++ b/src/arcade/eng/common/native/init-distro-rid.sh @@ -0,0 +1,110 @@ +#!/bin/sh + +# getNonPortableDistroRid +# +# Input: +# targetOs: (str) +# targetArch: (str) +# rootfsDir: (str) +# +# Return: +# non-portable rid +getNonPortableDistroRid() +{ + targetOs="$1" + targetArch="$2" + rootfsDir="$3" + nonPortableRid="" + + if [ "$targetOs" = "linux" ]; then + # shellcheck disable=SC1091 + if [ -e "${rootfsDir}/etc/os-release" ]; then + . "${rootfsDir}/etc/os-release" + if echo "${VERSION_ID:-}" | grep -qE '^([[:digit:]]|\.)+$'; then + nonPortableRid="${ID}.${VERSION_ID}-${targetArch}" + else + # Rolling release distros either do not set VERSION_ID, set it as blank or + # set it to non-version looking string (such as TEMPLATE_VERSION_ID on ArchLinux); + # so omit it here to be consistent with everything else. + nonPortableRid="${ID}-${targetArch}" + fi + elif [ -e "${rootfsDir}/android_platform" ]; then + # shellcheck disable=SC1091 + . "${rootfsDir}/android_platform" + nonPortableRid="$RID" + fi + fi + + if [ "$targetOs" = "freebsd" ]; then + # $rootfsDir can be empty. freebsd-version is a shell script and should always work. + __freebsd_major_version=$("$rootfsDir"/bin/freebsd-version | cut -d'.' -f1) + nonPortableRid="freebsd.$__freebsd_major_version-${targetArch}" + elif command -v getprop >/dev/null && getprop ro.product.system.model | grep -qi android; then + __android_sdk_version=$(getprop ro.build.version.sdk) + nonPortableRid="android.$__android_sdk_version-${targetArch}" + elif [ "$targetOs" = "illumos" ]; then + __uname_version=$(uname -v) + nonPortableRid="illumos-${targetArch}" + elif [ "$targetOs" = "solaris" ]; then + __uname_version=$(uname -v) + __solaris_major_version=$(echo "$__uname_version" | cut -d'.' -f1) + nonPortableRid="solaris.$__solaris_major_version-${targetArch}" + elif [ "$targetOs" = "haiku" ]; then + __uname_release="$(uname -r)" + nonPortableRid=haiku.r"$__uname_release"-"$targetArch" + fi + + echo "$nonPortableRid" | tr '[:upper:]' '[:lower:]' +} + +# initDistroRidGlobal +# +# Input: +# os: (str) +# arch: (str) +# rootfsDir?: (nullable:string) +# +# Return: +# None +# +# Notes: +# It is important to note that the function does not return anything, but it +# exports the following variables on success: +# __DistroRid : Non-portable rid of the target platform. +# __PortableTargetOS : OS-part of the portable rid that corresponds to the target platform. +initDistroRidGlobal() +{ + targetOs="$1" + targetArch="$2" + rootfsDir="" + if [ $# -ge 3 ]; then + rootfsDir="$3" + fi + + if [ -n "${rootfsDir}" ]; then + # We may have a cross build. Check for the existence of the rootfsDir + if [ ! -e "${rootfsDir}" ]; then + echo "Error: rootfsDir has been passed, but the location is not valid." + exit 1 + fi + fi + + __DistroRid=$(getNonPortableDistroRid "${targetOs}" "${targetArch}" "${rootfsDir}") + + if [ -z "${__PortableTargetOS:-}" ]; then + __PortableTargetOS="$targetOs" + + STRINGS="$(command -v strings || true)" + if [ -z "$STRINGS" ]; then + STRINGS="$(command -v llvm-strings || true)" + fi + + # Check for musl-based distros (e.g. Alpine Linux, Void Linux). + if "${rootfsDir}/usr/bin/ldd" --version 2>&1 | grep -q musl || + ( [ -n "$STRINGS" ] && "$STRINGS" "${rootfsDir}/usr/bin/ldd" 2>&1 | grep -q musl ); then + __PortableTargetOS="linux-musl" + fi + fi + + export __DistroRid __PortableTargetOS +} diff --git a/src/arcade/eng/common/native/init-os-and-arch.sh b/src/arcade/eng/common/native/init-os-and-arch.sh new file mode 100644 index 000000000000..38921d4338f7 --- /dev/null +++ b/src/arcade/eng/common/native/init-os-and-arch.sh @@ -0,0 +1,85 @@ +#!/bin/sh + +# Use uname to determine what the OS is. +OSName=$(uname -s | tr '[:upper:]' '[:lower:]') + +if command -v getprop && getprop ro.product.system.model 2>&1 | grep -qi android; then + OSName="android" +fi + +case "$OSName" in +freebsd|linux|netbsd|openbsd|sunos|android|haiku) + os="$OSName" ;; +darwin) + os=osx ;; +*) + echo "Unsupported OS $OSName detected!" + exit 1 ;; +esac + +# On Solaris, `uname -m` is discouraged, see https://docs.oracle.com/cd/E36784_01/html/E36870/uname-1.html +# and `uname -p` returns processor type (e.g. i386 on amd64). +# The appropriate tool to determine CPU is isainfo(1) https://docs.oracle.com/cd/E36784_01/html/E36870/isainfo-1.html. +if [ "$os" = "sunos" ]; then + if uname -o 2>&1 | grep -q illumos; then + os="illumos" + else + os="solaris" + fi + CPUName=$(isainfo -n) +else + # For the rest of the operating systems, use uname(1) to determine what the CPU is. + CPUName=$(uname -m) +fi + +case "$CPUName" in + arm64|aarch64) + arch=arm64 + if [ "$(getconf LONG_BIT)" -lt 64 ]; then + # This is 32-bit OS running on 64-bit CPU (for example Raspberry Pi OS) + arch=arm + fi + ;; + + loongarch64) + arch=loongarch64 + ;; + + riscv64) + arch=riscv64 + ;; + + amd64|x86_64) + arch=x64 + ;; + + armv7l|armv8l) + # shellcheck disable=SC1091 + if (NAME=""; . /etc/os-release; test "$NAME" = "Tizen"); then + arch=armel + else + arch=arm + fi + ;; + + armv6l) + arch=armv6 + ;; + + i[3-6]86) + echo "Unsupported CPU $CPUName detected, build might not succeed!" + arch=x86 + ;; + + s390x) + arch=s390x + ;; + + ppc64le) + arch=ppc64le + ;; + *) + echo "Unknown CPU $CPUName detected!" + exit 1 + ;; +esac diff --git a/src/arcade/eng/common/native/install-cmake-test.sh b/src/arcade/eng/common/native/install-cmake-test.sh new file mode 100644 index 000000000000..8a5e7cf0db5a --- /dev/null +++ b/src/arcade/eng/common/native/install-cmake-test.sh @@ -0,0 +1,117 @@ +#!/usr/bin/env bash + +source="${BASH_SOURCE[0]}" +scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" + +. $scriptroot/common-library.sh + +base_uri= +install_path= +version= +clean=false +force=false +download_retries=5 +retry_wait_time_seconds=30 + +while (($# > 0)); do + lowerI="$(echo $1 | tr "[:upper:]" "[:lower:]")" + case $lowerI in + --baseuri) + base_uri=$2 + shift 2 + ;; + --installpath) + install_path=$2 + shift 2 + ;; + --version) + version=$2 + shift 2 + ;; + --clean) + clean=true + shift 1 + ;; + --force) + force=true + shift 1 + ;; + --downloadretries) + download_retries=$2 + shift 2 + ;; + --retrywaittimeseconds) + retry_wait_time_seconds=$2 + shift 2 + ;; + --help) + echo "Common settings:" + echo " --baseuri Base file directory or Url wrom which to acquire tool archives" + echo " --installpath Base directory to install native tool to" + echo " --clean Don't install the tool, just clean up the current install of the tool" + echo " --force Force install of tools even if they previously exist" + echo " --help Print help and exit" + echo "" + echo "Advanced settings:" + echo " --downloadretries Total number of retry attempts" + echo " --retrywaittimeseconds Wait time between retry attempts in seconds" + echo "" + exit 0 + ;; + esac +done + +tool_name="cmake-test" +tool_os=$(GetCurrentOS) +tool_folder="$(echo $tool_os | tr "[:upper:]" "[:lower:]")" +tool_arch="x86_64" +tool_name_moniker="$tool_name-$version-$tool_os-$tool_arch" +tool_install_directory="$install_path/$tool_name/$version" +tool_file_path="$tool_install_directory/$tool_name_moniker/bin/$tool_name" +shim_path="$install_path/$tool_name.sh" +uri="${base_uri}/$tool_folder/$tool_name/$tool_name_moniker.tar.gz" + +# Clean up tool and installers +if [[ $clean = true ]]; then + echo "Cleaning $tool_install_directory" + if [[ -d $tool_install_directory ]]; then + rm -rf $tool_install_directory + fi + + echo "Cleaning $shim_path" + if [[ -f $shim_path ]]; then + rm -rf $shim_path + fi + + tool_temp_path=$(GetTempPathFileName $uri) + echo "Cleaning $tool_temp_path" + if [[ -f $tool_temp_path ]]; then + rm -rf $tool_temp_path + fi + + exit 0 +fi + +# Install tool +if [[ -f $tool_file_path ]] && [[ $force = false ]]; then + echo "$tool_name ($version) already exists, skipping install" + exit 0 +fi + +DownloadAndExtract $uri $tool_install_directory $force $download_retries $retry_wait_time_seconds + +if [[ $? != 0 ]]; then + Write-PipelineTelemetryError -category 'NativeToolsBootstrap' 'Installation failed' + exit 1 +fi + +# Generate Shim +# Always rewrite shims so that we are referencing the expected version +NewScriptShim $shim_path $tool_file_path true + +if [[ $? != 0 ]]; then + Write-PipelineTelemetryError -category 'NativeToolsBootstrap' 'Shim generation failed' + exit 1 +fi + +exit 0 diff --git a/src/arcade/eng/common/native/install-cmake.sh b/src/arcade/eng/common/native/install-cmake.sh new file mode 100644 index 000000000000..de496beebc5a --- /dev/null +++ b/src/arcade/eng/common/native/install-cmake.sh @@ -0,0 +1,117 @@ +#!/usr/bin/env bash + +source="${BASH_SOURCE[0]}" +scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" + +. $scriptroot/common-library.sh + +base_uri= +install_path= +version= +clean=false +force=false +download_retries=5 +retry_wait_time_seconds=30 + +while (($# > 0)); do + lowerI="$(echo $1 | tr "[:upper:]" "[:lower:]")" + case $lowerI in + --baseuri) + base_uri=$2 + shift 2 + ;; + --installpath) + install_path=$2 + shift 2 + ;; + --version) + version=$2 + shift 2 + ;; + --clean) + clean=true + shift 1 + ;; + --force) + force=true + shift 1 + ;; + --downloadretries) + download_retries=$2 + shift 2 + ;; + --retrywaittimeseconds) + retry_wait_time_seconds=$2 + shift 2 + ;; + --help) + echo "Common settings:" + echo " --baseuri Base file directory or Url wrom which to acquire tool archives" + echo " --installpath Base directory to install native tool to" + echo " --clean Don't install the tool, just clean up the current install of the tool" + echo " --force Force install of tools even if they previously exist" + echo " --help Print help and exit" + echo "" + echo "Advanced settings:" + echo " --downloadretries Total number of retry attempts" + echo " --retrywaittimeseconds Wait time between retry attempts in seconds" + echo "" + exit 0 + ;; + esac +done + +tool_name="cmake" +tool_os=$(GetCurrentOS) +tool_folder="$(echo $tool_os | tr "[:upper:]" "[:lower:]")" +tool_arch="x86_64" +tool_name_moniker="$tool_name-$version-$tool_os-$tool_arch" +tool_install_directory="$install_path/$tool_name/$version" +tool_file_path="$tool_install_directory/$tool_name_moniker/bin/$tool_name" +shim_path="$install_path/$tool_name.sh" +uri="${base_uri}/$tool_folder/$tool_name/$tool_name_moniker.tar.gz" + +# Clean up tool and installers +if [[ $clean = true ]]; then + echo "Cleaning $tool_install_directory" + if [[ -d $tool_install_directory ]]; then + rm -rf $tool_install_directory + fi + + echo "Cleaning $shim_path" + if [[ -f $shim_path ]]; then + rm -rf $shim_path + fi + + tool_temp_path=$(GetTempPathFileName $uri) + echo "Cleaning $tool_temp_path" + if [[ -f $tool_temp_path ]]; then + rm -rf $tool_temp_path + fi + + exit 0 +fi + +# Install tool +if [[ -f $tool_file_path ]] && [[ $force = false ]]; then + echo "$tool_name ($version) already exists, skipping install" + exit 0 +fi + +DownloadAndExtract $uri $tool_install_directory $force $download_retries $retry_wait_time_seconds + +if [[ $? != 0 ]]; then + Write-PipelineTelemetryError -category 'NativeToolsBootstrap' 'Installation failed' + exit 1 +fi + +# Generate Shim +# Always rewrite shims so that we are referencing the expected version +NewScriptShim $shim_path $tool_file_path true + +if [[ $? != 0 ]]; then + Write-PipelineTelemetryError -category 'NativeToolsBootstrap' 'Shim generation failed' + exit 1 +fi + +exit 0 diff --git a/src/arcade/eng/common/native/install-tool.ps1 b/src/arcade/eng/common/native/install-tool.ps1 new file mode 100644 index 000000000000..78f2d84a4e4b --- /dev/null +++ b/src/arcade/eng/common/native/install-tool.ps1 @@ -0,0 +1,132 @@ +<# +.SYNOPSIS +Install native tool + +.DESCRIPTION +Install cmake native tool from Azure blob storage + +.PARAMETER InstallPath +Base directory to install native tool to + +.PARAMETER BaseUri +Base file directory or Url from which to acquire tool archives + +.PARAMETER CommonLibraryDirectory +Path to folder containing common library modules + +.PARAMETER Force +Force install of tools even if they previously exist + +.PARAMETER Clean +Don't install the tool, just clean up the current install of the tool + +.PARAMETER DownloadRetries +Total number of retry attempts + +.PARAMETER RetryWaitTimeInSeconds +Wait time between retry attempts in seconds + +.NOTES +Returns 0 if install succeeds, 1 otherwise +#> +[CmdletBinding(PositionalBinding=$false)] +Param ( + [Parameter(Mandatory=$True)] + [string] $ToolName, + [Parameter(Mandatory=$True)] + [string] $InstallPath, + [Parameter(Mandatory=$True)] + [string] $BaseUri, + [Parameter(Mandatory=$True)] + [string] $Version, + [string] $CommonLibraryDirectory = $PSScriptRoot, + [switch] $Force = $False, + [switch] $Clean = $False, + [int] $DownloadRetries = 5, + [int] $RetryWaitTimeInSeconds = 30 +) + +. $PSScriptRoot\..\pipeline-logging-functions.ps1 + +# Import common library modules +Import-Module -Name (Join-Path $CommonLibraryDirectory "CommonLibrary.psm1") + +try { + # Define verbose switch if undefined + $Verbose = $VerbosePreference -Eq "Continue" + + $Arch = CommonLibrary\Get-MachineArchitecture + $ToolOs = "win64" + if($Arch -Eq "x32") { + $ToolOs = "win32" + } + $ToolNameMoniker = "$ToolName-$Version-$ToolOs-$Arch" + $ToolInstallDirectory = Join-Path $InstallPath "$ToolName\$Version\" + $Uri = "$BaseUri/windows/$ToolName/$ToolNameMoniker.zip" + $ShimPath = Join-Path $InstallPath "$ToolName.exe" + + if ($Clean) { + Write-Host "Cleaning $ToolInstallDirectory" + if (Test-Path $ToolInstallDirectory) { + Remove-Item $ToolInstallDirectory -Force -Recurse + } + Write-Host "Cleaning $ShimPath" + if (Test-Path $ShimPath) { + Remove-Item $ShimPath -Force + } + $ToolTempPath = CommonLibrary\Get-TempPathFilename -Path $Uri + Write-Host "Cleaning $ToolTempPath" + if (Test-Path $ToolTempPath) { + Remove-Item $ToolTempPath -Force + } + exit 0 + } + + # Install tool + if ((Test-Path $ToolInstallDirectory) -And (-Not $Force)) { + Write-Verbose "$ToolName ($Version) already exists, skipping install" + } + else { + $InstallStatus = CommonLibrary\DownloadAndExtract -Uri $Uri ` + -InstallDirectory $ToolInstallDirectory ` + -Force:$Force ` + -DownloadRetries $DownloadRetries ` + -RetryWaitTimeInSeconds $RetryWaitTimeInSeconds ` + -Verbose:$Verbose + + if ($InstallStatus -Eq $False) { + Write-PipelineTelemetryError "Installation failed" -Category "NativeToolsetBootstrapping" + exit 1 + } + } + + $ToolFilePath = Get-ChildItem $ToolInstallDirectory -Recurse -Filter "$ToolName.exe" | % { $_.FullName } + if (@($ToolFilePath).Length -Gt 1) { + Write-Error "There are multiple copies of $ToolName in $($ToolInstallDirectory): `n$(@($ToolFilePath | out-string))" + exit 1 + } elseif (@($ToolFilePath).Length -Lt 1) { + Write-Host "$ToolName was not found in $ToolInstallDirectory." + exit 1 + } + + # Generate shim + # Always rewrite shims so that we are referencing the expected version + $GenerateShimStatus = CommonLibrary\New-ScriptShim -ShimName $ToolName ` + -ShimDirectory $InstallPath ` + -ToolFilePath "$ToolFilePath" ` + -BaseUri $BaseUri ` + -Force:$Force ` + -Verbose:$Verbose + + if ($GenerateShimStatus -Eq $False) { + Write-PipelineTelemetryError "Generate shim failed" -Category "NativeToolsetBootstrapping" + return 1 + } + + exit 0 +} +catch { + Write-Host $_.ScriptStackTrace + Write-PipelineTelemetryError -Category "NativeToolsetBootstrapping" -Message $_ + exit 1 +} diff --git a/src/arcade/eng/common/pipeline-logging-functions.ps1 b/src/arcade/eng/common/pipeline-logging-functions.ps1 new file mode 100644 index 000000000000..8e422c561e4b --- /dev/null +++ b/src/arcade/eng/common/pipeline-logging-functions.ps1 @@ -0,0 +1,260 @@ +# Source for this file was taken from https://github.com/microsoft/azure-pipelines-task-lib/blob/11c9439d4af17e6475d9fe058e6b2e03914d17e6/powershell/VstsTaskSdk/LoggingCommandFunctions.ps1 and modified. + +# NOTE: You should not be calling these method directly as they are likely to change. Instead you should be calling the Write-Pipeline* functions defined in tools.ps1 + +$script:loggingCommandPrefix = '##vso[' +$script:loggingCommandEscapeMappings = @( # TODO: WHAT ABOUT "="? WHAT ABOUT "%"? + New-Object psobject -Property @{ Token = ';' ; Replacement = '%3B' } + New-Object psobject -Property @{ Token = "`r" ; Replacement = '%0D' } + New-Object psobject -Property @{ Token = "`n" ; Replacement = '%0A' } + New-Object psobject -Property @{ Token = "]" ; Replacement = '%5D' } +) +# TODO: BUG: Escape % ??? +# TODO: Add test to verify don't need to escape "=". + +# Specify "-Force" to force pipeline formatted output even if "$ci" is false or not set +function Write-PipelineTelemetryError { + [CmdletBinding()] + param( + [Parameter(Mandatory = $true)] + [string]$Category, + [Parameter(Mandatory = $true)] + [string]$Message, + [Parameter(Mandatory = $false)] + [string]$Type = 'error', + [string]$ErrCode, + [string]$SourcePath, + [string]$LineNumber, + [string]$ColumnNumber, + [switch]$AsOutput, + [switch]$Force) + + $PSBoundParameters.Remove('Category') | Out-Null + + if ($Force -Or ((Test-Path variable:ci) -And $ci)) { + $Message = "(NETCORE_ENGINEERING_TELEMETRY=$Category) $Message" + } + $PSBoundParameters.Remove('Message') | Out-Null + $PSBoundParameters.Add('Message', $Message) + Write-PipelineTaskError @PSBoundParameters +} + +# Specify "-Force" to force pipeline formatted output even if "$ci" is false or not set +function Write-PipelineTaskError { + [CmdletBinding()] + param( + [Parameter(Mandatory = $true)] + [string]$Message, + [Parameter(Mandatory = $false)] + [string]$Type = 'error', + [string]$ErrCode, + [string]$SourcePath, + [string]$LineNumber, + [string]$ColumnNumber, + [switch]$AsOutput, + [switch]$Force + ) + + if (!$Force -And (-Not (Test-Path variable:ci) -Or !$ci)) { + if ($Type -eq 'error') { + Write-Host $Message -ForegroundColor Red + return + } + elseif ($Type -eq 'warning') { + Write-Host $Message -ForegroundColor Yellow + return + } + } + + if (($Type -ne 'error') -and ($Type -ne 'warning')) { + Write-Host $Message + return + } + $PSBoundParameters.Remove('Force') | Out-Null + if (-not $PSBoundParameters.ContainsKey('Type')) { + $PSBoundParameters.Add('Type', 'error') + } + Write-LogIssue @PSBoundParameters +} + +function Write-PipelineSetVariable { + [CmdletBinding()] + param( + [Parameter(Mandatory = $true)] + [string]$Name, + [string]$Value, + [switch]$Secret, + [switch]$AsOutput, + [bool]$IsMultiJobVariable = $true) + + if ((Test-Path variable:ci) -And $ci) { + Write-LoggingCommand -Area 'task' -Event 'setvariable' -Data $Value -Properties @{ + 'variable' = $Name + 'isSecret' = $Secret + 'isOutput' = $IsMultiJobVariable + } -AsOutput:$AsOutput + } +} + +function Write-PipelinePrependPath { + [CmdletBinding()] + param( + [Parameter(Mandatory = $true)] + [string]$Path, + [switch]$AsOutput) + + if ((Test-Path variable:ci) -And $ci) { + Write-LoggingCommand -Area 'task' -Event 'prependpath' -Data $Path -AsOutput:$AsOutput + } +} + +function Write-PipelineSetResult { + [CmdletBinding()] + param( + [ValidateSet("Succeeded", "SucceededWithIssues", "Failed", "Cancelled", "Skipped")] + [Parameter(Mandatory = $true)] + [string]$Result, + [string]$Message) + if ((Test-Path variable:ci) -And $ci) { + Write-LoggingCommand -Area 'task' -Event 'complete' -Data $Message -Properties @{ + 'result' = $Result + } + } +} + +<######################################## +# Private functions. +########################################> +function Format-LoggingCommandData { + [CmdletBinding()] + param([string]$Value, [switch]$Reverse) + + if (!$Value) { + return '' + } + + if (!$Reverse) { + foreach ($mapping in $script:loggingCommandEscapeMappings) { + $Value = $Value.Replace($mapping.Token, $mapping.Replacement) + } + } + else { + for ($i = $script:loggingCommandEscapeMappings.Length - 1 ; $i -ge 0 ; $i--) { + $mapping = $script:loggingCommandEscapeMappings[$i] + $Value = $Value.Replace($mapping.Replacement, $mapping.Token) + } + } + + return $Value +} + +function Format-LoggingCommand { + [CmdletBinding()] + param( + [Parameter(Mandatory = $true)] + [string]$Area, + [Parameter(Mandatory = $true)] + [string]$Event, + [string]$Data, + [hashtable]$Properties) + + # Append the preamble. + [System.Text.StringBuilder]$sb = New-Object -TypeName System.Text.StringBuilder + $null = $sb.Append($script:loggingCommandPrefix).Append($Area).Append('.').Append($Event) + + # Append the properties. + if ($Properties) { + $first = $true + foreach ($key in $Properties.Keys) { + [string]$value = Format-LoggingCommandData $Properties[$key] + if ($value) { + if ($first) { + $null = $sb.Append(' ') + $first = $false + } + else { + $null = $sb.Append(';') + } + + $null = $sb.Append("$key=$value") + } + } + } + + # Append the tail and output the value. + $Data = Format-LoggingCommandData $Data + $sb.Append(']').Append($Data).ToString() +} + +function Write-LoggingCommand { + [CmdletBinding(DefaultParameterSetName = 'Parameters')] + param( + [Parameter(Mandatory = $true, ParameterSetName = 'Parameters')] + [string]$Area, + [Parameter(Mandatory = $true, ParameterSetName = 'Parameters')] + [string]$Event, + [Parameter(ParameterSetName = 'Parameters')] + [string]$Data, + [Parameter(ParameterSetName = 'Parameters')] + [hashtable]$Properties, + [Parameter(Mandatory = $true, ParameterSetName = 'Object')] + $Command, + [switch]$AsOutput) + + if ($PSCmdlet.ParameterSetName -eq 'Object') { + Write-LoggingCommand -Area $Command.Area -Event $Command.Event -Data $Command.Data -Properties $Command.Properties -AsOutput:$AsOutput + return + } + + $command = Format-LoggingCommand -Area $Area -Event $Event -Data $Data -Properties $Properties + if ($AsOutput) { + $command + } + else { + Write-Host $command + } +} + +function Write-LogIssue { + [CmdletBinding()] + param( + [ValidateSet('warning', 'error')] + [Parameter(Mandatory = $true)] + [string]$Type, + [string]$Message, + [string]$ErrCode, + [string]$SourcePath, + [string]$LineNumber, + [string]$ColumnNumber, + [switch]$AsOutput) + + $command = Format-LoggingCommand -Area 'task' -Event 'logissue' -Data $Message -Properties @{ + 'type' = $Type + 'code' = $ErrCode + 'sourcepath' = $SourcePath + 'linenumber' = $LineNumber + 'columnnumber' = $ColumnNumber + } + if ($AsOutput) { + return $command + } + + if ($Type -eq 'error') { + $foregroundColor = $host.PrivateData.ErrorForegroundColor + $backgroundColor = $host.PrivateData.ErrorBackgroundColor + if ($foregroundColor -isnot [System.ConsoleColor] -or $backgroundColor -isnot [System.ConsoleColor]) { + $foregroundColor = [System.ConsoleColor]::Red + $backgroundColor = [System.ConsoleColor]::Black + } + } + else { + $foregroundColor = $host.PrivateData.WarningForegroundColor + $backgroundColor = $host.PrivateData.WarningBackgroundColor + if ($foregroundColor -isnot [System.ConsoleColor] -or $backgroundColor -isnot [System.ConsoleColor]) { + $foregroundColor = [System.ConsoleColor]::Yellow + $backgroundColor = [System.ConsoleColor]::Black + } + } + + Write-Host $command -ForegroundColor $foregroundColor -BackgroundColor $backgroundColor +} diff --git a/src/arcade/eng/common/pipeline-logging-functions.sh b/src/arcade/eng/common/pipeline-logging-functions.sh new file mode 100644 index 000000000000..6a0b2255e911 --- /dev/null +++ b/src/arcade/eng/common/pipeline-logging-functions.sh @@ -0,0 +1,206 @@ +#!/usr/bin/env bash + +function Write-PipelineTelemetryError { + local telemetry_category='' + local force=false + local function_args=() + local message='' + while [[ $# -gt 0 ]]; do + opt="$(echo "${1/#--/-}" | tr "[:upper:]" "[:lower:]")" + case "$opt" in + -category|-c) + telemetry_category=$2 + shift + ;; + -force|-f) + force=true + ;; + -*) + function_args+=("$1 $2") + shift + ;; + *) + message=$* + ;; + esac + shift + done + + if [[ $force != true ]] && [[ "$ci" != true ]]; then + echo "$message" >&2 + return + fi + + if [[ $force == true ]]; then + function_args+=("-force") + fi + message="(NETCORE_ENGINEERING_TELEMETRY=$telemetry_category) $message" + function_args+=("$message") + Write-PipelineTaskError ${function_args[@]} +} + +function Write-PipelineTaskError { + local message_type="error" + local sourcepath='' + local linenumber='' + local columnnumber='' + local error_code='' + local force=false + + while [[ $# -gt 0 ]]; do + opt="$(echo "${1/#--/-}" | tr "[:upper:]" "[:lower:]")" + case "$opt" in + -type|-t) + message_type=$2 + shift + ;; + -sourcepath|-s) + sourcepath=$2 + shift + ;; + -linenumber|-ln) + linenumber=$2 + shift + ;; + -columnnumber|-cn) + columnnumber=$2 + shift + ;; + -errcode|-e) + error_code=$2 + shift + ;; + -force|-f) + force=true + ;; + *) + break + ;; + esac + + shift + done + + if [[ $force != true ]] && [[ "$ci" != true ]]; then + echo "$@" >&2 + return + fi + + local message="##vso[task.logissue" + + message="$message type=$message_type" + + if [ -n "$sourcepath" ]; then + message="$message;sourcepath=$sourcepath" + fi + + if [ -n "$linenumber" ]; then + message="$message;linenumber=$linenumber" + fi + + if [ -n "$columnnumber" ]; then + message="$message;columnnumber=$columnnumber" + fi + + if [ -n "$error_code" ]; then + message="$message;code=$error_code" + fi + + message="$message]$*" + echo "$message" +} + +function Write-PipelineSetVariable { + if [[ "$ci" != true ]]; then + return + fi + + local name='' + local value='' + local secret=false + local as_output=false + local is_multi_job_variable=true + + while [[ $# -gt 0 ]]; do + opt="$(echo "${1/#--/-}" | tr "[:upper:]" "[:lower:]")" + case "$opt" in + -name|-n) + name=$2 + shift + ;; + -value|-v) + value=$2 + shift + ;; + -secret|-s) + secret=true + ;; + -as_output|-a) + as_output=true + ;; + -is_multi_job_variable|-i) + is_multi_job_variable=$2 + shift + ;; + esac + shift + done + + value=${value/;/%3B} + value=${value/\\r/%0D} + value=${value/\\n/%0A} + value=${value/]/%5D} + + local message="##vso[task.setvariable variable=$name;isSecret=$secret;isOutput=$is_multi_job_variable]$value" + + if [[ "$as_output" == true ]]; then + $message + else + echo "$message" + fi +} + +function Write-PipelinePrependPath { + local prepend_path='' + + while [[ $# -gt 0 ]]; do + opt="$(echo "${1/#--/-}" | tr "[:upper:]" "[:lower:]")" + case "$opt" in + -path|-p) + prepend_path=$2 + shift + ;; + esac + shift + done + + export PATH="$prepend_path:$PATH" + + if [[ "$ci" == true ]]; then + echo "##vso[task.prependpath]$prepend_path" + fi +} + +function Write-PipelineSetResult { + local result='' + local message='' + + while [[ $# -gt 0 ]]; do + opt="$(echo "${1/#--/-}" | tr "[:upper:]" "[:lower:]")" + case "$opt" in + -result|-r) + result=$2 + shift + ;; + -message|-m) + message=$2 + shift + ;; + esac + shift + done + + if [[ "$ci" == true ]]; then + echo "##vso[task.complete result=$result;]$message" + fi +} diff --git a/src/arcade/eng/common/post-build/check-channel-consistency.ps1 b/src/arcade/eng/common/post-build/check-channel-consistency.ps1 new file mode 100644 index 000000000000..61208d2d1351 --- /dev/null +++ b/src/arcade/eng/common/post-build/check-channel-consistency.ps1 @@ -0,0 +1,48 @@ +param( + [Parameter(Mandatory=$true)][string] $PromoteToChannels, # List of channels that the build should be promoted to + [Parameter(Mandatory=$true)][array] $AvailableChannelIds # List of channel IDs available in the YAML implementation +) + +try { + $ErrorActionPreference = 'Stop' + Set-StrictMode -Version 2.0 + + # `tools.ps1` checks $ci to perform some actions. Since the post-build + # scripts don't necessarily execute in the same agent that run the + # build.ps1/sh script this variable isn't automatically set. + $ci = $true + $disableConfigureToolsetImport = $true + . $PSScriptRoot\..\tools.ps1 + + if ($PromoteToChannels -eq "") { + Write-PipelineTaskError -Type 'warning' -Message "This build won't publish assets as it's not configured to any Maestro channel. If that wasn't intended use Darc to configure a default channel using add-default-channel for this branch or to promote it to a channel using add-build-to-channel. See https://github.com/dotnet/arcade/blob/main/Documentation/Darc.md#assigning-an-individual-build-to-a-channel for more info." + ExitWithExitCode 0 + } + + # Check that every channel that Maestro told to promote the build to + # is available in YAML + $PromoteToChannelsIds = $PromoteToChannels -split "\D" | Where-Object { $_ } + + $hasErrors = $false + + foreach ($id in $PromoteToChannelsIds) { + if (($id -ne 0) -and ($id -notin $AvailableChannelIds)) { + Write-PipelineTaskError -Message "Channel $id is not present in the post-build YAML configuration! This is an error scenario. Please contact @dnceng." + $hasErrors = $true + } + } + + # The `Write-PipelineTaskError` doesn't error the script and we might report several errors + # in the previous lines. The check below makes sure that we return an error state from the + # script if we reported any validation error + if ($hasErrors) { + ExitWithExitCode 1 + } + + Write-Host 'done.' +} +catch { + Write-Host $_ + Write-PipelineTelemetryError -Category 'CheckChannelConsistency' -Message "There was an error while trying to check consistency of Maestro default channels for the build and post-build YAML configuration." + ExitWithExitCode 1 +} diff --git a/src/arcade/eng/common/post-build/nuget-validation.ps1 b/src/arcade/eng/common/post-build/nuget-validation.ps1 new file mode 100644 index 000000000000..e5de00c89836 --- /dev/null +++ b/src/arcade/eng/common/post-build/nuget-validation.ps1 @@ -0,0 +1,22 @@ +# This script validates NuGet package metadata information using this +# tool: https://github.com/NuGet/NuGetGallery/tree/jver-verify/src/VerifyMicrosoftPackage + +param( + [Parameter(Mandatory=$true)][string] $PackagesPath # Path to where the packages to be validated are +) + +# `tools.ps1` checks $ci to perform some actions. Since the post-build +# scripts don't necessarily execute in the same agent that run the +# build.ps1/sh script this variable isn't automatically set. +$ci = $true +$disableConfigureToolsetImport = $true +. $PSScriptRoot\..\tools.ps1 + +try { + & $PSScriptRoot\nuget-verification.ps1 ${PackagesPath}\*.nupkg +} +catch { + Write-Host $_.ScriptStackTrace + Write-PipelineTelemetryError -Category 'NuGetValidation' -Message $_ + ExitWithExitCode 1 +} diff --git a/src/arcade/eng/common/post-build/nuget-verification.ps1 b/src/arcade/eng/common/post-build/nuget-verification.ps1 new file mode 100644 index 000000000000..a365194a9389 --- /dev/null +++ b/src/arcade/eng/common/post-build/nuget-verification.ps1 @@ -0,0 +1,121 @@ +<# +.SYNOPSIS + Verifies that Microsoft NuGet packages have proper metadata. +.DESCRIPTION + Downloads a verification tool and runs metadata validation on the provided NuGet packages. This script writes an + error if any of the provided packages fail validation. All arguments provided to this PowerShell script that do not + match PowerShell parameters are passed on to the verification tool downloaded during the execution of this script. +.PARAMETER NuGetExePath + The path to the nuget.exe binary to use. If not provided, nuget.exe will be downloaded into the -DownloadPath + directory. +.PARAMETER PackageSource + The package source to use to download the verification tool. If not provided, nuget.org will be used. +.PARAMETER DownloadPath + The directory path to download the verification tool and nuget.exe to. If not provided, + %TEMP%\NuGet.VerifyNuGetPackage will be used. +.PARAMETER args + Arguments that will be passed to the verification tool. +.EXAMPLE + PS> .\verify.ps1 *.nupkg + Verifies the metadata of all .nupkg files in the currect working directory. +.EXAMPLE + PS> .\verify.ps1 --help + Displays the help text of the downloaded verifiction tool. +.LINK + https://github.com/NuGet/NuGetGallery/blob/master/src/VerifyMicrosoftPackage/README.md +#> + +# This script was copied from https://github.com/NuGet/NuGetGallery/blob/3e25ad135146676bcab0050a516939d9958bfa5d/src/VerifyMicrosoftPackage/verify.ps1 + +[CmdletBinding(PositionalBinding = $false)] +param( + [string]$NuGetExePath, + [string]$PackageSource = "https://api.nuget.org/v3/index.json", + [string]$DownloadPath, + [Parameter(ValueFromRemainingArguments = $true)] + [string[]]$args +) + +# The URL to download nuget.exe. +$nugetExeUrl = "https://dist.nuget.org/win-x86-commandline/v4.9.4/nuget.exe" + +# The package ID of the verification tool. +$packageId = "NuGet.VerifyMicrosoftPackage" + +# The location that nuget.exe and the verification tool will be downloaded to. +if (!$DownloadPath) { + $DownloadPath = (Join-Path $env:TEMP "NuGet.VerifyMicrosoftPackage") +} + +$fence = New-Object -TypeName string -ArgumentList '=', 80 + +# Create the download directory, if it doesn't already exist. +if (!(Test-Path $DownloadPath)) { + New-Item -ItemType Directory $DownloadPath | Out-Null +} +Write-Host "Using download path: $DownloadPath" + +if ($NuGetExePath) { + $nuget = $NuGetExePath +} else { + $downloadedNuGetExe = Join-Path $DownloadPath "nuget.exe" + + # Download nuget.exe, if it doesn't already exist. + if (!(Test-Path $downloadedNuGetExe)) { + Write-Host "Downloading nuget.exe from $nugetExeUrl..." + $ProgressPreference = 'SilentlyContinue' + try { + Invoke-WebRequest $nugetExeUrl -OutFile $downloadedNuGetExe + $ProgressPreference = 'Continue' + } catch { + $ProgressPreference = 'Continue' + Write-Error $_ + Write-Error "nuget.exe failed to download." + exit + } + } + + $nuget = $downloadedNuGetExe +} + +Write-Host "Using nuget.exe path: $nuget" +Write-Host " " + +# Download the latest version of the verification tool. +Write-Host "Downloading the latest version of $packageId from $packageSource..." +Write-Host $fence +& $nuget install $packageId ` + -Prerelease ` + -OutputDirectory $DownloadPath ` + -Source $PackageSource +Write-Host $fence +Write-Host " " + +if ($LASTEXITCODE -ne 0) { + Write-Error "nuget.exe failed to fetch the verify tool." + exit +} + +# Find the most recently downloaded tool +Write-Host "Finding the most recently downloaded verification tool." +$verifyProbePath = Join-Path $DownloadPath "$packageId.*" +$verifyPath = Get-ChildItem -Path $verifyProbePath -Directory ` + | Sort-Object -Property LastWriteTime -Descending ` + | Select-Object -First 1 +$verify = Join-Path $verifyPath "tools\NuGet.VerifyMicrosoftPackage.exe" +Write-Host "Using verification tool: $verify" +Write-Host " " + +# Execute the verification tool. +Write-Host "Executing the verify tool..." +Write-Host $fence +& $verify $args +Write-Host $fence +Write-Host " " + +# Respond to the exit code. +if ($LASTEXITCODE -ne 0) { + Write-Error "The verify tool found some problems." +} else { + Write-Output "The verify tool succeeded." +} diff --git a/src/arcade/eng/common/post-build/redact-logs.ps1 b/src/arcade/eng/common/post-build/redact-logs.ps1 new file mode 100644 index 000000000000..b7fc19591507 --- /dev/null +++ b/src/arcade/eng/common/post-build/redact-logs.ps1 @@ -0,0 +1,89 @@ +[CmdletBinding(PositionalBinding=$False)] +param( + [Parameter(Mandatory=$true, Position=0)][string] $InputPath, + [Parameter(Mandatory=$true)][string] $BinlogToolVersion, + [Parameter(Mandatory=$false)][string] $DotnetPath, + [Parameter(Mandatory=$false)][string] $PackageFeed = 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public/nuget/v3/index.json', + # File with strings to redact - separated by newlines. + # For comments start the line with '# ' - such lines are ignored + [Parameter(Mandatory=$false)][string] $TokensFilePath, + [Parameter(ValueFromRemainingArguments=$true)][String[]]$TokensToRedact +) + +try { + $ErrorActionPreference = 'Stop' + Set-StrictMode -Version 2.0 + + # `tools.ps1` checks $ci to perform some actions. Since the post-build + # scripts don't necessarily execute in the same agent that run the + # build.ps1/sh script this variable isn't automatically set. + $ci = $true + $disableConfigureToolsetImport = $true + . $PSScriptRoot\..\tools.ps1 + + $packageName = 'binlogtool' + + $dotnet = $DotnetPath + + if (!$dotnet) { + $dotnetRoot = InitializeDotNetCli -install:$true + $dotnet = "$dotnetRoot\dotnet.exe" + } + + $toolList = & "$dotnet" tool list -g + + if ($toolList -like "*$packageName*") { + & "$dotnet" tool uninstall $packageName -g + } + + $toolPath = "$PSScriptRoot\..\..\..\.tools" + $verbosity = 'minimal' + + New-Item -ItemType Directory -Force -Path $toolPath + + Push-Location -Path $toolPath + + try { + Write-Host "Installing Binlog redactor CLI..." + Write-Host "'$dotnet' new tool-manifest" + & "$dotnet" new tool-manifest + Write-Host "'$dotnet' tool install $packageName --local --add-source '$PackageFeed' -v $verbosity --version $BinlogToolVersion" + & "$dotnet" tool install $packageName --local --add-source "$PackageFeed" -v $verbosity --version $BinlogToolVersion + + if (Test-Path $TokensFilePath) { + Write-Host "Adding additional sensitive data for redaction from file: " $TokensFilePath + $TokensToRedact += Get-Content -Path $TokensFilePath | Foreach {$_.Trim()} | Where { $_ -notmatch "^# " } + } + + $optionalParams = [System.Collections.ArrayList]::new() + + Foreach ($p in $TokensToRedact) + { + if($p -match '^\$\(.*\)$') + { + Write-Host ("Ignoring token {0} as it is probably unexpanded AzDO variable" -f $p) + } + elseif($p) + { + $optionalParams.Add("-p:" + $p) | Out-Null + } + } + + & $dotnet binlogtool redact --input:$InputPath --recurse --in-place ` + @optionalParams + + if ($LastExitCode -ne 0) { + Write-PipelineTelemetryError -Category 'Redactor' -Type 'warning' -Message "Problems using Redactor tool (exit code: $LastExitCode). But ignoring them now." + } + } + finally { + Pop-Location + } + + Write-Host 'done.' +} +catch { + Write-Host $_ + Write-PipelineTelemetryError -Category 'Redactor' -Message "There was an error while trying to redact logs. Error: $_" + ExitWithExitCode 1 +} diff --git a/src/arcade/eng/common/post-build/sourcelink-validation.ps1 b/src/arcade/eng/common/post-build/sourcelink-validation.ps1 new file mode 100644 index 000000000000..1976ef70fb85 --- /dev/null +++ b/src/arcade/eng/common/post-build/sourcelink-validation.ps1 @@ -0,0 +1,327 @@ +param( + [Parameter(Mandatory=$true)][string] $InputPath, # Full path to directory where Symbols.NuGet packages to be checked are stored + [Parameter(Mandatory=$true)][string] $ExtractPath, # Full path to directory where the packages will be extracted during validation + [Parameter(Mandatory=$false)][string] $GHRepoName, # GitHub name of the repo including the Org. E.g., dotnet/arcade + [Parameter(Mandatory=$false)][string] $GHCommit, # GitHub commit SHA used to build the packages + [Parameter(Mandatory=$true)][string] $SourcelinkCliVersion # Version of SourceLink CLI to use +) + +$ErrorActionPreference = 'Stop' +Set-StrictMode -Version 2.0 + +# `tools.ps1` checks $ci to perform some actions. Since the post-build +# scripts don't necessarily execute in the same agent that run the +# build.ps1/sh script this variable isn't automatically set. +$ci = $true +$disableConfigureToolsetImport = $true +. $PSScriptRoot\..\tools.ps1 + +# Cache/HashMap (File -> Exist flag) used to consult whether a file exist +# in the repository at a specific commit point. This is populated by inserting +# all files present in the repo at a specific commit point. +$global:RepoFiles = @{} + +# Maximum number of jobs to run in parallel +$MaxParallelJobs = 16 + +$MaxRetries = 5 +$RetryWaitTimeInSeconds = 30 + +# Wait time between check for system load +$SecondsBetweenLoadChecks = 10 + +if (!$InputPath -or !(Test-Path $InputPath)){ + Write-Host "No files to validate." + ExitWithExitCode 0 +} + +$ValidatePackage = { + param( + [string] $PackagePath # Full path to a Symbols.NuGet package + ) + + . $using:PSScriptRoot\..\tools.ps1 + + # Ensure input file exist + if (!(Test-Path $PackagePath)) { + Write-Host "Input file does not exist: $PackagePath" + return [pscustomobject]@{ + result = 1 + packagePath = $PackagePath + } + } + + # Extensions for which we'll look for SourceLink information + # For now we'll only care about Portable & Embedded PDBs + $RelevantExtensions = @('.dll', '.exe', '.pdb') + + Write-Host -NoNewLine 'Validating ' ([System.IO.Path]::GetFileName($PackagePath)) '...' + + $PackageId = [System.IO.Path]::GetFileNameWithoutExtension($PackagePath) + $ExtractPath = Join-Path -Path $using:ExtractPath -ChildPath $PackageId + $FailedFiles = 0 + + Add-Type -AssemblyName System.IO.Compression.FileSystem + + [System.IO.Directory]::CreateDirectory($ExtractPath) | Out-Null + + try { + $zip = [System.IO.Compression.ZipFile]::OpenRead($PackagePath) + + $zip.Entries | + Where-Object {$RelevantExtensions -contains [System.IO.Path]::GetExtension($_.Name)} | + ForEach-Object { + $FileName = $_.FullName + $Extension = [System.IO.Path]::GetExtension($_.Name) + $FakeName = -Join((New-Guid), $Extension) + $TargetFile = Join-Path -Path $ExtractPath -ChildPath $FakeName + + # We ignore resource DLLs + if ($FileName.EndsWith('.resources.dll')) { + return [pscustomobject]@{ + result = 0 + packagePath = $PackagePath + } + } + + [System.IO.Compression.ZipFileExtensions]::ExtractToFile($_, $TargetFile, $true) + + $ValidateFile = { + param( + [string] $FullPath, # Full path to the module that has to be checked + [string] $RealPath, + [ref] $FailedFiles + ) + + $sourcelinkExe = "$env:USERPROFILE\.dotnet\tools" + $sourcelinkExe = Resolve-Path "$sourcelinkExe\sourcelink.exe" + $SourceLinkInfos = & $sourcelinkExe print-urls $FullPath | Out-String + + if ($LASTEXITCODE -eq 0 -and -not ([string]::IsNullOrEmpty($SourceLinkInfos))) { + $NumFailedLinks = 0 + + # We only care about Http addresses + $Matches = (Select-String '(http[s]?)(:\/\/)([^\s,]+)' -Input $SourceLinkInfos -AllMatches).Matches + + if ($Matches.Count -ne 0) { + $Matches.Value | + ForEach-Object { + $Link = $_ + $CommitUrl = "https://raw.githubusercontent.com/${using:GHRepoName}/${using:GHCommit}/" + + $FilePath = $Link.Replace($CommitUrl, "") + $Status = 200 + $Cache = $using:RepoFiles + + $attempts = 0 + + while ($attempts -lt $using:MaxRetries) { + if ( !($Cache.ContainsKey($FilePath)) ) { + try { + $Uri = $Link -as [System.URI] + + if ($Link -match "submodules") { + # Skip submodule links until sourcelink properly handles submodules + $Status = 200 + } + elseif ($Uri.AbsoluteURI -ne $null -and ($Uri.Host -match 'github' -or $Uri.Host -match 'githubusercontent')) { + # Only GitHub links are valid + $Status = (Invoke-WebRequest -Uri $Link -UseBasicParsing -Method HEAD -TimeoutSec 5).StatusCode + } + else { + # If it's not a github link, we want to break out of the loop and not retry. + $Status = 0 + $attempts = $using:MaxRetries + } + } + catch { + Write-Host $_ + $Status = 0 + } + } + + if ($Status -ne 200) { + $attempts++ + + if ($attempts -lt $using:MaxRetries) + { + $attemptsLeft = $using:MaxRetries - $attempts + Write-Warning "Download failed, $attemptsLeft attempts remaining, will retry in $using:RetryWaitTimeInSeconds seconds" + Start-Sleep -Seconds $using:RetryWaitTimeInSeconds + } + else { + if ($NumFailedLinks -eq 0) { + if ($FailedFiles.Value -eq 0) { + Write-Host + } + + Write-Host "`tFile $RealPath has broken links:" + } + + Write-Host "`t`tFailed to retrieve $Link" + + $NumFailedLinks++ + } + } + else { + break + } + } + } + } + + if ($NumFailedLinks -ne 0) { + $FailedFiles.value++ + $global:LASTEXITCODE = 1 + } + } + } + + &$ValidateFile $TargetFile $FileName ([ref]$FailedFiles) + } + } + catch { + Write-Host $_ + } + finally { + $zip.Dispose() + } + + if ($FailedFiles -eq 0) { + Write-Host 'Passed.' + return [pscustomobject]@{ + result = 0 + packagePath = $PackagePath + } + } + else { + Write-PipelineTelemetryError -Category 'SourceLink' -Message "$PackagePath has broken SourceLink links." + return [pscustomobject]@{ + result = 1 + packagePath = $PackagePath + } + } +} + +function CheckJobResult( + $result, + $packagePath, + [ref]$ValidationFailures, + [switch]$logErrors) { + if ($result -ne '0') { + if ($logErrors) { + Write-PipelineTelemetryError -Category 'SourceLink' -Message "$packagePath has broken SourceLink links." + } + $ValidationFailures.Value++ + } +} + +function ValidateSourceLinkLinks { + if ($GHRepoName -ne '' -and !($GHRepoName -Match '^[^\s\/]+/[^\s\/]+$')) { + if (!($GHRepoName -Match '^[^\s-]+-[^\s]+$')) { + Write-PipelineTelemetryError -Category 'SourceLink' -Message "GHRepoName should be in the format / or -. '$GHRepoName'" + ExitWithExitCode 1 + } + else { + $GHRepoName = $GHRepoName -replace '^([^\s-]+)-([^\s]+)$', '$1/$2'; + } + } + + if ($GHCommit -ne '' -and !($GHCommit -Match '^[0-9a-fA-F]{40}$')) { + Write-PipelineTelemetryError -Category 'SourceLink' -Message "GHCommit should be a 40 chars hexadecimal string. '$GHCommit'" + ExitWithExitCode 1 + } + + if ($GHRepoName -ne '' -and $GHCommit -ne '') { + $RepoTreeURL = -Join('http://api.github.com/repos/', $GHRepoName, '/git/trees/', $GHCommit, '?recursive=1') + $CodeExtensions = @('.cs', '.vb', '.fs', '.fsi', '.fsx', '.fsscript') + + try { + # Retrieve the list of files in the repo at that particular commit point and store them in the RepoFiles hash + $Data = Invoke-WebRequest $RepoTreeURL -UseBasicParsing | ConvertFrom-Json | Select-Object -ExpandProperty tree + + foreach ($file in $Data) { + $Extension = [System.IO.Path]::GetExtension($file.path) + + if ($CodeExtensions.Contains($Extension)) { + $RepoFiles[$file.path] = 1 + } + } + } + catch { + Write-Host "Problems downloading the list of files from the repo. Url used: $RepoTreeURL . Execution will proceed without caching." + } + } + elseif ($GHRepoName -ne '' -or $GHCommit -ne '') { + Write-Host 'For using the http caching mechanism both GHRepoName and GHCommit should be informed.' + } + + if (Test-Path $ExtractPath) { + Remove-Item $ExtractPath -Force -Recurse -ErrorAction SilentlyContinue + } + + $ValidationFailures = 0 + + # Process each NuGet package in parallel + Get-ChildItem "$InputPath\*.symbols.nupkg" | + ForEach-Object { + Write-Host "Starting $($_.FullName)" + Start-Job -ScriptBlock $ValidatePackage -ArgumentList $_.FullName | Out-Null + $NumJobs = @(Get-Job -State 'Running').Count + + while ($NumJobs -ge $MaxParallelJobs) { + Write-Host "There are $NumJobs validation jobs running right now. Waiting $SecondsBetweenLoadChecks seconds to check again." + sleep $SecondsBetweenLoadChecks + $NumJobs = @(Get-Job -State 'Running').Count + } + + foreach ($Job in @(Get-Job -State 'Completed')) { + $jobResult = Wait-Job -Id $Job.Id | Receive-Job + CheckJobResult $jobResult.result $jobResult.packagePath ([ref]$ValidationFailures) -LogErrors + Remove-Job -Id $Job.Id + } + } + + foreach ($Job in @(Get-Job)) { + $jobResult = Wait-Job -Id $Job.Id | Receive-Job + CheckJobResult $jobResult.result $jobResult.packagePath ([ref]$ValidationFailures) + Remove-Job -Id $Job.Id + } + if ($ValidationFailures -gt 0) { + Write-PipelineTelemetryError -Category 'SourceLink' -Message "$ValidationFailures package(s) failed validation." + ExitWithExitCode 1 + } +} + +function InstallSourcelinkCli { + $sourcelinkCliPackageName = 'sourcelink' + + $dotnetRoot = InitializeDotNetCli -install:$true + $dotnet = "$dotnetRoot\dotnet.exe" + $toolList = & "$dotnet" tool list --global + + if (($toolList -like "*$sourcelinkCliPackageName*") -and ($toolList -like "*$sourcelinkCliVersion*")) { + Write-Host "SourceLink CLI version $sourcelinkCliVersion is already installed." + } + else { + Write-Host "Installing SourceLink CLI version $sourcelinkCliVersion..." + Write-Host 'You may need to restart your command window if this is the first dotnet tool you have installed.' + & "$dotnet" tool install $sourcelinkCliPackageName --version $sourcelinkCliVersion --verbosity "minimal" --global + } +} + +try { + InstallSourcelinkCli + + foreach ($Job in @(Get-Job)) { + Remove-Job -Id $Job.Id + } + + ValidateSourceLinkLinks +} +catch { + Write-Host $_.Exception + Write-Host $_.ScriptStackTrace + Write-PipelineTelemetryError -Category 'SourceLink' -Message $_ + ExitWithExitCode 1 +} diff --git a/src/arcade/eng/common/post-build/symbols-validation.ps1 b/src/arcade/eng/common/post-build/symbols-validation.ps1 new file mode 100644 index 000000000000..7146e593ffae --- /dev/null +++ b/src/arcade/eng/common/post-build/symbols-validation.ps1 @@ -0,0 +1,337 @@ +param( + [Parameter(Mandatory = $true)][string] $InputPath, # Full path to directory where NuGet packages to be checked are stored + [Parameter(Mandatory = $true)][string] $ExtractPath, # Full path to directory where the packages will be extracted during validation + [Parameter(Mandatory = $true)][string] $DotnetSymbolVersion, # Version of dotnet symbol to use + [Parameter(Mandatory = $false)][switch] $CheckForWindowsPdbs, # If we should check for the existence of windows pdbs in addition to portable PDBs + [Parameter(Mandatory = $false)][switch] $ContinueOnError, # If we should keep checking symbols after an error + [Parameter(Mandatory = $false)][switch] $Clean, # Clean extracted symbols directory after checking symbols + [Parameter(Mandatory = $false)][string] $SymbolExclusionFile # Exclude the symbols in the file from publishing to symbol server +) + +. $PSScriptRoot\..\tools.ps1 +# Maximum number of jobs to run in parallel +$MaxParallelJobs = 16 + +# Max number of retries +$MaxRetry = 5 + +# Wait time between check for system load +$SecondsBetweenLoadChecks = 10 + +# Set error codes +Set-Variable -Name "ERROR_BADEXTRACT" -Option Constant -Value -1 +Set-Variable -Name "ERROR_FILEDOESNOTEXIST" -Option Constant -Value -2 + +$WindowsPdbVerificationParam = "" +if ($CheckForWindowsPdbs) { + $WindowsPdbVerificationParam = "--windows-pdbs" +} + +$ExclusionSet = New-Object System.Collections.Generic.HashSet[string]; + +if (!$InputPath -or !(Test-Path $InputPath)){ + Write-Host "No symbols to validate." + ExitWithExitCode 0 +} + +#Check if the path exists +if ($SymbolExclusionFile -and (Test-Path $SymbolExclusionFile)){ + [string[]]$Exclusions = Get-Content "$SymbolExclusionFile" + $Exclusions | foreach { if($_ -and $_.Trim()){$ExclusionSet.Add($_)} } +} +else{ + Write-Host "Symbol Exclusion file does not exists. No symbols to exclude." +} + +$CountMissingSymbols = { + param( + [string] $PackagePath, # Path to a NuGet package + [string] $WindowsPdbVerificationParam # If we should check for the existence of windows pdbs in addition to portable PDBs + ) + + Add-Type -AssemblyName System.IO.Compression.FileSystem + + Write-Host "Validating $PackagePath " + + # Ensure input file exist + if (!(Test-Path $PackagePath)) { + Write-PipelineTaskError "Input file does not exist: $PackagePath" + return [pscustomobject]@{ + result = $using:ERROR_FILEDOESNOTEXIST + packagePath = $PackagePath + } + } + + # Extensions for which we'll look for symbols + $RelevantExtensions = @('.dll', '.exe', '.so', '.dylib') + + # How many files are missing symbol information + $MissingSymbols = 0 + + $PackageId = [System.IO.Path]::GetFileNameWithoutExtension($PackagePath) + $PackageGuid = New-Guid + $ExtractPath = Join-Path -Path $using:ExtractPath -ChildPath $PackageGuid + $SymbolsPath = Join-Path -Path $ExtractPath -ChildPath 'Symbols' + + try { + [System.IO.Compression.ZipFile]::ExtractToDirectory($PackagePath, $ExtractPath) + } + catch { + Write-Host "Something went wrong extracting $PackagePath" + Write-Host $_ + return [pscustomobject]@{ + result = $using:ERROR_BADEXTRACT + packagePath = $PackagePath + } + } + + Get-ChildItem -Recurse $ExtractPath | + Where-Object { $RelevantExtensions -contains $_.Extension } | + ForEach-Object { + $FileName = $_.FullName + if ($FileName -Match '\\ref\\') { + Write-Host "`t Ignoring reference assembly file " $FileName + return + } + + $FirstMatchingSymbolDescriptionOrDefault = { + param( + [string] $FullPath, # Full path to the module that has to be checked + [string] $TargetServerParam, # Parameter to pass to `Symbol Tool` indicating the server to lookup for symbols + [string] $WindowsPdbVerificationParam, # Parameter to pass to potential check for windows-pdbs. + [string] $SymbolsPath + ) + + $FileName = [System.IO.Path]::GetFileName($FullPath) + $Extension = [System.IO.Path]::GetExtension($FullPath) + + # Those below are potential symbol files that the `dotnet symbol` might + # return. Which one will be returned depend on the type of file we are + # checking and which type of file was uploaded. + + # The file itself is returned + $SymbolPath = $SymbolsPath + '\' + $FileName + + # PDB file for the module + $PdbPath = $SymbolPath.Replace($Extension, '.pdb') + + # PDB file for R2R module (created by crossgen) + $NGenPdb = $SymbolPath.Replace($Extension, '.ni.pdb') + + # DBG file for a .so library + $SODbg = $SymbolPath.Replace($Extension, '.so.dbg') + + # DWARF file for a .dylib + $DylibDwarf = $SymbolPath.Replace($Extension, '.dylib.dwarf') + + $dotnetSymbolExe = "$env:USERPROFILE\.dotnet\tools" + $dotnetSymbolExe = Resolve-Path "$dotnetSymbolExe\dotnet-symbol.exe" + + $totalRetries = 0 + + while ($totalRetries -lt $using:MaxRetry) { + + # Save the output and get diagnostic output + $output = & $dotnetSymbolExe --symbols --modules $WindowsPdbVerificationParam $TargetServerParam $FullPath -o $SymbolsPath --diagnostics | Out-String + + if ((Test-Path $PdbPath) -and (Test-path $SymbolPath)) { + return 'Module and PDB for Module' + } + elseif ((Test-Path $NGenPdb) -and (Test-Path $PdbPath) -and (Test-Path $SymbolPath)) { + return 'Dll, PDB and NGen PDB' + } + elseif ((Test-Path $SODbg) -and (Test-Path $SymbolPath)) { + return 'So and DBG for SO' + } + elseif ((Test-Path $DylibDwarf) -and (Test-Path $SymbolPath)) { + return 'Dylib and Dwarf for Dylib' + } + elseif (Test-Path $SymbolPath) { + return 'Module' + } + else + { + $totalRetries++ + } + } + + return $null + } + + $FileRelativePath = $FileName.Replace("$ExtractPath\", "") + if (($($using:ExclusionSet) -ne $null) -and ($($using:ExclusionSet).Contains($FileRelativePath) -or ($($using:ExclusionSet).Contains($FileRelativePath.Replace("\", "/"))))){ + Write-Host "Skipping $FileName from symbol validation" + } + + else { + $FileGuid = New-Guid + $ExpandedSymbolsPath = Join-Path -Path $SymbolsPath -ChildPath $FileGuid + + $SymbolsOnMSDL = & $FirstMatchingSymbolDescriptionOrDefault ` + -FullPath $FileName ` + -TargetServerParam '--microsoft-symbol-server' ` + -SymbolsPath "$ExpandedSymbolsPath-msdl" ` + -WindowsPdbVerificationParam $WindowsPdbVerificationParam + $SymbolsOnSymWeb = & $FirstMatchingSymbolDescriptionOrDefault ` + -FullPath $FileName ` + -TargetServerParam '--internal-server' ` + -SymbolsPath "$ExpandedSymbolsPath-symweb" ` + -WindowsPdbVerificationParam $WindowsPdbVerificationParam + + Write-Host -NoNewLine "`t Checking file " $FileName "... " + + if ($SymbolsOnMSDL -ne $null -and $SymbolsOnSymWeb -ne $null) { + Write-Host "Symbols found on MSDL ($SymbolsOnMSDL) and SymWeb ($SymbolsOnSymWeb)" + } + else { + $MissingSymbols++ + + if ($SymbolsOnMSDL -eq $null -and $SymbolsOnSymWeb -eq $null) { + Write-Host 'No symbols found on MSDL or SymWeb!' + } + else { + if ($SymbolsOnMSDL -eq $null) { + Write-Host 'No symbols found on MSDL!' + } + else { + Write-Host 'No symbols found on SymWeb!' + } + } + } + } + } + + if ($using:Clean) { + Remove-Item $ExtractPath -Recurse -Force + } + + Pop-Location + + return [pscustomobject]@{ + result = $MissingSymbols + packagePath = $PackagePath + } +} + +function CheckJobResult( + $result, + $packagePath, + [ref]$DupedSymbols, + [ref]$TotalFailures) { + if ($result -eq $ERROR_BADEXTRACT) { + Write-PipelineTelemetryError -Category 'CheckSymbols' -Message "$packagePath has duplicated symbol files" + $DupedSymbols.Value++ + } + elseif ($result -eq $ERROR_FILEDOESNOTEXIST) { + Write-PipelineTelemetryError -Category 'CheckSymbols' -Message "$packagePath does not exist" + $TotalFailures.Value++ + } + elseif ($result -gt '0') { + Write-PipelineTelemetryError -Category 'CheckSymbols' -Message "Missing symbols for $result modules in the package $packagePath" + $TotalFailures.Value++ + } + else { + Write-Host "All symbols verified for package $packagePath" + } +} + +function CheckSymbolsAvailable { + if (Test-Path $ExtractPath) { + Remove-Item $ExtractPath -Force -Recurse -ErrorAction SilentlyContinue + } + + $TotalPackages = 0 + $TotalFailures = 0 + $DupedSymbols = 0 + + Get-ChildItem "$InputPath\*.nupkg" | + ForEach-Object { + $FileName = $_.Name + $FullName = $_.FullName + + # These packages from Arcade-Services include some native libraries that + # our current symbol uploader can't handle. Below is a workaround until + # we get issue: https://github.com/dotnet/arcade/issues/2457 sorted. + if ($FileName -Match 'Microsoft\.DotNet\.Darc\.') { + Write-Host "Ignoring Arcade-services file: $FileName" + Write-Host + return + } + elseif ($FileName -Match 'Microsoft\.DotNet\.Maestro\.Tasks\.') { + Write-Host "Ignoring Arcade-services file: $FileName" + Write-Host + return + } + + $TotalPackages++ + + Start-Job -ScriptBlock $CountMissingSymbols -ArgumentList @($FullName,$WindowsPdbVerificationParam) | Out-Null + + $NumJobs = @(Get-Job -State 'Running').Count + + while ($NumJobs -ge $MaxParallelJobs) { + Write-Host "There are $NumJobs validation jobs running right now. Waiting $SecondsBetweenLoadChecks seconds to check again." + sleep $SecondsBetweenLoadChecks + $NumJobs = @(Get-Job -State 'Running').Count + } + + foreach ($Job in @(Get-Job -State 'Completed')) { + $jobResult = Wait-Job -Id $Job.Id | Receive-Job + CheckJobResult $jobResult.result $jobResult.packagePath ([ref]$DupedSymbols) ([ref]$TotalFailures) + Remove-Job -Id $Job.Id + } + Write-Host + } + + foreach ($Job in @(Get-Job)) { + $jobResult = Wait-Job -Id $Job.Id | Receive-Job + CheckJobResult $jobResult.result $jobResult.packagePath ([ref]$DupedSymbols) ([ref]$TotalFailures) + } + + if ($TotalFailures -gt 0 -or $DupedSymbols -gt 0) { + if ($TotalFailures -gt 0) { + Write-PipelineTelemetryError -Category 'CheckSymbols' -Message "Symbols missing for $TotalFailures/$TotalPackages packages" + } + + if ($DupedSymbols -gt 0) { + Write-PipelineTelemetryError -Category 'CheckSymbols' -Message "$DupedSymbols/$TotalPackages packages had duplicated symbol files and could not be extracted" + } + + ExitWithExitCode 1 + } + else { + Write-Host "All symbols validated!" + } +} + +function InstallDotnetSymbol { + $dotnetSymbolPackageName = 'dotnet-symbol' + + $dotnetRoot = InitializeDotNetCli -install:$true + $dotnet = "$dotnetRoot\dotnet.exe" + $toolList = & "$dotnet" tool list --global + + if (($toolList -like "*$dotnetSymbolPackageName*") -and ($toolList -like "*$dotnetSymbolVersion*")) { + Write-Host "dotnet-symbol version $dotnetSymbolVersion is already installed." + } + else { + Write-Host "Installing dotnet-symbol version $dotnetSymbolVersion..." + Write-Host 'You may need to restart your command window if this is the first dotnet tool you have installed.' + & "$dotnet" tool install $dotnetSymbolPackageName --version $dotnetSymbolVersion --verbosity "minimal" --global + } +} + +try { + InstallDotnetSymbol + + foreach ($Job in @(Get-Job)) { + Remove-Job -Id $Job.Id + } + + CheckSymbolsAvailable +} +catch { + Write-Host $_.ScriptStackTrace + Write-PipelineTelemetryError -Category 'CheckSymbols' -Message $_ + ExitWithExitCode 1 +} diff --git a/src/arcade/eng/common/retain-build.ps1 b/src/arcade/eng/common/retain-build.ps1 new file mode 100644 index 000000000000..e7ba975adeb6 --- /dev/null +++ b/src/arcade/eng/common/retain-build.ps1 @@ -0,0 +1,45 @@ + +Param( +[Parameter(Mandatory=$true)][int] $buildId, +[Parameter(Mandatory=$true)][string] $azdoOrgUri, +[Parameter(Mandatory=$true)][string] $azdoProject, +[Parameter(Mandatory=$true)][string] $token +) + +$ErrorActionPreference = 'Stop' +Set-StrictMode -Version 2.0 + +function Get-AzDOHeaders( + [string] $token) +{ + $base64AuthInfo = [Convert]::ToBase64String([Text.Encoding]::ASCII.GetBytes(":${token}")) + $headers = @{"Authorization"="Basic $base64AuthInfo"} + return $headers +} + +function Update-BuildRetention( + [string] $azdoOrgUri, + [string] $azdoProject, + [int] $buildId, + [string] $token) +{ + $headers = Get-AzDOHeaders -token $token + $requestBody = "{ + `"keepForever`": `"true`" + }" + + $requestUri = "${azdoOrgUri}/${azdoProject}/_apis/build/builds/${buildId}?api-version=6.0" + write-Host "Attempting to retain build using the following URI: ${requestUri} ..." + + try { + Invoke-RestMethod -Uri $requestUri -Method Patch -Body $requestBody -Header $headers -contentType "application/json" + Write-Host "Updated retention settings for build ${buildId}." + } + catch { + Write-Error "Failed to update retention settings for build: $_.Exception.Response.StatusDescription" + exit 1 + } +} + +Update-BuildRetention -azdoOrgUri $azdoOrgUri -azdoProject $azdoProject -buildId $buildId -token $token +exit 0 diff --git a/src/arcade/eng/common/sdl/NuGet.config b/src/arcade/eng/common/sdl/NuGet.config new file mode 100644 index 000000000000..3849bdb3cf51 --- /dev/null +++ b/src/arcade/eng/common/sdl/NuGet.config @@ -0,0 +1,18 @@ + + + + + + + + + + + + + + + + + + diff --git a/src/arcade/eng/common/sdl/configure-sdl-tool.ps1 b/src/arcade/eng/common/sdl/configure-sdl-tool.ps1 new file mode 100644 index 000000000000..27f5a4115fc7 --- /dev/null +++ b/src/arcade/eng/common/sdl/configure-sdl-tool.ps1 @@ -0,0 +1,130 @@ +Param( + [string] $GuardianCliLocation, + [string] $WorkingDirectory, + [string] $TargetDirectory, + [string] $GdnFolder, + # The list of Guardian tools to configure. For each object in the array: + # - If the item is a [hashtable], it must contain these entries: + # - Name = The tool name as Guardian knows it. + # - Scenario = (Optional) Scenario-specific name for this configuration entry. It must be unique + # among all tool entries with the same Name. + # - Args = (Optional) Array of Guardian tool configuration args, like '@("Target > C:\temp")' + # - If the item is a [string] $v, it is treated as '@{ Name="$v" }' + [object[]] $ToolsList, + [string] $GuardianLoggerLevel='Standard', + # Optional: Additional params to add to any tool using CredScan. + [string[]] $CrScanAdditionalRunConfigParams, + # Optional: Additional params to add to any tool using PoliCheck. + [string[]] $PoliCheckAdditionalRunConfigParams, + # Optional: Additional params to add to any tool using CodeQL/Semmle. + [string[]] $CodeQLAdditionalRunConfigParams, + # Optional: Additional params to add to any tool using Binskim. + [string[]] $BinskimAdditionalRunConfigParams +) + +$ErrorActionPreference = 'Stop' +Set-StrictMode -Version 2.0 +$disableConfigureToolsetImport = $true +$global:LASTEXITCODE = 0 + +try { + # `tools.ps1` checks $ci to perform some actions. Since the SDL + # scripts don't necessarily execute in the same agent that run the + # build.ps1/sh script this variable isn't automatically set. + $ci = $true + . $PSScriptRoot\..\tools.ps1 + + # Normalize tools list: all in [hashtable] form with defined values for each key. + $ToolsList = $ToolsList | + ForEach-Object { + if ($_ -is [string]) { + $_ = @{ Name = $_ } + } + + if (-not ($_['Scenario'])) { $_.Scenario = "" } + if (-not ($_['Args'])) { $_.Args = @() } + $_ + } + + Write-Host "List of tools to configure:" + $ToolsList | ForEach-Object { $_ | Out-String | Write-Host } + + # We store config files in the r directory of .gdn + $gdnConfigPath = Join-Path $GdnFolder 'r' + $ValidPath = Test-Path $GuardianCliLocation + + if ($ValidPath -eq $False) + { + Write-PipelineTelemetryError -Force -Category 'Sdl' -Message "Invalid Guardian CLI Location." + ExitWithExitCode 1 + } + + foreach ($tool in $ToolsList) { + # Put together the name and scenario to make a unique key. + $toolConfigName = $tool.Name + if ($tool.Scenario) { + $toolConfigName += "_" + $tool.Scenario + } + + Write-Host "=== Configuring $toolConfigName..." + + $gdnConfigFile = Join-Path $gdnConfigPath "$toolConfigName-configure.gdnconfig" + + # For some tools, add default and automatic args. + switch -Exact ($tool.Name) { + 'credscan' { + if ($targetDirectory) { + $tool.Args += "`"TargetDirectory < $TargetDirectory`"" + } + $tool.Args += "`"OutputType < pre`"" + $tool.Args += $CrScanAdditionalRunConfigParams + } + 'policheck' { + if ($targetDirectory) { + $tool.Args += "`"Target < $TargetDirectory`"" + } + $tool.Args += $PoliCheckAdditionalRunConfigParams + } + {$_ -in 'semmle', 'codeql'} { + if ($targetDirectory) { + $tool.Args += "`"SourceCodeDirectory < $TargetDirectory`"" + } + $tool.Args += $CodeQLAdditionalRunConfigParams + } + 'binskim' { + if ($targetDirectory) { + # Binskim crashes due to specific PDBs. GitHub issue: https://github.com/microsoft/binskim/issues/924. + # We are excluding all `_.pdb` files from the scan. + $tool.Args += "`"Target < $TargetDirectory\**;-:file|$TargetDirectory\**\_.pdb`"" + } + $tool.Args += $BinskimAdditionalRunConfigParams + } + } + + # Create variable pointing to the args array directly so we can use splat syntax later. + $toolArgs = $tool.Args + + # Configure the tool. If args array is provided or the current tool has some default arguments + # defined, add "--args" and splat each element on the end. Arg format is "{Arg id} < {Value}", + # one per parameter. Doc page for "guardian configure": + # https://dev.azure.com/securitytools/SecurityIntegration/_wiki/wikis/Guardian/1395/configure + Exec-BlockVerbosely { + & $GuardianCliLocation configure ` + --working-directory $WorkingDirectory ` + --tool $tool.Name ` + --output-path $gdnConfigFile ` + --logger-level $GuardianLoggerLevel ` + --noninteractive ` + --force ` + $(if ($toolArgs) { "--args" }) @toolArgs + Exit-IfNZEC "Sdl" + } + + Write-Host "Created '$toolConfigName' configuration file: $gdnConfigFile" + } +} +catch { + Write-Host $_.ScriptStackTrace + Write-PipelineTelemetryError -Force -Category 'Sdl' -Message $_ + ExitWithExitCode 1 +} diff --git a/src/arcade/eng/common/sdl/execute-all-sdl-tools.ps1 b/src/arcade/eng/common/sdl/execute-all-sdl-tools.ps1 new file mode 100644 index 000000000000..4715d75e974d --- /dev/null +++ b/src/arcade/eng/common/sdl/execute-all-sdl-tools.ps1 @@ -0,0 +1,167 @@ +Param( + [string] $GuardianPackageName, # Required: the name of guardian CLI package (not needed if GuardianCliLocation is specified) + [string] $NugetPackageDirectory, # Required: directory where NuGet packages are installed (not needed if GuardianCliLocation is specified) + [string] $GuardianCliLocation, # Optional: Direct location of Guardian CLI executable if GuardianPackageName & NugetPackageDirectory are not specified + [string] $Repository=$env:BUILD_REPOSITORY_NAME, # Required: the name of the repository (e.g. dotnet/arcade) + [string] $BranchName=$env:BUILD_SOURCEBRANCH, # Optional: name of branch or version of gdn settings; defaults to master + [string] $SourceDirectory=$env:BUILD_SOURCESDIRECTORY, # Required: the directory where source files are located + [string] $ArtifactsDirectory = (Join-Path $env:BUILD_ARTIFACTSTAGINGDIRECTORY ('artifacts')), # Required: the directory where build artifacts are located + [string] $AzureDevOpsAccessToken, # Required: access token for dnceng; should be provided via KeyVault + + # Optional: list of SDL tools to run on source code. See 'configure-sdl-tool.ps1' for tools list + # format. + [object[]] $SourceToolsList, + # Optional: list of SDL tools to run on built artifacts. See 'configure-sdl-tool.ps1' for tools + # list format. + [object[]] $ArtifactToolsList, + # Optional: list of SDL tools to run without automatically specifying a target directory. See + # 'configure-sdl-tool.ps1' for tools list format. + [object[]] $CustomToolsList, + + [bool] $TsaPublish=$False, # Optional: true will publish results to TSA; only set to true after onboarding to TSA; TSA is the automated framework used to upload test results as bugs. + [string] $TsaBranchName=$env:BUILD_SOURCEBRANCH, # Optional: required for TSA publish; defaults to $(Build.SourceBranchName); TSA is the automated framework used to upload test results as bugs. + [string] $TsaRepositoryName=$env:BUILD_REPOSITORY_NAME, # Optional: TSA repository name; will be generated automatically if not submitted; TSA is the automated framework used to upload test results as bugs. + [string] $BuildNumber=$env:BUILD_BUILDNUMBER, # Optional: required for TSA publish; defaults to $(Build.BuildNumber) + [bool] $UpdateBaseline=$False, # Optional: if true, will update the baseline in the repository; should only be run after fixing any issues which need to be fixed + [bool] $TsaOnboard=$False, # Optional: if true, will onboard the repository to TSA; should only be run once; TSA is the automated framework used to upload test results as bugs. + [string] $TsaInstanceUrl, # Optional: only needed if TsaOnboard or TsaPublish is true; the instance-url registered with TSA; TSA is the automated framework used to upload test results as bugs. + [string] $TsaCodebaseName, # Optional: only needed if TsaOnboard or TsaPublish is true; the name of the codebase registered with TSA; TSA is the automated framework used to upload test results as bugs. + [string] $TsaProjectName, # Optional: only needed if TsaOnboard or TsaPublish is true; the name of the project registered with TSA; TSA is the automated framework used to upload test results as bugs. + [string] $TsaNotificationEmail, # Optional: only needed if TsaOnboard is true; the email(s) which will receive notifications of TSA bug filings (e.g. alias@microsoft.com); TSA is the automated framework used to upload test results as bugs. + [string] $TsaCodebaseAdmin, # Optional: only needed if TsaOnboard is true; the aliases which are admins of the TSA codebase (e.g. DOMAIN\alias); TSA is the automated framework used to upload test results as bugs. + [string] $TsaBugAreaPath, # Optional: only needed if TsaOnboard is true; the area path where TSA will file bugs in AzDO; TSA is the automated framework used to upload test results as bugs. + [string] $TsaIterationPath, # Optional: only needed if TsaOnboard is true; the iteration path where TSA will file bugs in AzDO; TSA is the automated framework used to upload test results as bugs. + [string] $GuardianLoggerLevel='Standard', # Optional: the logger level for the Guardian CLI; options are Trace, Verbose, Standard, Warning, and Error + [string[]] $CrScanAdditionalRunConfigParams, # Optional: Additional Params to custom build a CredScan run config in the format @("xyz:abc","sdf:1") + [string[]] $PoliCheckAdditionalRunConfigParams, # Optional: Additional Params to custom build a Policheck run config in the format @("xyz:abc","sdf:1") + [string[]] $CodeQLAdditionalRunConfigParams, # Optional: Additional Params to custom build a Semmle/CodeQL run config in the format @("xyz < abc","sdf < 1") + [string[]] $BinskimAdditionalRunConfigParams, # Optional: Additional Params to custom build a Binskim run config in the format @("xyz < abc","sdf < 1") + [bool] $BreakOnFailure=$False # Optional: Fail the build if there were errors during the run +) + +try { + $ErrorActionPreference = 'Stop' + Set-StrictMode -Version 2.0 + $disableConfigureToolsetImport = $true + $global:LASTEXITCODE = 0 + + # `tools.ps1` checks $ci to perform some actions. Since the SDL + # scripts don't necessarily execute in the same agent that run the + # build.ps1/sh script this variable isn't automatically set. + $ci = $true + . $PSScriptRoot\..\tools.ps1 + + #Replace repo names to the format of org/repo + if (!($Repository.contains('/'))) { + $RepoName = $Repository -replace '(.*?)-(.*)', '$1/$2'; + } + else{ + $RepoName = $Repository; + } + + if ($GuardianPackageName) { + $guardianCliLocation = Join-Path $NugetPackageDirectory (Join-Path $GuardianPackageName (Join-Path 'tools' 'guardian.cmd')) + } else { + $guardianCliLocation = $GuardianCliLocation + } + + $workingDirectory = (Split-Path $SourceDirectory -Parent) + $ValidPath = Test-Path $guardianCliLocation + + if ($ValidPath -eq $False) + { + Write-PipelineTelemetryError -Force -Category 'Sdl' -Message 'Invalid Guardian CLI Location.' + ExitWithExitCode 1 + } + + Exec-BlockVerbosely { + & $(Join-Path $PSScriptRoot 'init-sdl.ps1') -GuardianCliLocation $guardianCliLocation -Repository $RepoName -BranchName $BranchName -WorkingDirectory $workingDirectory -AzureDevOpsAccessToken $AzureDevOpsAccessToken -GuardianLoggerLevel $GuardianLoggerLevel + } + $gdnFolder = Join-Path $workingDirectory '.gdn' + + if ($TsaOnboard) { + if ($TsaCodebaseName -and $TsaNotificationEmail -and $TsaCodebaseAdmin -and $TsaBugAreaPath) { + Exec-BlockVerbosely { + & $guardianCliLocation tsa-onboard --codebase-name "$TsaCodebaseName" --notification-alias "$TsaNotificationEmail" --codebase-admin "$TsaCodebaseAdmin" --instance-url "$TsaInstanceUrl" --project-name "$TsaProjectName" --area-path "$TsaBugAreaPath" --iteration-path "$TsaIterationPath" --working-directory $workingDirectory --logger-level $GuardianLoggerLevel + } + if ($LASTEXITCODE -ne 0) { + Write-PipelineTelemetryError -Force -Category 'Sdl' -Message "Guardian tsa-onboard failed with exit code $LASTEXITCODE." + ExitWithExitCode $LASTEXITCODE + } + } else { + Write-PipelineTelemetryError -Force -Category 'Sdl' -Message 'Could not onboard to TSA -- not all required values ($TsaCodebaseName, $TsaNotificationEmail, $TsaCodebaseAdmin, $TsaBugAreaPath) were specified.' + ExitWithExitCode 1 + } + } + + # Configure a list of tools with a default target directory. Populates the ".gdn/r" directory. + function Configure-ToolsList([object[]] $tools, [string] $targetDirectory) { + if ($tools -and $tools.Count -gt 0) { + Exec-BlockVerbosely { + & $(Join-Path $PSScriptRoot 'configure-sdl-tool.ps1') ` + -GuardianCliLocation $guardianCliLocation ` + -WorkingDirectory $workingDirectory ` + -TargetDirectory $targetDirectory ` + -GdnFolder $gdnFolder ` + -ToolsList $tools ` + -AzureDevOpsAccessToken $AzureDevOpsAccessToken ` + -GuardianLoggerLevel $GuardianLoggerLevel ` + -CrScanAdditionalRunConfigParams $CrScanAdditionalRunConfigParams ` + -PoliCheckAdditionalRunConfigParams $PoliCheckAdditionalRunConfigParams ` + -CodeQLAdditionalRunConfigParams $CodeQLAdditionalRunConfigParams ` + -BinskimAdditionalRunConfigParams $BinskimAdditionalRunConfigParams + if ($BreakOnFailure) { + Exit-IfNZEC "Sdl" + } + } + } + } + + # Configure Artifact and Source tools with default Target directories. + Configure-ToolsList $ArtifactToolsList $ArtifactsDirectory + Configure-ToolsList $SourceToolsList $SourceDirectory + # Configure custom tools with no default Target directory. + Configure-ToolsList $CustomToolsList $null + + # At this point, all tools are configured in the ".gdn" directory. Run them all in a single call. + # (If we used "run" multiple times, each run would overwrite data from earlier runs.) + Exec-BlockVerbosely { + & $(Join-Path $PSScriptRoot 'run-sdl.ps1') ` + -GuardianCliLocation $guardianCliLocation ` + -WorkingDirectory $SourceDirectory ` + -UpdateBaseline $UpdateBaseline ` + -GdnFolder $gdnFolder + } + + if ($TsaPublish) { + if ($TsaBranchName -and $BuildNumber) { + if (-not $TsaRepositoryName) { + $TsaRepositoryName = "$($Repository)-$($BranchName)" + } + Exec-BlockVerbosely { + & $guardianCliLocation tsa-publish --all-tools --repository-name "$TsaRepositoryName" --branch-name "$TsaBranchName" --build-number "$BuildNumber" --onboard $True --codebase-name "$TsaCodebaseName" --notification-alias "$TsaNotificationEmail" --codebase-admin "$TsaCodebaseAdmin" --instance-url "$TsaInstanceUrl" --project-name "$TsaProjectName" --area-path "$TsaBugAreaPath" --iteration-path "$TsaIterationPath" --working-directory $workingDirectory --logger-level $GuardianLoggerLevel + } + if ($LASTEXITCODE -ne 0) { + Write-PipelineTelemetryError -Force -Category 'Sdl' -Message "Guardian tsa-publish failed with exit code $LASTEXITCODE." + ExitWithExitCode $LASTEXITCODE + } + } else { + Write-PipelineTelemetryError -Force -Category 'Sdl' -Message 'Could not publish to TSA -- not all required values ($TsaBranchName, $BuildNumber) were specified.' + ExitWithExitCode 1 + } + } + + if ($BreakOnFailure) { + Write-Host "Failing the build in case of breaking results..." + Exec-BlockVerbosely { + & $guardianCliLocation break --working-directory $workingDirectory --logger-level $GuardianLoggerLevel + } + } else { + Write-Host "Letting the build pass even if there were breaking results..." + } +} +catch { + Write-Host $_.ScriptStackTrace + Write-PipelineTelemetryError -Force -Category 'Sdl' -Message $_ + exit 1 +} diff --git a/src/arcade/eng/common/sdl/extract-artifact-archives.ps1 b/src/arcade/eng/common/sdl/extract-artifact-archives.ps1 new file mode 100644 index 000000000000..68da4fbf2571 --- /dev/null +++ b/src/arcade/eng/common/sdl/extract-artifact-archives.ps1 @@ -0,0 +1,63 @@ +# This script looks for each archive file in a directory and extracts it into the target directory. +# For example, the file "$InputPath/bin.tar.gz" extracts to "$ExtractPath/bin.tar.gz.extracted/**". +# Uses the "tar" utility added to Windows 10 / Windows 2019 that supports tar.gz and zip. +param( + # Full path to directory where archives are stored. + [Parameter(Mandatory=$true)][string] $InputPath, + # Full path to directory to extract archives into. May be the same as $InputPath. + [Parameter(Mandatory=$true)][string] $ExtractPath +) + +$ErrorActionPreference = 'Stop' +Set-StrictMode -Version 2.0 + +$disableConfigureToolsetImport = $true + +try { + # `tools.ps1` checks $ci to perform some actions. Since the SDL + # scripts don't necessarily execute in the same agent that run the + # build.ps1/sh script this variable isn't automatically set. + $ci = $true + . $PSScriptRoot\..\tools.ps1 + + Measure-Command { + $jobs = @() + + # Find archive files for non-Windows and Windows builds. + $archiveFiles = @( + Get-ChildItem (Join-Path $InputPath "*.tar.gz") + Get-ChildItem (Join-Path $InputPath "*.zip") + ) + + foreach ($targzFile in $archiveFiles) { + $jobs += Start-Job -ScriptBlock { + $file = $using:targzFile + $fileName = [System.IO.Path]::GetFileName($file) + $extractDir = Join-Path $using:ExtractPath "$fileName.extracted" + + New-Item $extractDir -ItemType Directory -Force | Out-Null + + Write-Host "Extracting '$file' to '$extractDir'..." + + # Pipe errors to stdout to prevent PowerShell detecting them and quitting the job early. + # This type of quit skips the catch, so we wouldn't be able to tell which file triggered the + # error. Save output so it can be stored in the exception string along with context. + $output = tar -xf $file -C $extractDir 2>&1 + # Handle NZEC manually rather than using Exit-IfNZEC: we are in a background job, so we + # don't have access to the outer scope. + if ($LASTEXITCODE -ne 0) { + throw "Error extracting '$file': non-zero exit code ($LASTEXITCODE). Output: '$output'" + } + + Write-Host "Extracted to $extractDir" + } + } + + Receive-Job $jobs -Wait + } +} +catch { + Write-Host $_ + Write-PipelineTelemetryError -Force -Category 'Sdl' -Message $_ + ExitWithExitCode 1 +} diff --git a/src/arcade/eng/common/sdl/extract-artifact-packages.ps1 b/src/arcade/eng/common/sdl/extract-artifact-packages.ps1 new file mode 100644 index 000000000000..f031ed5b25e9 --- /dev/null +++ b/src/arcade/eng/common/sdl/extract-artifact-packages.ps1 @@ -0,0 +1,82 @@ +param( + [Parameter(Mandatory=$true)][string] $InputPath, # Full path to directory where artifact packages are stored + [Parameter(Mandatory=$true)][string] $ExtractPath # Full path to directory where the packages will be extracted +) + +$ErrorActionPreference = 'Stop' +Set-StrictMode -Version 2.0 + +$disableConfigureToolsetImport = $true + +function ExtractArtifacts { + if (!(Test-Path $InputPath)) { + Write-Host "Input Path does not exist: $InputPath" + ExitWithExitCode 0 + } + $Jobs = @() + Get-ChildItem "$InputPath\*.nupkg" | + ForEach-Object { + $Jobs += Start-Job -ScriptBlock $ExtractPackage -ArgumentList $_.FullName + } + + foreach ($Job in $Jobs) { + Wait-Job -Id $Job.Id | Receive-Job + } +} + +try { + # `tools.ps1` checks $ci to perform some actions. Since the SDL + # scripts don't necessarily execute in the same agent that run the + # build.ps1/sh script this variable isn't automatically set. + $ci = $true + . $PSScriptRoot\..\tools.ps1 + + $ExtractPackage = { + param( + [string] $PackagePath # Full path to a NuGet package + ) + + if (!(Test-Path $PackagePath)) { + Write-PipelineTelemetryError -Category 'Build' -Message "Input file does not exist: $PackagePath" + ExitWithExitCode 1 + } + + $RelevantExtensions = @('.dll', '.exe', '.pdb') + Write-Host -NoNewLine 'Extracting ' ([System.IO.Path]::GetFileName($PackagePath)) '...' + + $PackageId = [System.IO.Path]::GetFileNameWithoutExtension($PackagePath) + $ExtractPath = Join-Path -Path $using:ExtractPath -ChildPath $PackageId + + Add-Type -AssemblyName System.IO.Compression.FileSystem + + [System.IO.Directory]::CreateDirectory($ExtractPath); + + try { + $zip = [System.IO.Compression.ZipFile]::OpenRead($PackagePath) + + $zip.Entries | + Where-Object {$RelevantExtensions -contains [System.IO.Path]::GetExtension($_.Name)} | + ForEach-Object { + $TargetPath = Join-Path -Path $ExtractPath -ChildPath (Split-Path -Path $_.FullName) + [System.IO.Directory]::CreateDirectory($TargetPath); + + $TargetFile = Join-Path -Path $ExtractPath -ChildPath $_.FullName + [System.IO.Compression.ZipFileExtensions]::ExtractToFile($_, $TargetFile) + } + } + catch { + Write-Host $_ + Write-PipelineTelemetryError -Force -Category 'Sdl' -Message $_ + ExitWithExitCode 1 + } + finally { + $zip.Dispose() + } + } + Measure-Command { ExtractArtifacts } +} +catch { + Write-Host $_ + Write-PipelineTelemetryError -Force -Category 'Sdl' -Message $_ + ExitWithExitCode 1 +} diff --git a/src/arcade/eng/common/sdl/init-sdl.ps1 b/src/arcade/eng/common/sdl/init-sdl.ps1 new file mode 100644 index 000000000000..3ac1d92b3700 --- /dev/null +++ b/src/arcade/eng/common/sdl/init-sdl.ps1 @@ -0,0 +1,55 @@ +Param( + [string] $GuardianCliLocation, + [string] $Repository, + [string] $BranchName='master', + [string] $WorkingDirectory, + [string] $AzureDevOpsAccessToken, + [string] $GuardianLoggerLevel='Standard' +) + +$ErrorActionPreference = 'Stop' +Set-StrictMode -Version 2.0 +$disableConfigureToolsetImport = $true +$global:LASTEXITCODE = 0 + +# `tools.ps1` checks $ci to perform some actions. Since the SDL +# scripts don't necessarily execute in the same agent that run the +# build.ps1/sh script this variable isn't automatically set. +$ci = $true +. $PSScriptRoot\..\tools.ps1 + +# Don't display the console progress UI - it's a huge perf hit +$ProgressPreference = 'SilentlyContinue' + +# Construct basic auth from AzDO access token; construct URI to the repository's gdn folder stored in that repository; construct location of zip file +$encodedPat = [Convert]::ToBase64String([System.Text.Encoding]::ASCII.GetBytes(":$AzureDevOpsAccessToken")) +$escapedRepository = [Uri]::EscapeDataString("/$Repository/$BranchName/.gdn") +$uri = "https://dev.azure.com/dnceng/internal/_apis/git/repositories/sdl-tool-cfg/Items?path=$escapedRepository&versionDescriptor[versionOptions]=0&`$format=zip&api-version=5.0" +$zipFile = "$WorkingDirectory/gdn.zip" + +Add-Type -AssemblyName System.IO.Compression.FileSystem +$gdnFolder = (Join-Path $WorkingDirectory '.gdn') + +try { + # if the folder does not exist, we'll do a guardian init and push it to the remote repository + Write-Host 'Initializing Guardian...' + Write-Host "$GuardianCliLocation init --working-directory $WorkingDirectory --logger-level $GuardianLoggerLevel" + & $GuardianCliLocation init --working-directory $WorkingDirectory --logger-level $GuardianLoggerLevel + if ($LASTEXITCODE -ne 0) { + Write-PipelineTelemetryError -Force -Category 'Build' -Message "Guardian init failed with exit code $LASTEXITCODE." + ExitWithExitCode $LASTEXITCODE + } + # We create the mainbaseline so it can be edited later + Write-Host "$GuardianCliLocation baseline --working-directory $WorkingDirectory --name mainbaseline" + & $GuardianCliLocation baseline --working-directory $WorkingDirectory --name mainbaseline + if ($LASTEXITCODE -ne 0) { + Write-PipelineTelemetryError -Force -Category 'Build' -Message "Guardian baseline failed with exit code $LASTEXITCODE." + ExitWithExitCode $LASTEXITCODE + } + ExitWithExitCode 0 +} +catch { + Write-Host $_.ScriptStackTrace + Write-PipelineTelemetryError -Force -Category 'Sdl' -Message $_ + ExitWithExitCode 1 +} diff --git a/src/arcade/eng/common/sdl/run-sdl.ps1 b/src/arcade/eng/common/sdl/run-sdl.ps1 new file mode 100644 index 000000000000..2eac8c78f103 --- /dev/null +++ b/src/arcade/eng/common/sdl/run-sdl.ps1 @@ -0,0 +1,49 @@ +Param( + [string] $GuardianCliLocation, + [string] $WorkingDirectory, + [string] $GdnFolder, + [string] $UpdateBaseline, + [string] $GuardianLoggerLevel='Standard' +) + +$ErrorActionPreference = 'Stop' +Set-StrictMode -Version 2.0 +$disableConfigureToolsetImport = $true +$global:LASTEXITCODE = 0 + +try { + # `tools.ps1` checks $ci to perform some actions. Since the SDL + # scripts don't necessarily execute in the same agent that run the + # build.ps1/sh script this variable isn't automatically set. + $ci = $true + . $PSScriptRoot\..\tools.ps1 + + # We store config files in the r directory of .gdn + $gdnConfigPath = Join-Path $GdnFolder 'r' + $ValidPath = Test-Path $GuardianCliLocation + + if ($ValidPath -eq $False) + { + Write-PipelineTelemetryError -Force -Category 'Sdl' -Message "Invalid Guardian CLI Location." + ExitWithExitCode 1 + } + + $gdnConfigFiles = Get-ChildItem $gdnConfigPath -Recurse -Include '*.gdnconfig' + Write-Host "Discovered Guardian config files:" + $gdnConfigFiles | Out-String | Write-Host + + Exec-BlockVerbosely { + & $GuardianCliLocation run ` + --working-directory $WorkingDirectory ` + --baseline mainbaseline ` + --update-baseline $UpdateBaseline ` + --logger-level $GuardianLoggerLevel ` + --config @gdnConfigFiles + Exit-IfNZEC "Sdl" + } +} +catch { + Write-Host $_.ScriptStackTrace + Write-PipelineTelemetryError -Force -Category 'Sdl' -Message $_ + ExitWithExitCode 1 +} diff --git a/src/arcade/eng/common/sdl/sdl.ps1 b/src/arcade/eng/common/sdl/sdl.ps1 new file mode 100644 index 000000000000..648c5068d7d6 --- /dev/null +++ b/src/arcade/eng/common/sdl/sdl.ps1 @@ -0,0 +1,38 @@ + +function Install-Gdn { + param( + [Parameter(Mandatory=$true)] + [string]$Path, + + # If omitted, install the latest version of Guardian, otherwise install that specific version. + [string]$Version + ) + + $ErrorActionPreference = 'Stop' + Set-StrictMode -Version 2.0 + $disableConfigureToolsetImport = $true + $global:LASTEXITCODE = 0 + + # `tools.ps1` checks $ci to perform some actions. Since the SDL + # scripts don't necessarily execute in the same agent that run the + # build.ps1/sh script this variable isn't automatically set. + $ci = $true + . $PSScriptRoot\..\tools.ps1 + + $argumentList = @("install", "Microsoft.Guardian.Cli", "-Source https://securitytools.pkgs.visualstudio.com/_packaging/Guardian/nuget/v3/index.json", "-OutputDirectory $Path", "-NonInteractive", "-NoCache") + + if ($Version) { + $argumentList += "-Version $Version" + } + + Start-Process nuget -Verbose -ArgumentList $argumentList -NoNewWindow -Wait + + $gdnCliPath = Get-ChildItem -Filter guardian.cmd -Recurse -Path $Path + + if (!$gdnCliPath) + { + Write-PipelineTelemetryError -Category 'Sdl' -Message 'Failure installing Guardian' + } + + return $gdnCliPath.FullName +} \ No newline at end of file diff --git a/src/arcade/eng/common/sdl/trim-assets-version.ps1 b/src/arcade/eng/common/sdl/trim-assets-version.ps1 new file mode 100644 index 000000000000..0daa2a9e9462 --- /dev/null +++ b/src/arcade/eng/common/sdl/trim-assets-version.ps1 @@ -0,0 +1,75 @@ +<# +.SYNOPSIS +Install and run the 'Microsoft.DotNet.VersionTools.Cli' tool with the 'trim-artifacts-version' command to trim the version from the NuGet assets file name. + +.PARAMETER InputPath +Full path to directory where artifact packages are stored + +.PARAMETER Recursive +Search for NuGet packages recursively + +#> + +Param( + [string] $InputPath, + [bool] $Recursive = $true +) + +$CliToolName = "Microsoft.DotNet.VersionTools.Cli" + +function Install-VersionTools-Cli { + param( + [Parameter(Mandatory=$true)][string]$Version + ) + + Write-Host "Installing the package '$CliToolName' with a version of '$version' ..." + $feed = "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json" + + $argumentList = @("tool", "install", "--local", "$CliToolName", "--add-source $feed", "--no-cache", "--version $Version", "--create-manifest-if-needed") + Start-Process "$dotnet" -Verbose -ArgumentList $argumentList -NoNewWindow -Wait +} + +# ------------------------------------------------------------------- + +if (!(Test-Path $InputPath)) { + Write-Host "Input Path '$InputPath' does not exist" + ExitWithExitCode 1 +} + +$ErrorActionPreference = 'Stop' +Set-StrictMode -Version 2.0 + +$disableConfigureToolsetImport = $true +$global:LASTEXITCODE = 0 + +# `tools.ps1` checks $ci to perform some actions. Since the SDL +# scripts don't necessarily execute in the same agent that run the +# build.ps1/sh script this variable isn't automatically set. +$ci = $true +. $PSScriptRoot\..\tools.ps1 + +try { + $dotnetRoot = InitializeDotNetCli -install:$true + $dotnet = "$dotnetRoot\dotnet.exe" + + $toolsetVersion = Read-ArcadeSdkVersion + Install-VersionTools-Cli -Version $toolsetVersion + + $cliToolFound = (& "$dotnet" tool list --local | Where-Object {$_.Split(' ')[0] -eq $CliToolName}) + if ($null -eq $cliToolFound) { + Write-PipelineTelemetryError -Force -Category 'Sdl' -Message "The '$CliToolName' tool is not installed." + ExitWithExitCode 1 + } + + Exec-BlockVerbosely { + & "$dotnet" $CliToolName trim-assets-version ` + --assets-path $InputPath ` + --recursive $Recursive + Exit-IfNZEC "Sdl" + } +} +catch { + Write-Host $_ + Write-PipelineTelemetryError -Force -Category 'Sdl' -Message $_ + ExitWithExitCode 1 +} diff --git a/src/arcade/eng/common/template-guidance.md b/src/arcade/eng/common/template-guidance.md new file mode 100644 index 000000000000..98bbc1ded0ba --- /dev/null +++ b/src/arcade/eng/common/template-guidance.md @@ -0,0 +1,133 @@ +# Overview + +Arcade provides templates for public (`/templates`) and 1ES pipeline templates (`/templates-official`) scenarios. Pipelines which are required to be managed by 1ES pipeline templates should reference `/templates-offical`, all other pipelines may reference `/templates`. + +## How to use + +Basic guidance is: + +- 1ES Pipeline Template or 1ES Microbuild template runs should reference `eng/common/templates-official`. Any internal production-graded pipeline should use these templates. + +- All other runs should reference `eng/common/templates`. + +See [azure-pipelines.yml](../../azure-pipelines.yml) (templates-official example) or [azure-pipelines-pr.yml](../../azure-pipelines-pr.yml) (templates example) for examples. + +#### The `templateIs1ESManaged` parameter + +The `templateIs1ESManaged` is available on most templates and affects which of the variants is used for nested templates. See [Development Notes](#development-notes) below for more information on the `templateIs1ESManaged1 parameter. + +- For templates under `job/`, `jobs/`, `steps`, or `post-build/`, this parameter must be explicitly set. + +## Multiple outputs + +1ES pipeline templates impose a policy where every publish artifact execution results in additional security scans being injected into your pipeline. When using `templates-official/jobs/jobs.yml`, Arcade reduces the number of additional security injections by gathering all publishing outputs into the [Build.ArtifactStagingDirectory](https://learn.microsoft.com/en-us/azure/devops/pipelines/build/variables?view=azure-devops&tabs=yaml#build-variables-devops-services), and utilizing the [outputParentDirectory](https://eng.ms/docs/cloud-ai-platform/devdiv/one-engineering-system-1es/1es-docs/1es-pipeline-templates/features/outputs#multiple-outputs) feature of 1ES pipeline templates. When implementing your pipeline, if you ensure publish artifacts are located in the `$(Build.ArtifactStagingDirectory)`, and utilize the 1ES provided template context, then you can reduce the number of security scans for your pipeline. + +Example: +``` yaml +# azure-pipelines.yml +extends: + template: azure-pipelines/MicroBuild.1ES.Official.yml@MicroBuildTemplate + parameters: + stages: + - stage: build + jobs: + - template: /eng/common/templates-official/jobs/jobs.yml@self + parameters: + # 1ES makes use of outputs to reduce security task injection overhead + templateContext: + outputs: + - output: pipelineArtifact + displayName: 'Publish logs from source' + continueOnError: true + condition: always() + targetPath: $(Build.ArtifactStagingDirectory)/artifacts/log + artifactName: Logs + jobs: + - job: Windows + steps: + - script: echo "friendly neighborhood" > artifacts/marvel/spiderman.txt + # copy build outputs to artifact staging directory for publishing + - task: CopyFiles@2 + displayName: Gather build output + inputs: + SourceFolder: '$(Build.SourcesDirectory)/artifacts/marvel' + Contents: '**' + TargetFolder: '$(Build.ArtifactStagingDirectory)/artifacts/marvel' +``` + +Note: Multiple outputs are ONLY applicable to 1ES PT publishing (only usable when referencing `templates-official`). + +## Development notes + +**Folder / file structure** + +``` text +eng\common\ + [templates || templates-official]\ + job\ + job.yml (shim + artifact publishing logic) + onelocbuild.yml (shim) + publish-build-assets.yml (shim) + source-build.yml (shim) + source-index-stage1.yml (shim) + jobs\ + codeql-build.yml (shim) + jobs.yml (shim) + source-build.yml (shim) + post-build\ + post-build.yml (shim) + common-variabls.yml (shim) + setup-maestro-vars.yml (shim) + steps\ + publish-build-artifacts.yml (logic) + publish-pipeline-artifacts.yml (logic) + component-governance.yml (shim) + generate-sbom.yml (shim) + publish-logs.yml (shim) + retain-build.yml (shim) + send-to-helix.yml (shim) + source-build.yml (shim) + variables\ + pool-providers.yml (logic + redirect) # templates/variables/pool-providers.yml will redirect to templates-official/variables/pool-providers.yml if you are running in the internal project + sdl-variables.yml (logic) + core-templates\ + job\ + job.yml (logic) + onelocbuild.yml (logic) + publish-build-assets.yml (logic) + source-build.yml (logic) + source-index-stage1.yml (logic) + jobs\ + codeql-build.yml (logic) + jobs.yml (logic) + source-build.yml (logic) + post-build\ + common-variabls.yml (logic) + post-build.yml (logic) + setup-maestro-vars.yml (logic) + steps\ + component-governance.yml (logic) + generate-sbom.yml (logic) + publish-build-artifacts.yml (redirect) + publish-logs.yml (logic) + publish-pipeline-artifacts.yml (redirect) + retain-build.yml (logic) + send-to-helix.yml (logic) + source-build.yml (logic) + variables\ + pool-providers.yml (redirect) +``` + +In the table above, a file is designated as "shim", "logic", or "redirect". + +- shim - represents a yaml file which is an intermediate step between pipeline logic and .Net Core Engineering's templates (`core-templates`) and defines the `is1ESPipeline` parameter value. + +- logic - represents actual base template logic. + +- redirect- represents a file in `core-templates` which redirects to the "logic" file in either `templates` or `templates-official`. + +Logic for Arcade's templates live **primarily** in the `core-templates` folder. The exceptions to the location of the logic files are around artifact publishing, which is handled differently between 1es pipeline templates and standard templates. `templates` and `templates-official` provide shim entry points which redirect to `core-templates` while also defining the `is1ESPipeline` parameter. If a shim is referenced in `templates`, then `is1ESPipeline` is set to `false`. If a shim is referenced in `templates-official`, then `is1ESPipeline` is set to `true`. + +Within `templates` and `templates-official`, the templates at the "stages", and "jobs" / "job" level have been replaced with shims. Templates at the "steps" and "variables" level are typically too granular to be replaced with shims and instead persist logic which is directly applicable to either scenario. + +Within `core-templates`, there are a handful of places where logic is dependent on which shim entry point was used. In those places, we redirect back to the respective logic file in `templates` or `templates-official`. diff --git a/src/arcade/eng/common/templates-official/job/onelocbuild.yml b/src/arcade/eng/common/templates-official/job/onelocbuild.yml new file mode 100644 index 000000000000..0f0c514b912d --- /dev/null +++ b/src/arcade/eng/common/templates-official/job/onelocbuild.yml @@ -0,0 +1,7 @@ +jobs: +- template: /eng/common/core-templates/job/onelocbuild.yml + parameters: + is1ESPipeline: true + + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} diff --git a/src/arcade/eng/common/templates-official/job/publish-build-assets.yml b/src/arcade/eng/common/templates-official/job/publish-build-assets.yml new file mode 100644 index 000000000000..d667a70e8de7 --- /dev/null +++ b/src/arcade/eng/common/templates-official/job/publish-build-assets.yml @@ -0,0 +1,7 @@ +jobs: +- template: /eng/common/core-templates/job/publish-build-assets.yml + parameters: + is1ESPipeline: true + + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} diff --git a/src/arcade/eng/common/templates-official/job/source-build.yml b/src/arcade/eng/common/templates-official/job/source-build.yml new file mode 100644 index 000000000000..1a480034b678 --- /dev/null +++ b/src/arcade/eng/common/templates-official/job/source-build.yml @@ -0,0 +1,7 @@ +jobs: +- template: /eng/common/core-templates/job/source-build.yml + parameters: + is1ESPipeline: true + + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} diff --git a/src/arcade/eng/common/templates-official/job/source-index-stage1.yml b/src/arcade/eng/common/templates-official/job/source-index-stage1.yml new file mode 100644 index 000000000000..6d5ead316f92 --- /dev/null +++ b/src/arcade/eng/common/templates-official/job/source-index-stage1.yml @@ -0,0 +1,7 @@ +jobs: +- template: /eng/common/core-templates/job/source-index-stage1.yml + parameters: + is1ESPipeline: true + + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} diff --git a/src/arcade/eng/common/templates-official/jobs/codeql-build.yml b/src/arcade/eng/common/templates-official/jobs/codeql-build.yml new file mode 100644 index 000000000000..a726322ecfe0 --- /dev/null +++ b/src/arcade/eng/common/templates-official/jobs/codeql-build.yml @@ -0,0 +1,7 @@ +jobs: +- template: /eng/common/core-templates/jobs/codeql-build.yml + parameters: + is1ESPipeline: true + + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} diff --git a/src/arcade/eng/common/templates-official/jobs/jobs.yml b/src/arcade/eng/common/templates-official/jobs/jobs.yml new file mode 100644 index 000000000000..007deddaea0f --- /dev/null +++ b/src/arcade/eng/common/templates-official/jobs/jobs.yml @@ -0,0 +1,7 @@ +jobs: +- template: /eng/common/core-templates/jobs/jobs.yml + parameters: + is1ESPipeline: true + + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} diff --git a/src/arcade/eng/common/templates-official/jobs/source-build.yml b/src/arcade/eng/common/templates-official/jobs/source-build.yml new file mode 100644 index 000000000000..483e7b611f34 --- /dev/null +++ b/src/arcade/eng/common/templates-official/jobs/source-build.yml @@ -0,0 +1,7 @@ +jobs: +- template: /eng/common/core-templates/jobs/source-build.yml + parameters: + is1ESPipeline: true + + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} \ No newline at end of file diff --git a/src/arcade/eng/common/templates-official/post-build/common-variables.yml b/src/arcade/eng/common/templates-official/post-build/common-variables.yml new file mode 100644 index 000000000000..c32fc49233f8 --- /dev/null +++ b/src/arcade/eng/common/templates-official/post-build/common-variables.yml @@ -0,0 +1,8 @@ +variables: +- template: /eng/common/core-templates/post-build/common-variables.yml + parameters: + # Specifies whether to use 1ES + is1ESPipeline: true + + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} \ No newline at end of file diff --git a/src/arcade/eng/common/templates-official/post-build/post-build.yml b/src/arcade/eng/common/templates-official/post-build/post-build.yml new file mode 100644 index 000000000000..2364c0fd4a52 --- /dev/null +++ b/src/arcade/eng/common/templates-official/post-build/post-build.yml @@ -0,0 +1,8 @@ +stages: +- template: /eng/common/core-templates/post-build/post-build.yml + parameters: + # Specifies whether to use 1ES + is1ESPipeline: true + + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} diff --git a/src/arcade/eng/common/templates-official/post-build/setup-maestro-vars.yml b/src/arcade/eng/common/templates-official/post-build/setup-maestro-vars.yml new file mode 100644 index 000000000000..024397d87864 --- /dev/null +++ b/src/arcade/eng/common/templates-official/post-build/setup-maestro-vars.yml @@ -0,0 +1,8 @@ +steps: +- template: /eng/common/core-templates/post-build/setup-maestro-vars.yml + parameters: + # Specifies whether to use 1ES + is1ESPipeline: true + + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} \ No newline at end of file diff --git a/src/arcade/eng/common/templates-official/steps/component-governance.yml b/src/arcade/eng/common/templates-official/steps/component-governance.yml new file mode 100644 index 000000000000..30bb3985ca2b --- /dev/null +++ b/src/arcade/eng/common/templates-official/steps/component-governance.yml @@ -0,0 +1,7 @@ +steps: +- template: /eng/common/core-templates/steps/component-governance.yml + parameters: + is1ESPipeline: true + + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} diff --git a/src/arcade/eng/common/templates-official/steps/enable-internal-runtimes.yml b/src/arcade/eng/common/templates-official/steps/enable-internal-runtimes.yml new file mode 100644 index 000000000000..f9dd238c6cd5 --- /dev/null +++ b/src/arcade/eng/common/templates-official/steps/enable-internal-runtimes.yml @@ -0,0 +1,9 @@ +# Obtains internal runtime download credentials and populates the 'dotnetbuilds-internal-container-read-token-base64' +# variable with the base64-encoded SAS token, by default +steps: +- template: /eng/common/core-templates/steps/enable-internal-runtimes.yml + parameters: + is1ESPipeline: true + + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} diff --git a/src/arcade/eng/common/templates-official/steps/enable-internal-sources.yml b/src/arcade/eng/common/templates-official/steps/enable-internal-sources.yml new file mode 100644 index 000000000000..e6d57182284d --- /dev/null +++ b/src/arcade/eng/common/templates-official/steps/enable-internal-sources.yml @@ -0,0 +1,7 @@ +steps: +- template: /eng/common/core-templates/steps/enable-internal-sources.yml + parameters: + is1ESPipeline: true + + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} \ No newline at end of file diff --git a/src/arcade/eng/common/templates-official/steps/generate-sbom.yml b/src/arcade/eng/common/templates-official/steps/generate-sbom.yml new file mode 100644 index 000000000000..9a89a4706d94 --- /dev/null +++ b/src/arcade/eng/common/templates-official/steps/generate-sbom.yml @@ -0,0 +1,7 @@ +steps: +- template: /eng/common/core-templates/steps/generate-sbom.yml + parameters: + is1ESPipeline: true + + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} diff --git a/src/arcade/eng/common/templates-official/steps/get-delegation-sas.yml b/src/arcade/eng/common/templates-official/steps/get-delegation-sas.yml new file mode 100644 index 000000000000..c5a9c1f8275c --- /dev/null +++ b/src/arcade/eng/common/templates-official/steps/get-delegation-sas.yml @@ -0,0 +1,7 @@ +steps: +- template: /eng/common/core-templates/steps/get-delegation-sas.yml + parameters: + is1ESPipeline: true + + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} diff --git a/src/arcade/eng/common/templates-official/steps/get-federated-access-token.yml b/src/arcade/eng/common/templates-official/steps/get-federated-access-token.yml new file mode 100644 index 000000000000..c8dcf6b81392 --- /dev/null +++ b/src/arcade/eng/common/templates-official/steps/get-federated-access-token.yml @@ -0,0 +1,7 @@ +steps: +- template: /eng/common/core-templates/steps/get-federated-access-token.yml + parameters: + is1ESPipeline: true + + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} \ No newline at end of file diff --git a/src/arcade/eng/common/templates-official/steps/publish-logs.yml b/src/arcade/eng/common/templates-official/steps/publish-logs.yml new file mode 100644 index 000000000000..579fd531e94c --- /dev/null +++ b/src/arcade/eng/common/templates-official/steps/publish-logs.yml @@ -0,0 +1,7 @@ +steps: +- template: /eng/common/core-templates/steps/publish-logs.yml + parameters: + is1ESPipeline: true + + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} diff --git a/src/arcade/eng/common/templates-official/steps/publish-pipeline-artifacts.yml b/src/arcade/eng/common/templates-official/steps/publish-pipeline-artifacts.yml new file mode 100644 index 000000000000..172f9f0fdc97 --- /dev/null +++ b/src/arcade/eng/common/templates-official/steps/publish-pipeline-artifacts.yml @@ -0,0 +1,28 @@ +parameters: +- name: is1ESPipeline + type: boolean + default: true + +- name: args + type: object + default: {} + +steps: +- ${{ if ne(parameters.is1ESPipeline, true) }}: + - 'eng/common/templates-official cannot be referenced from a non-1ES managed template': error +- task: 1ES.PublishPipelineArtifact@1 + displayName: ${{ coalesce(parameters.args.displayName, 'Publish to Build Artifact') }} + ${{ if parameters.args.condition }}: + condition: ${{ parameters.args.condition }} + ${{ else }}: + condition: succeeded() + ${{ if parameters.args.continueOnError }}: + continueOnError: ${{ parameters.args.continueOnError }} + inputs: + targetPath: ${{ parameters.args.targetPath }} + ${{ if parameters.args.artifactName }}: + artifactName: ${{ parameters.args.artifactName }} + ${{ if parameters.args.properties }}: + properties: ${{ parameters.args.properties }} + ${{ if parameters.args.sbomEnabled }}: + sbomEnabled: ${{ parameters.args.sbomEnabled }} diff --git a/src/arcade/eng/common/templates-official/steps/retain-build.yml b/src/arcade/eng/common/templates-official/steps/retain-build.yml new file mode 100644 index 000000000000..5594551508a3 --- /dev/null +++ b/src/arcade/eng/common/templates-official/steps/retain-build.yml @@ -0,0 +1,7 @@ +steps: +- template: /eng/common/core-templates/steps/retain-build.yml + parameters: + is1ESPipeline: true + + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} diff --git a/src/arcade/eng/common/templates-official/steps/send-to-helix.yml b/src/arcade/eng/common/templates-official/steps/send-to-helix.yml new file mode 100644 index 000000000000..6500f21bf845 --- /dev/null +++ b/src/arcade/eng/common/templates-official/steps/send-to-helix.yml @@ -0,0 +1,7 @@ +steps: +- template: /eng/common/core-templates/steps/send-to-helix.yml + parameters: + is1ESPipeline: true + + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} diff --git a/src/arcade/eng/common/templates-official/steps/source-build.yml b/src/arcade/eng/common/templates-official/steps/source-build.yml new file mode 100644 index 000000000000..8f92c49e7b06 --- /dev/null +++ b/src/arcade/eng/common/templates-official/steps/source-build.yml @@ -0,0 +1,7 @@ +steps: +- template: /eng/common/core-templates/steps/source-build.yml + parameters: + is1ESPipeline: true + + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} diff --git a/src/arcade/eng/common/templates-official/variables/pool-providers.yml b/src/arcade/eng/common/templates-official/variables/pool-providers.yml new file mode 100644 index 000000000000..1f308b24efc4 --- /dev/null +++ b/src/arcade/eng/common/templates-official/variables/pool-providers.yml @@ -0,0 +1,45 @@ +# Select a pool provider based off branch name. Anything with branch name containing 'release' must go into an -Svc pool, +# otherwise it should go into the "normal" pools. This separates out the queueing and billing of released branches. + +# Motivation: +# Once a given branch of a repository's output has been officially "shipped" once, it is then considered to be COGS +# (Cost of goods sold) and should be moved to a servicing pool provider. This allows both separation of queueing +# (allowing release builds and main PR builds to not intefere with each other) and billing (required for COGS. +# Additionally, the pool provider name itself may be subject to change when the .NET Core Engineering Services +# team needs to move resources around and create new and potentially differently-named pools. Using this template +# file from an Arcade-ified repo helps guard against both having to update one's release/* branches and renaming. + +# How to use: +# This yaml assumes your shipped product branches use the naming convention "release/..." (which many do). +# If we find alternate naming conventions in broad usage it can be added to the condition below. +# +# First, import the template in an arcade-ified repo to pick up the variables, e.g.: +# +# variables: +# - template: /eng/common/templates-official/variables/pool-providers.yml +# +# ... then anywhere specifying the pool provider use the runtime variables, +# $(DncEngInternalBuildPool) +# +# pool: +# name: $(DncEngInternalBuildPool) +# image: 1es-windows-2022 + +variables: + # Coalesce the target and source branches so we know when a PR targets a release branch + # If these variables are somehow missing, fall back to main (tends to have more capacity) + + # Any new -Svc alternative pools should have variables added here to allow for splitting work + + - name: DncEngInternalBuildPool + value: $[ + replace( + replace( + eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), + True, + 'NetCore1ESPool-Svc-Internal' + ), + False, + 'NetCore1ESPool-Internal' + ) + ] \ No newline at end of file diff --git a/src/arcade/eng/common/templates-official/variables/sdl-variables.yml b/src/arcade/eng/common/templates-official/variables/sdl-variables.yml new file mode 100644 index 000000000000..dbdd66d4a4b3 --- /dev/null +++ b/src/arcade/eng/common/templates-official/variables/sdl-variables.yml @@ -0,0 +1,7 @@ +variables: +# The Guardian version specified in 'eng/common/sdl/packages.config'. This value must be kept in +# sync with the packages.config file. +- name: DefaultGuardianVersion + value: 0.109.0 +- name: GuardianPackagesConfigFile + value: $(Build.SourcesDirectory)\eng\common\sdl\packages.config \ No newline at end of file diff --git a/src/arcade/eng/common/templates/job/onelocbuild.yml b/src/arcade/eng/common/templates/job/onelocbuild.yml new file mode 100644 index 000000000000..ff829dc4c700 --- /dev/null +++ b/src/arcade/eng/common/templates/job/onelocbuild.yml @@ -0,0 +1,7 @@ +jobs: +- template: /eng/common/core-templates/job/onelocbuild.yml + parameters: + is1ESPipeline: false + + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} diff --git a/src/arcade/eng/common/templates/job/publish-build-assets.yml b/src/arcade/eng/common/templates/job/publish-build-assets.yml new file mode 100644 index 000000000000..ab2edec2adb5 --- /dev/null +++ b/src/arcade/eng/common/templates/job/publish-build-assets.yml @@ -0,0 +1,7 @@ +jobs: +- template: /eng/common/core-templates/job/publish-build-assets.yml + parameters: + is1ESPipeline: false + + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} diff --git a/src/arcade/eng/common/templates/job/source-build.yml b/src/arcade/eng/common/templates/job/source-build.yml new file mode 100644 index 000000000000..e44d47b1d760 --- /dev/null +++ b/src/arcade/eng/common/templates/job/source-build.yml @@ -0,0 +1,7 @@ +jobs: +- template: /eng/common/core-templates/job/source-build.yml + parameters: + is1ESPipeline: false + + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} diff --git a/src/arcade/eng/common/templates/job/source-index-stage1.yml b/src/arcade/eng/common/templates/job/source-index-stage1.yml new file mode 100644 index 000000000000..89f3291593cb --- /dev/null +++ b/src/arcade/eng/common/templates/job/source-index-stage1.yml @@ -0,0 +1,7 @@ +jobs: +- template: /eng/common/core-templates/job/source-index-stage1.yml + parameters: + is1ESPipeline: false + + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} diff --git a/src/arcade/eng/common/templates/jobs/codeql-build.yml b/src/arcade/eng/common/templates/jobs/codeql-build.yml new file mode 100644 index 000000000000..517f24d6a52c --- /dev/null +++ b/src/arcade/eng/common/templates/jobs/codeql-build.yml @@ -0,0 +1,7 @@ +jobs: +- template: /eng/common/core-templates/jobs/codeql-build.yml + parameters: + is1ESPipeline: false + + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} diff --git a/src/arcade/eng/common/templates/jobs/jobs.yml b/src/arcade/eng/common/templates/jobs/jobs.yml new file mode 100644 index 000000000000..388e9037b3e6 --- /dev/null +++ b/src/arcade/eng/common/templates/jobs/jobs.yml @@ -0,0 +1,7 @@ +jobs: +- template: /eng/common/core-templates/jobs/jobs.yml + parameters: + is1ESPipeline: false + + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} diff --git a/src/arcade/eng/common/templates/jobs/source-build.yml b/src/arcade/eng/common/templates/jobs/source-build.yml new file mode 100644 index 000000000000..818d4c326dbb --- /dev/null +++ b/src/arcade/eng/common/templates/jobs/source-build.yml @@ -0,0 +1,7 @@ +jobs: +- template: /eng/common/core-templates/jobs/source-build.yml + parameters: + is1ESPipeline: false + + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} \ No newline at end of file diff --git a/src/arcade/eng/common/templates/post-build/common-variables.yml b/src/arcade/eng/common/templates/post-build/common-variables.yml new file mode 100644 index 000000000000..7fa105875592 --- /dev/null +++ b/src/arcade/eng/common/templates/post-build/common-variables.yml @@ -0,0 +1,8 @@ +variables: +- template: /eng/common/core-templates/post-build/common-variables.yml + parameters: + # Specifies whether to use 1ES + is1ESPipeline: false + + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} \ No newline at end of file diff --git a/src/arcade/eng/common/templates/post-build/post-build.yml b/src/arcade/eng/common/templates/post-build/post-build.yml new file mode 100644 index 000000000000..53ede714bdd2 --- /dev/null +++ b/src/arcade/eng/common/templates/post-build/post-build.yml @@ -0,0 +1,8 @@ +stages: +- template: /eng/common/core-templates/post-build/post-build.yml + parameters: + # Specifies whether to use 1ES + is1ESPipeline: false + + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} \ No newline at end of file diff --git a/src/arcade/eng/common/templates/post-build/setup-maestro-vars.yml b/src/arcade/eng/common/templates/post-build/setup-maestro-vars.yml new file mode 100644 index 000000000000..a79fab5b441e --- /dev/null +++ b/src/arcade/eng/common/templates/post-build/setup-maestro-vars.yml @@ -0,0 +1,8 @@ +steps: +- template: /eng/common/core-templates/post-build/setup-maestro-vars.yml + parameters: + # Specifies whether to use 1ES + is1ESPipeline: false + + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} \ No newline at end of file diff --git a/src/arcade/eng/common/templates/steps/component-governance.yml b/src/arcade/eng/common/templates/steps/component-governance.yml new file mode 100644 index 000000000000..c12a5f8d21d7 --- /dev/null +++ b/src/arcade/eng/common/templates/steps/component-governance.yml @@ -0,0 +1,7 @@ +steps: +- template: /eng/common/core-templates/steps/component-governance.yml + parameters: + is1ESPipeline: false + + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} diff --git a/src/arcade/eng/common/templates/steps/enable-internal-runtimes.yml b/src/arcade/eng/common/templates/steps/enable-internal-runtimes.yml new file mode 100644 index 000000000000..b21a8038cc1c --- /dev/null +++ b/src/arcade/eng/common/templates/steps/enable-internal-runtimes.yml @@ -0,0 +1,10 @@ +# Obtains internal runtime download credentials and populates the 'dotnetbuilds-internal-container-read-token-base64' +# variable with the base64-encoded SAS token, by default + +steps: +- template: /eng/common/core-templates/steps/enable-internal-runtimes.yml + parameters: + is1ESPipeline: false + + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} diff --git a/src/arcade/eng/common/templates/steps/enable-internal-sources.yml b/src/arcade/eng/common/templates/steps/enable-internal-sources.yml new file mode 100644 index 000000000000..5f87e9abb8aa --- /dev/null +++ b/src/arcade/eng/common/templates/steps/enable-internal-sources.yml @@ -0,0 +1,7 @@ +steps: +- template: /eng/common/core-templates/steps/enable-internal-sources.yml + parameters: + is1ESPipeline: false + + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} \ No newline at end of file diff --git a/src/arcade/eng/common/templates/steps/generate-sbom.yml b/src/arcade/eng/common/templates/steps/generate-sbom.yml new file mode 100644 index 000000000000..26dc00a2e0f3 --- /dev/null +++ b/src/arcade/eng/common/templates/steps/generate-sbom.yml @@ -0,0 +1,7 @@ +steps: +- template: /eng/common/core-templates/steps/generate-sbom.yml + parameters: + is1ESPipeline: false + + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} diff --git a/src/arcade/eng/common/templates/steps/get-delegation-sas.yml b/src/arcade/eng/common/templates/steps/get-delegation-sas.yml new file mode 100644 index 000000000000..83760c9798e3 --- /dev/null +++ b/src/arcade/eng/common/templates/steps/get-delegation-sas.yml @@ -0,0 +1,7 @@ +steps: +- template: /eng/common/core-templates/steps/get-delegation-sas.yml + parameters: + is1ESPipeline: false + + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} diff --git a/src/arcade/eng/common/templates/steps/get-federated-access-token.yml b/src/arcade/eng/common/templates/steps/get-federated-access-token.yml new file mode 100644 index 000000000000..31e151d9d9e7 --- /dev/null +++ b/src/arcade/eng/common/templates/steps/get-federated-access-token.yml @@ -0,0 +1,7 @@ +steps: +- template: /eng/common/core-templates/steps/get-federated-access-token.yml + parameters: + is1ESPipeline: false + + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} \ No newline at end of file diff --git a/src/arcade/eng/common/templates/steps/publish-logs.yml b/src/arcade/eng/common/templates/steps/publish-logs.yml new file mode 100644 index 000000000000..4ea86bd88235 --- /dev/null +++ b/src/arcade/eng/common/templates/steps/publish-logs.yml @@ -0,0 +1,7 @@ +steps: +- template: /eng/common/core-templates/steps/publish-logs.yml + parameters: + is1ESPipeline: false + + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} diff --git a/src/arcade/eng/common/templates/steps/publish-pipeline-artifacts.yml b/src/arcade/eng/common/templates/steps/publish-pipeline-artifacts.yml new file mode 100644 index 000000000000..5dd698b212fc --- /dev/null +++ b/src/arcade/eng/common/templates/steps/publish-pipeline-artifacts.yml @@ -0,0 +1,34 @@ +parameters: +- name: is1ESPipeline + type: boolean + default: false + +- name: args + type: object + default: {} + +steps: +- ${{ if eq(parameters.is1ESPipeline, true) }}: + - 'eng/common/templates cannot be referenced from a 1ES managed template': error +- task: PublishPipelineArtifact@1 + displayName: ${{ coalesce(parameters.args.displayName, 'Publish to Build Artifact') }} + ${{ if parameters.args.condition }}: + condition: ${{ parameters.args.condition }} + ${{ else }}: + condition: succeeded() + ${{ if parameters.args.continueOnError }}: + continueOnError: ${{ parameters.args.continueOnError }} + inputs: + targetPath: ${{ parameters.args.targetPath }} + ${{ if parameters.args.artifactName }}: + artifactName: ${{ parameters.args.artifactName }} + ${{ if parameters.args.publishLocation }}: + publishLocation: ${{ parameters.args.publishLocation }} + ${{ if parameters.args.fileSharePath }}: + fileSharePath: ${{ parameters.args.fileSharePath }} + ${{ if parameters.args.Parallel }}: + parallel: ${{ parameters.args.Parallel }} + ${{ if parameters.args.parallelCount }}: + parallelCount: ${{ parameters.args.parallelCount }} + ${{ if parameters.args.properties }}: + properties: ${{ parameters.args.properties }} \ No newline at end of file diff --git a/src/arcade/eng/common/templates/steps/retain-build.yml b/src/arcade/eng/common/templates/steps/retain-build.yml new file mode 100644 index 000000000000..8e841ace3d29 --- /dev/null +++ b/src/arcade/eng/common/templates/steps/retain-build.yml @@ -0,0 +1,7 @@ +steps: +- template: /eng/common/core-templates/steps/retain-build.yml + parameters: + is1ESPipeline: false + + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} diff --git a/src/arcade/eng/common/templates/steps/send-to-helix.yml b/src/arcade/eng/common/templates/steps/send-to-helix.yml new file mode 100644 index 000000000000..39f99fc2762d --- /dev/null +++ b/src/arcade/eng/common/templates/steps/send-to-helix.yml @@ -0,0 +1,7 @@ +steps: +- template: /eng/common/core-templates/steps/send-to-helix.yml + parameters: + is1ESPipeline: false + + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} diff --git a/src/arcade/eng/common/templates/steps/source-build.yml b/src/arcade/eng/common/templates/steps/source-build.yml new file mode 100644 index 000000000000..23c1d6f4e9f8 --- /dev/null +++ b/src/arcade/eng/common/templates/steps/source-build.yml @@ -0,0 +1,7 @@ +steps: +- template: /eng/common/core-templates/steps/source-build.yml + parameters: + is1ESPipeline: false + + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} diff --git a/src/arcade/eng/common/templates/variables/pool-providers.yml b/src/arcade/eng/common/templates/variables/pool-providers.yml new file mode 100644 index 000000000000..e0b19c14a073 --- /dev/null +++ b/src/arcade/eng/common/templates/variables/pool-providers.yml @@ -0,0 +1,59 @@ +# Select a pool provider based off branch name. Anything with branch name containing 'release' must go into an -Svc pool, +# otherwise it should go into the "normal" pools. This separates out the queueing and billing of released branches. + +# Motivation: +# Once a given branch of a repository's output has been officially "shipped" once, it is then considered to be COGS +# (Cost of goods sold) and should be moved to a servicing pool provider. This allows both separation of queueing +# (allowing release builds and main PR builds to not intefere with each other) and billing (required for COGS. +# Additionally, the pool provider name itself may be subject to change when the .NET Core Engineering Services +# team needs to move resources around and create new and potentially differently-named pools. Using this template +# file from an Arcade-ified repo helps guard against both having to update one's release/* branches and renaming. + +# How to use: +# This yaml assumes your shipped product branches use the naming convention "release/..." (which many do). +# If we find alternate naming conventions in broad usage it can be added to the condition below. +# +# First, import the template in an arcade-ified repo to pick up the variables, e.g.: +# +# variables: +# - template: /eng/common/templates/variables/pool-providers.yml +# +# ... then anywhere specifying the pool provider use the runtime variables, +# $(DncEngInternalBuildPool) and $ (DncEngPublicBuildPool), e.g.: +# +# pool: +# name: $(DncEngInternalBuildPool) +# demands: ImageOverride -equals windows.vs2019.amd64 +variables: + - ${{ if eq(variables['System.TeamProject'], 'internal') }}: + - template: /eng/common/templates-official/variables/pool-providers.yml + - ${{ else }}: + # Coalesce the target and source branches so we know when a PR targets a release branch + # If these variables are somehow missing, fall back to main (tends to have more capacity) + + # Any new -Svc alternative pools should have variables added here to allow for splitting work + - name: DncEngPublicBuildPool + value: $[ + replace( + replace( + eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), + True, + 'NetCore-Svc-Public' + ), + False, + 'NetCore-Public' + ) + ] + + - name: DncEngInternalBuildPool + value: $[ + replace( + replace( + eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), + True, + 'NetCore1ESPool-Svc-Internal' + ), + False, + 'NetCore1ESPool-Internal' + ) + ] From 575d30d32c7399985c8c28b92a0bcc9760003d0d Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Wed, 14 May 2025 09:50:54 +0000 Subject: [PATCH 44/57] Update dependencies from https://github.com/dotnet/scenario-tests build 20250514.1 Microsoft.SourceBuild.Intermediate.scenario-tests , Microsoft.DotNet.ScenarioTests.SdkTemplateTests From Version 9.0.0-preview.25257.1 -> To Version 9.0.0-preview.25264.1 --- eng/Version.Details.xml | 8 ++++---- eng/Versions.props | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index e8370ec1547b..4d3530be1ffe 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -631,14 +631,14 @@ https://github.com/dotnet/arcade-services 47e3672c762970073e4282bd563233da86bcca3e
- + https://github.com/dotnet/scenario-tests - db1f9548f1e32819b3d546faf1ba7d3fa6880ae7 + 45b878678cfbde6c460450ed1699de75a20d6e87 - + https://github.com/dotnet/scenario-tests - db1f9548f1e32819b3d546faf1ba7d3fa6880ae7 + 45b878678cfbde6c460450ed1699de75a20d6e87 From c50e4010f68e030743c64cc4422d7f9cffb78785 Mon Sep 17 00:00:00 2001 From: Matt Mitchell Date: Wed, 14 May 2025 09:59:34 -0700 Subject: [PATCH 45/57] Change ci.yml -> sb-ci.yml (#48953) --- .../eng/pipelines/{ci.yml => sb-ci.yml} | 33 +++++-------------- 1 file changed, 8 insertions(+), 25 deletions(-) rename src/SourceBuild/content/eng/pipelines/{ci.yml => sb-ci.yml} (69%) diff --git a/src/SourceBuild/content/eng/pipelines/ci.yml b/src/SourceBuild/content/eng/pipelines/sb-ci.yml similarity index 69% rename from src/SourceBuild/content/eng/pipelines/ci.yml rename to src/SourceBuild/content/eng/pipelines/sb-ci.yml index 16447ee3d731..3f8117714f50 100644 --- a/src/SourceBuild/content/eng/pipelines/ci.yml +++ b/src/SourceBuild/content/eng/pipelines/sb-ci.yml @@ -1,33 +1,16 @@ # This yml is used by these pipelines and triggers: -# NOTE: the triggers are defined in the Azure DevOps UI as they are too complex -# -# - dotnet-source-build (public) -# https://dev.azure.com/dnceng-public/public/_build?definitionId=240 -# - PR: ultralite build -# - CI: release/*, every batched commit, full build -# - Schedule: main only, full build -# -# - dotnet-unified-build (public) -# https://dev.azure.com/dnceng-public/public/_build?definitionId=278 -# - PR: lite build -# - CI: release/* only, every batched commit, full build -# - Schedule: main only, full build # # - dotnet-source-build (internal) # https://dev.azure.com/dnceng/internal/_build?definitionId=1219 -# - PR: ultralite build # - CI: release/* and internal/release/* only, every batched commit, full build -# - Schedule: main only, full build -# -# - dotnet-source-build-lite (internal) -# https://dev.azure.com/dnceng/internal/_build?definitionId=1299 -# - PR: release/* and main only, lite build, on-demand trigger -# - CI: main only, every batched commit, lite build -# -# - dotnet-unified-build (internal) -# https://dev.azure.com/dnceng/internal/_build?definitionId=1330 -# - PR: lite build -# - CI: release/*, internal/release/* and main only, every batched commit, full build + +trigger: + batch: true + branches: + include: + - main + - release/* + - internal/release/* variables: # enable source-only build for pipelines that contain the -source-build string From 63dab0db3653d92a8d7bac79f83258c1e45f85b9 Mon Sep 17 00:00:00 2001 From: Matt Mitchell Date: Wed, 14 May 2025 10:50:29 -0700 Subject: [PATCH 46/57] Update triggers to only target release/9.0.1xx (#48958) --- src/SourceBuild/content/eng/pipelines/sb-ci.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/SourceBuild/content/eng/pipelines/sb-ci.yml b/src/SourceBuild/content/eng/pipelines/sb-ci.yml index 3f8117714f50..752af447c879 100644 --- a/src/SourceBuild/content/eng/pipelines/sb-ci.yml +++ b/src/SourceBuild/content/eng/pipelines/sb-ci.yml @@ -8,9 +8,8 @@ trigger: batch: true branches: include: - - main - - release/* - - internal/release/* + - release/9.0.1xx + - internal/release/9.0.1xx variables: # enable source-only build for pipelines that contain the -source-build string From 60c5322757276126f3d7a612089502d636ac7cd6 Mon Sep 17 00:00:00 2001 From: Jason Zhai Date: Wed, 14 May 2025 19:57:33 -0700 Subject: [PATCH 47/57] Revert dotnet-templating feed --- NuGet.config | 1 + 1 file changed, 1 insertion(+) diff --git a/NuGet.config b/NuGet.config index fd2c5f1f2511..61e794b48bc8 100644 --- a/NuGet.config +++ b/NuGet.config @@ -47,6 +47,7 @@ + From f584fdabc48a880a89881f135707f626c2e315f3 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Thu, 15 May 2025 03:05:34 +0000 Subject: [PATCH 48/57] Update dependencies from https://github.com/dotnet/templating build 20250514.3 Microsoft.SourceBuild.Intermediate.templating , Microsoft.TemplateEngine.Abstractions , Microsoft.TemplateEngine.Mocks From Version 9.0.107-servicing.25258.2 -> To Version 9.0.107-servicing.25264.3 --- NuGet.config | 2 +- eng/Version.Details.xml | 10 +++++----- eng/Versions.props | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/NuGet.config b/NuGet.config index e6c5a0ccc913..4909569c98c6 100644 --- a/NuGet.config +++ b/NuGet.config @@ -35,7 +35,7 @@ - + diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 4d3530be1ffe..bd82cabcf754 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,16 +3,16 @@ https://github.com/dotnet/templating - 0c96b36ccec4428bbfdeafae392bda49c1667361 + c671e3de121ce56d69ccf5433f6a7f3ad1357983 - + https://github.com/dotnet/templating - 0c96b36ccec4428bbfdeafae392bda49c1667361 + c671e3de121ce56d69ccf5433f6a7f3ad1357983 - + https://github.com/dotnet/templating - 0c96b36ccec4428bbfdeafae392bda49c1667361 + c671e3de121ce56d69ccf5433f6a7f3ad1357983 diff --git a/eng/Versions.props b/eng/Versions.props index abeabc81de31..bf34590c2ebb 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -193,7 +193,7 @@ $(MicrosoftTemplateEngineAbstractionsPackageVersion) $(MicrosoftTemplateEngineAbstractionsPackageVersion) - 9.0.107-servicing.25258.2 + 9.0.107-servicing.25264.3 $(MicrosoftTemplateEngineMocksPackageVersion) $(MicrosoftTemplateEngineAbstractionsPackageVersion) $(MicrosoftTemplateEngineMocksPackageVersion) From 0e565b821aface5cf57fc21c840b89e56c0223c5 Mon Sep 17 00:00:00 2001 From: Matt Thalman Date: Thu, 15 May 2025 07:42:41 -0500 Subject: [PATCH 49/57] Update msbuild URI --- eng/Version.Details.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 82ed2eb4a03f..8e20d150b45f 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -78,7 +78,7 @@ 13c590f8d138d6a30b899a5172cc6ec494db930b - https://dev.azure.com/devdiv/DevDiv/_git/DotNet-msbuild-Trusted + https://github.com/dotnet/msbuild 13c590f8d138d6a30b899a5172cc6ec494db930b From 8cd51da5293e3e61a8b8b9b8262470999d0f2787 Mon Sep 17 00:00:00 2001 From: Matt Thalman Date: Thu, 15 May 2025 08:24:21 -0500 Subject: [PATCH 50/57] Fixing all instances of msbuild URI --- eng/Version.Details.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 8e20d150b45f..7691edc95a74 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -74,7 +74,7 @@ - https://dev.azure.com/devdiv/DevDiv/_git/DotNet-msbuild-Trusted + https://github.com/dotnet/msbuild 13c590f8d138d6a30b899a5172cc6ec494db930b @@ -83,7 +83,7 @@ - https://dev.azure.com/devdiv/DevDiv/_git/DotNet-msbuild-Trusted + https://github.com/dotnet/msbuild 13c590f8d138d6a30b899a5172cc6ec494db930b From caadbad289fd9b57e1489813edb7032bb1953f82 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Fri, 16 May 2025 02:36:29 +0000 Subject: [PATCH 51/57] Update dependencies from https://github.com/dotnet/templating build 20250515.4 Microsoft.SourceBuild.Intermediate.templating , Microsoft.TemplateEngine.Abstractions , Microsoft.TemplateEngine.Mocks From Version 9.0.107-servicing.25258.2 -> To Version 9.0.107-servicing.25265.4 --- NuGet.config | 2 +- eng/Version.Details.xml | 10 +++++----- eng/Versions.props | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/NuGet.config b/NuGet.config index 4909569c98c6..4153f23ac547 100644 --- a/NuGet.config +++ b/NuGet.config @@ -35,7 +35,7 @@ - + diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index bd82cabcf754..ad1a7f1fb51e 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,16 +3,16 @@ https://github.com/dotnet/templating - c671e3de121ce56d69ccf5433f6a7f3ad1357983 + 27b4c178f0891417129396ecb8f695200fcd1d50 - + https://github.com/dotnet/templating - c671e3de121ce56d69ccf5433f6a7f3ad1357983 + 27b4c178f0891417129396ecb8f695200fcd1d50 - + https://github.com/dotnet/templating - c671e3de121ce56d69ccf5433f6a7f3ad1357983 + 27b4c178f0891417129396ecb8f695200fcd1d50 diff --git a/eng/Versions.props b/eng/Versions.props index bf34590c2ebb..3cf9a967a6e2 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -193,7 +193,7 @@ $(MicrosoftTemplateEngineAbstractionsPackageVersion) $(MicrosoftTemplateEngineAbstractionsPackageVersion) - 9.0.107-servicing.25264.3 + 9.0.107-servicing.25265.4 $(MicrosoftTemplateEngineMocksPackageVersion) $(MicrosoftTemplateEngineAbstractionsPackageVersion) $(MicrosoftTemplateEngineMocksPackageVersion) From 8556b6cf2ea4f42a122841d1cb96a34d23225954 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Fri, 16 May 2025 12:44:22 +0000 Subject: [PATCH 52/57] Update dependencies from https://github.com/dotnet/scenario-tests build 20250516.2 Microsoft.SourceBuild.Intermediate.scenario-tests , Microsoft.DotNet.ScenarioTests.SdkTemplateTests From Version 9.0.0-preview.25264.1 -> To Version 9.0.0-preview.25266.2 --- NuGet.config | 35 ++++++++++++++++++++++++++++------- eng/Version.Details.xml | 8 ++++---- eng/Versions.props | 2 +- 3 files changed, 33 insertions(+), 12 deletions(-) diff --git a/NuGet.config b/NuGet.config index 61e794b48bc8..46ad5fc2253f 100644 --- a/NuGet.config +++ b/NuGet.config @@ -25,6 +25,10 @@ + + + + @@ -32,15 +36,22 @@ + + + + - + + + + @@ -48,14 +59,13 @@ - - - - - + + + + @@ -86,9 +96,12 @@ + + + + - @@ -97,6 +110,10 @@ + + + + @@ -104,6 +121,10 @@ + + + + diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 7691edc95a74..0b9e4629ea2f 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -631,14 +631,14 @@ https://github.com/dotnet/arcade-services 47e3672c762970073e4282bd563233da86bcca3e - + https://github.com/dotnet/scenario-tests - 45b878678cfbde6c460450ed1699de75a20d6e87 + a11e0d6f227feb7d0e2b619b48556d041237e59c - + https://github.com/dotnet/scenario-tests - 45b878678cfbde6c460450ed1699de75a20d6e87 + a11e0d6f227feb7d0e2b619b48556d041237e59c From 546d11abc748c10683200fe0f8fc0b1beb3051c6 Mon Sep 17 00:00:00 2001 From: ".NET Source-Build Bot" <102560831+dotnet-sb-bot@users.noreply.github.com> Date: Fri, 16 May 2025 10:02:46 -0500 Subject: [PATCH 53/57] .NET Source-Build 9.0.106 May 2025 Updates (#48919) Co-authored-by: Matt Thalman --- src/SourceBuild/content/eng/Version.Details.xml | 4 ++-- src/SourceBuild/content/eng/Versions.props | 4 ++-- src/SourceBuild/content/global.json | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/SourceBuild/content/eng/Version.Details.xml b/src/SourceBuild/content/eng/Version.Details.xml index afa91a48031d..07f6d37a92bd 100644 --- a/src/SourceBuild/content/eng/Version.Details.xml +++ b/src/SourceBuild/content/eng/Version.Details.xml @@ -2,9 +2,9 @@ - + https://github.com/dotnet/arcade - 5da211e1c42254cb35e7ef3d5a8428fb24853169 + aa61e8c20a869bcc994f8b29eb07d927d2bec6f4 diff --git a/src/SourceBuild/content/eng/Versions.props b/src/SourceBuild/content/eng/Versions.props index ec033700d78b..28e2abbaf298 100644 --- a/src/SourceBuild/content/eng/Versions.props +++ b/src/SourceBuild/content/eng/Versions.props @@ -23,8 +23,8 @@ of a .NET major or minor release, prebuilts may be needed. When the release is mature, prebuilts are not necessary, and this property is removed from the file. --> - 9.0.105 - 9.0.105-servicing.25165.1 + 9.0.106 + 9.0.106-servicing.25230.1 2.0.0-beta4.24126.1 diff --git a/src/SourceBuild/content/global.json b/src/SourceBuild/content/global.json index 28b3d0048c19..26e300e7646d 100644 --- a/src/SourceBuild/content/global.json +++ b/src/SourceBuild/content/global.json @@ -1,10 +1,10 @@ { "tools": { - "dotnet": "9.0.105" + "dotnet": "9.0.106" }, "msbuild-sdks": { "Microsoft.Build.NoTargets": "3.7.0", "Microsoft.Build.Traversal": "3.4.0", - "Microsoft.DotNet.Arcade.Sdk": "9.0.0-beta.25111.5" + "Microsoft.DotNet.Arcade.Sdk": "9.0.0-beta.25208.6" } } From df4c4064c79fa2fb245ba7a88a1c16229e057241 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Fri, 16 May 2025 15:06:45 +0000 Subject: [PATCH 54/57] Update dependencies from https://github.com/dotnet/source-build-reference-packages build 20250516.1 Microsoft.SourceBuild.Intermediate.source-build-reference-packages From Version 9.0.0-alpha.1.25223.3 -> To Version 9.0.0-alpha.1.25266.1 --- NuGet.config | 35 ++++++++++++++++++++++++++++------- eng/Version.Details.xml | 4 ++-- 2 files changed, 30 insertions(+), 9 deletions(-) diff --git a/NuGet.config b/NuGet.config index 61e794b48bc8..46ad5fc2253f 100644 --- a/NuGet.config +++ b/NuGet.config @@ -25,6 +25,10 @@ + + + + @@ -32,15 +36,22 @@ + + + + - + + + + @@ -48,14 +59,13 @@ - - - - - + + + + @@ -86,9 +96,12 @@ + + + + - @@ -97,6 +110,10 @@ + + + + @@ -104,6 +121,10 @@ + + + + diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 7691edc95a74..3ae2715f27f5 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -451,9 +451,9 @@ - + https://github.com/dotnet/source-build-reference-packages - 19eb5ea4e5f9c4e5256843a92805c8c9e942207d + 2bdf6b694572c45f6249c5406ae2303d678cda3f From a65b977271954d3e075f5cc0b1e5baeb8b4df287 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Mon, 19 May 2025 02:57:15 +0000 Subject: [PATCH 55/57] Update dependencies from https://github.com/dotnet/templating build 20250518.3 Microsoft.SourceBuild.Intermediate.templating , Microsoft.TemplateEngine.Abstractions , Microsoft.TemplateEngine.Mocks From Version 9.0.107-servicing.25258.2 -> To Version 9.0.107-servicing.25268.3 --- NuGet.config | 2 +- eng/Version.Details.xml | 10 +++++----- eng/Versions.props | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/NuGet.config b/NuGet.config index 70d46a3ad1bd..54496f8b8404 100644 --- a/NuGet.config +++ b/NuGet.config @@ -58,7 +58,7 @@ - + diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index e1e329819121..c30e7381e301 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,16 +3,16 @@ https://github.com/dotnet/templating - 27b4c178f0891417129396ecb8f695200fcd1d50 + 54d8e4f724f10c1605c7062c50e88a8e74a069e3 - + https://github.com/dotnet/templating - 27b4c178f0891417129396ecb8f695200fcd1d50 + 54d8e4f724f10c1605c7062c50e88a8e74a069e3 - + https://github.com/dotnet/templating - 27b4c178f0891417129396ecb8f695200fcd1d50 + 54d8e4f724f10c1605c7062c50e88a8e74a069e3 diff --git a/eng/Versions.props b/eng/Versions.props index 7a10eca1b5cf..dbc461adc72c 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -193,7 +193,7 @@ $(MicrosoftTemplateEngineAbstractionsPackageVersion) $(MicrosoftTemplateEngineAbstractionsPackageVersion) - 9.0.107-servicing.25265.4 + 9.0.107-servicing.25268.3 $(MicrosoftTemplateEngineMocksPackageVersion) $(MicrosoftTemplateEngineAbstractionsPackageVersion) $(MicrosoftTemplateEngineMocksPackageVersion) From 859166d347b99700765bbf4772438fbf10ed1b8b Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Mon, 19 May 2025 03:47:25 +0000 Subject: [PATCH 56/57] Update dependencies from https://github.com/dotnet/templating build 20250518.8 Microsoft.SourceBuild.Intermediate.templating , Microsoft.TemplateEngine.Abstractions , Microsoft.TemplateEngine.Mocks From Version 9.0.107-servicing.25258.2 -> To Version 9.0.107-servicing.25268.8 --- NuGet.config | 2 +- eng/Version.Details.xml | 10 +++++----- eng/Versions.props | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/NuGet.config b/NuGet.config index 54496f8b8404..84b2454cf740 100644 --- a/NuGet.config +++ b/NuGet.config @@ -58,7 +58,7 @@ - + diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index c30e7381e301..e0a9e262626d 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,16 +3,16 @@ https://github.com/dotnet/templating - 54d8e4f724f10c1605c7062c50e88a8e74a069e3 + a2da3a8446409db003c2868a6910c0323b113467 - + https://github.com/dotnet/templating - 54d8e4f724f10c1605c7062c50e88a8e74a069e3 + a2da3a8446409db003c2868a6910c0323b113467 - + https://github.com/dotnet/templating - 54d8e4f724f10c1605c7062c50e88a8e74a069e3 + a2da3a8446409db003c2868a6910c0323b113467 diff --git a/eng/Versions.props b/eng/Versions.props index dbc461adc72c..86a541ecbe10 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -193,7 +193,7 @@ $(MicrosoftTemplateEngineAbstractionsPackageVersion) $(MicrosoftTemplateEngineAbstractionsPackageVersion) - 9.0.107-servicing.25268.3 + 9.0.107-servicing.25268.8 $(MicrosoftTemplateEngineMocksPackageVersion) $(MicrosoftTemplateEngineAbstractionsPackageVersion) $(MicrosoftTemplateEngineMocksPackageVersion) From f8e88ae44b1a6b3aacf4d30f6bc2650ebac184ce Mon Sep 17 00:00:00 2001 From: Jason Zhai Date: Mon, 19 May 2025 18:47:23 -0700 Subject: [PATCH 57/57] Delete src/arcade/eng/common directory --- .../build-configuration.json | 4 - .../eng/common/PSScriptAnalyzerSettings.psd1 | 11 - src/arcade/eng/common/README.md | 28 -- src/arcade/eng/common/SetupNugetSources.ps1 | 171 -------- src/arcade/eng/common/SetupNugetSources.sh | 167 -------- src/arcade/eng/common/build.cmd | 3 - .../common/core-templates/job/onelocbuild.yml | 121 ------ .../core-templates/jobs/source-build.yml | 58 --- .../post-build/common-variables.yml | 22 - .../post-build/setup-maestro-vars.yml | 74 ---- .../steps/component-governance.yml | 16 - .../steps/enable-internal-runtimes.yml | 32 -- .../steps/enable-internal-sources.yml | 47 -- .../steps/get-federated-access-token.yml | 42 -- .../steps/publish-build-artifacts.yml | 20 - .../steps/publish-pipeline-artifacts.yml | 20 - .../core-templates/steps/retain-build.yml | 28 -- .../core-templates/steps/send-to-helix.yml | 93 ---- .../variables/pool-providers.yml | 8 - .../eng/common/cross/arm/tizen/tizen.patch | 9 - .../eng/common/cross/armel/tizen/tizen.patch | 9 - .../common/cross/riscv64/tizen/tizen.patch | 9 - .../eng/common/cross/tizen-build-rootfs.sh | 82 ---- .../eng/common/cross/x64/tizen/tizen.patch | 9 - .../eng/common/cross/x86/tizen/tizen.patch | 9 - src/arcade/eng/common/darc-init.ps1 | 47 -- src/arcade/eng/common/darc-init.sh | 82 ---- src/arcade/eng/common/dotnet-install.cmd | 2 - src/arcade/eng/common/dotnet-install.ps1 | 28 -- src/arcade/eng/common/dotnet-install.sh | 94 ---- .../common/enable-cross-org-publishing.ps1 | 13 - src/arcade/eng/common/generate-locproject.ps1 | 189 --------- src/arcade/eng/common/generate-sbom-prep.ps1 | 29 -- src/arcade/eng/common/generate-sbom-prep.sh | 39 -- src/arcade/eng/common/helixpublish.proj | 27 -- src/arcade/eng/common/init-tools-native.cmd | 3 - src/arcade/eng/common/init-tools-native.ps1 | 203 --------- src/arcade/eng/common/init-tools-native.sh | 238 ----------- .../eng/common/internal-feed-operations.ps1 | 132 ------ .../eng/common/internal-feed-operations.sh | 141 ------ .../eng/common/internal/Directory.Build.props | 11 - src/arcade/eng/common/internal/NuGet.config | 7 - src/arcade/eng/common/internal/Tools.csproj | 22 - .../common/loc/P22DotNetHtmlLocalization.lss | 29 -- src/arcade/eng/common/msbuild.ps1 | 28 -- src/arcade/eng/common/msbuild.sh | 58 --- .../eng/common/native/CommonLibrary.psm1 | 401 ------------------ .../eng/common/native/common-library.sh | 172 -------- src/arcade/eng/common/native/init-compiler.sh | 146 ------- .../eng/common/native/init-distro-rid.sh | 110 ----- .../eng/common/native/init-os-and-arch.sh | 85 ---- .../eng/common/native/install-cmake-test.sh | 117 ----- src/arcade/eng/common/native/install-cmake.sh | 117 ----- src/arcade/eng/common/native/install-tool.ps1 | 132 ------ .../eng/common/pipeline-logging-functions.ps1 | 260 ------------ .../eng/common/pipeline-logging-functions.sh | 206 --------- .../post-build/check-channel-consistency.ps1 | 48 --- .../common/post-build/nuget-validation.ps1 | 22 - .../common/post-build/nuget-verification.ps1 | 121 ------ .../eng/common/post-build/redact-logs.ps1 | 89 ---- .../post-build/sourcelink-validation.ps1 | 327 -------------- .../common/post-build/symbols-validation.ps1 | 337 --------------- src/arcade/eng/common/retain-build.ps1 | 45 -- src/arcade/eng/common/sdl/NuGet.config | 18 - .../eng/common/sdl/configure-sdl-tool.ps1 | 130 ------ .../eng/common/sdl/execute-all-sdl-tools.ps1 | 167 -------- .../common/sdl/extract-artifact-archives.ps1 | 63 --- .../common/sdl/extract-artifact-packages.ps1 | 82 ---- src/arcade/eng/common/sdl/init-sdl.ps1 | 55 --- src/arcade/eng/common/sdl/packages.config | 4 - src/arcade/eng/common/sdl/run-sdl.ps1 | 49 --- src/arcade/eng/common/sdl/sdl.ps1 | 38 -- .../eng/common/sdl/trim-assets-version.ps1 | 75 ---- src/arcade/eng/common/template-guidance.md | 133 ------ .../templates-official/job/onelocbuild.yml | 7 - .../job/publish-build-assets.yml | 7 - .../templates-official/job/source-build.yml | 7 - .../job/source-index-stage1.yml | 7 - .../templates-official/jobs/codeql-build.yml | 7 - .../common/templates-official/jobs/jobs.yml | 7 - .../templates-official/jobs/source-build.yml | 7 - .../post-build/common-variables.yml | 8 - .../post-build/post-build.yml | 8 - .../post-build/setup-maestro-vars.yml | 8 - .../steps/component-governance.yml | 7 - .../steps/enable-internal-runtimes.yml | 9 - .../steps/enable-internal-sources.yml | 7 - .../steps/generate-sbom.yml | 7 - .../steps/get-delegation-sas.yml | 7 - .../steps/get-federated-access-token.yml | 7 - .../templates-official/steps/publish-logs.yml | 7 - .../steps/publish-pipeline-artifacts.yml | 28 -- .../templates-official/steps/retain-build.yml | 7 - .../steps/send-to-helix.yml | 7 - .../templates-official/steps/source-build.yml | 7 - .../variables/pool-providers.yml | 45 -- .../variables/sdl-variables.yml | 7 - .../eng/common/templates/job/onelocbuild.yml | 7 - .../templates/job/publish-build-assets.yml | 7 - .../eng/common/templates/job/source-build.yml | 7 - .../templates/job/source-index-stage1.yml | 7 - .../common/templates/jobs/codeql-build.yml | 7 - src/arcade/eng/common/templates/jobs/jobs.yml | 7 - .../common/templates/jobs/source-build.yml | 7 - .../templates/post-build/common-variables.yml | 8 - .../templates/post-build/post-build.yml | 8 - .../post-build/setup-maestro-vars.yml | 8 - .../templates/steps/component-governance.yml | 7 - .../steps/enable-internal-runtimes.yml | 10 - .../steps/enable-internal-sources.yml | 7 - .../common/templates/steps/generate-sbom.yml | 7 - .../templates/steps/get-delegation-sas.yml | 7 - .../steps/get-federated-access-token.yml | 7 - .../common/templates/steps/publish-logs.yml | 7 - .../steps/publish-pipeline-artifacts.yml | 34 -- .../common/templates/steps/retain-build.yml | 7 - .../common/templates/steps/send-to-helix.yml | 7 - .../common/templates/steps/source-build.yml | 7 - .../templates/variables/pool-providers.yml | 59 --- 119 files changed, 6527 deletions(-) delete mode 100644 src/arcade/eng/common/BuildConfiguration/build-configuration.json delete mode 100644 src/arcade/eng/common/PSScriptAnalyzerSettings.psd1 delete mode 100644 src/arcade/eng/common/README.md delete mode 100644 src/arcade/eng/common/SetupNugetSources.ps1 delete mode 100644 src/arcade/eng/common/SetupNugetSources.sh delete mode 100644 src/arcade/eng/common/build.cmd delete mode 100644 src/arcade/eng/common/core-templates/job/onelocbuild.yml delete mode 100644 src/arcade/eng/common/core-templates/jobs/source-build.yml delete mode 100644 src/arcade/eng/common/core-templates/post-build/common-variables.yml delete mode 100644 src/arcade/eng/common/core-templates/post-build/setup-maestro-vars.yml delete mode 100644 src/arcade/eng/common/core-templates/steps/component-governance.yml delete mode 100644 src/arcade/eng/common/core-templates/steps/enable-internal-runtimes.yml delete mode 100644 src/arcade/eng/common/core-templates/steps/enable-internal-sources.yml delete mode 100644 src/arcade/eng/common/core-templates/steps/get-federated-access-token.yml delete mode 100644 src/arcade/eng/common/core-templates/steps/publish-build-artifacts.yml delete mode 100644 src/arcade/eng/common/core-templates/steps/publish-pipeline-artifacts.yml delete mode 100644 src/arcade/eng/common/core-templates/steps/retain-build.yml delete mode 100644 src/arcade/eng/common/core-templates/steps/send-to-helix.yml delete mode 100644 src/arcade/eng/common/core-templates/variables/pool-providers.yml delete mode 100644 src/arcade/eng/common/cross/arm/tizen/tizen.patch delete mode 100644 src/arcade/eng/common/cross/armel/tizen/tizen.patch delete mode 100644 src/arcade/eng/common/cross/riscv64/tizen/tizen.patch delete mode 100644 src/arcade/eng/common/cross/tizen-build-rootfs.sh delete mode 100644 src/arcade/eng/common/cross/x64/tizen/tizen.patch delete mode 100644 src/arcade/eng/common/cross/x86/tizen/tizen.patch delete mode 100644 src/arcade/eng/common/darc-init.ps1 delete mode 100644 src/arcade/eng/common/darc-init.sh delete mode 100644 src/arcade/eng/common/dotnet-install.cmd delete mode 100644 src/arcade/eng/common/dotnet-install.ps1 delete mode 100644 src/arcade/eng/common/dotnet-install.sh delete mode 100644 src/arcade/eng/common/enable-cross-org-publishing.ps1 delete mode 100644 src/arcade/eng/common/generate-locproject.ps1 delete mode 100644 src/arcade/eng/common/generate-sbom-prep.ps1 delete mode 100644 src/arcade/eng/common/generate-sbom-prep.sh delete mode 100644 src/arcade/eng/common/helixpublish.proj delete mode 100644 src/arcade/eng/common/init-tools-native.cmd delete mode 100644 src/arcade/eng/common/init-tools-native.ps1 delete mode 100644 src/arcade/eng/common/init-tools-native.sh delete mode 100644 src/arcade/eng/common/internal-feed-operations.ps1 delete mode 100644 src/arcade/eng/common/internal-feed-operations.sh delete mode 100644 src/arcade/eng/common/internal/Directory.Build.props delete mode 100644 src/arcade/eng/common/internal/NuGet.config delete mode 100644 src/arcade/eng/common/internal/Tools.csproj delete mode 100644 src/arcade/eng/common/loc/P22DotNetHtmlLocalization.lss delete mode 100644 src/arcade/eng/common/msbuild.ps1 delete mode 100644 src/arcade/eng/common/msbuild.sh delete mode 100644 src/arcade/eng/common/native/CommonLibrary.psm1 delete mode 100644 src/arcade/eng/common/native/common-library.sh delete mode 100644 src/arcade/eng/common/native/init-compiler.sh delete mode 100644 src/arcade/eng/common/native/init-distro-rid.sh delete mode 100644 src/arcade/eng/common/native/init-os-and-arch.sh delete mode 100644 src/arcade/eng/common/native/install-cmake-test.sh delete mode 100644 src/arcade/eng/common/native/install-cmake.sh delete mode 100644 src/arcade/eng/common/native/install-tool.ps1 delete mode 100644 src/arcade/eng/common/pipeline-logging-functions.ps1 delete mode 100644 src/arcade/eng/common/pipeline-logging-functions.sh delete mode 100644 src/arcade/eng/common/post-build/check-channel-consistency.ps1 delete mode 100644 src/arcade/eng/common/post-build/nuget-validation.ps1 delete mode 100644 src/arcade/eng/common/post-build/nuget-verification.ps1 delete mode 100644 src/arcade/eng/common/post-build/redact-logs.ps1 delete mode 100644 src/arcade/eng/common/post-build/sourcelink-validation.ps1 delete mode 100644 src/arcade/eng/common/post-build/symbols-validation.ps1 delete mode 100644 src/arcade/eng/common/retain-build.ps1 delete mode 100644 src/arcade/eng/common/sdl/NuGet.config delete mode 100644 src/arcade/eng/common/sdl/configure-sdl-tool.ps1 delete mode 100644 src/arcade/eng/common/sdl/execute-all-sdl-tools.ps1 delete mode 100644 src/arcade/eng/common/sdl/extract-artifact-archives.ps1 delete mode 100644 src/arcade/eng/common/sdl/extract-artifact-packages.ps1 delete mode 100644 src/arcade/eng/common/sdl/init-sdl.ps1 delete mode 100644 src/arcade/eng/common/sdl/packages.config delete mode 100644 src/arcade/eng/common/sdl/run-sdl.ps1 delete mode 100644 src/arcade/eng/common/sdl/sdl.ps1 delete mode 100644 src/arcade/eng/common/sdl/trim-assets-version.ps1 delete mode 100644 src/arcade/eng/common/template-guidance.md delete mode 100644 src/arcade/eng/common/templates-official/job/onelocbuild.yml delete mode 100644 src/arcade/eng/common/templates-official/job/publish-build-assets.yml delete mode 100644 src/arcade/eng/common/templates-official/job/source-build.yml delete mode 100644 src/arcade/eng/common/templates-official/job/source-index-stage1.yml delete mode 100644 src/arcade/eng/common/templates-official/jobs/codeql-build.yml delete mode 100644 src/arcade/eng/common/templates-official/jobs/jobs.yml delete mode 100644 src/arcade/eng/common/templates-official/jobs/source-build.yml delete mode 100644 src/arcade/eng/common/templates-official/post-build/common-variables.yml delete mode 100644 src/arcade/eng/common/templates-official/post-build/post-build.yml delete mode 100644 src/arcade/eng/common/templates-official/post-build/setup-maestro-vars.yml delete mode 100644 src/arcade/eng/common/templates-official/steps/component-governance.yml delete mode 100644 src/arcade/eng/common/templates-official/steps/enable-internal-runtimes.yml delete mode 100644 src/arcade/eng/common/templates-official/steps/enable-internal-sources.yml delete mode 100644 src/arcade/eng/common/templates-official/steps/generate-sbom.yml delete mode 100644 src/arcade/eng/common/templates-official/steps/get-delegation-sas.yml delete mode 100644 src/arcade/eng/common/templates-official/steps/get-federated-access-token.yml delete mode 100644 src/arcade/eng/common/templates-official/steps/publish-logs.yml delete mode 100644 src/arcade/eng/common/templates-official/steps/publish-pipeline-artifacts.yml delete mode 100644 src/arcade/eng/common/templates-official/steps/retain-build.yml delete mode 100644 src/arcade/eng/common/templates-official/steps/send-to-helix.yml delete mode 100644 src/arcade/eng/common/templates-official/steps/source-build.yml delete mode 100644 src/arcade/eng/common/templates-official/variables/pool-providers.yml delete mode 100644 src/arcade/eng/common/templates-official/variables/sdl-variables.yml delete mode 100644 src/arcade/eng/common/templates/job/onelocbuild.yml delete mode 100644 src/arcade/eng/common/templates/job/publish-build-assets.yml delete mode 100644 src/arcade/eng/common/templates/job/source-build.yml delete mode 100644 src/arcade/eng/common/templates/job/source-index-stage1.yml delete mode 100644 src/arcade/eng/common/templates/jobs/codeql-build.yml delete mode 100644 src/arcade/eng/common/templates/jobs/jobs.yml delete mode 100644 src/arcade/eng/common/templates/jobs/source-build.yml delete mode 100644 src/arcade/eng/common/templates/post-build/common-variables.yml delete mode 100644 src/arcade/eng/common/templates/post-build/post-build.yml delete mode 100644 src/arcade/eng/common/templates/post-build/setup-maestro-vars.yml delete mode 100644 src/arcade/eng/common/templates/steps/component-governance.yml delete mode 100644 src/arcade/eng/common/templates/steps/enable-internal-runtimes.yml delete mode 100644 src/arcade/eng/common/templates/steps/enable-internal-sources.yml delete mode 100644 src/arcade/eng/common/templates/steps/generate-sbom.yml delete mode 100644 src/arcade/eng/common/templates/steps/get-delegation-sas.yml delete mode 100644 src/arcade/eng/common/templates/steps/get-federated-access-token.yml delete mode 100644 src/arcade/eng/common/templates/steps/publish-logs.yml delete mode 100644 src/arcade/eng/common/templates/steps/publish-pipeline-artifacts.yml delete mode 100644 src/arcade/eng/common/templates/steps/retain-build.yml delete mode 100644 src/arcade/eng/common/templates/steps/send-to-helix.yml delete mode 100644 src/arcade/eng/common/templates/steps/source-build.yml delete mode 100644 src/arcade/eng/common/templates/variables/pool-providers.yml diff --git a/src/arcade/eng/common/BuildConfiguration/build-configuration.json b/src/arcade/eng/common/BuildConfiguration/build-configuration.json deleted file mode 100644 index 3d1cc89894c9..000000000000 --- a/src/arcade/eng/common/BuildConfiguration/build-configuration.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "RetryCountLimit": 1, - "RetryByAnyError": false -} diff --git a/src/arcade/eng/common/PSScriptAnalyzerSettings.psd1 b/src/arcade/eng/common/PSScriptAnalyzerSettings.psd1 deleted file mode 100644 index 4c1ea7c98ea4..000000000000 --- a/src/arcade/eng/common/PSScriptAnalyzerSettings.psd1 +++ /dev/null @@ -1,11 +0,0 @@ -@{ - IncludeRules=@('PSAvoidUsingCmdletAliases', - 'PSAvoidUsingWMICmdlet', - 'PSAvoidUsingPositionalParameters', - 'PSAvoidUsingInvokeExpression', - 'PSUseDeclaredVarsMoreThanAssignments', - 'PSUseCmdletCorrectly', - 'PSStandardDSCFunctionsInResource', - 'PSUseIdenticalMandatoryParametersForDSC', - 'PSUseIdenticalParametersForDSC') -} \ No newline at end of file diff --git a/src/arcade/eng/common/README.md b/src/arcade/eng/common/README.md deleted file mode 100644 index ff49c371527a..000000000000 --- a/src/arcade/eng/common/README.md +++ /dev/null @@ -1,28 +0,0 @@ -# Don't touch this folder - - uuuuuuuuuuuuuuuuuuuu - u" uuuuuuuuuuuuuuuuuu "u - u" u$$$$$$$$$$$$$$$$$$$$u "u - u" u$$$$$$$$$$$$$$$$$$$$$$$$u "u - u" u$$$$$$$$$$$$$$$$$$$$$$$$$$$$u "u - u" u$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$u "u - u" u$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$u "u - $ $$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$ $ - $ $$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$ $ - $ $$$" ... "$... ...$" ... "$$$ ... "$$$ $ - $ $$$u `"$$$$$$$ $$$ $$$$$ $$ $$$ $$$ $ - $ $$$$$$uu "$$$$ $$$ $$$$$ $$ """ u$$$ $ - $ $$$""$$$ $$$$ $$$u "$$$" u$$ $$$$$$$$ $ - $ $$$$....,$$$$$..$$$$$....,$$$$..$$$$$$$$ $ - $ $$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$ $ - "u "$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$" u" - "u "$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$" u" - "u "$$$$$$$$$$$$$$$$$$$$$$$$$$$$" u" - "u "$$$$$$$$$$$$$$$$$$$$$$$$" u" - "u "$$$$$$$$$$$$$$$$$$$$" u" - "u """""""""""""""""" u" - """""""""""""""""""" - -!!! Changes made in this directory are subject to being overwritten by automation !!! - -The files in this directory are shared by all Arcade repos and managed by automation. If you need to make changes to these files, open an issue or submit a pull request to https://github.com/dotnet/arcade first. diff --git a/src/arcade/eng/common/SetupNugetSources.ps1 b/src/arcade/eng/common/SetupNugetSources.ps1 deleted file mode 100644 index 5db4ad71ee2f..000000000000 --- a/src/arcade/eng/common/SetupNugetSources.ps1 +++ /dev/null @@ -1,171 +0,0 @@ -# This script adds internal feeds required to build commits that depend on internal package sources. For instance, -# dotnet6-internal would be added automatically if dotnet6 was found in the nuget.config file. In addition also enables -# disabled internal Maestro (darc-int*) feeds. -# -# Optionally, this script also adds a credential entry for each of the internal feeds if supplied. -# -# See example call for this script below. -# -# - task: PowerShell@2 -# displayName: Setup Private Feeds Credentials -# condition: eq(variables['Agent.OS'], 'Windows_NT') -# inputs: -# filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.ps1 -# arguments: -ConfigFile $(Build.SourcesDirectory)/NuGet.config -Password $Env:Token -# env: -# Token: $(dn-bot-dnceng-artifact-feeds-rw) -# -# Note that the NuGetAuthenticate task should be called after SetupNugetSources. -# This ensures that: -# - Appropriate creds are set for the added internal feeds (if not supplied to the scrupt) -# - The credential provider is installed. -# -# This logic is also abstracted into enable-internal-sources.yml. - -[CmdletBinding()] -param ( - [Parameter(Mandatory = $true)][string]$ConfigFile, - $Password -) - -$ErrorActionPreference = "Stop" -Set-StrictMode -Version 2.0 -[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 - -. $PSScriptRoot\tools.ps1 - -# Add source entry to PackageSources -function AddPackageSource($sources, $SourceName, $SourceEndPoint, $creds, $Username, $pwd) { - $packageSource = $sources.SelectSingleNode("add[@key='$SourceName']") - - if ($packageSource -eq $null) - { - $packageSource = $doc.CreateElement("add") - $packageSource.SetAttribute("key", $SourceName) - $packageSource.SetAttribute("value", $SourceEndPoint) - $sources.AppendChild($packageSource) | Out-Null - } - else { - Write-Host "Package source $SourceName already present." - } - - AddCredential -Creds $creds -Source $SourceName -Username $Username -pwd $pwd -} - -# Add a credential node for the specified source -function AddCredential($creds, $source, $username, $pwd) { - # If no cred supplied, don't do anything. - if (!$pwd) { - return; - } - - # Looks for credential configuration for the given SourceName. Create it if none is found. - $sourceElement = $creds.SelectSingleNode($Source) - if ($sourceElement -eq $null) - { - $sourceElement = $doc.CreateElement($Source) - $creds.AppendChild($sourceElement) | Out-Null - } - - # Add the node to the credential if none is found. - $usernameElement = $sourceElement.SelectSingleNode("add[@key='Username']") - if ($usernameElement -eq $null) - { - $usernameElement = $doc.CreateElement("add") - $usernameElement.SetAttribute("key", "Username") - $sourceElement.AppendChild($usernameElement) | Out-Null - } - $usernameElement.SetAttribute("value", $Username) - - # Add the to the credential if none is found. - # Add it as a clear text because there is no support for encrypted ones in non-windows .Net SDKs. - # -> https://github.com/NuGet/Home/issues/5526 - $passwordElement = $sourceElement.SelectSingleNode("add[@key='ClearTextPassword']") - if ($passwordElement -eq $null) - { - $passwordElement = $doc.CreateElement("add") - $passwordElement.SetAttribute("key", "ClearTextPassword") - $sourceElement.AppendChild($passwordElement) | Out-Null - } - - $passwordElement.SetAttribute("value", $pwd) -} - -function InsertMaestroPrivateFeedCredentials($Sources, $Creds, $Username, $pwd) { - $maestroPrivateSources = $Sources.SelectNodes("add[contains(@key,'darc-int')]") - - Write-Host "Inserting credentials for $($maestroPrivateSources.Count) Maestro's private feeds." - - ForEach ($PackageSource in $maestroPrivateSources) { - Write-Host "`tInserting credential for Maestro's feed:" $PackageSource.Key - AddCredential -Creds $creds -Source $PackageSource.Key -Username $Username -pwd $pwd - } -} - -function EnablePrivatePackageSources($DisabledPackageSources) { - $maestroPrivateSources = $DisabledPackageSources.SelectNodes("add[contains(@key,'darc-int')]") - ForEach ($DisabledPackageSource in $maestroPrivateSources) { - Write-Host "`tEnsuring private source '$($DisabledPackageSource.key)' is enabled by deleting it from disabledPackageSource" - # Due to https://github.com/NuGet/Home/issues/10291, we must actually remove the disabled entries - $DisabledPackageSources.RemoveChild($DisabledPackageSource) - } -} - -if (!(Test-Path $ConfigFile -PathType Leaf)) { - Write-PipelineTelemetryError -Category 'Build' -Message "Eng/common/SetupNugetSources.ps1 returned a non-zero exit code. Couldn't find the NuGet config file: $ConfigFile" - ExitWithExitCode 1 -} - -# Load NuGet.config -$doc = New-Object System.Xml.XmlDocument -$filename = (Get-Item $ConfigFile).FullName -$doc.Load($filename) - -# Get reference to or create one if none exist already -$sources = $doc.DocumentElement.SelectSingleNode("packageSources") -if ($sources -eq $null) { - $sources = $doc.CreateElement("packageSources") - $doc.DocumentElement.AppendChild($sources) | Out-Null -} - -$creds = $null -if ($Password) { - # Looks for a node. Create it if none is found. - $creds = $doc.DocumentElement.SelectSingleNode("packageSourceCredentials") - if ($creds -eq $null) { - $creds = $doc.CreateElement("packageSourceCredentials") - $doc.DocumentElement.AppendChild($creds) | Out-Null - } -} - -# Check for disabledPackageSources; we'll enable any darc-int ones we find there -$disabledSources = $doc.DocumentElement.SelectSingleNode("disabledPackageSources") -if ($disabledSources -ne $null) { - Write-Host "Checking for any darc-int disabled package sources in the disabledPackageSources node" - EnablePrivatePackageSources -DisabledPackageSources $disabledSources -} - -$userName = "dn-bot" - -# Insert credential nodes for Maestro's private feeds -InsertMaestroPrivateFeedCredentials -Sources $sources -Creds $creds -Username $userName -pwd $Password - -# 3.1 uses a different feed url format so it's handled differently here -$dotnet31Source = $sources.SelectSingleNode("add[@key='dotnet3.1']") -if ($dotnet31Source -ne $null) { - AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal/nuget/v2" -Creds $creds -Username $userName -pwd $Password - AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-transport/nuget/v2" -Creds $creds -Username $userName -pwd $Password -} - -$dotnetVersions = @('5','6','7','8','9') - -foreach ($dotnetVersion in $dotnetVersions) { - $feedPrefix = "dotnet" + $dotnetVersion; - $dotnetSource = $sources.SelectSingleNode("add[@key='$feedPrefix']") - if ($dotnetSource -ne $null) { - AddPackageSource -Sources $sources -SourceName "$feedPrefix-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/$feedPrefix-internal/nuget/v2" -Creds $creds -Username $userName -pwd $Password - AddPackageSource -Sources $sources -SourceName "$feedPrefix-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/$feedPrefix-internal-transport/nuget/v2" -Creds $creds -Username $userName -pwd $Password - } -} - -$doc.Save($filename) diff --git a/src/arcade/eng/common/SetupNugetSources.sh b/src/arcade/eng/common/SetupNugetSources.sh deleted file mode 100644 index 4604b61b0323..000000000000 --- a/src/arcade/eng/common/SetupNugetSources.sh +++ /dev/null @@ -1,167 +0,0 @@ -#!/usr/bin/env bash - -# This script adds internal feeds required to build commits that depend on internal package sources. For instance, -# dotnet6-internal would be added automatically if dotnet6 was found in the nuget.config file. In addition also enables -# disabled internal Maestro (darc-int*) feeds. -# -# Optionally, this script also adds a credential entry for each of the internal feeds if supplied. -# -# See example call for this script below. -# -# - task: Bash@3 -# displayName: Setup Internal Feeds -# inputs: -# filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh -# arguments: $(Build.SourcesDirectory)/NuGet.config -# condition: ne(variables['Agent.OS'], 'Windows_NT') -# - task: NuGetAuthenticate@1 -# -# Note that the NuGetAuthenticate task should be called after SetupNugetSources. -# This ensures that: -# - Appropriate creds are set for the added internal feeds (if not supplied to the scrupt) -# - The credential provider is installed. -# -# This logic is also abstracted into enable-internal-sources.yml. - -ConfigFile=$1 -CredToken=$2 -NL='\n' -TB=' ' - -source="${BASH_SOURCE[0]}" - -# resolve $source until the file is no longer a symlink -while [[ -h "$source" ]]; do - scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" - source="$(readlink "$source")" - # if $source was a relative symlink, we need to resolve it relative to the path where the - # symlink file was located - [[ $source != /* ]] && source="$scriptroot/$source" -done -scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" - -. "$scriptroot/tools.sh" - -if [ ! -f "$ConfigFile" ]; then - Write-PipelineTelemetryError -Category 'Build' "Error: Eng/common/SetupNugetSources.sh returned a non-zero exit code. Couldn't find the NuGet config file: $ConfigFile" - ExitWithExitCode 1 -fi - -if [[ `uname -s` == "Darwin" ]]; then - NL=$'\\\n' - TB='' -fi - -# Ensure there is a ... section. -grep -i "" $ConfigFile -if [ "$?" != "0" ]; then - echo "Adding ... section." - ConfigNodeHeader="" - PackageSourcesTemplate="${TB}${NL}${TB}" - - sed -i.bak "s|$ConfigNodeHeader|$ConfigNodeHeader${NL}$PackageSourcesTemplate|" $ConfigFile -fi - -# Ensure there is a ... section. -grep -i "" $ConfigFile -if [ "$?" != "0" ]; then - echo "Adding ... section." - - PackageSourcesNodeFooter="" - PackageSourceCredentialsTemplate="${TB}${NL}${TB}" - - sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourcesNodeFooter${NL}$PackageSourceCredentialsTemplate|" $ConfigFile -fi - -PackageSources=() - -# Ensure dotnet3.1-internal and dotnet3.1-internal-transport are in the packageSources if the public dotnet3.1 feeds are present -grep -i "" - - sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourceTemplate${NL}$PackageSourcesNodeFooter|" $ConfigFile - fi - PackageSources+=('dotnet3.1-internal') - - grep -i "" $ConfigFile - if [ "$?" != "0" ]; then - echo "Adding dotnet3.1-internal-transport to the packageSources." - PackageSourcesNodeFooter="" - PackageSourceTemplate="${TB}" - - sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourceTemplate${NL}$PackageSourcesNodeFooter|" $ConfigFile - fi - PackageSources+=('dotnet3.1-internal-transport') -fi - -DotNetVersions=('5' '6' '7' '8' '9') - -for DotNetVersion in ${DotNetVersions[@]} ; do - FeedPrefix="dotnet${DotNetVersion}"; - grep -i "" - - sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourceTemplate${NL}$PackageSourcesNodeFooter|" $ConfigFile - fi - PackageSources+=("$FeedPrefix-internal") - - grep -i "" $ConfigFile - if [ "$?" != "0" ]; then - echo "Adding $FeedPrefix-internal-transport to the packageSources." - PackageSourcesNodeFooter="" - PackageSourceTemplate="${TB}" - - sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourceTemplate${NL}$PackageSourcesNodeFooter|" $ConfigFile - fi - PackageSources+=("$FeedPrefix-internal-transport") - fi -done - -# I want things split line by line -PrevIFS=$IFS -IFS=$'\n' -PackageSources+="$IFS" -PackageSources+=$(grep -oh '"darc-int-[^"]*"' $ConfigFile | tr -d '"') -IFS=$PrevIFS - -if [ "$CredToken" ]; then - for FeedName in ${PackageSources[@]} ; do - # Check if there is no existing credential for this FeedName - grep -i "<$FeedName>" $ConfigFile - if [ "$?" != "0" ]; then - echo "Adding credentials for $FeedName." - - PackageSourceCredentialsNodeFooter="" - NewCredential="${TB}${TB}<$FeedName>${NL}${NL}${NL}" - - sed -i.bak "s|$PackageSourceCredentialsNodeFooter|$NewCredential${NL}$PackageSourceCredentialsNodeFooter|" $ConfigFile - fi - done -fi - -# Re-enable any entries in disabledPackageSources where the feed name contains darc-int -grep -i "" $ConfigFile -if [ "$?" == "0" ]; then - DisabledDarcIntSources=() - echo "Re-enabling any disabled \"darc-int\" package sources in $ConfigFile" - DisabledDarcIntSources+=$(grep -oh '"darc-int-[^"]*" value="true"' $ConfigFile | tr -d '"') - for DisabledSourceName in ${DisabledDarcIntSources[@]} ; do - if [[ $DisabledSourceName == darc-int* ]] - then - OldDisableValue="" - NewDisableValue="" - sed -i.bak "s|$OldDisableValue|$NewDisableValue|" $ConfigFile - echo "Neutralized disablePackageSources entry for '$DisabledSourceName'" - fi - done -fi diff --git a/src/arcade/eng/common/build.cmd b/src/arcade/eng/common/build.cmd deleted file mode 100644 index 99daf368abae..000000000000 --- a/src/arcade/eng/common/build.cmd +++ /dev/null @@ -1,3 +0,0 @@ -@echo off -powershell -ExecutionPolicy ByPass -NoProfile -command "& """%~dp0build.ps1""" %*" -exit /b %ErrorLevel% diff --git a/src/arcade/eng/common/core-templates/job/onelocbuild.yml b/src/arcade/eng/common/core-templates/job/onelocbuild.yml deleted file mode 100644 index 00feec8ebbc3..000000000000 --- a/src/arcade/eng/common/core-templates/job/onelocbuild.yml +++ /dev/null @@ -1,121 +0,0 @@ -parameters: - # Optional: dependencies of the job - dependsOn: '' - - # Optional: A defined YAML pool - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#pool - pool: '' - - CeapexPat: $(dn-bot-ceapex-package-r) # PAT for the loc AzDO instance https://dev.azure.com/ceapex - GithubPat: $(BotAccount-dotnet-bot-repo-PAT) - - SourcesDirectory: $(Build.SourcesDirectory) - CreatePr: true - AutoCompletePr: false - ReusePr: true - UseLfLineEndings: true - UseCheckedInLocProjectJson: false - SkipLocProjectJsonGeneration: false - LanguageSet: VS_Main_Languages - LclSource: lclFilesInRepo - LclPackageId: '' - RepoType: gitHub - GitHubOrg: dotnet - MirrorRepo: '' - MirrorBranch: main - condition: '' - JobNameSuffix: '' - is1ESPipeline: '' -jobs: -- job: OneLocBuild${{ parameters.JobNameSuffix }} - - dependsOn: ${{ parameters.dependsOn }} - - displayName: OneLocBuild${{ parameters.JobNameSuffix }} - - variables: - - group: OneLocBuildVariables # Contains the CeapexPat and GithubPat - - name: _GenerateLocProjectArguments - value: -SourcesDirectory ${{ parameters.SourcesDirectory }} - -LanguageSet "${{ parameters.LanguageSet }}" - -CreateNeutralXlfs - - ${{ if eq(parameters.UseCheckedInLocProjectJson, 'true') }}: - - name: _GenerateLocProjectArguments - value: ${{ variables._GenerateLocProjectArguments }} -UseCheckedInLocProjectJson - - template: /eng/common/core-templates/variables/pool-providers.yml - parameters: - is1ESPipeline: ${{ parameters.is1ESPipeline }} - - ${{ if ne(parameters.pool, '') }}: - pool: ${{ parameters.pool }} - ${{ if eq(parameters.pool, '') }}: - pool: - # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) - ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: - name: AzurePipelines-EO - image: 1ESPT-Windows2022 - demands: Cmd - os: windows - # If it's not devdiv, it's dnceng - ${{ if ne(variables['System.TeamProject'], 'DevDiv') }}: - name: $(DncEngInternalBuildPool) - image: 1es-windows-2022 - os: windows - - steps: - - ${{ if eq(parameters.is1ESPipeline, '') }}: - - 'Illegal entry point, is1ESPipeline is not defined. Repository yaml should not directly reference templates in core-templates folder.': error - - - ${{ if ne(parameters.SkipLocProjectJsonGeneration, 'true') }}: - - task: Powershell@2 - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/generate-locproject.ps1 - arguments: $(_GenerateLocProjectArguments) - displayName: Generate LocProject.json - condition: ${{ parameters.condition }} - - - task: OneLocBuild@2 - displayName: OneLocBuild - env: - SYSTEM_ACCESSTOKEN: $(System.AccessToken) - inputs: - locProj: eng/Localize/LocProject.json - outDir: $(Build.ArtifactStagingDirectory) - lclSource: ${{ parameters.LclSource }} - lclPackageId: ${{ parameters.LclPackageId }} - isCreatePrSelected: ${{ parameters.CreatePr }} - isAutoCompletePrSelected: ${{ parameters.AutoCompletePr }} - ${{ if eq(parameters.CreatePr, true) }}: - isUseLfLineEndingsSelected: ${{ parameters.UseLfLineEndings }} - ${{ if eq(parameters.RepoType, 'gitHub') }}: - isShouldReusePrSelected: ${{ parameters.ReusePr }} - packageSourceAuth: patAuth - patVariable: ${{ parameters.CeapexPat }} - ${{ if eq(parameters.RepoType, 'gitHub') }}: - repoType: ${{ parameters.RepoType }} - gitHubPatVariable: "${{ parameters.GithubPat }}" - ${{ if ne(parameters.MirrorRepo, '') }}: - isMirrorRepoSelected: true - gitHubOrganization: ${{ parameters.GitHubOrg }} - mirrorRepo: ${{ parameters.MirrorRepo }} - mirrorBranch: ${{ parameters.MirrorBranch }} - condition: ${{ parameters.condition }} - - - template: /eng/common/core-templates/steps/publish-build-artifacts.yml - parameters: - is1ESPipeline: ${{ parameters.is1ESPipeline }} - args: - displayName: Publish Localization Files - pathToPublish: '$(Build.ArtifactStagingDirectory)/loc' - publishLocation: Container - artifactName: Loc - condition: ${{ parameters.condition }} - - - template: /eng/common/core-templates/steps/publish-build-artifacts.yml - parameters: - is1ESPipeline: ${{ parameters.is1ESPipeline }} - args: - displayName: Publish LocProject.json - pathToPublish: '$(Build.SourcesDirectory)/eng/Localize/' - publishLocation: Container - artifactName: Loc - condition: ${{ parameters.condition }} \ No newline at end of file diff --git a/src/arcade/eng/common/core-templates/jobs/source-build.yml b/src/arcade/eng/common/core-templates/jobs/source-build.yml deleted file mode 100644 index a10ccfbee6de..000000000000 --- a/src/arcade/eng/common/core-templates/jobs/source-build.yml +++ /dev/null @@ -1,58 +0,0 @@ -parameters: - # This template adds arcade-powered source-build to CI. A job is created for each platform, as - # well as an optional server job that completes when all platform jobs complete. - - # The name of the "join" job for all source-build platforms. If set to empty string, the job is - # not included. Existing repo pipelines can use this job depend on all source-build jobs - # completing without maintaining a separate list of every single job ID: just depend on this one - # server job. By default, not included. Recommended name if used: 'Source_Build_Complete'. - allCompletedJobId: '' - - # See /eng/common/core-templates/job/source-build.yml - jobNamePrefix: 'Source_Build' - - # This is the default platform provided by Arcade, intended for use by a managed-only repo. - defaultManagedPlatform: - name: 'Managed' - container: 'mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream9' - - # Defines the platforms on which to run build jobs. One job is created for each platform, and the - # object in this array is sent to the job template as 'platform'. If no platforms are specified, - # one job runs on 'defaultManagedPlatform'. - platforms: [] - - is1ESPipeline: '' - - # If set to true and running on a non-public project, - # Internal nuget and blob storage locations will be enabled. - # This is not enabled by default because many repositories do not need internal sources - # and do not need to have the required service connections approved in the pipeline. - enableInternalSources: false - -jobs: - -- ${{ if ne(parameters.allCompletedJobId, '') }}: - - job: ${{ parameters.allCompletedJobId }} - displayName: Source-Build Complete - pool: server - dependsOn: - - ${{ each platform in parameters.platforms }}: - - ${{ parameters.jobNamePrefix }}_${{ platform.name }} - - ${{ if eq(length(parameters.platforms), 0) }}: - - ${{ parameters.jobNamePrefix }}_${{ parameters.defaultManagedPlatform.name }} - -- ${{ each platform in parameters.platforms }}: - - template: /eng/common/core-templates/job/source-build.yml - parameters: - is1ESPipeline: ${{ parameters.is1ESPipeline }} - jobNamePrefix: ${{ parameters.jobNamePrefix }} - platform: ${{ platform }} - enableInternalSources: ${{ parameters.enableInternalSources }} - -- ${{ if eq(length(parameters.platforms), 0) }}: - - template: /eng/common/core-templates/job/source-build.yml - parameters: - is1ESPipeline: ${{ parameters.is1ESPipeline }} - jobNamePrefix: ${{ parameters.jobNamePrefix }} - platform: ${{ parameters.defaultManagedPlatform }} - enableInternalSources: ${{ parameters.enableInternalSources }} diff --git a/src/arcade/eng/common/core-templates/post-build/common-variables.yml b/src/arcade/eng/common/core-templates/post-build/common-variables.yml deleted file mode 100644 index d5627a994ae5..000000000000 --- a/src/arcade/eng/common/core-templates/post-build/common-variables.yml +++ /dev/null @@ -1,22 +0,0 @@ -variables: - - group: Publish-Build-Assets - - # Whether the build is internal or not - - name: IsInternalBuild - value: ${{ and(ne(variables['System.TeamProject'], 'public'), contains(variables['Build.SourceBranch'], 'internal')) }} - - # Default Maestro++ API Endpoint and API Version - - name: MaestroApiEndPoint - value: "https://maestro.dot.net" - - name: MaestroApiVersion - value: "2020-02-20" - - - name: SourceLinkCLIVersion - value: 3.0.0 - - name: SymbolToolVersion - value: 1.0.1 - - name: BinlogToolVersion - value: 1.0.11 - - - name: runCodesignValidationInjection - value: false diff --git a/src/arcade/eng/common/core-templates/post-build/setup-maestro-vars.yml b/src/arcade/eng/common/core-templates/post-build/setup-maestro-vars.yml deleted file mode 100644 index f7602980dbe7..000000000000 --- a/src/arcade/eng/common/core-templates/post-build/setup-maestro-vars.yml +++ /dev/null @@ -1,74 +0,0 @@ -parameters: - BARBuildId: '' - PromoteToChannelIds: '' - is1ESPipeline: '' - -steps: - - ${{ if eq(parameters.is1ESPipeline, '') }}: - - 'Illegal entry point, is1ESPipeline is not defined. Repository yaml should not directly reference templates in core-templates folder.': error - - - ${{ if eq(coalesce(parameters.PromoteToChannelIds, 0), 0) }}: - - task: DownloadBuildArtifacts@0 - displayName: Download Release Configs - inputs: - buildType: current - artifactName: ReleaseConfigs - checkDownloadedFiles: true - - - task: AzureCLI@2 - name: setReleaseVars - displayName: Set Release Configs Vars - inputs: - azureSubscription: "Darc: Maestro Production" - scriptType: pscore - scriptLocation: inlineScript - inlineScript: | - try { - if (!$Env:PromoteToMaestroChannels -or $Env:PromoteToMaestroChannels.Trim() -eq '') { - $Content = Get-Content $(Build.StagingDirectory)/ReleaseConfigs/ReleaseConfigs.txt - - $BarId = $Content | Select -Index 0 - $Channels = $Content | Select -Index 1 - $IsStableBuild = $Content | Select -Index 2 - - $AzureDevOpsProject = $Env:System_TeamProject - $AzureDevOpsBuildDefinitionId = $Env:System_DefinitionId - $AzureDevOpsBuildId = $Env:Build_BuildId - } - else { - . $(Build.SourcesDirectory)\eng\common\tools.ps1 - $darc = Get-Darc - $buildInfo = & $darc get-build ` - --id ${{ parameters.BARBuildId }} ` - --extended ` - --output-format json ` - --ci ` - | convertFrom-Json - - $BarId = ${{ parameters.BARBuildId }} - $Channels = $Env:PromoteToMaestroChannels -split "," - $Channels = $Channels -join "][" - $Channels = "[$Channels]" - - $IsStableBuild = $buildInfo.stable - $AzureDevOpsProject = $buildInfo.azureDevOpsProject - $AzureDevOpsBuildDefinitionId = $buildInfo.azureDevOpsBuildDefinitionId - $AzureDevOpsBuildId = $buildInfo.azureDevOpsBuildId - } - - Write-Host "##vso[task.setvariable variable=BARBuildId]$BarId" - Write-Host "##vso[task.setvariable variable=TargetChannels]$Channels" - Write-Host "##vso[task.setvariable variable=IsStableBuild]$IsStableBuild" - - Write-Host "##vso[task.setvariable variable=AzDOProjectName]$AzureDevOpsProject" - Write-Host "##vso[task.setvariable variable=AzDOPipelineId]$AzureDevOpsBuildDefinitionId" - Write-Host "##vso[task.setvariable variable=AzDOBuildId]$AzureDevOpsBuildId" - } - catch { - Write-Host $_ - Write-Host $_.Exception - Write-Host $_.ScriptStackTrace - exit 1 - } - env: - PromoteToMaestroChannels: ${{ parameters.PromoteToChannelIds }} diff --git a/src/arcade/eng/common/core-templates/steps/component-governance.yml b/src/arcade/eng/common/core-templates/steps/component-governance.yml deleted file mode 100644 index cf0649aa9565..000000000000 --- a/src/arcade/eng/common/core-templates/steps/component-governance.yml +++ /dev/null @@ -1,16 +0,0 @@ -parameters: - disableComponentGovernance: false - componentGovernanceIgnoreDirectories: '' - is1ESPipeline: false - displayName: 'Component Detection' - -steps: -- ${{ if eq(parameters.disableComponentGovernance, 'true') }}: - - script: echo "##vso[task.setvariable variable=skipComponentGovernanceDetection]true" - displayName: Set skipComponentGovernanceDetection variable -- ${{ if ne(parameters.disableComponentGovernance, 'true') }}: - - task: ComponentGovernanceComponentDetection@0 - continueOnError: true - displayName: ${{ parameters.displayName }} - inputs: - ignoreDirectories: ${{ parameters.componentGovernanceIgnoreDirectories }} diff --git a/src/arcade/eng/common/core-templates/steps/enable-internal-runtimes.yml b/src/arcade/eng/common/core-templates/steps/enable-internal-runtimes.yml deleted file mode 100644 index 6bdbf62ac500..000000000000 --- a/src/arcade/eng/common/core-templates/steps/enable-internal-runtimes.yml +++ /dev/null @@ -1,32 +0,0 @@ -# Obtains internal runtime download credentials and populates the 'dotnetbuilds-internal-container-read-token-base64' -# variable with the base64-encoded SAS token, by default - -parameters: -- name: federatedServiceConnection - type: string - default: 'dotnetbuilds-internal-read' -- name: outputVariableName - type: string - default: 'dotnetbuilds-internal-container-read-token-base64' -- name: expiryInHours - type: number - default: 1 -- name: base64Encode - type: boolean - default: true -- name: is1ESPipeline - type: boolean - default: false - -steps: -- ${{ if ne(variables['System.TeamProject'], 'public') }}: - - template: /eng/common/core-templates/steps/get-delegation-sas.yml - parameters: - federatedServiceConnection: ${{ parameters.federatedServiceConnection }} - outputVariableName: ${{ parameters.outputVariableName }} - expiryInHours: ${{ parameters.expiryInHours }} - base64Encode: ${{ parameters.base64Encode }} - storageAccount: dotnetbuilds - container: internal - permissions: rl - is1ESPipeline: ${{ parameters.is1ESPipeline }} \ No newline at end of file diff --git a/src/arcade/eng/common/core-templates/steps/enable-internal-sources.yml b/src/arcade/eng/common/core-templates/steps/enable-internal-sources.yml deleted file mode 100644 index 64f881bffc3c..000000000000 --- a/src/arcade/eng/common/core-templates/steps/enable-internal-sources.yml +++ /dev/null @@ -1,47 +0,0 @@ -parameters: -# This is the Azure federated service connection that we log into to get an access token. -- name: nugetFederatedServiceConnection - type: string - default: 'dnceng-artifacts-feeds-read' -- name: is1ESPipeline - type: boolean - default: false -# Legacy parameters to allow for PAT usage -- name: legacyCredential - type: string - default: '' - -steps: -- ${{ if ne(variables['System.TeamProject'], 'public') }}: - - ${{ if ne(parameters.legacyCredential, '') }}: - - task: PowerShell@2 - displayName: Setup Internal Feeds - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.ps1 - arguments: -ConfigFile $(Build.SourcesDirectory)/NuGet.config -Password $Env:Token - env: - Token: ${{ parameters.legacyCredential }} - # If running on dnceng (internal project), just use the default behavior for NuGetAuthenticate. - # If running on DevDiv, NuGetAuthenticate is not really an option. It's scoped to a single feed, and we have many feeds that - # may be added. Instead, we'll use the traditional approach (add cred to nuget.config), but use an account token. - - ${{ else }}: - - ${{ if eq(variables['System.TeamProject'], 'internal') }}: - - task: PowerShell@2 - displayName: Setup Internal Feeds - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.ps1 - arguments: -ConfigFile $(Build.SourcesDirectory)/NuGet.config - - ${{ else }}: - - template: /eng/common/templates/steps/get-federated-access-token.yml - parameters: - federatedServiceConnection: ${{ parameters.nugetFederatedServiceConnection }} - outputVariableName: 'dnceng-artifacts-feeds-read-access-token' - - task: PowerShell@2 - displayName: Setup Internal Feeds - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.ps1 - arguments: -ConfigFile $(Build.SourcesDirectory)/NuGet.config -Password $(dnceng-artifacts-feeds-read-access-token) - # This is required in certain scenarios to install the ADO credential provider. - # It installed by default in some msbuild invocations (e.g. VS msbuild), but needs to be installed for others - # (e.g. dotnet msbuild). - - task: NuGetAuthenticate@1 diff --git a/src/arcade/eng/common/core-templates/steps/get-federated-access-token.yml b/src/arcade/eng/common/core-templates/steps/get-federated-access-token.yml deleted file mode 100644 index 3a4d4410c482..000000000000 --- a/src/arcade/eng/common/core-templates/steps/get-federated-access-token.yml +++ /dev/null @@ -1,42 +0,0 @@ -parameters: -- name: federatedServiceConnection - type: string -- name: outputVariableName - type: string -- name: is1ESPipeline - type: boolean -- name: stepName - type: string - default: 'getFederatedAccessToken' -- name: condition - type: string - default: '' -# Resource to get a token for. Common values include: -# - '499b84ac-1321-427f-aa17-267ca6975798' for Azure DevOps -# - 'https://storage.azure.com/' for storage -# Defaults to Azure DevOps -- name: resource - type: string - default: '499b84ac-1321-427f-aa17-267ca6975798' -- name: isStepOutputVariable - type: boolean - default: false - -steps: -- task: AzureCLI@2 - displayName: 'Getting federated access token for feeds' - name: ${{ parameters.stepName }} - ${{ if ne(parameters.condition, '') }}: - condition: ${{ parameters.condition }} - inputs: - azureSubscription: ${{ parameters.federatedServiceConnection }} - scriptType: 'pscore' - scriptLocation: 'inlineScript' - inlineScript: | - $accessToken = az account get-access-token --query accessToken --resource ${{ parameters.resource }} --output tsv - if ($LASTEXITCODE -ne 0) { - Write-Error "Failed to get access token for resource '${{ parameters.resource }}'" - exit 1 - } - Write-Host "Setting '${{ parameters.outputVariableName }}' with the access token value" - Write-Host "##vso[task.setvariable variable=${{ parameters.outputVariableName }};issecret=true;isOutput=${{ parameters.isStepOutputVariable }}]$accessToken" \ No newline at end of file diff --git a/src/arcade/eng/common/core-templates/steps/publish-build-artifacts.yml b/src/arcade/eng/common/core-templates/steps/publish-build-artifacts.yml deleted file mode 100644 index f24ce346684e..000000000000 --- a/src/arcade/eng/common/core-templates/steps/publish-build-artifacts.yml +++ /dev/null @@ -1,20 +0,0 @@ -parameters: -- name: is1ESPipeline - type: boolean - default: false -- name: args - type: object - default: {} -steps: -- ${{ if ne(parameters.is1ESPipeline, true) }}: - - template: /eng/common/templates/steps/publish-build-artifacts.yml - parameters: - is1ESPipeline: ${{ parameters.is1ESPipeline }} - ${{ each parameter in parameters.args }}: - ${{ parameter.key }}: ${{ parameter.value }} -- ${{ else }}: - - template: /eng/common/templates-official/steps/publish-build-artifacts.yml - parameters: - is1ESPipeline: ${{ parameters.is1ESPipeline }} - ${{ each parameter in parameters.args }}: - ${{ parameter.key }}: ${{ parameter.value }} \ No newline at end of file diff --git a/src/arcade/eng/common/core-templates/steps/publish-pipeline-artifacts.yml b/src/arcade/eng/common/core-templates/steps/publish-pipeline-artifacts.yml deleted file mode 100644 index 2efec04dc2c1..000000000000 --- a/src/arcade/eng/common/core-templates/steps/publish-pipeline-artifacts.yml +++ /dev/null @@ -1,20 +0,0 @@ -parameters: -- name: is1ESPipeline - type: boolean - default: false - -- name: args - type: object - default: {} - -steps: -- ${{ if ne(parameters.is1ESPipeline, true) }}: - - template: /eng/common/templates/steps/publish-pipeline-artifacts.yml - parameters: - ${{ each parameter in parameters }}: - ${{ parameter.key }}: ${{ parameter.value }} -- ${{ else }}: - - template: /eng/common/templates-official/steps/publish-pipeline-artifacts.yml - parameters: - ${{ each parameter in parameters }}: - ${{ parameter.key }}: ${{ parameter.value }} diff --git a/src/arcade/eng/common/core-templates/steps/retain-build.yml b/src/arcade/eng/common/core-templates/steps/retain-build.yml deleted file mode 100644 index 83d97a26a01f..000000000000 --- a/src/arcade/eng/common/core-templates/steps/retain-build.yml +++ /dev/null @@ -1,28 +0,0 @@ -parameters: - # Optional azure devops PAT with build execute permissions for the build's organization, - # only needed if the build that should be retained ran on a different organization than - # the pipeline where this template is executing from - Token: '' - # Optional BuildId to retain, defaults to the current running build - BuildId: '' - # Azure devops Organization URI for the build in the https://dev.azure.com/ format. - # Defaults to the organization the current pipeline is running on - AzdoOrgUri: '$(System.CollectionUri)' - # Azure devops project for the build. Defaults to the project the current pipeline is running on - AzdoProject: '$(System.TeamProject)' - -steps: - - task: powershell@2 - inputs: - targetType: 'filePath' - filePath: eng/common/retain-build.ps1 - pwsh: true - arguments: > - -AzdoOrgUri: ${{parameters.AzdoOrgUri}} - -AzdoProject ${{parameters.AzdoProject}} - -Token ${{coalesce(parameters.Token, '$env:SYSTEM_ACCESSTOKEN') }} - -BuildId ${{coalesce(parameters.BuildId, '$env:BUILD_ID')}} - displayName: Enable permanent build retention - env: - SYSTEM_ACCESSTOKEN: $(System.AccessToken) - BUILD_ID: $(Build.BuildId) \ No newline at end of file diff --git a/src/arcade/eng/common/core-templates/steps/send-to-helix.yml b/src/arcade/eng/common/core-templates/steps/send-to-helix.yml deleted file mode 100644 index 68fa739c4ab2..000000000000 --- a/src/arcade/eng/common/core-templates/steps/send-to-helix.yml +++ /dev/null @@ -1,93 +0,0 @@ -# Please remember to update the documentation if you make changes to these parameters! -parameters: - HelixSource: 'pr/default' # required -- sources must start with pr/, official/, prodcon/, or agent/ - HelixType: 'tests/default/' # required -- Helix telemetry which identifies what type of data this is; should include "test" for clarity and must end in '/' - HelixBuild: $(Build.BuildNumber) # required -- the build number Helix will use to identify this -- automatically set to the AzDO build number - HelixTargetQueues: '' # required -- semicolon-delimited list of Helix queues to test on; see https://helix.dot.net/ for a list of queues - HelixAccessToken: '' # required -- access token to make Helix API requests; should be provided by the appropriate variable group - HelixProjectPath: 'eng/common/helixpublish.proj' # optional -- path to the project file to build relative to BUILD_SOURCESDIRECTORY - HelixProjectArguments: '' # optional -- arguments passed to the build command - HelixConfiguration: '' # optional -- additional property attached to a job - HelixPreCommands: '' # optional -- commands to run before Helix work item execution - HelixPostCommands: '' # optional -- commands to run after Helix work item execution - WorkItemDirectory: '' # optional -- a payload directory to zip up and send to Helix; requires WorkItemCommand; incompatible with XUnitProjects - WorkItemCommand: '' # optional -- a command to execute on the payload; requires WorkItemDirectory; incompatible with XUnitProjects - WorkItemTimeout: '' # optional -- a timeout in TimeSpan.Parse-ready value (e.g. 00:02:00) for the work item command; requires WorkItemDirectory; incompatible with XUnitProjects - CorrelationPayloadDirectory: '' # optional -- a directory to zip up and send to Helix as a correlation payload - XUnitProjects: '' # optional -- semicolon-delimited list of XUnitProjects to parse and send to Helix; requires XUnitRuntimeTargetFramework, XUnitPublishTargetFramework, XUnitRunnerVersion, and IncludeDotNetCli=true - XUnitWorkItemTimeout: '' # optional -- the workitem timeout in seconds for all workitems created from the xUnit projects specified by XUnitProjects - XUnitPublishTargetFramework: '' # optional -- framework to use to publish your xUnit projects - XUnitRuntimeTargetFramework: '' # optional -- framework to use for the xUnit console runner - XUnitRunnerVersion: '' # optional -- version of the xUnit nuget package you wish to use on Helix; required for XUnitProjects - IncludeDotNetCli: false # optional -- true will download a version of the .NET CLI onto the Helix machine as a correlation payload; requires DotNetCliPackageType and DotNetCliVersion - DotNetCliPackageType: '' # optional -- either 'sdk', 'runtime' or 'aspnetcore-runtime'; determines whether the sdk or runtime will be sent to Helix; see https://raw.githubusercontent.com/dotnet/core/main/release-notes/releases-index.json - DotNetCliVersion: '' # optional -- version of the CLI to send to Helix; based on this: https://raw.githubusercontent.com/dotnet/core/main/release-notes/releases-index.json - WaitForWorkItemCompletion: true # optional -- true will make the task wait until work items have been completed and fail the build if work items fail. False is "fire and forget." - IsExternal: false # [DEPRECATED] -- doesn't do anything, jobs are external if HelixAccessToken is empty and Creator is set - HelixBaseUri: 'https://helix.dot.net/' # optional -- sets the Helix API base URI (allows targeting https://helix.int-dot.net ) - Creator: '' # optional -- if the build is external, use this to specify who is sending the job - DisplayNamePrefix: 'Run Tests' # optional -- rename the beginning of the displayName of the steps in AzDO - condition: succeeded() # optional -- condition for step to execute; defaults to succeeded() - continueOnError: false # optional -- determines whether to continue the build if the step errors; defaults to false - -steps: - - powershell: 'powershell "$env:BUILD_SOURCESDIRECTORY\eng\common\msbuild.ps1 $env:BUILD_SOURCESDIRECTORY/${{ parameters.HelixProjectPath }} /restore /p:TreatWarningsAsErrors=false ${{ parameters.HelixProjectArguments }} /t:Test /bl:$env:BUILD_SOURCESDIRECTORY\artifacts\log\$env:BuildConfig\SendToHelix.binlog"' - displayName: ${{ parameters.DisplayNamePrefix }} (Windows) - env: - BuildConfig: $(_BuildConfig) - HelixSource: ${{ parameters.HelixSource }} - HelixType: ${{ parameters.HelixType }} - HelixBuild: ${{ parameters.HelixBuild }} - HelixConfiguration: ${{ parameters.HelixConfiguration }} - HelixTargetQueues: ${{ parameters.HelixTargetQueues }} - HelixAccessToken: ${{ parameters.HelixAccessToken }} - HelixPreCommands: ${{ parameters.HelixPreCommands }} - HelixPostCommands: ${{ parameters.HelixPostCommands }} - WorkItemDirectory: ${{ parameters.WorkItemDirectory }} - WorkItemCommand: ${{ parameters.WorkItemCommand }} - WorkItemTimeout: ${{ parameters.WorkItemTimeout }} - CorrelationPayloadDirectory: ${{ parameters.CorrelationPayloadDirectory }} - XUnitProjects: ${{ parameters.XUnitProjects }} - XUnitWorkItemTimeout: ${{ parameters.XUnitWorkItemTimeout }} - XUnitPublishTargetFramework: ${{ parameters.XUnitPublishTargetFramework }} - XUnitRuntimeTargetFramework: ${{ parameters.XUnitRuntimeTargetFramework }} - XUnitRunnerVersion: ${{ parameters.XUnitRunnerVersion }} - IncludeDotNetCli: ${{ parameters.IncludeDotNetCli }} - DotNetCliPackageType: ${{ parameters.DotNetCliPackageType }} - DotNetCliVersion: ${{ parameters.DotNetCliVersion }} - WaitForWorkItemCompletion: ${{ parameters.WaitForWorkItemCompletion }} - HelixBaseUri: ${{ parameters.HelixBaseUri }} - Creator: ${{ parameters.Creator }} - SYSTEM_ACCESSTOKEN: $(System.AccessToken) - condition: and(${{ parameters.condition }}, eq(variables['Agent.Os'], 'Windows_NT')) - continueOnError: ${{ parameters.continueOnError }} - - script: $BUILD_SOURCESDIRECTORY/eng/common/msbuild.sh $BUILD_SOURCESDIRECTORY/${{ parameters.HelixProjectPath }} /restore /p:TreatWarningsAsErrors=false ${{ parameters.HelixProjectArguments }} /t:Test /bl:$BUILD_SOURCESDIRECTORY/artifacts/log/$BuildConfig/SendToHelix.binlog - displayName: ${{ parameters.DisplayNamePrefix }} (Unix) - env: - BuildConfig: $(_BuildConfig) - HelixSource: ${{ parameters.HelixSource }} - HelixType: ${{ parameters.HelixType }} - HelixBuild: ${{ parameters.HelixBuild }} - HelixConfiguration: ${{ parameters.HelixConfiguration }} - HelixTargetQueues: ${{ parameters.HelixTargetQueues }} - HelixAccessToken: ${{ parameters.HelixAccessToken }} - HelixPreCommands: ${{ parameters.HelixPreCommands }} - HelixPostCommands: ${{ parameters.HelixPostCommands }} - WorkItemDirectory: ${{ parameters.WorkItemDirectory }} - WorkItemCommand: ${{ parameters.WorkItemCommand }} - WorkItemTimeout: ${{ parameters.WorkItemTimeout }} - CorrelationPayloadDirectory: ${{ parameters.CorrelationPayloadDirectory }} - XUnitProjects: ${{ parameters.XUnitProjects }} - XUnitWorkItemTimeout: ${{ parameters.XUnitWorkItemTimeout }} - XUnitPublishTargetFramework: ${{ parameters.XUnitPublishTargetFramework }} - XUnitRuntimeTargetFramework: ${{ parameters.XUnitRuntimeTargetFramework }} - XUnitRunnerVersion: ${{ parameters.XUnitRunnerVersion }} - IncludeDotNetCli: ${{ parameters.IncludeDotNetCli }} - DotNetCliPackageType: ${{ parameters.DotNetCliPackageType }} - DotNetCliVersion: ${{ parameters.DotNetCliVersion }} - WaitForWorkItemCompletion: ${{ parameters.WaitForWorkItemCompletion }} - HelixBaseUri: ${{ parameters.HelixBaseUri }} - Creator: ${{ parameters.Creator }} - SYSTEM_ACCESSTOKEN: $(System.AccessToken) - condition: and(${{ parameters.condition }}, ne(variables['Agent.Os'], 'Windows_NT')) - continueOnError: ${{ parameters.continueOnError }} diff --git a/src/arcade/eng/common/core-templates/variables/pool-providers.yml b/src/arcade/eng/common/core-templates/variables/pool-providers.yml deleted file mode 100644 index 41053d382a2e..000000000000 --- a/src/arcade/eng/common/core-templates/variables/pool-providers.yml +++ /dev/null @@ -1,8 +0,0 @@ -parameters: - is1ESPipeline: false - -variables: - - ${{ if eq(parameters.is1ESPipeline, 'true') }}: - - template: /eng/common/templates-official/variables/pool-providers.yml - - ${{ else }}: - - template: /eng/common/templates/variables/pool-providers.yml \ No newline at end of file diff --git a/src/arcade/eng/common/cross/arm/tizen/tizen.patch b/src/arcade/eng/common/cross/arm/tizen/tizen.patch deleted file mode 100644 index fb12ade7250a..000000000000 --- a/src/arcade/eng/common/cross/arm/tizen/tizen.patch +++ /dev/null @@ -1,9 +0,0 @@ -diff -u -r a/usr/lib/libc.so b/usr/lib/libc.so ---- a/usr/lib/libc.so 2016-12-30 23:00:08.284951863 +0900 -+++ b/usr/lib/libc.so 2016-12-30 23:00:32.140951815 +0900 -@@ -2,4 +2,4 @@ - Use the shared library, but some functions are only in - the static library, so try that secondarily. */ - OUTPUT_FORMAT(elf32-littlearm) --GROUP ( /lib/libc.so.6 /usr/lib/libc_nonshared.a AS_NEEDED ( /lib/ld-linux-armhf.so.3 ) ) -+GROUP ( libc.so.6 libc_nonshared.a AS_NEEDED ( ld-linux-armhf.so.3 ) ) diff --git a/src/arcade/eng/common/cross/armel/tizen/tizen.patch b/src/arcade/eng/common/cross/armel/tizen/tizen.patch deleted file mode 100644 index ca7c7c1ff751..000000000000 --- a/src/arcade/eng/common/cross/armel/tizen/tizen.patch +++ /dev/null @@ -1,9 +0,0 @@ -diff -u -r a/usr/lib/libc.so b/usr/lib/libc.so ---- a/usr/lib/libc.so 2016-12-30 23:00:08.284951863 +0900 -+++ b/usr/lib/libc.so 2016-12-30 23:00:32.140951815 +0900 -@@ -2,4 +2,4 @@ - Use the shared library, but some functions are only in - the static library, so try that secondarily. */ - OUTPUT_FORMAT(elf32-littlearm) --GROUP ( /lib/libc.so.6 /usr/lib/libc_nonshared.a AS_NEEDED ( /lib/ld-linux.so.3 ) ) -+GROUP ( libc.so.6 libc_nonshared.a AS_NEEDED ( ld-linux.so.3 ) ) diff --git a/src/arcade/eng/common/cross/riscv64/tizen/tizen.patch b/src/arcade/eng/common/cross/riscv64/tizen/tizen.patch deleted file mode 100644 index eb6d1c07470b..000000000000 --- a/src/arcade/eng/common/cross/riscv64/tizen/tizen.patch +++ /dev/null @@ -1,9 +0,0 @@ -diff -u -r a/usr/lib/libc.so b/usr/lib/libc.so ---- a/usr/lib64/libc.so 2016-12-30 23:00:08.284951863 +0900 -+++ b/usr/lib64/libc.so 2016-12-30 23:00:32.140951815 +0900 -@@ -2,4 +2,4 @@ - Use the shared library, but some functions are only in - the static library, so try that secondarily. */ - OUTPUT_FORMAT(elf64-littleriscv) --GROUP ( /lib64/libc.so.6 /usr/lib64/libc_nonshared.a AS_NEEDED ( /lib64/ld-linux-riscv64-lp64d.so.1 ) ) -+GROUP ( libc.so.6 libc_nonshared.a AS_NEEDED ( ld-linux-riscv64-lp64d.so.1 ) ) diff --git a/src/arcade/eng/common/cross/tizen-build-rootfs.sh b/src/arcade/eng/common/cross/tizen-build-rootfs.sh deleted file mode 100644 index ba31c93285f6..000000000000 --- a/src/arcade/eng/common/cross/tizen-build-rootfs.sh +++ /dev/null @@ -1,82 +0,0 @@ -#!/usr/bin/env bash -set -e - -ARCH=$1 -LINK_ARCH=$ARCH - -case "$ARCH" in - arm) - TIZEN_ARCH="armv7hl" - ;; - armel) - TIZEN_ARCH="armv7l" - LINK_ARCH="arm" - ;; - arm64) - TIZEN_ARCH="aarch64" - ;; - x86) - TIZEN_ARCH="i686" - ;; - x64) - TIZEN_ARCH="x86_64" - LINK_ARCH="x86" - ;; - riscv64) - TIZEN_ARCH="riscv64" - LINK_ARCH="riscv" - ;; - *) - echo "Unsupported architecture for tizen: $ARCH" - exit 1 -esac - -__CrossDir=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) -__TIZEN_CROSSDIR="$__CrossDir/${ARCH}/tizen" - -if [[ -z "$ROOTFS_DIR" ]]; then - echo "ROOTFS_DIR is not defined." - exit 1; -fi - -TIZEN_TMP_DIR=$ROOTFS_DIR/tizen_tmp -mkdir -p $TIZEN_TMP_DIR - -# Download files -echo ">>Start downloading files" -VERBOSE=1 $__CrossDir/tizen-fetch.sh $TIZEN_TMP_DIR $TIZEN_ARCH -echo "<>Start constructing Tizen rootfs" -TIZEN_RPM_FILES=`ls $TIZEN_TMP_DIR/*.rpm` -cd $ROOTFS_DIR -for f in $TIZEN_RPM_FILES; do - rpm2cpio $f | cpio -idm --quiet -done -echo "<>Start configuring Tizen rootfs" -ln -sfn asm-${LINK_ARCH} ./usr/include/asm -patch -p1 < $__TIZEN_CROSSDIR/tizen.patch -if [[ "$TIZEN_ARCH" == "riscv64" ]]; then - echo "Fixing broken symlinks in $PWD" - rm ./usr/lib64/libresolv.so - ln -s ../../lib64/libresolv.so.2 ./usr/lib64/libresolv.so - rm ./usr/lib64/libpthread.so - ln -s ../../lib64/libpthread.so.0 ./usr/lib64/libpthread.so - rm ./usr/lib64/libdl.so - ln -s ../../lib64/libdl.so.2 ./usr/lib64/libdl.so - rm ./usr/lib64/libutil.so - ln -s ../../lib64/libutil.so.1 ./usr/lib64/libutil.so - rm ./usr/lib64/libm.so - ln -s ../../lib64/libm.so.6 ./usr/lib64/libm.so - rm ./usr/lib64/librt.so - ln -s ../../lib64/librt.so.1 ./usr/lib64/librt.so - rm ./lib/ld-linux-riscv64-lp64d.so.1 - ln -s ../lib64/ld-linux-riscv64-lp64d.so.1 ./lib/ld-linux-riscv64-lp64d.so.1 -fi -echo "< 0 ]]; do - opt="$(echo "$1" | tr "[:upper:]" "[:lower:]")" - case "$opt" in - --darcversion) - darcVersion=$2 - shift - ;; - --versionendpoint) - versionEndpoint=$2 - shift - ;; - --verbosity) - verbosity=$2 - shift - ;; - --toolpath) - toolpath=$2 - shift - ;; - *) - echo "Invalid argument: $1" - usage - exit 1 - ;; - esac - - shift -done - -# resolve $source until the file is no longer a symlink -while [[ -h "$source" ]]; do - scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" - source="$(readlink "$source")" - # if $source was a relative symlink, we need to resolve it relative to the path where the - # symlink file was located - [[ $source != /* ]] && source="$scriptroot/$source" -done -scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" - -. "$scriptroot/tools.sh" - -if [ -z "$darcVersion" ]; then - darcVersion=$(curl -X GET "$versionEndpoint" -H "accept: text/plain") -fi - -function InstallDarcCli { - local darc_cli_package_name="microsoft.dotnet.darc" - - InitializeDotNetCli true - local dotnet_root=$_InitializeDotNetCli - - if [ -z "$toolpath" ]; then - local tool_list=$($dotnet_root/dotnet tool list -g) - if [[ $tool_list = *$darc_cli_package_name* ]]; then - echo $($dotnet_root/dotnet tool uninstall $darc_cli_package_name -g) - fi - else - local tool_list=$($dotnet_root/dotnet tool list --tool-path "$toolpath") - if [[ $tool_list = *$darc_cli_package_name* ]]; then - echo $($dotnet_root/dotnet tool uninstall $darc_cli_package_name --tool-path "$toolpath") - fi - fi - - local arcadeServicesSource="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json" - - echo "Installing Darc CLI version $darcVersion..." - echo "You may need to restart your command shell if this is the first dotnet tool you have installed." - if [ -z "$toolpath" ]; then - echo $($dotnet_root/dotnet tool install $darc_cli_package_name --version $darcVersion --add-source "$arcadeServicesSource" -v $verbosity -g) - else - echo $($dotnet_root/dotnet tool install $darc_cli_package_name --version $darcVersion --add-source "$arcadeServicesSource" -v $verbosity --tool-path "$toolpath") - fi -} - -InstallDarcCli diff --git a/src/arcade/eng/common/dotnet-install.cmd b/src/arcade/eng/common/dotnet-install.cmd deleted file mode 100644 index b1c2642e76f7..000000000000 --- a/src/arcade/eng/common/dotnet-install.cmd +++ /dev/null @@ -1,2 +0,0 @@ -@echo off -powershell -ExecutionPolicy ByPass -NoProfile -command "& """%~dp0dotnet-install.ps1""" %*" \ No newline at end of file diff --git a/src/arcade/eng/common/dotnet-install.ps1 b/src/arcade/eng/common/dotnet-install.ps1 deleted file mode 100644 index 811f0f717f73..000000000000 --- a/src/arcade/eng/common/dotnet-install.ps1 +++ /dev/null @@ -1,28 +0,0 @@ -[CmdletBinding(PositionalBinding=$false)] -Param( - [string] $verbosity = 'minimal', - [string] $architecture = '', - [string] $version = 'Latest', - [string] $runtime = 'dotnet', - [string] $RuntimeSourceFeed = '', - [string] $RuntimeSourceFeedKey = '' -) - -. $PSScriptRoot\tools.ps1 - -$dotnetRoot = Join-Path $RepoRoot '.dotnet' - -$installdir = $dotnetRoot -try { - if ($architecture -and $architecture.Trim() -eq 'x86') { - $installdir = Join-Path $installdir 'x86' - } - InstallDotNet $installdir $version $architecture $runtime $true -RuntimeSourceFeed $RuntimeSourceFeed -RuntimeSourceFeedKey $RuntimeSourceFeedKey -} -catch { - Write-Host $_.ScriptStackTrace - Write-PipelineTelemetryError -Category 'InitializeToolset' -Message $_ - ExitWithExitCode 1 -} - -ExitWithExitCode 0 diff --git a/src/arcade/eng/common/dotnet-install.sh b/src/arcade/eng/common/dotnet-install.sh deleted file mode 100644 index 7b9d97e3bd4d..000000000000 --- a/src/arcade/eng/common/dotnet-install.sh +++ /dev/null @@ -1,94 +0,0 @@ -#!/usr/bin/env bash - -source="${BASH_SOURCE[0]}" -# resolve $source until the file is no longer a symlink -while [[ -h "$source" ]]; do - scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" - source="$(readlink "$source")" - # if $source was a relative symlink, we need to resolve it relative to the path where the - # symlink file was located - [[ $source != /* ]] && source="$scriptroot/$source" -done -scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" - -. "$scriptroot/tools.sh" - -version='Latest' -architecture='' -runtime='dotnet' -runtimeSourceFeed='' -runtimeSourceFeedKey='' -while [[ $# > 0 ]]; do - opt="$(echo "$1" | tr "[:upper:]" "[:lower:]")" - case "$opt" in - -version|-v) - shift - version="$1" - ;; - -architecture|-a) - shift - architecture="$1" - ;; - -runtime|-r) - shift - runtime="$1" - ;; - -runtimesourcefeed) - shift - runtimeSourceFeed="$1" - ;; - -runtimesourcefeedkey) - shift - runtimeSourceFeedKey="$1" - ;; - *) - Write-PipelineTelemetryError -Category 'Build' -Message "Invalid argument: $1" - exit 1 - ;; - esac - shift -done - -# Use uname to determine what the CPU is, see https://en.wikipedia.org/wiki/Uname#Examples -cpuname=$(uname -m) -case $cpuname in - arm64|aarch64) - buildarch=arm64 - if [ "$(getconf LONG_BIT)" -lt 64 ]; then - # This is 32-bit OS running on 64-bit CPU (for example Raspberry Pi OS) - buildarch=arm - fi - ;; - loongarch64) - buildarch=loongarch64 - ;; - amd64|x86_64) - buildarch=x64 - ;; - armv*l) - buildarch=arm - ;; - i[3-6]86) - buildarch=x86 - ;; - riscv64) - buildarch=riscv64 - ;; - *) - echo "Unknown CPU $cpuname detected, treating it as x64" - buildarch=x64 - ;; -esac - -dotnetRoot="${repo_root}.dotnet" -if [[ $architecture != "" ]] && [[ $architecture != $buildarch ]]; then - dotnetRoot="$dotnetRoot/$architecture" -fi - -InstallDotNet "$dotnetRoot" $version "$architecture" $runtime true $runtimeSourceFeed $runtimeSourceFeedKey || { - local exit_code=$? - Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "dotnet-install.sh failed (exit code '$exit_code')." >&2 - ExitWithExitCode $exit_code -} - -ExitWithExitCode 0 diff --git a/src/arcade/eng/common/enable-cross-org-publishing.ps1 b/src/arcade/eng/common/enable-cross-org-publishing.ps1 deleted file mode 100644 index da09da4f1fc4..000000000000 --- a/src/arcade/eng/common/enable-cross-org-publishing.ps1 +++ /dev/null @@ -1,13 +0,0 @@ -param( - [string] $token -) - - -. $PSScriptRoot\pipeline-logging-functions.ps1 - -# Write-PipelineSetVariable will no-op if a variable named $ci is not defined -# Since this script is only ever called in AzDO builds, just universally set it -$ci = $true - -Write-PipelineSetVariable -Name 'VSS_NUGET_ACCESSTOKEN' -Value $token -IsMultiJobVariable $false -Write-PipelineSetVariable -Name 'VSS_NUGET_URI_PREFIXES' -Value 'https://dnceng.pkgs.visualstudio.com/;https://pkgs.dev.azure.com/dnceng/;https://devdiv.pkgs.visualstudio.com/;https://pkgs.dev.azure.com/devdiv/' -IsMultiJobVariable $false diff --git a/src/arcade/eng/common/generate-locproject.ps1 b/src/arcade/eng/common/generate-locproject.ps1 deleted file mode 100644 index 524aaa57f2b7..000000000000 --- a/src/arcade/eng/common/generate-locproject.ps1 +++ /dev/null @@ -1,189 +0,0 @@ -Param( - [Parameter(Mandatory=$true)][string] $SourcesDirectory, # Directory where source files live; if using a Localize directory it should live in here - [string] $LanguageSet = 'VS_Main_Languages', # Language set to be used in the LocProject.json - [switch] $UseCheckedInLocProjectJson, # When set, generates a LocProject.json and compares it to one that already exists in the repo; otherwise just generates one - [switch] $CreateNeutralXlfs # Creates neutral xlf files. Only set to false when running locally -) - -# Generates LocProject.json files for the OneLocBuild task. OneLocBuildTask is described here: -# https://ceapex.visualstudio.com/CEINTL/_wiki/wikis/CEINTL.wiki/107/Localization-with-OneLocBuild-Task - -Set-StrictMode -Version 2.0 -$ErrorActionPreference = "Stop" -. $PSScriptRoot\pipeline-logging-functions.ps1 - -$exclusionsFilePath = "$SourcesDirectory\eng\Localize\LocExclusions.json" -$exclusions = @{ Exclusions = @() } -if (Test-Path -Path $exclusionsFilePath) -{ - $exclusions = Get-Content "$exclusionsFilePath" | ConvertFrom-Json -} - -Push-Location "$SourcesDirectory" # push location for Resolve-Path -Relative to work - -# Template files -$jsonFiles = @() -$jsonTemplateFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "\.template\.config\\localize\\.+\.en\.json" } # .NET templating pattern -$jsonTemplateFiles | ForEach-Object { - $null = $_.Name -Match "(.+)\.[\w-]+\.json" # matches '[filename].[langcode].json - - $destinationFile = "$($_.Directory.FullName)\$($Matches.1).json" - $jsonFiles += Copy-Item "$($_.FullName)" -Destination $destinationFile -PassThru -} - -$jsonWinformsTemplateFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "en\\strings\.json" } # current winforms pattern - -$wxlFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "\\.+\.wxl" -And -Not( $_.Directory.Name -Match "\d{4}" ) } # localized files live in four digit lang ID directories; this excludes them -if (-not $wxlFiles) { - $wxlEnFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "\\1033\\.+\.wxl" } # pick up en files (1033 = en) specifically so we can copy them to use as the neutral xlf files - if ($wxlEnFiles) { - $wxlFiles = @() - $wxlEnFiles | ForEach-Object { - $destinationFile = "$($_.Directory.Parent.FullName)\$($_.Name)" - $wxlFiles += Copy-Item "$($_.FullName)" -Destination $destinationFile -PassThru - } - } -} - -$macosHtmlEnFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "en\.lproj\\.+\.html$" } # add installer HTML files -$macosHtmlFiles = @() -if ($macosHtmlEnFiles) { - $macosHtmlEnFiles | ForEach-Object { - $destinationFile = "$($_.Directory.Parent.FullName)\$($_.Name)" - $macosHtmlFiles += Copy-Item "$($_.FullName)" -Destination $destinationFile -PassThru - } -} - -$xlfFiles = @() - -$allXlfFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory\*\*.xlf" -$langXlfFiles = @() -if ($allXlfFiles) { - $null = $allXlfFiles[0].FullName -Match "\.([\w-]+)\.xlf" # matches '[langcode].xlf' - $firstLangCode = $Matches.1 - $langXlfFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory\*\*.$firstLangCode.xlf" -} -$langXlfFiles | ForEach-Object { - $null = $_.Name -Match "(.+)\.[\w-]+\.xlf" # matches '[filename].[langcode].xlf - - $destinationFile = "$($_.Directory.FullName)\$($Matches.1).xlf" - $xlfFiles += Copy-Item "$($_.FullName)" -Destination $destinationFile -PassThru -} - -$locFiles = $jsonFiles + $jsonWinformsTemplateFiles + $xlfFiles - -$locJson = @{ - Projects = @( - @{ - LanguageSet = $LanguageSet - LocItems = @( - $locFiles | ForEach-Object { - $outputPath = "$(($_.DirectoryName | Resolve-Path -Relative) + "\")" - $continue = $true - foreach ($exclusion in $exclusions.Exclusions) { - if ($_.FullName.Contains($exclusion)) - { - $continue = $false - } - } - $sourceFile = ($_.FullName | Resolve-Path -Relative) - if (!$CreateNeutralXlfs -and $_.Extension -eq '.xlf') { - Remove-Item -Path $sourceFile - } - if ($continue) - { - if ($_.Directory.Name -eq 'en' -and $_.Extension -eq '.json') { - return @{ - SourceFile = $sourceFile - CopyOption = "LangIDOnPath" - OutputPath = "$($_.Directory.Parent.FullName | Resolve-Path -Relative)\" - } - } else { - return @{ - SourceFile = $sourceFile - CopyOption = "LangIDOnName" - OutputPath = $outputPath - } - } - } - } - ) - }, - @{ - LanguageSet = $LanguageSet - CloneLanguageSet = "WiX_CloneLanguages" - LssFiles = @( "wxl_loc.lss" ) - LocItems = @( - $wxlFiles | ForEach-Object { - $outputPath = "$($_.Directory.FullName | Resolve-Path -Relative)\" - $continue = $true - foreach ($exclusion in $exclusions.Exclusions) { - if ($_.FullName.Contains($exclusion)) { - $continue = $false - } - } - $sourceFile = ($_.FullName | Resolve-Path -Relative) - if ($continue) - { - return @{ - SourceFile = $sourceFile - CopyOption = "LangIDOnPath" - OutputPath = $outputPath - } - } - } - ) - }, - @{ - LanguageSet = $LanguageSet - CloneLanguageSet = "VS_macOS_CloneLanguages" - LssFiles = @( ".\eng\common\loc\P22DotNetHtmlLocalization.lss" ) - LocItems = @( - $macosHtmlFiles | ForEach-Object { - $outputPath = "$($_.Directory.FullName | Resolve-Path -Relative)\" - $continue = $true - foreach ($exclusion in $exclusions.Exclusions) { - if ($_.FullName.Contains($exclusion)) { - $continue = $false - } - } - $sourceFile = ($_.FullName | Resolve-Path -Relative) - $lciFile = $sourceFile + ".lci" - if ($continue) { - $result = @{ - SourceFile = $sourceFile - CopyOption = "LangIDOnPath" - OutputPath = $outputPath - } - if (Test-Path $lciFile -PathType Leaf) { - $result["LciFile"] = $lciFile - } - return $result - } - } - ) - } - ) -} - -$json = ConvertTo-Json $locJson -Depth 5 -Write-Host "LocProject.json generated:`n`n$json`n`n" -Pop-Location - -if (!$UseCheckedInLocProjectJson) { - New-Item "$SourcesDirectory\eng\Localize\LocProject.json" -Force # Need this to make sure the Localize directory is created - Set-Content "$SourcesDirectory\eng\Localize\LocProject.json" $json -} -else { - New-Item "$SourcesDirectory\eng\Localize\LocProject-generated.json" -Force # Need this to make sure the Localize directory is created - Set-Content "$SourcesDirectory\eng\Localize\LocProject-generated.json" $json - - if ((Get-FileHash "$SourcesDirectory\eng\Localize\LocProject-generated.json").Hash -ne (Get-FileHash "$SourcesDirectory\eng\Localize\LocProject.json").Hash) { - Write-PipelineTelemetryError -Category "OneLocBuild" -Message "Existing LocProject.json differs from generated LocProject.json. Download LocProject-generated.json and compare them." - - exit 1 - } - else { - Write-Host "Generated LocProject.json and current LocProject.json are identical." - } -} diff --git a/src/arcade/eng/common/generate-sbom-prep.ps1 b/src/arcade/eng/common/generate-sbom-prep.ps1 deleted file mode 100644 index a0c7d792a76f..000000000000 --- a/src/arcade/eng/common/generate-sbom-prep.ps1 +++ /dev/null @@ -1,29 +0,0 @@ -Param( - [Parameter(Mandatory=$true)][string] $ManifestDirPath # Manifest directory where sbom will be placed -) - -. $PSScriptRoot\pipeline-logging-functions.ps1 - -# Normally - we'd listen to the manifest path given, but 1ES templates will overwrite if this level gets uploaded directly -# with their own overwriting ours. So we create it as a sub directory of the requested manifest path. -$ArtifactName = "${env:SYSTEM_STAGENAME}_${env:AGENT_JOBNAME}_SBOM" -$SafeArtifactName = $ArtifactName -replace '["/:<>\\|?@*"() ]', '_' -$SbomGenerationDir = Join-Path $ManifestDirPath $SafeArtifactName - -Write-Host "Artifact name before : $ArtifactName" -Write-Host "Artifact name after : $SafeArtifactName" - -Write-Host "Creating dir $ManifestDirPath" - -# create directory for sbom manifest to be placed -if (!(Test-Path -path $SbomGenerationDir)) -{ - New-Item -ItemType Directory -path $SbomGenerationDir - Write-Host "Successfully created directory $SbomGenerationDir" -} -else{ - Write-PipelineTelemetryError -category 'Build' "Unable to create sbom folder." -} - -Write-Host "Updating artifact name" -Write-Host "##vso[task.setvariable variable=ARTIFACT_NAME]$SafeArtifactName" diff --git a/src/arcade/eng/common/generate-sbom-prep.sh b/src/arcade/eng/common/generate-sbom-prep.sh deleted file mode 100644 index b8ecca72bbf5..000000000000 --- a/src/arcade/eng/common/generate-sbom-prep.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/usr/bin/env bash - -source="${BASH_SOURCE[0]}" - -# resolve $SOURCE until the file is no longer a symlink -while [[ -h $source ]]; do - scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" - source="$(readlink "$source")" - - # if $source was a relative symlink, we need to resolve it relative to the path where the - # symlink file was located - [[ $source != /* ]] && source="$scriptroot/$source" -done -scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" -. $scriptroot/pipeline-logging-functions.sh - - -# replace all special characters with _, some builds use special characters like : in Agent.Jobname, that is not a permissible name while uploading artifacts. -artifact_name=$SYSTEM_STAGENAME"_"$AGENT_JOBNAME"_SBOM" -safe_artifact_name="${artifact_name//["/:<>\\|?@*$" ]/_}" -manifest_dir=$1 - -# Normally - we'd listen to the manifest path given, but 1ES templates will overwrite if this level gets uploaded directly -# with their own overwriting ours. So we create it as a sub directory of the requested manifest path. -sbom_generation_dir="$manifest_dir/$safe_artifact_name" - -if [ ! -d "$sbom_generation_dir" ] ; then - mkdir -p "$sbom_generation_dir" - echo "Sbom directory created." $sbom_generation_dir -else - Write-PipelineTelemetryError -category 'Build' "Unable to create sbom folder." -fi - -echo "Artifact name before : "$artifact_name -echo "Artifact name after : "$safe_artifact_name -export ARTIFACT_NAME=$safe_artifact_name -echo "##vso[task.setvariable variable=ARTIFACT_NAME]$safe_artifact_name" - -exit 0 diff --git a/src/arcade/eng/common/helixpublish.proj b/src/arcade/eng/common/helixpublish.proj deleted file mode 100644 index c1323bf41210..000000000000 --- a/src/arcade/eng/common/helixpublish.proj +++ /dev/null @@ -1,27 +0,0 @@ - - - - - msbuild - - - - - %(Identity) - - - - - - $(WorkItemDirectory) - $(WorkItemCommand) - $(WorkItemTimeout) - - - - - - - - - diff --git a/src/arcade/eng/common/init-tools-native.cmd b/src/arcade/eng/common/init-tools-native.cmd deleted file mode 100644 index 438cd548c452..000000000000 --- a/src/arcade/eng/common/init-tools-native.cmd +++ /dev/null @@ -1,3 +0,0 @@ -@echo off -powershell -NoProfile -NoLogo -ExecutionPolicy ByPass -command "& """%~dp0init-tools-native.ps1""" %*" -exit /b %ErrorLevel% \ No newline at end of file diff --git a/src/arcade/eng/common/init-tools-native.ps1 b/src/arcade/eng/common/init-tools-native.ps1 deleted file mode 100644 index 27ccdb9ecc95..000000000000 --- a/src/arcade/eng/common/init-tools-native.ps1 +++ /dev/null @@ -1,203 +0,0 @@ -<# -.SYNOPSIS -Entry point script for installing native tools - -.DESCRIPTION -Reads $RepoRoot\global.json file to determine native assets to install -and executes installers for those tools - -.PARAMETER BaseUri -Base file directory or Url from which to acquire tool archives - -.PARAMETER InstallDirectory -Directory to install native toolset. This is a command-line override for the default -Install directory precedence order: -- InstallDirectory command-line override -- NETCOREENG_INSTALL_DIRECTORY environment variable -- (default) %USERPROFILE%/.netcoreeng/native - -.PARAMETER Clean -Switch specifying to not install anything, but cleanup native asset folders - -.PARAMETER Force -Clean and then install tools - -.PARAMETER DownloadRetries -Total number of retry attempts - -.PARAMETER RetryWaitTimeInSeconds -Wait time between retry attempts in seconds - -.PARAMETER GlobalJsonFile -File path to global.json file - -.PARAMETER PathPromotion -Optional switch to enable either promote native tools specified in the global.json to the path (in Azure Pipelines) -or break the build if a native tool is not found on the path (on a local dev machine) - -.NOTES -#> -[CmdletBinding(PositionalBinding=$false)] -Param ( - [string] $BaseUri = 'https://netcorenativeassets.blob.core.windows.net/resource-packages/external', - [string] $InstallDirectory, - [switch] $Clean = $False, - [switch] $Force = $False, - [int] $DownloadRetries = 5, - [int] $RetryWaitTimeInSeconds = 30, - [string] $GlobalJsonFile, - [switch] $PathPromotion -) - -if (!$GlobalJsonFile) { - $GlobalJsonFile = Join-Path (Get-Item $PSScriptRoot).Parent.Parent.FullName 'global.json' -} - -Set-StrictMode -version 2.0 -$ErrorActionPreference='Stop' - -. $PSScriptRoot\pipeline-logging-functions.ps1 -Import-Module -Name (Join-Path $PSScriptRoot 'native\CommonLibrary.psm1') - -try { - # Define verbose switch if undefined - $Verbose = $VerbosePreference -Eq 'Continue' - - $EngCommonBaseDir = Join-Path $PSScriptRoot 'native\' - $NativeBaseDir = $InstallDirectory - if (!$NativeBaseDir) { - $NativeBaseDir = CommonLibrary\Get-NativeInstallDirectory - } - $Env:CommonLibrary_NativeInstallDir = $NativeBaseDir - $InstallBin = Join-Path $NativeBaseDir 'bin' - $InstallerPath = Join-Path $EngCommonBaseDir 'install-tool.ps1' - - # Process tools list - Write-Host "Processing $GlobalJsonFile" - If (-Not (Test-Path $GlobalJsonFile)) { - Write-Host "Unable to find '$GlobalJsonFile'" - exit 0 - } - $NativeTools = Get-Content($GlobalJsonFile) -Raw | - ConvertFrom-Json | - Select-Object -Expand 'native-tools' -ErrorAction SilentlyContinue - if ($NativeTools) { - if ($PathPromotion -eq $True) { - $ArcadeToolsDirectory = "$env:SYSTEMDRIVE\arcade-tools" - if (Test-Path $ArcadeToolsDirectory) { # if this directory exists, we should use native tools on machine - $NativeTools.PSObject.Properties | ForEach-Object { - $ToolName = $_.Name - $ToolVersion = $_.Value - $InstalledTools = @{} - - if ((Get-Command "$ToolName" -ErrorAction SilentlyContinue) -eq $null) { - if ($ToolVersion -eq "latest") { - $ToolVersion = "" - } - $ToolDirectories = (Get-ChildItem -Path "$ArcadeToolsDirectory" -Filter "$ToolName-$ToolVersion*" | Sort-Object -Descending) - if ($ToolDirectories -eq $null) { - Write-Error "Unable to find directory for $ToolName $ToolVersion; please make sure the tool is installed on this image." - exit 1 - } - $ToolDirectory = $ToolDirectories[0] - $BinPathFile = "$($ToolDirectory.FullName)\binpath.txt" - if (-not (Test-Path -Path "$BinPathFile")) { - Write-Error "Unable to find binpath.txt in '$($ToolDirectory.FullName)' ($ToolName $ToolVersion); artifact is either installed incorrectly or is not a bootstrappable tool." - exit 1 - } - $BinPath = Get-Content "$BinPathFile" - $ToolPath = Convert-Path -Path $BinPath - Write-Host "Adding $ToolName to the path ($ToolPath)..." - Write-Host "##vso[task.prependpath]$ToolPath" - $env:PATH = "$ToolPath;$env:PATH" - $InstalledTools += @{ $ToolName = $ToolDirectory.FullName } - } - } - return $InstalledTools - } else { - $NativeTools.PSObject.Properties | ForEach-Object { - $ToolName = $_.Name - $ToolVersion = $_.Value - - if ((Get-Command "$ToolName" -ErrorAction SilentlyContinue) -eq $null) { - Write-PipelineTelemetryError -Category 'NativeToolsBootstrap' -Message "$ToolName not found on path. Please install $ToolName $ToolVersion before proceeding." - Write-PipelineTelemetryError -Category 'NativeToolsBootstrap' -Message "If this is running on a build machine, the arcade-tools directory was not found, which means there's an error with the image." - } - } - exit 0 - } - } else { - $NativeTools.PSObject.Properties | ForEach-Object { - $ToolName = $_.Name - $ToolVersion = $_.Value - $LocalInstallerArguments = @{ ToolName = "$ToolName" } - $LocalInstallerArguments += @{ InstallPath = "$InstallBin" } - $LocalInstallerArguments += @{ BaseUri = "$BaseUri" } - $LocalInstallerArguments += @{ CommonLibraryDirectory = "$EngCommonBaseDir" } - $LocalInstallerArguments += @{ Version = "$ToolVersion" } - - if ($Verbose) { - $LocalInstallerArguments += @{ Verbose = $True } - } - if (Get-Variable 'Force' -ErrorAction 'SilentlyContinue') { - if($Force) { - $LocalInstallerArguments += @{ Force = $True } - } - } - if ($Clean) { - $LocalInstallerArguments += @{ Clean = $True } - } - - Write-Verbose "Installing $ToolName version $ToolVersion" - Write-Verbose "Executing '$InstallerPath $($LocalInstallerArguments.Keys.ForEach({"-$_ '$($LocalInstallerArguments.$_)'"}) -join ' ')'" - & $InstallerPath @LocalInstallerArguments - if ($LASTEXITCODE -Ne "0") { - $errMsg = "$ToolName installation failed" - if ((Get-Variable 'DoNotAbortNativeToolsInstallationOnFailure' -ErrorAction 'SilentlyContinue') -and $DoNotAbortNativeToolsInstallationOnFailure) { - $showNativeToolsWarning = $true - if ((Get-Variable 'DoNotDisplayNativeToolsInstallationWarnings' -ErrorAction 'SilentlyContinue') -and $DoNotDisplayNativeToolsInstallationWarnings) { - $showNativeToolsWarning = $false - } - if ($showNativeToolsWarning) { - Write-Warning $errMsg - } - $toolInstallationFailure = $true - } else { - # We cannot change this to Write-PipelineTelemetryError because of https://github.com/dotnet/arcade/issues/4482 - Write-Host $errMsg - exit 1 - } - } - } - - if ((Get-Variable 'toolInstallationFailure' -ErrorAction 'SilentlyContinue') -and $toolInstallationFailure) { - # We cannot change this to Write-PipelineTelemetryError because of https://github.com/dotnet/arcade/issues/4482 - Write-Host 'Native tools bootstrap failed' - exit 1 - } - } - } - else { - Write-Host 'No native tools defined in global.json' - exit 0 - } - - if ($Clean) { - exit 0 - } - if (Test-Path $InstallBin) { - Write-Host 'Native tools are available from ' (Convert-Path -Path $InstallBin) - Write-Host "##vso[task.prependpath]$(Convert-Path -Path $InstallBin)" - return $InstallBin - } - elseif (-not ($PathPromotion)) { - Write-PipelineTelemetryError -Category 'NativeToolsBootstrap' -Message 'Native tools install directory does not exist, installation failed' - exit 1 - } - exit 0 -} -catch { - Write-Host $_.ScriptStackTrace - Write-PipelineTelemetryError -Category 'NativeToolsBootstrap' -Message $_ - ExitWithExitCode 1 -} diff --git a/src/arcade/eng/common/init-tools-native.sh b/src/arcade/eng/common/init-tools-native.sh deleted file mode 100644 index 3e6a8d6acf2f..000000000000 --- a/src/arcade/eng/common/init-tools-native.sh +++ /dev/null @@ -1,238 +0,0 @@ -#!/usr/bin/env bash - -source="${BASH_SOURCE[0]}" -scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" - -base_uri='https://netcorenativeassets.blob.core.windows.net/resource-packages/external' -install_directory='' -clean=false -force=false -download_retries=5 -retry_wait_time_seconds=30 -global_json_file="$(dirname "$(dirname "${scriptroot}")")/global.json" -declare -a native_assets - -. $scriptroot/pipeline-logging-functions.sh -. $scriptroot/native/common-library.sh - -while (($# > 0)); do - lowerI="$(echo $1 | tr "[:upper:]" "[:lower:]")" - case $lowerI in - --baseuri) - base_uri=$2 - shift 2 - ;; - --installdirectory) - install_directory=$2 - shift 2 - ;; - --clean) - clean=true - shift 1 - ;; - --force) - force=true - shift 1 - ;; - --donotabortonfailure) - donotabortonfailure=true - shift 1 - ;; - --donotdisplaywarnings) - donotdisplaywarnings=true - shift 1 - ;; - --downloadretries) - download_retries=$2 - shift 2 - ;; - --retrywaittimeseconds) - retry_wait_time_seconds=$2 - shift 2 - ;; - --help) - echo "Common settings:" - echo " --installdirectory Directory to install native toolset." - echo " This is a command-line override for the default" - echo " Install directory precedence order:" - echo " - InstallDirectory command-line override" - echo " - NETCOREENG_INSTALL_DIRECTORY environment variable" - echo " - (default) %USERPROFILE%/.netcoreeng/native" - echo "" - echo " --clean Switch specifying not to install anything, but cleanup native asset folders" - echo " --donotabortonfailure Switch specifiying whether to abort native tools installation on failure" - echo " --donotdisplaywarnings Switch specifiying whether to display warnings during native tools installation on failure" - echo " --force Clean and then install tools" - echo " --help Print help and exit" - echo "" - echo "Advanced settings:" - echo " --baseuri Base URI for where to download native tools from" - echo " --downloadretries Number of times a download should be attempted" - echo " --retrywaittimeseconds Wait time between download attempts" - echo "" - exit 0 - ;; - esac -done - -function ReadGlobalJsonNativeTools { - # happy path: we have a proper JSON parsing tool `jq(1)` in PATH! - if command -v jq &> /dev/null; then - - # jq: read each key/value pair under "native-tools" entry and emit: - # KEY="" VALUE="" - # followed by a null byte. - # - # bash: read line with null byte delimeter and push to array (for later `eval`uation). - - while IFS= read -rd '' line; do - native_assets+=("$line") - done < <(jq -r '. | - select(has("native-tools")) | - ."native-tools" | - keys[] as $k | - @sh "KEY=\($k) VALUE=\(.[$k])\u0000"' "$global_json_file") - - return - fi - - # Warning: falling back to manually parsing JSON, which is not recommended. - - # Following routine matches the output and escaping logic of jq(1)'s @sh formatter used above. - # It has been tested with several weird strings with escaped characters in entries (key and value) - # and results were compared with the output of jq(1) in binary representation using xxd(1); - # just before the assignment to 'native_assets' array (above and below). - - # try to capture the section under "native-tools". - if [[ ! "$(cat "$global_json_file")" =~ \"native-tools\"[[:space:]\:\{]*([^\}]+) ]]; then - return - fi - - section="${BASH_REMATCH[1]}" - - parseStarted=0 - possibleEnd=0 - escaping=0 - escaped=0 - isKey=1 - - for (( i=0; i<${#section}; i++ )); do - char="${section:$i:1}" - if ! ((parseStarted)) && [[ "$char" =~ [[:space:],:] ]]; then continue; fi - - if ! ((escaping)) && [[ "$char" == "\\" ]]; then - escaping=1 - elif ((escaping)) && ! ((escaped)); then - escaped=1 - fi - - if ! ((parseStarted)) && [[ "$char" == "\"" ]]; then - parseStarted=1 - possibleEnd=0 - elif [[ "$char" == "'" ]]; then - token="$token'\\\''" - possibleEnd=0 - elif ((escaping)) || [[ "$char" != "\"" ]]; then - token="$token$char" - possibleEnd=1 - fi - - if ((possibleEnd)) && ! ((escaping)) && [[ "$char" == "\"" ]]; then - # Use printf to unescape token to match jq(1)'s @sh formatting rules. - # do not use 'token="$(printf "$token")"' syntax, as $() eats the trailing linefeed. - printf -v token "'$token'" - - if ((isKey)); then - KEY="$token" - isKey=0 - else - line="KEY=$KEY VALUE=$token" - native_assets+=("$line") - isKey=1 - fi - - # reset for next token - parseStarted=0 - token= - elif ((escaping)) && ((escaped)); then - escaping=0 - escaped=0 - fi - done -} - -native_base_dir=$install_directory -if [[ -z $install_directory ]]; then - native_base_dir=$(GetNativeInstallDirectory) -fi - -install_bin="${native_base_dir}/bin" -installed_any=false - -ReadGlobalJsonNativeTools - -if [[ ${#native_assets[@]} -eq 0 ]]; then - echo "No native tools defined in global.json" - exit 0; -else - native_installer_dir="$scriptroot/native" - for index in "${!native_assets[@]}"; do - eval "${native_assets["$index"]}" - - installer_path="$native_installer_dir/install-$KEY.sh" - installer_command="$installer_path" - installer_command+=" --baseuri $base_uri" - installer_command+=" --installpath $install_bin" - installer_command+=" --version $VALUE" - echo $installer_command - - if [[ $force = true ]]; then - installer_command+=" --force" - fi - - if [[ $clean = true ]]; then - installer_command+=" --clean" - fi - - if [[ -a $installer_path ]]; then - $installer_command - if [[ $? != 0 ]]; then - if [[ $donotabortonfailure = true ]]; then - if [[ $donotdisplaywarnings != true ]]; then - Write-PipelineTelemetryError -category 'NativeToolsBootstrap' "Execution Failed" - fi - else - Write-PipelineTelemetryError -category 'NativeToolsBootstrap' "Execution Failed" - exit 1 - fi - else - $installed_any = true - fi - else - if [[ $donotabortonfailure == true ]]; then - if [[ $donotdisplaywarnings != true ]]; then - Write-PipelineTelemetryError -category 'NativeToolsBootstrap' "Execution Failed: no install script" - fi - else - Write-PipelineTelemetryError -category 'NativeToolsBootstrap' "Execution Failed: no install script" - exit 1 - fi - fi - done -fi - -if [[ $clean = true ]]; then - exit 0 -fi - -if [[ -d $install_bin ]]; then - echo "Native tools are available from $install_bin" - echo "##vso[task.prependpath]$install_bin" -else - if [[ $installed_any = true ]]; then - Write-PipelineTelemetryError -category 'NativeToolsBootstrap' "Native tools install directory does not exist, installation failed" - exit 1 - fi -fi - -exit 0 diff --git a/src/arcade/eng/common/internal-feed-operations.ps1 b/src/arcade/eng/common/internal-feed-operations.ps1 deleted file mode 100644 index 92b77347d990..000000000000 --- a/src/arcade/eng/common/internal-feed-operations.ps1 +++ /dev/null @@ -1,132 +0,0 @@ -param( - [Parameter(Mandatory=$true)][string] $Operation, - [string] $AuthToken, - [string] $CommitSha, - [string] $RepoName, - [switch] $IsFeedPrivate -) - -$ErrorActionPreference = 'Stop' -Set-StrictMode -Version 2.0 -. $PSScriptRoot\tools.ps1 - -# Sets VSS_NUGET_EXTERNAL_FEED_ENDPOINTS based on the "darc-int-*" feeds defined in NuGet.config. This is needed -# in build agents by CredProvider to authenticate the restore requests to internal feeds as specified in -# https://github.com/microsoft/artifacts-credprovider/blob/0f53327cd12fd893d8627d7b08a2171bf5852a41/README.md#environment-variables. This should ONLY be called from identified -# internal builds -function SetupCredProvider { - param( - [string] $AuthToken - ) - - # Install the Cred Provider NuGet plugin - Write-Host 'Setting up Cred Provider NuGet plugin in the agent...' - Write-Host "Getting 'installcredprovider.ps1' from 'https://github.com/microsoft/artifacts-credprovider'..." - - $url = 'https://raw.githubusercontent.com/microsoft/artifacts-credprovider/master/helpers/installcredprovider.ps1' - - Write-Host "Writing the contents of 'installcredprovider.ps1' locally..." - Invoke-WebRequest $url -OutFile installcredprovider.ps1 - - Write-Host 'Installing plugin...' - .\installcredprovider.ps1 -Force - - Write-Host "Deleting local copy of 'installcredprovider.ps1'..." - Remove-Item .\installcredprovider.ps1 - - if (-Not("$env:USERPROFILE\.nuget\plugins\netcore")) { - Write-PipelineTelemetryError -Category 'Arcade' -Message 'CredProvider plugin was not installed correctly!' - ExitWithExitCode 1 - } - else { - Write-Host 'CredProvider plugin was installed correctly!' - } - - # Then, we set the 'VSS_NUGET_EXTERNAL_FEED_ENDPOINTS' environment variable to restore from the stable - # feeds successfully - - $nugetConfigPath = Join-Path $RepoRoot "NuGet.config" - - if (-Not (Test-Path -Path $nugetConfigPath)) { - Write-PipelineTelemetryError -Category 'Build' -Message 'NuGet.config file not found in repo root!' - ExitWithExitCode 1 - } - - $endpoints = New-Object System.Collections.ArrayList - $nugetConfigPackageSources = Select-Xml -Path $nugetConfigPath -XPath "//packageSources/add[contains(@key, 'darc-int-')]/@value" | foreach{$_.Node.Value} - - if (($nugetConfigPackageSources | Measure-Object).Count -gt 0 ) { - foreach ($stableRestoreResource in $nugetConfigPackageSources) { - $trimmedResource = ([string]$stableRestoreResource).Trim() - [void]$endpoints.Add(@{endpoint="$trimmedResource"; password="$AuthToken"}) - } - } - - if (($endpoints | Measure-Object).Count -gt 0) { - $endpointCredentials = @{endpointCredentials=$endpoints} | ConvertTo-Json -Compress - - # Create the environment variables the AzDo way - Write-LoggingCommand -Area 'task' -Event 'setvariable' -Data $endpointCredentials -Properties @{ - 'variable' = 'VSS_NUGET_EXTERNAL_FEED_ENDPOINTS' - 'issecret' = 'false' - } - - # We don't want sessions cached since we will be updating the endpoints quite frequently - Write-LoggingCommand -Area 'task' -Event 'setvariable' -Data 'False' -Properties @{ - 'variable' = 'NUGET_CREDENTIALPROVIDER_SESSIONTOKENCACHE_ENABLED' - 'issecret' = 'false' - } - } - else - { - Write-Host 'No internal endpoints found in NuGet.config' - } -} - -#Workaround for https://github.com/microsoft/msbuild/issues/4430 -function InstallDotNetSdkAndRestoreArcade { - $dotnetTempDir = Join-Path $RepoRoot "dotnet" - $dotnetSdkVersion="2.1.507" # After experimentation we know this version works when restoring the SDK (compared to 3.0.*) - $dotnet = "$dotnetTempDir\dotnet.exe" - $restoreProjPath = "$PSScriptRoot\restore.proj" - - Write-Host "Installing dotnet SDK version $dotnetSdkVersion to restore Arcade SDK..." - InstallDotNetSdk "$dotnetTempDir" "$dotnetSdkVersion" - - '' | Out-File "$restoreProjPath" - - & $dotnet restore $restoreProjPath - - Write-Host 'Arcade SDK restored!' - - if (Test-Path -Path $restoreProjPath) { - Remove-Item $restoreProjPath - } - - if (Test-Path -Path $dotnetTempDir) { - Remove-Item $dotnetTempDir -Recurse - } -} - -try { - Push-Location $PSScriptRoot - - if ($Operation -like 'setup') { - SetupCredProvider $AuthToken - } - elseif ($Operation -like 'install-restore') { - InstallDotNetSdkAndRestoreArcade - } - else { - Write-PipelineTelemetryError -Category 'Arcade' -Message "Unknown operation '$Operation'!" - ExitWithExitCode 1 - } -} -catch { - Write-Host $_.ScriptStackTrace - Write-PipelineTelemetryError -Category 'Arcade' -Message $_ - ExitWithExitCode 1 -} -finally { - Pop-Location -} diff --git a/src/arcade/eng/common/internal-feed-operations.sh b/src/arcade/eng/common/internal-feed-operations.sh deleted file mode 100644 index 9378223ba095..000000000000 --- a/src/arcade/eng/common/internal-feed-operations.sh +++ /dev/null @@ -1,141 +0,0 @@ -#!/usr/bin/env bash - -set -e - -# Sets VSS_NUGET_EXTERNAL_FEED_ENDPOINTS based on the "darc-int-*" feeds defined in NuGet.config. This is needed -# in build agents by CredProvider to authenticate the restore requests to internal feeds as specified in -# https://github.com/microsoft/artifacts-credprovider/blob/0f53327cd12fd893d8627d7b08a2171bf5852a41/README.md#environment-variables. -# This should ONLY be called from identified internal builds -function SetupCredProvider { - local authToken=$1 - - # Install the Cred Provider NuGet plugin - echo "Setting up Cred Provider NuGet plugin in the agent..."... - echo "Getting 'installcredprovider.ps1' from 'https://github.com/microsoft/artifacts-credprovider'..." - - local url="https://raw.githubusercontent.com/microsoft/artifacts-credprovider/master/helpers/installcredprovider.sh" - - echo "Writing the contents of 'installcredprovider.ps1' locally..." - local installcredproviderPath="installcredprovider.sh" - if command -v curl > /dev/null; then - curl $url > "$installcredproviderPath" - else - wget -q -O "$installcredproviderPath" "$url" - fi - - echo "Installing plugin..." - . "$installcredproviderPath" - - echo "Deleting local copy of 'installcredprovider.sh'..." - rm installcredprovider.sh - - if [ ! -d "$HOME/.nuget/plugins" ]; then - Write-PipelineTelemetryError -category 'Build' 'CredProvider plugin was not installed correctly!' - ExitWithExitCode 1 - else - echo "CredProvider plugin was installed correctly!" - fi - - # Then, we set the 'VSS_NUGET_EXTERNAL_FEED_ENDPOINTS' environment variable to restore from the stable - # feeds successfully - - local nugetConfigPath="{$repo_root}NuGet.config" - - if [ ! "$nugetConfigPath" ]; then - Write-PipelineTelemetryError -category 'Build' "NuGet.config file not found in repo's root!" - ExitWithExitCode 1 - fi - - local endpoints='[' - local nugetConfigPackageValues=`cat "$nugetConfigPath" | grep "key=\"darc-int-"` - local pattern="value=\"(.*)\"" - - for value in $nugetConfigPackageValues - do - if [[ $value =~ $pattern ]]; then - local endpoint="${BASH_REMATCH[1]}" - endpoints+="{\"endpoint\": \"$endpoint\", \"password\": \"$authToken\"}," - fi - done - - endpoints=${endpoints%?} - endpoints+=']' - - if [ ${#endpoints} -gt 2 ]; then - local endpointCredentials="{\"endpointCredentials\": "$endpoints"}" - - echo "##vso[task.setvariable variable=VSS_NUGET_EXTERNAL_FEED_ENDPOINTS]$endpointCredentials" - echo "##vso[task.setvariable variable=NUGET_CREDENTIALPROVIDER_SESSIONTOKENCACHE_ENABLED]False" - else - echo "No internal endpoints found in NuGet.config" - fi -} - -# Workaround for https://github.com/microsoft/msbuild/issues/4430 -function InstallDotNetSdkAndRestoreArcade { - local dotnetTempDir="$repo_root/dotnet" - local dotnetSdkVersion="2.1.507" # After experimentation we know this version works when restoring the SDK (compared to 3.0.*) - local restoreProjPath="$repo_root/eng/common/restore.proj" - - echo "Installing dotnet SDK version $dotnetSdkVersion to restore Arcade SDK..." - echo "" > "$restoreProjPath" - - InstallDotNetSdk "$dotnetTempDir" "$dotnetSdkVersion" - - local res=`$dotnetTempDir/dotnet restore $restoreProjPath` - echo "Arcade SDK restored!" - - # Cleanup - if [ "$restoreProjPath" ]; then - rm "$restoreProjPath" - fi - - if [ "$dotnetTempDir" ]; then - rm -r $dotnetTempDir - fi -} - -source="${BASH_SOURCE[0]}" -operation='' -authToken='' -repoName='' - -while [[ $# > 0 ]]; do - opt="$(echo "$1" | tr "[:upper:]" "[:lower:]")" - case "$opt" in - --operation) - operation=$2 - shift - ;; - --authtoken) - authToken=$2 - shift - ;; - *) - echo "Invalid argument: $1" - usage - exit 1 - ;; - esac - - shift -done - -while [[ -h "$source" ]]; do - scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" - source="$(readlink "$source")" - # if $source was a relative symlink, we need to resolve it relative to the path where the - # symlink file was located - [[ $source != /* ]] && source="$scriptroot/$source" -done -scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" - -. "$scriptroot/tools.sh" - -if [ "$operation" = "setup" ]; then - SetupCredProvider $authToken -elif [ "$operation" = "install-restore" ]; then - InstallDotNetSdkAndRestoreArcade -else - echo "Unknown operation '$operation'!" -fi diff --git a/src/arcade/eng/common/internal/Directory.Build.props b/src/arcade/eng/common/internal/Directory.Build.props deleted file mode 100644 index f1d041c33da5..000000000000 --- a/src/arcade/eng/common/internal/Directory.Build.props +++ /dev/null @@ -1,11 +0,0 @@ - - - - - false - false - - - - - diff --git a/src/arcade/eng/common/internal/NuGet.config b/src/arcade/eng/common/internal/NuGet.config deleted file mode 100644 index 19d3d311b166..000000000000 --- a/src/arcade/eng/common/internal/NuGet.config +++ /dev/null @@ -1,7 +0,0 @@ - - - - - - - diff --git a/src/arcade/eng/common/internal/Tools.csproj b/src/arcade/eng/common/internal/Tools.csproj deleted file mode 100644 index feaa6d20812d..000000000000 --- a/src/arcade/eng/common/internal/Tools.csproj +++ /dev/null @@ -1,22 +0,0 @@ - - - - - net472 - false - false - - - - - - - - - - - - - - - diff --git a/src/arcade/eng/common/loc/P22DotNetHtmlLocalization.lss b/src/arcade/eng/common/loc/P22DotNetHtmlLocalization.lss deleted file mode 100644 index 5d892d619398..000000000000 --- a/src/arcade/eng/common/loc/P22DotNetHtmlLocalization.lss +++ /dev/null @@ -1,29 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/src/arcade/eng/common/msbuild.ps1 b/src/arcade/eng/common/msbuild.ps1 deleted file mode 100644 index f041e5ddd958..000000000000 --- a/src/arcade/eng/common/msbuild.ps1 +++ /dev/null @@ -1,28 +0,0 @@ -[CmdletBinding(PositionalBinding=$false)] -Param( - [string] $verbosity = 'minimal', - [bool] $warnAsError = $true, - [bool] $nodeReuse = $true, - [switch] $ci, - [switch] $prepareMachine, - [switch] $excludePrereleaseVS, - [string] $msbuildEngine = $null, - [Parameter(ValueFromRemainingArguments=$true)][String[]]$extraArgs -) - -. $PSScriptRoot\tools.ps1 - -try { - if ($ci) { - $nodeReuse = $false - } - - MSBuild @extraArgs -} -catch { - Write-Host $_.ScriptStackTrace - Write-PipelineTelemetryError -Category 'Build' -Message $_ - ExitWithExitCode 1 -} - -ExitWithExitCode 0 \ No newline at end of file diff --git a/src/arcade/eng/common/msbuild.sh b/src/arcade/eng/common/msbuild.sh deleted file mode 100644 index 20d3dad54352..000000000000 --- a/src/arcade/eng/common/msbuild.sh +++ /dev/null @@ -1,58 +0,0 @@ -#!/usr/bin/env bash - -source="${BASH_SOURCE[0]}" - -# resolve $source until the file is no longer a symlink -while [[ -h "$source" ]]; do - scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" - source="$(readlink "$source")" - # if $source was a relative symlink, we need to resolve it relative to the path where the - # symlink file was located - [[ $source != /* ]] && source="$scriptroot/$source" -done -scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" - -verbosity='minimal' -warn_as_error=true -node_reuse=true -prepare_machine=false -extra_args='' - -while (($# > 0)); do - lowerI="$(echo $1 | tr "[:upper:]" "[:lower:]")" - case $lowerI in - --verbosity) - verbosity=$2 - shift 2 - ;; - --warnaserror) - warn_as_error=$2 - shift 2 - ;; - --nodereuse) - node_reuse=$2 - shift 2 - ;; - --ci) - ci=true - shift 1 - ;; - --preparemachine) - prepare_machine=true - shift 1 - ;; - *) - extra_args="$extra_args $1" - shift 1 - ;; - esac -done - -. "$scriptroot/tools.sh" - -if [[ "$ci" == true ]]; then - node_reuse=false -fi - -MSBuild $extra_args -ExitWithExitCode 0 diff --git a/src/arcade/eng/common/native/CommonLibrary.psm1 b/src/arcade/eng/common/native/CommonLibrary.psm1 deleted file mode 100644 index f71f6af6cdbc..000000000000 --- a/src/arcade/eng/common/native/CommonLibrary.psm1 +++ /dev/null @@ -1,401 +0,0 @@ -<# -.SYNOPSIS -Helper module to install an archive to a directory - -.DESCRIPTION -Helper module to download and extract an archive to a specified directory - -.PARAMETER Uri -Uri of artifact to download - -.PARAMETER InstallDirectory -Directory to extract artifact contents to - -.PARAMETER Force -Force download / extraction if file or contents already exist. Default = False - -.PARAMETER DownloadRetries -Total number of retry attempts. Default = 5 - -.PARAMETER RetryWaitTimeInSeconds -Wait time between retry attempts in seconds. Default = 30 - -.NOTES -Returns False if download or extraction fail, True otherwise -#> -function DownloadAndExtract { - [CmdletBinding(PositionalBinding=$false)] - Param ( - [Parameter(Mandatory=$True)] - [string] $Uri, - [Parameter(Mandatory=$True)] - [string] $InstallDirectory, - [switch] $Force = $False, - [int] $DownloadRetries = 5, - [int] $RetryWaitTimeInSeconds = 30 - ) - # Define verbose switch if undefined - $Verbose = $VerbosePreference -Eq "Continue" - - $TempToolPath = CommonLibrary\Get-TempPathFilename -Path $Uri - - # Download native tool - $DownloadStatus = CommonLibrary\Get-File -Uri $Uri ` - -Path $TempToolPath ` - -DownloadRetries $DownloadRetries ` - -RetryWaitTimeInSeconds $RetryWaitTimeInSeconds ` - -Force:$Force ` - -Verbose:$Verbose - - if ($DownloadStatus -Eq $False) { - Write-Error "Download failed from $Uri" - return $False - } - - # Extract native tool - $UnzipStatus = CommonLibrary\Expand-Zip -ZipPath $TempToolPath ` - -OutputDirectory $InstallDirectory ` - -Force:$Force ` - -Verbose:$Verbose - - if ($UnzipStatus -Eq $False) { - # Retry Download one more time with Force=true - $DownloadRetryStatus = CommonLibrary\Get-File -Uri $Uri ` - -Path $TempToolPath ` - -DownloadRetries 1 ` - -RetryWaitTimeInSeconds $RetryWaitTimeInSeconds ` - -Force:$True ` - -Verbose:$Verbose - - if ($DownloadRetryStatus -Eq $False) { - Write-Error "Last attempt of download failed as well" - return $False - } - - # Retry unzip again one more time with Force=true - $UnzipRetryStatus = CommonLibrary\Expand-Zip -ZipPath $TempToolPath ` - -OutputDirectory $InstallDirectory ` - -Force:$True ` - -Verbose:$Verbose - if ($UnzipRetryStatus -Eq $False) - { - Write-Error "Last attempt of unzip failed as well" - # Clean up partial zips and extracts - if (Test-Path $TempToolPath) { - Remove-Item $TempToolPath -Force - } - if (Test-Path $InstallDirectory) { - Remove-Item $InstallDirectory -Force -Recurse - } - return $False - } - } - - return $True -} - -<# -.SYNOPSIS -Download a file, retry on failure - -.DESCRIPTION -Download specified file and retry if attempt fails - -.PARAMETER Uri -Uri of file to download. If Uri is a local path, the file will be copied instead of downloaded - -.PARAMETER Path -Path to download or copy uri file to - -.PARAMETER Force -Overwrite existing file if present. Default = False - -.PARAMETER DownloadRetries -Total number of retry attempts. Default = 5 - -.PARAMETER RetryWaitTimeInSeconds -Wait time between retry attempts in seconds Default = 30 - -#> -function Get-File { - [CmdletBinding(PositionalBinding=$false)] - Param ( - [Parameter(Mandatory=$True)] - [string] $Uri, - [Parameter(Mandatory=$True)] - [string] $Path, - [int] $DownloadRetries = 5, - [int] $RetryWaitTimeInSeconds = 30, - [switch] $Force = $False - ) - $Attempt = 0 - - if ($Force) { - if (Test-Path $Path) { - Remove-Item $Path -Force - } - } - if (Test-Path $Path) { - Write-Host "File '$Path' already exists, skipping download" - return $True - } - - $DownloadDirectory = Split-Path -ErrorAction Ignore -Path "$Path" -Parent - if (-Not (Test-Path $DownloadDirectory)) { - New-Item -path $DownloadDirectory -force -itemType "Directory" | Out-Null - } - - $TempPath = "$Path.tmp" - if (Test-Path -IsValid -Path $Uri) { - Write-Verbose "'$Uri' is a file path, copying temporarily to '$TempPath'" - Copy-Item -Path $Uri -Destination $TempPath - Write-Verbose "Moving temporary file to '$Path'" - Move-Item -Path $TempPath -Destination $Path - return $? - } - else { - Write-Verbose "Downloading $Uri" - # Don't display the console progress UI - it's a huge perf hit - $ProgressPreference = 'SilentlyContinue' - while($Attempt -Lt $DownloadRetries) - { - try { - Invoke-WebRequest -UseBasicParsing -Uri $Uri -OutFile $TempPath - Write-Verbose "Downloaded to temporary location '$TempPath'" - Move-Item -Path $TempPath -Destination $Path - Write-Verbose "Moved temporary file to '$Path'" - return $True - } - catch { - $Attempt++ - if ($Attempt -Lt $DownloadRetries) { - $AttemptsLeft = $DownloadRetries - $Attempt - Write-Warning "Download failed, $AttemptsLeft attempts remaining, will retry in $RetryWaitTimeInSeconds seconds" - Start-Sleep -Seconds $RetryWaitTimeInSeconds - } - else { - Write-Error $_ - Write-Error $_.Exception - } - } - } - } - - return $False -} - -<# -.SYNOPSIS -Generate a shim for a native tool - -.DESCRIPTION -Creates a wrapper script (shim) that passes arguments forward to native tool assembly - -.PARAMETER ShimName -The name of the shim - -.PARAMETER ShimDirectory -The directory where shims are stored - -.PARAMETER ToolFilePath -Path to file that shim forwards to - -.PARAMETER Force -Replace shim if already present. Default = False - -.NOTES -Returns $True if generating shim succeeds, $False otherwise -#> -function New-ScriptShim { - [CmdletBinding(PositionalBinding=$false)] - Param ( - [Parameter(Mandatory=$True)] - [string] $ShimName, - [Parameter(Mandatory=$True)] - [string] $ShimDirectory, - [Parameter(Mandatory=$True)] - [string] $ToolFilePath, - [Parameter(Mandatory=$True)] - [string] $BaseUri, - [switch] $Force - ) - try { - Write-Verbose "Generating '$ShimName' shim" - - if (-Not (Test-Path $ToolFilePath)){ - Write-Error "Specified tool file path '$ToolFilePath' does not exist" - return $False - } - - # WinShimmer is a small .NET Framework program that creates .exe shims to bootstrapped programs - # Many of the checks for installed programs expect a .exe extension for Windows tools, rather - # than a .bat or .cmd file. - # Source: https://github.com/dotnet/arcade/tree/master/src/WinShimmer - if (-Not (Test-Path "$ShimDirectory\WinShimmer\winshimmer.exe")) { - $InstallStatus = DownloadAndExtract -Uri "$BaseUri/windows/winshimmer/WinShimmer.zip" ` - -InstallDirectory $ShimDirectory\WinShimmer ` - -Force:$Force ` - -DownloadRetries 2 ` - -RetryWaitTimeInSeconds 5 ` - -Verbose:$Verbose - } - - if ((Test-Path (Join-Path $ShimDirectory "$ShimName.exe"))) { - Write-Host "$ShimName.exe already exists; replacing..." - Remove-Item (Join-Path $ShimDirectory "$ShimName.exe") - } - - & "$ShimDirectory\WinShimmer\winshimmer.exe" $ShimName $ToolFilePath $ShimDirectory - return $True - } - catch { - Write-Host $_ - Write-Host $_.Exception - return $False - } -} - -<# -.SYNOPSIS -Returns the machine architecture of the host machine - -.NOTES -Returns 'x64' on 64 bit machines - Returns 'x86' on 32 bit machines -#> -function Get-MachineArchitecture { - $ProcessorArchitecture = $Env:PROCESSOR_ARCHITECTURE - $ProcessorArchitectureW6432 = $Env:PROCESSOR_ARCHITEW6432 - if($ProcessorArchitecture -Eq "X86") - { - if(($ProcessorArchitectureW6432 -Eq "") -Or - ($ProcessorArchitectureW6432 -Eq "X86")) { - return "x86" - } - $ProcessorArchitecture = $ProcessorArchitectureW6432 - } - if (($ProcessorArchitecture -Eq "AMD64") -Or - ($ProcessorArchitecture -Eq "IA64") -Or - ($ProcessorArchitecture -Eq "ARM64") -Or - ($ProcessorArchitecture -Eq "LOONGARCH64") -Or - ($ProcessorArchitecture -Eq "RISCV64")) { - return "x64" - } - return "x86" -} - -<# -.SYNOPSIS -Get the name of a temporary folder under the native install directory -#> -function Get-TempDirectory { - return Join-Path (Get-NativeInstallDirectory) "temp/" -} - -function Get-TempPathFilename { - [CmdletBinding(PositionalBinding=$false)] - Param ( - [Parameter(Mandatory=$True)] - [string] $Path - ) - $TempDir = CommonLibrary\Get-TempDirectory - $TempFilename = Split-Path $Path -leaf - $TempPath = Join-Path $TempDir $TempFilename - return $TempPath -} - -<# -.SYNOPSIS -Returns the base directory to use for native tool installation - -.NOTES -Returns the value of the NETCOREENG_INSTALL_DIRECTORY if that environment variable -is set, or otherwise returns an install directory under the %USERPROFILE% -#> -function Get-NativeInstallDirectory { - $InstallDir = $Env:NETCOREENG_INSTALL_DIRECTORY - if (!$InstallDir) { - $InstallDir = Join-Path $Env:USERPROFILE ".netcoreeng/native/" - } - return $InstallDir -} - -<# -.SYNOPSIS -Unzip an archive - -.DESCRIPTION -Powershell module to unzip an archive to a specified directory - -.PARAMETER ZipPath (Required) -Path to archive to unzip - -.PARAMETER OutputDirectory (Required) -Output directory for archive contents - -.PARAMETER Force -Overwrite output directory contents if they already exist - -.NOTES -- Returns True and does not perform an extraction if output directory already exists but Overwrite is not True. -- Returns True if unzip operation is successful -- Returns False if Overwrite is True and it is unable to remove contents of OutputDirectory -- Returns False if unable to extract zip archive -#> -function Expand-Zip { - [CmdletBinding(PositionalBinding=$false)] - Param ( - [Parameter(Mandatory=$True)] - [string] $ZipPath, - [Parameter(Mandatory=$True)] - [string] $OutputDirectory, - [switch] $Force - ) - - Write-Verbose "Extracting '$ZipPath' to '$OutputDirectory'" - try { - if ((Test-Path $OutputDirectory) -And (-Not $Force)) { - Write-Host "Directory '$OutputDirectory' already exists, skipping extract" - return $True - } - if (Test-Path $OutputDirectory) { - Write-Verbose "'Force' is 'True', but '$OutputDirectory' exists, removing directory" - Remove-Item $OutputDirectory -Force -Recurse - if ($? -Eq $False) { - Write-Error "Unable to remove '$OutputDirectory'" - return $False - } - } - - $TempOutputDirectory = Join-Path "$(Split-Path -Parent $OutputDirectory)" "$(Split-Path -Leaf $OutputDirectory).tmp" - if (Test-Path $TempOutputDirectory) { - Remove-Item $TempOutputDirectory -Force -Recurse - } - New-Item -Path $TempOutputDirectory -Force -ItemType "Directory" | Out-Null - - Add-Type -assembly "system.io.compression.filesystem" - [io.compression.zipfile]::ExtractToDirectory("$ZipPath", "$TempOutputDirectory") - if ($? -Eq $False) { - Write-Error "Unable to extract '$ZipPath'" - return $False - } - - Move-Item -Path $TempOutputDirectory -Destination $OutputDirectory - } - catch { - Write-Host $_ - Write-Host $_.Exception - - return $False - } - return $True -} - -export-modulemember -function DownloadAndExtract -export-modulemember -function Expand-Zip -export-modulemember -function Get-File -export-modulemember -function Get-MachineArchitecture -export-modulemember -function Get-NativeInstallDirectory -export-modulemember -function Get-TempDirectory -export-modulemember -function Get-TempPathFilename -export-modulemember -function New-ScriptShim diff --git a/src/arcade/eng/common/native/common-library.sh b/src/arcade/eng/common/native/common-library.sh deleted file mode 100644 index 080c2c283ae4..000000000000 --- a/src/arcade/eng/common/native/common-library.sh +++ /dev/null @@ -1,172 +0,0 @@ -#!/usr/bin/env bash - -function GetNativeInstallDirectory { - local install_dir - - if [[ -z $NETCOREENG_INSTALL_DIRECTORY ]]; then - install_dir=$HOME/.netcoreeng/native/ - else - install_dir=$NETCOREENG_INSTALL_DIRECTORY - fi - - echo $install_dir - return 0 -} - -function GetTempDirectory { - - echo $(GetNativeInstallDirectory)temp/ - return 0 -} - -function ExpandZip { - local zip_path=$1 - local output_directory=$2 - local force=${3:-false} - - echo "Extracting $zip_path to $output_directory" - if [[ -d $output_directory ]] && [[ $force = false ]]; then - echo "Directory '$output_directory' already exists, skipping extract" - return 0 - fi - - if [[ -d $output_directory ]]; then - echo "'Force flag enabled, but '$output_directory' exists. Removing directory" - rm -rf $output_directory - if [[ $? != 0 ]]; then - Write-PipelineTelemetryError -category 'NativeToolsBootstrap' "Unable to remove '$output_directory'" - return 1 - fi - fi - - echo "Creating directory: '$output_directory'" - mkdir -p $output_directory - - echo "Extracting archive" - tar -xf $zip_path -C $output_directory - if [[ $? != 0 ]]; then - Write-PipelineTelemetryError -category 'NativeToolsBootstrap' "Unable to extract '$zip_path'" - return 1 - fi - - return 0 -} - -function GetCurrentOS { - local unameOut="$(uname -s)" - case $unameOut in - Linux*) echo "Linux";; - Darwin*) echo "MacOS";; - esac - return 0 -} - -function GetFile { - local uri=$1 - local path=$2 - local force=${3:-false} - local download_retries=${4:-5} - local retry_wait_time_seconds=${5:-30} - - if [[ -f $path ]]; then - if [[ $force = false ]]; then - echo "File '$path' already exists. Skipping download" - return 0 - else - rm -rf $path - fi - fi - - if [[ -f $uri ]]; then - echo "'$uri' is a file path, copying file to '$path'" - cp $uri $path - return $? - fi - - echo "Downloading $uri" - # Use curl if available, otherwise use wget - if command -v curl > /dev/null; then - curl "$uri" -sSL --retry $download_retries --retry-delay $retry_wait_time_seconds --create-dirs -o "$path" --fail - else - wget -q -O "$path" "$uri" --tries="$download_retries" - fi - - return $? -} - -function GetTempPathFileName { - local path=$1 - - local temp_dir=$(GetTempDirectory) - local temp_file_name=$(basename $path) - echo $temp_dir$temp_file_name - return 0 -} - -function DownloadAndExtract { - local uri=$1 - local installDir=$2 - local force=${3:-false} - local download_retries=${4:-5} - local retry_wait_time_seconds=${5:-30} - - local temp_tool_path=$(GetTempPathFileName $uri) - - echo "downloading to: $temp_tool_path" - - # Download file - GetFile "$uri" "$temp_tool_path" $force $download_retries $retry_wait_time_seconds - if [[ $? != 0 ]]; then - Write-PipelineTelemetryError -category 'NativeToolsBootstrap' "Failed to download '$uri' to '$temp_tool_path'." - return 1 - fi - - # Extract File - echo "extracting from $temp_tool_path to $installDir" - ExpandZip "$temp_tool_path" "$installDir" $force $download_retries $retry_wait_time_seconds - if [[ $? != 0 ]]; then - Write-PipelineTelemetryError -category 'NativeToolsBootstrap' "Failed to extract '$temp_tool_path' to '$installDir'." - return 1 - fi - - return 0 -} - -function NewScriptShim { - local shimpath=$1 - local tool_file_path=$2 - local force=${3:-false} - - echo "Generating '$shimpath' shim" - if [[ -f $shimpath ]]; then - if [[ $force = false ]]; then - echo "File '$shimpath' already exists." >&2 - return 1 - else - rm -rf $shimpath - fi - fi - - if [[ ! -f $tool_file_path ]]; then - # try to see if the path is lower cased - tool_file_path="$(echo $tool_file_path | tr "[:upper:]" "[:lower:]")" - if [[ ! -f $tool_file_path ]]; then - Write-PipelineTelemetryError -category 'NativeToolsBootstrap' "Specified tool file path:'$tool_file_path' does not exist" - return 1 - fi - fi - - local shim_contents=$'#!/usr/bin/env bash\n' - shim_contents+="SHIMARGS="$'$1\n' - shim_contents+="$tool_file_path"$' $SHIMARGS\n' - - # Write shim file - echo "$shim_contents" > $shimpath - - chmod +x $shimpath - - echo "Finished generating shim '$shimpath'" - - return $? -} - diff --git a/src/arcade/eng/common/native/init-compiler.sh b/src/arcade/eng/common/native/init-compiler.sh deleted file mode 100644 index 9a0e1f2b4567..000000000000 --- a/src/arcade/eng/common/native/init-compiler.sh +++ /dev/null @@ -1,146 +0,0 @@ -#!/bin/sh -# -# This file detects the C/C++ compiler and exports it to the CC/CXX environment variables -# -# NOTE: some scripts source this file and rely on stdout being empty, make sure -# to not output *anything* here, unless it is an error message that fails the -# build. - -if [ -z "$build_arch" ] || [ -z "$compiler" ]; then - echo "Usage..." - echo "build_arch= compiler= init-compiler.sh" - echo "Specify the target architecture." - echo "Specify the name of compiler (clang or gcc)." - exit 1 -fi - -case "$compiler" in - clang*|-clang*|--clang*) - # clangx.y or clang-x.y - version="$(echo "$compiler" | tr -d '[:alpha:]-=')" - majorVersion="${version%%.*}" - - # LLVM based on v18 released in early 2024, with two releases per year - maxVersion="$((18 + ((($(date +%Y) - 2024) * 12 + $(date +%-m) - 3) / 6)))" - compiler=clang - ;; - - gcc*|-gcc*|--gcc*) - # gccx.y or gcc-x.y - version="$(echo "$compiler" | tr -d '[:alpha:]-=')" - majorVersion="${version%%.*}" - - # GCC based on v14 released in early 2024, with one release per year - maxVersion="$((14 + ((($(date +%Y) - 2024) * 12 + $(date +%-m) - 3) / 12)))" - compiler=gcc - ;; -esac - -cxxCompiler="$compiler++" - -# clear the existing CC and CXX from environment -CC= -CXX= -LDFLAGS= - -if [ "$compiler" = "gcc" ]; then cxxCompiler="g++"; fi - -check_version_exists() { - desired_version=-1 - - # Set up the environment to be used for building with the desired compiler. - if command -v "$compiler-$1" > /dev/null; then - desired_version="-$1" - elif command -v "$compiler$1" > /dev/null; then - desired_version="$1" - fi - - echo "$desired_version" -} - -__baseOS="$(uname)" -set_compiler_version_from_CC() { - if [ "$__baseOS" = "Darwin" ]; then - # On Darwin, the versions from -version/-dumpversion refer to Xcode - # versions, not llvm versions, so we can't rely on them. - return - fi - - version="$("$CC" -dumpversion)" - if [ -z "$version" ]; then - echo "Error: $CC -dumpversion didn't provide a version" - exit 1 - fi - - # gcc and clang often display 3 part versions. However, gcc can show only 1 part in some environments. - IFS=. read -r majorVersion _ < /dev/null; then - echo "Error: No compatible version of $compiler was found within the range of $minVersion to $maxVersion. Please upgrade your toolchain or specify the compiler explicitly using CLR_CC and CLR_CXX environment variables." - exit 1 - fi - - CC="$(command -v "$compiler" 2> /dev/null)" - CXX="$(command -v "$cxxCompiler" 2> /dev/null)" - set_compiler_version_from_CC - fi - else - desired_version="$(check_version_exists "$majorVersion")" - if [ "$desired_version" = "-1" ]; then - echo "Error: Could not find specific version of $compiler: $majorVersion." - exit 1 - fi - fi - - if [ -z "$CC" ]; then - CC="$(command -v "$compiler$desired_version" 2> /dev/null)" - CXX="$(command -v "$cxxCompiler$desired_version" 2> /dev/null)" - if [ -z "$CXX" ]; then CXX="$(command -v "$cxxCompiler" 2> /dev/null)"; fi - set_compiler_version_from_CC - fi -else - if [ ! -f "$CLR_CC" ]; then - echo "Error: CLR_CC is set but path '$CLR_CC' does not exist" - exit 1 - fi - CC="$CLR_CC" - CXX="$CLR_CXX" - set_compiler_version_from_CC -fi - -if [ -z "$CC" ]; then - echo "Error: Unable to find $compiler." - exit 1 -fi - -if [ "$__baseOS" != "Darwin" ]; then - # On Darwin, we always want to use the Apple linker. - - # Only lld version >= 9 can be considered stable. lld supports s390x starting from 18.0. - if [ "$compiler" = "clang" ] && [ -n "$majorVersion" ] && [ "$majorVersion" -ge 9 ] && { [ "$build_arch" != "s390x" ] || [ "$majorVersion" -ge 18 ]; }; then - if "$CC" -fuse-ld=lld -Wl,--version >/dev/null 2>&1; then - LDFLAGS="-fuse-ld=lld" - fi - fi -fi - -SCAN_BUILD_COMMAND="$(command -v "scan-build$desired_version" 2> /dev/null)" - -export CC CXX LDFLAGS SCAN_BUILD_COMMAND diff --git a/src/arcade/eng/common/native/init-distro-rid.sh b/src/arcade/eng/common/native/init-distro-rid.sh deleted file mode 100644 index 83ea7aab0e08..000000000000 --- a/src/arcade/eng/common/native/init-distro-rid.sh +++ /dev/null @@ -1,110 +0,0 @@ -#!/bin/sh - -# getNonPortableDistroRid -# -# Input: -# targetOs: (str) -# targetArch: (str) -# rootfsDir: (str) -# -# Return: -# non-portable rid -getNonPortableDistroRid() -{ - targetOs="$1" - targetArch="$2" - rootfsDir="$3" - nonPortableRid="" - - if [ "$targetOs" = "linux" ]; then - # shellcheck disable=SC1091 - if [ -e "${rootfsDir}/etc/os-release" ]; then - . "${rootfsDir}/etc/os-release" - if echo "${VERSION_ID:-}" | grep -qE '^([[:digit:]]|\.)+$'; then - nonPortableRid="${ID}.${VERSION_ID}-${targetArch}" - else - # Rolling release distros either do not set VERSION_ID, set it as blank or - # set it to non-version looking string (such as TEMPLATE_VERSION_ID on ArchLinux); - # so omit it here to be consistent with everything else. - nonPortableRid="${ID}-${targetArch}" - fi - elif [ -e "${rootfsDir}/android_platform" ]; then - # shellcheck disable=SC1091 - . "${rootfsDir}/android_platform" - nonPortableRid="$RID" - fi - fi - - if [ "$targetOs" = "freebsd" ]; then - # $rootfsDir can be empty. freebsd-version is a shell script and should always work. - __freebsd_major_version=$("$rootfsDir"/bin/freebsd-version | cut -d'.' -f1) - nonPortableRid="freebsd.$__freebsd_major_version-${targetArch}" - elif command -v getprop >/dev/null && getprop ro.product.system.model | grep -qi android; then - __android_sdk_version=$(getprop ro.build.version.sdk) - nonPortableRid="android.$__android_sdk_version-${targetArch}" - elif [ "$targetOs" = "illumos" ]; then - __uname_version=$(uname -v) - nonPortableRid="illumos-${targetArch}" - elif [ "$targetOs" = "solaris" ]; then - __uname_version=$(uname -v) - __solaris_major_version=$(echo "$__uname_version" | cut -d'.' -f1) - nonPortableRid="solaris.$__solaris_major_version-${targetArch}" - elif [ "$targetOs" = "haiku" ]; then - __uname_release="$(uname -r)" - nonPortableRid=haiku.r"$__uname_release"-"$targetArch" - fi - - echo "$nonPortableRid" | tr '[:upper:]' '[:lower:]' -} - -# initDistroRidGlobal -# -# Input: -# os: (str) -# arch: (str) -# rootfsDir?: (nullable:string) -# -# Return: -# None -# -# Notes: -# It is important to note that the function does not return anything, but it -# exports the following variables on success: -# __DistroRid : Non-portable rid of the target platform. -# __PortableTargetOS : OS-part of the portable rid that corresponds to the target platform. -initDistroRidGlobal() -{ - targetOs="$1" - targetArch="$2" - rootfsDir="" - if [ $# -ge 3 ]; then - rootfsDir="$3" - fi - - if [ -n "${rootfsDir}" ]; then - # We may have a cross build. Check for the existence of the rootfsDir - if [ ! -e "${rootfsDir}" ]; then - echo "Error: rootfsDir has been passed, but the location is not valid." - exit 1 - fi - fi - - __DistroRid=$(getNonPortableDistroRid "${targetOs}" "${targetArch}" "${rootfsDir}") - - if [ -z "${__PortableTargetOS:-}" ]; then - __PortableTargetOS="$targetOs" - - STRINGS="$(command -v strings || true)" - if [ -z "$STRINGS" ]; then - STRINGS="$(command -v llvm-strings || true)" - fi - - # Check for musl-based distros (e.g. Alpine Linux, Void Linux). - if "${rootfsDir}/usr/bin/ldd" --version 2>&1 | grep -q musl || - ( [ -n "$STRINGS" ] && "$STRINGS" "${rootfsDir}/usr/bin/ldd" 2>&1 | grep -q musl ); then - __PortableTargetOS="linux-musl" - fi - fi - - export __DistroRid __PortableTargetOS -} diff --git a/src/arcade/eng/common/native/init-os-and-arch.sh b/src/arcade/eng/common/native/init-os-and-arch.sh deleted file mode 100644 index 38921d4338f7..000000000000 --- a/src/arcade/eng/common/native/init-os-and-arch.sh +++ /dev/null @@ -1,85 +0,0 @@ -#!/bin/sh - -# Use uname to determine what the OS is. -OSName=$(uname -s | tr '[:upper:]' '[:lower:]') - -if command -v getprop && getprop ro.product.system.model 2>&1 | grep -qi android; then - OSName="android" -fi - -case "$OSName" in -freebsd|linux|netbsd|openbsd|sunos|android|haiku) - os="$OSName" ;; -darwin) - os=osx ;; -*) - echo "Unsupported OS $OSName detected!" - exit 1 ;; -esac - -# On Solaris, `uname -m` is discouraged, see https://docs.oracle.com/cd/E36784_01/html/E36870/uname-1.html -# and `uname -p` returns processor type (e.g. i386 on amd64). -# The appropriate tool to determine CPU is isainfo(1) https://docs.oracle.com/cd/E36784_01/html/E36870/isainfo-1.html. -if [ "$os" = "sunos" ]; then - if uname -o 2>&1 | grep -q illumos; then - os="illumos" - else - os="solaris" - fi - CPUName=$(isainfo -n) -else - # For the rest of the operating systems, use uname(1) to determine what the CPU is. - CPUName=$(uname -m) -fi - -case "$CPUName" in - arm64|aarch64) - arch=arm64 - if [ "$(getconf LONG_BIT)" -lt 64 ]; then - # This is 32-bit OS running on 64-bit CPU (for example Raspberry Pi OS) - arch=arm - fi - ;; - - loongarch64) - arch=loongarch64 - ;; - - riscv64) - arch=riscv64 - ;; - - amd64|x86_64) - arch=x64 - ;; - - armv7l|armv8l) - # shellcheck disable=SC1091 - if (NAME=""; . /etc/os-release; test "$NAME" = "Tizen"); then - arch=armel - else - arch=arm - fi - ;; - - armv6l) - arch=armv6 - ;; - - i[3-6]86) - echo "Unsupported CPU $CPUName detected, build might not succeed!" - arch=x86 - ;; - - s390x) - arch=s390x - ;; - - ppc64le) - arch=ppc64le - ;; - *) - echo "Unknown CPU $CPUName detected!" - exit 1 - ;; -esac diff --git a/src/arcade/eng/common/native/install-cmake-test.sh b/src/arcade/eng/common/native/install-cmake-test.sh deleted file mode 100644 index 8a5e7cf0db5a..000000000000 --- a/src/arcade/eng/common/native/install-cmake-test.sh +++ /dev/null @@ -1,117 +0,0 @@ -#!/usr/bin/env bash - -source="${BASH_SOURCE[0]}" -scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" - -. $scriptroot/common-library.sh - -base_uri= -install_path= -version= -clean=false -force=false -download_retries=5 -retry_wait_time_seconds=30 - -while (($# > 0)); do - lowerI="$(echo $1 | tr "[:upper:]" "[:lower:]")" - case $lowerI in - --baseuri) - base_uri=$2 - shift 2 - ;; - --installpath) - install_path=$2 - shift 2 - ;; - --version) - version=$2 - shift 2 - ;; - --clean) - clean=true - shift 1 - ;; - --force) - force=true - shift 1 - ;; - --downloadretries) - download_retries=$2 - shift 2 - ;; - --retrywaittimeseconds) - retry_wait_time_seconds=$2 - shift 2 - ;; - --help) - echo "Common settings:" - echo " --baseuri Base file directory or Url wrom which to acquire tool archives" - echo " --installpath Base directory to install native tool to" - echo " --clean Don't install the tool, just clean up the current install of the tool" - echo " --force Force install of tools even if they previously exist" - echo " --help Print help and exit" - echo "" - echo "Advanced settings:" - echo " --downloadretries Total number of retry attempts" - echo " --retrywaittimeseconds Wait time between retry attempts in seconds" - echo "" - exit 0 - ;; - esac -done - -tool_name="cmake-test" -tool_os=$(GetCurrentOS) -tool_folder="$(echo $tool_os | tr "[:upper:]" "[:lower:]")" -tool_arch="x86_64" -tool_name_moniker="$tool_name-$version-$tool_os-$tool_arch" -tool_install_directory="$install_path/$tool_name/$version" -tool_file_path="$tool_install_directory/$tool_name_moniker/bin/$tool_name" -shim_path="$install_path/$tool_name.sh" -uri="${base_uri}/$tool_folder/$tool_name/$tool_name_moniker.tar.gz" - -# Clean up tool and installers -if [[ $clean = true ]]; then - echo "Cleaning $tool_install_directory" - if [[ -d $tool_install_directory ]]; then - rm -rf $tool_install_directory - fi - - echo "Cleaning $shim_path" - if [[ -f $shim_path ]]; then - rm -rf $shim_path - fi - - tool_temp_path=$(GetTempPathFileName $uri) - echo "Cleaning $tool_temp_path" - if [[ -f $tool_temp_path ]]; then - rm -rf $tool_temp_path - fi - - exit 0 -fi - -# Install tool -if [[ -f $tool_file_path ]] && [[ $force = false ]]; then - echo "$tool_name ($version) already exists, skipping install" - exit 0 -fi - -DownloadAndExtract $uri $tool_install_directory $force $download_retries $retry_wait_time_seconds - -if [[ $? != 0 ]]; then - Write-PipelineTelemetryError -category 'NativeToolsBootstrap' 'Installation failed' - exit 1 -fi - -# Generate Shim -# Always rewrite shims so that we are referencing the expected version -NewScriptShim $shim_path $tool_file_path true - -if [[ $? != 0 ]]; then - Write-PipelineTelemetryError -category 'NativeToolsBootstrap' 'Shim generation failed' - exit 1 -fi - -exit 0 diff --git a/src/arcade/eng/common/native/install-cmake.sh b/src/arcade/eng/common/native/install-cmake.sh deleted file mode 100644 index de496beebc5a..000000000000 --- a/src/arcade/eng/common/native/install-cmake.sh +++ /dev/null @@ -1,117 +0,0 @@ -#!/usr/bin/env bash - -source="${BASH_SOURCE[0]}" -scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" - -. $scriptroot/common-library.sh - -base_uri= -install_path= -version= -clean=false -force=false -download_retries=5 -retry_wait_time_seconds=30 - -while (($# > 0)); do - lowerI="$(echo $1 | tr "[:upper:]" "[:lower:]")" - case $lowerI in - --baseuri) - base_uri=$2 - shift 2 - ;; - --installpath) - install_path=$2 - shift 2 - ;; - --version) - version=$2 - shift 2 - ;; - --clean) - clean=true - shift 1 - ;; - --force) - force=true - shift 1 - ;; - --downloadretries) - download_retries=$2 - shift 2 - ;; - --retrywaittimeseconds) - retry_wait_time_seconds=$2 - shift 2 - ;; - --help) - echo "Common settings:" - echo " --baseuri Base file directory or Url wrom which to acquire tool archives" - echo " --installpath Base directory to install native tool to" - echo " --clean Don't install the tool, just clean up the current install of the tool" - echo " --force Force install of tools even if they previously exist" - echo " --help Print help and exit" - echo "" - echo "Advanced settings:" - echo " --downloadretries Total number of retry attempts" - echo " --retrywaittimeseconds Wait time between retry attempts in seconds" - echo "" - exit 0 - ;; - esac -done - -tool_name="cmake" -tool_os=$(GetCurrentOS) -tool_folder="$(echo $tool_os | tr "[:upper:]" "[:lower:]")" -tool_arch="x86_64" -tool_name_moniker="$tool_name-$version-$tool_os-$tool_arch" -tool_install_directory="$install_path/$tool_name/$version" -tool_file_path="$tool_install_directory/$tool_name_moniker/bin/$tool_name" -shim_path="$install_path/$tool_name.sh" -uri="${base_uri}/$tool_folder/$tool_name/$tool_name_moniker.tar.gz" - -# Clean up tool and installers -if [[ $clean = true ]]; then - echo "Cleaning $tool_install_directory" - if [[ -d $tool_install_directory ]]; then - rm -rf $tool_install_directory - fi - - echo "Cleaning $shim_path" - if [[ -f $shim_path ]]; then - rm -rf $shim_path - fi - - tool_temp_path=$(GetTempPathFileName $uri) - echo "Cleaning $tool_temp_path" - if [[ -f $tool_temp_path ]]; then - rm -rf $tool_temp_path - fi - - exit 0 -fi - -# Install tool -if [[ -f $tool_file_path ]] && [[ $force = false ]]; then - echo "$tool_name ($version) already exists, skipping install" - exit 0 -fi - -DownloadAndExtract $uri $tool_install_directory $force $download_retries $retry_wait_time_seconds - -if [[ $? != 0 ]]; then - Write-PipelineTelemetryError -category 'NativeToolsBootstrap' 'Installation failed' - exit 1 -fi - -# Generate Shim -# Always rewrite shims so that we are referencing the expected version -NewScriptShim $shim_path $tool_file_path true - -if [[ $? != 0 ]]; then - Write-PipelineTelemetryError -category 'NativeToolsBootstrap' 'Shim generation failed' - exit 1 -fi - -exit 0 diff --git a/src/arcade/eng/common/native/install-tool.ps1 b/src/arcade/eng/common/native/install-tool.ps1 deleted file mode 100644 index 78f2d84a4e4b..000000000000 --- a/src/arcade/eng/common/native/install-tool.ps1 +++ /dev/null @@ -1,132 +0,0 @@ -<# -.SYNOPSIS -Install native tool - -.DESCRIPTION -Install cmake native tool from Azure blob storage - -.PARAMETER InstallPath -Base directory to install native tool to - -.PARAMETER BaseUri -Base file directory or Url from which to acquire tool archives - -.PARAMETER CommonLibraryDirectory -Path to folder containing common library modules - -.PARAMETER Force -Force install of tools even if they previously exist - -.PARAMETER Clean -Don't install the tool, just clean up the current install of the tool - -.PARAMETER DownloadRetries -Total number of retry attempts - -.PARAMETER RetryWaitTimeInSeconds -Wait time between retry attempts in seconds - -.NOTES -Returns 0 if install succeeds, 1 otherwise -#> -[CmdletBinding(PositionalBinding=$false)] -Param ( - [Parameter(Mandatory=$True)] - [string] $ToolName, - [Parameter(Mandatory=$True)] - [string] $InstallPath, - [Parameter(Mandatory=$True)] - [string] $BaseUri, - [Parameter(Mandatory=$True)] - [string] $Version, - [string] $CommonLibraryDirectory = $PSScriptRoot, - [switch] $Force = $False, - [switch] $Clean = $False, - [int] $DownloadRetries = 5, - [int] $RetryWaitTimeInSeconds = 30 -) - -. $PSScriptRoot\..\pipeline-logging-functions.ps1 - -# Import common library modules -Import-Module -Name (Join-Path $CommonLibraryDirectory "CommonLibrary.psm1") - -try { - # Define verbose switch if undefined - $Verbose = $VerbosePreference -Eq "Continue" - - $Arch = CommonLibrary\Get-MachineArchitecture - $ToolOs = "win64" - if($Arch -Eq "x32") { - $ToolOs = "win32" - } - $ToolNameMoniker = "$ToolName-$Version-$ToolOs-$Arch" - $ToolInstallDirectory = Join-Path $InstallPath "$ToolName\$Version\" - $Uri = "$BaseUri/windows/$ToolName/$ToolNameMoniker.zip" - $ShimPath = Join-Path $InstallPath "$ToolName.exe" - - if ($Clean) { - Write-Host "Cleaning $ToolInstallDirectory" - if (Test-Path $ToolInstallDirectory) { - Remove-Item $ToolInstallDirectory -Force -Recurse - } - Write-Host "Cleaning $ShimPath" - if (Test-Path $ShimPath) { - Remove-Item $ShimPath -Force - } - $ToolTempPath = CommonLibrary\Get-TempPathFilename -Path $Uri - Write-Host "Cleaning $ToolTempPath" - if (Test-Path $ToolTempPath) { - Remove-Item $ToolTempPath -Force - } - exit 0 - } - - # Install tool - if ((Test-Path $ToolInstallDirectory) -And (-Not $Force)) { - Write-Verbose "$ToolName ($Version) already exists, skipping install" - } - else { - $InstallStatus = CommonLibrary\DownloadAndExtract -Uri $Uri ` - -InstallDirectory $ToolInstallDirectory ` - -Force:$Force ` - -DownloadRetries $DownloadRetries ` - -RetryWaitTimeInSeconds $RetryWaitTimeInSeconds ` - -Verbose:$Verbose - - if ($InstallStatus -Eq $False) { - Write-PipelineTelemetryError "Installation failed" -Category "NativeToolsetBootstrapping" - exit 1 - } - } - - $ToolFilePath = Get-ChildItem $ToolInstallDirectory -Recurse -Filter "$ToolName.exe" | % { $_.FullName } - if (@($ToolFilePath).Length -Gt 1) { - Write-Error "There are multiple copies of $ToolName in $($ToolInstallDirectory): `n$(@($ToolFilePath | out-string))" - exit 1 - } elseif (@($ToolFilePath).Length -Lt 1) { - Write-Host "$ToolName was not found in $ToolInstallDirectory." - exit 1 - } - - # Generate shim - # Always rewrite shims so that we are referencing the expected version - $GenerateShimStatus = CommonLibrary\New-ScriptShim -ShimName $ToolName ` - -ShimDirectory $InstallPath ` - -ToolFilePath "$ToolFilePath" ` - -BaseUri $BaseUri ` - -Force:$Force ` - -Verbose:$Verbose - - if ($GenerateShimStatus -Eq $False) { - Write-PipelineTelemetryError "Generate shim failed" -Category "NativeToolsetBootstrapping" - return 1 - } - - exit 0 -} -catch { - Write-Host $_.ScriptStackTrace - Write-PipelineTelemetryError -Category "NativeToolsetBootstrapping" -Message $_ - exit 1 -} diff --git a/src/arcade/eng/common/pipeline-logging-functions.ps1 b/src/arcade/eng/common/pipeline-logging-functions.ps1 deleted file mode 100644 index 8e422c561e4b..000000000000 --- a/src/arcade/eng/common/pipeline-logging-functions.ps1 +++ /dev/null @@ -1,260 +0,0 @@ -# Source for this file was taken from https://github.com/microsoft/azure-pipelines-task-lib/blob/11c9439d4af17e6475d9fe058e6b2e03914d17e6/powershell/VstsTaskSdk/LoggingCommandFunctions.ps1 and modified. - -# NOTE: You should not be calling these method directly as they are likely to change. Instead you should be calling the Write-Pipeline* functions defined in tools.ps1 - -$script:loggingCommandPrefix = '##vso[' -$script:loggingCommandEscapeMappings = @( # TODO: WHAT ABOUT "="? WHAT ABOUT "%"? - New-Object psobject -Property @{ Token = ';' ; Replacement = '%3B' } - New-Object psobject -Property @{ Token = "`r" ; Replacement = '%0D' } - New-Object psobject -Property @{ Token = "`n" ; Replacement = '%0A' } - New-Object psobject -Property @{ Token = "]" ; Replacement = '%5D' } -) -# TODO: BUG: Escape % ??? -# TODO: Add test to verify don't need to escape "=". - -# Specify "-Force" to force pipeline formatted output even if "$ci" is false or not set -function Write-PipelineTelemetryError { - [CmdletBinding()] - param( - [Parameter(Mandatory = $true)] - [string]$Category, - [Parameter(Mandatory = $true)] - [string]$Message, - [Parameter(Mandatory = $false)] - [string]$Type = 'error', - [string]$ErrCode, - [string]$SourcePath, - [string]$LineNumber, - [string]$ColumnNumber, - [switch]$AsOutput, - [switch]$Force) - - $PSBoundParameters.Remove('Category') | Out-Null - - if ($Force -Or ((Test-Path variable:ci) -And $ci)) { - $Message = "(NETCORE_ENGINEERING_TELEMETRY=$Category) $Message" - } - $PSBoundParameters.Remove('Message') | Out-Null - $PSBoundParameters.Add('Message', $Message) - Write-PipelineTaskError @PSBoundParameters -} - -# Specify "-Force" to force pipeline formatted output even if "$ci" is false or not set -function Write-PipelineTaskError { - [CmdletBinding()] - param( - [Parameter(Mandatory = $true)] - [string]$Message, - [Parameter(Mandatory = $false)] - [string]$Type = 'error', - [string]$ErrCode, - [string]$SourcePath, - [string]$LineNumber, - [string]$ColumnNumber, - [switch]$AsOutput, - [switch]$Force - ) - - if (!$Force -And (-Not (Test-Path variable:ci) -Or !$ci)) { - if ($Type -eq 'error') { - Write-Host $Message -ForegroundColor Red - return - } - elseif ($Type -eq 'warning') { - Write-Host $Message -ForegroundColor Yellow - return - } - } - - if (($Type -ne 'error') -and ($Type -ne 'warning')) { - Write-Host $Message - return - } - $PSBoundParameters.Remove('Force') | Out-Null - if (-not $PSBoundParameters.ContainsKey('Type')) { - $PSBoundParameters.Add('Type', 'error') - } - Write-LogIssue @PSBoundParameters -} - -function Write-PipelineSetVariable { - [CmdletBinding()] - param( - [Parameter(Mandatory = $true)] - [string]$Name, - [string]$Value, - [switch]$Secret, - [switch]$AsOutput, - [bool]$IsMultiJobVariable = $true) - - if ((Test-Path variable:ci) -And $ci) { - Write-LoggingCommand -Area 'task' -Event 'setvariable' -Data $Value -Properties @{ - 'variable' = $Name - 'isSecret' = $Secret - 'isOutput' = $IsMultiJobVariable - } -AsOutput:$AsOutput - } -} - -function Write-PipelinePrependPath { - [CmdletBinding()] - param( - [Parameter(Mandatory = $true)] - [string]$Path, - [switch]$AsOutput) - - if ((Test-Path variable:ci) -And $ci) { - Write-LoggingCommand -Area 'task' -Event 'prependpath' -Data $Path -AsOutput:$AsOutput - } -} - -function Write-PipelineSetResult { - [CmdletBinding()] - param( - [ValidateSet("Succeeded", "SucceededWithIssues", "Failed", "Cancelled", "Skipped")] - [Parameter(Mandatory = $true)] - [string]$Result, - [string]$Message) - if ((Test-Path variable:ci) -And $ci) { - Write-LoggingCommand -Area 'task' -Event 'complete' -Data $Message -Properties @{ - 'result' = $Result - } - } -} - -<######################################## -# Private functions. -########################################> -function Format-LoggingCommandData { - [CmdletBinding()] - param([string]$Value, [switch]$Reverse) - - if (!$Value) { - return '' - } - - if (!$Reverse) { - foreach ($mapping in $script:loggingCommandEscapeMappings) { - $Value = $Value.Replace($mapping.Token, $mapping.Replacement) - } - } - else { - for ($i = $script:loggingCommandEscapeMappings.Length - 1 ; $i -ge 0 ; $i--) { - $mapping = $script:loggingCommandEscapeMappings[$i] - $Value = $Value.Replace($mapping.Replacement, $mapping.Token) - } - } - - return $Value -} - -function Format-LoggingCommand { - [CmdletBinding()] - param( - [Parameter(Mandatory = $true)] - [string]$Area, - [Parameter(Mandatory = $true)] - [string]$Event, - [string]$Data, - [hashtable]$Properties) - - # Append the preamble. - [System.Text.StringBuilder]$sb = New-Object -TypeName System.Text.StringBuilder - $null = $sb.Append($script:loggingCommandPrefix).Append($Area).Append('.').Append($Event) - - # Append the properties. - if ($Properties) { - $first = $true - foreach ($key in $Properties.Keys) { - [string]$value = Format-LoggingCommandData $Properties[$key] - if ($value) { - if ($first) { - $null = $sb.Append(' ') - $first = $false - } - else { - $null = $sb.Append(';') - } - - $null = $sb.Append("$key=$value") - } - } - } - - # Append the tail and output the value. - $Data = Format-LoggingCommandData $Data - $sb.Append(']').Append($Data).ToString() -} - -function Write-LoggingCommand { - [CmdletBinding(DefaultParameterSetName = 'Parameters')] - param( - [Parameter(Mandatory = $true, ParameterSetName = 'Parameters')] - [string]$Area, - [Parameter(Mandatory = $true, ParameterSetName = 'Parameters')] - [string]$Event, - [Parameter(ParameterSetName = 'Parameters')] - [string]$Data, - [Parameter(ParameterSetName = 'Parameters')] - [hashtable]$Properties, - [Parameter(Mandatory = $true, ParameterSetName = 'Object')] - $Command, - [switch]$AsOutput) - - if ($PSCmdlet.ParameterSetName -eq 'Object') { - Write-LoggingCommand -Area $Command.Area -Event $Command.Event -Data $Command.Data -Properties $Command.Properties -AsOutput:$AsOutput - return - } - - $command = Format-LoggingCommand -Area $Area -Event $Event -Data $Data -Properties $Properties - if ($AsOutput) { - $command - } - else { - Write-Host $command - } -} - -function Write-LogIssue { - [CmdletBinding()] - param( - [ValidateSet('warning', 'error')] - [Parameter(Mandatory = $true)] - [string]$Type, - [string]$Message, - [string]$ErrCode, - [string]$SourcePath, - [string]$LineNumber, - [string]$ColumnNumber, - [switch]$AsOutput) - - $command = Format-LoggingCommand -Area 'task' -Event 'logissue' -Data $Message -Properties @{ - 'type' = $Type - 'code' = $ErrCode - 'sourcepath' = $SourcePath - 'linenumber' = $LineNumber - 'columnnumber' = $ColumnNumber - } - if ($AsOutput) { - return $command - } - - if ($Type -eq 'error') { - $foregroundColor = $host.PrivateData.ErrorForegroundColor - $backgroundColor = $host.PrivateData.ErrorBackgroundColor - if ($foregroundColor -isnot [System.ConsoleColor] -or $backgroundColor -isnot [System.ConsoleColor]) { - $foregroundColor = [System.ConsoleColor]::Red - $backgroundColor = [System.ConsoleColor]::Black - } - } - else { - $foregroundColor = $host.PrivateData.WarningForegroundColor - $backgroundColor = $host.PrivateData.WarningBackgroundColor - if ($foregroundColor -isnot [System.ConsoleColor] -or $backgroundColor -isnot [System.ConsoleColor]) { - $foregroundColor = [System.ConsoleColor]::Yellow - $backgroundColor = [System.ConsoleColor]::Black - } - } - - Write-Host $command -ForegroundColor $foregroundColor -BackgroundColor $backgroundColor -} diff --git a/src/arcade/eng/common/pipeline-logging-functions.sh b/src/arcade/eng/common/pipeline-logging-functions.sh deleted file mode 100644 index 6a0b2255e911..000000000000 --- a/src/arcade/eng/common/pipeline-logging-functions.sh +++ /dev/null @@ -1,206 +0,0 @@ -#!/usr/bin/env bash - -function Write-PipelineTelemetryError { - local telemetry_category='' - local force=false - local function_args=() - local message='' - while [[ $# -gt 0 ]]; do - opt="$(echo "${1/#--/-}" | tr "[:upper:]" "[:lower:]")" - case "$opt" in - -category|-c) - telemetry_category=$2 - shift - ;; - -force|-f) - force=true - ;; - -*) - function_args+=("$1 $2") - shift - ;; - *) - message=$* - ;; - esac - shift - done - - if [[ $force != true ]] && [[ "$ci" != true ]]; then - echo "$message" >&2 - return - fi - - if [[ $force == true ]]; then - function_args+=("-force") - fi - message="(NETCORE_ENGINEERING_TELEMETRY=$telemetry_category) $message" - function_args+=("$message") - Write-PipelineTaskError ${function_args[@]} -} - -function Write-PipelineTaskError { - local message_type="error" - local sourcepath='' - local linenumber='' - local columnnumber='' - local error_code='' - local force=false - - while [[ $# -gt 0 ]]; do - opt="$(echo "${1/#--/-}" | tr "[:upper:]" "[:lower:]")" - case "$opt" in - -type|-t) - message_type=$2 - shift - ;; - -sourcepath|-s) - sourcepath=$2 - shift - ;; - -linenumber|-ln) - linenumber=$2 - shift - ;; - -columnnumber|-cn) - columnnumber=$2 - shift - ;; - -errcode|-e) - error_code=$2 - shift - ;; - -force|-f) - force=true - ;; - *) - break - ;; - esac - - shift - done - - if [[ $force != true ]] && [[ "$ci" != true ]]; then - echo "$@" >&2 - return - fi - - local message="##vso[task.logissue" - - message="$message type=$message_type" - - if [ -n "$sourcepath" ]; then - message="$message;sourcepath=$sourcepath" - fi - - if [ -n "$linenumber" ]; then - message="$message;linenumber=$linenumber" - fi - - if [ -n "$columnnumber" ]; then - message="$message;columnnumber=$columnnumber" - fi - - if [ -n "$error_code" ]; then - message="$message;code=$error_code" - fi - - message="$message]$*" - echo "$message" -} - -function Write-PipelineSetVariable { - if [[ "$ci" != true ]]; then - return - fi - - local name='' - local value='' - local secret=false - local as_output=false - local is_multi_job_variable=true - - while [[ $# -gt 0 ]]; do - opt="$(echo "${1/#--/-}" | tr "[:upper:]" "[:lower:]")" - case "$opt" in - -name|-n) - name=$2 - shift - ;; - -value|-v) - value=$2 - shift - ;; - -secret|-s) - secret=true - ;; - -as_output|-a) - as_output=true - ;; - -is_multi_job_variable|-i) - is_multi_job_variable=$2 - shift - ;; - esac - shift - done - - value=${value/;/%3B} - value=${value/\\r/%0D} - value=${value/\\n/%0A} - value=${value/]/%5D} - - local message="##vso[task.setvariable variable=$name;isSecret=$secret;isOutput=$is_multi_job_variable]$value" - - if [[ "$as_output" == true ]]; then - $message - else - echo "$message" - fi -} - -function Write-PipelinePrependPath { - local prepend_path='' - - while [[ $# -gt 0 ]]; do - opt="$(echo "${1/#--/-}" | tr "[:upper:]" "[:lower:]")" - case "$opt" in - -path|-p) - prepend_path=$2 - shift - ;; - esac - shift - done - - export PATH="$prepend_path:$PATH" - - if [[ "$ci" == true ]]; then - echo "##vso[task.prependpath]$prepend_path" - fi -} - -function Write-PipelineSetResult { - local result='' - local message='' - - while [[ $# -gt 0 ]]; do - opt="$(echo "${1/#--/-}" | tr "[:upper:]" "[:lower:]")" - case "$opt" in - -result|-r) - result=$2 - shift - ;; - -message|-m) - message=$2 - shift - ;; - esac - shift - done - - if [[ "$ci" == true ]]; then - echo "##vso[task.complete result=$result;]$message" - fi -} diff --git a/src/arcade/eng/common/post-build/check-channel-consistency.ps1 b/src/arcade/eng/common/post-build/check-channel-consistency.ps1 deleted file mode 100644 index 61208d2d1351..000000000000 --- a/src/arcade/eng/common/post-build/check-channel-consistency.ps1 +++ /dev/null @@ -1,48 +0,0 @@ -param( - [Parameter(Mandatory=$true)][string] $PromoteToChannels, # List of channels that the build should be promoted to - [Parameter(Mandatory=$true)][array] $AvailableChannelIds # List of channel IDs available in the YAML implementation -) - -try { - $ErrorActionPreference = 'Stop' - Set-StrictMode -Version 2.0 - - # `tools.ps1` checks $ci to perform some actions. Since the post-build - # scripts don't necessarily execute in the same agent that run the - # build.ps1/sh script this variable isn't automatically set. - $ci = $true - $disableConfigureToolsetImport = $true - . $PSScriptRoot\..\tools.ps1 - - if ($PromoteToChannels -eq "") { - Write-PipelineTaskError -Type 'warning' -Message "This build won't publish assets as it's not configured to any Maestro channel. If that wasn't intended use Darc to configure a default channel using add-default-channel for this branch or to promote it to a channel using add-build-to-channel. See https://github.com/dotnet/arcade/blob/main/Documentation/Darc.md#assigning-an-individual-build-to-a-channel for more info." - ExitWithExitCode 0 - } - - # Check that every channel that Maestro told to promote the build to - # is available in YAML - $PromoteToChannelsIds = $PromoteToChannels -split "\D" | Where-Object { $_ } - - $hasErrors = $false - - foreach ($id in $PromoteToChannelsIds) { - if (($id -ne 0) -and ($id -notin $AvailableChannelIds)) { - Write-PipelineTaskError -Message "Channel $id is not present in the post-build YAML configuration! This is an error scenario. Please contact @dnceng." - $hasErrors = $true - } - } - - # The `Write-PipelineTaskError` doesn't error the script and we might report several errors - # in the previous lines. The check below makes sure that we return an error state from the - # script if we reported any validation error - if ($hasErrors) { - ExitWithExitCode 1 - } - - Write-Host 'done.' -} -catch { - Write-Host $_ - Write-PipelineTelemetryError -Category 'CheckChannelConsistency' -Message "There was an error while trying to check consistency of Maestro default channels for the build and post-build YAML configuration." - ExitWithExitCode 1 -} diff --git a/src/arcade/eng/common/post-build/nuget-validation.ps1 b/src/arcade/eng/common/post-build/nuget-validation.ps1 deleted file mode 100644 index e5de00c89836..000000000000 --- a/src/arcade/eng/common/post-build/nuget-validation.ps1 +++ /dev/null @@ -1,22 +0,0 @@ -# This script validates NuGet package metadata information using this -# tool: https://github.com/NuGet/NuGetGallery/tree/jver-verify/src/VerifyMicrosoftPackage - -param( - [Parameter(Mandatory=$true)][string] $PackagesPath # Path to where the packages to be validated are -) - -# `tools.ps1` checks $ci to perform some actions. Since the post-build -# scripts don't necessarily execute in the same agent that run the -# build.ps1/sh script this variable isn't automatically set. -$ci = $true -$disableConfigureToolsetImport = $true -. $PSScriptRoot\..\tools.ps1 - -try { - & $PSScriptRoot\nuget-verification.ps1 ${PackagesPath}\*.nupkg -} -catch { - Write-Host $_.ScriptStackTrace - Write-PipelineTelemetryError -Category 'NuGetValidation' -Message $_ - ExitWithExitCode 1 -} diff --git a/src/arcade/eng/common/post-build/nuget-verification.ps1 b/src/arcade/eng/common/post-build/nuget-verification.ps1 deleted file mode 100644 index a365194a9389..000000000000 --- a/src/arcade/eng/common/post-build/nuget-verification.ps1 +++ /dev/null @@ -1,121 +0,0 @@ -<# -.SYNOPSIS - Verifies that Microsoft NuGet packages have proper metadata. -.DESCRIPTION - Downloads a verification tool and runs metadata validation on the provided NuGet packages. This script writes an - error if any of the provided packages fail validation. All arguments provided to this PowerShell script that do not - match PowerShell parameters are passed on to the verification tool downloaded during the execution of this script. -.PARAMETER NuGetExePath - The path to the nuget.exe binary to use. If not provided, nuget.exe will be downloaded into the -DownloadPath - directory. -.PARAMETER PackageSource - The package source to use to download the verification tool. If not provided, nuget.org will be used. -.PARAMETER DownloadPath - The directory path to download the verification tool and nuget.exe to. If not provided, - %TEMP%\NuGet.VerifyNuGetPackage will be used. -.PARAMETER args - Arguments that will be passed to the verification tool. -.EXAMPLE - PS> .\verify.ps1 *.nupkg - Verifies the metadata of all .nupkg files in the currect working directory. -.EXAMPLE - PS> .\verify.ps1 --help - Displays the help text of the downloaded verifiction tool. -.LINK - https://github.com/NuGet/NuGetGallery/blob/master/src/VerifyMicrosoftPackage/README.md -#> - -# This script was copied from https://github.com/NuGet/NuGetGallery/blob/3e25ad135146676bcab0050a516939d9958bfa5d/src/VerifyMicrosoftPackage/verify.ps1 - -[CmdletBinding(PositionalBinding = $false)] -param( - [string]$NuGetExePath, - [string]$PackageSource = "https://api.nuget.org/v3/index.json", - [string]$DownloadPath, - [Parameter(ValueFromRemainingArguments = $true)] - [string[]]$args -) - -# The URL to download nuget.exe. -$nugetExeUrl = "https://dist.nuget.org/win-x86-commandline/v4.9.4/nuget.exe" - -# The package ID of the verification tool. -$packageId = "NuGet.VerifyMicrosoftPackage" - -# The location that nuget.exe and the verification tool will be downloaded to. -if (!$DownloadPath) { - $DownloadPath = (Join-Path $env:TEMP "NuGet.VerifyMicrosoftPackage") -} - -$fence = New-Object -TypeName string -ArgumentList '=', 80 - -# Create the download directory, if it doesn't already exist. -if (!(Test-Path $DownloadPath)) { - New-Item -ItemType Directory $DownloadPath | Out-Null -} -Write-Host "Using download path: $DownloadPath" - -if ($NuGetExePath) { - $nuget = $NuGetExePath -} else { - $downloadedNuGetExe = Join-Path $DownloadPath "nuget.exe" - - # Download nuget.exe, if it doesn't already exist. - if (!(Test-Path $downloadedNuGetExe)) { - Write-Host "Downloading nuget.exe from $nugetExeUrl..." - $ProgressPreference = 'SilentlyContinue' - try { - Invoke-WebRequest $nugetExeUrl -OutFile $downloadedNuGetExe - $ProgressPreference = 'Continue' - } catch { - $ProgressPreference = 'Continue' - Write-Error $_ - Write-Error "nuget.exe failed to download." - exit - } - } - - $nuget = $downloadedNuGetExe -} - -Write-Host "Using nuget.exe path: $nuget" -Write-Host " " - -# Download the latest version of the verification tool. -Write-Host "Downloading the latest version of $packageId from $packageSource..." -Write-Host $fence -& $nuget install $packageId ` - -Prerelease ` - -OutputDirectory $DownloadPath ` - -Source $PackageSource -Write-Host $fence -Write-Host " " - -if ($LASTEXITCODE -ne 0) { - Write-Error "nuget.exe failed to fetch the verify tool." - exit -} - -# Find the most recently downloaded tool -Write-Host "Finding the most recently downloaded verification tool." -$verifyProbePath = Join-Path $DownloadPath "$packageId.*" -$verifyPath = Get-ChildItem -Path $verifyProbePath -Directory ` - | Sort-Object -Property LastWriteTime -Descending ` - | Select-Object -First 1 -$verify = Join-Path $verifyPath "tools\NuGet.VerifyMicrosoftPackage.exe" -Write-Host "Using verification tool: $verify" -Write-Host " " - -# Execute the verification tool. -Write-Host "Executing the verify tool..." -Write-Host $fence -& $verify $args -Write-Host $fence -Write-Host " " - -# Respond to the exit code. -if ($LASTEXITCODE -ne 0) { - Write-Error "The verify tool found some problems." -} else { - Write-Output "The verify tool succeeded." -} diff --git a/src/arcade/eng/common/post-build/redact-logs.ps1 b/src/arcade/eng/common/post-build/redact-logs.ps1 deleted file mode 100644 index b7fc19591507..000000000000 --- a/src/arcade/eng/common/post-build/redact-logs.ps1 +++ /dev/null @@ -1,89 +0,0 @@ -[CmdletBinding(PositionalBinding=$False)] -param( - [Parameter(Mandatory=$true, Position=0)][string] $InputPath, - [Parameter(Mandatory=$true)][string] $BinlogToolVersion, - [Parameter(Mandatory=$false)][string] $DotnetPath, - [Parameter(Mandatory=$false)][string] $PackageFeed = 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public/nuget/v3/index.json', - # File with strings to redact - separated by newlines. - # For comments start the line with '# ' - such lines are ignored - [Parameter(Mandatory=$false)][string] $TokensFilePath, - [Parameter(ValueFromRemainingArguments=$true)][String[]]$TokensToRedact -) - -try { - $ErrorActionPreference = 'Stop' - Set-StrictMode -Version 2.0 - - # `tools.ps1` checks $ci to perform some actions. Since the post-build - # scripts don't necessarily execute in the same agent that run the - # build.ps1/sh script this variable isn't automatically set. - $ci = $true - $disableConfigureToolsetImport = $true - . $PSScriptRoot\..\tools.ps1 - - $packageName = 'binlogtool' - - $dotnet = $DotnetPath - - if (!$dotnet) { - $dotnetRoot = InitializeDotNetCli -install:$true - $dotnet = "$dotnetRoot\dotnet.exe" - } - - $toolList = & "$dotnet" tool list -g - - if ($toolList -like "*$packageName*") { - & "$dotnet" tool uninstall $packageName -g - } - - $toolPath = "$PSScriptRoot\..\..\..\.tools" - $verbosity = 'minimal' - - New-Item -ItemType Directory -Force -Path $toolPath - - Push-Location -Path $toolPath - - try { - Write-Host "Installing Binlog redactor CLI..." - Write-Host "'$dotnet' new tool-manifest" - & "$dotnet" new tool-manifest - Write-Host "'$dotnet' tool install $packageName --local --add-source '$PackageFeed' -v $verbosity --version $BinlogToolVersion" - & "$dotnet" tool install $packageName --local --add-source "$PackageFeed" -v $verbosity --version $BinlogToolVersion - - if (Test-Path $TokensFilePath) { - Write-Host "Adding additional sensitive data for redaction from file: " $TokensFilePath - $TokensToRedact += Get-Content -Path $TokensFilePath | Foreach {$_.Trim()} | Where { $_ -notmatch "^# " } - } - - $optionalParams = [System.Collections.ArrayList]::new() - - Foreach ($p in $TokensToRedact) - { - if($p -match '^\$\(.*\)$') - { - Write-Host ("Ignoring token {0} as it is probably unexpanded AzDO variable" -f $p) - } - elseif($p) - { - $optionalParams.Add("-p:" + $p) | Out-Null - } - } - - & $dotnet binlogtool redact --input:$InputPath --recurse --in-place ` - @optionalParams - - if ($LastExitCode -ne 0) { - Write-PipelineTelemetryError -Category 'Redactor' -Type 'warning' -Message "Problems using Redactor tool (exit code: $LastExitCode). But ignoring them now." - } - } - finally { - Pop-Location - } - - Write-Host 'done.' -} -catch { - Write-Host $_ - Write-PipelineTelemetryError -Category 'Redactor' -Message "There was an error while trying to redact logs. Error: $_" - ExitWithExitCode 1 -} diff --git a/src/arcade/eng/common/post-build/sourcelink-validation.ps1 b/src/arcade/eng/common/post-build/sourcelink-validation.ps1 deleted file mode 100644 index 1976ef70fb85..000000000000 --- a/src/arcade/eng/common/post-build/sourcelink-validation.ps1 +++ /dev/null @@ -1,327 +0,0 @@ -param( - [Parameter(Mandatory=$true)][string] $InputPath, # Full path to directory where Symbols.NuGet packages to be checked are stored - [Parameter(Mandatory=$true)][string] $ExtractPath, # Full path to directory where the packages will be extracted during validation - [Parameter(Mandatory=$false)][string] $GHRepoName, # GitHub name of the repo including the Org. E.g., dotnet/arcade - [Parameter(Mandatory=$false)][string] $GHCommit, # GitHub commit SHA used to build the packages - [Parameter(Mandatory=$true)][string] $SourcelinkCliVersion # Version of SourceLink CLI to use -) - -$ErrorActionPreference = 'Stop' -Set-StrictMode -Version 2.0 - -# `tools.ps1` checks $ci to perform some actions. Since the post-build -# scripts don't necessarily execute in the same agent that run the -# build.ps1/sh script this variable isn't automatically set. -$ci = $true -$disableConfigureToolsetImport = $true -. $PSScriptRoot\..\tools.ps1 - -# Cache/HashMap (File -> Exist flag) used to consult whether a file exist -# in the repository at a specific commit point. This is populated by inserting -# all files present in the repo at a specific commit point. -$global:RepoFiles = @{} - -# Maximum number of jobs to run in parallel -$MaxParallelJobs = 16 - -$MaxRetries = 5 -$RetryWaitTimeInSeconds = 30 - -# Wait time between check for system load -$SecondsBetweenLoadChecks = 10 - -if (!$InputPath -or !(Test-Path $InputPath)){ - Write-Host "No files to validate." - ExitWithExitCode 0 -} - -$ValidatePackage = { - param( - [string] $PackagePath # Full path to a Symbols.NuGet package - ) - - . $using:PSScriptRoot\..\tools.ps1 - - # Ensure input file exist - if (!(Test-Path $PackagePath)) { - Write-Host "Input file does not exist: $PackagePath" - return [pscustomobject]@{ - result = 1 - packagePath = $PackagePath - } - } - - # Extensions for which we'll look for SourceLink information - # For now we'll only care about Portable & Embedded PDBs - $RelevantExtensions = @('.dll', '.exe', '.pdb') - - Write-Host -NoNewLine 'Validating ' ([System.IO.Path]::GetFileName($PackagePath)) '...' - - $PackageId = [System.IO.Path]::GetFileNameWithoutExtension($PackagePath) - $ExtractPath = Join-Path -Path $using:ExtractPath -ChildPath $PackageId - $FailedFiles = 0 - - Add-Type -AssemblyName System.IO.Compression.FileSystem - - [System.IO.Directory]::CreateDirectory($ExtractPath) | Out-Null - - try { - $zip = [System.IO.Compression.ZipFile]::OpenRead($PackagePath) - - $zip.Entries | - Where-Object {$RelevantExtensions -contains [System.IO.Path]::GetExtension($_.Name)} | - ForEach-Object { - $FileName = $_.FullName - $Extension = [System.IO.Path]::GetExtension($_.Name) - $FakeName = -Join((New-Guid), $Extension) - $TargetFile = Join-Path -Path $ExtractPath -ChildPath $FakeName - - # We ignore resource DLLs - if ($FileName.EndsWith('.resources.dll')) { - return [pscustomobject]@{ - result = 0 - packagePath = $PackagePath - } - } - - [System.IO.Compression.ZipFileExtensions]::ExtractToFile($_, $TargetFile, $true) - - $ValidateFile = { - param( - [string] $FullPath, # Full path to the module that has to be checked - [string] $RealPath, - [ref] $FailedFiles - ) - - $sourcelinkExe = "$env:USERPROFILE\.dotnet\tools" - $sourcelinkExe = Resolve-Path "$sourcelinkExe\sourcelink.exe" - $SourceLinkInfos = & $sourcelinkExe print-urls $FullPath | Out-String - - if ($LASTEXITCODE -eq 0 -and -not ([string]::IsNullOrEmpty($SourceLinkInfos))) { - $NumFailedLinks = 0 - - # We only care about Http addresses - $Matches = (Select-String '(http[s]?)(:\/\/)([^\s,]+)' -Input $SourceLinkInfos -AllMatches).Matches - - if ($Matches.Count -ne 0) { - $Matches.Value | - ForEach-Object { - $Link = $_ - $CommitUrl = "https://raw.githubusercontent.com/${using:GHRepoName}/${using:GHCommit}/" - - $FilePath = $Link.Replace($CommitUrl, "") - $Status = 200 - $Cache = $using:RepoFiles - - $attempts = 0 - - while ($attempts -lt $using:MaxRetries) { - if ( !($Cache.ContainsKey($FilePath)) ) { - try { - $Uri = $Link -as [System.URI] - - if ($Link -match "submodules") { - # Skip submodule links until sourcelink properly handles submodules - $Status = 200 - } - elseif ($Uri.AbsoluteURI -ne $null -and ($Uri.Host -match 'github' -or $Uri.Host -match 'githubusercontent')) { - # Only GitHub links are valid - $Status = (Invoke-WebRequest -Uri $Link -UseBasicParsing -Method HEAD -TimeoutSec 5).StatusCode - } - else { - # If it's not a github link, we want to break out of the loop and not retry. - $Status = 0 - $attempts = $using:MaxRetries - } - } - catch { - Write-Host $_ - $Status = 0 - } - } - - if ($Status -ne 200) { - $attempts++ - - if ($attempts -lt $using:MaxRetries) - { - $attemptsLeft = $using:MaxRetries - $attempts - Write-Warning "Download failed, $attemptsLeft attempts remaining, will retry in $using:RetryWaitTimeInSeconds seconds" - Start-Sleep -Seconds $using:RetryWaitTimeInSeconds - } - else { - if ($NumFailedLinks -eq 0) { - if ($FailedFiles.Value -eq 0) { - Write-Host - } - - Write-Host "`tFile $RealPath has broken links:" - } - - Write-Host "`t`tFailed to retrieve $Link" - - $NumFailedLinks++ - } - } - else { - break - } - } - } - } - - if ($NumFailedLinks -ne 0) { - $FailedFiles.value++ - $global:LASTEXITCODE = 1 - } - } - } - - &$ValidateFile $TargetFile $FileName ([ref]$FailedFiles) - } - } - catch { - Write-Host $_ - } - finally { - $zip.Dispose() - } - - if ($FailedFiles -eq 0) { - Write-Host 'Passed.' - return [pscustomobject]@{ - result = 0 - packagePath = $PackagePath - } - } - else { - Write-PipelineTelemetryError -Category 'SourceLink' -Message "$PackagePath has broken SourceLink links." - return [pscustomobject]@{ - result = 1 - packagePath = $PackagePath - } - } -} - -function CheckJobResult( - $result, - $packagePath, - [ref]$ValidationFailures, - [switch]$logErrors) { - if ($result -ne '0') { - if ($logErrors) { - Write-PipelineTelemetryError -Category 'SourceLink' -Message "$packagePath has broken SourceLink links." - } - $ValidationFailures.Value++ - } -} - -function ValidateSourceLinkLinks { - if ($GHRepoName -ne '' -and !($GHRepoName -Match '^[^\s\/]+/[^\s\/]+$')) { - if (!($GHRepoName -Match '^[^\s-]+-[^\s]+$')) { - Write-PipelineTelemetryError -Category 'SourceLink' -Message "GHRepoName should be in the format / or -. '$GHRepoName'" - ExitWithExitCode 1 - } - else { - $GHRepoName = $GHRepoName -replace '^([^\s-]+)-([^\s]+)$', '$1/$2'; - } - } - - if ($GHCommit -ne '' -and !($GHCommit -Match '^[0-9a-fA-F]{40}$')) { - Write-PipelineTelemetryError -Category 'SourceLink' -Message "GHCommit should be a 40 chars hexadecimal string. '$GHCommit'" - ExitWithExitCode 1 - } - - if ($GHRepoName -ne '' -and $GHCommit -ne '') { - $RepoTreeURL = -Join('http://api.github.com/repos/', $GHRepoName, '/git/trees/', $GHCommit, '?recursive=1') - $CodeExtensions = @('.cs', '.vb', '.fs', '.fsi', '.fsx', '.fsscript') - - try { - # Retrieve the list of files in the repo at that particular commit point and store them in the RepoFiles hash - $Data = Invoke-WebRequest $RepoTreeURL -UseBasicParsing | ConvertFrom-Json | Select-Object -ExpandProperty tree - - foreach ($file in $Data) { - $Extension = [System.IO.Path]::GetExtension($file.path) - - if ($CodeExtensions.Contains($Extension)) { - $RepoFiles[$file.path] = 1 - } - } - } - catch { - Write-Host "Problems downloading the list of files from the repo. Url used: $RepoTreeURL . Execution will proceed without caching." - } - } - elseif ($GHRepoName -ne '' -or $GHCommit -ne '') { - Write-Host 'For using the http caching mechanism both GHRepoName and GHCommit should be informed.' - } - - if (Test-Path $ExtractPath) { - Remove-Item $ExtractPath -Force -Recurse -ErrorAction SilentlyContinue - } - - $ValidationFailures = 0 - - # Process each NuGet package in parallel - Get-ChildItem "$InputPath\*.symbols.nupkg" | - ForEach-Object { - Write-Host "Starting $($_.FullName)" - Start-Job -ScriptBlock $ValidatePackage -ArgumentList $_.FullName | Out-Null - $NumJobs = @(Get-Job -State 'Running').Count - - while ($NumJobs -ge $MaxParallelJobs) { - Write-Host "There are $NumJobs validation jobs running right now. Waiting $SecondsBetweenLoadChecks seconds to check again." - sleep $SecondsBetweenLoadChecks - $NumJobs = @(Get-Job -State 'Running').Count - } - - foreach ($Job in @(Get-Job -State 'Completed')) { - $jobResult = Wait-Job -Id $Job.Id | Receive-Job - CheckJobResult $jobResult.result $jobResult.packagePath ([ref]$ValidationFailures) -LogErrors - Remove-Job -Id $Job.Id - } - } - - foreach ($Job in @(Get-Job)) { - $jobResult = Wait-Job -Id $Job.Id | Receive-Job - CheckJobResult $jobResult.result $jobResult.packagePath ([ref]$ValidationFailures) - Remove-Job -Id $Job.Id - } - if ($ValidationFailures -gt 0) { - Write-PipelineTelemetryError -Category 'SourceLink' -Message "$ValidationFailures package(s) failed validation." - ExitWithExitCode 1 - } -} - -function InstallSourcelinkCli { - $sourcelinkCliPackageName = 'sourcelink' - - $dotnetRoot = InitializeDotNetCli -install:$true - $dotnet = "$dotnetRoot\dotnet.exe" - $toolList = & "$dotnet" tool list --global - - if (($toolList -like "*$sourcelinkCliPackageName*") -and ($toolList -like "*$sourcelinkCliVersion*")) { - Write-Host "SourceLink CLI version $sourcelinkCliVersion is already installed." - } - else { - Write-Host "Installing SourceLink CLI version $sourcelinkCliVersion..." - Write-Host 'You may need to restart your command window if this is the first dotnet tool you have installed.' - & "$dotnet" tool install $sourcelinkCliPackageName --version $sourcelinkCliVersion --verbosity "minimal" --global - } -} - -try { - InstallSourcelinkCli - - foreach ($Job in @(Get-Job)) { - Remove-Job -Id $Job.Id - } - - ValidateSourceLinkLinks -} -catch { - Write-Host $_.Exception - Write-Host $_.ScriptStackTrace - Write-PipelineTelemetryError -Category 'SourceLink' -Message $_ - ExitWithExitCode 1 -} diff --git a/src/arcade/eng/common/post-build/symbols-validation.ps1 b/src/arcade/eng/common/post-build/symbols-validation.ps1 deleted file mode 100644 index 7146e593ffae..000000000000 --- a/src/arcade/eng/common/post-build/symbols-validation.ps1 +++ /dev/null @@ -1,337 +0,0 @@ -param( - [Parameter(Mandatory = $true)][string] $InputPath, # Full path to directory where NuGet packages to be checked are stored - [Parameter(Mandatory = $true)][string] $ExtractPath, # Full path to directory where the packages will be extracted during validation - [Parameter(Mandatory = $true)][string] $DotnetSymbolVersion, # Version of dotnet symbol to use - [Parameter(Mandatory = $false)][switch] $CheckForWindowsPdbs, # If we should check for the existence of windows pdbs in addition to portable PDBs - [Parameter(Mandatory = $false)][switch] $ContinueOnError, # If we should keep checking symbols after an error - [Parameter(Mandatory = $false)][switch] $Clean, # Clean extracted symbols directory after checking symbols - [Parameter(Mandatory = $false)][string] $SymbolExclusionFile # Exclude the symbols in the file from publishing to symbol server -) - -. $PSScriptRoot\..\tools.ps1 -# Maximum number of jobs to run in parallel -$MaxParallelJobs = 16 - -# Max number of retries -$MaxRetry = 5 - -# Wait time between check for system load -$SecondsBetweenLoadChecks = 10 - -# Set error codes -Set-Variable -Name "ERROR_BADEXTRACT" -Option Constant -Value -1 -Set-Variable -Name "ERROR_FILEDOESNOTEXIST" -Option Constant -Value -2 - -$WindowsPdbVerificationParam = "" -if ($CheckForWindowsPdbs) { - $WindowsPdbVerificationParam = "--windows-pdbs" -} - -$ExclusionSet = New-Object System.Collections.Generic.HashSet[string]; - -if (!$InputPath -or !(Test-Path $InputPath)){ - Write-Host "No symbols to validate." - ExitWithExitCode 0 -} - -#Check if the path exists -if ($SymbolExclusionFile -and (Test-Path $SymbolExclusionFile)){ - [string[]]$Exclusions = Get-Content "$SymbolExclusionFile" - $Exclusions | foreach { if($_ -and $_.Trim()){$ExclusionSet.Add($_)} } -} -else{ - Write-Host "Symbol Exclusion file does not exists. No symbols to exclude." -} - -$CountMissingSymbols = { - param( - [string] $PackagePath, # Path to a NuGet package - [string] $WindowsPdbVerificationParam # If we should check for the existence of windows pdbs in addition to portable PDBs - ) - - Add-Type -AssemblyName System.IO.Compression.FileSystem - - Write-Host "Validating $PackagePath " - - # Ensure input file exist - if (!(Test-Path $PackagePath)) { - Write-PipelineTaskError "Input file does not exist: $PackagePath" - return [pscustomobject]@{ - result = $using:ERROR_FILEDOESNOTEXIST - packagePath = $PackagePath - } - } - - # Extensions for which we'll look for symbols - $RelevantExtensions = @('.dll', '.exe', '.so', '.dylib') - - # How many files are missing symbol information - $MissingSymbols = 0 - - $PackageId = [System.IO.Path]::GetFileNameWithoutExtension($PackagePath) - $PackageGuid = New-Guid - $ExtractPath = Join-Path -Path $using:ExtractPath -ChildPath $PackageGuid - $SymbolsPath = Join-Path -Path $ExtractPath -ChildPath 'Symbols' - - try { - [System.IO.Compression.ZipFile]::ExtractToDirectory($PackagePath, $ExtractPath) - } - catch { - Write-Host "Something went wrong extracting $PackagePath" - Write-Host $_ - return [pscustomobject]@{ - result = $using:ERROR_BADEXTRACT - packagePath = $PackagePath - } - } - - Get-ChildItem -Recurse $ExtractPath | - Where-Object { $RelevantExtensions -contains $_.Extension } | - ForEach-Object { - $FileName = $_.FullName - if ($FileName -Match '\\ref\\') { - Write-Host "`t Ignoring reference assembly file " $FileName - return - } - - $FirstMatchingSymbolDescriptionOrDefault = { - param( - [string] $FullPath, # Full path to the module that has to be checked - [string] $TargetServerParam, # Parameter to pass to `Symbol Tool` indicating the server to lookup for symbols - [string] $WindowsPdbVerificationParam, # Parameter to pass to potential check for windows-pdbs. - [string] $SymbolsPath - ) - - $FileName = [System.IO.Path]::GetFileName($FullPath) - $Extension = [System.IO.Path]::GetExtension($FullPath) - - # Those below are potential symbol files that the `dotnet symbol` might - # return. Which one will be returned depend on the type of file we are - # checking and which type of file was uploaded. - - # The file itself is returned - $SymbolPath = $SymbolsPath + '\' + $FileName - - # PDB file for the module - $PdbPath = $SymbolPath.Replace($Extension, '.pdb') - - # PDB file for R2R module (created by crossgen) - $NGenPdb = $SymbolPath.Replace($Extension, '.ni.pdb') - - # DBG file for a .so library - $SODbg = $SymbolPath.Replace($Extension, '.so.dbg') - - # DWARF file for a .dylib - $DylibDwarf = $SymbolPath.Replace($Extension, '.dylib.dwarf') - - $dotnetSymbolExe = "$env:USERPROFILE\.dotnet\tools" - $dotnetSymbolExe = Resolve-Path "$dotnetSymbolExe\dotnet-symbol.exe" - - $totalRetries = 0 - - while ($totalRetries -lt $using:MaxRetry) { - - # Save the output and get diagnostic output - $output = & $dotnetSymbolExe --symbols --modules $WindowsPdbVerificationParam $TargetServerParam $FullPath -o $SymbolsPath --diagnostics | Out-String - - if ((Test-Path $PdbPath) -and (Test-path $SymbolPath)) { - return 'Module and PDB for Module' - } - elseif ((Test-Path $NGenPdb) -and (Test-Path $PdbPath) -and (Test-Path $SymbolPath)) { - return 'Dll, PDB and NGen PDB' - } - elseif ((Test-Path $SODbg) -and (Test-Path $SymbolPath)) { - return 'So and DBG for SO' - } - elseif ((Test-Path $DylibDwarf) -and (Test-Path $SymbolPath)) { - return 'Dylib and Dwarf for Dylib' - } - elseif (Test-Path $SymbolPath) { - return 'Module' - } - else - { - $totalRetries++ - } - } - - return $null - } - - $FileRelativePath = $FileName.Replace("$ExtractPath\", "") - if (($($using:ExclusionSet) -ne $null) -and ($($using:ExclusionSet).Contains($FileRelativePath) -or ($($using:ExclusionSet).Contains($FileRelativePath.Replace("\", "/"))))){ - Write-Host "Skipping $FileName from symbol validation" - } - - else { - $FileGuid = New-Guid - $ExpandedSymbolsPath = Join-Path -Path $SymbolsPath -ChildPath $FileGuid - - $SymbolsOnMSDL = & $FirstMatchingSymbolDescriptionOrDefault ` - -FullPath $FileName ` - -TargetServerParam '--microsoft-symbol-server' ` - -SymbolsPath "$ExpandedSymbolsPath-msdl" ` - -WindowsPdbVerificationParam $WindowsPdbVerificationParam - $SymbolsOnSymWeb = & $FirstMatchingSymbolDescriptionOrDefault ` - -FullPath $FileName ` - -TargetServerParam '--internal-server' ` - -SymbolsPath "$ExpandedSymbolsPath-symweb" ` - -WindowsPdbVerificationParam $WindowsPdbVerificationParam - - Write-Host -NoNewLine "`t Checking file " $FileName "... " - - if ($SymbolsOnMSDL -ne $null -and $SymbolsOnSymWeb -ne $null) { - Write-Host "Symbols found on MSDL ($SymbolsOnMSDL) and SymWeb ($SymbolsOnSymWeb)" - } - else { - $MissingSymbols++ - - if ($SymbolsOnMSDL -eq $null -and $SymbolsOnSymWeb -eq $null) { - Write-Host 'No symbols found on MSDL or SymWeb!' - } - else { - if ($SymbolsOnMSDL -eq $null) { - Write-Host 'No symbols found on MSDL!' - } - else { - Write-Host 'No symbols found on SymWeb!' - } - } - } - } - } - - if ($using:Clean) { - Remove-Item $ExtractPath -Recurse -Force - } - - Pop-Location - - return [pscustomobject]@{ - result = $MissingSymbols - packagePath = $PackagePath - } -} - -function CheckJobResult( - $result, - $packagePath, - [ref]$DupedSymbols, - [ref]$TotalFailures) { - if ($result -eq $ERROR_BADEXTRACT) { - Write-PipelineTelemetryError -Category 'CheckSymbols' -Message "$packagePath has duplicated symbol files" - $DupedSymbols.Value++ - } - elseif ($result -eq $ERROR_FILEDOESNOTEXIST) { - Write-PipelineTelemetryError -Category 'CheckSymbols' -Message "$packagePath does not exist" - $TotalFailures.Value++ - } - elseif ($result -gt '0') { - Write-PipelineTelemetryError -Category 'CheckSymbols' -Message "Missing symbols for $result modules in the package $packagePath" - $TotalFailures.Value++ - } - else { - Write-Host "All symbols verified for package $packagePath" - } -} - -function CheckSymbolsAvailable { - if (Test-Path $ExtractPath) { - Remove-Item $ExtractPath -Force -Recurse -ErrorAction SilentlyContinue - } - - $TotalPackages = 0 - $TotalFailures = 0 - $DupedSymbols = 0 - - Get-ChildItem "$InputPath\*.nupkg" | - ForEach-Object { - $FileName = $_.Name - $FullName = $_.FullName - - # These packages from Arcade-Services include some native libraries that - # our current symbol uploader can't handle. Below is a workaround until - # we get issue: https://github.com/dotnet/arcade/issues/2457 sorted. - if ($FileName -Match 'Microsoft\.DotNet\.Darc\.') { - Write-Host "Ignoring Arcade-services file: $FileName" - Write-Host - return - } - elseif ($FileName -Match 'Microsoft\.DotNet\.Maestro\.Tasks\.') { - Write-Host "Ignoring Arcade-services file: $FileName" - Write-Host - return - } - - $TotalPackages++ - - Start-Job -ScriptBlock $CountMissingSymbols -ArgumentList @($FullName,$WindowsPdbVerificationParam) | Out-Null - - $NumJobs = @(Get-Job -State 'Running').Count - - while ($NumJobs -ge $MaxParallelJobs) { - Write-Host "There are $NumJobs validation jobs running right now. Waiting $SecondsBetweenLoadChecks seconds to check again." - sleep $SecondsBetweenLoadChecks - $NumJobs = @(Get-Job -State 'Running').Count - } - - foreach ($Job in @(Get-Job -State 'Completed')) { - $jobResult = Wait-Job -Id $Job.Id | Receive-Job - CheckJobResult $jobResult.result $jobResult.packagePath ([ref]$DupedSymbols) ([ref]$TotalFailures) - Remove-Job -Id $Job.Id - } - Write-Host - } - - foreach ($Job in @(Get-Job)) { - $jobResult = Wait-Job -Id $Job.Id | Receive-Job - CheckJobResult $jobResult.result $jobResult.packagePath ([ref]$DupedSymbols) ([ref]$TotalFailures) - } - - if ($TotalFailures -gt 0 -or $DupedSymbols -gt 0) { - if ($TotalFailures -gt 0) { - Write-PipelineTelemetryError -Category 'CheckSymbols' -Message "Symbols missing for $TotalFailures/$TotalPackages packages" - } - - if ($DupedSymbols -gt 0) { - Write-PipelineTelemetryError -Category 'CheckSymbols' -Message "$DupedSymbols/$TotalPackages packages had duplicated symbol files and could not be extracted" - } - - ExitWithExitCode 1 - } - else { - Write-Host "All symbols validated!" - } -} - -function InstallDotnetSymbol { - $dotnetSymbolPackageName = 'dotnet-symbol' - - $dotnetRoot = InitializeDotNetCli -install:$true - $dotnet = "$dotnetRoot\dotnet.exe" - $toolList = & "$dotnet" tool list --global - - if (($toolList -like "*$dotnetSymbolPackageName*") -and ($toolList -like "*$dotnetSymbolVersion*")) { - Write-Host "dotnet-symbol version $dotnetSymbolVersion is already installed." - } - else { - Write-Host "Installing dotnet-symbol version $dotnetSymbolVersion..." - Write-Host 'You may need to restart your command window if this is the first dotnet tool you have installed.' - & "$dotnet" tool install $dotnetSymbolPackageName --version $dotnetSymbolVersion --verbosity "minimal" --global - } -} - -try { - InstallDotnetSymbol - - foreach ($Job in @(Get-Job)) { - Remove-Job -Id $Job.Id - } - - CheckSymbolsAvailable -} -catch { - Write-Host $_.ScriptStackTrace - Write-PipelineTelemetryError -Category 'CheckSymbols' -Message $_ - ExitWithExitCode 1 -} diff --git a/src/arcade/eng/common/retain-build.ps1 b/src/arcade/eng/common/retain-build.ps1 deleted file mode 100644 index e7ba975adeb6..000000000000 --- a/src/arcade/eng/common/retain-build.ps1 +++ /dev/null @@ -1,45 +0,0 @@ - -Param( -[Parameter(Mandatory=$true)][int] $buildId, -[Parameter(Mandatory=$true)][string] $azdoOrgUri, -[Parameter(Mandatory=$true)][string] $azdoProject, -[Parameter(Mandatory=$true)][string] $token -) - -$ErrorActionPreference = 'Stop' -Set-StrictMode -Version 2.0 - -function Get-AzDOHeaders( - [string] $token) -{ - $base64AuthInfo = [Convert]::ToBase64String([Text.Encoding]::ASCII.GetBytes(":${token}")) - $headers = @{"Authorization"="Basic $base64AuthInfo"} - return $headers -} - -function Update-BuildRetention( - [string] $azdoOrgUri, - [string] $azdoProject, - [int] $buildId, - [string] $token) -{ - $headers = Get-AzDOHeaders -token $token - $requestBody = "{ - `"keepForever`": `"true`" - }" - - $requestUri = "${azdoOrgUri}/${azdoProject}/_apis/build/builds/${buildId}?api-version=6.0" - write-Host "Attempting to retain build using the following URI: ${requestUri} ..." - - try { - Invoke-RestMethod -Uri $requestUri -Method Patch -Body $requestBody -Header $headers -contentType "application/json" - Write-Host "Updated retention settings for build ${buildId}." - } - catch { - Write-Error "Failed to update retention settings for build: $_.Exception.Response.StatusDescription" - exit 1 - } -} - -Update-BuildRetention -azdoOrgUri $azdoOrgUri -azdoProject $azdoProject -buildId $buildId -token $token -exit 0 diff --git a/src/arcade/eng/common/sdl/NuGet.config b/src/arcade/eng/common/sdl/NuGet.config deleted file mode 100644 index 3849bdb3cf51..000000000000 --- a/src/arcade/eng/common/sdl/NuGet.config +++ /dev/null @@ -1,18 +0,0 @@ - - - - - - - - - - - - - - - - - - diff --git a/src/arcade/eng/common/sdl/configure-sdl-tool.ps1 b/src/arcade/eng/common/sdl/configure-sdl-tool.ps1 deleted file mode 100644 index 27f5a4115fc7..000000000000 --- a/src/arcade/eng/common/sdl/configure-sdl-tool.ps1 +++ /dev/null @@ -1,130 +0,0 @@ -Param( - [string] $GuardianCliLocation, - [string] $WorkingDirectory, - [string] $TargetDirectory, - [string] $GdnFolder, - # The list of Guardian tools to configure. For each object in the array: - # - If the item is a [hashtable], it must contain these entries: - # - Name = The tool name as Guardian knows it. - # - Scenario = (Optional) Scenario-specific name for this configuration entry. It must be unique - # among all tool entries with the same Name. - # - Args = (Optional) Array of Guardian tool configuration args, like '@("Target > C:\temp")' - # - If the item is a [string] $v, it is treated as '@{ Name="$v" }' - [object[]] $ToolsList, - [string] $GuardianLoggerLevel='Standard', - # Optional: Additional params to add to any tool using CredScan. - [string[]] $CrScanAdditionalRunConfigParams, - # Optional: Additional params to add to any tool using PoliCheck. - [string[]] $PoliCheckAdditionalRunConfigParams, - # Optional: Additional params to add to any tool using CodeQL/Semmle. - [string[]] $CodeQLAdditionalRunConfigParams, - # Optional: Additional params to add to any tool using Binskim. - [string[]] $BinskimAdditionalRunConfigParams -) - -$ErrorActionPreference = 'Stop' -Set-StrictMode -Version 2.0 -$disableConfigureToolsetImport = $true -$global:LASTEXITCODE = 0 - -try { - # `tools.ps1` checks $ci to perform some actions. Since the SDL - # scripts don't necessarily execute in the same agent that run the - # build.ps1/sh script this variable isn't automatically set. - $ci = $true - . $PSScriptRoot\..\tools.ps1 - - # Normalize tools list: all in [hashtable] form with defined values for each key. - $ToolsList = $ToolsList | - ForEach-Object { - if ($_ -is [string]) { - $_ = @{ Name = $_ } - } - - if (-not ($_['Scenario'])) { $_.Scenario = "" } - if (-not ($_['Args'])) { $_.Args = @() } - $_ - } - - Write-Host "List of tools to configure:" - $ToolsList | ForEach-Object { $_ | Out-String | Write-Host } - - # We store config files in the r directory of .gdn - $gdnConfigPath = Join-Path $GdnFolder 'r' - $ValidPath = Test-Path $GuardianCliLocation - - if ($ValidPath -eq $False) - { - Write-PipelineTelemetryError -Force -Category 'Sdl' -Message "Invalid Guardian CLI Location." - ExitWithExitCode 1 - } - - foreach ($tool in $ToolsList) { - # Put together the name and scenario to make a unique key. - $toolConfigName = $tool.Name - if ($tool.Scenario) { - $toolConfigName += "_" + $tool.Scenario - } - - Write-Host "=== Configuring $toolConfigName..." - - $gdnConfigFile = Join-Path $gdnConfigPath "$toolConfigName-configure.gdnconfig" - - # For some tools, add default and automatic args. - switch -Exact ($tool.Name) { - 'credscan' { - if ($targetDirectory) { - $tool.Args += "`"TargetDirectory < $TargetDirectory`"" - } - $tool.Args += "`"OutputType < pre`"" - $tool.Args += $CrScanAdditionalRunConfigParams - } - 'policheck' { - if ($targetDirectory) { - $tool.Args += "`"Target < $TargetDirectory`"" - } - $tool.Args += $PoliCheckAdditionalRunConfigParams - } - {$_ -in 'semmle', 'codeql'} { - if ($targetDirectory) { - $tool.Args += "`"SourceCodeDirectory < $TargetDirectory`"" - } - $tool.Args += $CodeQLAdditionalRunConfigParams - } - 'binskim' { - if ($targetDirectory) { - # Binskim crashes due to specific PDBs. GitHub issue: https://github.com/microsoft/binskim/issues/924. - # We are excluding all `_.pdb` files from the scan. - $tool.Args += "`"Target < $TargetDirectory\**;-:file|$TargetDirectory\**\_.pdb`"" - } - $tool.Args += $BinskimAdditionalRunConfigParams - } - } - - # Create variable pointing to the args array directly so we can use splat syntax later. - $toolArgs = $tool.Args - - # Configure the tool. If args array is provided or the current tool has some default arguments - # defined, add "--args" and splat each element on the end. Arg format is "{Arg id} < {Value}", - # one per parameter. Doc page for "guardian configure": - # https://dev.azure.com/securitytools/SecurityIntegration/_wiki/wikis/Guardian/1395/configure - Exec-BlockVerbosely { - & $GuardianCliLocation configure ` - --working-directory $WorkingDirectory ` - --tool $tool.Name ` - --output-path $gdnConfigFile ` - --logger-level $GuardianLoggerLevel ` - --noninteractive ` - --force ` - $(if ($toolArgs) { "--args" }) @toolArgs - Exit-IfNZEC "Sdl" - } - - Write-Host "Created '$toolConfigName' configuration file: $gdnConfigFile" - } -} -catch { - Write-Host $_.ScriptStackTrace - Write-PipelineTelemetryError -Force -Category 'Sdl' -Message $_ - ExitWithExitCode 1 -} diff --git a/src/arcade/eng/common/sdl/execute-all-sdl-tools.ps1 b/src/arcade/eng/common/sdl/execute-all-sdl-tools.ps1 deleted file mode 100644 index 4715d75e974d..000000000000 --- a/src/arcade/eng/common/sdl/execute-all-sdl-tools.ps1 +++ /dev/null @@ -1,167 +0,0 @@ -Param( - [string] $GuardianPackageName, # Required: the name of guardian CLI package (not needed if GuardianCliLocation is specified) - [string] $NugetPackageDirectory, # Required: directory where NuGet packages are installed (not needed if GuardianCliLocation is specified) - [string] $GuardianCliLocation, # Optional: Direct location of Guardian CLI executable if GuardianPackageName & NugetPackageDirectory are not specified - [string] $Repository=$env:BUILD_REPOSITORY_NAME, # Required: the name of the repository (e.g. dotnet/arcade) - [string] $BranchName=$env:BUILD_SOURCEBRANCH, # Optional: name of branch or version of gdn settings; defaults to master - [string] $SourceDirectory=$env:BUILD_SOURCESDIRECTORY, # Required: the directory where source files are located - [string] $ArtifactsDirectory = (Join-Path $env:BUILD_ARTIFACTSTAGINGDIRECTORY ('artifacts')), # Required: the directory where build artifacts are located - [string] $AzureDevOpsAccessToken, # Required: access token for dnceng; should be provided via KeyVault - - # Optional: list of SDL tools to run on source code. See 'configure-sdl-tool.ps1' for tools list - # format. - [object[]] $SourceToolsList, - # Optional: list of SDL tools to run on built artifacts. See 'configure-sdl-tool.ps1' for tools - # list format. - [object[]] $ArtifactToolsList, - # Optional: list of SDL tools to run without automatically specifying a target directory. See - # 'configure-sdl-tool.ps1' for tools list format. - [object[]] $CustomToolsList, - - [bool] $TsaPublish=$False, # Optional: true will publish results to TSA; only set to true after onboarding to TSA; TSA is the automated framework used to upload test results as bugs. - [string] $TsaBranchName=$env:BUILD_SOURCEBRANCH, # Optional: required for TSA publish; defaults to $(Build.SourceBranchName); TSA is the automated framework used to upload test results as bugs. - [string] $TsaRepositoryName=$env:BUILD_REPOSITORY_NAME, # Optional: TSA repository name; will be generated automatically if not submitted; TSA is the automated framework used to upload test results as bugs. - [string] $BuildNumber=$env:BUILD_BUILDNUMBER, # Optional: required for TSA publish; defaults to $(Build.BuildNumber) - [bool] $UpdateBaseline=$False, # Optional: if true, will update the baseline in the repository; should only be run after fixing any issues which need to be fixed - [bool] $TsaOnboard=$False, # Optional: if true, will onboard the repository to TSA; should only be run once; TSA is the automated framework used to upload test results as bugs. - [string] $TsaInstanceUrl, # Optional: only needed if TsaOnboard or TsaPublish is true; the instance-url registered with TSA; TSA is the automated framework used to upload test results as bugs. - [string] $TsaCodebaseName, # Optional: only needed if TsaOnboard or TsaPublish is true; the name of the codebase registered with TSA; TSA is the automated framework used to upload test results as bugs. - [string] $TsaProjectName, # Optional: only needed if TsaOnboard or TsaPublish is true; the name of the project registered with TSA; TSA is the automated framework used to upload test results as bugs. - [string] $TsaNotificationEmail, # Optional: only needed if TsaOnboard is true; the email(s) which will receive notifications of TSA bug filings (e.g. alias@microsoft.com); TSA is the automated framework used to upload test results as bugs. - [string] $TsaCodebaseAdmin, # Optional: only needed if TsaOnboard is true; the aliases which are admins of the TSA codebase (e.g. DOMAIN\alias); TSA is the automated framework used to upload test results as bugs. - [string] $TsaBugAreaPath, # Optional: only needed if TsaOnboard is true; the area path where TSA will file bugs in AzDO; TSA is the automated framework used to upload test results as bugs. - [string] $TsaIterationPath, # Optional: only needed if TsaOnboard is true; the iteration path where TSA will file bugs in AzDO; TSA is the automated framework used to upload test results as bugs. - [string] $GuardianLoggerLevel='Standard', # Optional: the logger level for the Guardian CLI; options are Trace, Verbose, Standard, Warning, and Error - [string[]] $CrScanAdditionalRunConfigParams, # Optional: Additional Params to custom build a CredScan run config in the format @("xyz:abc","sdf:1") - [string[]] $PoliCheckAdditionalRunConfigParams, # Optional: Additional Params to custom build a Policheck run config in the format @("xyz:abc","sdf:1") - [string[]] $CodeQLAdditionalRunConfigParams, # Optional: Additional Params to custom build a Semmle/CodeQL run config in the format @("xyz < abc","sdf < 1") - [string[]] $BinskimAdditionalRunConfigParams, # Optional: Additional Params to custom build a Binskim run config in the format @("xyz < abc","sdf < 1") - [bool] $BreakOnFailure=$False # Optional: Fail the build if there were errors during the run -) - -try { - $ErrorActionPreference = 'Stop' - Set-StrictMode -Version 2.0 - $disableConfigureToolsetImport = $true - $global:LASTEXITCODE = 0 - - # `tools.ps1` checks $ci to perform some actions. Since the SDL - # scripts don't necessarily execute in the same agent that run the - # build.ps1/sh script this variable isn't automatically set. - $ci = $true - . $PSScriptRoot\..\tools.ps1 - - #Replace repo names to the format of org/repo - if (!($Repository.contains('/'))) { - $RepoName = $Repository -replace '(.*?)-(.*)', '$1/$2'; - } - else{ - $RepoName = $Repository; - } - - if ($GuardianPackageName) { - $guardianCliLocation = Join-Path $NugetPackageDirectory (Join-Path $GuardianPackageName (Join-Path 'tools' 'guardian.cmd')) - } else { - $guardianCliLocation = $GuardianCliLocation - } - - $workingDirectory = (Split-Path $SourceDirectory -Parent) - $ValidPath = Test-Path $guardianCliLocation - - if ($ValidPath -eq $False) - { - Write-PipelineTelemetryError -Force -Category 'Sdl' -Message 'Invalid Guardian CLI Location.' - ExitWithExitCode 1 - } - - Exec-BlockVerbosely { - & $(Join-Path $PSScriptRoot 'init-sdl.ps1') -GuardianCliLocation $guardianCliLocation -Repository $RepoName -BranchName $BranchName -WorkingDirectory $workingDirectory -AzureDevOpsAccessToken $AzureDevOpsAccessToken -GuardianLoggerLevel $GuardianLoggerLevel - } - $gdnFolder = Join-Path $workingDirectory '.gdn' - - if ($TsaOnboard) { - if ($TsaCodebaseName -and $TsaNotificationEmail -and $TsaCodebaseAdmin -and $TsaBugAreaPath) { - Exec-BlockVerbosely { - & $guardianCliLocation tsa-onboard --codebase-name "$TsaCodebaseName" --notification-alias "$TsaNotificationEmail" --codebase-admin "$TsaCodebaseAdmin" --instance-url "$TsaInstanceUrl" --project-name "$TsaProjectName" --area-path "$TsaBugAreaPath" --iteration-path "$TsaIterationPath" --working-directory $workingDirectory --logger-level $GuardianLoggerLevel - } - if ($LASTEXITCODE -ne 0) { - Write-PipelineTelemetryError -Force -Category 'Sdl' -Message "Guardian tsa-onboard failed with exit code $LASTEXITCODE." - ExitWithExitCode $LASTEXITCODE - } - } else { - Write-PipelineTelemetryError -Force -Category 'Sdl' -Message 'Could not onboard to TSA -- not all required values ($TsaCodebaseName, $TsaNotificationEmail, $TsaCodebaseAdmin, $TsaBugAreaPath) were specified.' - ExitWithExitCode 1 - } - } - - # Configure a list of tools with a default target directory. Populates the ".gdn/r" directory. - function Configure-ToolsList([object[]] $tools, [string] $targetDirectory) { - if ($tools -and $tools.Count -gt 0) { - Exec-BlockVerbosely { - & $(Join-Path $PSScriptRoot 'configure-sdl-tool.ps1') ` - -GuardianCliLocation $guardianCliLocation ` - -WorkingDirectory $workingDirectory ` - -TargetDirectory $targetDirectory ` - -GdnFolder $gdnFolder ` - -ToolsList $tools ` - -AzureDevOpsAccessToken $AzureDevOpsAccessToken ` - -GuardianLoggerLevel $GuardianLoggerLevel ` - -CrScanAdditionalRunConfigParams $CrScanAdditionalRunConfigParams ` - -PoliCheckAdditionalRunConfigParams $PoliCheckAdditionalRunConfigParams ` - -CodeQLAdditionalRunConfigParams $CodeQLAdditionalRunConfigParams ` - -BinskimAdditionalRunConfigParams $BinskimAdditionalRunConfigParams - if ($BreakOnFailure) { - Exit-IfNZEC "Sdl" - } - } - } - } - - # Configure Artifact and Source tools with default Target directories. - Configure-ToolsList $ArtifactToolsList $ArtifactsDirectory - Configure-ToolsList $SourceToolsList $SourceDirectory - # Configure custom tools with no default Target directory. - Configure-ToolsList $CustomToolsList $null - - # At this point, all tools are configured in the ".gdn" directory. Run them all in a single call. - # (If we used "run" multiple times, each run would overwrite data from earlier runs.) - Exec-BlockVerbosely { - & $(Join-Path $PSScriptRoot 'run-sdl.ps1') ` - -GuardianCliLocation $guardianCliLocation ` - -WorkingDirectory $SourceDirectory ` - -UpdateBaseline $UpdateBaseline ` - -GdnFolder $gdnFolder - } - - if ($TsaPublish) { - if ($TsaBranchName -and $BuildNumber) { - if (-not $TsaRepositoryName) { - $TsaRepositoryName = "$($Repository)-$($BranchName)" - } - Exec-BlockVerbosely { - & $guardianCliLocation tsa-publish --all-tools --repository-name "$TsaRepositoryName" --branch-name "$TsaBranchName" --build-number "$BuildNumber" --onboard $True --codebase-name "$TsaCodebaseName" --notification-alias "$TsaNotificationEmail" --codebase-admin "$TsaCodebaseAdmin" --instance-url "$TsaInstanceUrl" --project-name "$TsaProjectName" --area-path "$TsaBugAreaPath" --iteration-path "$TsaIterationPath" --working-directory $workingDirectory --logger-level $GuardianLoggerLevel - } - if ($LASTEXITCODE -ne 0) { - Write-PipelineTelemetryError -Force -Category 'Sdl' -Message "Guardian tsa-publish failed with exit code $LASTEXITCODE." - ExitWithExitCode $LASTEXITCODE - } - } else { - Write-PipelineTelemetryError -Force -Category 'Sdl' -Message 'Could not publish to TSA -- not all required values ($TsaBranchName, $BuildNumber) were specified.' - ExitWithExitCode 1 - } - } - - if ($BreakOnFailure) { - Write-Host "Failing the build in case of breaking results..." - Exec-BlockVerbosely { - & $guardianCliLocation break --working-directory $workingDirectory --logger-level $GuardianLoggerLevel - } - } else { - Write-Host "Letting the build pass even if there were breaking results..." - } -} -catch { - Write-Host $_.ScriptStackTrace - Write-PipelineTelemetryError -Force -Category 'Sdl' -Message $_ - exit 1 -} diff --git a/src/arcade/eng/common/sdl/extract-artifact-archives.ps1 b/src/arcade/eng/common/sdl/extract-artifact-archives.ps1 deleted file mode 100644 index 68da4fbf2571..000000000000 --- a/src/arcade/eng/common/sdl/extract-artifact-archives.ps1 +++ /dev/null @@ -1,63 +0,0 @@ -# This script looks for each archive file in a directory and extracts it into the target directory. -# For example, the file "$InputPath/bin.tar.gz" extracts to "$ExtractPath/bin.tar.gz.extracted/**". -# Uses the "tar" utility added to Windows 10 / Windows 2019 that supports tar.gz and zip. -param( - # Full path to directory where archives are stored. - [Parameter(Mandatory=$true)][string] $InputPath, - # Full path to directory to extract archives into. May be the same as $InputPath. - [Parameter(Mandatory=$true)][string] $ExtractPath -) - -$ErrorActionPreference = 'Stop' -Set-StrictMode -Version 2.0 - -$disableConfigureToolsetImport = $true - -try { - # `tools.ps1` checks $ci to perform some actions. Since the SDL - # scripts don't necessarily execute in the same agent that run the - # build.ps1/sh script this variable isn't automatically set. - $ci = $true - . $PSScriptRoot\..\tools.ps1 - - Measure-Command { - $jobs = @() - - # Find archive files for non-Windows and Windows builds. - $archiveFiles = @( - Get-ChildItem (Join-Path $InputPath "*.tar.gz") - Get-ChildItem (Join-Path $InputPath "*.zip") - ) - - foreach ($targzFile in $archiveFiles) { - $jobs += Start-Job -ScriptBlock { - $file = $using:targzFile - $fileName = [System.IO.Path]::GetFileName($file) - $extractDir = Join-Path $using:ExtractPath "$fileName.extracted" - - New-Item $extractDir -ItemType Directory -Force | Out-Null - - Write-Host "Extracting '$file' to '$extractDir'..." - - # Pipe errors to stdout to prevent PowerShell detecting them and quitting the job early. - # This type of quit skips the catch, so we wouldn't be able to tell which file triggered the - # error. Save output so it can be stored in the exception string along with context. - $output = tar -xf $file -C $extractDir 2>&1 - # Handle NZEC manually rather than using Exit-IfNZEC: we are in a background job, so we - # don't have access to the outer scope. - if ($LASTEXITCODE -ne 0) { - throw "Error extracting '$file': non-zero exit code ($LASTEXITCODE). Output: '$output'" - } - - Write-Host "Extracted to $extractDir" - } - } - - Receive-Job $jobs -Wait - } -} -catch { - Write-Host $_ - Write-PipelineTelemetryError -Force -Category 'Sdl' -Message $_ - ExitWithExitCode 1 -} diff --git a/src/arcade/eng/common/sdl/extract-artifact-packages.ps1 b/src/arcade/eng/common/sdl/extract-artifact-packages.ps1 deleted file mode 100644 index f031ed5b25e9..000000000000 --- a/src/arcade/eng/common/sdl/extract-artifact-packages.ps1 +++ /dev/null @@ -1,82 +0,0 @@ -param( - [Parameter(Mandatory=$true)][string] $InputPath, # Full path to directory where artifact packages are stored - [Parameter(Mandatory=$true)][string] $ExtractPath # Full path to directory where the packages will be extracted -) - -$ErrorActionPreference = 'Stop' -Set-StrictMode -Version 2.0 - -$disableConfigureToolsetImport = $true - -function ExtractArtifacts { - if (!(Test-Path $InputPath)) { - Write-Host "Input Path does not exist: $InputPath" - ExitWithExitCode 0 - } - $Jobs = @() - Get-ChildItem "$InputPath\*.nupkg" | - ForEach-Object { - $Jobs += Start-Job -ScriptBlock $ExtractPackage -ArgumentList $_.FullName - } - - foreach ($Job in $Jobs) { - Wait-Job -Id $Job.Id | Receive-Job - } -} - -try { - # `tools.ps1` checks $ci to perform some actions. Since the SDL - # scripts don't necessarily execute in the same agent that run the - # build.ps1/sh script this variable isn't automatically set. - $ci = $true - . $PSScriptRoot\..\tools.ps1 - - $ExtractPackage = { - param( - [string] $PackagePath # Full path to a NuGet package - ) - - if (!(Test-Path $PackagePath)) { - Write-PipelineTelemetryError -Category 'Build' -Message "Input file does not exist: $PackagePath" - ExitWithExitCode 1 - } - - $RelevantExtensions = @('.dll', '.exe', '.pdb') - Write-Host -NoNewLine 'Extracting ' ([System.IO.Path]::GetFileName($PackagePath)) '...' - - $PackageId = [System.IO.Path]::GetFileNameWithoutExtension($PackagePath) - $ExtractPath = Join-Path -Path $using:ExtractPath -ChildPath $PackageId - - Add-Type -AssemblyName System.IO.Compression.FileSystem - - [System.IO.Directory]::CreateDirectory($ExtractPath); - - try { - $zip = [System.IO.Compression.ZipFile]::OpenRead($PackagePath) - - $zip.Entries | - Where-Object {$RelevantExtensions -contains [System.IO.Path]::GetExtension($_.Name)} | - ForEach-Object { - $TargetPath = Join-Path -Path $ExtractPath -ChildPath (Split-Path -Path $_.FullName) - [System.IO.Directory]::CreateDirectory($TargetPath); - - $TargetFile = Join-Path -Path $ExtractPath -ChildPath $_.FullName - [System.IO.Compression.ZipFileExtensions]::ExtractToFile($_, $TargetFile) - } - } - catch { - Write-Host $_ - Write-PipelineTelemetryError -Force -Category 'Sdl' -Message $_ - ExitWithExitCode 1 - } - finally { - $zip.Dispose() - } - } - Measure-Command { ExtractArtifacts } -} -catch { - Write-Host $_ - Write-PipelineTelemetryError -Force -Category 'Sdl' -Message $_ - ExitWithExitCode 1 -} diff --git a/src/arcade/eng/common/sdl/init-sdl.ps1 b/src/arcade/eng/common/sdl/init-sdl.ps1 deleted file mode 100644 index 3ac1d92b3700..000000000000 --- a/src/arcade/eng/common/sdl/init-sdl.ps1 +++ /dev/null @@ -1,55 +0,0 @@ -Param( - [string] $GuardianCliLocation, - [string] $Repository, - [string] $BranchName='master', - [string] $WorkingDirectory, - [string] $AzureDevOpsAccessToken, - [string] $GuardianLoggerLevel='Standard' -) - -$ErrorActionPreference = 'Stop' -Set-StrictMode -Version 2.0 -$disableConfigureToolsetImport = $true -$global:LASTEXITCODE = 0 - -# `tools.ps1` checks $ci to perform some actions. Since the SDL -# scripts don't necessarily execute in the same agent that run the -# build.ps1/sh script this variable isn't automatically set. -$ci = $true -. $PSScriptRoot\..\tools.ps1 - -# Don't display the console progress UI - it's a huge perf hit -$ProgressPreference = 'SilentlyContinue' - -# Construct basic auth from AzDO access token; construct URI to the repository's gdn folder stored in that repository; construct location of zip file -$encodedPat = [Convert]::ToBase64String([System.Text.Encoding]::ASCII.GetBytes(":$AzureDevOpsAccessToken")) -$escapedRepository = [Uri]::EscapeDataString("/$Repository/$BranchName/.gdn") -$uri = "https://dev.azure.com/dnceng/internal/_apis/git/repositories/sdl-tool-cfg/Items?path=$escapedRepository&versionDescriptor[versionOptions]=0&`$format=zip&api-version=5.0" -$zipFile = "$WorkingDirectory/gdn.zip" - -Add-Type -AssemblyName System.IO.Compression.FileSystem -$gdnFolder = (Join-Path $WorkingDirectory '.gdn') - -try { - # if the folder does not exist, we'll do a guardian init and push it to the remote repository - Write-Host 'Initializing Guardian...' - Write-Host "$GuardianCliLocation init --working-directory $WorkingDirectory --logger-level $GuardianLoggerLevel" - & $GuardianCliLocation init --working-directory $WorkingDirectory --logger-level $GuardianLoggerLevel - if ($LASTEXITCODE -ne 0) { - Write-PipelineTelemetryError -Force -Category 'Build' -Message "Guardian init failed with exit code $LASTEXITCODE." - ExitWithExitCode $LASTEXITCODE - } - # We create the mainbaseline so it can be edited later - Write-Host "$GuardianCliLocation baseline --working-directory $WorkingDirectory --name mainbaseline" - & $GuardianCliLocation baseline --working-directory $WorkingDirectory --name mainbaseline - if ($LASTEXITCODE -ne 0) { - Write-PipelineTelemetryError -Force -Category 'Build' -Message "Guardian baseline failed with exit code $LASTEXITCODE." - ExitWithExitCode $LASTEXITCODE - } - ExitWithExitCode 0 -} -catch { - Write-Host $_.ScriptStackTrace - Write-PipelineTelemetryError -Force -Category 'Sdl' -Message $_ - ExitWithExitCode 1 -} diff --git a/src/arcade/eng/common/sdl/packages.config b/src/arcade/eng/common/sdl/packages.config deleted file mode 100644 index e5f543ea68c2..000000000000 --- a/src/arcade/eng/common/sdl/packages.config +++ /dev/null @@ -1,4 +0,0 @@ - - - - diff --git a/src/arcade/eng/common/sdl/run-sdl.ps1 b/src/arcade/eng/common/sdl/run-sdl.ps1 deleted file mode 100644 index 2eac8c78f103..000000000000 --- a/src/arcade/eng/common/sdl/run-sdl.ps1 +++ /dev/null @@ -1,49 +0,0 @@ -Param( - [string] $GuardianCliLocation, - [string] $WorkingDirectory, - [string] $GdnFolder, - [string] $UpdateBaseline, - [string] $GuardianLoggerLevel='Standard' -) - -$ErrorActionPreference = 'Stop' -Set-StrictMode -Version 2.0 -$disableConfigureToolsetImport = $true -$global:LASTEXITCODE = 0 - -try { - # `tools.ps1` checks $ci to perform some actions. Since the SDL - # scripts don't necessarily execute in the same agent that run the - # build.ps1/sh script this variable isn't automatically set. - $ci = $true - . $PSScriptRoot\..\tools.ps1 - - # We store config files in the r directory of .gdn - $gdnConfigPath = Join-Path $GdnFolder 'r' - $ValidPath = Test-Path $GuardianCliLocation - - if ($ValidPath -eq $False) - { - Write-PipelineTelemetryError -Force -Category 'Sdl' -Message "Invalid Guardian CLI Location." - ExitWithExitCode 1 - } - - $gdnConfigFiles = Get-ChildItem $gdnConfigPath -Recurse -Include '*.gdnconfig' - Write-Host "Discovered Guardian config files:" - $gdnConfigFiles | Out-String | Write-Host - - Exec-BlockVerbosely { - & $GuardianCliLocation run ` - --working-directory $WorkingDirectory ` - --baseline mainbaseline ` - --update-baseline $UpdateBaseline ` - --logger-level $GuardianLoggerLevel ` - --config @gdnConfigFiles - Exit-IfNZEC "Sdl" - } -} -catch { - Write-Host $_.ScriptStackTrace - Write-PipelineTelemetryError -Force -Category 'Sdl' -Message $_ - ExitWithExitCode 1 -} diff --git a/src/arcade/eng/common/sdl/sdl.ps1 b/src/arcade/eng/common/sdl/sdl.ps1 deleted file mode 100644 index 648c5068d7d6..000000000000 --- a/src/arcade/eng/common/sdl/sdl.ps1 +++ /dev/null @@ -1,38 +0,0 @@ - -function Install-Gdn { - param( - [Parameter(Mandatory=$true)] - [string]$Path, - - # If omitted, install the latest version of Guardian, otherwise install that specific version. - [string]$Version - ) - - $ErrorActionPreference = 'Stop' - Set-StrictMode -Version 2.0 - $disableConfigureToolsetImport = $true - $global:LASTEXITCODE = 0 - - # `tools.ps1` checks $ci to perform some actions. Since the SDL - # scripts don't necessarily execute in the same agent that run the - # build.ps1/sh script this variable isn't automatically set. - $ci = $true - . $PSScriptRoot\..\tools.ps1 - - $argumentList = @("install", "Microsoft.Guardian.Cli", "-Source https://securitytools.pkgs.visualstudio.com/_packaging/Guardian/nuget/v3/index.json", "-OutputDirectory $Path", "-NonInteractive", "-NoCache") - - if ($Version) { - $argumentList += "-Version $Version" - } - - Start-Process nuget -Verbose -ArgumentList $argumentList -NoNewWindow -Wait - - $gdnCliPath = Get-ChildItem -Filter guardian.cmd -Recurse -Path $Path - - if (!$gdnCliPath) - { - Write-PipelineTelemetryError -Category 'Sdl' -Message 'Failure installing Guardian' - } - - return $gdnCliPath.FullName -} \ No newline at end of file diff --git a/src/arcade/eng/common/sdl/trim-assets-version.ps1 b/src/arcade/eng/common/sdl/trim-assets-version.ps1 deleted file mode 100644 index 0daa2a9e9462..000000000000 --- a/src/arcade/eng/common/sdl/trim-assets-version.ps1 +++ /dev/null @@ -1,75 +0,0 @@ -<# -.SYNOPSIS -Install and run the 'Microsoft.DotNet.VersionTools.Cli' tool with the 'trim-artifacts-version' command to trim the version from the NuGet assets file name. - -.PARAMETER InputPath -Full path to directory where artifact packages are stored - -.PARAMETER Recursive -Search for NuGet packages recursively - -#> - -Param( - [string] $InputPath, - [bool] $Recursive = $true -) - -$CliToolName = "Microsoft.DotNet.VersionTools.Cli" - -function Install-VersionTools-Cli { - param( - [Parameter(Mandatory=$true)][string]$Version - ) - - Write-Host "Installing the package '$CliToolName' with a version of '$version' ..." - $feed = "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json" - - $argumentList = @("tool", "install", "--local", "$CliToolName", "--add-source $feed", "--no-cache", "--version $Version", "--create-manifest-if-needed") - Start-Process "$dotnet" -Verbose -ArgumentList $argumentList -NoNewWindow -Wait -} - -# ------------------------------------------------------------------- - -if (!(Test-Path $InputPath)) { - Write-Host "Input Path '$InputPath' does not exist" - ExitWithExitCode 1 -} - -$ErrorActionPreference = 'Stop' -Set-StrictMode -Version 2.0 - -$disableConfigureToolsetImport = $true -$global:LASTEXITCODE = 0 - -# `tools.ps1` checks $ci to perform some actions. Since the SDL -# scripts don't necessarily execute in the same agent that run the -# build.ps1/sh script this variable isn't automatically set. -$ci = $true -. $PSScriptRoot\..\tools.ps1 - -try { - $dotnetRoot = InitializeDotNetCli -install:$true - $dotnet = "$dotnetRoot\dotnet.exe" - - $toolsetVersion = Read-ArcadeSdkVersion - Install-VersionTools-Cli -Version $toolsetVersion - - $cliToolFound = (& "$dotnet" tool list --local | Where-Object {$_.Split(' ')[0] -eq $CliToolName}) - if ($null -eq $cliToolFound) { - Write-PipelineTelemetryError -Force -Category 'Sdl' -Message "The '$CliToolName' tool is not installed." - ExitWithExitCode 1 - } - - Exec-BlockVerbosely { - & "$dotnet" $CliToolName trim-assets-version ` - --assets-path $InputPath ` - --recursive $Recursive - Exit-IfNZEC "Sdl" - } -} -catch { - Write-Host $_ - Write-PipelineTelemetryError -Force -Category 'Sdl' -Message $_ - ExitWithExitCode 1 -} diff --git a/src/arcade/eng/common/template-guidance.md b/src/arcade/eng/common/template-guidance.md deleted file mode 100644 index 98bbc1ded0ba..000000000000 --- a/src/arcade/eng/common/template-guidance.md +++ /dev/null @@ -1,133 +0,0 @@ -# Overview - -Arcade provides templates for public (`/templates`) and 1ES pipeline templates (`/templates-official`) scenarios. Pipelines which are required to be managed by 1ES pipeline templates should reference `/templates-offical`, all other pipelines may reference `/templates`. - -## How to use - -Basic guidance is: - -- 1ES Pipeline Template or 1ES Microbuild template runs should reference `eng/common/templates-official`. Any internal production-graded pipeline should use these templates. - -- All other runs should reference `eng/common/templates`. - -See [azure-pipelines.yml](../../azure-pipelines.yml) (templates-official example) or [azure-pipelines-pr.yml](../../azure-pipelines-pr.yml) (templates example) for examples. - -#### The `templateIs1ESManaged` parameter - -The `templateIs1ESManaged` is available on most templates and affects which of the variants is used for nested templates. See [Development Notes](#development-notes) below for more information on the `templateIs1ESManaged1 parameter. - -- For templates under `job/`, `jobs/`, `steps`, or `post-build/`, this parameter must be explicitly set. - -## Multiple outputs - -1ES pipeline templates impose a policy where every publish artifact execution results in additional security scans being injected into your pipeline. When using `templates-official/jobs/jobs.yml`, Arcade reduces the number of additional security injections by gathering all publishing outputs into the [Build.ArtifactStagingDirectory](https://learn.microsoft.com/en-us/azure/devops/pipelines/build/variables?view=azure-devops&tabs=yaml#build-variables-devops-services), and utilizing the [outputParentDirectory](https://eng.ms/docs/cloud-ai-platform/devdiv/one-engineering-system-1es/1es-docs/1es-pipeline-templates/features/outputs#multiple-outputs) feature of 1ES pipeline templates. When implementing your pipeline, if you ensure publish artifacts are located in the `$(Build.ArtifactStagingDirectory)`, and utilize the 1ES provided template context, then you can reduce the number of security scans for your pipeline. - -Example: -``` yaml -# azure-pipelines.yml -extends: - template: azure-pipelines/MicroBuild.1ES.Official.yml@MicroBuildTemplate - parameters: - stages: - - stage: build - jobs: - - template: /eng/common/templates-official/jobs/jobs.yml@self - parameters: - # 1ES makes use of outputs to reduce security task injection overhead - templateContext: - outputs: - - output: pipelineArtifact - displayName: 'Publish logs from source' - continueOnError: true - condition: always() - targetPath: $(Build.ArtifactStagingDirectory)/artifacts/log - artifactName: Logs - jobs: - - job: Windows - steps: - - script: echo "friendly neighborhood" > artifacts/marvel/spiderman.txt - # copy build outputs to artifact staging directory for publishing - - task: CopyFiles@2 - displayName: Gather build output - inputs: - SourceFolder: '$(Build.SourcesDirectory)/artifacts/marvel' - Contents: '**' - TargetFolder: '$(Build.ArtifactStagingDirectory)/artifacts/marvel' -``` - -Note: Multiple outputs are ONLY applicable to 1ES PT publishing (only usable when referencing `templates-official`). - -## Development notes - -**Folder / file structure** - -``` text -eng\common\ - [templates || templates-official]\ - job\ - job.yml (shim + artifact publishing logic) - onelocbuild.yml (shim) - publish-build-assets.yml (shim) - source-build.yml (shim) - source-index-stage1.yml (shim) - jobs\ - codeql-build.yml (shim) - jobs.yml (shim) - source-build.yml (shim) - post-build\ - post-build.yml (shim) - common-variabls.yml (shim) - setup-maestro-vars.yml (shim) - steps\ - publish-build-artifacts.yml (logic) - publish-pipeline-artifacts.yml (logic) - component-governance.yml (shim) - generate-sbom.yml (shim) - publish-logs.yml (shim) - retain-build.yml (shim) - send-to-helix.yml (shim) - source-build.yml (shim) - variables\ - pool-providers.yml (logic + redirect) # templates/variables/pool-providers.yml will redirect to templates-official/variables/pool-providers.yml if you are running in the internal project - sdl-variables.yml (logic) - core-templates\ - job\ - job.yml (logic) - onelocbuild.yml (logic) - publish-build-assets.yml (logic) - source-build.yml (logic) - source-index-stage1.yml (logic) - jobs\ - codeql-build.yml (logic) - jobs.yml (logic) - source-build.yml (logic) - post-build\ - common-variabls.yml (logic) - post-build.yml (logic) - setup-maestro-vars.yml (logic) - steps\ - component-governance.yml (logic) - generate-sbom.yml (logic) - publish-build-artifacts.yml (redirect) - publish-logs.yml (logic) - publish-pipeline-artifacts.yml (redirect) - retain-build.yml (logic) - send-to-helix.yml (logic) - source-build.yml (logic) - variables\ - pool-providers.yml (redirect) -``` - -In the table above, a file is designated as "shim", "logic", or "redirect". - -- shim - represents a yaml file which is an intermediate step between pipeline logic and .Net Core Engineering's templates (`core-templates`) and defines the `is1ESPipeline` parameter value. - -- logic - represents actual base template logic. - -- redirect- represents a file in `core-templates` which redirects to the "logic" file in either `templates` or `templates-official`. - -Logic for Arcade's templates live **primarily** in the `core-templates` folder. The exceptions to the location of the logic files are around artifact publishing, which is handled differently between 1es pipeline templates and standard templates. `templates` and `templates-official` provide shim entry points which redirect to `core-templates` while also defining the `is1ESPipeline` parameter. If a shim is referenced in `templates`, then `is1ESPipeline` is set to `false`. If a shim is referenced in `templates-official`, then `is1ESPipeline` is set to `true`. - -Within `templates` and `templates-official`, the templates at the "stages", and "jobs" / "job" level have been replaced with shims. Templates at the "steps" and "variables" level are typically too granular to be replaced with shims and instead persist logic which is directly applicable to either scenario. - -Within `core-templates`, there are a handful of places where logic is dependent on which shim entry point was used. In those places, we redirect back to the respective logic file in `templates` or `templates-official`. diff --git a/src/arcade/eng/common/templates-official/job/onelocbuild.yml b/src/arcade/eng/common/templates-official/job/onelocbuild.yml deleted file mode 100644 index 0f0c514b912d..000000000000 --- a/src/arcade/eng/common/templates-official/job/onelocbuild.yml +++ /dev/null @@ -1,7 +0,0 @@ -jobs: -- template: /eng/common/core-templates/job/onelocbuild.yml - parameters: - is1ESPipeline: true - - ${{ each parameter in parameters }}: - ${{ parameter.key }}: ${{ parameter.value }} diff --git a/src/arcade/eng/common/templates-official/job/publish-build-assets.yml b/src/arcade/eng/common/templates-official/job/publish-build-assets.yml deleted file mode 100644 index d667a70e8de7..000000000000 --- a/src/arcade/eng/common/templates-official/job/publish-build-assets.yml +++ /dev/null @@ -1,7 +0,0 @@ -jobs: -- template: /eng/common/core-templates/job/publish-build-assets.yml - parameters: - is1ESPipeline: true - - ${{ each parameter in parameters }}: - ${{ parameter.key }}: ${{ parameter.value }} diff --git a/src/arcade/eng/common/templates-official/job/source-build.yml b/src/arcade/eng/common/templates-official/job/source-build.yml deleted file mode 100644 index 1a480034b678..000000000000 --- a/src/arcade/eng/common/templates-official/job/source-build.yml +++ /dev/null @@ -1,7 +0,0 @@ -jobs: -- template: /eng/common/core-templates/job/source-build.yml - parameters: - is1ESPipeline: true - - ${{ each parameter in parameters }}: - ${{ parameter.key }}: ${{ parameter.value }} diff --git a/src/arcade/eng/common/templates-official/job/source-index-stage1.yml b/src/arcade/eng/common/templates-official/job/source-index-stage1.yml deleted file mode 100644 index 6d5ead316f92..000000000000 --- a/src/arcade/eng/common/templates-official/job/source-index-stage1.yml +++ /dev/null @@ -1,7 +0,0 @@ -jobs: -- template: /eng/common/core-templates/job/source-index-stage1.yml - parameters: - is1ESPipeline: true - - ${{ each parameter in parameters }}: - ${{ parameter.key }}: ${{ parameter.value }} diff --git a/src/arcade/eng/common/templates-official/jobs/codeql-build.yml b/src/arcade/eng/common/templates-official/jobs/codeql-build.yml deleted file mode 100644 index a726322ecfe0..000000000000 --- a/src/arcade/eng/common/templates-official/jobs/codeql-build.yml +++ /dev/null @@ -1,7 +0,0 @@ -jobs: -- template: /eng/common/core-templates/jobs/codeql-build.yml - parameters: - is1ESPipeline: true - - ${{ each parameter in parameters }}: - ${{ parameter.key }}: ${{ parameter.value }} diff --git a/src/arcade/eng/common/templates-official/jobs/jobs.yml b/src/arcade/eng/common/templates-official/jobs/jobs.yml deleted file mode 100644 index 007deddaea0f..000000000000 --- a/src/arcade/eng/common/templates-official/jobs/jobs.yml +++ /dev/null @@ -1,7 +0,0 @@ -jobs: -- template: /eng/common/core-templates/jobs/jobs.yml - parameters: - is1ESPipeline: true - - ${{ each parameter in parameters }}: - ${{ parameter.key }}: ${{ parameter.value }} diff --git a/src/arcade/eng/common/templates-official/jobs/source-build.yml b/src/arcade/eng/common/templates-official/jobs/source-build.yml deleted file mode 100644 index 483e7b611f34..000000000000 --- a/src/arcade/eng/common/templates-official/jobs/source-build.yml +++ /dev/null @@ -1,7 +0,0 @@ -jobs: -- template: /eng/common/core-templates/jobs/source-build.yml - parameters: - is1ESPipeline: true - - ${{ each parameter in parameters }}: - ${{ parameter.key }}: ${{ parameter.value }} \ No newline at end of file diff --git a/src/arcade/eng/common/templates-official/post-build/common-variables.yml b/src/arcade/eng/common/templates-official/post-build/common-variables.yml deleted file mode 100644 index c32fc49233f8..000000000000 --- a/src/arcade/eng/common/templates-official/post-build/common-variables.yml +++ /dev/null @@ -1,8 +0,0 @@ -variables: -- template: /eng/common/core-templates/post-build/common-variables.yml - parameters: - # Specifies whether to use 1ES - is1ESPipeline: true - - ${{ each parameter in parameters }}: - ${{ parameter.key }}: ${{ parameter.value }} \ No newline at end of file diff --git a/src/arcade/eng/common/templates-official/post-build/post-build.yml b/src/arcade/eng/common/templates-official/post-build/post-build.yml deleted file mode 100644 index 2364c0fd4a52..000000000000 --- a/src/arcade/eng/common/templates-official/post-build/post-build.yml +++ /dev/null @@ -1,8 +0,0 @@ -stages: -- template: /eng/common/core-templates/post-build/post-build.yml - parameters: - # Specifies whether to use 1ES - is1ESPipeline: true - - ${{ each parameter in parameters }}: - ${{ parameter.key }}: ${{ parameter.value }} diff --git a/src/arcade/eng/common/templates-official/post-build/setup-maestro-vars.yml b/src/arcade/eng/common/templates-official/post-build/setup-maestro-vars.yml deleted file mode 100644 index 024397d87864..000000000000 --- a/src/arcade/eng/common/templates-official/post-build/setup-maestro-vars.yml +++ /dev/null @@ -1,8 +0,0 @@ -steps: -- template: /eng/common/core-templates/post-build/setup-maestro-vars.yml - parameters: - # Specifies whether to use 1ES - is1ESPipeline: true - - ${{ each parameter in parameters }}: - ${{ parameter.key }}: ${{ parameter.value }} \ No newline at end of file diff --git a/src/arcade/eng/common/templates-official/steps/component-governance.yml b/src/arcade/eng/common/templates-official/steps/component-governance.yml deleted file mode 100644 index 30bb3985ca2b..000000000000 --- a/src/arcade/eng/common/templates-official/steps/component-governance.yml +++ /dev/null @@ -1,7 +0,0 @@ -steps: -- template: /eng/common/core-templates/steps/component-governance.yml - parameters: - is1ESPipeline: true - - ${{ each parameter in parameters }}: - ${{ parameter.key }}: ${{ parameter.value }} diff --git a/src/arcade/eng/common/templates-official/steps/enable-internal-runtimes.yml b/src/arcade/eng/common/templates-official/steps/enable-internal-runtimes.yml deleted file mode 100644 index f9dd238c6cd5..000000000000 --- a/src/arcade/eng/common/templates-official/steps/enable-internal-runtimes.yml +++ /dev/null @@ -1,9 +0,0 @@ -# Obtains internal runtime download credentials and populates the 'dotnetbuilds-internal-container-read-token-base64' -# variable with the base64-encoded SAS token, by default -steps: -- template: /eng/common/core-templates/steps/enable-internal-runtimes.yml - parameters: - is1ESPipeline: true - - ${{ each parameter in parameters }}: - ${{ parameter.key }}: ${{ parameter.value }} diff --git a/src/arcade/eng/common/templates-official/steps/enable-internal-sources.yml b/src/arcade/eng/common/templates-official/steps/enable-internal-sources.yml deleted file mode 100644 index e6d57182284d..000000000000 --- a/src/arcade/eng/common/templates-official/steps/enable-internal-sources.yml +++ /dev/null @@ -1,7 +0,0 @@ -steps: -- template: /eng/common/core-templates/steps/enable-internal-sources.yml - parameters: - is1ESPipeline: true - - ${{ each parameter in parameters }}: - ${{ parameter.key }}: ${{ parameter.value }} \ No newline at end of file diff --git a/src/arcade/eng/common/templates-official/steps/generate-sbom.yml b/src/arcade/eng/common/templates-official/steps/generate-sbom.yml deleted file mode 100644 index 9a89a4706d94..000000000000 --- a/src/arcade/eng/common/templates-official/steps/generate-sbom.yml +++ /dev/null @@ -1,7 +0,0 @@ -steps: -- template: /eng/common/core-templates/steps/generate-sbom.yml - parameters: - is1ESPipeline: true - - ${{ each parameter in parameters }}: - ${{ parameter.key }}: ${{ parameter.value }} diff --git a/src/arcade/eng/common/templates-official/steps/get-delegation-sas.yml b/src/arcade/eng/common/templates-official/steps/get-delegation-sas.yml deleted file mode 100644 index c5a9c1f8275c..000000000000 --- a/src/arcade/eng/common/templates-official/steps/get-delegation-sas.yml +++ /dev/null @@ -1,7 +0,0 @@ -steps: -- template: /eng/common/core-templates/steps/get-delegation-sas.yml - parameters: - is1ESPipeline: true - - ${{ each parameter in parameters }}: - ${{ parameter.key }}: ${{ parameter.value }} diff --git a/src/arcade/eng/common/templates-official/steps/get-federated-access-token.yml b/src/arcade/eng/common/templates-official/steps/get-federated-access-token.yml deleted file mode 100644 index c8dcf6b81392..000000000000 --- a/src/arcade/eng/common/templates-official/steps/get-federated-access-token.yml +++ /dev/null @@ -1,7 +0,0 @@ -steps: -- template: /eng/common/core-templates/steps/get-federated-access-token.yml - parameters: - is1ESPipeline: true - - ${{ each parameter in parameters }}: - ${{ parameter.key }}: ${{ parameter.value }} \ No newline at end of file diff --git a/src/arcade/eng/common/templates-official/steps/publish-logs.yml b/src/arcade/eng/common/templates-official/steps/publish-logs.yml deleted file mode 100644 index 579fd531e94c..000000000000 --- a/src/arcade/eng/common/templates-official/steps/publish-logs.yml +++ /dev/null @@ -1,7 +0,0 @@ -steps: -- template: /eng/common/core-templates/steps/publish-logs.yml - parameters: - is1ESPipeline: true - - ${{ each parameter in parameters }}: - ${{ parameter.key }}: ${{ parameter.value }} diff --git a/src/arcade/eng/common/templates-official/steps/publish-pipeline-artifacts.yml b/src/arcade/eng/common/templates-official/steps/publish-pipeline-artifacts.yml deleted file mode 100644 index 172f9f0fdc97..000000000000 --- a/src/arcade/eng/common/templates-official/steps/publish-pipeline-artifacts.yml +++ /dev/null @@ -1,28 +0,0 @@ -parameters: -- name: is1ESPipeline - type: boolean - default: true - -- name: args - type: object - default: {} - -steps: -- ${{ if ne(parameters.is1ESPipeline, true) }}: - - 'eng/common/templates-official cannot be referenced from a non-1ES managed template': error -- task: 1ES.PublishPipelineArtifact@1 - displayName: ${{ coalesce(parameters.args.displayName, 'Publish to Build Artifact') }} - ${{ if parameters.args.condition }}: - condition: ${{ parameters.args.condition }} - ${{ else }}: - condition: succeeded() - ${{ if parameters.args.continueOnError }}: - continueOnError: ${{ parameters.args.continueOnError }} - inputs: - targetPath: ${{ parameters.args.targetPath }} - ${{ if parameters.args.artifactName }}: - artifactName: ${{ parameters.args.artifactName }} - ${{ if parameters.args.properties }}: - properties: ${{ parameters.args.properties }} - ${{ if parameters.args.sbomEnabled }}: - sbomEnabled: ${{ parameters.args.sbomEnabled }} diff --git a/src/arcade/eng/common/templates-official/steps/retain-build.yml b/src/arcade/eng/common/templates-official/steps/retain-build.yml deleted file mode 100644 index 5594551508a3..000000000000 --- a/src/arcade/eng/common/templates-official/steps/retain-build.yml +++ /dev/null @@ -1,7 +0,0 @@ -steps: -- template: /eng/common/core-templates/steps/retain-build.yml - parameters: - is1ESPipeline: true - - ${{ each parameter in parameters }}: - ${{ parameter.key }}: ${{ parameter.value }} diff --git a/src/arcade/eng/common/templates-official/steps/send-to-helix.yml b/src/arcade/eng/common/templates-official/steps/send-to-helix.yml deleted file mode 100644 index 6500f21bf845..000000000000 --- a/src/arcade/eng/common/templates-official/steps/send-to-helix.yml +++ /dev/null @@ -1,7 +0,0 @@ -steps: -- template: /eng/common/core-templates/steps/send-to-helix.yml - parameters: - is1ESPipeline: true - - ${{ each parameter in parameters }}: - ${{ parameter.key }}: ${{ parameter.value }} diff --git a/src/arcade/eng/common/templates-official/steps/source-build.yml b/src/arcade/eng/common/templates-official/steps/source-build.yml deleted file mode 100644 index 8f92c49e7b06..000000000000 --- a/src/arcade/eng/common/templates-official/steps/source-build.yml +++ /dev/null @@ -1,7 +0,0 @@ -steps: -- template: /eng/common/core-templates/steps/source-build.yml - parameters: - is1ESPipeline: true - - ${{ each parameter in parameters }}: - ${{ parameter.key }}: ${{ parameter.value }} diff --git a/src/arcade/eng/common/templates-official/variables/pool-providers.yml b/src/arcade/eng/common/templates-official/variables/pool-providers.yml deleted file mode 100644 index 1f308b24efc4..000000000000 --- a/src/arcade/eng/common/templates-official/variables/pool-providers.yml +++ /dev/null @@ -1,45 +0,0 @@ -# Select a pool provider based off branch name. Anything with branch name containing 'release' must go into an -Svc pool, -# otherwise it should go into the "normal" pools. This separates out the queueing and billing of released branches. - -# Motivation: -# Once a given branch of a repository's output has been officially "shipped" once, it is then considered to be COGS -# (Cost of goods sold) and should be moved to a servicing pool provider. This allows both separation of queueing -# (allowing release builds and main PR builds to not intefere with each other) and billing (required for COGS. -# Additionally, the pool provider name itself may be subject to change when the .NET Core Engineering Services -# team needs to move resources around and create new and potentially differently-named pools. Using this template -# file from an Arcade-ified repo helps guard against both having to update one's release/* branches and renaming. - -# How to use: -# This yaml assumes your shipped product branches use the naming convention "release/..." (which many do). -# If we find alternate naming conventions in broad usage it can be added to the condition below. -# -# First, import the template in an arcade-ified repo to pick up the variables, e.g.: -# -# variables: -# - template: /eng/common/templates-official/variables/pool-providers.yml -# -# ... then anywhere specifying the pool provider use the runtime variables, -# $(DncEngInternalBuildPool) -# -# pool: -# name: $(DncEngInternalBuildPool) -# image: 1es-windows-2022 - -variables: - # Coalesce the target and source branches so we know when a PR targets a release branch - # If these variables are somehow missing, fall back to main (tends to have more capacity) - - # Any new -Svc alternative pools should have variables added here to allow for splitting work - - - name: DncEngInternalBuildPool - value: $[ - replace( - replace( - eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), - True, - 'NetCore1ESPool-Svc-Internal' - ), - False, - 'NetCore1ESPool-Internal' - ) - ] \ No newline at end of file diff --git a/src/arcade/eng/common/templates-official/variables/sdl-variables.yml b/src/arcade/eng/common/templates-official/variables/sdl-variables.yml deleted file mode 100644 index dbdd66d4a4b3..000000000000 --- a/src/arcade/eng/common/templates-official/variables/sdl-variables.yml +++ /dev/null @@ -1,7 +0,0 @@ -variables: -# The Guardian version specified in 'eng/common/sdl/packages.config'. This value must be kept in -# sync with the packages.config file. -- name: DefaultGuardianVersion - value: 0.109.0 -- name: GuardianPackagesConfigFile - value: $(Build.SourcesDirectory)\eng\common\sdl\packages.config \ No newline at end of file diff --git a/src/arcade/eng/common/templates/job/onelocbuild.yml b/src/arcade/eng/common/templates/job/onelocbuild.yml deleted file mode 100644 index ff829dc4c700..000000000000 --- a/src/arcade/eng/common/templates/job/onelocbuild.yml +++ /dev/null @@ -1,7 +0,0 @@ -jobs: -- template: /eng/common/core-templates/job/onelocbuild.yml - parameters: - is1ESPipeline: false - - ${{ each parameter in parameters }}: - ${{ parameter.key }}: ${{ parameter.value }} diff --git a/src/arcade/eng/common/templates/job/publish-build-assets.yml b/src/arcade/eng/common/templates/job/publish-build-assets.yml deleted file mode 100644 index ab2edec2adb5..000000000000 --- a/src/arcade/eng/common/templates/job/publish-build-assets.yml +++ /dev/null @@ -1,7 +0,0 @@ -jobs: -- template: /eng/common/core-templates/job/publish-build-assets.yml - parameters: - is1ESPipeline: false - - ${{ each parameter in parameters }}: - ${{ parameter.key }}: ${{ parameter.value }} diff --git a/src/arcade/eng/common/templates/job/source-build.yml b/src/arcade/eng/common/templates/job/source-build.yml deleted file mode 100644 index e44d47b1d760..000000000000 --- a/src/arcade/eng/common/templates/job/source-build.yml +++ /dev/null @@ -1,7 +0,0 @@ -jobs: -- template: /eng/common/core-templates/job/source-build.yml - parameters: - is1ESPipeline: false - - ${{ each parameter in parameters }}: - ${{ parameter.key }}: ${{ parameter.value }} diff --git a/src/arcade/eng/common/templates/job/source-index-stage1.yml b/src/arcade/eng/common/templates/job/source-index-stage1.yml deleted file mode 100644 index 89f3291593cb..000000000000 --- a/src/arcade/eng/common/templates/job/source-index-stage1.yml +++ /dev/null @@ -1,7 +0,0 @@ -jobs: -- template: /eng/common/core-templates/job/source-index-stage1.yml - parameters: - is1ESPipeline: false - - ${{ each parameter in parameters }}: - ${{ parameter.key }}: ${{ parameter.value }} diff --git a/src/arcade/eng/common/templates/jobs/codeql-build.yml b/src/arcade/eng/common/templates/jobs/codeql-build.yml deleted file mode 100644 index 517f24d6a52c..000000000000 --- a/src/arcade/eng/common/templates/jobs/codeql-build.yml +++ /dev/null @@ -1,7 +0,0 @@ -jobs: -- template: /eng/common/core-templates/jobs/codeql-build.yml - parameters: - is1ESPipeline: false - - ${{ each parameter in parameters }}: - ${{ parameter.key }}: ${{ parameter.value }} diff --git a/src/arcade/eng/common/templates/jobs/jobs.yml b/src/arcade/eng/common/templates/jobs/jobs.yml deleted file mode 100644 index 388e9037b3e6..000000000000 --- a/src/arcade/eng/common/templates/jobs/jobs.yml +++ /dev/null @@ -1,7 +0,0 @@ -jobs: -- template: /eng/common/core-templates/jobs/jobs.yml - parameters: - is1ESPipeline: false - - ${{ each parameter in parameters }}: - ${{ parameter.key }}: ${{ parameter.value }} diff --git a/src/arcade/eng/common/templates/jobs/source-build.yml b/src/arcade/eng/common/templates/jobs/source-build.yml deleted file mode 100644 index 818d4c326dbb..000000000000 --- a/src/arcade/eng/common/templates/jobs/source-build.yml +++ /dev/null @@ -1,7 +0,0 @@ -jobs: -- template: /eng/common/core-templates/jobs/source-build.yml - parameters: - is1ESPipeline: false - - ${{ each parameter in parameters }}: - ${{ parameter.key }}: ${{ parameter.value }} \ No newline at end of file diff --git a/src/arcade/eng/common/templates/post-build/common-variables.yml b/src/arcade/eng/common/templates/post-build/common-variables.yml deleted file mode 100644 index 7fa105875592..000000000000 --- a/src/arcade/eng/common/templates/post-build/common-variables.yml +++ /dev/null @@ -1,8 +0,0 @@ -variables: -- template: /eng/common/core-templates/post-build/common-variables.yml - parameters: - # Specifies whether to use 1ES - is1ESPipeline: false - - ${{ each parameter in parameters }}: - ${{ parameter.key }}: ${{ parameter.value }} \ No newline at end of file diff --git a/src/arcade/eng/common/templates/post-build/post-build.yml b/src/arcade/eng/common/templates/post-build/post-build.yml deleted file mode 100644 index 53ede714bdd2..000000000000 --- a/src/arcade/eng/common/templates/post-build/post-build.yml +++ /dev/null @@ -1,8 +0,0 @@ -stages: -- template: /eng/common/core-templates/post-build/post-build.yml - parameters: - # Specifies whether to use 1ES - is1ESPipeline: false - - ${{ each parameter in parameters }}: - ${{ parameter.key }}: ${{ parameter.value }} \ No newline at end of file diff --git a/src/arcade/eng/common/templates/post-build/setup-maestro-vars.yml b/src/arcade/eng/common/templates/post-build/setup-maestro-vars.yml deleted file mode 100644 index a79fab5b441e..000000000000 --- a/src/arcade/eng/common/templates/post-build/setup-maestro-vars.yml +++ /dev/null @@ -1,8 +0,0 @@ -steps: -- template: /eng/common/core-templates/post-build/setup-maestro-vars.yml - parameters: - # Specifies whether to use 1ES - is1ESPipeline: false - - ${{ each parameter in parameters }}: - ${{ parameter.key }}: ${{ parameter.value }} \ No newline at end of file diff --git a/src/arcade/eng/common/templates/steps/component-governance.yml b/src/arcade/eng/common/templates/steps/component-governance.yml deleted file mode 100644 index c12a5f8d21d7..000000000000 --- a/src/arcade/eng/common/templates/steps/component-governance.yml +++ /dev/null @@ -1,7 +0,0 @@ -steps: -- template: /eng/common/core-templates/steps/component-governance.yml - parameters: - is1ESPipeline: false - - ${{ each parameter in parameters }}: - ${{ parameter.key }}: ${{ parameter.value }} diff --git a/src/arcade/eng/common/templates/steps/enable-internal-runtimes.yml b/src/arcade/eng/common/templates/steps/enable-internal-runtimes.yml deleted file mode 100644 index b21a8038cc1c..000000000000 --- a/src/arcade/eng/common/templates/steps/enable-internal-runtimes.yml +++ /dev/null @@ -1,10 +0,0 @@ -# Obtains internal runtime download credentials and populates the 'dotnetbuilds-internal-container-read-token-base64' -# variable with the base64-encoded SAS token, by default - -steps: -- template: /eng/common/core-templates/steps/enable-internal-runtimes.yml - parameters: - is1ESPipeline: false - - ${{ each parameter in parameters }}: - ${{ parameter.key }}: ${{ parameter.value }} diff --git a/src/arcade/eng/common/templates/steps/enable-internal-sources.yml b/src/arcade/eng/common/templates/steps/enable-internal-sources.yml deleted file mode 100644 index 5f87e9abb8aa..000000000000 --- a/src/arcade/eng/common/templates/steps/enable-internal-sources.yml +++ /dev/null @@ -1,7 +0,0 @@ -steps: -- template: /eng/common/core-templates/steps/enable-internal-sources.yml - parameters: - is1ESPipeline: false - - ${{ each parameter in parameters }}: - ${{ parameter.key }}: ${{ parameter.value }} \ No newline at end of file diff --git a/src/arcade/eng/common/templates/steps/generate-sbom.yml b/src/arcade/eng/common/templates/steps/generate-sbom.yml deleted file mode 100644 index 26dc00a2e0f3..000000000000 --- a/src/arcade/eng/common/templates/steps/generate-sbom.yml +++ /dev/null @@ -1,7 +0,0 @@ -steps: -- template: /eng/common/core-templates/steps/generate-sbom.yml - parameters: - is1ESPipeline: false - - ${{ each parameter in parameters }}: - ${{ parameter.key }}: ${{ parameter.value }} diff --git a/src/arcade/eng/common/templates/steps/get-delegation-sas.yml b/src/arcade/eng/common/templates/steps/get-delegation-sas.yml deleted file mode 100644 index 83760c9798e3..000000000000 --- a/src/arcade/eng/common/templates/steps/get-delegation-sas.yml +++ /dev/null @@ -1,7 +0,0 @@ -steps: -- template: /eng/common/core-templates/steps/get-delegation-sas.yml - parameters: - is1ESPipeline: false - - ${{ each parameter in parameters }}: - ${{ parameter.key }}: ${{ parameter.value }} diff --git a/src/arcade/eng/common/templates/steps/get-federated-access-token.yml b/src/arcade/eng/common/templates/steps/get-federated-access-token.yml deleted file mode 100644 index 31e151d9d9e7..000000000000 --- a/src/arcade/eng/common/templates/steps/get-federated-access-token.yml +++ /dev/null @@ -1,7 +0,0 @@ -steps: -- template: /eng/common/core-templates/steps/get-federated-access-token.yml - parameters: - is1ESPipeline: false - - ${{ each parameter in parameters }}: - ${{ parameter.key }}: ${{ parameter.value }} \ No newline at end of file diff --git a/src/arcade/eng/common/templates/steps/publish-logs.yml b/src/arcade/eng/common/templates/steps/publish-logs.yml deleted file mode 100644 index 4ea86bd88235..000000000000 --- a/src/arcade/eng/common/templates/steps/publish-logs.yml +++ /dev/null @@ -1,7 +0,0 @@ -steps: -- template: /eng/common/core-templates/steps/publish-logs.yml - parameters: - is1ESPipeline: false - - ${{ each parameter in parameters }}: - ${{ parameter.key }}: ${{ parameter.value }} diff --git a/src/arcade/eng/common/templates/steps/publish-pipeline-artifacts.yml b/src/arcade/eng/common/templates/steps/publish-pipeline-artifacts.yml deleted file mode 100644 index 5dd698b212fc..000000000000 --- a/src/arcade/eng/common/templates/steps/publish-pipeline-artifacts.yml +++ /dev/null @@ -1,34 +0,0 @@ -parameters: -- name: is1ESPipeline - type: boolean - default: false - -- name: args - type: object - default: {} - -steps: -- ${{ if eq(parameters.is1ESPipeline, true) }}: - - 'eng/common/templates cannot be referenced from a 1ES managed template': error -- task: PublishPipelineArtifact@1 - displayName: ${{ coalesce(parameters.args.displayName, 'Publish to Build Artifact') }} - ${{ if parameters.args.condition }}: - condition: ${{ parameters.args.condition }} - ${{ else }}: - condition: succeeded() - ${{ if parameters.args.continueOnError }}: - continueOnError: ${{ parameters.args.continueOnError }} - inputs: - targetPath: ${{ parameters.args.targetPath }} - ${{ if parameters.args.artifactName }}: - artifactName: ${{ parameters.args.artifactName }} - ${{ if parameters.args.publishLocation }}: - publishLocation: ${{ parameters.args.publishLocation }} - ${{ if parameters.args.fileSharePath }}: - fileSharePath: ${{ parameters.args.fileSharePath }} - ${{ if parameters.args.Parallel }}: - parallel: ${{ parameters.args.Parallel }} - ${{ if parameters.args.parallelCount }}: - parallelCount: ${{ parameters.args.parallelCount }} - ${{ if parameters.args.properties }}: - properties: ${{ parameters.args.properties }} \ No newline at end of file diff --git a/src/arcade/eng/common/templates/steps/retain-build.yml b/src/arcade/eng/common/templates/steps/retain-build.yml deleted file mode 100644 index 8e841ace3d29..000000000000 --- a/src/arcade/eng/common/templates/steps/retain-build.yml +++ /dev/null @@ -1,7 +0,0 @@ -steps: -- template: /eng/common/core-templates/steps/retain-build.yml - parameters: - is1ESPipeline: false - - ${{ each parameter in parameters }}: - ${{ parameter.key }}: ${{ parameter.value }} diff --git a/src/arcade/eng/common/templates/steps/send-to-helix.yml b/src/arcade/eng/common/templates/steps/send-to-helix.yml deleted file mode 100644 index 39f99fc2762d..000000000000 --- a/src/arcade/eng/common/templates/steps/send-to-helix.yml +++ /dev/null @@ -1,7 +0,0 @@ -steps: -- template: /eng/common/core-templates/steps/send-to-helix.yml - parameters: - is1ESPipeline: false - - ${{ each parameter in parameters }}: - ${{ parameter.key }}: ${{ parameter.value }} diff --git a/src/arcade/eng/common/templates/steps/source-build.yml b/src/arcade/eng/common/templates/steps/source-build.yml deleted file mode 100644 index 23c1d6f4e9f8..000000000000 --- a/src/arcade/eng/common/templates/steps/source-build.yml +++ /dev/null @@ -1,7 +0,0 @@ -steps: -- template: /eng/common/core-templates/steps/source-build.yml - parameters: - is1ESPipeline: false - - ${{ each parameter in parameters }}: - ${{ parameter.key }}: ${{ parameter.value }} diff --git a/src/arcade/eng/common/templates/variables/pool-providers.yml b/src/arcade/eng/common/templates/variables/pool-providers.yml deleted file mode 100644 index e0b19c14a073..000000000000 --- a/src/arcade/eng/common/templates/variables/pool-providers.yml +++ /dev/null @@ -1,59 +0,0 @@ -# Select a pool provider based off branch name. Anything with branch name containing 'release' must go into an -Svc pool, -# otherwise it should go into the "normal" pools. This separates out the queueing and billing of released branches. - -# Motivation: -# Once a given branch of a repository's output has been officially "shipped" once, it is then considered to be COGS -# (Cost of goods sold) and should be moved to a servicing pool provider. This allows both separation of queueing -# (allowing release builds and main PR builds to not intefere with each other) and billing (required for COGS. -# Additionally, the pool provider name itself may be subject to change when the .NET Core Engineering Services -# team needs to move resources around and create new and potentially differently-named pools. Using this template -# file from an Arcade-ified repo helps guard against both having to update one's release/* branches and renaming. - -# How to use: -# This yaml assumes your shipped product branches use the naming convention "release/..." (which many do). -# If we find alternate naming conventions in broad usage it can be added to the condition below. -# -# First, import the template in an arcade-ified repo to pick up the variables, e.g.: -# -# variables: -# - template: /eng/common/templates/variables/pool-providers.yml -# -# ... then anywhere specifying the pool provider use the runtime variables, -# $(DncEngInternalBuildPool) and $ (DncEngPublicBuildPool), e.g.: -# -# pool: -# name: $(DncEngInternalBuildPool) -# demands: ImageOverride -equals windows.vs2019.amd64 -variables: - - ${{ if eq(variables['System.TeamProject'], 'internal') }}: - - template: /eng/common/templates-official/variables/pool-providers.yml - - ${{ else }}: - # Coalesce the target and source branches so we know when a PR targets a release branch - # If these variables are somehow missing, fall back to main (tends to have more capacity) - - # Any new -Svc alternative pools should have variables added here to allow for splitting work - - name: DncEngPublicBuildPool - value: $[ - replace( - replace( - eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), - True, - 'NetCore-Svc-Public' - ), - False, - 'NetCore-Public' - ) - ] - - - name: DncEngInternalBuildPool - value: $[ - replace( - replace( - eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), - True, - 'NetCore1ESPool-Svc-Internal' - ), - False, - 'NetCore1ESPool-Internal' - ) - ]