Use Arcade's publishing logic to publish a given VMR vertical (#47076)

Co-authored-by: root <root@jekoritz-devbox.>
Co-authored-by: Viktor Hofer <[email protected]>
Co-authored-by: Matt Mitchell (.NET) <[email protected]>

Author: 4 people

eng/pipelines/templates/jobs/vmr-build.yml

@@ -192,6 +192,9 @@ jobs:
   - name: artifactsStagingDir
     value: $(Build.ArtifactStagingDirectory)/artifacts
 
+  - name: artifactsPrepublishDir
+    value: $(Build.ArtifactStagingDirectory)/prepublish
+
   - name: successfulJobArtifactName
     value: $(Agent.JobName)_Artifacts
 
@@ -310,7 +313,7 @@ jobs:
       sbomEnabled: true
 
     - output: pipelineArtifact
-      path: $(Build.ArtifactStagingDirectory)
+      path: $(artifactsPrepublishDir)
       artifact: $(failedJobArtifactName)
       displayName: Publish Artifacts (On Failure)
       condition: failed()
@@ -639,6 +642,19 @@ jobs:
     continueOnError: true
     condition: succeededOrFailed()
 
+  - task: CopyFiles@2
+    displayName: Copy unmerged artifacts to staging directory
+    inputs:
+      SourceFolder: '$(sourcesPath)/artifacts'
+      Contents: |
+        packages/**/*
+        assets/**/*
+        obj/manifests/**/*
+      TargetFolder: '$(artifactsPrepublishDir)'
+      CleanTargetFolder: true
+    condition: failed()
+    continueOnError: true
+
   - ${{ if eq(parameters.buildSourceOnly, 'True') }}:
     - task: CopyFiles@2
       displayName: Copy prebuilt-report to BuildLogs
@@ -692,7 +708,7 @@ jobs:
       condition: succeeded()
       continueOnError: true
 
-    - publish: $(Build.ArtifactStagingDirectory)
+    - publish: $(artifactsPrepublishDir)
       artifact: $(failedJobArtifactName)
       displayName: Publish Artifacts (On Failure)
       condition: failed()

eng/pipelines/templates/stages/vmr-build-with-join.yml

@@ -113,6 +113,11 @@ stages:
             ArtifactName: BlobArtifacts
             displayName: Publish Blob Artifacts
             sbomEnabled: false
+          - output: buildArtifacts
+            PathtoPublish: $(Build.ArtifactStagingDirectory)/artifacts/PDB
+            ArtifactName: PdbArtifacts
+            displayName: Publish PDB Artifacts
+            sbomEnabled: false
           - output: buildArtifacts
             PathtoPublish: $(Build.ArtifactStagingDirectory)/artifacts/packages
             ArtifactName: PackageArtifacts

src/SourceBuild/content/Directory.Build.props

@@ -7,74 +7,7 @@
     <Nullable>enable</Nullable>
   </PropertyGroup>
 
-  <PropertyGroup Label="CalculateTargetOS">
-    <BuildOS>linux</BuildOS>
-    <BuildOS Condition="$([MSBuild]::IsOSPlatform('OSX'))">osx</BuildOS>
-    <BuildOS Condition="$([MSBuild]::IsOSPlatform('FREEBSD'))">freebsd</BuildOS>
-    <BuildOS Condition="$([MSBuild]::IsOSPlatform('NETBSD'))">netbsd</BuildOS>
-    <BuildOS Condition="$([MSBuild]::IsOSPlatform('ILLUMOS'))">illumos</BuildOS>
-    <BuildOS Condition="$([MSBuild]::IsOSPlatform('SOLARIS'))">solaris</BuildOS>
-    <BuildOS Condition="$([MSBuild]::IsOSPlatform('HAIKU'))">haiku</BuildOS>
-    <BuildOS Condition="$([MSBuild]::IsOSPlatform('WINDOWS'))">windows</BuildOS>
-    <TargetOS Condition="'$(TargetOS)' == ''">$(BuildOS)</TargetOS>
-    <HostOS Condition="'$(HostOS)' == ''">$(TargetOS)</HostOS>
-  </PropertyGroup>
-
-  <!-- This is a list of cases where we aren't producing a whole SDK, just a runtime. This list should be kept in sync with https://github.com/dotnet/runtime/blob/main/eng/DotNetBuild.props#L26 -->
-  <PropertyGroup Label="ShortStacks">
-    <ShortStack Condition="'$(TargetOS)' == 'wasi'">true</ShortStack>
-    <ShortStack Condition="'$(TargetOS)' == 'browser'">true</ShortStack>
-    <ShortStack Condition="'$(TargetOS)' == 'ios'">true</ShortStack>
-    <ShortStack Condition="'$(TargetOS)' == 'iossimulator'">true</ShortStack>
-    <ShortStack Condition="'$(TargetOS)' == 'tvos'">true</ShortStack>
-    <ShortStack Condition="'$(TargetOS)' == 'tvossimulator'">true</ShortStack>
-    <ShortStack Condition="'$(TargetOS)' == 'maccatalyst'">true</ShortStack>
-    <ShortStack Condition="'$(TargetOS)' == 'android'">true</ShortStack>
-    <ShortStack Condition="'$(TargetOS)' == 'linux-bionic'">true</ShortStack>
-    <!-- Mono LLVM builds are short -->
-    <ShortStack Condition="'$(DotNetBuildMonoEnableLLVM)' == 'true' or '$(DotNetBuildMonoAOTEnableLLVM)' == 'true'">true</ShortStack>
-    <!-- Short stack builds stop at runtime, not the whole SDK -->
-    <RootRepo Condition="'$(ShortStack)' == 'true'">runtime</RootRepo>
-  </PropertyGroup>
-
-  <!-- See https://github.com/dotnet/arcade/blob/main/Documentation/UnifiedBuild/Unified-Build-Controls.md#output-controls for
-       control set definition. -->
-  <PropertyGroup Label="CalculateArch">
-    <!-- Build architecture is what we are building on. -->
-    <BuildArchitecture>$([System.Runtime.InteropServices.RuntimeInformation]::ProcessArchitecture.ToString().ToLowerInvariant)</BuildArchitecture>
-    <!-- The target architecture is the what the customer is targeting their outputs to run on. -->
-    <TargetArchitecture Condition="'$(TargetArchitecture)' == ''">$(BuildArchitecture)</TargetArchitecture>
-    <!-- The host architecture is the what the customer will build on. Much of the time, Host==Target. -->
-    <HostArchitecture Condition="'$(HostArchitecture)' == ''">$(TargetArchitecture)</HostArchitecture>
-
-    <Platform Condition="'$(Platform)' == ''">AnyCPU</Platform>
-  </PropertyGroup>
-
-  <PropertyGroup Label="CalculateRID">
-    <!-- Use current machine distro RID if set. Otherwise, fall back to RuntimeInformation.RuntimeIdentifier -->
-    <BuildRid>$(__DistroRid)</BuildRid>
-    <BuildRid Condition="'$(BuildRid)' == '' and '$(MSBuildRuntimeType)' == 'core'">$([System.Runtime.InteropServices.RuntimeInformation]::RuntimeIdentifier)</BuildRid>
-    <BuildRid Condition="'$(BuildRid)' == '' and '$(MSBuildRuntimeType)' != 'core'">win-$([System.Runtime.InteropServices.RuntimeInformation]::OSArchitecture.ToString().ToLowerInvariant)</BuildRid>
-
-    <TargetRid Condition="'$(TargetRid)' == ''">$(BuildRid.Substring(0, $(BuildRid.LastIndexOf('-'))))-$(TargetArchitecture)</TargetRid>
-    <HostRid Condition="'$(HostRid)' == ''">$(TargetRid)</HostRid>
-
-    <!-- Source-only builds are non portable, except for cross-builds.
-         Source-only cross-builds default to the portable configuration so the resulting SDK works on a wider range of distros. -->
-    <PortableBuild Condition="'$(PortableBuild)' == '' and '$(DotNetBuildSourceOnly)' == 'true' and '$(BuildArchitecture)' == '$(TargetArchitecture)'">false</PortableBuild>
-    <PortableBuild Condition="'$(PortableBuild)' == ''">true</PortableBuild>
-
-    <PortableRid Condition="'$(__PortableTargetOS)' != ''">$(__PortableTargetOS)-$(TargetArchitecture)</PortableRid>
-    <PortableRid Condition="'$(PortableRid)' == '' and '$(TargetOS)' == 'freebsd'">freebsd-$(TargetArchitecture)</PortableRid>
-    <PortableRid Condition="'$(PortableRid)' == '' and '$(TargetOS)' == 'osx'">osx-$(TargetArchitecture)</PortableRid>
-    <PortableRid Condition="'$(PortableRid)' == '' and '$(TargetOS)' == 'linux'">linux-$(TargetArchitecture)</PortableRid>
-    <PortableRid Condition="$(TargetRid.StartsWith('linux-musl')) or $(TargetRid.StartsWith('alpine'))">linux-musl-$(TargetArchitecture)</PortableRid>
-    <PortableRid Condition="'$(PortableRid)' == '' and '$(TargetOS)' == 'windows'">win-$(TargetArchitecture)</PortableRid>
-
-    <TargetRid Condition="'$(PortableBuild)' == 'true' and '$(PortableRid)' != ''">$(PortableRid)</TargetRid>
-    <TargetRid Condition="'$(ShortStack)' == 'true' and '$(TargetOS)' != 'windows'">$(TargetOS)-$(TargetArchitecture)</TargetRid>
-    <TargetRid Condition="'$(ShortStack)' == 'true' and '$(TargetOS)' == 'windows'">win-$(TargetArchitecture)</TargetRid>
-  </PropertyGroup>
+  <Import Project="$(MSBuildThisFileDirectory)eng/RuntimeIdentifier.props" />
 
   <Import Project="Sdk.props" Sdk="Microsoft.DotNet.Arcade.Sdk" Condition="'$(SkipArcadeSdkImport)' != 'true'" />
 
@@ -138,74 +71,7 @@
 
     <ShellExtension Condition="'$(BuildOS)' == 'windows'">.cmd</ShellExtension>
     <ShellExtension Condition="'$(BuildOS)' != 'windows'">.sh</ShellExtension>
-
-    <!-- While the tarball infrastructure currently only supports non Windows, these properties
-         are already defined for more flexibility in the future. -->
-    <ArchiveExtension Condition="'$(BuildOS)' == 'windows'">.zip</ArchiveExtension>
-    <ArchiveExtension Condition="'$(BuildOS)' != 'windows'">.tar.gz</ArchiveExtension>
-  </PropertyGroup>
-
-  <PropertyGroup>
-    <SrcDir>$([MSBuild]::NormalizeDirectory('$(RepoRoot)', 'src'))</SrcDir>
-    <ToolsDir>$([MSBuild]::NormalizeDirectory('$(RepositoryEngineeringDir)', 'tools'))</ToolsDir>
-    <TasksDir>$([MSBuild]::NormalizeDirectory('$(ToolsDir)', 'tasks'))</TasksDir>
-    <RepoProjectsDir>$([MSBuild]::NormalizeDirectory('$(RepoRoot)', 'repo-projects'))</RepoProjectsDir>
-
-    <PrereqsDir>$([MSBuild]::NormalizeDirectory('$(RepoRoot)', 'prereqs'))</PrereqsDir>
-    <KeysDir>$([MSBuild]::NormalizeDirectory('$(PrereqsDir)', 'keys'))</KeysDir>
-    <PrereqsPackagesDir>$([MSBuild]::NormalizeDirectory('$(PrereqsDir)', 'packages'))</PrereqsPackagesDir>
-
-    <!-- When SB mode, the initial toolsets are unzipped to the "BootstrapPackages" dir.
-         When running in non-source-build, the bootstrap package is the one restored at the beginning of the build. -->
-    <BootstrapPackagesDir Condition="'$(DotNetBuildSourceOnly)' == 'true'">$([MSBuild]::NormalizeDirectory('$(NuGetPackageRoot)', 'BootstrapPackages'))</BootstrapPackagesDir>
-    <BootstrapPackagesDir Condition="'$(DotNetBuildSourceOnly)' != 'true'">$(NuGetPackageRoot)</BootstrapPackagesDir>
-
-    <VSMSBuildSdkResolversDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsDir)', 'toolset', 'VSSdkResolvers'))</VSMSBuildSdkResolversDir>
-    <IntermediateSymbolsRootDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsObjDir)', 'Symbols'))</IntermediateSymbolsRootDir>
-
-    <ArtifactsAssetManifestsDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsDir)', 'manifests', '$(Configuration)'))</ArtifactsAssetManifestsDir>
-    <AssetManifestsIntermediateDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsObjDir)', 'manifests', '$(Configuration)'))</AssetManifestsIntermediateDir>
-    <ArtifactsAssetsDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsDir)', 'assets', '$(Configuration)'))</ArtifactsAssetsDir>
-
-    <DotNetSdkExtractDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsObjDir)', 'extracted-dotnet-sdk'))</DotNetSdkExtractDir>
-
-    <PrebuiltPackagesPath>$([MSBuild]::NormalizeDirectory('$(PrereqsPackagesDir)', 'prebuilt'))</PrebuiltPackagesPath>
-    <PreviouslyRestoredPackagesPath>$([MSBuild]::NormalizeDirectory('$(PrereqsPackagesDir)', 'previouslyRestored'))</PreviouslyRestoredPackagesPath>
-    <PreviouslySourceBuiltPackagesPath>$([MSBuild]::NormalizeDirectory('$(PrereqsPackagesDir)', 'previously-source-built'))</PreviouslySourceBuiltPackagesPath>
-    <PreviouslySourceBuiltPackagesPath Condition="'$(CustomPreviouslySourceBuiltPackagesPath)' != ''">$([MSBuild]::EnsureTrailingSlash('$(CustomPreviouslySourceBuiltPackagesPath)'))</PreviouslySourceBuiltPackagesPath>
-
-    <PackageReportDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsDir)', 'prebuilt-report'))</PackageReportDir>
-    <ResultingPrebuiltPackagesDir>$([MSBuild]::NormalizeDirectory('$(PackageReportDir)', 'prebuilt-packages'))</ResultingPrebuiltPackagesDir>
-    <SbrpRepoSrcDir>$([MSBuild]::NormalizeDirectory('$(SrcDir)', 'source-build-reference-packages', 'src'))</SbrpRepoSrcDir>
-    <ReferencePackagesDir>$([MSBuild]::NormalizeDirectory('$(PrereqsPackagesDir)', 'reference'))</ReferencePackagesDir>
-    <PreviouslySourceBuiltReferencePackagesDirName>SourceBuildReferencePackages</PreviouslySourceBuiltReferencePackagesDirName>
-    <PreviouslySourceBuiltReferencePackagesDir>$([MSBuild]::NormalizeDirectory('$(PreviouslySourceBuiltPackagesPath)', '$(PreviouslySourceBuiltReferencePackagesDirName)'))</PreviouslySourceBuiltReferencePackagesDir>
-    <SourceBuiltArtifactsTarballName>Private.SourceBuilt.Artifacts</SourceBuiltArtifactsTarballName>
-    <SourceBuiltPrebuiltsTarballName>Private.SourceBuilt.Prebuilts</SourceBuiltPrebuiltsTarballName>
-
-    <BaselineDataFile>$(ToolsDir)prebuilt-baseline.xml</BaselineDataFile>
-  </PropertyGroup>
-
-  <!-- Build task assembly paths -->
-  <PropertyGroup>
-    <MicrosoftDotNetUnifiedBuildTasksAssembly>$([MSBuild]::NormalizePath('$(ArtifactsBinDir)', 'Microsoft.DotNet.UnifiedBuild.Tasks', '$(Configuration)', 'Microsoft.DotNet.UnifiedBuild.Tasks.dll'))</MicrosoftDotNetUnifiedBuildTasksAssembly>
-    <MicrosoftDotNetSourceBuildTasksLeakDetectionAssembly>$([MSBuild]::NormalizePath('$(ArtifactsBinDir)', 'Microsoft.DotNet.SourceBuild.Tasks.LeakDetection', '$(Configuration)', 'Microsoft.DotNet.SourceBuild.Tasks.LeakDetection.dll'))</MicrosoftDotNetSourceBuildTasksLeakDetectionAssembly>
-  </PropertyGroup>
-
-  <PropertyGroup Condition="'$(EnablePoison)' == 'true'">
-    <PoisonMarkerFile>.prebuilt.xml</PoisonMarkerFile>
-    <PoisonReportDataFile>$(PackageReportDir)poison-catalog.xml</PoisonReportDataFile>
-    <PoisonedReportFile>$(PackageReportDir)poisoned.txt</PoisonedReportFile>
-    <PoisonUsageReportFile>$(PackageReportDir)poison-usage.xml</PoisonUsageReportFile>
-  </PropertyGroup>
-
-  <PropertyGroup>
-    <!-- Respect the VerticalName property which is set to a unique identifier in CI.
-         Don't use that when building source-only to allow the build to find the previously built manifest file regardless of the vertical name.
-         Default to VerticalManifest.xml. -->
-    <MergedAssetManifestName Condition="'$(VerticalName)' != ''">$(VerticalName).xml</MergedAssetManifestName>
-    <MergedAssetManifestName Condition="'$(DotNetBuildSourceOnly)' == 'true' or '$(MergedAssetManifestName)' == ''">VerticalManifest.xml</MergedAssetManifestName>
-    <MergedAssetManifestOutputPath>$(ArtifactsAssetManifestsDir)$(MergedAssetManifestName)</MergedAssetManifestOutputPath>
   </PropertyGroup>
 
+  <Import Project="$(RepositoryEngineeringDir)VmrLayout.props" />
 </Project>

src/SourceBuild/content/Directory.Build.targets

@@ -61,35 +61,4 @@
       <SourceBuiltRuntimeVersion>$(RuntimeFilename.Replace('$(RuntimeFilenamePrefix)','').Replace('-$(TargetRid)$(ArchiveExtension)',''))</SourceBuiltRuntimeVersion>
     </PropertyGroup>
   </Target>
-
-  <ItemGroup>
-    <!-- Binplace all assets and packages to ArtifactsStagingDir. -->
-    <BinPlaceDir Include="$(ArtifactsStagingDir)" Condition="'$(ArtifactsStagingDir)' != ''" />
-  </ItemGroup>
-
-  <Target Name="BinPlaceFiles"
-          AfterTargets="Build"
-          Condition="'@(BinPlaceDir)' != '' and '@(BinPlaceFile)' != ''"
-          Inputs="@(BinPlaceDir);%(BinPlaceDir.Identity)"
-          Outputs="unused">
-    <PropertyGroup>
-      <_BinPlaceDir>%(BinPlaceDir.Identity)</_BinPlaceDir>
-    </PropertyGroup>
-
-    <ItemGroup>
-      <_BinPlaceFileWithFullTargetPath Include="@(BinPlaceFile)">
-        <TargetPath Condition="'%(BinPlaceFile.TargetPath)' != ''">%(BinPlaceFile.TargetPath)</TargetPath>
-        <TargetPath Condition="'%(BinPlaceFile.TargetPath)' == ''">$([MSBuild]::MakeRelative('$(ArtifactsDir)', '%(BinPlaceFile.Identity)'))</TargetPath>
-      </_BinPlaceFileWithFullTargetPath>
-    </ItemGroup>
-
-    <Copy SourceFiles="@(_BinPlaceFileWithFullTargetPath)"
-          DestinationFiles="@(_BinPlaceFileWithFullTargetPath -> '$([MSBuild]::EnsureTrailingSlash('$(_BinPlaceDir)'))%(TargetPath)')"
-          SkipUnchangedFiles="true"
-          OverwriteReadOnlyFiles="true"
-          Retries="$(CopyRetryCount)"
-          RetryDelayMilliseconds="$(CopyRetryDelayMilliseconds)"
-          UseHardlinksIfPossible="true" />
-  </Target>
-
 </Project>

src/SourceBuild/content/build.proj

@@ -21,8 +21,7 @@
 
     <ProjectReference Include="$(RepoProjectsDir)$(RootRepo).proj" />
 
-    <!-- Post-build: Source-only validation, packaging and publishing -->
-    <ProjectReference Include="$(RepositoryEngineeringDir)merge-asset-manifests.proj" />
+    <!-- Post-build: Source-only validation -->
     <ProjectReference Include="$(RepositoryEngineeringDir)finish-source-only.proj" Condition="'$(DotNetBuildSourceOnly)' == 'true'" />
   </ItemGroup>
 

src/SourceBuild/content/build.sh

@@ -135,7 +135,7 @@ while [[ $# > 0 ]]; do
       test=true
       ;;
     -sign)
-      properties+=( "/p:Sign=true" )
+      properties+=( "/p:DotNetBuildSign=true" )
       ;;
 
     # Source-only settings
@@ -222,13 +222,12 @@ fi
 . "$scriptroot/eng/common/tools.sh"
 
 project="$scriptroot/build.proj"
-targets="/t:Build"
+actions=( "/p:Restore=true" "/p:Build=true" "/p:Publish=true")
 
-# This repo uses the VSTest integration instead of the Arcade Test target
 if [[ "$test" == true ]]; then
   project="$scriptroot/test/tests.proj"
-  targets="$targets;VSTest"
-  properties+=( "/p:Test=true" )
+  actions=( "/p:Restore=true" "/p:Build=true" "/p:Test=true" )
+  properties+=( "/p:IsTestRun=true" )
 
   # Workaround for vstest hangs (https://github.com/microsoft/vstest/issues/5091) [TODO]
   export MSBUILDENSURESTDOUTFORTASKPROCESSES=1
@@ -252,10 +251,13 @@ function Build {
     fi
 
     MSBuild --restore \
-      $project \
-      $targets \
+      $_InitializeToolset \
+      "/p:Projects=$project" \
       $bl \
       /p:Configuration=$configuration \
+      "/p:RepoRoot=$scriptroot/" \
+      "-tl:off" \
+      "${actions[@]}" \
       "${properties[@]}"
 
     ExitWithExitCode 0
@@ -275,6 +277,9 @@ function Build {
     # kill off the MSBuild server so that on future invocations we pick up our custom SDK Resolver
     "$CLI_ROOT/dotnet" build-server shutdown --msbuild
 
+    local bootstrapArcadeDir=$(cat "$scriptroot/artifacts/toolset/bootstrap-sdks.txt" | grep "microsoft.dotnet.arcade.sdk")
+    local arcadeBuildStepsDir="$bootstrapArcadeDir/tools/"
+
     # Point MSBuild to the custom SDK resolvers folder, so it will pick up our custom SDK Resolver
     export MSBUILDADDITIONALSDKRESOLVERSFOLDER="$scriptroot/artifacts/toolset/VSSdkResolvers/"
 
@@ -283,7 +288,17 @@ function Build {
       bl="/bl:\"$log_dir/Build.binlog\""
     fi
 
-    "$CLI_ROOT/dotnet" msbuild --restore "$project" $bl $targets "${properties[@]}"
+    "$CLI_ROOT/dotnet" \
+      msbuild \
+      --restore \
+      "$arcadeBuildStepsDir/Build.proj" \
+      "/p:Projects=$project" \
+      /p:Configuration=$configuration \
+      "/p:RepoRoot=$scriptroot/" \
+      "-tl:off" \
+      $bl \
+      "${actions[@]}" \
+      "${properties[@]}"
   fi
 }