Use Arcade's publishing logic to publish a given VMR vertical

src/SourceBuild/content/Directory.Build.props

@@ -198,12 +198,14 @@
   </PropertyGroup>
 
   <PropertyGroup>
+    <ArtifactsStagingDir Condition="'$(ArtifactsStagingDir)' == ''">$(ArtifactsDir)/staging/</ArtifactsStagingDir>
+    <SourceBuiltAssetsDir>$(ArtifactsStagingDir)/assets/$(Configuration)</SourceBuiltAssetsDir>
     <!-- Respect the VerticalName property which is set to a unique identifier in CI.
          Don't use that when building source-only to allow the build to find the previously built manifest file regardless of the vertical name.
          Default to VerticalManifest.xml. -->
     <MergedAssetManifestName Condition="'$(VerticalName)' != ''">$(VerticalName).xml</MergedAssetManifestName>
     <MergedAssetManifestName Condition="'$(DotNetBuildSourceOnly)' == 'true' or '$(MergedAssetManifestName)' == ''">VerticalManifest.xml</MergedAssetManifestName>
-    <MergedAssetManifestOutputPath>$(ArtifactsAssetManifestsDir)$(MergedAssetManifestName)</MergedAssetManifestOutputPath>
+    <MergedAssetManifestOutputPath>$(ArtifactsStagingDir)/manifests/$(Configuration)/$(MergedAssetManifestName)</MergedAssetManifestOutputPath>
   </PropertyGroup>
 
 </Project>

src/SourceBuild/content/build.proj

@@ -20,10 +20,6 @@
     <ProjectReference Include="$(ToolsDir)tools.proj" BuildInParallel="true" />
 
     <ProjectReference Include="$(RepoProjectsDir)$(RootRepo).proj" />
-
-    <!-- Post-build: Source-only validation, packaging and publishing -->
-    <ProjectReference Include="$(RepositoryEngineeringDir)merge-asset-manifests.proj" />
-    <ProjectReference Include="$(RepositoryEngineeringDir)finish-source-only.proj" Condition="'$(DotNetBuildSourceOnly)' == 'true'" />
   </ItemGroup>
 
   <Target Name="PrintInfo" BeforeTargets="Build">

src/SourceBuild/content/build.sh

@@ -127,7 +127,7 @@ while [[ $# > 0 ]]; do
       test=true
       ;;
     -sign)
-      properties+=( "/p:Sign=true" )
+      properties+=( "/p:DotNetBuildSign=true" )
       ;;
 
     # Source-only settings
@@ -214,13 +214,12 @@ fi
 . "$scriptroot/eng/common/tools.sh"
 
 project="$scriptroot/build.proj"
-targets="/t:Build"
+actions=( "/p:Restore=true" "/p:Build=true" "/p:Publish=true")
 
 # This repo uses the VSTest integration instead of the Arcade Test target
 if [[ "$test" == true ]]; then
   project="$scriptroot/test/tests.proj"
-  targets="$targets;VSTest"
-  properties+=( "/p:Test=true" )
+  actions=( "/p:Restore=true" "/p:Build=true" "/p:Test=true" )
 
   # Workaround for vstest hangs (https://github.com/microsoft/vstest/issues/5091) [TODO]
   export MSBUILDENSURESTDOUTFORTASKPROCESSES=1
@@ -244,10 +243,14 @@ function Build {
     fi
 
     MSBuild --restore \
-      $project \
-      $targets \
+      $_InitializeToolset \
+      "/p:Projects=$project" \
       $bl \
       /p:Configuration=$configuration \
+      "/p:RepoRoot=$scriptroot/" \
+      "/p:DotNetPublishUsingPipelines=true" \
+      "-tl:off" \
+      "${actions[@]}" \
       "${properties[@]}"
 
     ExitWithExitCode 0
@@ -267,6 +270,9 @@ function Build {
     # kill off the MSBuild server so that on future invocations we pick up our custom SDK Resolver
     "$CLI_ROOT/dotnet" build-server shutdown --msbuild
 
+    local bootstrapArcadeDir=$(cat "$scriptroot/artifacts/toolset/bootstrap-sdks.txt" | grep "microsoft.dotnet.arcade.sdk")
+    local arcadeBuildStepsDir="$bootstrapArcadeDir/tools/"
+
     # Point MSBuild to the custom SDK resolvers folder, so it will pick up our custom SDK Resolver
     export MSBUILDADDITIONALSDKRESOLVERSFOLDER="$scriptroot/artifacts/toolset/VSSdkResolvers/"
 
@@ -275,7 +281,18 @@ function Build {
       bl="/bl:\"$log_dir/Build.binlog\""
     fi
 
-    "$CLI_ROOT/dotnet" msbuild --restore "$project" $bl $targets "${properties[@]}"
+    "$CLI_ROOT/dotnet" \
+      msbuild \
+      --restore \
+      "$arcadeBuildStepsDir/Build.proj" \
+      "/p:Projects=$project" \
+      /p:Configuration=$configuration \
+      "/p:RepoRoot=$scriptroot/" \
+      "/p:DotNetPublishUsingPipelines=true" \
+      "-tl:off" \
+      $bl \
+      "${actions[@]}" \
+      "${properties[@]}"
   fi
 }
 

src/SourceBuild/content/eng/Publishing.props

@@ -0,0 +1,92 @@
+<Project InitialTargets="DiscoverArtifacts">
+  <PropertyGroup>
+    <EnableDefaultArtifacts>false</EnableDefaultArtifacts>
+    <PushToLocalStorage>true</PushToLocalStorage>
+    <PreserveRepoOrigin>true</PreserveRepoOrigin>
+    <ArtifactsAssetsDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsDir)', 'assets', '$(Configuration)'))</ArtifactsAssetsDir>
+    <ArtifactsStagingDir Condition="'$(ArtifactsStagingDir)' == ''">$(ArtifactsDir)/staging/</ArtifactsStagingDir>
+    <SourceBuiltAssetsDir>$(ArtifactsStagingDir)/assets/$(Configuration)</SourceBuiltAssetsDir>
+    <SourceBuiltAssetManifestsDir>$(ArtifactsStagingDir)/manifests/$(Configuration)</SourceBuiltAssetManifestsDir>
+    <SourceBuiltPdbArtifactsDir>$(ArtifactsStagingDir)/PDB/$(Configuration)</SourceBuiltPdbArtifactsDir>
+    <PrereqsDir>$([MSBuild]::NormalizeDirectory('$(RepoRoot)', 'prereqs'))</PrereqsDir>
+    <PrereqsPackagesDir>$([MSBuild]::NormalizeDirectory('$(PrereqsDir)', 'packages'))</PrereqsPackagesDir>
+    <ReferencePackagesDir>$([MSBuild]::NormalizeDirectory('$(PrereqsPackagesDir)', 'reference'))</ReferencePackagesDir>
+    <PublishingUseHardlinksIfPossible>true</PublishingUseHardlinksIfPossible>
+    <AssetManifestFileName Condition="'$(VerticalName)' != ''">$(VerticalName).xml</AssetManifestFileName>
+    <AssetManifestFileName Condition="'$(DotNetBuildSourceOnly)' == 'true' or '$(AssetManifestName)' == ''">VerticalManifest.xml</AssetManifestFileName>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(DotNetBuildSourceOnly)' != 'true'">
+    <SourceBuiltShippingPackagesDir>$(ArtifactsStagingDir)/packages/$(Configuration)/Shipping</SourceBuiltShippingPackagesDir>
+    <SourceBuiltNonShippingPackagesDir>$(ArtifactsStagingDir)/packages/$(Configuration)/NonShipping</SourceBuiltNonShippingPackagesDir>
+  </PropertyGroup>
+  <!--
+    When building from source, the Private.SourceBuilt.Artifacts archive will contain the nuget packages, so don't push them where we'll upload from.
+  -->
+  <PropertyGroup Condition="'$(DotNetBuildSourceOnly)' == 'true'">
+    <SourceBuiltShippingPackagesDir>$(ArtifactsTmpDir)/staging/packages/$(Configuration)/Shipping</SourceBuiltShippingPackagesDir>
+    <SourceBuiltNonShippingPackagesDir>$(ArtifactsTmpDir)/staging/packages/$(Configuration)/NonShipping</SourceBuiltNonShippingPackagesDir>
+  </PropertyGroup>
+  <PropertyGroup>
+    <AssetManifestsIntermediateDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsObjDir)', 'manifests', '$(Configuration)'))</AssetManifestsIntermediateDir>
+    <IntermediateSymbolsRootDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsObjDir)', 'Symbols'))</IntermediateSymbolsRootDir>
+  </PropertyGroup>
+
+  <!--
+    Don't binplace any artifacts with vertical visibility. These are only used for building other repos within this vertical
+    and shouldn't be pushed to build storage.
+  -->
+  <ItemGroup>
+    <ArtifactVisibilityToPublish Include="Internal;External" />
+  </ItemGroup>
+
+  <!-- Build task assembly paths -->
+  <PropertyGroup>
+    <MicrosoftDotNetUnifiedBuildTasksAssembly>$([MSBuild]::NormalizePath('$(ArtifactsBinDir)', 'Microsoft.DotNet.UnifiedBuild.Tasks', '$(Configuration)', 'Microsoft.DotNet.UnifiedBuild.Tasks.dll'))</MicrosoftDotNetUnifiedBuildTasksAssembly>
+  </PropertyGroup>
+
+  <UsingTask TaskName="Microsoft.DotNet.UnifiedBuild.Tasks.GetKnownArtifactsFromAssetManifests" AssemblyFile="$(MicrosoftDotNetUnifiedBuildTasksAssembly)" TaskFactory="TaskHostFactory" />
+
+  <Target Name="DiscoverArtifacts">
+    <ItemGroup>
+      <!-- Repo manifests from individual repos -->
+      <RepoManifests Include="$(AssetManifestsIntermediateDir)/**/*.xml" />
+    </ItemGroup>
+    <!-- Get produced packages and assets from the manifests -->
+    <GetKnownArtifactsFromAssetManifests AssetManifests="@(RepoManifests)">
+      <Output TaskParameter="KnownPackages" ItemName="ProducedPackage" />
+      <Output TaskParameter="KnownBlobs" ItemName="ProducedAsset" />
+    </GetKnownArtifactsFromAssetManifests>
+    <ItemGroup>
+      <ProducedPackage>
+        <IsShipping Condition="'%(ProducedPackage.NonShipping)' != 'true'">true</IsShipping>
+        <IsShipping Condition="'%(ProducedPackage.NonShipping)' == 'true'">false</IsShipping>
+        <Kind>Package</Kind>
+        <PublishFlatContainer>false</PublishFlatContainer>
+      </ProducedPackage>
+      <ProducedPackage>
+        <ShippingFolder Condition="'%(ProducedPackage.NonShipping)' != 'true'">Shipping</ShippingFolder>
+        <ShippingFolder Condition="'%(ProducedPackage.NonShipping)' == 'true'">NonShipping</ShippingFolder>
+      </ProducedPackage>
+
+      <ReferencePackage Include="@(ProducedPackage->'$(ReferencePackagesDir)%(Identity).%(Version).nupkg')" Condition="'%(RepoOrigin)' == 'source-build-reference-packages'" />
+    </ItemGroup>
+
+    <!-- Create Artifact items for produced packages and assets -->
+    <ItemGroup>
+      <Artifact Include="@(ProducedPackage->'$(ArtifactsPackagesDir)%(ShippingFolder)/%(RepoOrigin)/%(Identity).%(Version).nupkg')" Condition="'%(RepoOrigin)' != 'source-build-reference-packages'" />
+      <Artifact Include="@(ReferencePackage)" RepoOrigin="SourceBuildReferencePackages" />
+      <Artifact Include="@(ProducedAsset->'$(ArtifactsAssetsDir)%(Identity)')">
+        <Kind>Asset</Kind>
+      </Artifact>
+    </ItemGroup>
+  </Target>
+
+  <Target Name="FinishSourceBuild" AfterTargets="Publish" Condition="'$(DotNetBuildSourceOnly)' == 'true'">
+    <MSBuild
+      Projects="$(RepositoryEngineeringDir)finish-source-only.proj"
+      Targets="Restore"
+      Properties="MSBuildRestoreSessionId=$([System.Guid]::NewGuid())" />
+    <MSBuild
+      Projects="$(RepositoryEngineeringDir)finish-source-only.proj" />
+  </Target>
+</Project>

src/SourceBuild/content/eng/build.ps1

@@ -53,26 +53,25 @@ if ($help) {
 }
 
 $project = Join-Path $RepoRoot "build.proj"
-$arguments = @()
-$targets = "/t:Build"
+$actions = @("/p:Restore=true", "/p:Build=true", "/p:Publish=true")
 
 # This repo uses the VSTest integration instead of the Arcade Test target
 if ($test) {
   $project = Join-Path (Join-Path $RepoRoot "test") "tests.proj"
-  $targets += ";VSTest"
-  $arguments += "/p:Test=true"
+  $actions = @("/p:Restore=true", "/p:Build=true", "/p:Publish=true")
 
   # Workaround for vstest hangs (https://github.com/microsoft/vstest/issues/5091) [TODO]
   $env:MSBUILDENSURESTDOUTFORTASKPROCESSES="1"
 }
 
+$arguments = @()
 # Override project if specified on cmd-line
 if ($projects) {
   $project = $projects
 }
 
 if ($sign) {
-  $arguments += "/p:Sign=true"
+  $arguments += "/p:DotNetBuildSign=true"
 }
 
 if ($buildRepoTests) {
@@ -84,15 +83,19 @@ if ($cleanWhileBuilding) {
 }
 
 function Build {
-  InitializeToolset
+  $buildProj = InitializeToolset
 
   $bl = if ($binaryLog) { '/bl:' + (Join-Path $LogDir 'Build.binlog') } else { '' }
 
   MSBuild -restore `
-    $project `
+    $buildProj `
+    /p:Projects=$project `
+    /p:RepoRoot=$RepoRoot `
+    "-tl:off" `
     $bl `
-    $targets `
     /p:Configuration=$configuration `
+    /p:DotNetPublishUsingPipelines=true `
+    @actions `
     @properties `
     @arguments
 }

src/SourceBuild/content/eng/finish-source-only.proj

@@ -11,7 +11,6 @@
     <ProjectReference Include="$(TasksDir)Microsoft.DotNet.UnifiedBuild.Tasks\Microsoft.DotNet.UnifiedBuild.Tasks.csproj" />
     <ProjectReference Include="$(TasksDir)Microsoft.DotNet.SourceBuild.Tasks.LeakDetection\Microsoft.DotNet.SourceBuild.Tasks.LeakDetection.csproj" />
     <ProjectReference Include="$(RepositoryEngineeringDir)extract-sdk-archive.proj" />
-    <ProjectReference Include="$(RepositoryEngineeringDir)merge-asset-manifests.proj" />
   </ItemGroup>
 
   <!-- After building, generate a prebuilt usage report. -->
@@ -132,7 +131,7 @@
       <AssetToCheck Include="$(ArtifactsAssetsDir)*$(ArchiveExtension)" />
       <AssetToCheck Remove="$(ArtifactsAssetsDir)$(SourceBuiltArtifactsTarballName)*" />
       <!-- Include shipping nuget packages. -->
-      <ShippingPackageToCheck Include="$(ArtifactsShippingPackagesDir)**/*.nupkg" />
+      <ShippingPackageToCheck Include="$(ArtifactsTmpDir)/staging/packages/$(Configuration)/Shipping/*.nupkg" />
       <!-- Add and mark SBRP packages to validate that they have the correct poison attribute. -->
       <SbrpPackageToCheck Include="$(ReferencePackagesDir)**\*.nupkg" IsSourceBuildReferencePackage="true" />
     </ItemGroup>
@@ -226,8 +225,8 @@
           DependsOnTargets="DetermineSourceBuiltSdkNonStableVersion;ResolveProjectReferences">
     <!-- Inputs: Packages to include in the tarball -->
     <ItemGroup>
-      <ArtifactsPackageToBundle Include="$(ArtifactsShippingPackagesDir)**;
-                                          $(ArtifactsNonShippingPackagesDir)**"
+      <ArtifactsPackageToBundle Include="$(ArtifactsTmpDir)/staging/packages/$(Configuration)/Shipping/**;
+                                          $(ArtifactsTmpDir)/staging/packages/$(Configuration)/NonShipping/**"
                                 Condition="!$([System.String]::Copy('%(Identity)').EndsWith('.symbols.nupkg'))" />
       <ReferencePackageToBundle Include="$(ReferencePackagesDir)**"
                                 Condition="!$([System.String]::Copy('%(Identity)').EndsWith('.symbols.nupkg'))" />

src/SourceBuild/content/eng/init-source-only.proj

@@ -100,9 +100,17 @@
       <ToolsetPackage Include="Microsoft.Build.Traversal" SourceFolder="$(ReferencePackagesDir)" Version="$(TRAVERSAL_BOOTSTRAP_VERSION)" />
     </ItemGroup>
 
+    <ItemGroup>
+      <ToolsetPackage Destination="$(BootstrapPackagesDir)$([System.String]::Copy('%(ToolsetPackage.Identity)').ToLowerInvariant())/%(ToolsetPackage.Version)" />
+    </ItemGroup>
+
     <Unzip SourceFiles="%(ToolsetPackage.SourceFolder)%(ToolsetPackage.Identity).%(ToolsetPackage.Version).nupkg"
-           DestinationFolder="$(BootstrapPackagesDir)$([System.String]::Copy('%(ToolsetPackage.Identity)').ToLowerInvariant())/%(ToolsetPackage.Version)"
+           DestinationFolder="%(ToolsetPackage.Destination)"
            SkipUnchangedFiles="true" />
+
+    <WriteLinesToFile File="$(ArtifactsDir)/toolset/bootstrap-sdks.txt"
+                      Lines="@(ToolsetPackage->Metadata('Destination'))"
+                      Overwrite="true" />
   </Target>
 
 </Project>