Add baselining mechanism to the build comparison tool

Accepts an optiona; json file with a list of baseline entries. Baseline a few well known items.

Author: mmitche

eng/pipelines/templates/steps/vmr-validate-asset-baseline.yml

@@ -48,6 +48,7 @@ steps:
     -vmrAssetBasePath "$(Build.ArtifactStagingDirectory)\vmr-assets"
     -msftAssetBasePath "$(Build.ArtifactStagingDirectory)\base-assets"
     -report "$(Build.SourcesDirectory)\artifacts\AssetBaselines\BaselineComparison.xml"
+    -baseline "$(Build.SourcesDirectory)\src\sdk\eng\vmr-msft-comparison-baseline.json"
   displayName: Validate Asset Baselines
 
 - task: 1ES.PublishPipelineArtifact@1

eng/vmr-msft-comparison-baseline.json

@@ -0,0 +1,22 @@
+[
+  {
+    "issueType": "MissingPackageContent",
+    "descriptionMatch": "mscordaccore_(arm64|amd64)_(arm64|amd64|x86)_10.0.\\d{2}.\\d+.dll",
+    "justification": "mscordaccoree's version is baked into the file name and varies build to build."
+  },
+  {
+    "issueType": "ExtraPackageContent",
+    "descriptionMatch": "mscordaccore_(arm64|amd64)_(arm64|amd64|x86)_10.0.\\d{2}.\\d+.dll",
+    "justification": "mscordaccoree's version is baked into the file name and varies build to build."
+  },
+  {
+    "issueType": "AssemblyVersionMismatch",
+    "descriptionMatch": ".resources.dll",
+    "justification": "Resource dlls get per-build versioning but this doesn't matter."
+  },
+  {
+    "issueType": "MissingShipping",
+    "descriptionMatch": "Aspire.*",
+    "justification": "Aspire to be removed from the VMR. Also .NET does not ship these packages."
+  }
+]

src/SourceBuild/content/eng/tools/BuildComparer/AssetMapping.cs

@@ -0,0 +1,68 @@
+using System.Xml.Linq;
+using System.Xml.Serialization;
+/// <summary>
+/// Represents the mapping between base build and VMR build for a specific asset.
+/// </summary>
+public class AssetMapping
+{
+    /// <summary>
+    /// Gets or sets the identifier of the asset.
+    /// </summary>
+    [XmlAttribute("Id")]
+    public string Id { get; set; }
+
+    /// <summary>
+    /// Gets or sets the type of the asset.
+    /// </summary>
+    [XmlAttribute("Type")]
+    public AssetType AssetType { get; set; } = AssetType.Unknown;
+    
+    /// <summary>
+    /// Gets a value indicating whether a corresponding element was found in the diff manifest.
+    /// </summary>
+    [XmlIgnore]
+    public bool DiffElementFound { get => DiffManifestElement != null; }
+    
+    /// <summary>
+    /// Gets a value indicating whether a corresponding file was found in the diff build.
+    /// </summary>
+    [XmlIgnore]
+    public bool DiffFileFound { get => DiffFilePath != null; }
+
+    /// <summary>
+    /// Gets or sets the path to the diff file.
+    /// </summary>
+    [XmlElement("DiffFile")]
+    public string DiffFilePath { get; set; }
+    
+    /// <summary>
+    /// Gets or sets the XML element from the diff manifest.
+    /// </summary>
+    [XmlIgnore]
+    public XElement DiffManifestElement { get; set; }
+
+    /// <summary>
+    /// Gets or sets the path to the base build file.
+    /// </summary>
+    [XmlElement("BaseFile")]
+    public string BaseBuildFilePath { get; set; }
+
+    /// <summary>
+    /// Gets or sets the XML element from the base build manifest.
+    /// </summary>
+    [XmlIgnore]
+    public XElement BaseBuildManifestElement
+    {
+        get; set;
+    }
+
+    /// <summary>
+    /// Gets or sets the list of errors encountered during evaluation.
+    /// </summary>
+    public List<string> EvaluationErrors { get; set; } = new List<string>();
+
+    /// <summary>
+    /// Gets or sets the list of issues identified for this asset.
+    /// </summary>
+    public List<Issue> Issues { get; set; } = new List<Issue>();
+}

src/SourceBuild/content/eng/tools/BuildComparer/AssetType.cs

@@ -0,0 +1,21 @@
+
+/// <summary>
+/// Defines the type of asset being processed in the build comparison tool.
+/// </summary>
+public enum AssetType
+{
+    /// <summary>
+    /// Represents a random non-package file in the build.
+    /// </summary>
+    Blob,
+    
+    /// <summary>
+    /// Represents a NuGet package asset.
+    /// </summary>
+    Package,
+    
+    /// <summary>
+    /// Represents an asset of unknown type.
+    /// </summary>
+    Unknown
+}

src/SourceBuild/content/eng/tools/BuildComparer/BaselineEntry.cs

@@ -0,0 +1,106 @@
+using System.Text.RegularExpressions;
+using System.Text.Json;
+using System.Text.Json.Serialization;
+using System.Xml.Serialization;
+
+public class BaselineEntry
+{
+    [XmlIgnore]
+    /// <summary>
+    /// Regex used to match the ID of the asset.
+    /// </summary>
+    public Regex IdMatch { get; set; }
+    [XmlIgnore]
+    /// <summary>
+    /// Issue type to baseline
+    /// </summary>
+    public IssueType? IssueType { get; set; }
+    [XmlIgnore]
+    /// <summary>
+    /// Regex used to match the description of the asset.
+    /// </summary>
+    public Regex DescriptionMatch { get; set; }
+    [XmlAttribute]
+    /// <summary>
+    /// Justification for the baseline.
+    /// </summary>
+    public string Justification { get; set; }
+}
+
+public class Baseline
+{
+    private List<BaselineEntry> _entries;
+    public Baseline(string filePath)
+    {
+        if (string.IsNullOrEmpty(filePath) || !File.Exists(filePath))
+        {
+            throw new ArgumentException($"Baseline file not found: {filePath}");
+        }
+
+        string jsonContent = File.ReadAllText(filePath);
+
+        // Configure JSON deserialization options with custom converters
+        var options = new JsonSerializerOptions
+        {
+            PropertyNameCaseInsensitive = true,
+            Converters =
+            {
+                new RegexJsonConverter(),
+                new JsonStringEnumConverter()
+            }
+        };
+
+        // Deserialize the JSON into a list of BaselineEntry objects
+        _entries = JsonSerializer.Deserialize<List<BaselineEntry>>(jsonContent, options) ?? new List<BaselineEntry>();
+
+        // Validate the baseline entries. An entry must have a Justification and an IssueType,
+        // and must have at least one of IdMatch or DescriptionMatch.
+
+        foreach (var entry in _entries)
+        {
+            if (string.IsNullOrEmpty(entry.Justification))
+            {
+                throw new ArgumentException("Justification cannot be null or empty.");
+            }
+            if (entry.IssueType == null)
+            {
+                throw new ArgumentException("IssueType cannot be null.");
+            }
+            if (entry.IdMatch == null && entry.DescriptionMatch == null)
+            {
+                throw new ArgumentException("At least one of IdMatch or DescriptionMatch must be provided.");
+            }
+        }
+    }
+
+    // Check which baseline entries match against the given asset issue.
+    public List<BaselineEntry> GetMatchingBaselineEntries(Issue assetIssue, AssetMapping assetMapping)
+    {
+        var matchingEntries = new List<BaselineEntry>();
+        foreach (var entry in _entries)
+        {
+            if (entry.IssueType == assetIssue.IssueType &&
+                (entry.IdMatch == null || entry.IdMatch.IsMatch(assetMapping.Id)) &&
+                (entry.DescriptionMatch == null || entry.DescriptionMatch.IsMatch(assetIssue.Description)))
+            {
+                matchingEntries.Add(entry);
+            }
+        }
+        return matchingEntries;
+    }
+
+    // Custom JSON converter for Regex objects
+    private class RegexJsonConverter : JsonConverter<Regex>
+    {
+        public override Regex Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options)
+        {
+            string pattern = reader.GetString();
+            return pattern != null ? new Regex(pattern, RegexOptions.Compiled) : null;
+        }
+
+        public override void Write(Utf8JsonWriter writer, Regex value, JsonSerializerOptions options)
+        {
+            writer.WriteStringValue(value?.ToString());
+        }
+    }
+}

src/SourceBuild/content/eng/tools/BuildComparer/ComparisonReport.cs

@@ -0,0 +1,37 @@
+using System.Xml.Serialization;
+/// <summary>
+/// Contains the results of a build comparison between Microsoft and VMR builds.
+/// </summary>
+public class ComparisonReport
+{
+    /// <summary>
+    /// Gets the number of assets with identified issues.
+    /// </summary>
+    public int IssueCount { get => AssetsWithIssues.Sum(a => a.Issues.Where(i => i.Baseline == null).Count()); }
+
+    /// <summary>
+    /// Gets the number of assets with evaluation errors.
+    /// </summary>
+    public int ErrorCount { get => AssetsWithErrors.Sum(a => a.EvaluationErrors.Count); }
+
+    /// <summary>
+    /// Gets the total number of assets analyzed in the report.
+    /// </summary>
+    public int BaselineCount { get => AssetsWithIssues.Sum(a => a.Issues.Where(i => i.Baseline != null).Count()) +
+                                      AssetsWithErrors.Sum(a => a.Issues.Where(i => i.Baseline != null).Count()) +
+                                      AssetsWithoutIssues.Sum(a => a.Issues.Where(i => i.Baseline != null).Count()); }
+    /// <summary>
+    /// Gets or sets the list of assets that have issues.
+    /// </summary>
+    public List<AssetMapping> AssetsWithIssues { get; set; }
+
+    /// <summary>
+    /// Gets or sets the list of assets that have issues.
+    /// </summary>
+    public List<AssetMapping> AssetsWithErrors { get; set; }
+
+    /// <summary>
+    /// Gets or sets the list of assets that have issues.
+    /// </summary>
+    public List<AssetMapping> AssetsWithoutIssues { get; set; }
+}

src/SourceBuild/content/eng/tools/BuildComparer/Issue.cs

@@ -0,0 +1,23 @@
+using System.Xml.Serialization;
+/// <summary>
+/// Represents an issue identified during asset comparison.
+/// </summary>
+public class Issue
+{
+    /// <summary>
+    /// Gets or sets the type of issue.
+    /// </summary>
+    [XmlAttribute("Type")]
+    public IssueType IssueType { get; set; }
+    
+    /// <summary>
+    /// Gets or sets a description of the issue.
+    /// </summary>
+    [XmlAttribute("Description")]
+    public string Description { get; set; }
+
+    /// <summary>
+    /// Matching baseline entries for this issue.
+    /// </summary>
+    public BaselineEntry Baseline { get; set; }
+}

src/SourceBuild/content/eng/tools/BuildComparer/IssueType.cs

@@ -0,0 +1,28 @@
+
+/// <summary>
+/// Defines types of issues that can be identified during asset comparison.
+/// </summary>
+public enum IssueType
+{
+    /// <summary>
+    /// Indicates a shipping asset is missing in the VMR build.
+    /// </summary>
+    MissingShipping,
+    
+    /// <summary>
+    /// Indicates a non-shipping asset is missing in the VMR build.
+    /// </summary>
+    MissingNonShipping,
+    
+    /// <summary>
+    /// Indicates an asset is classified differently between base and VMR builds.
+    /// </summary>
+    MisclassifiedAsset,
+    
+    /// <summary>
+    /// Indicates a version mismatch between assemblies in base and VMR builds.
+    /// </summary>
+    AssemblyVersionMismatch,
+    MissingPackageContent,
+    ExtraPackageContent,
+}