chore: Dotty refactor (#2618)

Author: tippmar-nr

.github/workflows/nuget_slack_notifications.yml

@@ -2,12 +2,12 @@ name: Check for new core technologies
 
 on:
   schedule:
-   - cron:  '0 10 * * *'
+   - cron:  '0 10 * * 1' # Every Monday at 10:00 AM
   workflow_dispatch:
     inputs:
       daysToSearch:
-        description: "Days of NuGet history to search for package updates"
-        default: "1"
+        description: "Days of NuGet history to search for package updates (0 to search since last run of this workflow on main)"
+        default: "0"
         type: string
       testMode:
         description: "If checked, no notification message will be sent to the team channel, nor will any Github issues be created."
@@ -49,7 +49,27 @@ jobs:
           dotnet add nugetSlackNotifications.csproj package NewRelic.Agent
           dotnet publish -o ${{ env.scan-tool-publish-path }}
 
+      - name: Find timestamp of most recent run of this workflow and set as an environment variable
+        if: inputs.daysToSearch == '0' || github.event_name == 'schedule'
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+           echo "LAST_RUN_TIMESTAMP=$(gh run list --workflow nuget_slack_notifications.yml --branch main --status completed --limit 1 --json updatedAt | jq -r '.[0].updatedAt')" >> $GITHUB_ENV
+        shell: bash
+
       - name: Check for updates to core technology packages
+        env:
+          DOTTY_WEBHOOK: ${{ secrets.SLACK_NUGET_NOTIFICATIONS_WEBHOOK }}
+          DOTTY_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          CORECLR_ENABLE_PROFILING: 1
+          CORECLR_NEWRELIC_HOME: ${{ env.scan-tool-publish-path }}/newrelic
+          CORECLR_PROFILER: "{36032161-FFC0-4B61-B559-F6C5D41BAE5A}"
+          CORECLR_PROFILER_PATH: ${{ env.scan-tool-publish-path }}/newrelic/libNewRelicProfiler.so
+          NEW_RELIC_APP_NAME: Dotty
+          NEW_RELIC_HOST: staging-collector.newrelic.com
+          NEW_RELIC_LICENSE_KEY: ${{ secrets.STAGING_LICENSE_KEY }}
+          DOTTY_LAST_RUN_TIMESTAMP: ${{ env.LAST_RUN_TIMESTAMP }}
+
         run: |
           if [ ${{ inputs.daysToSearch }} != "" ]; then
             export DOTTY_DAYS_TO_SEARCH=${{ inputs.daysToSearch }}
@@ -60,46 +80,4 @@ jobs:
           cd ${{ env.scan-tool-publish-path }}
           dotnet ./nugetSlackNotifications.dll ${{ env.nugets }}
         shell: bash
-
-        env:
-            DOTTY_WEBHOOK: ${{ secrets.SLACK_NUGET_NOTIFICATIONS_WEBHOOK }}
-            DOTTY_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-            CORECLR_ENABLE_PROFILING: 1
-            CORECLR_NEWRELIC_HOME: ${{ env.scan-tool-publish-path }}/newrelic
-            CORECLR_PROFILER: "{36032161-FFC0-4B61-B559-F6C5D41BAE5A}"
-            CORECLR_PROFILER_PATH: ${{ env.scan-tool-publish-path }}/newrelic/libNewRelicProfiler.so
-            NEW_RELIC_APP_NAME: Dotty
-            NEW_RELIC_HOST: staging-collector.newrelic.com
-            NEW_RELIC_LICENSE_KEY: ${{ secrets.STAGING_LICENSE_KEY }}
-            nugets:
-                 "amazon.lambda.apigatewayevents
-                 amazon.lambda.applicationloadbalancerevents
-                 amazon.lambda.cloudwatchevents
-                 amazon.lambda.dynamodbevents
-                 amazon.lambda.kinesisevents
-                 amazon.lambda.kinesisfirehoseevents
-                 amazon.lambda.s3events
-                 amazon.lambda.simpleemailevents
-                 amazon.lambda.snsevents
-                 amazon.lambda.sqsevents
-                 elasticsearch.net
-                 elastic.clients.elasticsearch
-                 log4net
-                 microsoft.extensions.logging
-                 microsoft.data.sqlclient
-                 microsoft.net.http
-                 mongodb.driver
-                 mysql.data
-                 mysqlconnector
-                 nest
-                 nlog
-                 rabbitmq.client
-                 restsharp
-                 serilog
-                 serilog.extensions.logging
-                 serilog.aspnetcore
-                 serilog.sinks.file
-                 serilog.sinks.console
-                 stackexchange.redis
-                 system.data.sqlclient"
-                 
+        

.github/workflows/scripts/nugetSlackNotifications/Program.cs

@@ -3,85 +3,133 @@
 using Serilog;
 using System;
 using System.Collections.Generic;
+using System.Linq;
 using System.Net;
 using System.Net.Http;
 using System.Text;
 using System.Text.Json;
 using System.Text.Json.Serialization;
 using System.Threading.Tasks;
-
-#pragma warning disable CS8618
+using NuGet.Common;
+using NuGet.Configuration;
+using NuGet.Protocol;
+using NuGet.Protocol.Core.Types;
+using Repository = NuGet.Protocol.Core.Types.Repository;
 
 namespace nugetSlackNotifications
 {
     public class Program
     {
-        // the semver2 registration endpoint returns gzip encoded json
-        private static readonly HttpClient _client = new(new HttpClientHandler() { AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate });
+        private static readonly HttpClient _client = new();
         private static List<NugetVersionData> _newVersions = new();
 
-        private static readonly int _daysToSearch = int.TryParse(Environment.GetEnvironmentVariable("DOTTY_DAYS_TO_SEARCH"), out var days) ? days : 1; // How many days of package release history to scan for changes
+        private static readonly int _daysToSearch = int.TryParse(Environment.GetEnvironmentVariable("DOTTY_DAYS_TO_SEARCH"), out var days) ? int.Max(1, days) : 1; // How many days of package release history to scan for changes
         private static readonly bool _testMode = bool.TryParse(Environment.GetEnvironmentVariable("DOTTY_TEST_MODE"), out var testMode) ? testMode : false;
-        private static readonly string? _webhook = Environment.GetEnvironmentVariable("DOTTY_WEBHOOK");
-        private static readonly string? _githubToken = Environment.GetEnvironmentVariable("DOTTY_TOKEN");
+        private static readonly string _webhook = Environment.GetEnvironmentVariable("DOTTY_WEBHOOK");
+        private static readonly string _githubToken = Environment.GetEnvironmentVariable("DOTTY_TOKEN");
+        private static readonly DateTimeOffset _lastRunTimestamp = DateTimeOffset.TryParse(Environment.GetEnvironmentVariable("DOTTY_LAST_RUN_TIMESTAMP"), out var timestamp) ? timestamp : DateTimeOffset.MinValue;
+        private const string PackageInfoFilename = "packageInfo.json";
 
 
-        static async Task Main(string[] args)
+        static async Task Main()
         {
             Log.Logger = new LoggerConfiguration().WriteTo.Console().CreateLogger();
 
-            foreach (string packageName in args)
+            // searchTime is the date to search for package updates from.
+            // If _lastRunTimestamp is not set, search from _daysToSearch days ago.
+            // Otherwise, search from _lastRunTimestamp.
+            var searchTime = _lastRunTimestamp == DateTimeOffset.MinValue ? DateTimeOffset.UtcNow.Date.AddDays(-_daysToSearch) : _lastRunTimestamp;
+
+            Log.Information($"Searching for package updates since {searchTime.ToUniversalTime():s}Z.");
+
+            // initialize nuget repo
+            var ps = new PackageSource("https://api.nuget.org/v3/index.json");
+            var sourceRepository = Repository.Factory.GetCoreV3(ps);
+            var metadataResource = await sourceRepository.GetResourceAsync<PackageMetadataResource>();
+            var sourceCacheContext = new SourceCacheContext();
+
+            if (!System.IO.File.Exists(PackageInfoFilename))
+            {
+                Log.Error($"{PackageInfoFilename} not found in the current directory. Exiting.");
+                return;
+            }
+
+            var packageInfoJson = await System.IO.File.ReadAllTextAsync(PackageInfoFilename);
+            var packageInfos = JsonSerializer.Deserialize<PackageInfo[]>(packageInfoJson);
+
+            foreach (var package in packageInfos)
             {
                 try
                 {
-                    await CheckPackage(packageName);
+                    await CheckPackage(package, metadataResource, sourceCacheContext, searchTime);
                 }
                 catch (Exception ex)
                 {
-                    Log.Error(ex, $"Caught exception while checking {packageName} for updates.");
-                    await SendSlackNotification($"Dotty: caught exception while checking {packageName} for updates: {ex}");
+                    Log.Error(ex, $"Caught exception while checking {package.PackageName} for updates.");
+                    await SendSlackNotification($"Dotty: caught exception while checking {package.PackageName} for updates: {ex}");
                 }
             }
 
             await AlertOnNewVersions();
             await CreateGithubIssuesForNewVersions();
-
         }
 
         [Transaction]
-        static async Task CheckPackage(string packageName)
+        static async Task CheckPackage(PackageInfo package, PackageMetadataResource metadataResource,            SourceCacheContext sourceCacheContext, DateTimeOffset searchTime)
         {
-            var response = await _client.GetStringAsync($"https://api.nuget.org/v3/registration5-gz-semver2/{packageName}/index.json");
+            var packageName = package.PackageName;
+
+            var metaData = (await metadataResource.GetMetadataAsync(packageName, false, false, sourceCacheContext, NullLogger.Instance, System.Threading.CancellationToken.None)).OrderByDescending(p => p.Identity.Version).ToList();
 
-            SearchResult? searchResult = JsonSerializer.Deserialize<SearchResult>(response);
-            if (searchResult is null)
+            if (!metaData.Any())
             {
-                Log.Warning($"CheckPackage: null search result for package {packageName}");
+                Log.Warning($"CheckPackage: No metadata found for package {packageName}");
                 return;
             }
 
-            Item item = searchResult.items[^1]; // get the most recent group
+            // get the most recent version of the package
+            var latest = metaData.First();
+            packageName = latest.Identity.Id;
 
-            // need to make another request to get the page with individual release listings
-            Page? page = JsonSerializer.Deserialize<Page>(await _client.GetStringAsync(item.id));
-            if (page is null)
+            // get the second most recent version of the package (if there is one)
+            var previous = metaData.Skip(1).FirstOrDefault();
+
+            // check publish date
+            if (latest.Published >= searchTime)
             {
-                Log.Warning($"CheckPackage: null page result for package {packageName}, item id {item.id}");
-                return;
-            }
+                if (previous != null && (package.IgnorePatch || package.IgnoreMinor))
+                {
+                    var previousVersion = previous.Identity.Version;
+                    var latestVersion = latest.Identity.Version;
 
-            // need to get the most recent and previous catalog entries to display previous and new version
-            Catalogentry? latestCatalogEntry = GetCatalogentry(packageName, page, 1);
-            Catalogentry? previousCatalogEntry = GetCatalogentry(packageName, page, 2);
+                    if (package.IgnorePatch)
+                    {
+                        if (previousVersion.Major == latestVersion.Major && previousVersion.Minor == latestVersion.Minor)
+                        {
+                            Log.Information($"Package {packageName} ignores Patch version updates; the Minor version ({latestVersion.Major}.{latestVersion.Minor:2}) has not been updated.");
+                            return;
+                        }
+                    }
+
+                    if (package.IgnoreMinor)
+                    {
 
-            if (latestCatalogEntry?.published > DateTime.Now.AddDays(-_daysToSearch) && !await latestCatalogEntry.isPrerelease())
-            {
-                Log.Information($"Package {packageName} has been updated in the past {_daysToSearch} days.");
-                _newVersions.Add(new NugetVersionData(packageName, previousCatalogEntry?.version ?? "Unknown", latestCatalogEntry.version, $"https://www.nuget.org/packages/{packageName}/"));
+                        if (previousVersion.Major == latestVersion.Major)
+                        {
+                            Log.Information($"Package {packageName} ignores Minor version updates; the Major version ({latestVersion.Major}) has not been updated.");
+                            return;
+                        }
+                    }
+                }
+
+                var previousVersionDescription = previous?.Identity.Version.ToNormalizedString() ?? "Unknown";
+                var latestVersionDescription = latest.Identity.Version.ToNormalizedString();
+                Log.Information($"Package {packageName} was updated from {previousVersionDescription} to {latestVersionDescription}.");
+                _newVersions.Add(new NugetVersionData(packageName, previousVersionDescription, latestVersionDescription, latest.PackageDetailsUrl.ToString(), latest.Published.Value.Date));
             }
             else
             {
-                Log.Information($"Package {packageName} has not been updated in the past {_daysToSearch} days.");
+                Log.Information($"Package {packageName} has NOT been updated.");
             }
         }
 
@@ -91,10 +139,10 @@ static async Task AlertOnNewVersions()
 
             if (_newVersions.Count > 0 && _webhook != null && !_testMode) // only message channel if there's package updates to report AND we have a webhook from the environment AND we're not in test mode
             {
-                string msg = "Hi team! Dotty here :technologist::pager:\nThere's some new NuGet releases you should know about :arrow_heading_down::sparkles:";
+                var msg = "Hi team! Dotty here :technologist::pager:\nThere's some new NuGet releases you should know about :arrow_heading_down::sparkles:";
                 foreach (var versionData in _newVersions)
                 {
-                    msg += $"\n\t:package: {char.ToUpper(versionData.PackageName[0]) + versionData.PackageName[1..]} {versionData.OldVersion} :point_right: <{versionData.Url}|{versionData.NewVersion}>";
+                    msg += $"\n\t:package: {versionData.PackageName} {versionData.OldVersion} :point_right: <{versionData.Url}|{versionData.NewVersion}>";
                 }
                 msg += $"\nThanks and have a wonderful {DateTime.Now.DayOfWeek}.";
 
@@ -117,11 +165,14 @@ static async Task CreateGithubIssuesForNewVersions()
                 ghClient.Credentials = tokenAuth;
                 foreach (var versionData in _newVersions)
                 {
-                    var newIssue = new NewIssue($"Dotty: update tests for {versionData.PackageName} from {versionData.OldVersion} to {versionData.NewVersion}");
-                    newIssue.Body = versionData.Url;
+                    var newIssue = new NewIssue($"Dotty: update tests for {versionData.PackageName} from {versionData.OldVersion} to {versionData.NewVersion}")
+                    {
+                        Body = $"Package [{versionData.PackageName}]({versionData.Url}) was updated from {versionData.OldVersion} to {versionData.NewVersion} on {versionData.PublishDate.ToShortDateString()}."
+                    };
                     newIssue.Labels.Add("testing");
                     newIssue.Labels.Add("Core Technologies");
                     var issue = await ghClient.Issue.Create("newrelic", "newrelic-dotnet-agent", newIssue);
+                    Log.Information($"Created issue #{issue.Id} for {versionData.PackageName} update to {versionData.NewVersion} in newrelic/newrelic-dotnet-agent.");
                 }
             }
             else
@@ -160,29 +211,6 @@ static async Task SendSlackNotification(string msg)
                 Log.Error($"SendSlackNotification called but _webhook is null.  msg={msg}");
             }
         }
-
-        [Trace]
-        static Catalogentry? GetCatalogentry(string packageName, Page page, int releaseIndex)
-        {
-            // release index = 1 for most recent release, 2 for the previous release, etc.
-            try
-            {
-                // alternative json structure (see mysql.data)
-                if (page.items[^1].catalogEntry is null)
-                {
-                    return page.items[^1].items[^releaseIndex].catalogEntry; // latest release
-                }
-                else // standard structure
-                {
-                    return page.items[^releaseIndex].catalogEntry;
-                }
-            }
-            catch (IndexOutOfRangeException)
-            {
-                Log.Warning($"GetCatalogEntry: array index issue for package {packageName} and releaseIndex {releaseIndex}");
-                return null;
-            }
-        }
     }
 
     public class NugetVersionData
@@ -191,65 +219,27 @@ public class NugetVersionData
         public string OldVersion { get; set; }
         public string NewVersion { get; set; }
         public string Url { get; set; }
+        public DateTime PublishDate { get; set; }
 
-        public NugetVersionData(string packageName, string oldVersion, string newVersion, string url)
+        public NugetVersionData(string packageName, string oldVersion, string newVersion, string url, DateTime publishDate)
         {
             PackageName = packageName;
             OldVersion = oldVersion;
             NewVersion = newVersion;
             Url = url;
+            PublishDate = publishDate;
         }
     }
 
-    public class SearchResult
-    {
-        public int count { get; set; }
-        public Item[] items { get; set; }
-    }
-
-    public class Item
-    {
-        [JsonPropertyName("@id")]
-        public string id { get; set; }
-        public DateTime commitTimeStamp { get; set; }
-    }
-
-    public class Page
-    {
-        public Release[] items { get; set; }
-    }
-
-    public class Release
+    public class PackageInfo
     {
-        [JsonPropertyName("@id")]
-        public string id { get; set; }
-        public Catalogentry catalogEntry { get; set; }
-
-        // only packages with alternative json structure (i.e. mysql.data) will have this
-        public Release[] items { get; set; }
-    }
-
-    public class Version
-    {
-        public bool isPrerelease { get; set; }
-    }
-
-    public class Catalogentry
-    {
-        [JsonPropertyName("@id")]
-        public string id { get; set; }
-        public DateTime published { get; set; }
-        public string version { get; set; }
-
-        public async Task<bool> isPrerelease()
-        {
-            HttpClient client = new();
-            string response = await client.GetStringAsync(id);
-            Version? version = JsonSerializer.Deserialize<Version>(response);
-
-            return version is null ? false : version.isPrerelease;
-        }
+        [JsonPropertyName("packageName")]
+        public string PackageName { get; set; }
+        [JsonPropertyName("ignorePatch")]
+        public bool IgnorePatch { get; set; }
+        [JsonPropertyName("ignoreMinor")]
+        public bool IgnoreMinor { get; set; }
+        [JsonPropertyName("ignoreReason")]
+        public string IgnoreReason {get; set;}
     }
 }
-
-#pragma warning restore CS8618