User/aamaini/2281511 (#1409)

* Use Go CLI scan instead of go.sum if available

* Track project roots with successful CLI scan

* Add unit test for new CLI logic

* CR: Fix method name that filters go.sum

* CR: Bump Go117 version

---------

Co-authored-by: Aayush Maini <[email protected]>

Author: AMaini503

src/Microsoft.ComponentDetection.Detectors/go/Go117ComponentDetector.cs

@@ -48,7 +48,7 @@ public Go117ComponentDetector(
 
     public override IEnumerable<ComponentType> SupportedComponentTypes { get; } = [ComponentType.Go];
 
-    public override int Version => 1;
+    public override int Version => 2;
 
     protected override Task<IObservable<ProcessRequest>> OnPrepareDetectionAsync(
         IObservable<ProcessRequest> processRequests,
@@ -75,7 +75,7 @@ protected override Task<IObservable<ProcessRequest>> OnPrepareDetectionAsync(
                     return true;
                 }
 
-                return GoDetectorUtils.ShouldRemoveGoSumFromDetection(goSumFilePath: processRequest.ComponentStream.Location, goModFile, this.Logger);
+                return GoDetectorUtils.ShouldIncludeGoSumFromDetection(goSumFilePath: processRequest.ComponentStream.Location, goModFile, this.Logger);
             }
             finally
             {
@@ -97,7 +97,11 @@ protected override async Task OnFileFoundAsync(ProcessRequest processRequest, ID
             return;
         }
 
-        var record = new GoGraphTelemetryRecord();
+        using var record = new GoGraphTelemetryRecord();
+        var wasGoCliDisabled = this.IsGoCliManuallyDisabled();
+        record.WasGoCliDisabled = wasGoCliDisabled;
+        record.WasGoFallbackStrategyUsed = false;
+
         var fileExtension = Path.GetExtension(file.Location).ToUpperInvariant();
         switch (fileExtension)
         {
@@ -123,7 +127,29 @@ await GoDependencyGraphUtility.GenerateAndPopulateDependencyGraphAsync(
             case ".SUM":
             {
                 this.Logger.LogDebug("Found Go.sum: {Location}", file.Location);
-                await this.goParserFactory.CreateParser(GoParserType.GoSum, this.Logger).ParseAsync(singleFileComponentRecorder, file, record);
+
+                // check if we can use Go CLI instead
+                var wasGoCliScanSuccessful = false;
+                if (!wasGoCliDisabled)
+                {
+                    wasGoCliScanSuccessful = await this.goParserFactory.CreateParser(GoParserType.GoCLI, this.Logger).ParseAsync(singleFileComponentRecorder, file, record);
+                }
+
+                this.Logger.LogDebug("Status of Go CLI scan when considering {GoSumLocation}: {Status}", file.Location, wasGoCliScanSuccessful);
+
+                // If Go CLI scan was not successful/disabled, scan go.sum because this go.sum was recorded due to go.mod
+                // containing go < 1.17. So go.mod is incomplete. We need to parse go.sum to make list of dependencies complete
+                if (!wasGoCliScanSuccessful)
+                {
+                    record.WasGoFallbackStrategyUsed = true;
+                    this.Logger.LogDebug("Go CLI scan when considering {GoSumLocation} was not successful. Falling back to scanning go.sum", file.Location);
+                    await this.goParserFactory.CreateParser(GoParserType.GoSum, this.Logger).ParseAsync(singleFileComponentRecorder, file, record);
+                }
+                else
+                {
+                    this.projectRoots.Add(projectRootDirectory.FullName);
+                }
+
                 break;
             }
 

src/Microsoft.ComponentDetection.Detectors/go/GoComponentDetector.cs

@@ -94,7 +94,7 @@ protected override Task<IObservable<ProcessRequest>> OnPrepareDetectionAsync(
                     return true;
                 }
 
-                return GoDetectorUtils.ShouldRemoveGoSumFromDetection(goSumFilePath: processRequest.ComponentStream.Location, goModFile, this.Logger);
+                return GoDetectorUtils.ShouldIncludeGoSumFromDetection(goSumFilePath: processRequest.ComponentStream.Location, goModFile, this.Logger);
             }
             finally
             {

src/Microsoft.ComponentDetection.Detectors/go/Utils/GoDetectorUtils.cs

@@ -15,7 +15,7 @@ public static class GoDetectorUtils
     /// <param name="adjacentGoModFile">Component stream representing the adjacent go.mod file.</param>
     /// <param name="logger">The logger to use for logging messages.</param>
     /// <returns>True if the adjacent go.mod file is present and has a go version >= 1.17.</returns>
-    public static bool ShouldRemoveGoSumFromDetection(string goSumFilePath, ComponentStream adjacentGoModFile, ILogger logger)
+    public static bool ShouldIncludeGoSumFromDetection(string goSumFilePath, ComponentStream adjacentGoModFile, ILogger logger)
     {
         using var reader = new StreamReader(adjacentGoModFile.Stream);
         var goModFileContents = reader.ReadToEnd();

test/Microsoft.ComponentDetection.Detectors.Tests/Go117ComponentDetectorTests.cs

@@ -139,29 +139,57 @@ public async Task Go117ModDetector_GoModFileFound_GoModParserIsExecuted()
         goModParserMock.Verify(parser => parser.ParseAsync(It.IsAny<ISingleFileComponentRecorder>(), It.IsAny<IComponentStream>(), It.IsAny<GoGraphTelemetryRecord>()), Times.Once);
     }
 
-    [TestMethod]
-    public async Task Go117ModDetector_GoSumFileFound_GoSumParserIsExecuted()
+    /// <summary>
+    /// Verifies that if Go CLI is enabled/available and succeeds, go.sum file is not parsed and vice-versa.
+    /// </summary>
+    /// <returns>Task.</returns>
+    [DataTestMethod]
+    [DataRow(true)]
+    [DataRow(false)]
+    public async Task Go117Detector_GoSum_GoSumParserExecuted(bool goCliSucceeds)
     {
+        var nInvocationsOfSumParser = goCliSucceeds ? 0 : 1;
         var goSumParserMock = new Mock<IGoParser>();
+        var goCliParserMock = new Mock<IGoParser>();
         this.mockParserFactory.Setup(x => x.CreateParser(GoParserType.GoSum, It.IsAny<ILogger>())).Returns(goSumParserMock.Object);
+        this.mockParserFactory.Setup(x => x.CreateParser(GoParserType.GoCLI, It.IsAny<ILogger>())).Returns(goCliParserMock.Object);
 
-        this.commandLineMock.Setup(x => x.CanCommandBeLocatedAsync("go", null, null, It.Is<string[]>(p => p.SequenceEqual(new List<string> { "version" }.ToArray()))))
-        .ReturnsAsync(true);
+        // Setup go cli parser to succeed/fail
+        goCliParserMock.Setup(p => p.ParseAsync(It.IsAny<ISingleFileComponentRecorder>(), It.IsAny<IComponentStream>(), It.IsAny<GoGraphTelemetryRecord>())).ReturnsAsync(goCliSucceeds);
 
-        this.commandLineMock.Setup(x => x.ExecuteCommandAsync("go", null, null, default, It.Is<string[]>(p => p.SequenceEqual(new List<string> { "version" }.ToArray()))))
-        .ReturnsAsync(new CommandLineExecutionResult
-        {
-            ExitCode = 0,
-            StdOut = "go version go1.10.6 windows/amd64",
-        });
+        // Setup go sum parser to succeed
+        goSumParserMock.Setup(p => p.ParseAsync(It.IsAny<ISingleFileComponentRecorder>(), It.IsAny<IComponentStream>(), It.IsAny<GoGraphTelemetryRecord>())).ReturnsAsync(true);
 
         var (scanResult, componentRecorder) = await this.DetectorTestUtility
             .WithFile("go.sum", string.Empty)
             .ExecuteDetectorAsync();
 
         scanResult.ResultCode.Should().Be(ProcessingResultCode.Success);
+        this.mockParserFactory.Verify(clm => clm.CreateParser(GoParserType.GoSum, It.IsAny<ILogger>()), nInvocationsOfSumParser == 0 ? Times.Never : Times.Once);
+    }
+
+    /// <summary>
+    /// Verifies that if Go CLI is disabled, go.sum is parsed.
+    /// </summary>
+    /// <returns>Task.</returns>
+    [TestMethod]
+    public async Task Go117Detector_GoSum_GoSumParserExecutedIfCliDisabled()
+    {
+        var goSumParserMock = new Mock<IGoParser>();
+        this.mockParserFactory.Setup(x => x.CreateParser(GoParserType.GoSum, It.IsAny<ILogger>())).Returns(goSumParserMock.Object);
+
+        // Setup environment variable to disable CLI scan
+        this.envVarService.Setup(s => s.IsEnvironmentVariableValueTrue("DisableGoCliScan")).Returns(true);
+
+        // Setup go sum parser to succed
+        goSumParserMock.Setup(p => p.ParseAsync(It.IsAny<ISingleFileComponentRecorder>(), It.IsAny<IComponentStream>(), It.IsAny<GoGraphTelemetryRecord>())).ReturnsAsync(true);
 
-        goSumParserMock.Verify(parser => parser.ParseAsync(It.IsAny<ISingleFileComponentRecorder>(), It.IsAny<IComponentStream>(), It.IsAny<GoGraphTelemetryRecord>()), Times.Once);
+        var (scanResult, componentRecorder) = await this.DetectorTestUtility
+            .WithFile("go.sum", string.Empty)
+            .ExecuteDetectorAsync();
+
+        scanResult.ResultCode.Should().Be(ProcessingResultCode.Success);
+        this.mockParserFactory.Verify(clm => clm.CreateParser(GoParserType.GoSum, It.IsAny<ILogger>()), Times.Once);
     }
 
     [TestMethod]