Load analyzers and generators in isolated ALCs in our OOP process

src/LanguageServer/Microsoft.CodeAnalysis.LanguageServer/HostWorkspace/LanguageServerWorkspaceFactory.cs

@@ -57,7 +57,12 @@ public LanguageServerWorkspaceFactory(
     public async Task InitializeSolutionLevelAnalyzersAsync(ImmutableArray<string> analyzerPaths)
     {
         var references = new List<AnalyzerFileReference>();
-        var analyzerLoader = Workspace.Services.GetRequiredService<IAnalyzerAssemblyLoaderProvider>().SharedShadowCopyLoader;
+        var loaderProvider = Workspace.Services.GetRequiredService<IAnalyzerAssemblyLoaderProvider>();
+
+        // Load all analyzers into a fresh shadow copied load context.  In the future, if we want to support reloading
+        // of solution-level analyzer references, we should just need to listen for changes to those analyzer paths and
+        // then call back into this method to update the solution accordingly.
+        var analyzerLoader = loaderProvider.CreateNewShadowCopyLoader();
 
         foreach (var analyzerPath in analyzerPaths)
         {

src/VisualStudio/Core/Test.Next/Remote/SerializationValidator.cs

@@ -110,7 +110,8 @@ public async Task<T> GetValueAsync<T>(Checksum checksum)
 
         public async Task<Solution> GetSolutionAsync(SolutionAssetStorage.Scope scope)
         {
-            var solutionInfo = await new AssetProvider(this).CreateSolutionInfoAsync(scope.SolutionChecksum, CancellationToken.None).ConfigureAwait(false);
+            var solutionInfo = await new AssetProvider(this).CreateSolutionInfoAsync(
+                scope.SolutionChecksum, this.Services.SolutionServices, CancellationToken.None).ConfigureAwait(false);
 
             var workspace = new AdhocWorkspace(Services.HostServices);
             return workspace.AddSolution(solutionInfo);

src/VisualStudio/Core/Test.Next/Services/SolutionServiceTests.cs

@@ -15,6 +15,7 @@
 using Microsoft.CodeAnalysis.Diagnostics;
 using Microsoft.CodeAnalysis.Editor.Test;
 using Microsoft.CodeAnalysis.Formatting;
+using Microsoft.CodeAnalysis.Host;
 using Microsoft.CodeAnalysis.Remote;
 using Microsoft.CodeAnalysis.Remote.Testing;
 using Microsoft.CodeAnalysis.Serialization;
@@ -446,7 +447,10 @@ public async Task TestRemoteWorkspace()
         await Verify(remoteWorkspace, currentSolution, remoteSolution3);
 
         // move to new solution backward
-        var solutionInfo2 = await assetProvider.CreateSolutionInfoAsync(await solution1.CompilationState.GetChecksumAsync(CancellationToken.None), CancellationToken.None);
+        var solutionInfo2 = await assetProvider.CreateSolutionInfoAsync(
+            await solution1.CompilationState.GetChecksumAsync(CancellationToken.None),
+            remoteWorkspace.Services.SolutionServices,
+            CancellationToken.None);
         var solution2 = remoteWorkspace.GetTestAccessor().CreateSolutionFromInfo(solutionInfo2);
 
         // move to new solution forward

src/Workspaces/Core/Portable/Serialization/SerializerService_Reference.cs

@@ -8,7 +8,6 @@
 using System.Diagnostics.CodeAnalysis;
 using System.IO;
 using System.Reflection.Metadata;
-using System.Threading;
 using Microsoft.CodeAnalysis.Diagnostics;
 using Microsoft.CodeAnalysis.Host;
 using Microsoft.CodeAnalysis.Shared.Extensions;
@@ -54,15 +53,24 @@ private static Checksum CreateChecksum(MetadataReference reference)
 
     protected virtual Checksum CreateChecksum(AnalyzerReference reference)
     {
+#if NET
+        // If we're in the oop side and we're being asked to produce our local checksum (so we can compare it to the
+        // host checksum), then we want to just defer to the underlying analyzer reference of our isolated reference.
+        // This underlying reference corresponds to the reference that the host has, and we do not want to make any
+        // changes as long as they're both in agreement.
+        if (reference is IsolatedAnalyzerFileReference { UnderlyingAnalyzerFileReference: var underlyingReference })
+            reference = underlyingReference;
+#endif
+
         using var stream = SerializableBytes.CreateWritableStream();
 
         using (var writer = new ObjectWriter(stream, leaveOpen: true))
         {
             switch (reference)
             {
-                case AnalyzerFileReference file:
-                    writer.WriteString(file.FullPath);
-                    writer.WriteGuid(TryGetAnalyzerFileReferenceMvid(file));
+                case AnalyzerFileReference fileReference:
+                    writer.WriteString(fileReference.FullPath);
+                    writer.WriteGuid(TryGetAnalyzerFileReferenceMvid(fileReference));
                     break;
 
                 case AnalyzerImageReference analyzerImageReference:
@@ -109,11 +117,11 @@ protected virtual void WriteAnalyzerReferenceTo(AnalyzerReference reference, Obj
     {
         switch (reference)
         {
-            case AnalyzerFileReference file:
+            case AnalyzerFileReference fileReference:
                 writer.WriteString(nameof(AnalyzerFileReference));
-                writer.WriteString(file.FullPath);
+                writer.WriteString(fileReference.FullPath);
 
-                // Note: it is intentional that we are not writing the MVID of the analyzer file reference over (even
+                // Note: it is intentional that we are not writing the MVID of the analyzer file reference over in (even
                 // though we mixed it into the checksum).  We don't actually need the data on the other side as it will
                 // be read out from the file itself.  So the flow is as follows when an analyzer-file-reference changes:
                 //
@@ -150,8 +158,10 @@ protected virtual AnalyzerReference ReadAnalyzerReferenceFrom(ObjectReader reade
         switch (reader.ReadString())
         {
             case nameof(AnalyzerFileReference):
-                var fullPath = reader.ReadRequiredString();
-                return new AnalyzerFileReference(fullPath, _analyzerLoaderProvider.SharedShadowCopyLoader);
+                // Rehydrate the analyzer file reference with the simple shared shadow copy loader.  Note: we won't
+                // actually use this instance we create.  Instead, the caller will use create an IsolatedAssemblyReferenceSet
+                // from these to ensure that all the types can be safely loaded into their own ALC.
+                return new AnalyzerFileReference(reader.ReadRequiredString(), _analyzerLoaderProvider.SharedShadowCopyLoader);
 
             case nameof(AnalyzerImageReference):
                 var guid = reader.ReadGuid();
@@ -286,7 +296,7 @@ private PortableExecutableReference ReadPortableExecutableReferenceFrom(ObjectRe
         // so that we can put xml doc comment as part of snapshot. but until we believe that is necessary,
         // it will go with simpler approach
         var documentProvider = filePath != null && _documentationService != null ?
-            _documentationService.GetDocumentationProvider(filePath) : XmlDocumentationProvider.Default;
+            _documentationService.GetDocumentationProvider(filePath) : DocumentationProvider.Default;
 
         return new SerializedPortableExecutableReference(
             properties, filePath, metadata, storageHandles, documentProvider);

src/Workspaces/Core/Portable/Workspace/Host/Metadata/IAnalyzerAssemblyLoaderProvider.cs

@@ -9,11 +9,24 @@
 using System.IO;
 using Microsoft.CodeAnalysis.Host.Mef;
 
+#if NET
+using Microsoft.CodeAnalysis.Diagnostics;
+using System.Runtime.Loader;
+#endif
+
 namespace Microsoft.CodeAnalysis.Host;
 
 internal interface IAnalyzerAssemblyLoaderProvider : IWorkspaceService
 {
     IAnalyzerAssemblyLoaderInternal SharedShadowCopyLoader { get; }
+
+#if NET
+    /// <summary>
+    /// Creates a fresh shadow copying loader that will load all <see cref="AnalyzerReference"/>s and <see
+    /// cref="ISourceGenerator"/>s in a fresh <see cref="AssemblyLoadContext"/>.
+    /// </summary>
+    IAnalyzerAssemblyLoaderInternal CreateNewShadowCopyLoader();
+#endif
 }
 
 /// <summary>
@@ -28,13 +41,13 @@ internal abstract class AbstractAnalyzerAssemblyLoaderProvider : IAnalyzerAssemb
     public AbstractAnalyzerAssemblyLoaderProvider(IEnumerable<IAnalyzerAssemblyResolver> externalResolvers)
     {
         _externalResolvers = externalResolvers.ToImmutableArray();
-        _shadowCopyLoader = new(CreateShadowCopyLoader);
+        _shadowCopyLoader = new(CreateNewShadowCopyLoader);
     }
 
     public IAnalyzerAssemblyLoaderInternal SharedShadowCopyLoader
         => _shadowCopyLoader.Value;
 
-    private IAnalyzerAssemblyLoaderInternal CreateShadowCopyLoader()
+    public IAnalyzerAssemblyLoaderInternal CreateNewShadowCopyLoader()
         => this.WrapLoader(DefaultAnalyzerAssemblyLoader.CreateNonLockingLoader(
                 Path.Combine(Path.GetTempPath(), nameof(Roslyn), "AnalyzerAssemblyLoader"),
                 _externalResolvers));

src/Workspaces/Core/Portable/Workspace/IsolatedAnalyzerFileReference.cs

@@ -0,0 +1,114 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+#if NET
+
+using System;
+using System.Collections.Immutable;
+using System.Runtime.CompilerServices;
+using Microsoft.CodeAnalysis.Diagnostics;
+using System.Runtime.Loader;
+using Roslyn.Utilities;
+
+namespace Microsoft.CodeAnalysis;
+
+/// <summary>
+/// Wrapper around a real <see cref="AnalyzerFileReference"/>.  An "isolated" analyzer reference is an analyzer
+/// reference associated with an <see cref="AssemblyLoadContext"/> that is connected to a set of other "isolated"
+/// analyzer references.  This allows for loading the analyzers and generators from it in a way that is associated with
+/// that load context, keeping them separate from other analyzers and generators loaded in other load contexts, while
+/// also allowing all of those instances to be collected when no longer needed.  Being isolated means that if any of the
+/// underlying assembly references change, that they can be loaded side by side with the prior references.  This enables
+/// functionality like live reloading of analyzers and generators when they change on disk.  Note: this is only
+/// supported on .Net Core, and not .Net Framework, as only the former has <see cref="AssemblyLoadContext"/>s.
+/// </summary>
+/// <remarks>
+/// The purpose of this type is to allow passing out a <see cref="AnalyzerReference"/> to the rest of the system that
+/// then ensures that as long as it is alive (or any <see cref="DiagnosticAnalyzer"/> or <see cref="ISourceGenerator"/>
+/// it passes out is alive), that the <see cref="IsolatedAnalyzerReferenceSet"/> (and its corresponding <see
+/// cref="AssemblyLoadContext"/>) is kept alive as well.
+/// </remarks>
+internal sealed class IsolatedAnalyzerFileReference(
+    IsolatedAnalyzerReferenceSet isolatedAnalyzerReferenceSet,
+    AnalyzerFileReference underlyingAnalyzerReference)
+    : AnalyzerReference
+{
+    /// <summary>
+    /// Conditional weak tables that ensure that as long as a particular <see cref="DiagnosticAnalyzer"/> or <see
+    /// cref="ISourceGenerator"/> is alive, that the corresponding <see cref="IsolatedAnalyzerReferenceSet"/> (and its
+    /// corresponding <see cref="AssemblyLoadContext"/> is kept alive.
+    /// </summary>
+    private static readonly ConditionalWeakTable<DiagnosticAnalyzer, IsolatedAnalyzerReferenceSet> s_analyzerToPinnedReferenceSet = [];
+
+    /// <inheritdoc cref="s_analyzerToPinnedReferenceSet"/>
+    private static readonly ConditionalWeakTable<ISourceGenerator, IsolatedAnalyzerReferenceSet> s_generatorToPinnedReferenceSet = [];
+
+    /// <summary>
+    /// We keep a strong reference here.  As long as this <see cref="IsolatedAnalyzerFileReference"/> is passed out and
+    /// held onto (say by a Project instance), it should keep the IsolatedAssemblyReferenceSet (and its ALC) alive.
+    /// </summary>
+    private readonly IsolatedAnalyzerReferenceSet _isolatedAnalyzerReferenceSet = isolatedAnalyzerReferenceSet;
+
+    /// <summary>
+    /// The actual real <see cref="AnalyzerReference"/> we defer our operations to.
+    /// </summary>
+    public readonly AnalyzerFileReference UnderlyingAnalyzerFileReference = underlyingAnalyzerReference;
+
+    public override string Display => UnderlyingAnalyzerFileReference.Display;
+    public override string? FullPath => UnderlyingAnalyzerFileReference.FullPath;
+    public override object Id => UnderlyingAnalyzerFileReference.Id;
+
+    public override ImmutableArray<DiagnosticAnalyzer> GetAnalyzers(string language)
+        => PinAnalyzers(static (reference, language) => reference.GetAnalyzers(language), language);
+
+    public override ImmutableArray<DiagnosticAnalyzer> GetAnalyzersForAllLanguages()
+        => PinAnalyzers(static (reference, _) => reference.GetAnalyzersForAllLanguages(), default(VoidResult));
+
+    [Obsolete]
+    public override ImmutableArray<ISourceGenerator> GetGenerators()
+        => PinGenerators(static (reference, _) => reference.GetGenerators(), default(VoidResult));
+
+    public override ImmutableArray<ISourceGenerator> GetGenerators(string language)
+        => PinGenerators(static (reference, language) => reference.GetGenerators(language), language);
+
+    public override ImmutableArray<ISourceGenerator> GetGeneratorsForAllLanguages()
+        => PinGenerators(static (reference, _) => reference.GetGeneratorsForAllLanguages(), default(VoidResult));
+
+    private ImmutableArray<DiagnosticAnalyzer> PinAnalyzers<TArg>(Func<AnalyzerReference, TArg, ImmutableArray<DiagnosticAnalyzer>> getItems, TArg arg)
+        => PinItems(s_analyzerToPinnedReferenceSet, getItems, arg);
+
+    private ImmutableArray<ISourceGenerator> PinGenerators<TArg>(Func<AnalyzerReference, TArg, ImmutableArray<ISourceGenerator>> getItems, TArg arg)
+        => PinItems(s_generatorToPinnedReferenceSet, getItems, arg);
+
+    private ImmutableArray<TItem> PinItems<TItem, TArg>(
+        ConditionalWeakTable<TItem, IsolatedAnalyzerReferenceSet> table,
+        Func<AnalyzerReference, TArg, ImmutableArray<TItem>> getItems,
+        TArg arg)
+        where TItem : class
+    {
+        // Keep a reference from each generator to the IsolatedAssemblyReferenceSet.  This will ensure it (and the ALC
+        // it points at) stays alive as long as the generator instance stays alive.
+        var items = getItems(this.UnderlyingAnalyzerFileReference, arg);
+
+        foreach (var item in items)
+            table.TryAdd(item, _isolatedAnalyzerReferenceSet);
+
+        // Note: we want to keep ourselves alive during this call so that neither we nor our reference set get GC'ed
+        // while we're computing the items.
+        GC.KeepAlive(this);
+
+        return items;
+    }
+
+    public override bool Equals(object? obj)
+        => ReferenceEquals(this, obj);
+
+    public override int GetHashCode()
+        => RuntimeHelpers.GetHashCode(this);
+
+    public override string ToString()
+        => $"{nameof(IsolatedAnalyzerFileReference)}({UnderlyingAnalyzerFileReference})";
+}
+
+#endif