File based programs IDE support

.vscode/tasks.json

@@ -169,7 +169,7 @@
       "type": "process",
       "options": {
         "env": {
-          "DOTNET_ROSLYN_SERVER_PATH": "${workspaceRoot}/artifacts/bin/Microsoft.CodeAnalysis.LanguageServer/Debug/net8.0/Microsoft.CodeAnalysis.LanguageServer.dll"
+          "DOTNET_ROSLYN_SERVER_PATH": "${workspaceRoot}/artifacts/bin/Microsoft.CodeAnalysis.LanguageServer/Debug/net9.0/Microsoft.CodeAnalysis.LanguageServer.dll"
         }
       },
       "dependsOn": [ "build language server" ]

docs/features/file-based-programs-vscode.md

@@ -0,0 +1,67 @@
+# File-based programs VS Code support
+
+See also [dotnet-run-file.md](https://github.com/dotnet/sdk/blob/main/documentation/general/dotnet-run-file.md).
+
+## Feature overview
+
+A file-based program embeds a subset of MSBuild project capabilities into C# code, allowing single files to stand alone as ordinary projects.
+
+The following is a file-based program:
+
+```cs
+Console.WriteLine("Hello World!");
+```
+
+So is the following:
+
+```cs
+#!/usr/bin/env dotnet run
+#:sdk Microsoft.Net.Sdk
+#:package [email protected]
+#:property LangVersion=preview
+
+using Newtonsoft.Json;
+
+Main();
+
+void Main()
+{
+    if (args is not [_, var jsonPath, ..])
+    {
+        Console.Error.WriteLine("Usage: app <json-file>");
+        return;
+    }
+
+    var json = File.ReadAllText(jsonPath);
+    var data = JsonConvert.DeserializeObject<Data>(json);
+    // ...
+}
+
+record Data(string field1, int field2);
+```
+
+This basically works by having the `dotnet` command line interpret the `#:` directives in source files, produce a C# project XML document in memory, and pass it off to MSBuild. The in-memory project is sometimes called a "virtual project".
+
+## Miscellaneous files changes
+
+There is a long-standing backlog item to enhance the experience of working with miscellaneous files ("loose files" not associated with any project). We think that as part of the "file-based program" work, we can enable the following in such files without substantial issues:
+- Syntax diagnostics.
+- Intellisense for the "default" set of references. e.g. those references which are included in the project created by `dotnet new console` with the current SDK.
+
+## High-level IDE layer changes
+
+The following outline is for "single-file" scenarios. We are interested in expanding to multi-file as well, and this document will be expanded in the future with additional explanation of how to handle that scenario.
+
+### Prerequisite work
+- MSBuildHost is updated to allow passing project content along with a file path. (https://github.com/dotnet/roslyn/pull/78303)
+- LanguageServerProjectSystem has a base type extracted so that a FileBasedProgramProjectSystem can be added. The latter handles loading file-based program projects, and makes the appropriate call to obtain the file-based program project XML. (https://github.com/dotnet/roslyn/pull/78329)
+
+### Heuristic
+The IDE considers a file to be a file-based program, if:
+- It has any `#:` directives which configure the file-based program project, or,
+- It has any top-level statements.
+- Any of the above is met, and, the file is not included in an ordinary `.csproj` project (i.e. it is not part of any ordinary project's list of `Compile` items).
+
+### Opt-out
+
+We added an opt-out flag with option name `dotnet.projects.enableFileBasedPrograms`. If issues arise with the file-based program experience, then VS Code users should set the corresponding setting `"dotnet.projects.enableFileBasedPrograms": false` to revert back to the old miscellaneous files experience.

src/LanguageServer/Microsoft.CodeAnalysis.LanguageServer.UnitTests/WorkspaceProjectFactoryServiceTests.cs

@@ -48,7 +48,7 @@ public async Task CreateProjectAndBatch()
         await batch.ApplyAsync(CancellationToken.None);
 
         // Verify it actually did something; we won't exclusively test each method since those are tested at lower layers
-        var project = workspaceFactory.Workspace.CurrentSolution.Projects.Single();
+        var project = workspaceFactory.HostWorkspace.CurrentSolution.Projects.Single();
 
         var document = Assert.Single(project.Documents);
         Assert.Equal(sourceFilePath, document.FilePath);

src/LanguageServer/Microsoft.CodeAnalysis.LanguageServer/HostWorkspace/FileBasedProgramsProjectSystem.cs

@@ -0,0 +1,183 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System.Collections.Immutable;
+using System.Security;
+using Microsoft.CodeAnalysis.Features.Workspaces;
+using Microsoft.CodeAnalysis.Host;
+using Microsoft.CodeAnalysis.LanguageServer.HostWorkspace.ProjectTelemetry;
+using Microsoft.CodeAnalysis.MetadataAsSource;
+using Microsoft.CodeAnalysis.MSBuild;
+using Microsoft.CodeAnalysis.Options;
+using Microsoft.CodeAnalysis.ProjectSystem;
+using Microsoft.CodeAnalysis.Shared.Extensions;
+using Microsoft.CodeAnalysis.Shared.TestHooks;
+using Microsoft.CodeAnalysis.Text;
+using Microsoft.CodeAnalysis.Workspaces.ProjectSystem;
+using Microsoft.CommonLanguageServerProtocol.Framework;
+using Microsoft.Extensions.Logging;
+using Microsoft.VisualStudio.Composition;
+using Roslyn.LanguageServer.Protocol;
+using Roslyn.Utilities;
+using static Microsoft.CodeAnalysis.MSBuild.BuildHostProcessManager;
+
+namespace Microsoft.CodeAnalysis.LanguageServer.HostWorkspace;
+
+internal sealed class FileBasedProgramsProjectSystem : LanguageServerProjectLoader, ILspMiscellaneousFilesWorkspaceProvider
+{
+    private readonly ILspServices _lspServices;
+    private readonly ILogger<FileBasedProgramsProjectSystem> _logger;
+    private readonly IMetadataAsSourceFileService _metadataAsSourceFileService;
+
+    public FileBasedProgramsProjectSystem(
+        ILspServices lspServices,
+        IMetadataAsSourceFileService metadataAsSourceFileService,
+        LanguageServerWorkspaceFactory workspaceFactory,
+        IFileChangeWatcher fileChangeWatcher,
+        IGlobalOptionService globalOptionService,
+        ILoggerFactory loggerFactory,
+        IAsynchronousOperationListenerProvider listenerProvider,
+        ProjectLoadTelemetryReporter projectLoadTelemetry,
+        ServerConfigurationFactory serverConfigurationFactory,
+        BinlogNamer binlogNamer)
+            : base(
+                workspaceFactory.FileBasedProgramsProjectFactory,
+                workspaceFactory.TargetFrameworkManager,
+                workspaceFactory.ProjectSystemHostInfo,
+                fileChangeWatcher,
+                globalOptionService,
+                loggerFactory,
+                listenerProvider,
+                projectLoadTelemetry,
+                serverConfigurationFactory,
+                binlogNamer)
+    {
+        _lspServices = lspServices;
+        _logger = loggerFactory.CreateLogger<FileBasedProgramsProjectSystem>();
+        _metadataAsSourceFileService = metadataAsSourceFileService;
+    }
+
+    public Workspace Workspace => ProjectFactory.Workspace;
+
+    public async Task<TextDocument?> AddMiscellaneousDocumentAsync(DocumentUri uri, SourceText documentText, string languageId, ILspLogger logger)
+    {
+        var (documentPath, isFile) = uri.ParsedUri is { } parsedUri
+            ? (ProtocolConversions.GetDocumentFilePathFromUri(parsedUri), isFile: parsedUri.IsFile)
+            : (uri.UriString, isFile: false);
+
+        var container = documentText.Container;
+        if (_metadataAsSourceFileService.TryAddDocumentToWorkspace(documentPath, container, out var documentId))
+        {
+            var metadataWorkspace = _metadataAsSourceFileService.TryGetWorkspace();
+            Contract.ThrowIfNull(metadataWorkspace);
+            var metadataDoc = metadataWorkspace.CurrentSolution.GetRequiredDocument(documentId);
+            return metadataDoc;
+        }
+
+        var languageInfoProvider = _lspServices.GetRequiredService<ILanguageInfoProvider>();
+        if (!languageInfoProvider.TryGetLanguageInformation(uri, languageId, out var languageInformation))
+        {
+            // Only log here since throwing here could take down the LSP server.
+            logger.LogError($"Could not find language information for {uri} with absolute path {documentPath}");
+            return null;
+        }
+
+        if (!isFile || !GlobalOptionService.GetOption(LanguageServerProjectSystemOptionsStorage.EnableFileBasedPrograms))
+        {
+            // For now, we cannot provide intellisense etc on files which are not on disk or are not C#.
+            var sourceTextLoader = new SourceTextLoader(documentText, documentPath);
+            var projectInfo = MiscellaneousFileUtilities.CreateMiscellaneousProjectInfoForDocument(
+                Workspace, documentPath, sourceTextLoader, languageInformation, documentText.ChecksumAlgorithm, Workspace.CurrentSolution.Services, []);
+            await ProjectFactory.ApplyChangeToWorkspaceAsync(ws => ws.OnProjectAdded(projectInfo), cancellationToken: default);
+
+            var newSolution = Workspace.CurrentSolution;
+            if (languageInformation.LanguageName == "Razor")
+            {
+                var docId = projectInfo.AdditionalDocuments.Single().Id;
+                return newSolution.GetRequiredAdditionalDocument(docId);
+            }
+
+            var id = projectInfo.Documents.Single().Id;
+            return newSolution.GetRequiredDocument(id);
+        }
+
+        // We have a file on disk. Light up the file-based program experience.
+        // For Razor files we need to override the language name to C# as that's what code is generated
+        var isRazor = languageInformation.LanguageName == "Razor";
+        var languageName = isRazor ? LanguageNames.CSharp : languageInformation.LanguageName;
+        var documentFileInfo = new DocumentFileInfo(documentPath, logicalPath: documentPath, isLinked: false, isGenerated: false, folders: default);
+        var projectFileInfo = new ProjectFileInfo()
+        {
+            Language = languageName,
+            FilePath = VirtualProject.GetVirtualProjectPath(documentPath),
+            CommandLineArgs = ["/langversion:preview", "/features:FileBasedProgram=true"],
+            Documents = isRazor ? [] : [documentFileInfo],
+            AdditionalDocuments = isRazor ? [documentFileInfo] : [],
+            AnalyzerConfigDocuments = [],
+            ProjectReferences = [],
+            PackageReferences = [],
+            ProjectCapabilities = [],
+            ContentFilePaths = [],
+            FileGlobs = []
+        };
+
+        var projectSet = AddLoadedProjectSet(documentPath);
+        Project workspaceProject;
+        using (await projectSet.Semaphore.DisposableWaitAsync())
+        {
+            var loadedProject = await this.CreateAndTrackInitialProjectAsync_NoLock(projectSet, documentPath, language: languageName);
+            await loadedProject.UpdateWithNewProjectInfoAsync(projectFileInfo, hasAllInformation: false, _logger);
+
+            ProjectsToLoadAndReload.AddWork(new ProjectToLoad(documentPath, ProjectGuid: null, ReportTelemetry: true));
+            loadedProject.NeedsReload += (_, _) => ProjectsToLoadAndReload.AddWork(new ProjectToLoad(documentPath, ProjectGuid: null, ReportTelemetry: false));
+            workspaceProject = ProjectFactory.Workspace.CurrentSolution.GetRequiredProject(loadedProject.ProjectId);
+        }
+
+        var document = isRazor ? workspaceProject.AdditionalDocuments.Single() : workspaceProject.Documents.Single();
+
+        _ = Task.Run(async () =>
+        {
+            await ProjectsToLoadAndReload.WaitUntilCurrentBatchCompletesAsync();
+            await ProjectInitializationHandler.SendProjectInitializationCompleteNotificationAsync();
+        });
+
+        Contract.ThrowIfFalse(document.FilePath == documentPath);
+        return document;
+    }
+
+    public async ValueTask TryRemoveMiscellaneousDocumentAsync(DocumentUri uri, bool removeFromMetadataWorkspace)
+    {
+        var documentPath = uri.ParsedUri is { } parsedUri ? ProtocolConversions.GetDocumentFilePathFromUri(parsedUri) : uri.UriString;
+        await TryUnloadProjectSetAsync(documentPath);
+
+        // also do an unload in case this was the non-file scenario
+        if (removeFromMetadataWorkspace && uri.ParsedUri is not null && _metadataAsSourceFileService.TryRemoveDocumentFromWorkspace(ProtocolConversions.GetDocumentFilePathFromUri(uri.ParsedUri)))
+        {
+            return;
+        }
+
+        var matchingDocument = Workspace.CurrentSolution.GetDocumentIds(uri).SingleOrDefault();
+        if (matchingDocument != null)
+        {
+            var project = Workspace.CurrentSolution.GetRequiredProject(matchingDocument.ProjectId);
+            Workspace.OnProjectRemoved(project.Id);
+        }
+    }
+
+    protected override async Task<(RemoteProjectFile? projectFile, bool hasAllInformation, BuildHostProcessKind preferred, BuildHostProcessKind actual)> TryLoadProjectAsync(
+        BuildHostProcessManager buildHostProcessManager, string documentPath, CancellationToken cancellationToken)
+    {
+        const BuildHostProcessKind buildHostKind = BuildHostProcessKind.NetCore;
+        var buildHost = await buildHostProcessManager.GetBuildHostAsync(buildHostKind, cancellationToken);
+        Contract.ThrowIfFalse(Path.GetExtension(documentPath) == ".cs");
+
+        var fakeProjectPath = VirtualProject.GetVirtualProjectPath(documentPath);
+        var loader = ProjectFactory.CreateFileTextLoader(documentPath);
+        var textAndVersion = await loader.LoadTextAsync(new LoadTextOptions(SourceHashAlgorithms.Default), cancellationToken: default);
+        var (contentToLoad, isFileBasedProgram) = VirtualProject.MakeVirtualProjectContent(documentPath, textAndVersion.Text);
+
+        var loadedFile = await buildHost.LoadProjectAsync(fakeProjectPath, contentToLoad, languageName: LanguageNames.CSharp, cancellationToken);
+        return (loadedFile, hasAllInformation: isFileBasedProgram, preferred: buildHostKind, actual: buildHostKind);
+    }
+}

src/LanguageServer/Microsoft.CodeAnalysis.LanguageServer/HostWorkspace/FileBasedProgramsWorkspaceProviderFactory.cs

@@ -0,0 +1,42 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System.Composition;
+using Microsoft.CodeAnalysis.Host;
+using Microsoft.CodeAnalysis.Host.Mef;
+using Microsoft.CodeAnalysis.LanguageServer.Handler;
+using Microsoft.CodeAnalysis.LanguageServer.HostWorkspace.ProjectTelemetry;
+using Microsoft.CodeAnalysis.MetadataAsSource;
+using Microsoft.CodeAnalysis.Options;
+using Microsoft.CodeAnalysis.ProjectSystem;
+using Microsoft.CodeAnalysis.Shared.TestHooks;
+using Microsoft.CommonLanguageServerProtocol.Framework;
+using Microsoft.Extensions.Logging;
+
+namespace Microsoft.CodeAnalysis.LanguageServer.HostWorkspace;
+
+/// <summary>
+/// Service to create <see cref="LspMiscellaneousFilesWorkspaceProvider"/> instances.
+/// This is not exported as a <see cref="ILspServiceFactory"/> as it requires
+/// special base language server dependencies such as the <see cref="HostServices"/>
+/// </summary>
+[ExportCSharpVisualBasicStatelessLspService(typeof(ILspMiscellaneousFilesWorkspaceProviderFactory), WellKnownLspServerKinds.CSharpVisualBasicLspServer), Shared]
+[method: ImportingConstructor]
+[method: Obsolete(MefConstruction.ImportingConstructorMessage, error: true)]
+internal sealed class FileBasedProgramsWorkspaceProviderFactory(
+    IMetadataAsSourceFileService metadataAsSourceFileService,
+    LanguageServerWorkspaceFactory workspaceFactory,
+    IFileChangeWatcher fileChangeWatcher,
+    IGlobalOptionService globalOptionService,
+    ILoggerFactory loggerFactory,
+    IAsynchronousOperationListenerProvider listenerProvider,
+    ProjectLoadTelemetryReporter projectLoadTelemetry,
+    ServerConfigurationFactory serverConfigurationFactory,
+    BinlogNamer binlogNamer) : ILspMiscellaneousFilesWorkspaceProviderFactory
+{
+    public ILspMiscellaneousFilesWorkspaceProvider CreateLspMiscellaneousFilesWorkspaceProvider(ILspServices lspServices, HostServices hostServices)
+    {
+        return new FileBasedProgramsProjectSystem(lspServices, metadataAsSourceFileService, workspaceFactory, fileChangeWatcher, globalOptionService, loggerFactory, listenerProvider, projectLoadTelemetry, serverConfigurationFactory, binlogNamer);
+    }
+}