Merge pull request #22151 from heejaechang/handleTimeout

made timeout to 7 days so that sleep won't affect it. for master, we …

Author: heejaechang

src/VisualStudio/Core/Def/Implementation/Remote/RemoteHostOptions.cs

@@ -34,9 +34,12 @@ internal static class RemoteHostOptions
         // this is our timeout on how long we will try keep connecting. so far I saw over 2-3 seconds before connection made 
         // when there are many (over 10+ requests) at the same time. one of reasons of this is we put our service hub process as "Below Normal" priority.
         // normally response time is within 10s ms. at most 100ms. if priority is changed to "Normal", most of time 10s ms.
+        // 
+        // also another reason why timeout is so big is that, if user put this computer sleep, then timeout can happen. so for now, until we have
+        // sleep aware timer, we put very long timeout for request service (https://github.com/dotnet/roslyn/pull/22151)
         [ExportOption]
         public static readonly Option<int> RequestServiceTimeoutInMS = new Option<int>(
-            nameof(InternalFeatureOnOffOptions), nameof(RequestServiceTimeoutInMS), defaultValue: 10 * 60 * 1000,
+            nameof(InternalFeatureOnOffOptions), nameof(RequestServiceTimeoutInMS), defaultValue: 7 * 24 * 60 * 60 * 1000 /* 7 days */,
             storageLocations: new LocalUserProfileStorageLocation(InternalFeatureOnOffOptions.LocalRegistryPath + nameof(RequestServiceTimeoutInMS)));
 
         // This options allow users to restart OOP when it is killed by users

src/VisualStudio/Core/Next/Remote/ServiceHubRemoteHostClient.cs

@@ -9,6 +9,7 @@
 using Microsoft.CodeAnalysis.Editor.Shared.Utilities;
 using Microsoft.CodeAnalysis.ErrorReporting;
 using Microsoft.CodeAnalysis.Execution;
+using Microsoft.CodeAnalysis.Extensions;
 using Microsoft.CodeAnalysis.Internal.Log;
 using Microsoft.CodeAnalysis.Remote;
 using Microsoft.ServiceHub.Client;
@@ -175,27 +176,46 @@ private static async Task<TResult> RetryRemoteCallAsync<TException, TResult>(
         {
             const int retry_delayInMS = 50;
 
-            var start = DateTime.UtcNow;
-            while (DateTime.UtcNow - start < timeout)
+            using (var pooledStopwatch = SharedPools.Default<Stopwatch>().GetPooledObject())
             {
-                cancellationToken.ThrowIfCancellationRequested();
+                var watch = pooledStopwatch.Object;
+                watch.Start();
 
-                try
+                while (watch.Elapsed < timeout)
                 {
-                    return await funcAsync().ConfigureAwait(false);
-                }
-                catch (TException)
-                {
-                    // throw cancellation token if operation is cancelled
                     cancellationToken.ThrowIfCancellationRequested();
-                }
 
-                // wait for retry_delayInMS before next try
-                await Task.Delay(retry_delayInMS, cancellationToken).ConfigureAwait(false);
+                    try
+                    {
+                        return await funcAsync().ConfigureAwait(false);
+                    }
+                    catch (TException)
+                    {
+                        // throw cancellation token if operation is cancelled
+                        cancellationToken.ThrowIfCancellationRequested();
+                    }
+
+                    // wait for retry_delayInMS before next try
+                    await Task.Delay(retry_delayInMS, cancellationToken).ConfigureAwait(false);
+
+                    ReportTimeout(watch);
+                }
             }
 
             // operation timed out, more than we are willing to wait
-            throw new TimeoutException("RequestServiceAsync timed out");
+            ShowInfoBar();
+
+            // user didn't ask for cancellation, but we can't fullfill this request. so we
+            // create our own cancellation token and then throw it. this doesn't guarantee
+            // 100% that we won't crash, but this is at least safest way we know until user
+            // restart VS (with info bar)
+            using (var ownCancellationSource = new CancellationTokenSource())
+            {
+                ownCancellationSource.Cancel();
+                ownCancellationSource.Token.ThrowIfCancellationRequested();
+            }
+
+            throw ExceptionUtilities.Unreachable;
         }
 
         private static async Task<Stream> RequestServiceAsync(
@@ -256,6 +276,7 @@ private static async Task<Stream> RequestServiceAsync(
             throw ExceptionUtilities.Unreachable;
         }
 
+        #region code related to make diagnosis easier later
         private static int ReportDetailInfo(IFaultUtility faultUtility)
         {
             // 0 means send watson, otherwise, cancel watson
@@ -264,6 +285,14 @@ private static int ReportDetailInfo(IFaultUtility faultUtility)
 
             try
             {
+                // add service hub process.
+                // we will record dumps for all service hub processes
+                foreach (var p in Process.GetProcessesByName("ServiceHub.RoslynCodeAnalysisService32"))
+                {
+                    // include all remote host processes
+                    faultUtility.AddProcessDump(p.Id);
+                }
+
                 var logPath = Path.Combine(Path.GetTempPath(), "servicehub", "logs");
                 if (!Directory.Exists(logPath))
                 {
@@ -303,5 +332,37 @@ private static bool ReportNonIOException(Exception ex)
             // catch all exception. not worth crashing VS.
             return true;
         }
+
+        private static readonly TimeSpan s_reportTimeout = TimeSpan.FromMinutes(10);
+        private static bool s_timeoutReported = false;
+
+        private static void ReportTimeout(Stopwatch watch)
+        {
+            // if we tried for 10 min and still couldn't connect. NFW (non fatal watson) some data
+            if (!s_timeoutReported && watch.Elapsed > s_reportTimeout)
+            {
+                s_timeoutReported = true;
+
+                // report service hub logs along with dump
+                WatsonReporter.Report("RequestServiceAsync Timeout", new Exception("RequestServiceAsync Timeout"), ReportDetailInfo);
+            }
+        }
+
+        private static bool s_infoBarReported = false;
+
+        private static void ShowInfoBar()
+        {
+            // use info bar to show warning to users
+            if (CodeAnalysis.PrimaryWorkspace.Workspace != null && !s_infoBarReported)
+            {
+                // do not report it multiple times
+                s_infoBarReported = true;
+
+                // use info bar to show warning to users
+                CodeAnalysis.PrimaryWorkspace.Workspace.Services.GetService<IErrorReportingService>()?.ShowGlobalErrorInfo(
+                    ServicesVSResources.Unfortunately_a_process_used_by_Visual_Studio_has_encountered_an_unrecoverable_error_We_recommend_saving_your_work_and_then_closing_and_restarting_Visual_Studio);
+            }
+        }
+        #endregion
     }
 }

src/Workspaces/Core/Portable/Utilities/ObjectPools/Extensions.cs

@@ -1,6 +1,7 @@
 // Copyright (c) Microsoft.  All Rights Reserved.  Licensed under the Apache License, Version 2.0.  See License.txt in the project root for license information.
 
 using System.Collections.Generic;
+using System.Diagnostics;
 using System.Text;
 
 namespace Roslyn.Utilities
@@ -14,6 +15,11 @@ public static PooledObject<StringBuilder> GetPooledObject(this ObjectPool<String
             return PooledObject<StringBuilder>.Create(pool);
         }
 
+        public static PooledObject<Stopwatch> GetPooledObject(this ObjectPool<Stopwatch> pool)
+        {
+            return PooledObject<Stopwatch>.Create(pool);
+        }
+
         public static PooledObject<Stack<TItem>> GetPooledObject<TItem>(this ObjectPool<Stack<TItem>> pool)
         {
             return PooledObject<Stack<TItem>>.Create(pool);
@@ -52,6 +58,14 @@ public static StringBuilder AllocateAndClear(this ObjectPool<StringBuilder> pool
             return sb;
         }
 
+        public static Stopwatch AllocateAndClear(this ObjectPool<Stopwatch> pool)
+        {
+            var watch = pool.Allocate();
+            watch.Reset();
+
+            return watch;
+        }
+
         public static Stack<T> AllocateAndClear<T>(this ObjectPool<Stack<T>> pool)
         {
             var set = pool.Allocate();
@@ -109,6 +123,17 @@ public static void ClearAndFree(this ObjectPool<StringBuilder> pool, StringBuild
             pool.Free(sb);
         }
 
+        public static void ClearAndFree(this ObjectPool<Stopwatch> pool, Stopwatch watch)
+        {
+            if (watch == null)
+            {
+                return;
+            }
+
+            watch.Reset();
+            pool.Free(watch);
+        }
+
         public static void ClearAndFree<T>(this ObjectPool<HashSet<T>> pool, HashSet<T> set)
         {
             if (set == null)

src/Workspaces/Core/Portable/Utilities/ObjectPools/PooledObject.cs

@@ -2,6 +2,7 @@
 
 using System;
 using System.Collections.Generic;
+using System.Diagnostics;
 using System.Text;
 
 namespace Roslyn.Utilities
@@ -40,6 +41,14 @@ public static PooledObject<StringBuilder> Create(ObjectPool<StringBuilder> pool)
                 pool,
                 p => Allocator(p),
                 (p, sb) => Releaser(p, sb));
+        } 
+
+        public static PooledObject<Stopwatch> Create(ObjectPool<Stopwatch> pool)
+        {
+            return new PooledObject<Stopwatch>(
+                pool,
+                p => Allocator(p),
+                (p, sb) => Releaser(p, sb));
         }
 
         public static PooledObject<Stack<TItem>> Create<TItem>(ObjectPool<Stack<TItem>> pool)
@@ -94,6 +103,16 @@ private static void Releaser(ObjectPool<StringBuilder> pool, StringBuilder sb)
             pool.ClearAndFree(sb);
         }
 
+        private static Stopwatch Allocator(ObjectPool<Stopwatch> pool)
+        {
+            return pool.AllocateAndClear();
+        }
+
+        private static void Releaser(ObjectPool<Stopwatch> pool, Stopwatch sb)
+        {
+            pool.ClearAndFree(sb);
+        }
+
         private static Stack<TItem> Allocator<TItem>(ObjectPool<Stack<TItem>> pool)
         {
             return pool.AllocateAndClear();