Add some retry support to Renderer::PaintFrame (#2830)

If _PaintFrameForEngine returns E_PENDING, we'll give it another two
tries to get itself straight. If it continues to fail, we'll take down
the application.

We observed that the DX renderer was failing to present the swap chain
and failfast'ing when it did so; however, there are some errors from
which DXGI guidance suggests we try to recover. We'll now return
E_PENDING (and destroy our device resources) when we hit those errors.

Fixes #2265.

(cherry picked from commit 277acc3)

Author: DHowett

src/renderer/base/renderer.cpp

@@ -10,6 +10,8 @@
 using namespace Microsoft::Console::Render;
 using namespace Microsoft::Console::Types;
 
+static constexpr auto maxRetriesForRenderEngine = 3;
+
 // Routine Description:
 // - Creates a new renderer controller for a console.
 // Arguments:
@@ -62,7 +64,21 @@ Renderer::~Renderer()
 
     for (IRenderEngine* const pEngine : _rgpEngines)
     {
-        LOG_IF_FAILED(_PaintFrameForEngine(pEngine));
+        auto tries = maxRetriesForRenderEngine;
+        while (tries > 0)
+        {
+            const auto hr = _PaintFrameForEngine(pEngine);
+            if (E_PENDING == hr)
+            {
+                if (--tries == 0)
+                {
+                    FAIL_FAST_HR_MSG(E_UNEXPECTED, "A rendering engine required too many retries.");
+                }
+                continue;
+            }
+            LOG_IF_FAILED(hr);
+            break;
+        }
     }
 
     return S_OK;

src/renderer/dx/DxRenderer.cpp

@@ -866,15 +866,31 @@ void DxEngine::_InvalidOr(RECT rc) noexcept
 // Arguments:
 // - <none>
 // Return Value:
-// - S_OK or relevant DirectX error
+// - S_OK on success, E_PENDING to indicate a retry or a relevant DirectX error
 [[nodiscard]] HRESULT DxEngine::Present() noexcept
 {
     if (_presentReady)
     {
         try
         {
-            FAIL_FAST_IF_FAILED(_dxgiSwapChain->Present(1, 0));
-            /*FAIL_FAST_IF_FAILED(_dxgiSwapChain->Present1(1, 0, &_presentParams));*/
+            HRESULT hr = S_OK;
+
+            hr = _dxgiSwapChain->Present(1, 0);
+            /*hr = _dxgiSwapChain->Present1(1, 0, &_presentParams);*/
+
+            if (FAILED(hr))
+            {
+                // These two error codes are indicated for destroy-and-recreate
+                if (hr == DXGI_ERROR_DEVICE_REMOVED || hr == DXGI_ERROR_DEVICE_RESET)
+                {
+                    // We don't need to end painting here, as the renderer has done it for us.
+                    _ReleaseDeviceResources();
+                    FAIL_FAST_IF_FAILED(InvalidateAll());
+                    return E_PENDING; // Indicate a retry to the renderer.
+                }
+
+                FAIL_FAST_HR(hr);
+            }
 
             RETURN_IF_FAILED(_CopyFrontToBack());
             _presentReady = false;