Fix lock up that occurs when WinForms is executed on the pipeline thread (#1149)

* Ensure PSRL static constructor really only run once

* Fix await thread marshalling error

Co-Authored-By: Patrick Meinecke <[email protected]>

Author: rjmholt

src/PowerShellEditorServices/Server/PsesDebugServer.cs

@@ -5,6 +5,7 @@
 
 using System;
 using System.IO;
+using System.Threading;
 using System.Threading.Tasks;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Logging;
@@ -21,7 +22,10 @@ namespace Microsoft.PowerShell.EditorServices.Server
     /// </summary>
     internal class PsesDebugServer : IDisposable
     {
-        private static bool s_hasRunPsrlStaticCtor = false;
+        /// <summary>
+        /// This is a bool but must be an int, since Interlocked.Exchange can't handle a bool
+        /// </summary>
+        private static int s_hasRunPsrlStaticCtor = 0;
 
         private readonly Stream _inputStream;
         private readonly Stream _outputStream;
@@ -64,20 +68,21 @@ public async Task StartAsync()
                 options.Serializer = new DapProtocolSerializer();
                 options.Reciever = new DapReciever();
                 options.LoggerFactory = _loggerFactory;
-                Extensions.Logging.ILogger logger = options.LoggerFactory.CreateLogger("DebugOptionsStartup");
+                ILogger logger = options.LoggerFactory.CreateLogger("DebugOptionsStartup");
 
                 // We need to let the PowerShell Context Service know that we are in a debug session
                 // so that it doesn't send the powerShell/startDebugger message.
                 _powerShellContextService = ServiceProvider.GetService<PowerShellContextService>();
                 _powerShellContextService.IsDebugServerActive = true;
 
                 // Needed to make sure PSReadLine's static properties are initialized in the pipeline thread.
-                if (!s_hasRunPsrlStaticCtor && _usePSReadLine)
+                if (_usePSReadLine && Interlocked.Exchange(ref s_hasRunPsrlStaticCtor, 1) == 0)
                 {
-                    s_hasRunPsrlStaticCtor = true;
+                    // This must be run synchronously to ensure debugging works
                     _powerShellContextService
                         .ExecuteScriptStringAsync("[System.Runtime.CompilerServices.RuntimeHelpers]::RunClassConstructor([Microsoft.PowerShell.PSConsoleReadLine].TypeHandle)")
-                        .Wait();
+                        .GetAwaiter()
+                        .GetResult();
                 }
 
                 options.Services = new ServiceCollection()

src/PowerShellEditorServices/Services/PowerShellContext/PowerShellContextService.cs

@@ -2356,18 +2356,18 @@ private void OnDebuggerStop(object sender, DebuggerStopEventArgs e)
                     PowerShellExecutionResult.Stopped,
                     null));
 
-                // Get the session details and push the current
-                // runspace if the session has changed
-                SessionDetails sessionDetails = null;
-                try
-                {
-                    sessionDetails = this.GetSessionDetailsInDebugger();
-                }
-                catch (InvalidOperationException)
-                {
-                    this.logger.LogTrace(
-                        "Attempting to get session details failed, most likely due to a running pipeline that is attempting to stop.");
-                }
+            // Get the session details and push the current
+            // runspace if the session has changed
+            SessionDetails sessionDetails = null;
+            try
+            {
+                sessionDetails = this.GetSessionDetailsInDebugger();
+            }
+            catch (InvalidOperationException)
+            {
+                this.logger.LogTrace(
+                    "Attempting to get session details failed, most likely due to a running pipeline that is attempting to stop.");
+            }
 
             if (!localThreadController.FrameExitTask.Task.IsCompleted)
             {

src/PowerShellEditorServices/Services/PowerShellContext/Session/ThreadController.cs

@@ -78,9 +78,9 @@ internal async Task<IEnumerable<TResult>> RequestPipelineExecutionAsync<TResult>
         /// A task object representing the asynchronous operation. The Result property will return
         /// the retrieved pipeline execution request.
         /// </returns>
-        internal async Task<IPipelineExecutionRequest> TakeExecutionRequestAsync()
+        internal Task<IPipelineExecutionRequest> TakeExecutionRequestAsync()
         {
-            return await PipelineRequestQueue.DequeueAsync();
+            return PipelineRequestQueue.DequeueAsync();
         }
 
         /// <summary>

src/PowerShellEditorServices/Utility/AsyncQueue.cs

@@ -146,7 +146,7 @@ public async Task<T> DequeueAsync(CancellationToken cancellationToken)
         {
             Task<T> requestTask;
 
-            using (await queueLock.LockAsync(cancellationToken))
+            using (await queueLock.LockAsync(cancellationToken).ConfigureAwait(false))
             {
                 if (this.itemQueue.Count > 0)
                 {
@@ -171,7 +171,7 @@ public async Task<T> DequeueAsync(CancellationToken cancellationToken)
             }
 
             // Wait for the request task to complete outside of the lock
-            return await requestTask;
+            return await requestTask.ConfigureAwait(false);
         }
 
         /// <summary>