Merge pull request #1459 from PowerShell/async-ps-consumer

Reimplement the way PowerShell is run and hosted in PSES with a dedicated pipeline thread consumer

Author: andyleejordan

.vsts-ci/azure-pipelines-ci.yml

@@ -8,14 +8,6 @@ variables:
   - name: DOTNET_CLI_TELEMETRY_OPTOUT
     value: 'true'
 
-trigger:
-  branches:
-    include:
-      - master
-
-pr:
-- master
-
 jobs:
 - job: PS51_Win2016
   displayName: PowerShell 5.1 - Windows Server 2016

CHANGELOG.md

@@ -1,5 +1,80 @@
 # PowerShell Editor Services Release History
 
+## v3.0.0
+### Thursday, October 28, 2021
+
+This preview release includes a complete overhaul of the core PowerShell engine
+of PowerShell Editor Services.
+This represents over a year's work,
+tracked in [PSES #1295](https://github.com/PowerShell/PowerShellEditorServices/issues/1295)
+and implemented in [PSES #1459](https://github.com/PowerShell/PowerShellEditorServices/pull/1459),
+and is our answer to many, many issues
+opened by users over the last few years.
+We're hoping you'll see a marked improvement
+in the reliability, performance and footprint
+of the extension as a result.
+
+Previously the Integrated Console was run
+by setting threadpool tasks on a shared main runspace,
+and where LSP servicing was done with PowerShell idle events.
+This lead to overhead, threading issues
+and a complex implementation intended to work around
+the asymmetry between PowerShell as a synchronous,
+single-threaded runtime and a language server
+as an asynchronous, multi-threaded service.
+
+Now, PowerShell Editor Services maintains its own dedicated pipeline thread,
+which is able to service requests similar to JavaScript's event loop,
+meaning we can run everything synchronously on the correct thread.
+We also get more efficiency because we can directly call
+PowerShell APIs and code written in C# from this thread,
+without the overhead of a PowerShell pipeline.
+
+This change has overhauled how we service LSP requests,
+how the Integrated Console works,
+how PSReadLine is integrated,
+how debugging is implemented,
+how remoting is handled,
+and a long tail of other features in PowerShell Editor Services.
+
+Also, in making it, while 6,000 lines of code were added,
+we removed 12,000,
+for a more maintainable, more efficient
+and easier to understand extension backend.
+
+While most of our testing has been re-enabled
+(and we're working on adding more),
+there are bound to be issues with this new implementation.
+Please give this a try and let us know if you run into anything.
+
+We also want to thank [@SeeminglyScience](https://github.com/SeeminglyScience)
+for his help and knowledge as we've made this migration.
+
+Finally, a crude breakdown of the work from the commits:
+
+- An initial dedicated pipeline thread consumer implementation
+- Implement the console REPL
+- Implement PSRL idle handling
+- Implement completions
+- Move to invoking PSRL as a C# delegate
+- Implement cancellation and <kbd>Ctrl</kbd>+<kbd>C</kbd>
+- Make <kbd>F8</kbd> work again
+- Ensure execution policy is set correctly
+- Implement $PROFILE support
+- Make nested prompts work
+- Implement REPL debugging
+- Implement remote debugging in the REPL
+- Hook up the debugging UI
+- Implement a new concurrent priority queue for PowerShell tasks
+- Reimplement the REPL synchronously rather than on its own thread
+- Really get debugging working...
+- Implement DSC breakpoint support
+- Reimplement legacy readline support
+- Ensure stdio is still supported as an LSP transport
+- Remove PowerShellContextService and other defunct code
+- Get integration tests working again (and improve diagnosis of PSES failures)
+- Get unit testing working again (except debug service tests)
+
 ## v2.5.2
 ### Monday, October 18, 2021
 

PowerShellEditorServices.Common.props

@@ -1,6 +1,6 @@
 <Project>
   <PropertyGroup>
-    <VersionPrefix>2.5.2</VersionPrefix>
+    <VersionPrefix>3.0.0</VersionPrefix>
     <VersionSuffix></VersionSuffix>
     <Company>Microsoft</Company>
     <Copyright>© Microsoft Corporation.</Copyright>

module/PowerShellEditorServices/PowerShellEditorServices.psd1

@@ -19,7 +19,7 @@ RootModule = if ($PSEdition -eq 'Core')
     }
 
 # Version number of this module.
-ModuleVersion = '2.5.2'
+ModuleVersion = '3.0.0'
 
 # ID used to uniquely identify this module
 GUID = '9ca15887-53a2-479a-9cda-48d26bcb6c47'

modules.json

@@ -6,6 +6,7 @@
         "Version": "1.1.3"
     },
     "PSReadLine": {
-        "Version": "2.1.0"
+        "Version": "2.2.0-beta4",
+        "AllowPrerelease": true
     }
 }

src/PowerShellEditorServices.Hosting/Commands/InvokeReadLineConstructorCommand.cs

@@ -120,6 +120,11 @@ public static EditorServicesLoader Create(
             {
                 AppDomain.CurrentDomain.AssemblyLoad += (object sender, AssemblyLoadEventArgs args) =>
                 {
+                    if (args.LoadedAssembly.IsDynamic)
+                    {
+                        return;
+                    }
+
                     logger.Log(
                         PsesLogLevel.Diagnostic,
                         $"Loaded '{args.LoadedAssembly.GetName()}' from '{args.LoadedAssembly.Location}'");

src/PowerShellEditorServices.Hosting/Commands/InvokeReadLineForEditorServicesCommand.cs

@@ -4,7 +4,7 @@
 using System;
 using System.Threading;
 using System.Threading.Tasks;
-using Microsoft.PowerShell.EditorServices.Handlers;
+using Microsoft.PowerShell.EditorServices.Services.Extension;
 using OmniSharp.Extensions.LanguageServer.Protocol.Server;
 
 namespace Microsoft.PowerShell.EditorServices.Extensions.Services

src/PowerShellEditorServices.Hosting/EditorServicesLoader.cs

@@ -5,7 +5,7 @@
 using System.Linq.Expressions;
 using System.Reflection;
 using Microsoft.Extensions.DependencyInjection;
-using Microsoft.PowerShell.EditorServices.Services;
+using Microsoft.PowerShell.EditorServices.Services.Extension;
 using Microsoft.PowerShell.EditorServices.Utility;
 using OmniSharp.Extensions.LanguageServer.Protocol.Server;
 

src/PowerShellEditorServices/Extensions/Api/EditorContextService.cs

@@ -6,7 +6,7 @@
 using System.Linq;
 using System.Threading;
 using System.Threading.Tasks;
-using Microsoft.PowerShell.EditorServices.Services.PowerShellContext;
+using Microsoft.PowerShell.EditorServices.Services.Extension;
 using OmniSharp.Extensions.LanguageServer.Protocol.Server;
 
 namespace Microsoft.PowerShell.EditorServices.Extensions.Services
@@ -116,7 +116,9 @@ public async Task<string> PromptInputAsync(string message)
                 new ShowInputPromptRequest
                 {
                     Name = message,
-                }).Returning<ShowInputPromptResponse>(CancellationToken.None).ConfigureAwait(false);
+                })
+                .Returning<ShowInputPromptResponse>(CancellationToken.None)
+                .ConfigureAwait(false);
 
             if (response.PromptCancelled)
             {
@@ -142,7 +144,9 @@ public async Task<IReadOnlyList<string>> PromptMultipleSelectionAsync(string mes
                     Message = message,
                     Choices = choiceDetails,
                     DefaultChoices = defaultChoiceIndexes?.ToArray(),
-                }).Returning<ShowChoicePromptResponse>(CancellationToken.None).ConfigureAwait(false);
+                })
+                .Returning<ShowChoicePromptResponse>(CancellationToken.None)
+                .ConfigureAwait(false);
 
             if (response.PromptCancelled)
             {
@@ -168,7 +172,9 @@ public async Task<string> PromptSelectionAsync(string message, IReadOnlyList<Pro
                     Message = message,
                     Choices = choiceDetails,
                     DefaultChoices = defaultChoiceIndex > -1 ? new[] { defaultChoiceIndex } : null,
-                }).Returning<ShowChoicePromptResponse>(CancellationToken.None).ConfigureAwait(false);
+                })
+                .Returning<ShowChoicePromptResponse>(CancellationToken.None)
+                .ConfigureAwait(false);
 
             if (response.PromptCancelled)
             {

src/PowerShellEditorServices/Extensions/Api/EditorExtensionServiceProvider.cs

@@ -1,7 +1,7 @@
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT License.
 
-using Microsoft.PowerShell.EditorServices.Services;
+using Microsoft.PowerShell.EditorServices.Services.Extension;
 using System;
 using System.Collections.Generic;
 using System.Threading.Tasks;

src/PowerShellEditorServices/Extensions/Api/EditorUIService.cs

@@ -1,7 +1,6 @@
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT License.
 
-using MediatR;
 using OmniSharp.Extensions.LanguageServer.Protocol.Server;
 using System.Threading;
 using System.Threading.Tasks;

src/PowerShellEditorServices/Extensions/Api/ExtensionCommandService.cs

@@ -5,7 +5,6 @@
 using System;
 using System.Collections.Generic;
 using System.Management.Automation.Language;
-using System.Threading.Tasks;
 
 namespace Microsoft.PowerShell.EditorServices.Extensions.Services
 {

src/PowerShellEditorServices/Extensions/Api/LanguageServerService.cs

@@ -1,7 +1,6 @@
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT License.
 
-using System;
 using Microsoft.PowerShell.EditorServices.Services.TextDocument;
 
 namespace Microsoft.PowerShell.EditorServices.Extensions

src/PowerShellEditorServices/Extensions/Api/WorkspaceService.cs

@@ -1,7 +1,7 @@
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT License.
 
-using Microsoft.PowerShell.EditorServices.Handlers;
+using Microsoft.PowerShell.EditorServices.Services.Extension;
 using Microsoft.PowerShell.EditorServices.Services.TextDocument;
 using OmniSharp.Extensions.LanguageServer.Protocol.Models;
 using System;

src/PowerShellEditorServices/Extensions/EditorContext.cs

@@ -5,7 +5,7 @@
 using System.Reflection;
 using System.Threading.Tasks;
 using Microsoft.PowerShell.EditorServices.Extensions.Services;
-using Microsoft.PowerShell.EditorServices.Services;
+using Microsoft.PowerShell.EditorServices.Services.Extension;
 
 namespace Microsoft.PowerShell.EditorServices.Extensions
 {

src/PowerShellEditorServices/Extensions/EditorFileRanges.cs

@@ -1,8 +1,6 @@
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT License.
 
-using System.Threading.Tasks;
-
 namespace Microsoft.PowerShell.EditorServices.Extensions
 {
     /// <summary>

src/PowerShellEditorServices/Extensions/EditorObject.cs

@@ -8,10 +8,10 @@
 using Microsoft.Extensions.Logging;
 using Microsoft.PowerShell.EditorServices.Logging;
 using Microsoft.PowerShell.EditorServices.Server;
-using Microsoft.PowerShell.EditorServices.Services;
 using Serilog;
 using Serilog.Events;
 using OmniSharp.Extensions.LanguageServer.Protocol.Server;
+using Microsoft.PowerShell.EditorServices.Services.Extension;
 
 #if DEBUG
 using Serilog.Debugging;

src/PowerShellEditorServices/Extensions/EditorWorkspace.cs

@@ -3,8 +3,6 @@
 
 using Microsoft.Extensions.Logging;
 using System;
-using System.Collections.Generic;
-using System.Text;
 
 namespace Microsoft.PowerShell.EditorServices.Logging
 {

src/PowerShellEditorServices/Hosting/EditorServicesServerFactory.cs

@@ -42,6 +42,9 @@
   </ItemGroup>
 
   <ItemGroup>
-    <Compile Remove="Extensions\Api\DocumentSymbolService.cs"/>
+    <Compile Remove="Extensions\Api\DocumentSymbolService.cs" />
+    <Compile Remove="Services\Extension\Templating\**" />
+    <EmbeddedResource Remove="Services\Extension\Templating\**" />
+    <None Remove="Services\Extension\Templating\**" />
   </ItemGroup>
 </Project>