diff --git a/src/DurableSDK/Commands/InvokeDurableActivityCommand.cs b/src/DurableSDK/Commands/InvokeDurableActivityCommand.cs index ba9b429f..4cb8639e 100644 --- a/src/DurableSDK/Commands/InvokeDurableActivityCommand.cs +++ b/src/DurableSDK/Commands/InvokeDurableActivityCommand.cs @@ -43,10 +43,8 @@ protected override void EndProcessing() { var privateData = (Hashtable)MyInvocation.MyCommand.Module.PrivateData; var context = (OrchestrationContext)privateData[SetFunctionInvocationContextCommand.ContextKey]; - var loadedFunctions = FunctionLoader.GetLoadedFunctions(); var task = new ActivityInvocationTask(FunctionName, Input, RetryOptions); - ActivityInvocationTask.ValidateTask(task, loadedFunctions); _durableTaskHandler.StopAndInitiateDurableTaskOrReplay( task, context, NoWait.IsPresent, diff --git a/src/DurableSDK/Commands/SetFunctionInvocationContextCommand.cs b/src/DurableSDK/Commands/SetFunctionInvocationContextCommand.cs index 943e8362..3430be16 100644 --- a/src/DurableSDK/Commands/SetFunctionInvocationContextCommand.cs +++ b/src/DurableSDK/Commands/SetFunctionInvocationContextCommand.cs @@ -9,6 +9,7 @@ namespace Microsoft.Azure.Functions.PowerShellWorker.Durable.Commands { using System.Collections; using System.Management.Automation; + using Microsoft.PowerShell.Commands; /// /// Set the orchestration context. diff --git a/src/DurableSDK/DurableTaskHandler.cs b/src/DurableSDK/DurableTaskHandler.cs index c65b1275..a35546b8 100644 --- a/src/DurableSDK/DurableTaskHandler.cs +++ b/src/DurableSDK/DurableTaskHandler.cs @@ -6,10 +6,12 @@ namespace Microsoft.Azure.Functions.PowerShellWorker.Durable { using System; + using System.Collections; using System.Collections.Generic; + using System.Management.Automation; using System.Threading; using Microsoft.Azure.Functions.PowerShellWorker.Durable.Tasks; - using Utility; + using Microsoft.PowerShell.Commands; internal class DurableTaskHandler { @@ -83,7 +85,7 @@ public void StopAndInitiateDurableTaskOrReplay( retryOptions.MaxNumberOfAttempts, onSuccess: result => { - output(TypeExtensions.ConvertFromJson(result)); + output(ConvertFromJson(result)); }, onFailure); @@ -232,15 +234,41 @@ private static object GetEventResult(HistoryEvent historyEvent) if (historyEvent.EventType == HistoryEventType.TaskCompleted) { - return TypeExtensions.ConvertFromJson(historyEvent.Result); + return ConvertFromJson(historyEvent.Result); } else if (historyEvent.EventType == HistoryEventType.EventRaised) { - return TypeExtensions.ConvertFromJson(historyEvent.Input); + return ConvertFromJson(historyEvent.Input); } return null; } + public static object ConvertFromJson(string json) + { + object retObj = JsonObject.ConvertFromJson(json, returnHashtable: true, error: out _); + + if (retObj is PSObject psObj) + { + retObj = psObj.BaseObject; + } + + if (retObj is Hashtable hashtable) + { + try + { + // ConvertFromJson returns case-sensitive Hashtable by design -- JSON may contain keys that only differ in case. + // We try casting the Hashtable to a case-insensitive one, but if that fails, we keep using the original one. + retObj = new Hashtable(hashtable, StringComparer.OrdinalIgnoreCase); + } + catch + { + retObj = hashtable; + } + } + + return retObj; + } + private void InitiateAndWaitForStop(OrchestrationContext context) { context.OrchestrationActionCollector.Stop(); diff --git a/src/DurableSDK/ExternalInvoker.cs b/src/DurableSDK/ExternalInvoker.cs new file mode 100644 index 00000000..fa8a31c6 --- /dev/null +++ b/src/DurableSDK/ExternalInvoker.cs @@ -0,0 +1,26 @@ +// +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. +// + +namespace Microsoft.Azure.Functions.PowerShellWorker.Durable +{ + using System; + using System.Collections; + using System.Management.Automation; + + internal class ExternalInvoker : IExternalInvoker + { + private readonly Func _externalSDKInvokerFunction; + + public ExternalInvoker(Func invokerFunction) + { + _externalSDKInvokerFunction = invokerFunction; + } + + public Hashtable Invoke(IPowerShellServices powerShellServices) + { + return (Hashtable)_externalSDKInvokerFunction.Invoke(powerShellServices.GetPowerShell()); + } + } +} diff --git a/src/DurableSDK/IExternalInvoker.cs b/src/DurableSDK/IExternalInvoker.cs new file mode 100644 index 00000000..3a703f3d --- /dev/null +++ b/src/DurableSDK/IExternalInvoker.cs @@ -0,0 +1,16 @@ +// +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. +// + +namespace Microsoft.Azure.Functions.PowerShellWorker.Durable +{ + using System.Collections; + + // Represents a contract for the + internal interface IExternalInvoker + { + // Method to invoke an orchestration using the external Durable SDK + Hashtable Invoke(IPowerShellServices powerShellServices); + } +} diff --git a/src/DurableSDK/IOrchestrationInvoker.cs b/src/DurableSDK/IOrchestrationInvoker.cs index 7e80aba3..36011b56 100644 --- a/src/DurableSDK/IOrchestrationInvoker.cs +++ b/src/DurableSDK/IOrchestrationInvoker.cs @@ -10,5 +10,6 @@ namespace Microsoft.Azure.Functions.PowerShellWorker.Durable internal interface IOrchestrationInvoker { Hashtable Invoke(OrchestrationBindingInfo orchestrationBindingInfo, IPowerShellServices pwsh); + void SetExternalInvoker(IExternalInvoker externalInvoker); } } diff --git a/src/DurableSDK/IPowerShellServices.cs b/src/DurableSDK/IPowerShellServices.cs index a8cf897b..cdd850bc 100644 --- a/src/DurableSDK/IPowerShellServices.cs +++ b/src/DurableSDK/IPowerShellServices.cs @@ -5,17 +5,26 @@ namespace Microsoft.Azure.Functions.PowerShellWorker.Durable { + using Microsoft.Azure.WebJobs.Script.Grpc.Messages; using System; using System.Management.Automation; internal interface IPowerShellServices { + PowerShell GetPowerShell(); + + bool UseExternalDurableSDK(); + void SetDurableClient(object durableClient); - void SetOrchestrationContext(OrchestrationContext orchestrationContext); + OrchestrationBindingInfo SetOrchestrationContext(ParameterBinding context, out IExternalInvoker externalInvoker); void ClearOrchestrationContext(); + void TracePipelineObject(); + + void AddParameter(string name, object value); + IAsyncResult BeginInvoke(PSDataCollection output); void EndInvoke(IAsyncResult asyncResult); diff --git a/src/DurableSDK/OrchestrationActionCollector.cs b/src/DurableSDK/OrchestrationActionCollector.cs index b62fbc4b..1542c2bf 100644 --- a/src/DurableSDK/OrchestrationActionCollector.cs +++ b/src/DurableSDK/OrchestrationActionCollector.cs @@ -11,6 +11,7 @@ namespace Microsoft.Azure.Functions.PowerShellWorker.Durable using System.Threading; using Microsoft.Azure.Functions.PowerShellWorker.Durable.Actions; + using Newtonsoft.Json; internal class OrchestrationActionCollector { diff --git a/src/DurableSDK/OrchestrationInvoker.cs b/src/DurableSDK/OrchestrationInvoker.cs index fef557ba..b71116b4 100644 --- a/src/DurableSDK/OrchestrationInvoker.cs +++ b/src/DurableSDK/OrchestrationInvoker.cs @@ -11,58 +11,98 @@ namespace Microsoft.Azure.Functions.PowerShellWorker.Durable using System.Linq; using System.Management.Automation; - using PowerShellWorker.Utility; using Microsoft.Azure.Functions.PowerShellWorker.Durable.Actions; internal class OrchestrationInvoker : IOrchestrationInvoker { - public Hashtable Invoke(OrchestrationBindingInfo orchestrationBindingInfo, IPowerShellServices pwsh) + private IExternalInvoker _externalInvoker; + internal static string isOrchestrationFailureKey = "IsOrchestrationFailure"; + + public Hashtable Invoke( + OrchestrationBindingInfo orchestrationBindingInfo, + IPowerShellServices powerShellServices) { try { - var outputBuffer = new PSDataCollection(); - var context = orchestrationBindingInfo.Context; + if (powerShellServices.UseExternalDurableSDK()) + { + return InvokeExternalDurableSDK(powerShellServices); + } + return InvokeInternalDurableSDK(orchestrationBindingInfo, powerShellServices); + } + catch (Exception ex) + { + ex.Data.Add(isOrchestrationFailureKey, true); + throw; + } + finally + { + powerShellServices.ClearStreamsAndCommands(); + } + } + + public Hashtable InvokeExternalDurableSDK(IPowerShellServices powerShellServices) + { + return _externalInvoker.Invoke(powerShellServices); + } + + public Hashtable InvokeInternalDurableSDK( + OrchestrationBindingInfo orchestrationBindingInfo, + IPowerShellServices powerShellServices) + { + var outputBuffer = new PSDataCollection(); + var context = orchestrationBindingInfo.Context; + + // context.History should never be null when initializing CurrentUtcDateTime + var orchestrationStart = context.History.First( + e => e.EventType == HistoryEventType.OrchestratorStarted); + context.CurrentUtcDateTime = orchestrationStart.Timestamp.ToUniversalTime(); - // context.History should never be null when initializing CurrentUtcDateTime - var orchestrationStart = context.History.First( - e => e.EventType == HistoryEventType.OrchestratorStarted); - context.CurrentUtcDateTime = orchestrationStart.Timestamp.ToUniversalTime(); + // Marks the first OrchestratorStarted event as processed + orchestrationStart.IsProcessed = true; - // Marks the first OrchestratorStarted event as processed - orchestrationStart.IsProcessed = true; - - var asyncResult = pwsh.BeginInvoke(outputBuffer); + // Finish initializing the Function invocation + powerShellServices.AddParameter(orchestrationBindingInfo.ParameterName, context); + powerShellServices.TracePipelineObject(); - var (shouldStop, actions) = - orchestrationBindingInfo.Context.OrchestrationActionCollector.WaitForActions(asyncResult.AsyncWaitHandle); + var asyncResult = powerShellServices.BeginInvoke(outputBuffer); - if (shouldStop) + var (shouldStop, actions) = + orchestrationBindingInfo.Context.OrchestrationActionCollector.WaitForActions(asyncResult.AsyncWaitHandle); + + if (shouldStop) + { + // The orchestration function should be stopped and restarted + powerShellServices.StopInvoke(); + // return (Hashtable)orchestrationBindingInfo.Context.OrchestrationActionCollector.output; + return CreateOrchestrationResult(isDone: false, actions, output: null, context.CustomStatus); + } + else + { + try { - // The orchestration function should be stopped and restarted - pwsh.StopInvoke(); - return CreateOrchestrationResult(isDone: false, actions, output: null, context.CustomStatus); + // The orchestration function completed + powerShellServices.EndInvoke(asyncResult); + var result = CreateReturnValueFromFunctionOutput(outputBuffer); + return CreateOrchestrationResult(isDone: true, actions, output: result, context.CustomStatus); } - else + catch (Exception e) { - try - { - // The orchestration function completed - pwsh.EndInvoke(asyncResult); - var result = FunctionReturnValueBuilder.CreateReturnValueFromFunctionOutput(outputBuffer); - return CreateOrchestrationResult(isDone: true, actions, output: result, context.CustomStatus); - } - catch (Exception e) - { - // The orchestrator code has thrown an unhandled exception: - // this should be treated as an entire orchestration failure - throw new OrchestrationFailureException(actions, context.CustomStatus, e); - } + // The orchestrator code has thrown an unhandled exception: + // this should be treated as an entire orchestration failure + throw new OrchestrationFailureException(actions, context.CustomStatus, e); } } - finally + } + + public static object CreateReturnValueFromFunctionOutput(IList pipelineItems) + { + if (pipelineItems == null || pipelineItems.Count <= 0) { - pwsh.ClearStreamsAndCommands(); + return null; } + + return pipelineItems.Count == 1 ? pipelineItems[0] : pipelineItems.ToArray(); } private static Hashtable CreateOrchestrationResult( @@ -72,7 +112,12 @@ private static Hashtable CreateOrchestrationResult( object customStatus) { var orchestrationMessage = new OrchestrationMessage(isDone, actions, output, customStatus); - return new Hashtable { { AzFunctionInfo.DollarReturn, orchestrationMessage } }; + return new Hashtable { { "$return", orchestrationMessage } }; + } + + public void SetExternalInvoker(IExternalInvoker externalInvoker) + { + _externalInvoker = externalInvoker; } } } diff --git a/src/DurableSDK/PowerShellExtensions.cs b/src/DurableSDK/PowerShellExtensions.cs new file mode 100644 index 00000000..a5225188 --- /dev/null +++ b/src/DurableSDK/PowerShellExtensions.cs @@ -0,0 +1,69 @@ +// +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. +// + +using System.Collections; +using System.Collections.ObjectModel; + +namespace Microsoft.Azure.Functions.PowerShellWorker.Durable +{ + using System.Management.Automation; + + internal static class PowerShellExtensions + { + public static void InvokeAndClearCommands(this PowerShell pwsh) + { + try + { + pwsh.Invoke(); + } + finally + { + pwsh.Streams.ClearStreams(); + pwsh.Commands.Clear(); + } + } + + public static void InvokeAndClearCommands(this PowerShell pwsh, IEnumerable input) + { + try + { + pwsh.Invoke(input); + } + finally + { + pwsh.Streams.ClearStreams(); + pwsh.Commands.Clear(); + } + } + + public static Collection InvokeAndClearCommands(this PowerShell pwsh) + { + try + { + var result = pwsh.Invoke(); + return result; + } + finally + { + pwsh.Streams.ClearStreams(); + pwsh.Commands.Clear(); + } + } + + public static Collection InvokeAndClearCommands(this PowerShell pwsh, IEnumerable input) + { + try + { + var result = pwsh.Invoke(input); + return result; + } + finally + { + pwsh.Streams.ClearStreams(); + pwsh.Commands.Clear(); + } + } + } +} diff --git a/src/DurableSDK/PowerShellServices.cs b/src/DurableSDK/PowerShellServices.cs index 0efb681d..fda8d80d 100644 --- a/src/DurableSDK/PowerShellServices.cs +++ b/src/DurableSDK/PowerShellServices.cs @@ -6,43 +6,132 @@ namespace Microsoft.Azure.Functions.PowerShellWorker.Durable { using System; + using System.Collections.ObjectModel; + using System.Linq; using System.Management.Automation; - using PowerShell; + using Microsoft.Azure.Functions.PowerShellWorker.Utility; + using Microsoft.Azure.WebJobs.Script.Grpc.Messages; + using Newtonsoft.Json; internal class PowerShellServices : IPowerShellServices { - private const string SetFunctionInvocationContextCommand = - "Microsoft.Azure.Functions.PowerShellWorker\\Set-FunctionInvocationContext"; + private readonly string SetFunctionInvocationContextCommand; + private const string ExternalDurableSDKName = "DurableSDK"; + private const string InternalDurableSDKName = "Microsoft.Azure.Functions.PowerShellWorker"; private readonly PowerShell _pwsh; - private bool _hasSetOrchestrationContext = false; + private bool _hasInitializedDurableFunction = false; + private readonly bool _useExternalDurableSDK = false; public PowerShellServices(PowerShell pwsh) { + //This logic will be commented out until the external SDK is published on the PS Gallery + + // We attempt to import the external SDK upon construction of the PowerShellServices object. + // We maintain the boolean member _useExternalDurableSDK in this object rather than + // DurableController because the expected input and functionality of SetFunctionInvocationContextCommand + // may differ between the internal and external implementations. + + try + { + pwsh.AddCommand(Utils.ImportModuleCmdletInfo) + .AddParameter("Name", ExternalDurableSDKName) + .AddParameter("ErrorAction", ActionPreference.Stop) + .InvokeAndClearCommands(); + _useExternalDurableSDK = true; + } + catch (Exception e) + { + // Check to see if ExternalDurableSDK is among the modules imported or + // available to be imported: if it is, then something went wrong with + // the Import-Module statement and we should throw an Exception. + // Otherwise, we use the InternalDurableSDK + var availableModules = pwsh.AddCommand(Utils.GetModuleCmdletInfo) + .AddParameter("Name", ExternalDurableSDKName) + .InvokeAndClearCommands(); + if (availableModules.Count() > 0) + { + // TODO: evaluate if there is a better error message or exception type to be throwing. + // Ideally, this should never happen. + throw new InvalidOperationException("The external Durable SDK was detected, but unable to be imported.", e); + } + _useExternalDurableSDK = false; + } + //_useExternalDurableSDK = false; + + if (_useExternalDurableSDK) + { + SetFunctionInvocationContextCommand = $"{ExternalDurableSDKName}\\Set-FunctionInvocationContext"; + } + else + { + SetFunctionInvocationContextCommand = $"{InternalDurableSDKName}\\Set-FunctionInvocationContext"; + } _pwsh = pwsh; } + public bool UseExternalDurableSDK() + { + return _useExternalDurableSDK; + } + + public PowerShell GetPowerShell() + { + return this._pwsh; + } + public void SetDurableClient(object durableClient) { _pwsh.AddCommand(SetFunctionInvocationContextCommand) .AddParameter("DurableClient", durableClient) .InvokeAndClearCommands(); - - _hasSetOrchestrationContext = true; + _hasInitializedDurableFunction = true; } - public void SetOrchestrationContext(OrchestrationContext orchestrationContext) + public OrchestrationBindingInfo SetOrchestrationContext( + ParameterBinding context, + out IExternalInvoker externalInvoker) { - _pwsh.AddCommand(SetFunctionInvocationContextCommand) - .AddParameter("OrchestrationContext", orchestrationContext) - .InvokeAndClearCommands(); + externalInvoker = null; + OrchestrationBindingInfo orchestrationBindingInfo = new OrchestrationBindingInfo( + context.Name, + JsonConvert.DeserializeObject(context.Data.String)); + + if (_useExternalDurableSDK) + { + Collection> output = _pwsh.AddCommand(SetFunctionInvocationContextCommand) + // The external SetFunctionInvocationContextCommand expects a .json string to deserialize + // and writes an invoker function to the output pipeline. + .AddParameter("OrchestrationContext", context.Data.String) + .InvokeAndClearCommands>(); + if (output.Count() == 1) + { + externalInvoker = new ExternalInvoker(output[0]); + } + else + { + throw new InvalidOperationException($"Only a single output was expected for an invocation of {SetFunctionInvocationContextCommand}"); + } + } + else + { + _pwsh.AddCommand(SetFunctionInvocationContextCommand) + .AddParameter("OrchestrationContext", orchestrationBindingInfo.Context) + .InvokeAndClearCommands(); + } + _hasInitializedDurableFunction = true; + return orchestrationBindingInfo; + } + - _hasSetOrchestrationContext = true; + public void AddParameter(string name, object value) + { + _pwsh.AddParameter(name, value); } public void ClearOrchestrationContext() { - if (_hasSetOrchestrationContext) + if (_hasInitializedDurableFunction) { _pwsh.AddCommand(SetFunctionInvocationContextCommand) .AddParameter("Clear", true) @@ -50,6 +139,11 @@ public void ClearOrchestrationContext() } } + public void TracePipelineObject() + { + _pwsh.AddCommand("Microsoft.Azure.Functions.PowerShellWorker\\Trace-PipelineObject"); + } + public IAsyncResult BeginInvoke(PSDataCollection output) { return _pwsh.BeginInvoke(input: null, output); diff --git a/src/DurableSDK/Tasks/ActivityInvocationTask.cs b/src/DurableSDK/Tasks/ActivityInvocationTask.cs index 35b31fd2..5f9f6a33 100644 --- a/src/DurableSDK/Tasks/ActivityInvocationTask.cs +++ b/src/DurableSDK/Tasks/ActivityInvocationTask.cs @@ -7,13 +7,7 @@ namespace Microsoft.Azure.Functions.PowerShellWorker.Durable.Tasks { - using System; using System.Linq; - using System.Collections.Generic; - - using WebJobs.Script.Grpc.Messages; - - using Microsoft.Azure.Functions.PowerShellWorker; using Microsoft.Azure.Functions.PowerShellWorker.Durable; using Microsoft.Azure.Functions.PowerShellWorker.Durable.Actions; using Microsoft.Azure.Functions.PowerShellWorker.DurableWorker; @@ -62,24 +56,5 @@ internal override OrchestrationAction CreateOrchestrationAction() ? new CallActivityAction(FunctionName, Input) : new CallActivityWithRetryAction(FunctionName, Input, RetryOptions); } - - internal static void ValidateTask(ActivityInvocationTask task, IEnumerable loadedFunctions) - { - var functionInfo = loadedFunctions.FirstOrDefault(fi => fi.FuncName == task.FunctionName); - if (functionInfo == null) - { - var message = string.Format(PowerShellWorkerStrings.FunctionNotFound, task.FunctionName); - throw new InvalidOperationException(message); - } - - var activityTriggerBinding = functionInfo.InputBindings.FirstOrDefault( - entry => DurableBindings.IsActivityTrigger(entry.Value.Type) - && entry.Value.Direction == BindingInfo.Types.Direction.In); - if (activityTriggerBinding.Key == null) - { - var message = string.Format(PowerShellWorkerStrings.FunctionDoesNotHaveProperActivityFunctionBinding, task.FunctionName); - throw new InvalidOperationException(message); - } - } } } diff --git a/src/DurableWorker/DurableController.cs b/src/DurableWorker/DurableController.cs index 7b7f1b4e..3d756e15 100644 --- a/src/DurableWorker/DurableController.cs +++ b/src/DurableWorker/DurableController.cs @@ -5,7 +5,6 @@ namespace Microsoft.Azure.Functions.PowerShellWorker.Durable { - using System; using System.Collections; using System.Collections.Generic; using System.Collections.ObjectModel; @@ -48,9 +47,14 @@ internal DurableController( _orchestrationInvoker = orchestrationInvoker; } - public void BeforeFunctionInvocation(IList inputData) + public string GetOrchestrationParameterName() { - // If the function is an orchestration client, then we set the DurableClient + return _orchestrationBindingInfo?.ParameterName; + } + + public void InitializeBindings(IList inputData) + { + // If the function is an durable client, then we set the DurableClient // in the module context for the 'Start-DurableOrchestration' function to use. if (_durableFunctionInfo.IsDurableClient) { @@ -59,11 +63,14 @@ public void BeforeFunctionInvocation(IList inputData) .Data.ToObject(); _powerShellServices.SetDurableClient(durableClient); + } else if (_durableFunctionInfo.IsOrchestrationFunction) { - _orchestrationBindingInfo = CreateOrchestrationBindingInfo(inputData); - _powerShellServices.SetOrchestrationContext(_orchestrationBindingInfo.Context); + _orchestrationBindingInfo = _powerShellServices.SetOrchestrationContext( + inputData[0], + out IExternalInvoker externalInvoker); + _orchestrationInvoker.SetExternalInvoker(externalInvoker); } } @@ -88,46 +95,22 @@ public bool TryGetInputBindingParameterValue(string bindingName, out object valu public void AddPipelineOutputIfNecessary(Collection pipelineItems, Hashtable result) { - var shouldAddPipelineOutput = - _durableFunctionInfo.Type == DurableFunctionType.ActivityFunction; - - if (shouldAddPipelineOutput) + + if (ShouldSuppressPipelineTraces()) { var returnValue = FunctionReturnValueBuilder.CreateReturnValueFromFunctionOutput(pipelineItems); result.Add(AzFunctionInfo.DollarReturn, returnValue); } } - public bool TryInvokeOrchestrationFunction(out Hashtable result) + public Hashtable InvokeOrchestrationFunction() { - if (!_durableFunctionInfo.IsOrchestrationFunction) - { - result = null; - return false; - } - - result = _orchestrationInvoker.Invoke(_orchestrationBindingInfo, _powerShellServices); - return true; + return _orchestrationInvoker.Invoke(_orchestrationBindingInfo, _powerShellServices); } public bool ShouldSuppressPipelineTraces() { return _durableFunctionInfo.Type == DurableFunctionType.ActivityFunction; } - - private static OrchestrationBindingInfo CreateOrchestrationBindingInfo(IList inputData) - { - // Quote from https://docs.microsoft.com/en-us/azure/azure-functions/durable-functions-bindings: - // - // "Orchestrator functions should never use any input or output bindings other than the orchestration trigger binding. - // Doing so has the potential to cause problems with the Durable Task extension because those bindings may not obey the single-threading and I/O rules." - // - // Therefore, it's by design that input data contains only one item, which is the metadata of the orchestration context. - var context = inputData[0]; - - return new OrchestrationBindingInfo( - context.Name, - JsonConvert.DeserializeObject(context.Data.String)); - } } } diff --git a/src/DurableWorker/DurableFunctionInfo.cs b/src/DurableWorker/DurableFunctionInfo.cs index 8665cfe8..05c3650d 100644 --- a/src/DurableWorker/DurableFunctionInfo.cs +++ b/src/DurableWorker/DurableFunctionInfo.cs @@ -13,9 +13,12 @@ public DurableFunctionInfo(DurableFunctionType type, string durableClientBinding DurableClientBindingName = durableClientBindingName; } + public bool IsActivityFunction => Type == DurableFunctionType.ActivityFunction; + public bool IsDurableClient => DurableClientBindingName != null; public bool IsOrchestrationFunction => Type == DurableFunctionType.OrchestrationFunction; + public string DurableClientBindingName { get; } diff --git a/src/Modules/Microsoft.Azure.Functions.PowerShellWorker/Microsoft.Azure.Functions.PowerShellWorker.psm1 b/src/Modules/Microsoft.Azure.Functions.PowerShellWorker/Microsoft.Azure.Functions.PowerShellWorker.psm1 index c87f08b6..be368a48 100644 --- a/src/Modules/Microsoft.Azure.Functions.PowerShellWorker/Microsoft.Azure.Functions.PowerShellWorker.psm1 +++ b/src/Modules/Microsoft.Azure.Functions.PowerShellWorker/Microsoft.Azure.Functions.PowerShellWorker.psm1 @@ -11,7 +11,7 @@ Set-Alias -Name Start-NewOrchestration -Value Start-DurableOrchestration function GetDurableClientFromModulePrivateData { $PrivateData = $PSCmdlet.MyInvocation.MyCommand.Module.PrivateData - if ($PrivateData -eq $null -or $PrivateData['DurableClient'] -eq $null) { + if ($null -eq $PrivateData -or $null -eq $PrivateData['DurableClient']) { throw "No binding of the type 'durableClient' was defined." } else { diff --git a/src/PowerShell/PowerShellManager.cs b/src/PowerShell/PowerShellManager.cs index 7e458914..0344c05d 100644 --- a/src/PowerShell/PowerShellManager.cs +++ b/src/PowerShell/PowerShellManager.cs @@ -15,6 +15,7 @@ namespace Microsoft.Azure.Functions.PowerShellWorker.PowerShell { + using Microsoft.Azure.Functions.PowerShellWorker.DurableWorker; using System.Management.Automation; using System.Text; @@ -204,32 +205,37 @@ public Hashtable InvokeFunction( FunctionInvocationPerformanceStopwatch stopwatch) { var outputBindings = FunctionMetadata.GetOutputBindingHashtable(_pwsh.Runspace.InstanceId); - - var durableController = new DurableController(functionInfo.DurableFunctionInfo, _pwsh); + var durableFunctionsUtils = new DurableController(functionInfo.DurableFunctionInfo, _pwsh); try { - durableController.BeforeFunctionInvocation(inputData); + durableFunctionsUtils.InitializeBindings(inputData); AddEntryPointInvocationCommand(functionInfo); stopwatch.OnCheckpoint(FunctionInvocationPerformanceStopwatch.Checkpoint.FunctionCodeReady); - SetInputBindingParameterValues(functionInfo, inputData, durableController, triggerMetadata, traceContext, retryContext); + var orchestrationParamName = durableFunctionsUtils.GetOrchestrationParameterName(); + SetInputBindingParameterValues(functionInfo, inputData, orchestrationParamName, triggerMetadata, traceContext, retryContext); stopwatch.OnCheckpoint(FunctionInvocationPerformanceStopwatch.Checkpoint.InputBindingValuesReady); - if (!durableController.ShouldSuppressPipelineTraces()) - { - _pwsh.AddCommand("Microsoft.Azure.Functions.PowerShellWorker\\Trace-PipelineObject"); - } - stopwatch.OnCheckpoint(FunctionInvocationPerformanceStopwatch.Checkpoint.InvokingFunctionCode); Logger.Log(isUserOnlyLog: false, LogLevel.Trace, CreateInvocationPerformanceReportMessage(functionInfo.FuncName, stopwatch)); try { - return durableController.TryInvokeOrchestrationFunction(out var result) - ? result - : InvokeNonOrchestrationFunction(durableController, outputBindings); + if(functionInfo.DurableFunctionInfo.IsOrchestrationFunction) + { + return durableFunctionsUtils.InvokeOrchestrationFunction(); + } + else + { + var isActivityFunction = functionInfo.DurableFunctionInfo.IsActivityFunction; + if (!isActivityFunction) + { + _pwsh.AddCommand("Microsoft.Azure.Functions.PowerShellWorker\\Trace-PipelineObject"); + } + return ExecuteUserCode(isActivityFunction, outputBindings); + } } catch (RuntimeException e) { @@ -237,9 +243,9 @@ public Hashtable InvokeFunction( Logger.Log(isUserOnlyLog: true, LogLevel.Error, GetFunctionExceptionMessage(e)); throw; } - catch (OrchestrationFailureException e) + catch (Exception e) { - if (e.InnerException is IContainsErrorRecord inner) + if (e.Data.Contains(OrchestrationInvoker.isOrchestrationFailureKey) && e.InnerException is IContainsErrorRecord inner) { Logger.Log(isUserOnlyLog: true, LogLevel.Error, GetFunctionExceptionMessage(inner)); } @@ -248,7 +254,7 @@ public Hashtable InvokeFunction( } finally { - durableController.AfterFunctionInvocation(); + durableFunctionsUtils.AfterFunctionInvocation(); outputBindings.Clear(); ResetRunspace(); } @@ -257,7 +263,7 @@ public Hashtable InvokeFunction( private void SetInputBindingParameterValues( AzFunctionInfo functionInfo, IEnumerable inputData, - DurableController durableController, + string orchParamName, Hashtable triggerMetadata, TraceContext traceContext, RetryContext retryContext) @@ -266,13 +272,12 @@ private void SetInputBindingParameterValues( { if (functionInfo.FuncParameters.TryGetValue(binding.Name, out var paramInfo)) { - if (!durableController.TryGetInputBindingParameterValue(binding.Name, out var valueToUse)) + if (string.CompareOrdinal(binding.Name, orchParamName) != 0) { var bindingInfo = functionInfo.InputBindings[binding.Name]; - valueToUse = Utils.TransformInBindingValueAsNeeded(paramInfo, bindingInfo, binding.Data.ToObject()); + var valueToUse = Utils.TransformInBindingValueAsNeeded(paramInfo, bindingInfo, binding.Data.ToObject()); + _pwsh.AddParameter(binding.Name, valueToUse); } - - _pwsh.AddParameter(binding.Name, valueToUse); } } @@ -296,11 +301,15 @@ private void SetInputBindingParameterValues( /// /// Execution a function fired by a trigger or an activity function scheduled by an orchestration. /// - private Hashtable InvokeNonOrchestrationFunction(DurableController durableController, IDictionary outputBindings) + private Hashtable ExecuteUserCode(bool addPipelineOutput, IDictionary outputBindings) { var pipelineItems = _pwsh.InvokeAndClearCommands(); var result = new Hashtable(outputBindings, StringComparer.OrdinalIgnoreCase); - durableController.AddPipelineOutputIfNecessary(pipelineItems, result); + if (addPipelineOutput) + { + var returnValue = FunctionReturnValueBuilder.CreateReturnValueFromFunctionOutput(pipelineItems); + result.Add(AzFunctionInfo.DollarReturn, returnValue); + } return result; } diff --git a/src/Utility/TypeExtensions.cs b/src/Utility/TypeExtensions.cs index 2f3c4186..b3bcb9e3 100644 --- a/src/Utility/TypeExtensions.cs +++ b/src/Utility/TypeExtensions.cs @@ -142,7 +142,7 @@ public static object ConvertFromJson(string json) private static string ConvertToJson(object fromObj) { var context = new JsonObject.ConvertToJsonContext( - maxDepth: 4, + maxDepth: 10, //TODO: fix enumsAsStrings: false, compressOutput: true); diff --git a/src/Utility/Utils.cs b/src/Utility/Utils.cs index 549ba5de..21ec68b8 100644 --- a/src/Utility/Utils.cs +++ b/src/Utility/Utils.cs @@ -17,6 +17,7 @@ namespace Microsoft.Azure.Functions.PowerShellWorker.Utility internal class Utils { + internal readonly static CmdletInfo GetModuleCmdletInfo = new CmdletInfo("Get-Module", typeof(GetModuleCommand)); internal readonly static CmdletInfo ImportModuleCmdletInfo = new CmdletInfo("Import-Module", typeof(ImportModuleCommand)); internal readonly static CmdletInfo RemoveModuleCmdletInfo = new CmdletInfo("Remove-Module", typeof(RemoveModuleCommand)); internal readonly static CmdletInfo RemoveJobCmdletInfo = new CmdletInfo("Remove-Job", typeof(RemoveJobCommand)); diff --git a/test/E2E/TestFunctionApp/profile.ps1 b/test/E2E/TestFunctionApp/profile.ps1 new file mode 100644 index 00000000..9afdf1da --- /dev/null +++ b/test/E2E/TestFunctionApp/profile.ps1 @@ -0,0 +1,22 @@ +# Azure Functions profile.ps1 +# +# This profile.ps1 will get executed every "cold start" of your Function App. +# "cold start" occurs when: +# +# * A Function App starts up for the very first time +# * A Function App starts up after being de-allocated due to inactivity +# +# You can define helper functions, run commands, or specify environment variables +# NOTE: any variables defined that are not environment variables will get reset after the first execution + +# Authenticate with Azure PowerShell using MSI. +# Remove this if you are not planning on using MSI or Azure PowerShell. +if ($env:MSI_SECRET) { + Disable-AzContextAutosave -Scope Process | Out-Null + Connect-AzAccount -Identity +} + +# Uncomment the next line to enable legacy AzureRm alias in Azure PowerShell. +# Enable-AzureRmAlias + +# You can also define functions or aliases that can be referenced in any of your PowerShell functions. \ No newline at end of file diff --git a/test/Unit/Durable/ActivityInvocationTaskTests.cs b/test/Unit/Durable/ActivityInvocationTaskTests.cs index 530711ca..96e27571 100644 --- a/test/Unit/Durable/ActivityInvocationTaskTests.cs +++ b/test/Unit/Durable/ActivityInvocationTaskTests.cs @@ -163,59 +163,6 @@ public void StopAndInitiateDurableTaskOrReplay_OutputsActivityInvocationTask_Whe Assert.Equal(FunctionName, allOutput.Single().FunctionName); } - [Fact] - public void ValidateTask_Throws_WhenActivityFunctionDoesNotExist() - { - var history = CreateHistory(scheduled: false, completed: false, failed: false, output: InvocationResultJson); - var orchestrationContext = new OrchestrationContext { History = history }; - - var loadedFunctions = new[] - { - DurableTestUtilities.CreateFakeAzFunctionInfo(FunctionName, "fakeTriggerBindingName", ActivityTriggerBindingType, BindingInfo.Types.Direction.In) - }; - - const string wrongFunctionName = "AnotherFunction"; - - var durableTaskHandler = new DurableTaskHandler(); - - var exception = - Assert.Throws( - () => ActivityInvocationTask.ValidateTask( - new ActivityInvocationTask(wrongFunctionName, FunctionInput), loadedFunctions)); - - Assert.Contains(wrongFunctionName, exception.Message); - Assert.DoesNotContain(ActivityTriggerBindingType, exception.Message); - - DurableTestUtilities.VerifyNoActionAdded(orchestrationContext); - } - - [Theory] - [InlineData("IncorrectBindingType", BindingInfo.Types.Direction.In)] - [InlineData(ActivityTriggerBindingType, BindingInfo.Types.Direction.Out)] - public void ValidateTask_Throws_WhenActivityFunctionHasNoProperBinding( - string bindingType, BindingInfo.Types.Direction bindingDirection) - { - var history = CreateHistory(scheduled: false, completed: false, failed: false, output: InvocationResultJson); - var orchestrationContext = new OrchestrationContext { History = history }; - - var loadedFunctions = new[] - { - DurableTestUtilities.CreateFakeAzFunctionInfo(FunctionName, "fakeTriggerBindingName", bindingType, bindingDirection) - }; - - var durableTaskHandler = new DurableTaskHandler(); - - var exception = - Assert.Throws( - () => ActivityInvocationTask.ValidateTask( - new ActivityInvocationTask(FunctionName, FunctionInput), loadedFunctions)); - - Assert.Contains(FunctionName, exception.Message); - Assert.Contains(ActivityTriggerBindingType, exception.Message); - - DurableTestUtilities.VerifyNoActionAdded(orchestrationContext); - } - [Theory] [InlineData(false)] [InlineData(true)] diff --git a/test/Unit/Durable/DurableControllerTests.cs b/test/Unit/Durable/DurableControllerTests.cs index 5ec3244c..b57cb767 100644 --- a/test/Unit/Durable/DurableControllerTests.cs +++ b/test/Unit/Durable/DurableControllerTests.cs @@ -23,9 +23,12 @@ public class DurableControllerTests { private readonly Mock _mockPowerShellServices = new Mock(MockBehavior.Strict); private readonly Mock _mockOrchestrationInvoker = new Mock(MockBehavior.Strict); + private const string _contextParameterName = "ParameterName"; + private static readonly OrchestrationContext _orchestrationContext = new OrchestrationContext { InstanceId = Guid.NewGuid().ToString() }; + private static readonly OrchestrationBindingInfo _orchestrationBindingInfo = new OrchestrationBindingInfo(_contextParameterName, _orchestrationContext); [Fact] - public void BeforeFunctionInvocation_SetsDurableClient_ForDurableClientFunction() + public void InitializeBindings_SetsDurableClient_ForDurableClientFunction() { var durableController = CreateDurableController(DurableFunctionType.None, "DurableClientBindingName"); @@ -39,7 +42,7 @@ public void BeforeFunctionInvocation_SetsDurableClient_ForDurableClientFunction( _mockPowerShellServices.Setup(_ => _.SetDurableClient(It.IsAny())); - durableController.BeforeFunctionInvocation(inputData); + durableController.InitializeBindings(inputData); _mockPowerShellServices.Verify( _ => _.SetDurableClient( @@ -48,50 +51,50 @@ public void BeforeFunctionInvocation_SetsDurableClient_ForDurableClientFunction( } [Fact] - public void BeforeFunctionInvocation_SetsOrchestrationContext_ForOrchestrationFunction() + public void InitializeBindings_SetsOrchestrationContext_ForOrchestrationFunction() { var durableController = CreateDurableController(DurableFunctionType.OrchestrationFunction); - - var orchestrationContext = new OrchestrationContext { InstanceId = Guid.NewGuid().ToString() }; var inputData = new[] { - CreateParameterBinding("ParameterName", orchestrationContext) + CreateParameterBinding("ParameterName", _orchestrationContext) }; + + _mockPowerShellServices.Setup(_ => _.SetOrchestrationContext(It.IsAny(), + out It.Ref.IsAny)) + .Returns(_orchestrationBindingInfo); + _mockOrchestrationInvoker.Setup(_ => _.SetExternalInvoker(It.IsAny())); - _mockPowerShellServices.Setup(_ => _.SetOrchestrationContext(It.IsAny())); - - durableController.BeforeFunctionInvocation(inputData); + durableController.InitializeBindings(inputData); _mockPowerShellServices.Verify( _ => _.SetOrchestrationContext( - It.Is(c => c.InstanceId == orchestrationContext.InstanceId)), + It.Is(c => c.Data.ToString().Contains(_orchestrationContext.InstanceId)), + out It.Ref.IsAny), Times.Once); } [Fact] - public void BeforeFunctionInvocation_Throws_OnOrchestrationFunctionWithoutContextParameter() + public void InitializeBindings_Throws_OnOrchestrationFunctionWithoutContextParameter() { var durableController = CreateDurableController(DurableFunctionType.OrchestrationFunction); var inputData = new ParameterBinding[0]; - Assert.ThrowsAny(() => durableController.BeforeFunctionInvocation(inputData)); + Assert.ThrowsAny(() => durableController.InitializeBindings(inputData)); } [Theory] [InlineData(DurableFunctionType.None)] [InlineData(DurableFunctionType.ActivityFunction)] - internal void BeforeFunctionInvocation_DoesNothing_ForNonOrchestrationFunction(DurableFunctionType durableFunctionType) + internal void InitializeBindings_DoesNothing_ForNonOrchestrationFunction(DurableFunctionType durableFunctionType) { var durableController = CreateDurableController(durableFunctionType); - var orchestrationContext = new OrchestrationContext { InstanceId = Guid.NewGuid().ToString() }; - var inputData = new[] { // Even if a parameter similar to orchestration context is passed: - CreateParameterBinding("ParameterName", orchestrationContext) + CreateParameterBinding("ParameterName", _orchestrationContext) }; - durableController.BeforeFunctionInvocation(inputData); + durableController.InitializeBindings(inputData); } [Theory] @@ -112,19 +115,19 @@ internal void AfterFunctionInvocation_ClearsOrchestrationContext(DurableFunction public void TryGetInputBindingParameterValue_RetrievesOrchestrationContextParameter_ForOrchestrationFunction() { var durableController = CreateDurableController(DurableFunctionType.OrchestrationFunction); - - var orchestrationContext = new OrchestrationContext { InstanceId = Guid.NewGuid().ToString() }; - const string contextParameterName = "ParameterName"; var inputData = new[] { - CreateParameterBinding(contextParameterName, orchestrationContext) + CreateParameterBinding(_contextParameterName, _orchestrationContext) }; - - _mockPowerShellServices.Setup(_ => _.SetOrchestrationContext(It.IsAny())); - durableController.BeforeFunctionInvocation(inputData); - - Assert.True(durableController.TryGetInputBindingParameterValue(contextParameterName, out var value)); - Assert.Equal(orchestrationContext.InstanceId, ((OrchestrationContext)value).InstanceId); + _mockPowerShellServices.Setup(_ => _.SetOrchestrationContext( + It.IsAny(), + out It.Ref.IsAny)) + .Returns(_orchestrationBindingInfo); + _mockOrchestrationInvoker.Setup(_ => _.SetExternalInvoker(It.IsAny())); + durableController.InitializeBindings(inputData); + + Assert.True(durableController.TryGetInputBindingParameterValue(_contextParameterName, out var value)); + Assert.Equal(_orchestrationContext.InstanceId, ((OrchestrationContext)value).InstanceId); } [Theory] @@ -133,70 +136,52 @@ public void TryGetInputBindingParameterValue_RetrievesOrchestrationContextParame internal void TryGetInputBindingParameterValue_RetrievesNothing_ForNonOrchestrationFunction(DurableFunctionType durableFunctionType) { var durableController = CreateDurableController(durableFunctionType); - - var orchestrationContext = new OrchestrationContext { InstanceId = Guid.NewGuid().ToString() }; - const string contextParameterName = "ParameterName"; var inputData = new[] { - CreateParameterBinding(contextParameterName, orchestrationContext) + CreateParameterBinding(_contextParameterName, _orchestrationContext) }; - _mockPowerShellServices.Setup(_ => _.SetOrchestrationContext(It.IsAny())); - durableController.BeforeFunctionInvocation(inputData); + _mockPowerShellServices.Setup(_ => _.SetOrchestrationContext( + It.IsAny(), + out It.Ref.IsAny)) + .Returns(_orchestrationBindingInfo); + _mockOrchestrationInvoker.Setup(_ => _.SetExternalInvoker(It.IsAny())); + durableController.InitializeBindings(inputData); - Assert.False(durableController.TryGetInputBindingParameterValue(contextParameterName, out var value)); + Assert.False(durableController.TryGetInputBindingParameterValue(_contextParameterName, out var value)); Assert.Null(value); } [Fact] public void TryInvokeOrchestrationFunction_InvokesOrchestrationFunction() { - var contextParameterName = "ParameterName"; - var orchestrationContext = new OrchestrationContext { InstanceId = Guid.NewGuid().ToString() }; - var inputData = new[] { CreateParameterBinding(contextParameterName, orchestrationContext) }; - + var inputData = new[] { CreateParameterBinding(_contextParameterName, _orchestrationContext) }; var durableController = CreateDurableController(DurableFunctionType.OrchestrationFunction); - _mockPowerShellServices.Setup(_ => _.SetOrchestrationContext(It.IsAny())); - durableController.BeforeFunctionInvocation(inputData); + _mockPowerShellServices.Setup(_ => _.SetOrchestrationContext( + It.IsAny(), + out It.Ref.IsAny)) + .Returns(_orchestrationBindingInfo); + _mockOrchestrationInvoker.Setup(_ => _.SetExternalInvoker(It.IsAny())); + durableController.InitializeBindings(inputData); var expectedResult = new Hashtable(); _mockOrchestrationInvoker.Setup( _ => _.Invoke(It.IsAny(), It.IsAny())) .Returns(expectedResult); - var invoked = durableController.TryInvokeOrchestrationFunction(out var actualResult); - Assert.True(invoked); + var actualResult = durableController.InvokeOrchestrationFunction(); Assert.Same(expectedResult, actualResult); _mockOrchestrationInvoker.Verify( _ => _.Invoke( It.Is( - bindingInfo => bindingInfo.Context.InstanceId == orchestrationContext.InstanceId - && bindingInfo.ParameterName == contextParameterName), + bindingInfo => bindingInfo.Context.InstanceId == _orchestrationContext.InstanceId + && bindingInfo.ParameterName == _contextParameterName), _mockPowerShellServices.Object), Times.Once); } - [Theory] - [InlineData(DurableFunctionType.None)] - [InlineData(DurableFunctionType.ActivityFunction)] - internal void TryInvokeOrchestrationFunction_DoesNotInvokeNonOrchestrationFunction(DurableFunctionType durableFunctionType) - { - var contextParameterName = "ParameterName"; - var orchestrationContext = new OrchestrationContext { InstanceId = Guid.NewGuid().ToString() }; - var inputData = new[] { CreateParameterBinding(contextParameterName, orchestrationContext) }; - - var durableController = CreateDurableController(durableFunctionType); - - _mockPowerShellServices.Setup(_ => _.SetOrchestrationContext(It.IsAny())); - durableController.BeforeFunctionInvocation(inputData); - - var invoked = durableController.TryInvokeOrchestrationFunction(out var actualResult); - Assert.False(invoked); - Assert.Null(actualResult); - } - [Fact] public void AddPipelineOutputIfNecessary_AddsDollarReturn_ForActivityFunction() { diff --git a/test/Unit/Durable/OrchestrationInvokerTests.cs b/test/Unit/Durable/OrchestrationInvokerTests.cs index 65c4b24d..2a2db694 100644 --- a/test/Unit/Durable/OrchestrationInvokerTests.cs +++ b/test/Unit/Durable/OrchestrationInvokerTests.cs @@ -34,12 +34,16 @@ public void InvocationRunsToCompletionIfNotStopped() { var invocationAsyncResult = DurableTestUtilities.CreateInvocationResult(completed: true); DurableTestUtilities.ExpectBeginInvoke(_mockPowerShellServices, invocationAsyncResult); + _mockPowerShellServices.Setup(_ => _.UseExternalDurableSDK()).Returns(false); _orchestrationInvoker.Invoke(_orchestrationBindingInfo, _mockPowerShellServices.Object); _mockPowerShellServices.Verify(_ => _.BeginInvoke(It.IsAny>()), Times.Once); _mockPowerShellServices.Verify(_ => _.EndInvoke(invocationAsyncResult), Times.Once); _mockPowerShellServices.Verify(_ => _.ClearStreamsAndCommands(), Times.Once); + _mockPowerShellServices.Verify(_ => _.TracePipelineObject(), Times.Once); + _mockPowerShellServices.Verify(_ => _.AddParameter(It.IsAny(), It.IsAny()), Times.Once); + _mockPowerShellServices.Verify(_ => _.UseExternalDurableSDK(), Times.Once); _mockPowerShellServices.VerifyNoOtherCalls(); } @@ -47,10 +51,14 @@ public void InvocationRunsToCompletionIfNotStopped() public void InvocationStopsOnStopEvent() { InvokeOrchestration(completed: false); + _mockPowerShellServices.Setup(_ => _.UseExternalDurableSDK()).Returns(false); _mockPowerShellServices.Verify(_ => _.BeginInvoke(It.IsAny>()), Times.Once); _mockPowerShellServices.Verify(_ => _.StopInvoke(), Times.Once); _mockPowerShellServices.Verify(_ => _.ClearStreamsAndCommands(), Times.Once); + _mockPowerShellServices.Verify(_ => _.TracePipelineObject(), Times.Once); + _mockPowerShellServices.Verify(_ => _.AddParameter(It.IsAny(), It.IsAny()), Times.Once); + _mockPowerShellServices.Verify(_ => _.UseExternalDurableSDK(), Times.Once); _mockPowerShellServices.VerifyNoOtherCalls(); }