# This is a combination of 37 commits.

# This is the 1st commit message:

separate DF SDK classes from DF worker classes

# This is the commit message #2:

fix typo

# This is the commit message #3:

DurableSDK now compiles by itself

# This is the commit message #4:

Allow ExternalSDK to handle orchestration

# This is the commit message #5:

document next steps

# This is the commit message #6:

allow external SDK to set the user-code's input. Still need to refactor this logic for the worker to continue working with old SDK

# This is the commit message #7:

add import module

# This is the commit message #8:

supress traces

# This is the commit message #9:

avoid nullptr

# This is the commit message #10:

pass tests

# This is the commit message #11:

fix E2E tests

# This is the commit message #12:

develop E2E tests

# This is the commit message #13:

Enabled external durable client (#765)

Co-authored-by: Michael Peng <[email protected]>
# This is the commit message #14:

bindings work

# This is the commit message #15:

conditional binding intialization

# This is the commit message #16:

conditional import

# This is the commit message #17:

Added exception handling logic

# This is the commit message #18:

Revert durableController name to durableFunctionsUtils

# This is the commit message #19:

Ensure unit tests are functioning properly

# This is the commit message #20:

Corrected unit test names

# This is the commit message #21:

Turned repeated variables in unit tests into static members

# This is the commit message #22:

Fixed issue with building the worker

# This is the commit message #23:

Fix E2E test

# This is the commit message #24:

Fixed unit test setup

# This is the commit message #25:

Fixed another unit test setup

# This is the commit message #26:

Remove string representation of booleans

# This is the commit message #27:

patch e2e test
# This is the commit message #28:

remove typo in toString
# This is the commit message #29:

Update PowerShell language worker pipelines (#750)

* Install .Net to a global location

* Remove .Net installation tasks

* Update install .Net 6 task

* Update Windows image to use windows-latest
# This is the commit message #30:

Make throughput warning message visible for tooling diagnosis (#757)


# This is the commit message #31:

Update grpc.tools to version 2.43.0

# This is the commit message #32:

Update Google.Protobuf.Tools to version 3.19.4

# This is the commit message #33:

Revert "Update Google.Protobuf.Tools to version 3.19.4"

This reverts commit bcbd022.

# This is the commit message #34:

Revert "Update grpc.tools to version 2.43.0"

This reverts commit ccb323a.

# This is the commit message #35:

Update Google.Protobuf to 3.19.4 and grpc.tools to  2.43.0 (#762)

* Update grpc.tools to version 2.43.0

* Update Google.Protobuf.Tools to version 3.19.4
# This is the commit message #36:

Switch from Grpc.Core to Grpc.Net.Client (#758)

* Upgraded protobuf versions and removed Grpc.Core dependency

* Updated channel and option types used

* Change channel credentials

* Added http prefix to url

* Add valid URL check and explicitly include credentials
# This is the commit message #37:

Update pipeline logic to generate the SBOM for release builds (#767)

Author: davidmrdavid

azure-pipelines-e2e-integration-tests.yml

@@ -11,22 +11,16 @@ strategy:
     linux:
       imageName: 'ubuntu-latest'
     windows:
-      imageName: 'vs2017-win2016'
+      imageName: 'windows-latest'
 
 pool:
     vmImage: $(imageName)
 
 steps:
 - pwsh: |
-    Invoke-WebRequest 'https://raw.githubusercontent.com/dotnet/cli/master/scripts/obtain/dotnet-install.ps1' -OutFile 'dotnet-install.ps1'
-    ./dotnet-install.ps1 -InstallDir "$env:ProgramFiles/dotnet" -Version "6.0.100" -Channel 'release'
-  displayName: 'Install the .Net version used by the Core Tools for Windows'
-  condition: eq( variables['Agent.OS'], 'Windows_NT' )
-
-- bash: |
-    curl -sSL https://raw.githubusercontent.com/dotnet/cli/master/scripts/obtain/dotnet-install.sh | bash /dev/stdin -v '6.0.100' -c 'release' --install-dir /usr/share/dotnet
-  displayName: 'Install the .Net version used by the Core Tools for Linux'
-  condition: eq( variables['Agent.OS'], 'Linux' )
+    Import-Module "./tools/helper.psm1" -Force
+    Install-Dotnet
+  displayName: 'Install .NET 6.0'
 
 - pwsh: ./test/E2E/Start-E2ETest.ps1 -UseCoreToolsBuildFromIntegrationTests
   env:

azure-pipelines.yml

@@ -43,32 +43,14 @@ steps:
 
 - pwsh: |
       $ErrorActionPreference = "Stop"
-      if ($isReleaseBuild)
-      {
-        ./build.ps1 -Clean -Configuration Release -BuildNumber "$(buildNumber)" -AddSBOM -SBOMUtilSASUrl $env:SBOMUtilSASUrl
-      }
-      else
-      {
-        ./build.ps1 -Clean -Configuration Release -BuildNumber "$(buildNumber)"
-      }
+      $shouldAddSBOM = [bool]"$(IsReleaseBuild)"
+
+      ./build.ps1 -Clean -Configuration Release -BuildNumber "$(buildNumber)" -AddSBOM:$shouldAddSBOM -SBOMUtilSASUrl "$(SBOMUtilSASUrl)"
   displayName: 'Build worker code'
-  env:
-    SBOMUtilSASUrl: $(SBOMUtilSASUrl)
 
 - pwsh: ./build.ps1 -NoBuild -Test
   displayName: 'Running UnitTest'
 
-- pwsh: |
-    Invoke-WebRequest 'https://raw.githubusercontent.com/dotnet/cli/master/scripts/obtain/dotnet-install.ps1' -OutFile 'dotnet-install.ps1'
-    ./dotnet-install.ps1 -InstallDir "$env:ProgramFiles/dotnet" -Version "6.0.100" -Channel 'release'
-  displayName: 'Install the .Net version used by the Core Tools for Windows'
-  condition: eq( variables['Agent.OS'], 'Windows_NT' )
-
-- bash: |
-    curl -sSL https://raw.githubusercontent.com/dotnet/cli/master/scripts/obtain/dotnet-install.sh | bash /dev/stdin -v '6.0.100' -c 'release' --install-dir /usr/share/dotnet
-  displayName: 'Install the .Net version used by the Core Tools for Linux'
-  condition: eq( variables['Agent.OS'], 'Linux' )
-
 - pwsh: ./test/E2E/Start-E2ETest.ps1
   env:
     AzureWebJobsStorage: $(AzureWebJobsStorage)

src/Durable/PowerShellServices.cs

@@ -43,10 +43,8 @@ protected override void EndProcessing()
         {
             var privateData = (Hashtable)MyInvocation.MyCommand.Module.PrivateData;
             var context = (OrchestrationContext)privateData[SetFunctionInvocationContextCommand.ContextKey];
-            var loadedFunctions = FunctionLoader.GetLoadedFunctions();
 
             var task = new ActivityInvocationTask(FunctionName, Input, RetryOptions);
-            ActivityInvocationTask.ValidateTask(task, loadedFunctions);
 
             _durableTaskHandler.StopAndInitiateDurableTaskOrReplay(
                 task, context, NoWait.IsPresent,

src/Durable/Actions/ActionType.cs renamed to src/DurableSDK/Actions/ActionType.cs

@@ -9,6 +9,7 @@ namespace Microsoft.Azure.Functions.PowerShellWorker.Durable.Commands
 {
     using System.Collections;
     using System.Management.Automation;
+    using Microsoft.PowerShell.Commands;
 
     /// <summary>
     /// Set the orchestration context.

src/Durable/Actions/CallActivityAction.cs renamed to src/DurableSDK/Actions/CallActivityAction.cs

@@ -6,10 +6,12 @@
 namespace Microsoft.Azure.Functions.PowerShellWorker.Durable
 {
     using System;
+    using System.Collections;
     using System.Collections.Generic;
+    using System.Management.Automation;
     using System.Threading;
     using Microsoft.Azure.Functions.PowerShellWorker.Durable.Tasks;
-    using Utility;
+    using Microsoft.PowerShell.Commands;
 
     internal class DurableTaskHandler
     {
@@ -47,6 +49,7 @@ public void StopAndInitiateDurableTaskOrReplay(
                     }
 
                     completedHistoryEvent.IsProcessed = true;
+                    context.IsReplaying = completedHistoryEvent.IsPlayed;
 
                     switch (completedHistoryEvent.EventType)
                     {
@@ -57,6 +60,13 @@ public void StopAndInitiateDurableTaskOrReplay(
                                 output(eventResult);
                             }
                             break;
+                        case HistoryEventType.EventRaised:
+                            var eventRaisedResult = GetEventResult(completedHistoryEvent);
+                            if (eventRaisedResult != null)
+                            {
+                                output(eventRaisedResult);
+                            }
+                            break;
 
                         case HistoryEventType.TaskFailed:
                             if (retryOptions == null)
@@ -76,7 +86,7 @@ public void StopAndInitiateDurableTaskOrReplay(
                                         retryOptions.MaxNumberOfAttempts,
                                         onSuccess:
                                             result => {
-                                                output(TypeExtensions.ConvertFromJson(result));
+                                                output(ConvertFromJson(result));
                                             },
                                         onFailure);
 
@@ -126,6 +136,7 @@ public void WaitAll(
             var allTasksCompleted = completedEvents.Count == tasksToWaitFor.Count;
             if (allTasksCompleted)
             {
+                context.IsReplaying = completedEvents.Count == 0 ? false : completedEvents[0].IsPlayed;
                 CurrentUtcDateTimeUpdater.UpdateCurrentUtcDateTime(context);
 
                 foreach (var completedHistoryEvent in completedEvents)
@@ -185,6 +196,7 @@ public void WaitAny(
             var anyTaskCompleted = completedTasks.Count > 0;
             if (anyTaskCompleted)
             {
+                context.IsReplaying = context.History[firstCompletedHistoryEventIndex].IsPlayed;
                 CurrentUtcDateTimeUpdater.UpdateCurrentUtcDateTime(context);
                 // Return a reference to the first completed task
                 output(firstCompletedTask);
@@ -206,15 +218,41 @@ private static object GetEventResult(HistoryEvent historyEvent)
 
             if (historyEvent.EventType == HistoryEventType.TaskCompleted)
             {
-                return TypeExtensions.ConvertFromJson(historyEvent.Result);
+                return ConvertFromJson(historyEvent.Result);
             }
             else if (historyEvent.EventType == HistoryEventType.EventRaised)
             {
-                return TypeExtensions.ConvertFromJson(historyEvent.Input);
+                return ConvertFromJson(historyEvent.Input);
             }
             return null;
         }
 
+        public static object ConvertFromJson(string json)
+        {
+            object retObj = JsonObject.ConvertFromJson(json, returnHashtable: true, error: out _);
+
+            if (retObj is PSObject psObj)
+            {
+                retObj = psObj.BaseObject;
+            }
+
+            if (retObj is Hashtable hashtable)
+            {
+                try
+                {
+                    // ConvertFromJson returns case-sensitive Hashtable by design -- JSON may contain keys that only differ in case.
+                    // We try casting the Hashtable to a case-insensitive one, but if that fails, we keep using the original one.
+                    retObj = new Hashtable(hashtable, StringComparer.OrdinalIgnoreCase);
+                }
+                catch
+                {
+                    retObj = hashtable;
+                }
+            }
+
+            return retObj;
+        }
+
         private void InitiateAndWaitForStop(OrchestrationContext context)
         {
             context.OrchestrationActionCollector.Stop();

src/Durable/Actions/CallActivityWithRetryAction.cs renamed to src/DurableSDK/Actions/CallActivityWithRetryAction.cs

@@ -5,10 +5,13 @@
 
 namespace Microsoft.Azure.Functions.PowerShellWorker.Durable
 {
+    using System;
     using System.Collections;
+    using System.Management.Automation;
 
     internal interface IOrchestrationInvoker
     {
         Hashtable Invoke(OrchestrationBindingInfo orchestrationBindingInfo, IPowerShellServices pwsh);
+        void SetExternalInvoker(Action<PowerShell> externalInvoker);
     }
 }

src/Durable/Actions/CreateDurableTimerAction.cs renamed to src/DurableSDK/Actions/CreateDurableTimerAction.cs

@@ -5,17 +5,26 @@
 
 namespace Microsoft.Azure.Functions.PowerShellWorker.Durable
 {
+    using Microsoft.Azure.WebJobs.Script.Grpc.Messages;
     using System;
     using System.Management.Automation;
 
     internal interface IPowerShellServices
     {
+        PowerShell GetPowerShell();
+
+        bool UseExternalDurableSDK();
+
         void SetDurableClient(object durableClient);
 
-        void SetOrchestrationContext(OrchestrationContext orchestrationContext);
+        OrchestrationBindingInfo SetOrchestrationContext(ParameterBinding orchestrationContext, out Action<object> externalInvoker);
 
         void ClearOrchestrationContext();
 
+        public void TracePipelineObject();
+        public void AddParameter(string name, object value);
+
+
         IAsyncResult BeginInvoke(PSDataCollection<object> output);
 
         void EndInvoke(IAsyncResult asyncResult);

src/Durable/Actions/ExternalEventAction.cs renamed to src/DurableSDK/Actions/ExternalEventAction.cs

@@ -11,6 +11,7 @@ namespace Microsoft.Azure.Functions.PowerShellWorker.Durable
     using System.Threading;
 
     using Microsoft.Azure.Functions.PowerShellWorker.Durable.Actions;
+    using Newtonsoft.Json;
 
     internal class OrchestrationActionCollector
     {

src/Durable/Actions/OrchestrationAction.cs renamed to src/DurableSDK/Actions/OrchestrationAction.cs

@@ -20,13 +20,13 @@ public class OrchestrationContext
         public object Input { get; internal set; }
 
         [DataMember]
-        internal string InstanceId { get; set; }
+        public string InstanceId { get; set; }
 
         [DataMember]
         internal string ParentInstanceId { get; set; }
 
         [DataMember]
-        internal bool IsReplaying { get; set; }
+        public bool IsReplaying { get; set; }
 
         [DataMember]
         internal HistoryEvent[] History { get; set; }
@@ -35,6 +35,15 @@ public class OrchestrationContext
 
         internal OrchestrationActionCollector OrchestrationActionCollector { get; } = new OrchestrationActionCollector();
 
+        internal object ExternalResult;
+        internal bool ExternalIsError;
+
+        internal void SetExternalResult(object result, bool isError)
+        {
+            this.ExternalResult = result;
+            this.ExternalIsError = isError;
+        }
+
         internal object CustomStatus { get; set; }
     }
 }