Merge branch 'dev' into log_thread

Author: YunchuWang

.ci/e2e_integration_test/pipeline.yml

@@ -11,7 +11,6 @@ pool:
 
 variables:
     DOTNET_VERSION: '3.1.x'
-    DOTNET_VERSION_5: '5.0.x'
     DOTNET_VERSION_6: '6.x'
     PYTHON_VERSION: '3.9'
 
@@ -26,11 +25,6 @@ steps:
   inputs:
     packageType: 'sdk'
     version: $(DOTNET_VERSION)
-- task: UseDotNet@2
-  displayName: 'Install DotNet 5'
-  inputs:
-    packageType: 'sdk'
-    version: $(DOTNET_VERSION_5)
 - task: UseDotNet@2
   displayName: 'Install DotNet 6'
   inputs:
@@ -42,6 +36,7 @@ steps:
     AzureWebJobsCosmosDBConnectionString: $(AzureWebJobsCosmosDBConnectionString)
     AzureWebJobsEventHubConnectionString: $(AzureWebJobsEventHubConnectionString)
     AzureWebJobsServiceBusConnectionString: $(AzureWebJobsServiceBusConnectionString)
+    AzureWebJobsSqlConnectionString: $(AzureWebJobsSqlConnectionString)
     AzureWebJobsEventGridTopicUri: $(AzureWebJobsEventGridTopicUri)
     AzureWebJobsEventGridConnectionKey: $(AzureWebJobsEventGridConnectionKey)
     PythonVersion: $(PYTHON_VERSION)

.ci/e2e_integration_test/start-e2e.ps1

@@ -88,7 +88,7 @@ Write-Host "--------------------------------------------------------------------
 Write-Host "Preparing E2E integration tests..." -ForegroundColor Green
 Write-Host "-----------------------------------------------------------------------------`n" -ForegroundColor Green
 python -m pip install -U pip
-pip install -e .[dev]
+python -m pip install -U -e .[dev]
 python setup.py build
 python setup.py extension
 Write-Host "-----------------------------------------------------------------------------`n" -ForegroundColor Green
@@ -97,5 +97,5 @@ Write-Host "--------------------------------------------------------------------
 Write-Host "Running E2E integration tests..." -ForegroundColor Green
 Write-Host "-----------------------------------------------------------------------------`n" -ForegroundColor Green
 $env:CORE_TOOLS_EXE_PATH = "$funcExePath"
-pytest --junitxml=e2e-integration-test-report.xml --cov=./azure_functions_worker --cov-branch --cov-append tests/endtoend --cov-report xml --cov-report html
+python -m pytest --junitxml=e2e-integration-test-report.xml --reruns 4 tests/endtoend
 Write-Host "-----------------------------------------------------------------------------`n" -ForegroundColor Green

.ci/linux_devops_e2e_tests.sh

@@ -5,6 +5,7 @@ export AzureWebJobsStorage=$LINUXSTORAGECONNECTIONSTRING
 export AzureWebJobsCosmosDBConnectionString=$LINUXCOSMOSDBCONNECTIONSTRING
 export AzureWebJobsEventHubConnectionString=$LINUXEVENTHUBCONNECTIONSTRING
 export AzureWebJobsServiceBusConnectionString=$LINUXSERVICEBUSCONNECTIONSTRING
+export AzureWebJobsSqlConnectionString=$LINUXSQLCONNECTIONSTRING
 export AzureWebJobsEventGridTopicUri=$LINUXEVENTGRIDTOPICURI
 export AzureWebJobsEventGridConnectionKey=$LINUXEVENTGRIDTOPICCONNECTIONKEY
 

.ci/perf_tests/dockerfiles/perf_tests.Dockerfile

@@ -1,4 +1,4 @@
-ARG PYTHON_VERSION=3.8
+ARG PYTHON_VERSION=3.11
 
 FROM mcr.microsoft.com/azure-functions/python:4-python$PYTHON_VERSION
 
@@ -15,7 +15,8 @@ ENV AzureWebJobsScriptRoot=/home/site/wwwroot \
     FUNCTIONS_WORKER_SHARED_MEMORY_DATA_TRANSFER_ENABLED=1
 
 RUN apt-get --quiet update && \
-    apt-get install --quiet -y git && \
+    apt-get install --quiet -y git procps && \
+    # Procps is required for displaying worker and profiling processes info
     cd /home && \
     git clone https://github.com/vrdmr/AzFunctionsPythonPerformance.git && \
     mkdir -p /home/site/wwwroot/ && \

.ci/perf_tests/dockerfiles/perf_tests_3.6.Dockerfile

@@ -194,7 +194,8 @@ VOLUME ["/azure-functions-host/workers/python/3.8/LINUX/X64/azure_functions_work
 RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 379CE192D401AB61 && \
     echo "deb https://dl.bintray.com/loadimpact/deb stable main" | tee -a /etc/apt/sources.list  && \
     apt-get update && \
-    apt-get install -y git k6 && \
+    apt-get install -y git k6 procps && \
+    # Procps is required for displaying worker and profiling processes info
     cd /home && \
     git clone https://github.com/vrdmr/AzFunctionsPythonPerformance.git && \
     mkdir -p /home/site/wwwroot/ && \

.ci/perf_tests/dockerfiles/perf_tests_3.7.Dockerfile

@@ -35,7 +35,7 @@ export let options = {
 
 // Main function
 export default function () {
-    let response = http.get(`${PROTOCOL}://${HOSTNAME}:${PORT}/api/SyncHttpTriggerHelloWorld`);
+    let response = http.get(`${PROTOCOL}://${HOSTNAME}:${PORT}/api/SyncHelloWorld`);
 
     // check() returns false if any of the specified conditions fail
     let checkRes = check(response, {
@@ -45,4 +45,4 @@ export default function () {
 
     // We reverse the check() result since we want to count the failures
     failureRate.add(!checkRes);
-}
+}

.ci/perf_tests/dockerfiles/perf_tests_3.9.Dockerfile

@@ -0,0 +1,21 @@
+# See here for image contents: https://github.com/microsoft/vscode-dev-containers/tree/v0.234.0/containers/python-3/.devcontainer/base.Dockerfile
+
+# [Choice] Python version (use -bullseye variants on local arm64/Apple Silicon): 3, 3.10, 3.9, 3.8, 3.7, 3.6, 3-bullseye, 3.10-bullseye, 3.9-bullseye, 3.8-bullseye, 3.7-bullseye, 3.6-bullseye, 3-buster, 3.10-buster, 3.9-buster, 3.8-buster, 3.7-buster, 3.6-buster
+ARG VARIANT="3.9"
+FROM mcr.microsoft.com/vscode/devcontainers/python:0-${VARIANT}
+
+# [Choice] Node.js version: none, lts/*, 16, 14, 12, 10
+ARG NODE_VERSION="none"
+RUN if [ "${NODE_VERSION}" != "none" ]; then su vscode -c "umask 0002 && . /usr/local/share/nvm/nvm.sh && nvm install ${NODE_VERSION} 2>&1"; fi
+
+# [Optional] If your pip requirements rarely change, uncomment this section to add them to the image.
+# COPY requirements.txt /tmp/pip-tmp/
+# RUN pip3 --disable-pip-version-check --no-cache-dir install -r /tmp/pip-tmp/requirements.txt \
+#    && rm -rf /tmp/pip-tmp
+
+# [Optional] Uncomment this section to install additional OS packages.
+# RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
+#     && apt-get -y install --no-install-recommends <your-package-list-here>
+
+# [Optional] Uncomment this line to install global node packages.
+# RUN su vscode -c "source /usr/local/share/nvm/nvm.sh && npm install -g <your-package-here>" 2>&1

.ci/perf_tests/k6scripts/SyncHttpTriggerHelloWorld.js renamed to .ci/perf_tests/k6scripts/SyncHelloWorld.js

@@ -0,0 +1,57 @@
+// For format details, see https://aka.ms/devcontainer.json. For config options, see the README at:
+// https://github.com/microsoft/vscode-dev-containers/tree/v0.234.0/containers/python-3
+{
+	"name": "Python 3",
+	"build": {
+		"dockerfile": "Dockerfile",
+		"context": "..",
+		"args": { 
+			// Update 'VARIANT' to pick a Python version: 3, 3.10, 3.9, 3.8, 3.7
+			// Append -bullseye or -buster to pin to an OS version.
+			// Use -bullseye variants on local on arm64/Apple Silicon.
+			"VARIANT": "3.9",
+			// Options
+			"NODE_VERSION": "none"
+		}
+	},
+
+	// Set *default* container specific settings.json values on container create.
+	"settings": { 
+		"python.defaultInterpreterPath": "/usr/local/bin/python",
+		"python.linting.enabled": true,
+		"python.linting.pylintEnabled": true,
+		"python.formatting.autopep8Path": "/usr/local/py-utils/bin/autopep8",
+		"python.formatting.blackPath": "/usr/local/py-utils/bin/black",
+		"python.formatting.yapfPath": "/usr/local/py-utils/bin/yapf",
+		"python.linting.banditPath": "/usr/local/py-utils/bin/bandit",
+		"python.linting.flake8Path": "/usr/local/py-utils/bin/flake8",
+		"python.linting.mypyPath": "/usr/local/py-utils/bin/mypy",
+		"python.linting.pycodestylePath": "/usr/local/py-utils/bin/pycodestyle",
+		"python.linting.pydocstylePath": "/usr/local/py-utils/bin/pydocstyle",
+		"python.linting.pylintPath": "/usr/local/py-utils/bin/pylint",
+		"python.testing.pytestArgs": [
+			"tests/unittests"
+		],
+		"python.testing.unittestEnabled": false,
+		"python.testing.pytestEnabled": true
+	},
+
+	// Add the IDs of extensions you want installed when the container is created.
+	"extensions": [
+		"ms-python.python",
+		"ms-python.vscode-pylance"
+	],
+
+	// Use 'forwardPorts' to make a list of ports inside the container available locally.
+	// "forwardPorts": [],
+
+	// Use 'postCreateCommand' to run commands after the container is created.
+	"postCreateCommand": "sudo python -m pip install -U pip && sudo python -m pip install -U -e .[dev] && sudo python setup.py webhost",
+
+	// Comment out to connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root.
+	"remoteUser": "vscode",
+
+	"features": {
+		"dotnet": "latest"
+	}
+}

.devcontainer/Dockerfile

@@ -12,4 +12,4 @@ exclude = .git, __pycache__, build, dist, .eggs, .github, .local, docs/,
           tests/unittests/broken_functions/syntax_error/main.py,
           .env*, .vscode, venv*, *.venv*
 
-max-line-length = 80
+max-line-length = 88

.devcontainer/devcontainer.json

@@ -0,0 +1,69 @@
+title: "[Survey] "
+labels: ["Survey"]
+body:
+  - type: checkboxes
+    attributes:
+      label: What stack do you primarily use?
+      options:
+        - label: Python
+        - label: Node
+        - label: PowerShell
+        - label: .NET
+        - label: Java
+        - label: Go
+        - label: Other (Please specify below)
+  - type: textarea
+    id: role
+    attributes:
+      label: What is your current role in which you use Azure Functions?
+      description: Software Developer, Machine Learning Engineer, Full Stack Engineer, etc.
+      render: bash
+    validations:
+      required: false
+  - type: textarea
+    id: scenarios
+    attributes:
+      label: What are the use cases and scenarios in which you use Azure Functions today?
+      description: Describe how you are using Azure Functions, and expected end goals. 
+      render: bash
+    validations:
+      required: false
+  - type: textarea
+    id: pain-points
+    attributes:
+      label: What are the pain points you have experienced using Azure Functions?
+      render: bash
+    validations:
+      required: false
+  - type: textarea
+    id: improvements
+    attributes:
+      label: Please share any suggestions for product improvements.
+      render: bash
+    validations:
+      required: false
+  - type: textarea
+    id: tooling
+    attributes:
+      label: What developer tools do you use? 
+      description: Visual Studio Code, Command Line, etc.
+      render: bash
+    validations:
+      required: false
+  - type: checkboxes
+    attributes:
+      label: Would you be interested in meeting with the Azure Functions team to provide additional feedback?
+      options:
+        - label: "Yes"
+        - label: "No"
+  - type: textarea
+    id: comments
+    attributes:
+      label: Please provide any other comments or feedback here, as well as your email if you indicated 'Yes' to being contacted from the Azure Functions team.
+      render: bash
+    validations:
+      required: false
+  - type: markdown
+    attributes:
+      value: |
+        "Thank you for your feedback! For any specific questions or to report a bug, please create an issue [here](https://github.com/Azure/azure-functions-python-worker/issues)."

.flake8

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+python -m pytest -q -n auto --dist loadfile --reruns 4 --instafail --cov=./azure_functions_worker --cov-report xml --cov-branch --cov-append tests/endtoend/test_worker_process_count_functions.py tests/endtoend/test_threadpool_thread_count_functions.py
+python -m pytest -q -n auto --dist loadfile --reruns 4 --instafail --cov=./azure_functions_worker --cov-report xml --cov-branch --cov-append --ignore=tests/endtoend/test_worker_process_count_functions.py --ignore=tests/endtoend/test_threadpool_thread_count_functions.py tests/endtoend