diff --git a/.github/ISSUE_TEMPLATE/sdk_bug_report.md b/.github/ISSUE_TEMPLATE/sdk_bug_report.md new file mode 100644 index 000000000..38422b9d9 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/sdk_bug_report.md @@ -0,0 +1,31 @@ +--- +name: Bug report +about: Create a report to help us improve +title: '' +labels: '' +assignees: '' +--- + +## SDK Bindings Bug Report +This is a bug report for the newly added SDK bindings feature for Python Azure Functions. + +- **Package Name**: +- **Package Version**: +- **Operating System**: +- **Python Version**: + +**Describe the bug** +A clear and concise description of what the bug is. + +**To Reproduce** +Steps to reproduce the behavior: +1. + +**Expected behavior** +A clear and concise description of what you expected to happen. + +**Screenshots** +If applicable, add screenshots to help explain your problem. + +**Additional context** +Add any other context about the problem here. \ No newline at end of file diff --git a/.github/ISSUE_TEMPLATE/sdk_feature_request.md b/.github/ISSUE_TEMPLATE/sdk_feature_request.md new file mode 100644 index 000000000..3861bbdbd --- /dev/null +++ b/.github/ISSUE_TEMPLATE/sdk_feature_request.md @@ -0,0 +1,23 @@ +--- +name: Feature request +about: Suggest an idea for this project +title: '' +labels: '' +assignees: '' + +--- + +## SDK Bindings Feature Request +This is a feature request for the newly added SDK bindings feature for Python Azure Functions. + +- **Is your feature request related to a problem? Please describe.** +A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] + +- **Describe the solution you'd like** +A clear and concise description of what you want to happen. + +- **Describe alternatives you've considered** +A clear and concise description of any alternative solutions or features you've considered. + +- **Additional context** +Add any other context or screenshots about the feature request here. \ No newline at end of file diff --git a/.github/workflows/ci_deferred_bindings_workflow.yml b/.github/workflows/ci_deferred_bindings_workflow.yml new file mode 100644 index 000000000..f5cbe0838 --- /dev/null +++ b/.github/workflows/ci_deferred_bindings_workflow.yml @@ -0,0 +1,123 @@ +# This workflow will install Python dependencies and run end to end tests with single version of Python +# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions + +name: CI Deferred Bindings tests + +on: + workflow_dispatch: + inputs: + archive_webhost_logging: + description: "For debugging purposes, archive test webhost logs" + required: false + default: "false" + push: + branches: [dev, main, release/*] + pull_request: + branches: [dev, main, release/*] + schedule: + # Monday to Thursday 1 AM PDT build + # * is a special character in YAML so you have to quote this string + - cron: "0 8 * * 1,2,3,4" + +jobs: + build: + name: "Python Deferred Bindings CI Run" + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: [3.9, "3.10", "3.11"] + permissions: read-all + steps: + - name: Checkout code. + uses: actions/checkout@v4 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + - name: Set up Dotnet 6.x + uses: actions/setup-dotnet@v4 + with: + dotnet-version: "6.x" + - name: Install dependencies and the worker + run: | + retry() { + local -r -i max_attempts="$1"; shift + local -r cmd="$@" + local -i attempt_num=1 + until $cmd + do + if (( attempt_num == max_attempts )) + then + echo "Attempt $attempt_num failed and there are no more attempts left!" + return 1 + else + echo "Attempt $attempt_num failed! Trying again in $attempt_num seconds..." + sleep 1 + fi + done + } + + python -m pip install --upgrade pip + python -m pip install --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple -U azure-functions --pre + python -m pip install -U -e .[dev] + python -m pip install --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple --pre -U -e .[deferred-bindings] + + # Retry a couple times to avoid certificate issue + retry 5 python setup.py build + retry 5 python setup.py webhost --branch-name=dev + retry 5 python setup.py extension --test-type=deferred-bindings + mkdir logs + - name: Running 3.9 Tests + if: matrix.python-version == 3.9 + env: + AzureWebJobsStorage: ${{ secrets.LinuxStorageConnectionString39 }} + AzureWebJobsCosmosDBConnectionString: ${{ secrets.LinuxCosmosDBConnectionString39 }} + AzureWebJobsEventHubConnectionString: ${{ secrets.LinuxEventHubConnectionString39 }} + AzureWebJobsServiceBusConnectionString: ${{ secrets.LinuxServiceBusConnectionString39 }} + AzureWebJobsSqlConnectionString: ${{ secrets.LinuxSqlConnectionString39 }} + AzureWebJobsEventGridTopicUri: ${{ secrets.LinuxEventGridTopicUriString39 }} + AzureWebJobsEventGridConnectionKey: ${{ secrets.LinuxEventGridConnectionKeyString39 }} + ARCHIVE_WEBHOST_LOGS: ${{ github.event.inputs.archive_webhost_logging }} + run: | + python -m pytest -q -n auto --dist loadfile --reruns 4 --instafail --cov=./azure_functions_worker --cov-report xml --cov-branch --cov-append tests/extension_tests/deferred_bindings_tests + - name: Running 3.10 Tests + if: matrix.python-version == 3.10 + env: + AzureWebJobsStorage: ${{ secrets.LinuxStorageConnectionString310 }} + AzureWebJobsCosmosDBConnectionString: ${{ secrets.LinuxCosmosDBConnectionString310 }} + AzureWebJobsEventHubConnectionString: ${{ secrets.LinuxEventHubConnectionString310 }} + AzureWebJobsServiceBusConnectionString: ${{ secrets.LinuxServiceBusConnectionString310 }} + AzureWebJobsSqlConnectionString: ${{ secrets.LinuxSqlConnectionString310 }} + AzureWebJobsEventGridTopicUri: ${{ secrets.LinuxEventGridTopicUriString310 }} + AzureWebJobsEventGridConnectionKey: ${{ secrets.LinuxEventGridConnectionKeyString310 }} + ARCHIVE_WEBHOST_LOGS: ${{ github.event.inputs.archive_webhost_logging }} + run: | + python -m pytest -q -n auto --dist loadfile --reruns 4 --instafail --cov=./azure_functions_worker --cov-report xml --cov-branch --cov-append tests/extension_tests/deferred_bindings_tests + - name: Running 3.11 Tests + if: matrix.python-version == 3.11 + env: + AzureWebJobsStorage: ${{ secrets.LinuxStorageConnectionString311 }} + AzureWebJobsCosmosDBConnectionString: ${{ secrets.LinuxCosmosDBConnectionString311 }} + AzureWebJobsEventHubConnectionString: ${{ secrets.LinuxEventHubConnectionString311 }} + AzureWebJobsServiceBusConnectionString: ${{ secrets.LinuxServiceBusConnectionString311 }} + AzureWebJobsSqlConnectionString: ${{ secrets.LinuxSqlConnectionString311 }} + AzureWebJobsEventGridTopicUri: ${{ secrets.LinuxEventGridTopicUriString311 }} + AzureWebJobsEventGridConnectionKey: ${{ secrets.LinuxEventGridConnectionKeyString311 }} + ARCHIVE_WEBHOST_LOGS: ${{ github.event.inputs.archive_webhost_logging }} + run: | + python -m pytest -q -n auto --dist loadfile --reruns 4 --instafail --cov=./azure_functions_worker --cov-report xml --cov-branch --cov-append tests/extension_tests/deferred_bindings_tests + - name: Codecov + uses: codecov/codecov-action@v3 + with: + file: ./coverage.xml # optional + flags: unittests # optional + name: codecov # optional + fail_ci_if_error: false # optional (default = false) + - name: Publish Logs to Artifact + if: failure() + uses: actions/upload-artifact@v4 + with: + name: Test WebHost Logs ${{ github.run_id }} ${{ matrix.python-version }} + path: logs/*.log + if-no-files-found: ignore diff --git a/azure_functions_worker/bindings/datumdef.py b/azure_functions_worker/bindings/datumdef.py index b420fbf3b..d367b2fc2 100644 --- a/azure_functions_worker/bindings/datumdef.py +++ b/azure_functions_worker/bindings/datumdef.py @@ -93,6 +93,8 @@ def from_typed_data(cls, td: protos.TypedData): val = td.collection_string elif tt == 'collection_sint64': val = td.collection_sint64 + elif tt == 'model_binding_data': + val = td.model_binding_data elif tt is None: return None else: diff --git a/azure_functions_worker/bindings/meta.py b/azure_functions_worker/bindings/meta.py index f7a810145..42a065a99 100644 --- a/azure_functions_worker/bindings/meta.py +++ b/azure_functions_worker/bindings/meta.py @@ -4,6 +4,7 @@ import typing from .. import protos +from ..constants import BASE_EXT_SUPPORTED_PY_MINOR_VERSION from . import datumdef from . import generic @@ -13,6 +14,9 @@ PB_TYPE_DATA = 'data' PB_TYPE_RPC_SHARED_MEMORY = 'rpc_shared_memory' BINDING_REGISTRY = None +SDK_BINDING_REGISTRY = None +DEFERRED_BINDINGS_ENABLED = False +SDK_CACHE = {} def load_binding_registry() -> None: @@ -25,12 +29,25 @@ def load_binding_registry() -> None: global BINDING_REGISTRY BINDING_REGISTRY = func.get_binding_registry() - -def get_binding(bind_name: str) -> object: - binding = None - registry = BINDING_REGISTRY - if registry is not None: - binding = registry.get(bind_name) + # The SDKs only support python 3.8+ + if sys.version_info.minor >= BASE_EXT_SUPPORTED_PY_MINOR_VERSION: + # Import the base extension + try: + import azure.functions.extension.base as clients + global SDK_BINDING_REGISTRY + SDK_BINDING_REGISTRY = clients.get_binding_registry() + except ImportError: + # This will throw a ModuleNotFoundError in env reload because + # azure.functions.extension isn't loaded in + pass + + +def get_binding(bind_name: str, pytype: typing.Optional[type] = None) -> object: + binding = get_deferred_binding(bind_name=bind_name, pytype=pytype) + + # Either cx didn't import library or didn't define sdk type + if binding is None and BINDING_REGISTRY is not None: + binding = BINDING_REGISTRY.get(bind_name) if binding is None: binding = generic.GenericBinding @@ -43,7 +60,8 @@ def is_trigger_binding(bind_name: str) -> bool: def check_input_type_annotation(bind_name: str, pytype: type) -> bool: - binding = get_binding(bind_name) + # check that needs to pass for sdk bindings -- pass in pytype + binding = get_binding(bind_name, pytype) return binding.check_input_type_annotation(pytype) @@ -72,7 +90,7 @@ def from_incoming_proto( pytype: typing.Optional[type], trigger_metadata: typing.Optional[typing.Dict[str, protos.TypedData]], shmem_mgr: SharedMemoryManager) -> typing.Any: - binding = get_binding(binding) + binding = get_binding(binding, pytype) if trigger_metadata: metadata = { k: datumdef.Datum.from_typed_data(v) @@ -93,6 +111,14 @@ def from_incoming_proto( raise TypeError(f'Unknown ParameterBindingType: {pb_type}') try: + # if the binding is an sdk type binding + if (SDK_BINDING_REGISTRY is not None + and SDK_BINDING_REGISTRY.check_supported_type(pytype)): + return deferred_bindings_decode(binding=binding, + pb=pb, + pytype=pytype, + datum=datum, + metadata=metadata) return binding.decode(datum, trigger_metadata=metadata) except NotImplementedError: # Binding does not support the data. @@ -184,3 +210,41 @@ def to_outgoing_param_binding(binding: str, obj: typing.Any, *, return protos.ParameterBinding( name=out_name, data=rpc_val) + + +def get_deferred_binding(bind_name: str, + pytype: typing.Optional[type] = None) -> object: + binding = None + + # Checks if pytype is a supported sdk type + if (SDK_BINDING_REGISTRY is not None + and SDK_BINDING_REGISTRY.check_supported_type(pytype)): + # Set flag once + global DEFERRED_BINDINGS_ENABLED + if not DEFERRED_BINDINGS_ENABLED: + DEFERRED_BINDINGS_ENABLED = True + binding = SDK_BINDING_REGISTRY.get(bind_name) + + # This will return None if not a supported type + return binding + + +def deferred_bindings_decode(binding: typing.Any, + pb: protos.ParameterBinding, *, + pytype: typing.Optional[type], + datum: typing.Any, + metadata: typing.Any): + global SDK_CACHE + # Check is the object is already in the cache + # If dict is empty or key doesn't exist, obj is None + obj = SDK_CACHE.get((pb.name, pytype, datum.value.content), None) + + # If the object is in the cache, return it + if obj is not None: + return obj + # If the object is not in the cache, create and add it to the cache + else: + obj = binding.decode(datum, trigger_metadata=metadata, + pytype=pytype) + SDK_CACHE[(pb.name, pytype, datum.value.content)] = obj + return obj diff --git a/azure_functions_worker/constants.py b/azure_functions_worker/constants.py index b6cc668b6..fa2cc56d2 100644 --- a/azure_functions_worker/constants.py +++ b/azure_functions_worker/constants.py @@ -56,6 +56,9 @@ # Paths CUSTOMER_PACKAGES_PATH = "/home/site/wwwroot/.python_packages/lib/site-packages" +# Base extension supported Python minor version +BASE_EXT_SUPPORTED_PY_MINOR_VERSION = 8 + # Flag to index functions in handle init request PYTHON_ENABLE_INIT_INDEXING = "PYTHON_ENABLE_INIT_INDEXING" diff --git a/azure_functions_worker/loader.py b/azure_functions_worker/loader.py index 938fde64c..f73cd8a06 100644 --- a/azure_functions_worker/loader.py +++ b/azure_functions_worker/loader.py @@ -14,7 +14,7 @@ from google.protobuf.duration_pb2 import Duration -from . import protos, functions +from . import protos, functions, bindings from .bindings.retrycontext import RetryPolicy from .utils.common import get_app_setting from .constants import MODULE_NOT_FOUND_TS_URL, PYTHON_SCRIPT_FILE_NAME, \ @@ -130,6 +130,9 @@ def process_indexed_function(functions_registry: functions.Registry, binding_protos = build_binding_protos(indexed_function) retry_protos = build_retry_protos(indexed_function) + raw_bindings = get_fx_raw_bindings(indexed_function=indexed_function, + function_info=function_info) + function_metadata = protos.RpcFunctionMetadata( name=function_info.name, function_id=function_info.function_id, @@ -140,7 +143,7 @@ def process_indexed_function(functions_registry: functions.Registry, is_proxy=False, # not supported in V4 language=PYTHON_LANGUAGE_RUNTIME, bindings=binding_protos, - raw_bindings=indexed_function.get_raw_bindings(), + raw_bindings=raw_bindings, retry_options=retry_protos, properties={METADATA_PROPERTIES_WORKER_INDEXED: "True"}) @@ -239,3 +242,17 @@ def index_function_app(function_path: str): f"{script_file_name}.") return app.get_functions() + + +def get_fx_raw_bindings(indexed_function, function_info): + # Check if deferred bindings is enabled + if (bindings.meta is not None + and bindings.meta.DEFERRED_BINDINGS_ENABLED + and bindings.meta.SDK_BINDING_REGISTRY is not None): + # Reset the flag + bindings.meta.DEFERRED_BINDINGS_ENABLED = False + return bindings.meta.SDK_BINDING_REGISTRY.get_raw_bindings( + indexed_function, function_info.input_types) + + else: + return indexed_function.get_raw_bindings() diff --git a/azure_functions_worker/logging.py b/azure_functions_worker/logging.py index adb5ff294..d86c39c81 100644 --- a/azure_functions_worker/logging.py +++ b/azure_functions_worker/logging.py @@ -13,7 +13,6 @@ SDK_LOG_PREFIX = "azure.functions" SYSTEM_ERROR_LOG_PREFIX = "azure_functions_worker_errors" - logger: logging.Logger = logging.getLogger(SYSTEM_LOG_PREFIX) error_logger: logging.Logger = ( logging.getLogger(SYSTEM_ERROR_LOG_PREFIX)) diff --git a/setup.py b/setup.py index a3970fe19..478176cc8 100644 --- a/setup.py +++ b/setup.py @@ -20,8 +20,10 @@ from setuptools import setup from setuptools.command import develop +from azure_functions_worker.constants import BASE_EXT_SUPPORTED_PY_MINOR_VERSION from azure_functions_worker.version import VERSION -from tests.utils.constants import EXTENSIONS_CSPROJ_TEMPLATE +from tests.utils.constants import (DEFERRED_BINDINGS_CSPROJ_TEMPLATE, + EXTENSIONS_CSPROJ_TEMPLATE) # The GitHub repository of the Azure Functions Host WEBHOST_GITHUB_API = "https://api.github.com/repos/Azure/azure-functions-host" @@ -82,6 +84,9 @@ ("protobuf~=4.22.0", "grpcio-tools~=1.54.2", "grpcio~=1.54.2") ) +if sys.version_info.minor >= BASE_EXT_SUPPORTED_PY_MINOR_VERSION: + INSTALL_REQUIRES.append("azure-functions-extension-base") + EXTRA_REQUIRES = { "dev": [ "azure-eventhub~=5.7.0", # Used for EventHub E2E tests @@ -109,6 +114,9 @@ "pandas", "numpy", "pre-commit" + ], + "deferred-bindings": [ + "azure-functions-extension-blob" ] } @@ -218,12 +226,18 @@ class Extension(distutils.cmd.Command): "extensions-dir", None, "A path to the directory where extension should be installed", + ), + ( + "test-type=", + None, + "The type of test to be run", ) ] def __init__(self, dist: Distribution): super().__init__(dist) self.extensions_dir = None + self.test_type = None def initialize_options(self): pass @@ -241,8 +255,12 @@ def _install_extensions(self): print("{}", file=f) if not (self.extensions_dir / "extensions.csproj").exists(): - with open(self.extensions_dir / "extensions.csproj", "w") as f: - print(EXTENSIONS_CSPROJ_TEMPLATE, file=f) + if self.test_type == "deferred-bindings": + with open(self.extensions_dir / "extensions.csproj", "w") as f: + print(DEFERRED_BINDINGS_CSPROJ_TEMPLATE, file=f) + else: + with open(self.extensions_dir / "extensions.csproj", "w") as f: + print(EXTENSIONS_CSPROJ_TEMPLATE, file=f) with open(self.extensions_dir / "NuGet.config", "w") as f: print(NUGET_CONFIG, file=f) diff --git a/tests/extension_tests/deferred_bindings_tests/deferred_bindings_blob_functions/function_app.py b/tests/extension_tests/deferred_bindings_tests/deferred_bindings_blob_functions/function_app.py new file mode 100644 index 000000000..6fac92113 --- /dev/null +++ b/tests/extension_tests/deferred_bindings_tests/deferred_bindings_blob_functions/function_app.py @@ -0,0 +1,240 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +import json + +import azure.functions as func +import azure.functions.extension.blob as bindings + +app = func.FunctionApp(http_auth_level=func.AuthLevel.ANONYMOUS) + + +@app.function_name(name="put_bc_trigger") +@app.blob_output(arg_name="file", + path="python-worker-tests/test-blobclient-trigger.txt", + connection="AzureWebJobsStorage") +@app.route(route="put_bc_trigger") +def put_bc_trigger(req: func.HttpRequest, file: func.Out[str]) -> str: + file.set(req.get_body()) + return 'OK' + + +@app.function_name(name="bc_blob_trigger") +@app.blob_trigger(arg_name="client", + path="python-worker-tests/test-blobclient-trigger.txt", + connection="AzureWebJobsStorage") +@app.blob_output(arg_name="$return", + path="python-worker-tests/test-blobclient-triggered.txt", + connection="AzureWebJobsStorage") +def bc_blob_trigger(client: bindings.BlobClient) -> str: + blob_properties = client.get_blob_properties() + file = client.download_blob(encoding='utf-8').readall() + return json.dumps({ + 'name': blob_properties.name, + 'length': blob_properties.size, + 'content': file + }) + + +@app.function_name(name="get_bc_blob_triggered") +@app.blob_input(arg_name="client", + path="python-worker-tests/test-blobclient-triggered.txt", + connection="AzureWebJobsStorage") +@app.route(route="get_bc_blob_triggered") +def get_bc_blob_triggered(req: func.HttpRequest, + client: bindings.BlobClient) -> str: + return client.download_blob(encoding='utf-8').readall() + + +@app.function_name(name="put_cc_trigger") +@app.blob_output(arg_name="file", + path="python-worker-tests/test-containerclient-trigger.txt", + connection="AzureWebJobsStorage") +@app.route(route="put_cc_trigger") +def put_cc_trigger(req: func.HttpRequest, file: func.Out[str]) -> str: + file.set(req.get_body()) + return 'OK' + + +@app.function_name(name="cc_blob_trigger") +@app.blob_trigger(arg_name="client", + path="python-worker-tests/test-containerclient-trigger.txt", + connection="AzureWebJobsStorage") +@app.blob_output(arg_name="$return", + path="python-worker-tests/test-containerclient-triggered.txt", + connection="AzureWebJobsStorage") +def cc_blob_trigger(client: bindings.ContainerClient) -> str: + container_properties = client.get_container_properties() + file = client.download_blob("test-containerclient-trigger.txt", + encoding='utf-8').readall() + return json.dumps({ + 'name': container_properties.name, + 'content': file + }) + + +@app.function_name(name="get_cc_blob_triggered") +@app.blob_input(arg_name="client", + path="python-worker-tests/test-containerclient-triggered.txt", + connection="AzureWebJobsStorage") +@app.route(route="get_cc_blob_triggered") +def get_cc_blob_triggered(req: func.HttpRequest, + client: bindings.ContainerClient) -> str: + return client.download_blob("test-containerclient-triggered.txt", + encoding='utf-8').readall() + + +@app.function_name(name="put_ssd_trigger") +@app.blob_output(arg_name="file", + path="python-worker-tests/test-ssd-trigger.txt", + connection="AzureWebJobsStorage") +@app.route(route="put_ssd_trigger") +def put_ssd_trigger(req: func.HttpRequest, file: func.Out[str]) -> str: + file.set(req.get_body()) + return 'OK' + + +@app.function_name(name="ssd_blob_trigger") +@app.blob_trigger(arg_name="stream", + path="python-worker-tests/test-ssd-trigger.txt", + connection="AzureWebJobsStorage") +@app.blob_output(arg_name="$return", + path="python-worker-tests/test-ssd-triggered.txt", + connection="AzureWebJobsStorage") +def ssd_blob_trigger(stream: bindings.StorageStreamDownloader) -> str: + # testing chunking + file = "" + for chunk in stream.chunks(): + file += chunk.decode("utf-8") + return json.dumps({ + 'content': file + }) + + +@app.function_name(name="get_ssd_blob_triggered") +@app.blob_input(arg_name="stream", + path="python-worker-tests/test-ssd-triggered.txt", + connection="AzureWebJobsStorage") +@app.route(route="get_ssd_blob_triggered") +def get_ssd_blob_triggered(req: func.HttpRequest, + stream: bindings.StorageStreamDownloader) -> str: + return stream.readall().decode('utf-8') + + +@app.function_name(name="get_bc_bytes") +@app.route(route="get_bc_bytes") +@app.blob_input(arg_name="client", + path="python-worker-tests/test-blob-extension-bytes.txt", + connection="AzureWebJobsStorage") +def get_bc_bytes(req: func.HttpRequest, client: bindings.BlobClient) -> str: + return client.download_blob(encoding='utf-8').readall() + + +@app.function_name(name="get_cc_bytes") +@app.route(route="get_cc_bytes") +@app.blob_input(arg_name="client", + path="python-worker-tests/test-blob-extension-bytes.txt", + connection="AzureWebJobsStorage") +def get_cc_bytes(req: func.HttpRequest, + client: bindings.ContainerClient) -> str: + return client.download_blob("test-blob-extension-bytes.txt", + encoding='utf-8').readall() + + +@app.function_name(name="get_ssd_bytes") +@app.route(route="get_ssd_bytes") +@app.blob_input(arg_name="stream", + path="python-worker-tests/test-blob-extension-bytes.txt", + connection="AzureWebJobsStorage") +def get_ssd_bytes(req: func.HttpRequest, + stream: bindings.StorageStreamDownloader) -> str: + return stream.readall().decode('utf-8') + + +@app.function_name(name="get_bc_str") +@app.route(route="get_bc_str") +@app.blob_input(arg_name="client", + path="python-worker-tests/test-blob-extension-str.txt", + connection="AzureWebJobsStorage") +def get_bc_str(req: func.HttpRequest, client: bindings.BlobClient) -> str: + return client.download_blob(encoding='utf-8').readall() + + +@app.function_name(name="get_cc_str") +@app.route(route="get_cc_str") +@app.blob_input(arg_name="client", + path="python-worker-tests", + connection="AzureWebJobsStorage") +def get_cc_str(req: func.HttpRequest, client: bindings.ContainerClient) -> str: + return client.download_blob("test-blob-extension-str.txt", + encoding='utf-8').readall() + + +@app.function_name(name="get_ssd_str") +@app.route(route="get_ssd_str") +@app.blob_input(arg_name="stream", + path="python-worker-tests/test-blob-extension-str.txt", + connection="AzureWebJobsStorage") +def get_ssd_str(req: func.HttpRequest, stream: bindings.StorageStreamDownloader) -> str: + return stream.readall().decode('utf-8') + + +@app.function_name(name="bc_and_inputstream_input") +@app.route(route="bc_and_inputstream_input") +@app.blob_input(arg_name="client", + path="python-worker-tests/test-blob-extension-str.txt", + data_type="STRING", + connection="AzureWebJobsStorage") +@app.blob_input(arg_name="blob", + path="python-worker-tests/test-blob-extension-str.txt", + data_type="STRING", + connection="AzureWebJobsStorage") +def bc_and_inputstream_input(req: func.HttpRequest, client: bindings.BlobClient, + blob: func.InputStream) -> str: + output_msg = "" + file = blob.read().decode('utf-8') + client_file = client.download_blob(encoding='utf-8').readall() + output_msg = file + " - input stream " + client_file + " - blob client" + return output_msg + + +@app.function_name(name="type_undefined") +@app.route(route="type_undefined") +@app.blob_input(arg_name="file", + path="python-worker-tests/test-blob-extension-str.txt", + data_type="STRING", + connection="AzureWebJobsStorage") +def type_undefined(req: func.HttpRequest, file) -> str: + assert not isinstance(file, bindings.BlobClient) + assert not isinstance(file, bindings.ContainerClient) + assert not isinstance(file, bindings.StorageStreamDownloader) + return file.read().decode('utf-8') + + +@app.function_name(name="put_blob_str") +@app.blob_output(arg_name="file", + path="python-worker-tests/test-blob-extension-str.txt", + connection="AzureWebJobsStorage") +@app.route(route="put_blob_str") +def put_blob_str(req: func.HttpRequest, file: func.Out[str]) -> str: + file.set(req.get_body()) + return 'OK' + + +@app.function_name(name="put_blob_bytes") +@app.blob_output(arg_name="file", + path="python-worker-tests/test-blob-extension-bytes.txt", + connection="AzureWebJobsStorage") +@app.route(route="put_blob_bytes") +def put_blob_bytes(req: func.HttpRequest, file: func.Out[bytes]) -> str: + file.set(req.get_body()) + return 'OK' + + +@app.function_name(name="blob_cache") +@app.blob_input(arg_name="client", + path="python-worker-tests/test-blobclient-triggered.txt", + connection="AzureWebJobsStorage") +@app.route(route="blob_cache") +def blob_cache(req: func.HttpRequest, + client: bindings.BlobClient) -> str: + return client.download_blob(encoding='utf-8').readall() diff --git a/tests/extension_tests/deferred_bindings_tests/deferred_bindings_functions/deferred_bindings_disabled/function_app.py b/tests/extension_tests/deferred_bindings_tests/deferred_bindings_functions/deferred_bindings_disabled/function_app.py new file mode 100644 index 000000000..19034479b --- /dev/null +++ b/tests/extension_tests/deferred_bindings_tests/deferred_bindings_functions/deferred_bindings_disabled/function_app.py @@ -0,0 +1,21 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +import json +import azure.functions as func + +app = func.FunctionApp(http_auth_level=func.AuthLevel.ANONYMOUS) + + +@app.function_name(name="blob_trigger") +@app.blob_trigger(arg_name="file", + path="python-worker-tests/test-blob-trigger.txt", + connection="AzureWebJobsStorage") +@app.blob_output(arg_name="$return", + path="python-worker-tests/test-blob-triggered.txt", + connection="AzureWebJobsStorage") +def blob_trigger(file: func.InputStream) -> str: + return json.dumps({ + 'name': file.name, + 'length': file.length, + 'content': file.read().decode('utf-8') + }) diff --git a/tests/extension_tests/deferred_bindings_tests/deferred_bindings_functions/deferred_bindings_enabled/function_app.py b/tests/extension_tests/deferred_bindings_tests/deferred_bindings_functions/deferred_bindings_enabled/function_app.py new file mode 100644 index 000000000..233b2611e --- /dev/null +++ b/tests/extension_tests/deferred_bindings_tests/deferred_bindings_functions/deferred_bindings_enabled/function_app.py @@ -0,0 +1,16 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +import azure.functions as func +import azure.functions.extension.blob as bindings + +app = func.FunctionApp(http_auth_level=func.AuthLevel.ANONYMOUS) + + +@app.function_name(name="get_bc_blob_triggered") +@app.blob_input(arg_name="client", + path="python-worker-tests/test-blobclient-triggered.txt", + connection="AzureWebJobsStorage") +@app.route(route="get_bc_blob_triggered") +def get_bc_blob_triggered(req: func.HttpRequest, + client: bindings.BlobClient) -> str: + return client.download_blob(encoding='utf-8').readall() diff --git a/tests/extension_tests/deferred_bindings_tests/deferred_bindings_functions/deferred_bindings_enabled_dual/function_app.py b/tests/extension_tests/deferred_bindings_tests/deferred_bindings_functions/deferred_bindings_enabled_dual/function_app.py new file mode 100644 index 000000000..625fdabb3 --- /dev/null +++ b/tests/extension_tests/deferred_bindings_tests/deferred_bindings_functions/deferred_bindings_enabled_dual/function_app.py @@ -0,0 +1,33 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +import json + +import azure.functions as func +import azure.functions.extension.blob as bindings + +app = func.FunctionApp(http_auth_level=func.AuthLevel.ANONYMOUS) + + +@app.function_name(name="get_bc_blob_triggered") +@app.blob_input(arg_name="client", + path="python-worker-tests/test-blobclient-triggered.txt", + connection="AzureWebJobsStorage") +@app.route(route="get_bc_blob_triggered") +def get_bc_blob_triggered(req: func.HttpRequest, + client: bindings.BlobClient) -> str: + return client.download_blob(encoding='utf-8').readall() + + +@app.function_name(name="blob_trigger") +@app.blob_trigger(arg_name="file", + path="python-worker-tests/test-blob-trigger.txt", + connection="AzureWebJobsStorage") +@app.blob_output(arg_name="$return", + path="python-worker-tests/test-blob-triggered.txt", + connection="AzureWebJobsStorage") +def blob_trigger(file: func.InputStream) -> str: + return json.dumps({ + 'name': file.name, + 'length': file.length, + 'content': file.read().decode('utf-8') + }) diff --git a/tests/extension_tests/deferred_bindings_tests/test_deferred_bindings.py b/tests/extension_tests/deferred_bindings_tests/test_deferred_bindings.py new file mode 100644 index 000000000..543f62fa9 --- /dev/null +++ b/tests/extension_tests/deferred_bindings_tests/test_deferred_bindings.py @@ -0,0 +1,108 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +from azure_functions_worker import protos +from azure_functions_worker.bindings import datumdef, meta +from tests.utils import testutils +from azure.functions.extension.blob import BlobClient, BlobClientConverter + +DEFERRED_BINDINGS_ENABLED_DIR = testutils.EXTENSION_TESTS_FOLDER / \ + 'deferred_bindings_tests' / \ + 'deferred_bindings_functions' / \ + 'deferred_bindings_enabled' +DEFERRED_BINDINGS_DISABLED_DIR = testutils.EXTENSION_TESTS_FOLDER / \ + 'deferred_bindings_tests' / \ + 'deferred_bindings_functions' / \ + 'deferred_bindings_disabled' + +DEFERRED_BINDINGS_ENABLED_DUAL_DIR = testutils.EXTENSION_TESTS_FOLDER / \ + 'deferred_bindings_tests' / \ + 'deferred_bindings_functions' / \ + 'deferred_bindings_enabled_dual' + + +class MockMBD: + def __init__(self, version: str, source: str, + content_type: str, content: str): + self.version = version + self.source = source + self.content_type = content_type + self.content = content + + +class TestDeferredBindingsEnabled(testutils.AsyncTestCase): + + async def test_deferred_bindings_metadata(self): + async with testutils.start_mockhost( + script_root=DEFERRED_BINDINGS_ENABLED_DIR) as host: + await host.init_worker() + r = await host.get_functions_metadata() + self.assertIsInstance(r.response, protos.FunctionMetadataResponse) + self.assertEqual(r.response.result.status, + protos.StatusResult.Success) + + +class TestDeferredBindingsEnabledDual(testutils.AsyncTestCase): + + async def test_deferred_bindings_dual_metadata(self): + async with testutils.start_mockhost( + script_root=DEFERRED_BINDINGS_ENABLED_DUAL_DIR) as host: + await host.init_worker() + r = await host.get_functions_metadata() + self.assertIsInstance(r.response, protos.FunctionMetadataResponse) + self.assertEqual(r.response.result.status, + protos.StatusResult.Success) + + +class TestDeferredBindingsDisabled(testutils.AsyncTestCase): + + async def test_non_deferred_bindings_metadata(self): + async with testutils.start_mockhost( + script_root=DEFERRED_BINDINGS_DISABLED_DIR) as host: + await host.init_worker() + r = await host.get_functions_metadata() + self.assertIsInstance(r.response, protos.FunctionMetadataResponse) + self.assertEqual(r.response.result.status, + protos.StatusResult.Success) + self.assertFalse(meta.DEFERRED_BINDINGS_ENABLED) + + +class TestDeferredBindingsHelpers(testutils.AsyncTestCase): + + async def test_get_deferred_binding(self): + async with testutils.start_mockhost( + script_root=DEFERRED_BINDINGS_DISABLED_DIR) as host: + await host.init_worker() + bind_name = 'blob' + pytype = BlobClient + binding = meta.get_deferred_binding(bind_name=bind_name, pytype=pytype) + self.assertEquals(binding, BlobClientConverter) + + async def test_get_non_deferred_binding(self): + async with testutils.start_mockhost( + script_root=DEFERRED_BINDINGS_DISABLED_DIR) as host: + await host.init_worker() + bind_name = 'blob' + pytype = str + binding = meta.get_deferred_binding(bind_name=bind_name, pytype=pytype) + self.assertEquals(binding, None) + + def test_deferred_bindings_decode(self): + binding = BlobClientConverter + pb = protos.ParameterBinding(name='test', + data=protos.TypedData( + string='test')) + sample_mbd = MockMBD(version="1.0", + source="AzureStorageBlobs", + content_type="application/json", + content="{\"Connection\":\"AzureWebJobsStorage\"," + "\"ContainerName\":" + "\"python-worker-tests\"," + "\"BlobName\":" + "\"test-blobclient-trigger.txt\"}") + datum = datumdef.Datum(value=sample_mbd, type='model_binding_data') + + obj = meta.deferred_bindings_decode(binding=binding, pb=pb, + pytype=BlobClient, datum=datum, metadata={}) + + self.assertIsNotNone(obj) diff --git a/tests/extension_tests/deferred_bindings_tests/test_deferred_bindings_blob_functions.py b/tests/extension_tests/deferred_bindings_tests/test_deferred_bindings_blob_functions.py new file mode 100644 index 000000000..006318b35 --- /dev/null +++ b/tests/extension_tests/deferred_bindings_tests/test_deferred_bindings_blob_functions.py @@ -0,0 +1,164 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +import time + +from tests.utils import testutils +from azure_functions_worker.bindings import meta + + +class TestSdkBlobFunctions(testutils.WebHostTestCase): + + @classmethod + def get_script_dir(cls): + return testutils.EXTENSION_TESTS_FOLDER / 'deferred_bindings_tests' / \ + 'deferred_bindings_blob_functions' + + def test_blob_str(self): + r = self.webhost.request('POST', 'put_blob_str', data='test-data') + self.assertEqual(r.status_code, 200) + self.assertEqual(r.text, 'OK') + + time.sleep(5) + + r = self.webhost.request('GET', 'get_bc_str') + self.assertEqual(r.status_code, 200) + self.assertEqual(r.text, 'test-data') + + r = self.webhost.request('GET', 'get_cc_str') + self.assertEqual(r.status_code, 200) + self.assertEqual(r.text, 'test-data') + + r = self.webhost.request('GET', 'get_ssd_str') + self.assertEqual(r.status_code, 200) + self.assertEqual(r.text, 'test-data') + + def test_blob_bytes(self): + r = self.webhost.request('POST', 'put_blob_bytes', + data='test-dată'.encode('utf-8')) + self.assertEqual(r.status_code, 200) + self.assertEqual(r.text, 'OK') + + time.sleep(5) + + r = self.webhost.request('POST', 'get_bc_bytes') + self.assertEqual(r.status_code, 200) + self.assertEqual(r.text, 'test-dată') + + r = self.webhost.request('POST', 'get_cc_bytes') + self.assertEqual(r.status_code, 200) + self.assertEqual(r.text, 'test-dată') + + r = self.webhost.request('POST', 'get_ssd_bytes') + self.assertEqual(r.status_code, 200) + self.assertEqual(r.text, 'test-dată') + + def test_bc_blob_trigger(self): + data = "DummyData" + + r = self.webhost.request('POST', 'put_bc_trigger', + data=data.encode('utf-8')) + self.assertEqual(r.status_code, 200) + self.assertEqual(r.text, 'OK') + + # Blob trigger may be processed after some delay + # We check it every 2 seconds to allow the trigger to be fired + max_retries = 10 + for try_no in range(max_retries): + time.sleep(5) + + try: + # Check that the trigger has fired + r = self.webhost.request('GET', 'get_bc_blob_triggered') + self.assertEqual(r.status_code, 200) + response = r.json() + + self.assertEqual(response['name'], + 'test-blobclient-trigger.txt') + self.assertEqual(response['content'], data) + + break + except AssertionError: + if try_no == max_retries - 1: + raise + + def test_cc_blob_trigger(self): + data = "DummyData" + + r = self.webhost.request('POST', 'put_cc_trigger', + data=data.encode('utf-8')) + self.assertEqual(r.status_code, 200) + self.assertEqual(r.text, 'OK') + + # Blob trigger may be processed after some delay + # We check it every 2 seconds to allow the trigger to be fired + max_retries = 10 + for try_no in range(max_retries): + time.sleep(5) + + try: + # Check that the trigger has fired + r = self.webhost.request('GET', 'get_cc_blob_triggered') + self.assertEqual(r.status_code, 200) + response = r.json() + + self.assertEqual(response['name'], + 'python-worker-tests') + self.assertEqual(response['content'], data) + + break + except AssertionError: + if try_no == max_retries - 1: + raise + + def test_ssd_blob_trigger(self): + data = "DummyData" + + r = self.webhost.request('POST', 'put_ssd_trigger', + data=data.encode('utf-8')) + self.assertEqual(r.status_code, 200) + self.assertEqual(r.text, 'OK') + + # Blob trigger may be processed after some delay + # We check it every 2 seconds to allow the trigger to be fired + max_retries = 10 + for try_no in range(max_retries): + time.sleep(5) + + try: + # Check that the trigger has fired + r = self.webhost.request('GET', 'get_ssd_blob_triggered') + self.assertEqual(r.status_code, 200) + response = r.json() + + self.assertEqual(response['content'], data) + + break + except AssertionError: + if try_no == max_retries - 1: + raise + + def test_bc_and_inputstream_input(self): + r = self.webhost.request('POST', 'put_blob_str', data='test-data') + self.assertEqual(r.status_code, 200) + self.assertEqual(r.text, 'OK') + + r = self.webhost.request('GET', 'bc_and_inputstream_input') + self.assertEqual(r.status_code, 200) + self.assertEqual(r.text, 'test-data - input stream test-data - blob client') + + def test_type_undefined(self): + r = self.webhost.request('POST', 'put_blob_str', data='test-data') + self.assertEqual(r.status_code, 200) + self.assertEqual(r.text, 'OK') + + r = self.webhost.request('GET', 'type_undefined') + self.assertEqual(r.status_code, 200) + self.assertEqual(r.text, 'test-data') + + self.assertFalse(meta.DEFERRED_BINDINGS_ENABLED) + + def test_caching(self): + # Cache is empty at the start + self.assertEqual(meta.SDK_CACHE, {}) + r = self.webhost.request('GET', 'blob_cache') + self.assertEqual(r.status_code, 200) diff --git a/tests/unittests/test_types.py b/tests/unittests/test_types.py index 1b76ea45a..e083c503f 100644 --- a/tests/unittests/test_types.py +++ b/tests/unittests/test_types.py @@ -5,6 +5,17 @@ from azure import functions as azf from azure.functions import http as bind_http from azure.functions import meta as bind_meta +from azure_functions_worker import protos +from azure_functions_worker.bindings import datumdef + + +class MockMBD: + def __init__(self, version: str, source: str, + content_type: str, content: str): + self.version = version + self.source = source + self.content_type = content_type + self.content = content class TestFunctions(unittest.TestCase): @@ -162,3 +173,23 @@ def test_scalar_typed_data_decoder_not_ok(self): with self.assertRaisesRegex(exc, msg): Converter._decode_trigger_metadata_field( metadata, field, python_type=pytype) + + def test_model_binding_data_datum_ok(self): + sample_mbd = MockMBD(version="1.0", + source="AzureStorageBlobs", + content_type="application/json", + content="{\"Connection\":\"python-worker-tests\"," + "\"ContainerName\":\"test-blob\"," + "\"BlobName\":\"test.txt\"}") + + datum: bind_meta.Datum = bind_meta.Datum(value=sample_mbd, + type='model_binding_data') + + self.assertEqual(datum.value, sample_mbd) + self.assertEqual(datum.type, "model_binding_data") + + def test_model_binding_data_td_ok(self): + mock_mbd = protos.TypedData(model_binding_data={'version': '1.0'}) + mbd_datum = datumdef.Datum.from_typed_data(mock_mbd) + + self.assertEqual(mbd_datum.type, 'model_binding_data') diff --git a/tests/utils/constants.py b/tests/utils/constants.py index eb567a7b4..4a55f647b 100644 --- a/tests/utils/constants.py +++ b/tests/utils/constants.py @@ -2,6 +2,51 @@ # Licensed under the MIT License. import pathlib +# Extensions necessary for non-core bindings. +DEFERRED_BINDINGS_CSPROJ_TEMPLATE = """\ + + + + net60 + + ** + + + + + + + + + + + + + + + + + + + +""" + # Extensions necessary for non-core bindings. EXTENSIONS_CSPROJ_TEMPLATE = """\ diff --git a/tests/utils/testutils.py b/tests/utils/testutils.py index 57946f1eb..61dbf6dc8 100644 --- a/tests/utils/testutils.py +++ b/tests/utils/testutils.py @@ -57,6 +57,7 @@ E2E_TESTS_ROOT = TESTS_ROOT / E2E_TESTS_FOLDER UNIT_TESTS_FOLDER = pathlib.Path('unittests') UNIT_TESTS_ROOT = TESTS_ROOT / UNIT_TESTS_FOLDER +EXTENSION_TESTS_FOLDER = pathlib.Path('extension_tests') WEBHOST_DLL = "Microsoft.Azure.WebJobs.Script.WebHost.dll" DEFAULT_WEBHOST_DLL_PATH = ( PROJECT_ROOT / 'build' / 'webhost' / 'bin' / WEBHOST_DLL @@ -241,7 +242,7 @@ def setUpClass(cls): except Exception: raise - if not cls.webhost.is_healthy(): + if not cls.webhost.is_healthy() and cls.host_stdout is not None: cls.host_out = cls.host_stdout.read() if cls.host_out is not None and len(cls.host_out) > 0: error_message = 'WebHost is not started correctly.'