From 0bbb9475a2628e2da6ec48a4d7bb25a37e172167 Mon Sep 17 00:00:00 2001 From: "Hanzhang Zeng (Roger)" Date: Mon, 16 Mar 2020 10:53:04 -0700 Subject: [PATCH 1/7] Increase message size to sys.maxsize --- azure_functions_worker/main.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/azure_functions_worker/main.py b/azure_functions_worker/main.py index 1ddb194ab..2e595fbc0 100644 --- a/azure_functions_worker/main.py +++ b/azure_functions_worker/main.py @@ -2,6 +2,7 @@ import argparse +import sys from . import aio_compat from . import dispatcher @@ -21,8 +22,6 @@ def parse_args(): parser.add_argument('--log-to', type=str, default=None, help='log destination: stdout, stderr, ' 'syslog, or a file path') - parser.add_argument('--grpcMaxMessageLength', type=int, - dest='grpc_max_msg_len') return parser.parse_args() @@ -36,17 +35,16 @@ def main(): try: return aio_compat.run(start_async( - args.host, args.port, args.worker_id, args.request_id, - args.grpc_max_msg_len)) + args.host, args.port, args.worker_id, args.request_id)) except Exception: error_logger.exception('unhandled error in functions worker') raise -async def start_async(host, port, worker_id, request_id, grpc_max_msg_len): +async def start_async(host, port, worker_id, request_id): disp = await dispatcher.Dispatcher.connect( host, port, worker_id, request_id, - connect_timeout=5.0, max_msg_len=grpc_max_msg_len) + connect_timeout=5.0, max_msg_len=sys.maxsize) disp.load_bindings() From e4ccd6d399ef2d4c6b1dbbd4c18f99404fc3661c Mon Sep 17 00:00:00 2001 From: "Hanzhang Zeng (Roger)" Date: Mon, 16 Mar 2020 11:05:40 -0700 Subject: [PATCH 2/7] Update testing function host to 2.0.13036 --- azure_functions_worker/main.py | 2 ++ setup.py | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/azure_functions_worker/main.py b/azure_functions_worker/main.py index 2e595fbc0..b834980fb 100644 --- a/azure_functions_worker/main.py +++ b/azure_functions_worker/main.py @@ -22,6 +22,8 @@ def parse_args(): parser.add_argument('--log-to', type=str, default=None, help='log destination: stdout, stderr, ' 'syslog, or a file path') + parser.add_argument('--grpcMaxMessageLength', type=int, + dest='grpc_max_msg_len') return parser.parse_args() diff --git a/setup.py b/setup.py index 07329f37f..5fed8baa0 100644 --- a/setup.py +++ b/setup.py @@ -17,9 +17,9 @@ # TODO: change this to something more stable when available. -WEBHOST_URL = ('https://ci.appveyor.com/api/buildjobs/1fqp92o5h2gks7xe' +WEBHOST_URL = ('https://ci.appveyor.com/api/buildjobs/19gqd7drpxhkedea' '/artifacts' - '/Functions.Binaries.2.0.12888.no-runtime.zip') + '/Functions.Binaries.2.0.13036.no-runtime.zip') # Extensions necessary for non-core bindings. AZURE_EXTENSIONS = """\ From 30f6d125b1c5e1c871a3985058643972aea53ea9 Mon Sep 17 00:00:00 2001 From: "Hanzhang Zeng (Roger)" Date: Mon, 16 Mar 2020 14:34:50 -0700 Subject: [PATCH 3/7] Set message limitation on -1 --- azure_functions_worker/dispatcher.py | 3 ++- azure_functions_worker/main.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/azure_functions_worker/dispatcher.py b/azure_functions_worker/dispatcher.py index 69f7d5d84..7e9c8fb69 100644 --- a/azure_functions_worker/dispatcher.py +++ b/azure_functions_worker/dispatcher.py @@ -44,7 +44,7 @@ class Dispatcher(metaclass=DispatcherMeta): _GRPC_STOP_RESPONSE = object() def __init__(self, loop, host, port, worker_id, request_id, - grpc_connect_timeout, grpc_max_msg_len): + grpc_connect_timeout, grpc_max_msg_len=-1): self._loop = loop self._host = host self._port = port @@ -65,6 +65,7 @@ def __init__(self, loop, host, port, worker_id, request_id, max_workers=1) self._grpc_connect_timeout = grpc_connect_timeout + # This is set to -1 by default to remove the limitation on msg size self._grpc_max_msg_len = grpc_max_msg_len self._grpc_resp_queue: queue.Queue = queue.Queue() self._grpc_connected_fut = loop.create_future() diff --git a/azure_functions_worker/main.py b/azure_functions_worker/main.py index b834980fb..880b9a43f 100644 --- a/azure_functions_worker/main.py +++ b/azure_functions_worker/main.py @@ -46,7 +46,8 @@ def main(): async def start_async(host, port, worker_id, request_id): disp = await dispatcher.Dispatcher.connect( host, port, worker_id, request_id, - connect_timeout=5.0, max_msg_len=sys.maxsize) + connect_timeout=5.0) + disp.load_bindings() From b51e0c9bf8882101504ff67c8be0d161718ffec4 Mon Sep 17 00:00:00 2001 From: "Hanzhang Zeng (Roger)" Date: Mon, 16 Mar 2020 14:54:33 -0700 Subject: [PATCH 4/7] Fix code quality --- azure_functions_worker/main.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/azure_functions_worker/main.py b/azure_functions_worker/main.py index 880b9a43f..4f180b0f7 100644 --- a/azure_functions_worker/main.py +++ b/azure_functions_worker/main.py @@ -2,7 +2,6 @@ import argparse -import sys from . import aio_compat from . import dispatcher @@ -48,7 +47,6 @@ async def start_async(host, port, worker_id, request_id): host, port, worker_id, request_id, connect_timeout=5.0) - disp.load_bindings() await disp.dispatch_forever() From 2acaf9313d240eb6c2e393d7f82d4d8b10f20a6c Mon Sep 17 00:00:00 2001 From: "Hanzhang Zeng (Roger)" Date: Mon, 16 Mar 2020 18:02:03 -0700 Subject: [PATCH 5/7] Improve debug flag consistency --- azure_functions_worker/constants.py | 3 +++ azure_functions_worker/testutils.py | 11 ++++++++--- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/azure_functions_worker/constants.py b/azure_functions_worker/constants.py index a67eb144b..15e171ffe 100644 --- a/azure_functions_worker/constants.py +++ b/azure_functions_worker/constants.py @@ -4,5 +4,8 @@ RPC_HTTP_BODY_ONLY = "RpcHttpBodyOnly" RPC_HTTP_TRIGGER_METADATA_REMOVED = "RpcHttpTriggerMetadataRemoved" +# Debug Flags +PYAZURE_WEBHOST_DEBUG = "PYAZURE_WEBHOST_DEBUG" + # Feature Flags (app settings) PYTHON_ROLLBACK_CWD_PATH = "PYTHON_ROLLBACK_CWD_PATH" diff --git a/azure_functions_worker/testutils.py b/azure_functions_worker/testutils.py index 480c83284..219f27fa0 100644 --- a/azure_functions_worker/testutils.py +++ b/azure_functions_worker/testutils.py @@ -33,6 +33,8 @@ from . import aio_compat from . import dispatcher from . import protos +from .utils.common import is_envvar_true +from .constants import PYAZURE_WEBHOST_DEBUG PROJECT_ROOT = pathlib.Path(__file__).parent.parent @@ -137,7 +139,10 @@ def __new__(mcls, name, bases, dct): def wrapper(self, *args, __meth__=test_case, __check_log__=check_log_case, **kwargs): - if __check_log__ is not None and callable(__check_log__): + if (__check_log__ is not None + and callable(__check_log__) + and not is_envvar_true(PYAZURE_WEBHOST_DEBUG)): + # Check logging output for unit test scenarios result = self._run_test(__meth__, *args, **kwargs) @@ -177,7 +182,7 @@ def get_script_dir(cls): @classmethod def setUpClass(cls): script_dir = pathlib.Path(cls.get_script_dir()) - if os.environ.get('PYAZURE_WEBHOST_DEBUG'): + if is_envvar_true(PYAZURE_WEBHOST_DEBUG): cls.host_stdout = None else: cls.host_stdout = tempfile.NamedTemporaryFile('w+t') @@ -660,7 +665,7 @@ def start_webhost(*, script_dir=None, stdout=None): script_root = FUNCS_PATH if stdout is None: - if os.environ.get('PYAZURE_WEBHOST_DEBUG'): + if is_envvar_true(PYAZURE_WEBHOST_DEBUG): stdout = sys.stdout else: stdout = subprocess.DEVNULL From 1269103aab38180cf3d31bb218bf4790e0b82d0c Mon Sep 17 00:00:00 2001 From: "Hanzhang Zeng (Roger)" Date: Mon, 16 Mar 2020 18:10:12 -0700 Subject: [PATCH 6/7] Fix DOTNET_VERSION incompatibility --- azure-pipelines.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 8879231b9..2bb6c3dad 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -5,12 +5,12 @@ trigger: - master variables: - DOTNET_VERSION: '2.2.x' + DOTNET_VERSION: '2.2.207' jobs: - job: Tests pool: - vmImage: 'ubuntu-16.04' + vmImage: 'ubuntu-18.04' strategy: matrix: Python36: @@ -52,7 +52,7 @@ jobs: - job: Build_WINDOWS_X64 dependsOn: 'Tests' pool: - vmImage: 'vs2017-win2016' + vmImage: 'windows-2019' strategy: matrix: Python36V2: @@ -80,7 +80,7 @@ jobs: - job: Build_WINDOWS_X86 dependsOn: 'Tests' pool: - vmImage: 'vs2017-win2016' + vmImage: 'windows-2019' strategy: matrix: Python37V2: @@ -102,7 +102,7 @@ jobs: - job: Build_LINUX_X64 dependsOn: 'Tests' pool: - vmImage: 'ubuntu-16.04' + vmImage: 'ubuntu-18.04' strategy: matrix: Python36V2: @@ -129,7 +129,7 @@ jobs: - job: Build_OSX_X64 dependsOn: 'Tests' pool: - vmImage: 'macOS-10.13' + vmImage: 'macOS-10.15' strategy: matrix: Python36V2: @@ -161,7 +161,7 @@ jobs: 'Build_OSX_X64' ] pool: - vmImage: 'vs2017-win2016' + vmImage: 'windows-2019' strategy: matrix: V2PythonWorker: From 325b17b2845f69e6046ac05013a4aa441c1afe9a Mon Sep 17 00:00:00 2001 From: "Hanzhang Zeng (Roger)" Date: Mon, 16 Mar 2020 23:50:38 -0700 Subject: [PATCH 7/7] Upgrade pip before resolving dependencies --- pack/scripts/win_deps.ps1 | 4 +++- setup.py | 4 ++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/pack/scripts/win_deps.ps1 b/pack/scripts/win_deps.ps1 index 6dcfd9bb2..d674915d1 100644 --- a/pack/scripts/win_deps.ps1 +++ b/pack/scripts/win_deps.ps1 @@ -1,5 +1,7 @@ python -m venv .env -.env\scripts\activate +.env\Scripts\Activate.ps1 +python -m pip install --upgrade pip + python -m pip install . $depsPath = Join-Path -Path $env:BUILD_SOURCESDIRECTORY -ChildPath "deps" diff --git a/setup.py b/setup.py index 5fed8baa0..381e2bb7f 100644 --- a/setup.py +++ b/setup.py @@ -274,8 +274,8 @@ def run(self): 'azure_functions_worker.bindings', 'azure_functions_worker.utils'], install_requires=[ - 'grpcio==1.26.0', - 'grpcio-tools==1.26.0', + 'grpcio~=1.26.0', + 'grpcio-tools~=1.26.0', ], extras_require={ 'dev': [