table & eh, unit, lint

Author: hallvictoria

.github/workflows/ci_ut_workflow.yml

@@ -63,6 +63,7 @@ jobs:
         # Retry a couple times to avoid certificate issue
         retry 5 python setup.py build
         retry 5 python setup.py webhost --branch-name=dev
+        retry 5 python setup.py extension
         mkdir logs
     - name: Test with pytest
       env:

tests/endtoend/eventhub_batch_functions/eventhub_batch_functions_stein/function_app.py

@@ -18,10 +18,10 @@
     connection="AzureWebJobsEventHubConnectionString",
     data_type="string",
     cardinality="many")
-@app.table_output(arg_name="$return",
-                  connection="AzureWebJobsStorage",
-                  table_name="EventHubBatchTest")
-def eventhub_multiple(events):
+@app.blob_output(arg_name="$return",
+                 path="python-worker-tests/test-eventhub-batch-triggered.txt",
+                 connection="AzureWebJobsStorage")
+def eventhub_multiple(events) -> str:
     table_entries = []
     for event in events:
         json_entry = event.get_body()
@@ -46,13 +46,12 @@ def eventhub_output_batch(req: func.HttpRequest, out: func.Out[str]) -> str:
 
 # Retrieve the event data from storage blob and return it as Http response
 @app.function_name(name="get_eventhub_batch_triggered")
-@app.route(route="get_eventhub_batch_triggered/{id}")
-@app.table_input(arg_name="testEntities",
-                 connection="AzureWebJobsStorage",
-                 table_name="EventHubBatchTest",
-                 partition_key="{id}")
-def get_eventhub_batch_triggered(req: func.HttpRequest, testEntities):
-    return func.HttpResponse(status_code=200, body=testEntities)
+@app.route(route="get_eventhub_batch_triggered")
+@app.blob_input(arg_name="testEntities",
+                path="python-worker-tests/test-eventhub-batch-triggered.txt",
+                connection="AzureWebJobsStorage")
+def get_eventhub_batch_triggered(req: func.HttpRequest, testEntities: func.InputStream):
+    return func.HttpResponse(status_code=200, body=testEntities.read().decode('utf-8'))
 
 
 # Retrieve the event data from storage blob and return it as Http response

tests/endtoend/eventhub_batch_functions/eventhub_multiple/__init__.py

@@ -5,7 +5,7 @@
 
 # This is an actual EventHub trigger which handles Eventhub events in batches.
 # It serializes multiple event data into a json and store it into a blob.
-def main(events):
+def main(events) -> str:
     table_entries = []
     for event in events:
         json_entry = event.get_body()

tests/endtoend/eventhub_batch_functions/eventhub_multiple/function.json

@@ -13,9 +13,9 @@
     },
     {
       "direction": "out",
-      "type": "table",
+      "type": "blob",
       "name": "$return",
-      "tableName": "EventHubBatchTest",
+      "path": "python-worker-tests/test-eventhub-batch-triggered.txt",
       "connection": "AzureWebJobsStorage"
     }
   ]

tests/endtoend/eventhub_batch_functions/get_eventhub_batch_triggered/__init__.py

@@ -4,5 +4,5 @@
 
 
 # Retrieve the event data from storage blob and return it as Http response
-def main(req: func.HttpRequest, testEntities):
-    return func.HttpResponse(status_code=200, body=testEntities)
+def main(req: func.HttpRequest, testEntities: func.InputStream):
+    return func.HttpResponse(status_code=200, body=testEntities.read().decode('utf-8'))

tests/endtoend/eventhub_batch_functions/get_eventhub_batch_triggered/function.json

@@ -12,10 +12,9 @@
     },
     {
       "direction": "in",
-      "type": "table",
+      "type": "blob",
       "name": "testEntities",
-      "partitionKey": "WillBePopulated",
-      "tableName": "EventHubBatchTest",
+      "path": "python-worker-tests/test-eventhub-batch-triggered.txt",
       "connection": "AzureWebJobsStorage"
     },
     {

tests/endtoend/table_functions/table_functions_stein/function_app.py

@@ -15,8 +15,7 @@
                  row_key='{id}',
                  partition_key="test")
 def table_in_binding(req: func.HttpRequest, testEntity):
-    headers_dict = json.loads(testEntity)
-    return func.HttpResponse(status_code=200, headers=headers_dict)
+    return func.HttpResponse(status_code=200, body=testEntity)
 
 
 @app.function_name(name="table_out_binding")
@@ -28,6 +27,5 @@ def table_out_binding(req: func.HttpRequest, resp: func.Out[func.HttpResponse]):
     row_key_uuid = str(uuid.uuid4())
     table_dict = {'PartitionKey': 'test', 'RowKey': row_key_uuid}
     table_json = json.dumps(table_dict)
-    http_resp = func.HttpResponse(status_code=200, headers=table_dict)
-    resp.set(http_resp)
+    resp.set(table_json)
     return table_json

tests/endtoend/table_functions/table_functions_stein/generic/function_app.py

@@ -19,8 +19,7 @@
     row_key="{id}",
     partition_key="test")
 def table_in_binding(req: func.HttpRequest, testEntity):
-    headers_dict = json.loads(testEntity)
-    return func.HttpResponse(status_code=200, headers=headers_dict)
+    return func.HttpResponse(status_code=200, body=testEntity)
 
 
 @app.function_name(name="table_out_binding")
@@ -36,6 +35,5 @@ def table_out_binding(req: func.HttpRequest, resp: func.Out[func.HttpResponse]):
     row_key_uuid = str(uuid.uuid4())
     table_dict = {'PartitionKey': 'test', 'RowKey': row_key_uuid}
     table_json = json.dumps(table_dict)
-    http_resp = func.HttpResponse(status_code=200, headers=table_dict)
-    resp.set(http_resp)
+    resp.set(table_json)
     return table_json

tests/endtoend/table_functions/table_in_binding/__init__.py

@@ -1,10 +1,7 @@
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # Licensed under the MIT License.
-import json
-
 import azure.functions as func
 
 
 def main(req: func.HttpRequest, testEntity):
-    headers_dict = json.loads(testEntity)
-    return func.HttpResponse(status_code=200, headers=headers_dict)
+    return func.HttpResponse(status_code=200, body=testEntity)

tests/endtoend/table_functions/table_out_binding/__init__.py

@@ -10,6 +10,5 @@ def main(req: func.HttpRequest, resp: func.Out[func.HttpResponse]):
     row_key_uuid = str(uuid.uuid4())
     table_dict = {'PartitionKey': 'test', 'RowKey': row_key_uuid}
     table_json = json.dumps(table_dict)
-    http_resp = func.HttpResponse(status_code=200, headers=table_dict)
-    resp.set(http_resp)
+    resp.set(table_json)
     return table_json

tests/endtoend/test_eventhub_batch_functions.py

@@ -2,7 +2,6 @@
 # Licensed under the MIT License.
 import json
 import time
-import pathlib
 from datetime import datetime
 from unittest import skipIf
 
@@ -37,14 +36,9 @@ def get_libraries_to_install(cls):
 
     def test_eventhub_multiple(self):
         NUM_EVENTS = 3
-        all_row_keys_seen = dict([(str(i), True) for i in range(NUM_EVENTS)])
+        all_row_keys_seen = dict([(i, True) for i in range(NUM_EVENTS)])
         partition_key = str(round(time.time()))
 
-        # Dynamically rewrite function.json to point to new partition key
-        # for recording EventHub state
-        old_partition_key = self._get_table_partition_key()
-        self._set_table_partition_key(partition_key)
-
         # wait for host to restart after change
         time.sleep(5)
 
@@ -57,29 +51,26 @@ def test_eventhub_multiple(self):
                                  data=json.dumps(docs))
         self.assertEqual(r.status_code, 200)
 
-        row_keys = [str(i) for i in range(NUM_EVENTS)]
+        row_keys = [i for i in range(NUM_EVENTS)]
         seen = [False] * NUM_EVENTS
         row_keys_seen = dict(zip(row_keys, seen))
 
         # Allow trigger to fire.
         time.sleep(5)
 
-        try:
-            r = self.webhost.request('GET', 'get_eventhub_batch_triggered')
+        r = self.webhost.request('GET', 'get_eventhub_batch_triggered')
 
-            # Waiting for the blob get updated with the latest data from the
-            # eventhub output binding
-            time.sleep(2)
-            self.assertEqual(r.status_code, 200)
-            entries = r.json()
-            for entry in entries:
-                self.assertEqual(entry['PartitionKey'], partition_key)
-                row_key = entry['RowKey']
-                row_keys_seen[row_key] = True
+        # Waiting for the blob get updated with the latest data from the
+        # eventhub output binding
+        time.sleep(2)
+        self.assertEqual(r.status_code, 200)
+        entries = r.json()
+        for entry in entries:
+            self.assertEqual(entry['PartitionKey'], partition_key)
+            row_key = entry['RowKey']
+            row_keys_seen[row_key] = True
 
-            self.assertDictEqual(all_row_keys_seen, row_keys_seen)
-        finally:
-            self._cleanup(old_partition_key)
+        self.assertDictEqual(all_row_keys_seen, row_keys_seen)
 
     def test_eventhub_multiple_with_metadata(self):
         # Generate a unique event body for EventHub event
@@ -142,36 +133,6 @@ def test_eventhub_multiple_with_metadata(self):
             self.assertGreaterEqual(sys_props['SequenceNumber'], 0)
             self.assertIsNotNone(sys_props['Offset'])
 
-    def _cleanup(self, old_partition_key):
-        self._set_table_partition_key(old_partition_key)
-
-    def _get_table_partition_key(self):
-        func_dict = self._get_table_function_json_dict()
-        partition_key = func_dict['bindings'][1]['partitionKey']
-        return partition_key
-
-    def _set_table_partition_key(self, partition_key):
-        full_json_path = self._get_table_function_json_path()
-
-        func_dict = self._get_table_function_json_dict()
-        func_dict['bindings'][1]['partitionKey'] = partition_key
-
-        with open(full_json_path, 'w') as f:
-            json.dump(func_dict, f, indent=2)
-
-    def _get_table_function_json_dict(self):
-        full_json_path = self._get_table_function_json_path()
-
-        with open(full_json_path, 'r') as f:
-            func_dict = json.load(f)
-
-        return func_dict
-
-    def _get_table_function_json_path(self):
-        script_dir = pathlib.Path(self.get_script_dir())
-        json_path = pathlib.Path('get_eventhub_batch_triggered/function.json')
-        return testutils.TESTS_ROOT / script_dir / json_path
-
 
 @skipIf(is_envvar_true(DEDICATED_DOCKER_TEST)
         or is_envvar_true(CONSUMPTION_DOCKER_TEST),
@@ -191,7 +152,7 @@ def get_libraries_to_install(cls):
 
     def test_eventhub_multiple(self):
         NUM_EVENTS = 3
-        all_row_keys_seen = dict([(str(i), True) for i in range(NUM_EVENTS)])
+        all_row_keys_seen = dict([(i, True) for i in range(NUM_EVENTS)])
         partition_key = str(round(time.time()))
 
         docs = []
@@ -203,7 +164,7 @@ def test_eventhub_multiple(self):
                                  data=json.dumps(docs))
         self.assertEqual(r.status_code, 200)
 
-        row_keys = [str(i) for i in range(NUM_EVENTS)]
+        row_keys = [i for i in range(NUM_EVENTS)]
         seen = [False] * NUM_EVENTS
         row_keys_seen = dict(zip(row_keys, seen))
 
@@ -212,7 +173,7 @@ def test_eventhub_multiple(self):
 
         r = self.webhost.request(
             'GET',
-            f'get_eventhub_batch_triggered/{partition_key}')
+            'get_eventhub_batch_triggered')
         self.assertEqual(r.status_code, 200)
         entries = r.json()
         for entry in entries:

tests/endtoend/test_table_functions.py

@@ -24,7 +24,7 @@ def get_script_dir(cls):
     def test_table_bindings(self):
         out_resp = self.webhost.request('POST', 'table_out_binding')
         self.assertEqual(out_resp.status_code, 200)
-        row_key = out_resp.headers['rowKey']
+        row_key = json.loads(out_resp.text)['RowKey']
 
         script_dir = pathlib.Path(self.get_script_dir())
         json_path = pathlib.Path('table_in_binding/function.json')
@@ -42,8 +42,12 @@ def test_table_bindings(self):
 
         in_resp = self.webhost.request('GET', 'table_in_binding')
         self.assertEqual(in_resp.status_code, 200)
-        in_row_key = in_resp.headers['rowKey']
-        self.assertEqual(in_row_key, row_key)
+        row_key_present = False
+        for row in json.loads(in_resp.text):
+            if row["RowKey"] == row_key:
+                row_key_present = True
+                break
+        self.assertTrue(row_key_present)
 
 
 @skipIf(is_envvar_true(DEDICATED_DOCKER_TEST)
@@ -60,12 +64,16 @@ def get_script_dir(cls):
     def test_table_bindings(self):
         out_resp = self.webhost.request('POST', 'table_out_binding')
         self.assertEqual(out_resp.status_code, 200)
-        row_key = out_resp.headers['rowKey']
+        row_key = json.loads(out_resp.text)['RowKey']
 
         in_resp = self.webhost.request('GET', f'table_in_binding/{row_key}')
         self.assertEqual(in_resp.status_code, 200)
-        in_row_key = in_resp.headers['rowKey']
-        self.assertEqual(in_row_key, row_key)
+        row_key_present = False
+        for row in json.loads(in_resp.text):
+            if row["RowKey"] == row_key:
+                row_key_present = True
+                break
+        self.assertTrue(row_key_present)
 
 
 @skipIf(is_envvar_true(DEDICATED_DOCKER_TEST)

tests/utils/constants.py

@@ -35,7 +35,7 @@
         Version="4.0.1" />
       <PackageReference Include="Microsoft.Azure.WebJobs.Extensions.DurableTask"
         Version="2.13.2" />
-      <PackageReference Include="Microsoft.Azure.WebJobs.Extensions.Tables" 
+      <PackageReference Include="Microsoft.Azure.WebJobs.Extensions.Tables"
         Version="1.2.1" />
       <PackageReference Include="System.Drawing.Common"
         Version="4.7.3" />

tests/utils/testutils_lc.py

@@ -278,8 +278,9 @@ def _encrypt_context(cls, encryption_key: str, plain_text: str) -> str:
         return f'{iv_base64}.{encrypted_base64}.{key_sha256_base64}'
 
     def __enter__(self):
-        mesh_image = (os.environ.get(_CUSTOM_IMAGE) or
-                      self._find_latest_mesh_image(self._host_version, self._py_version))
+        mesh_image = (os.environ.get(_CUSTOM_IMAGE)
+                      or self._find_latest_mesh_image(self._host_version,
+                                                      self._py_version))
         self.spawn_container(image=mesh_image)
         return self