Moved consumption tests in a separate pipeline (#1148)

* Moved consumption tests in separate pipeline

* Installing pytest

* Updated dependencies

* Added xdist to pytest

* Installing dependencies

* Installing additional dependencies

* Formatted yml file

* Added __init__

* Added requests to dependencies

* Added crypto to dependencies

* Install all dependencies

* Unskipped tests for py310

* Undo last change

* Give permissions to temp dir

* Temp: check user

* Testing build worker

* Fix syntax error

* Ignore errors in cleanup

* Added dependency isolation flag

* Building webhost

* Deleting temp dir

* Adding logs

* Check if dir is empty

* Removing library mounting

* Fixing flake8 test

* Skipped downloading library

* list dirs

* Update ci_consumption_workflow.yml

* Updated yml to checkout code

* Skipping protobuf tests for py310

Co-authored-by: Gavin Aguiar <gavin@GavinPC>
Co-authored-by: peterstone2017 <[email protected]>

Author: 3 people

.github/workflows/ci_consumption_workflow.yml

@@ -0,0 +1,54 @@
+# This workflow will run all tests in tests/consumption_tests in Docker using a consumption image
+# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
+
+name: CI Consumption tests
+
+on:
+  workflow_dispatch:
+  push:
+    branches: [ dev, master, main, release/* ]
+  pull_request:
+    branches: [ dev, master, main, release/* ]
+
+jobs:
+  build:
+    name: "Python Consumption CI Run"
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: [ 3.7, 3.8, 3.9, "3.10" ]
+
+    steps:
+      - name: Checkout code.
+        uses: actions/checkout@v2
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install dependencies
+        run: |
+          python -m pip install --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple -U -e .[dev]
+          python setup.py build
+          python setup.py webhost --branch-name=dev
+          python setup.py extension
+      - name: Running 3.7 Tests
+        if: matrix.python-version == 3.7
+        env:
+          AzureWebJobsStorage: ${{ secrets.LinuxStorageConnectionString37 }}
+        run: python -m pytest -n auto --instafail tests/consumption_tests
+      - name: Running 3.8 Tests
+        if: matrix.python-version == 3.8
+        env:
+          AzureWebJobsStorage: ${{ secrets.LinuxStorageConnectionString38 }}
+        run: python -m pytest -n auto --instafail tests/consumption_tests
+      - name: Running 3.9 Tests
+        if: matrix.python-version == 3.9
+        env:
+          AzureWebJobsStorage: ${{ secrets.LinuxStorageConnectionString39 }}
+        run: python -m pytest -n auto --instafail tests/consumption_tests
+      - name: Running 3.10 Tests
+        if: matrix.python-version == 3.10
+        env:
+          AzureWebJobsStorage: ${{ secrets.LinuxStorageConnectionString310 }}
+        run: python -m pytest -n auto --instafail tests/consumption_tests

.github/workflows/ci_e2e_workflow.yml

@@ -65,18 +65,6 @@ jobs:
           retry 5 python setup.py build
           retry 5 python setup.py webhost --branch-name=dev
           retry 5 python setup.py extension
-      - name: Running 3.6 Tests
-        if: matrix.python-version == 3.6
-        env:
-          AzureWebJobsStorage: ${{ secrets.LinuxStorageConnectionString36 }}
-          AzureWebJobsCosmosDBConnectionString: ${{ secrets.LinuxCosmosDBConnectionString36 }}
-          AzureWebJobsEventHubConnectionString: ${{ secrets.LinuxEventHubConnectionString36 }}
-          AzureWebJobsServiceBusConnectionString: ${{ secrets.LinuxServiceBusConnectionString36 }}
-          AzureWebJobsSqlConnectionString: ${{ secrets.LinuxSqlConnectionString36 }}
-          AzureWebJobsEventGridTopicUri: ${{ secrets.LinuxEventGridTopicUriString36 }}
-          AzureWebJobsEventGridConnectionKey: ${{ secrets.LinuxEventGridConnectionKeyString36 }}
-        run: |
-          python -m pytest -n auto  --instafail --cov=./azure_functions_worker --cov-report xml --cov-branch --cov-append tests/endtoend
       - name: Running 3.7 Tests
         if: matrix.python-version == 3.7
         env:

.github/workflows/ut_ci_workflow.yml renamed to .github/workflows/ci_ut_workflow.yml

@@ -13,8 +13,6 @@
 _DEFAULT_HOST_VERSION = "4"
 
 
-@skipIf(sys.version_info >= (3, 10, 0),
-        "Skip the tests for Python 3.10 and above")
 class TestLinuxConsumption(TestCase):
     """Test worker behaviors on specific scenarios.
 
@@ -92,6 +90,8 @@ def test_common_libraries(self):
             self.assertIn('pyodbc', content)
             self.assertIn('requests', content)
 
+    @skipIf(sys.version_info.minor == 10,
+            "Protobuf pinning fails during remote build")
     def test_new_protobuf(self):
         """A function app with the following requirements.txt:
 
@@ -105,7 +105,8 @@ def test_new_protobuf(self):
                                                self._py_version) as ctrl:
             ctrl.assign_container(env={
                 "AzureWebJobsStorage": self._storage,
-                "SCM_RUN_FROM_PACKAGE": self._get_blob_url("NewProtobuf")
+                "SCM_RUN_FROM_PACKAGE": self._get_blob_url("NewProtobuf"),
+                "PYTHON_ISOLATE_WORKER_DEPENDENCIES": "1"
             })
             req = Request('GET', f'{ctrl.url}/api/HttpTrigger')
             resp = ctrl.send_request(req)
@@ -119,6 +120,8 @@ def test_new_protobuf(self):
             self.assertEqual(content['google.protobuf'], '3.15.8')
             self.assertEqual(content['grpc'], '1.33.2')
 
+    @skipIf(sys.version_info.minor == 10,
+            "Protobuf pinning fails during remote build")
     def test_old_protobuf(self):
         """A function app with the following requirements.txt:
 
@@ -132,7 +135,8 @@ def test_old_protobuf(self):
                                                self._py_version) as ctrl:
             ctrl.assign_container(env={
                 "AzureWebJobsStorage": self._storage,
-                "SCM_RUN_FROM_PACKAGE": self._get_blob_url("OldProtobuf")
+                "SCM_RUN_FROM_PACKAGE": self._get_blob_url("OldProtobuf"),
+                "PYTHON_ISOLATE_WORKER_DEPENDENCIES": "1"
             })
             req = Request('GET', f'{ctrl.url}/api/HttpTrigger')
             resp = ctrl.send_request(req)
@@ -213,6 +217,7 @@ def test_pinning_functions_to_older_version(self):
                 "AzureWebJobsStorage": self._storage,
                 "SCM_RUN_FROM_PACKAGE": self._get_blob_url(
                     "PinningFunctions"),
+                "PYTHON_ISOLATE_WORKER_DEPENDENCIES": "1"
             })
             req = Request('GET', f'{ctrl.url}/api/HttpTrigger1')
             resp = ctrl.send_request(req)

tests/endtoend/test_linux_consumption.py renamed to tests/consumption_tests/test_linux_consumption.py

@@ -162,18 +162,14 @@ def spawn_container(self,
         """
         # Construct environment variables and start the docker container
         worker_path = os.path.join(PROJECT_ROOT, 'azure_functions_worker')
-        library_path = os.path.join(tempfile.gettempdir(), _FUNC_FILE_NAME,
-                                    'azure', 'functions')
-        self._download_azure_functions()
+
+        # TODO: Mount library in docker container
+        # self._download_azure_functions()
 
         container_worker_path = (
             f"/azure-functions-host/workers/python/{self._py_version}/"
             "LINUX/X64/azure_functions_worker"
         )
-        container_library_path = (
-            f"/azure-functions-host/workers/python/{self._py_version}/"
-            "LINUX/X64/azure/functions"
-        )
 
         run_cmd = []
         run_cmd.extend([self._docker_cmd, "run", "-p", "0:80", "-d"])
@@ -183,9 +179,7 @@ def spawn_container(self,
         run_cmd.extend(["-e", f"CONTAINER_NAME={self._uuid}"])
         run_cmd.extend(["-e", f"CONTAINER_ENCRYPTION_KEY={_DUMMY_CONT_KEY}"])
         run_cmd.extend(["-e", "WEBSITE_PLACEHOLDER_MODE=1"])
-        run_cmd.extend(["-e", "PYTHON_ISOLATE_WORKER_DEPENDENCIES=1"])
         run_cmd.extend(["-v", f'{worker_path}:{container_worker_path}'])
-        run_cmd.extend(["-v", f'{library_path}:{container_library_path}'])
 
         for key, value in env.items():
             run_cmd.extend(["-e", f"{key}={value}"])
@@ -295,7 +289,8 @@ def __enter__(self):
     def __exit__(self, exc_type, exc_value, traceback):
         logs = self.get_container_logs()
         self.safe_kill_container()
-        shutil.rmtree(os.path.join(tempfile.gettempdir(), _FUNC_FILE_NAME))
+        shutil.rmtree(os.path.join(tempfile.gettempdir(), _FUNC_FILE_NAME),
+                      ignore_errors=True)
 
         if traceback:
             print(f'Test failed with container logs: {logs}',