Run test for Python-3.11

Run unit tests for Python3.11
exclude hive extra from it because of issue cloudera/python-sasl#30

Author: pankajastro

.circleci/config.yml

@@ -42,6 +42,10 @@ workflows:
           <<: *main_and_release_branches
       - build-docs:
           <<: *all_branches_and_version_tag
+      - test-python3-11:
+          <<: *all_branches_and_version_tag
+          requires:
+            - markdown-link-check
       - test:
           name: test-python<< matrix.python_version >>
           matrix:
@@ -164,6 +168,37 @@ jobs:
             - ~/.pyenv/versions/
           key: docs-{{ .Branch }}-{{ checksum "setup.cfg" }}-{{ checksum ".readthedocs.yaml" }}
 
+  test-python3-11:
+    description: "Test Python-3.11"
+    executor:
+      name: docker-executor
+      python_version: "3.11"
+    parallelism: 4
+    steps:
+      - checkout
+      - restore_cache:
+          keys:
+            - deps-{{ .Branch }}-{{ checksum "setup.cfg" }}-{{ checksum "/home/circleci/.pyenv/version" }}
+            - deps-main-{{ checksum "setup.cfg" }}-{{ checksum "/home/circleci/.pyenv/version" }}
+      - run:
+          name: Install Dependencies
+          command: pip install -U -e .[test_python_3_11,tests]
+      - run:
+          name: Run tests
+          command: |
+            set -e
+            TEST_FILES=$(circleci tests \
+                          glob "tests/**/test_*.py" | sed '/tests\/apache\/hive/d' \
+                          | circleci tests split --split-by=timings \
+                        )
+            pytest --junit-xml=test-report/report.xml  $TEST_FILES
+      - store_test_results:
+          path: test-report
+      - save_cache:
+          paths:
+            - ~/.cache/pip
+            - ~/.pyenv/versions/
+          key: deps-{{ .Branch }}-{{ checksum "setup.cfg" }}-{{ checksum "/home/circleci/.pyenv/version" }}
 
   test:
     parameters:

.circleci/scripts/pre_commit_readme_extra.py

@@ -10,7 +10,7 @@
 config.read(repo_dir / "setup.cfg")
 
 all_extra = []
-extra_to_exclude = {"tests", "mypy", "docs"}
+extra_to_exclude = {"tests", "mypy", "docs", "test_python_3_11"}
 all_extras = set(config["options.extras_require"].keys()) - extra_to_exclude
 
 readme_path = repo_dir / "README.rst"

.circleci/scripts/pre_commit_setup_cfg_python_3_11_all_extra.py

@@ -0,0 +1,35 @@
+#!/usr/bin/env python3
+"""
+Pre-commit hook to sync an all Python3.11 extra in setup.cfg.
+It will contain all the dependencies apart from tests and mypy.
+"""
+import configparser
+from pathlib import Path
+
+repo_dir = Path(__file__).parent.parent.parent
+
+config = configparser.ConfigParser(strict=False)
+config.read(repo_dir / "setup.cfg")
+
+all_extra = []
+extra_to_exclude = {"tests", "mypy", "docs", "all", "test_python_3_11", "apache.hive"}
+for k in config["options.extras_require"].keys():
+    if k in extra_to_exclude:
+        continue
+    reqs = config["options.extras_require"][k].split()
+    all_extra.extend(reqs)
+
+expected_all_extra = set(all_extra)
+found_all_extra = set(config["options.extras_require"].get("test_python_3_11", "").split())
+
+if not found_all_extra:
+    raise SystemExit("Missing 'all' extra in setup.cfg")
+
+"""
+Use XOR operator ^ to find the missing dependencies instead of set A - set B
+set A - set B will only show difference of set A from set B, but we want see overall diff
+"""
+diff_extras = expected_all_extra ^ found_all_extra
+if diff_extras:
+    diff_extras_str = "\n \t" + "\n \t".join(sorted(diff_extras))
+    raise SystemExit(f"'all' extra in setup.cfg is missing some dependencies:\n {diff_extras_str}")

.pre-commit-config.yaml

@@ -106,6 +106,12 @@ repos:
         files: ^setup.cfg$
         pass_filenames: false
         entry: .circleci/scripts/pre_commit_setup_cfg_all_extra.py
+      - id: sync-all-python-3-11-extras-setup.cfg
+        name: Sync python-3-11 extra in setup.cfg
+        language: python
+        files: ^setup.cfg$
+        pass_filenames: false
+        entry: .circleci/scripts/pre_commit_setup_cfg_python_3_11_all_extra.py
       - id: sync-version-setup.cfg-docs-conf
         name: Sync "version" of the package in setup.cfg and docs/conf.py
         language: python

astronomer/providers/microsoft/azure/hooks/wasb.py

@@ -55,9 +55,7 @@ def get_conn(self) -> BlobServiceClient:
             app_id = conn.login
             app_secret = conn.password
             token_credential = ClientSecretCredential(tenant, app_id, app_secret)
-            return BlobServiceClient(
-                account_url=conn.host, credential=token_credential, **extra  # type:ignore[arg-type]
-            )
+            return BlobServiceClient(account_url=conn.host, credential=token_credential, **extra)
 
         sas_token = extra.pop("sas_token", extra.pop("extra__wasb__sas_token", None))
         if sas_token:
@@ -133,7 +131,7 @@ async def get_blobs_list_async(
         blob_list = []
         blobs = container.walk_blobs(name_starts_with=prefix, include=include, delimiter=delimiter, **kwargs)
         async for blob in blobs:
-            blob_list.append(blob.name)
+            blob_list.append(blob)
         return blob_list
 
     async def check_for_prefix_async(self, container_name: str, prefix: str, **kwargs: Any) -> bool:

setup.cfg

@@ -22,6 +22,7 @@ classifiers =
     Programming Language :: Python :: 3.8
     Programming Language :: Python :: 3.9
     Programming Language :: Python :: 3.10
+    Programming Language :: Python :: 3.11
 project_urls =
     Source Code=https://github.com/astronomer/astronomer-providers/
     Homepage=https://astronomer.io/
@@ -138,6 +139,31 @@ all =
     paramiko
     snowflake-sqlalchemy>=1.4.4  # Temporary solution for https://github.com/astronomer/astronomer-providers/issues/958, we should pin apache-airflow-providers-snowflake version after it pins this package to great than or equal to 1.4.4.
 
+# Hive provider does not work for python-3-11
+# See GihHub issue https://github.com/cloudera/python-sasl/issues/30
+# So creating this target for testing
+test_python_3_11 =
+    aiobotocore>=2.1.1
+    apache-airflow-providers-amazon>=3.0.0
+    apache-airflow-providers-apache-livy
+    apache-airflow-providers-cncf-kubernetes>=4
+    apache-airflow-providers-databricks>=2.2.0
+    apache-airflow-providers-google>=8.1.0
+    apache-airflow-providers-http
+    apache-airflow-providers-snowflake
+    apache-airflow-providers-sftp
+    apache-airflow-providers-microsoft-azure
+    asyncssh>=2.12.0
+    databricks-sql-connector>=2.0.4;python_version>='3.10'
+    apache-airflow-providers-dbt-cloud>=2.1.0
+    gcloud-aio-bigquery
+    gcloud-aio-storage
+    kubernetes_asyncio
+    openlineage-airflow>=0.12.0
+    paramiko
+    snowflake-sqlalchemy>=1.4.4  # Temporary solution for https://github.com/astronomer/astronomer-providers/issues/958, we should pin apache-airflow-providers-snowflake version after it pins this package to great than or equal to 1.4.4.
+
+
 [options.packages.find]
 include =
     astronomer.*

tests/core/triggers/test_external_task.py

@@ -2,10 +2,9 @@
 from unittest import mock
 from unittest.mock import AsyncMock
 
-import asynctest
 import pytest
-from airflow import AirflowException
-from airflow.operators.dummy import DummyOperator
+from airflow.exceptions import AirflowException
+from airflow.operators.empty import EmptyOperator
 from airflow.triggers.base import TriggerEvent
 from airflow.utils.state import DagRunState, TaskInstanceState
 
@@ -35,7 +34,7 @@ async def test_task_state_trigger(self, session, dag):
         session.add(dag_run)
         session.commit()
 
-        external_task = DummyOperator(task_id=self.TASK_ID, dag=dag)
+        external_task = EmptyOperator(task_id=self.TASK_ID, dag=dag)
         instance = get_task_instance(external_task)
         session.add(instance)
         session.commit()
@@ -222,7 +221,7 @@ async def test_deployment_task_exception(self, mock_run):
         assert TriggerEvent({"state": "error", "message": "Test exception"}) == actual
 
     @pytest.mark.asyncio
-    @asynctest.patch("astronomer.providers.http.hooks.http.HttpHookAsync.run")
+    @mock.patch("astronomer.providers.http.hooks.http.HttpHookAsync.run")
     async def test_deployment_complete(self, mock_run):
         """Assert ExternalDeploymentTaskTrigger runs and complete the run in success state"""
         mock.AsyncMock(HttpHookAsync)