Removing deprecated code in hive provider (#38859)

* Removing deprecated code in hive provider

* fixing tests and removing property

* removing some more tests

Author: amoghrajesh

airflow/providers/apache/hive/CHANGELOG.rst

@@ -37,6 +37,14 @@ Breaking changes
 Changed the default value of ``use_beeline`` in hive cli connection to True.
 Beeline will be always enabled by default in this connection type.
 
+Removed deprecated parameter ``authMechanism`` from HiveHook and dependent operators.
+Use the already existing ``auth_mechanism`` instead in your ``extra``.
+
+Removed the method ``get_hook`` from hive operator. Use the ``hook`` property instead.
+
+Removed the deprecated ``col_blacklist`` property from HiveStatsCollectionOperator.
+Please rename it to ``excluded_columns`` instead.
+
 7.0.1
 .....
 

airflow/providers/apache/hive/hooks/hive.py

@@ -23,12 +23,9 @@
 import socket
 import subprocess
 import time
-import warnings
 from tempfile import NamedTemporaryFile, TemporaryDirectory
 from typing import TYPE_CHECKING, Any, Iterable, Mapping
 
-from airflow.exceptions import AirflowProviderDeprecationWarning
-
 if TYPE_CHECKING:
     import pandas as pd
 
@@ -561,15 +558,6 @@ def get_metastore_client(self) -> Any:
         if not host:
             raise AirflowException("Failed to locate the valid server.")
 
-        if "authMechanism" in conn.extra_dejson:
-            warnings.warn(
-                "The 'authMechanism' option is deprecated. Please use 'auth_mechanism'.",
-                AirflowProviderDeprecationWarning,
-                stacklevel=2,
-            )
-            conn.extra_dejson["auth_mechanism"] = conn.extra_dejson["authMechanism"]
-            del conn.extra_dejson["authMechanism"]
-
         auth_mechanism = conn.extra_dejson.get("auth_mechanism", "NOSASL")
 
         if conf.get("core", "security") == "kerberos":
@@ -872,15 +860,6 @@ def get_conn(self, schema: str | None = None) -> Any:
 
         db = self.get_connection(self.hiveserver2_conn_id)  # type: ignore
 
-        if "authMechanism" in db.extra_dejson:
-            warnings.warn(
-                "The 'authMechanism' option is deprecated. Please use 'auth_mechanism'.",
-                AirflowProviderDeprecationWarning,
-                stacklevel=2,
-            )
-            db.extra_dejson["auth_mechanism"] = db.extra_dejson["authMechanism"]
-            del db.extra_dejson["authMechanism"]
-
         auth_mechanism = db.extra_dejson.get("auth_mechanism", "NONE")
         if auth_mechanism == "NONE" and db.login is None:
             # we need to give a username

airflow/providers/apache/hive/operators/hive.py

@@ -22,10 +22,7 @@
 from functools import cached_property
 from typing import TYPE_CHECKING, Any, Sequence
 
-from deprecated.classic import deprecated
-
 from airflow.configuration import conf
-from airflow.exceptions import AirflowProviderDeprecationWarning
 from airflow.models import BaseOperator
 from airflow.providers.apache.hive.hooks.hive import HiveCliHook
 from airflow.utils import operator_helpers
@@ -131,11 +128,6 @@ def hook(self) -> HiveCliHook:
             proxy_user=self.proxy_user,
         )
 
-    @deprecated(reason="use `hook` property instead.", category=AirflowProviderDeprecationWarning)
-    def get_hook(self) -> HiveCliHook:
-        """Get Hive cli hook."""
-        return self.hook
-
     def prepare_template(self) -> None:
         if self.hiveconf_jinja_translate:
             self.hql = re.sub(r"(\$\{(hiveconf:)?([ a-zA-Z0-9_]*)\})", r"{{ \g<3> }}", self.hql)

airflow/providers/apache/hive/operators/hive_stats.py

@@ -18,15 +18,13 @@
 from __future__ import annotations
 
 import json
-import warnings
 from typing import TYPE_CHECKING, Any, Callable, Sequence
 
-from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning
+from airflow.exceptions import AirflowException
 from airflow.models import BaseOperator
 from airflow.providers.apache.hive.hooks.hive import HiveMetastoreHook
 from airflow.providers.mysql.hooks.mysql import MySqlHook
 from airflow.providers.presto.hooks.presto import PrestoHook
-from airflow.utils.types import NOTSET, ArgNotSet
 
 if TYPE_CHECKING:
     from airflow.utils.context import Context
@@ -79,18 +77,8 @@ def __init__(
         mysql_conn_id: str = "airflow_db",
         ds: str = "{{ ds }}",
         dttm: str = "{{ logical_date.isoformat() }}",
-        col_blacklist: list[str] | None | ArgNotSet = NOTSET,
         **kwargs: Any,
     ) -> None:
-        if col_blacklist is not NOTSET:
-            warnings.warn(
-                f"col_blacklist kwarg passed to {self.__class__.__name__} "
-                f"(task_id: {kwargs.get('task_id')}) is deprecated, "
-                f"please rename it to excluded_columns instead",
-                category=AirflowProviderDeprecationWarning,
-                stacklevel=2,
-            )
-            excluded_columns = col_blacklist  # type: ignore[assignment]
         super().__init__(**kwargs)
         self.table = table
         self.partition = partition

tests/providers/apache/hive/operators/test_hive_stats.py

@@ -23,11 +23,9 @@
 
 import pytest
 
-from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning
+from airflow.exceptions import AirflowException
 from airflow.providers.apache.hive.operators.hive_stats import HiveStatsCollectionOperator
 from airflow.providers.presto.hooks.presto import PrestoHook
-from airflow.utils import timezone
-from airflow.utils.task_instance_session import set_current_task_instance_session
 from tests.providers.apache.hive import (
     DEFAULT_DATE,
     DEFAULT_DATE_DS,
@@ -372,41 +370,3 @@ def test_runs_for_hive_stats(self, mock_hive_metastore_hook):
                 "value",
             ],
         )
-
-    def test_col_blacklist_deprecation(self):
-        warn_message = "col_blacklist kwarg passed to.*task_id: fake-task-id.*is deprecated"
-        with pytest.warns(AirflowProviderDeprecationWarning, match=warn_message):
-            HiveStatsCollectionOperator(
-                task_id="fake-task-id",
-                table="airflow.static_babynames_partitioned",
-                partition={"ds": DEFAULT_DATE_DS},
-                col_blacklist=["foo", "bar"],
-            )
-
-    @pytest.mark.db_test
-    @pytest.mark.parametrize(
-        "col_blacklist",
-        [pytest.param(None, id="none"), pytest.param(["foo", "bar"], id="list")],
-    )
-    def test_partial_col_blacklist_deprecation(self, col_blacklist, dag_maker, session):
-        with dag_maker(
-            dag_id="test_partial_col_blacklist_deprecation",
-            start_date=timezone.datetime(2024, 1, 1),
-            session=session,
-        ):
-            HiveStatsCollectionOperator.partial(
-                task_id="fake-task-id",
-                partition={"ds": DEFAULT_DATE_DS},
-                col_blacklist=col_blacklist,
-                excluded_columns=["spam", "egg"],
-            ).expand(table=["airflow.table1", "airflow.table2"])
-
-        dr = dag_maker.create_dagrun(execution_date=None)
-        tis = dr.get_task_instances(session=session)
-        with set_current_task_instance_session(session=session):
-            warn_message = "col_blacklist kwarg passed to.*task_id: fake-task-id.*is deprecated"
-            for ti in tis:
-                with pytest.warns(AirflowProviderDeprecationWarning, match=warn_message):
-                    ti.render_templates()
-                expected = col_blacklist or []
-                assert ti.task.excluded_columns == expected