Replace to broad exceptions into the dev (#38489)

* Replace to broad exceptions into the Dev

* Add changes according to the review

Co-authored-by: Jarek Potiuk <[email protected]>
Co-authored-by: Tzu-ping Chung <[email protected]>

* RuntimeError -> SystemExit if detect it run in __main__

Co-authored-by: Tzu-ping Chung <[email protected]>

---------

Co-authored-by: Jarek Potiuk <[email protected]>
Co-authored-by: Tzu-ping Chung <[email protected]>

Author: 3 people

dev/breeze/src/airflow_breeze/commands/release_management_commands.py

@@ -1143,7 +1143,7 @@ def _get_all_providers_in_dist(
     for file in DIST_DIR.glob(f"{filename_prefix}*.tar.gz"):
         matched = filename_pattern.match(file.name)
         if not matched:
-            raise Exception(f"Cannot parse provider package name from {file.name}")
+            raise SystemExit(f"Cannot parse provider package name from {file.name}")
         provider_package_id = matched.group(1).replace("_", ".")
         yield provider_package_id
 
@@ -1168,7 +1168,7 @@ def get_all_providers_in_dist(package_format: str, install_selected_providers: s
             )
         )
     else:
-        raise Exception(f"Unknown package format {package_format}")
+        raise SystemExit(f"Unknown package format {package_format}")
     if install_selected_providers:
         filter_list = install_selected_providers.split(",")
         return [provider for provider in all_found_providers if provider in filter_list]
@@ -1293,11 +1293,12 @@ def install_provider_packages(
                     if dependency not in chunk:
                         chunk.append(dependency)
         if len(list_of_all_providers) != total_num_providers:
-            raise Exception(
+            msg = (
                 f"Total providers {total_num_providers} is different "
                 f"than {len(list_of_all_providers)} (just to be sure"
                 f" no rounding errors crippled in)"
             )
+            raise RuntimeError(msg)
         parallelism = min(parallelism, len(provider_chunks))
         with ci_group(f"Installing providers in {parallelism} chunks"):
             all_params = [f"Chunk {n}" for n in range(parallelism)]

dev/breeze/src/airflow_breeze/global_constants.py

@@ -364,7 +364,7 @@ def get_airflow_version():
                 airflow_version = line.split()[2][1:-1]
                 break
     if airflow_version == "unknown":
-        raise Exception("Unable to determine Airflow version")
+        raise RuntimeError("Unable to determine Airflow version")
     return airflow_version
 
 

dev/breeze/src/airflow_breeze/utils/docs_publisher.py

@@ -58,15 +58,19 @@ def _build_dir(self) -> str:
     @property
     def _current_version(self):
         if not self.is_versioned:
-            raise Exception("This documentation package is not versioned")
+            msg = (
+                "This documentation package is not versioned. "
+                "Make sure to add version in `provider.yaml` for the package."
+            )
+            raise RuntimeError(msg)
         if self.package_name == "apache-airflow":
             return get_airflow_version()
         if self.package_name.startswith("apache-airflow-providers-"):
             provider = get_provider_packages_metadata().get(get_short_package_name(self.package_name))
             return provider["versions"][0]
         if self.package_name == "helm-chart":
             return chart_version()
-        return Exception(f"Unsupported package: {self.package_name}")
+        raise SystemExit(f"Unsupported package: {self.package_name}")
 
     @property
     def _publish_dir(self) -> str:

dev/breeze/src/airflow_breeze/utils/kubernetes_utils.py

@@ -92,7 +92,8 @@ def get_architecture_string_for_urls() -> str:
         return "amd64"
     if architecture == Architecture.ARM:
         return "arm64"
-    raise Exception(f"The architecture {architecture} is not supported when downloading kubernetes tools!")
+    msg = f"The architecture {architecture} is not supported when downloading kubernetes tools!"
+    raise SystemExit(msg)
 
 
 def _download_with_retries(num_tries, path, tool, url):

dev/breeze/src/airflow_breeze/utils/packages.py

@@ -132,8 +132,9 @@ def refresh_provider_metadata_from_yaml_file(provider_yaml_path: Path):
 
         try:
             jsonschema.validate(provider, schema=schema)
-        except jsonschema.ValidationError:
-            raise Exception(f"Unable to parse: {provider_yaml_path}.")
+        except jsonschema.ValidationError as ex:
+            msg = f"Unable to parse: {provider_yaml_path}. Original error {type(ex).__name__}: {ex}"
+            raise RuntimeError(msg)
     except ImportError:
         # we only validate the schema if jsonschema is available. This is needed for autocomplete
         # to not fail with import error if jsonschema is not installed
@@ -176,12 +177,12 @@ def validate_provider_info_with_runtime_schema(provider_info: dict[str, Any]) ->
     try:
         jsonschema.validate(provider_info, schema=schema)
     except jsonschema.ValidationError as ex:
-        get_console().print("[red]Provider info not validated against runtime schema[/]")
-        raise Exception(
-            "Error when validating schema. The schema must be compatible with "
-            "airflow/provider_info.schema.json.",
-            ex,
+        get_console().print(
+            "[red]Error when validating schema. The schema must be compatible with "
+            "[bold]'airflow/provider_info.schema.json'[/bold].\n"
+            f"Original exception [bold]{type(ex).__name__}: {ex}[/]"
         )
+        raise SystemExit(1)
 
 
 def get_provider_info_dict(provider_id: str) -> dict[str, Any]:

docs/conf.py

@@ -72,7 +72,7 @@
             if provider_yaml["package-name"] == PACKAGE_NAME
         )
     except StopIteration:
-        raise Exception(f"Could not find provider.yaml file for package: {PACKAGE_NAME}")
+        raise RuntimeError(f"Could not find provider.yaml file for package: {PACKAGE_NAME}")
     PACKAGE_DIR = pathlib.Path(CURRENT_PROVIDER["package-dir"])
     PACKAGE_VERSION = CURRENT_PROVIDER["versions"][0]
     SYSTEM_TESTS_DIR = CURRENT_PROVIDER["system-tests-dir"]

docs/exts/docs_build/docs_builder.py

@@ -33,7 +33,6 @@
     PROCESS_TIMEOUT,
 )
 from docs.exts.docs_build.errors import DocBuildError, parse_sphinx_warnings
-from docs.exts.docs_build.helm_chart_utils import chart_version
 from docs.exts.docs_build.spelling_checks import SpellingError, parse_spelling_warnings
 
 console = Console(force_terminal=True, color_system="standard", width=CONSOLE_WIDTH)
@@ -88,21 +87,6 @@ def log_build_warning_filename(self) -> str:
         """Warnings from build job."""
         return os.path.join(self._build_dir, f"warning-build-{self.package_name}.log")
 
-    @property
-    def _current_version(self):
-        if not self.is_versioned:
-            raise Exception("This documentation package is not versioned")
-        if self.package_name == "apache-airflow":
-            from airflow.version import version as airflow_version
-
-            return airflow_version
-        if self.package_name.startswith("apache-airflow-providers-"):
-            provider = next(p for p in ALL_PROVIDER_YAMLS if p["package-name"] == self.package_name)
-            return provider["versions"][0]
-        if self.package_name == "helm-chart":
-            return chart_version()
-        return Exception(f"Unsupported package: {self.package_name}")
-
     @property
     def _src_dir(self) -> str:
         return f"{DOCS_DIR}/{self.package_name}"

docs/exts/operators_and_hooks_ref.py

@@ -69,10 +69,10 @@ def _render_template(template_name, **kwargs):
 
 def _docs_path(filepath: str):
     if not filepath.startswith("/docs/"):
-        raise Exception(f"The path must starts with '/docs/'. Current value: {filepath}")
+        raise RuntimeError(f"The path must starts with '/docs/'. Current value: {filepath}")
 
     if not filepath.endswith(".rst"):
-        raise Exception(f"The path must ends with '.rst'. Current value: {filepath}")
+        raise RuntimeError(f"The path must ends with '.rst'. Current value: {filepath}")
 
     if filepath.startswith("/docs/apache-airflow-providers-"):
         _, _, provider, rest = filepath.split("/", maxsplit=3)

docs/exts/provider_yaml_utils.py

@@ -67,8 +67,9 @@ def load_package_data(include_suspended: bool = False) -> list[dict[str, Any]]:
             provider = yaml.safe_load(yaml_file)
         try:
             jsonschema.validate(provider, schema=schema)
-        except jsonschema.ValidationError:
-            raise Exception(f"Unable to parse: {provider_yaml_path}.")
+        except jsonschema.ValidationError as ex:
+            msg = f"Unable to parse: {provider_yaml_path}. Original error {type(ex).__name__}: {ex}"
+            raise RuntimeError(msg)
         if provider["state"] == "suspended" and not include_suspended:
             continue
         provider_yaml_dir = os.path.dirname(provider_yaml_path)

hatch_build.py

@@ -592,11 +592,12 @@ def get_provider_requirement(provider_spec: str) -> str:
     if PROVIDER_DEPENDENCIES[provider_id]["state"] not in ["ready", "suspended", "removed"]:
         for dependency in PROVIDER_DEPENDENCIES[provider_id]["deps"]:
             if dependency.startswith("apache-airflow-providers"):
-                raise Exception(
+                msg = (
                     f"The provider {provider_id} is pre-installed and it has as dependency "
                     f"to another provider {dependency}. This is not allowed. Pre-installed"
                     f"providers should only have 'apache-airflow' and regular dependencies."
                 )
+                raise SystemExit(msg)
             if not dependency.startswith("apache-airflow"):
                 PREINSTALLED_NOT_READY_DEPS.append(dependency)
 

scripts/ci/pre_commit/generate_pypi_readme.py

@@ -54,7 +54,7 @@ def extract_section(content, section_name):
     if section_match:
         return section_match.group(1)
     else:
-        raise Exception(f"Cannot find section {section_name} in README.md")
+        raise RuntimeError(f"Cannot find section {section_name} in README.md")
 
 
 if __name__ == "__main__":

scripts/ci/pre_commit/insert_extras.py

@@ -42,7 +42,7 @@ def get_header_and_footer(extra_type: str, file_format: str) -> tuple[str, str]:
     elif file_format == "txt":
         return f"# START {extra_type.upper()} HERE", f"# END {extra_type.upper()} HERE"
     else:
-        raise Exception(f"Bad format {format} passed. Only rst and txt are supported")
+        raise ValueError(f"Bad format {format} passed. Only rst and txt are supported")
 
 
 def get_wrapped_list(extras_set: list[str]) -> list[str]:
@@ -81,7 +81,7 @@ def process_documentation_files() -> bool:
     extra_type_dict = get_extra_types_dict()
     for file, file_format, add_comment in FILES_TO_UPDATE:
         if not file.exists():
-            raise Exception(f"File {file} does not exist")
+            raise FileNotFoundError(f"File {file} does not exist")
         for extra_type_description, extra_list in extra_type_dict.items():
             header, footer = get_header_and_footer(extra_type_description, file_format)
             if insert_documentation(

scripts/ci/pre_commit/json_schema.py

@@ -30,7 +30,7 @@
 from jsonschema.validators import extend, validator_for
 
 if __name__ != "__main__":
-    raise Exception(
+    raise SystemExit(
         "This file is intended to be executed as an executable program. You cannot use it as a module."
         "To run this script, run the ./build_docs.py command"
     )
@@ -150,7 +150,7 @@ def _load_spec(spec_file: str | None, spec_url: str | None):
     if spec_url:
         spec_file = fetch_and_cache(url=spec_url, output_filename=re.sub(r"[^a-zA-Z0-9]", "-", spec_url))
     if not spec_file:
-        raise Exception(f"The {spec_file} was None and {spec_url} did not lead to any file loading.")
+        raise ValueError(f"The {spec_file} was None and {spec_url} did not lead to any file loading.")
     with open(spec_file) as schema_file:
         schema = json.loads(schema_file.read())
     return schema

scripts/ci/pre_commit/replace_bad_characters.py

@@ -25,7 +25,7 @@
 from rich.console import Console
 
 if __name__ != "__main__":
-    raise Exception(
+    raise SystemExit(
         "This file is intended to be executed as an executable program. You cannot use it as a module."
         f"To run this script, run the {__file__} command"
     )

scripts/ci/pre_commit/update_chart_dependencies.py

@@ -36,7 +36,7 @@ def get_latest_prometheus_statsd_exporter_version() -> str:
     for version in quay_data["tags"]:
         if version["name"].startswith("v"):
             return version["name"]
-    raise Exception("ERROR! No version found")
+    raise RuntimeError("ERROR! No version found")
 
 
 if __name__ == "__main__":
@@ -64,7 +64,7 @@ def get_latest_prometheus_statsd_exporter_version() -> str:
                 next_line = f"    tag: {latest_prometheus_statsd_exporter_version}"
                 replace = False
             else:
-                raise Exception(
+                raise ValueError(
                     f"ERROR! The next line after repository: should be tag: - "
                     f"index {index} in {VALUES_YAML_FILE}"
                 )

scripts/ci/pre_commit/update_versions.py

@@ -31,7 +31,7 @@ def update_version(pattern: re.Pattern, v: str, file_path: Path):
     with file_path.open("r+") as f:
         file_content = f.read()
         if not pattern.search(file_content):
-            raise Exception(f"Pattern {pattern!r} doesn't found in {file_path!r} file")
+            raise RuntimeError(f"Pattern {pattern!r} doesn't found in {file_path!r} file")
         new_content = pattern.sub(rf"\g<1>{v}\g<2>", file_content)
         if file_content == new_content:
             return

scripts/in_container/run_provider_yaml_files_check.py

@@ -62,7 +62,7 @@
     from yaml import SafeLoader  # type: ignore
 
 if __name__ != "__main__":
-    raise Exception(
+    raise SystemExit(
         "This file is intended to be executed as an executable program. You cannot use it as a module."
     )
 
@@ -112,8 +112,9 @@ def _load_package_data(package_paths: Iterable[str]):
         rel_path = pathlib.Path(provider_yaml_path).relative_to(ROOT_DIR).as_posix()
         try:
             jsonschema.validate(provider, schema=schema)
-        except jsonschema.ValidationError:
-            raise Exception(f"Unable to parse: {rel_path}.")
+        except jsonschema.ValidationError as ex:
+            msg = f"Unable to parse: {provider_yaml_path}. Original error {type(ex).__name__}: {ex}"
+            raise RuntimeError(msg)
         if not provider["state"] == "suspended":
             result[rel_path] = provider
         else:

scripts/in_container/update_quarantined_test_status.py

@@ -202,14 +202,14 @@ def get_table(history_map: dict[str, TestHistory]) -> str:
     print(f"Token: {token}")
     github_repository = os.environ.get("GITHUB_REPOSITORY")
     if not github_repository:
-        raise Exception("GitHub Repository must be defined!")
+        raise RuntimeError("GitHub Repository must be defined!")
     user, repo = github_repository.split("/")
     print(f"User: {user}, Repo: {repo}")
     issue_id = int(os.environ.get("ISSUE_ID", 0))
     num_runs = int(os.environ.get("NUM_RUNS", 10))
 
     if issue_id == 0:
-        raise Exception("You need to define ISSUE_ID as environment variable")
+        raise RuntimeError("You need to define ISSUE_ID as environment variable")
 
     gh = login(token=token)
 

scripts/tools/check_if_limited_dependencies.py

@@ -27,7 +27,7 @@
 program = f"./{__file__}" if not __file__.startswith("./") else __file__
 
 if __name__ != "__main__":
-    raise Exception(
+    raise SystemExit(
         "This file is intended to be used as an executable program. You cannot use it as a module."
         f"To execute this script, run the '{program}' command"
     )

scripts/tools/list-integrations.py

@@ -34,7 +34,7 @@
 program = f"./{__file__}" if not __file__.startswith("./") else __file__
 
 if __name__ != "__main__":
-    raise Exception(
+    raise SystemExit(
         "This file is intended to be used as an executable program. You cannot use it as a module."
         f"To execute this script, run the '{program}' command"
     )