Merge pull request #342 from tekktrik/dev/ci-rerun-update

Tooling update

Author: kattni

.gitignore

@@ -11,6 +11,7 @@ _build
 env.sh
 *.swp
 .libraries/*
+.gitlibs/*
 .cp_org/*
 .blinka/*
 .vscode

tools/ci_status.py

@@ -16,8 +16,10 @@
 
 from typing import Optional
 import argparse
+import time
 from github.Repository import Repository
 from github.Workflow import Workflow
+from github.WorkflowRun import WorkflowRun
 from github.GithubException import GithubException
 from library_functions import StrPath
 from iterate_libraries import (
@@ -56,6 +58,46 @@ def run_gh_rest_check(
     return workflow_runs[0].conclusion
 
 
+def run_gh_rest_rerun(
+    lib_repo: Repository,
+    user: Optional[str] = None,
+    branch: Optional[str] = None,
+    workflow_filename: Optional[str] = "build.yml",
+    rerun_level: int = 0,
+) -> bool:
+    """Uses ``PyGithub`` to rerun the CI status of a repository
+
+    :param Repository lib_repo: The repo as a github.Repository.Repository object
+    :param str|None user: The user that triggered the run; if `None` is
+        provided, any user is acceptable
+    :param str|None branch: The branch name to specifically check; if `None` is
+        provided, all branches are allowed; this is the default
+    :param str|None workflow_filename: The filename of the workflow; if `None` is
+        provided, any workflow name is acceptable; the default is ``"build.yml"``
+    :param int rerun_level: The level at which rerun should occur (0 = none,
+        1 = failed, 2 = all)
+    :return: The requested runs conclusion
+    :rtype: bool
+    """
+    if not rerun_level:
+        return False
+    if rerun_level == 1:
+        result = (
+            run_gh_rest_check(lib_repo, user, branch, workflow_filename) == "success"
+        )
+    if rerun_level == 2 or not result:
+        arg_dict = {}
+        if user is not None:
+            arg_dict["actor"] = user
+        if branch is not None:
+            arg_dict["branch"] = branch
+        workflow: Workflow = lib_repo.get_workflow(workflow_filename)
+        latest_run: WorkflowRun = workflow.get_runs(**arg_dict)[0]
+        latest_run.rerun()
+        return True
+    return False
+
+
 def check_build_status(
     lib_repo: Repository,
     user: Optional[str] = None,
@@ -105,13 +147,63 @@ def check_build_status(
         return None
 
 
+# pylint: disable=too-many-arguments
+def rerun_workflow(
+    lib_repo: Repository,
+    user: Optional[str] = None,
+    branch: Optional[str] = None,
+    workflow_filename: Optional[str] = "build.yml",
+    rerun_level: int = 0,
+    debug: bool = False,
+):
+    """Uses ``PyGithub`` to rerun the CI of the Adafruit
+    CircuitPython Bundle repositories
+
+    :param Repository lib_repo: The repo as a github.Repository.Repository object
+    :param str|None user: The user that triggered the run; if `None` is
+        provided, any user is acceptable
+    :param str|None branch: The branch name to specifically check; if `None` is
+        provided, all branches are allowed; this is the default
+    :param str|None workflow_filename: The filename of the workflow; if `None`
+        is provided, any workflow name is acceptable; the defail is `"build.yml"`
+    :param int rerun_level: The level at which rerun should occur (0 = none,
+        1 = failed, 2 = all)
+    :param bool debug: Whether debug statements should be printed to the standard
+        output
+    :return: The result of the workflow run, or ``None`` if it could not be
+        determined
+    :rtype: bool|None
+    """
+    if lib_repo.archived:
+        return False
+
+    try:
+        result = run_gh_rest_rerun(
+            lib_repo, user, branch, workflow_filename, rerun_level
+        )
+        if debug and result:
+            print("***", "Library", lib_repo.name, "workflow was rerun!", "***")
+        return result
+    except GithubException:
+        if debug:
+            print(
+                "???",
+                "Library",
+                lib_repo.name,
+                "had an issue occur",
+                "???",
+            )
+        return None
+
+
 def check_build_statuses(
     gh_token: str,
     user: Optional[str] = None,
     branch: Optional[str] = "main",
     workflow_filename: Optional[str] = "build.yml",
     *,
     debug: bool = False,
+    local_folder: str = "",
 ) -> list[RemoteLibFunc_IterResult[bool]]:
     """Checks all the libraries in the Adafruit CircuitPython Bundle to get the
     latest build status with the requested information
@@ -125,6 +217,7 @@ def check_build_statuses(
         provided, any workflow name is acceptable; the defail is `"build.yml"`
     :param bool debug: Whether debug statements should be printed to
         the standard output
+    :param str local_folder: A path to a local folder containing extra repositories
     :return: A list of tuples containing paired Repoistory objects and build
         statuses
     :rtype: list
@@ -133,6 +226,49 @@ def check_build_statuses(
     return iter_remote_bundle_with_func(
         gh_token,
         [(check_build_status, (user, branch, workflow_filename), {"debug": debug})],
+        local_folder=local_folder,
+    )
+
+
+def rerun_workflows(
+    gh_token: str,
+    user: Optional[str] = None,
+    branch: Optional[str] = "main",
+    workflow_filename: Optional[str] = "build.yml",
+    rerun_level: int = 0,
+    *,
+    debug: bool = False,
+    local_folder: str = "",
+) -> list[RemoteLibFunc_IterResult[bool]]:
+    """Reruns the CI of all the libraries in the Adafruit CircuitPython Bundle.
+
+    :param str gh_token: The Github token to be used for with the Github API
+    :param str|None user: The user that triggered the run; if `None` is
+        provided, any user is acceptable
+    :param str|None branch: The branch name to specifically check; if `None` is
+        provided, all branches are allowed; this is the default
+    :param str|None workflow_filename: The filename of the workflow; if `None` is
+        provided, any workflow name is acceptable; the defail is `"build.yml"`
+    :param int rerun_level: The level at which reruns should occur (0 = none,
+        1 = failed, 2 = all)
+    :param bool debug: Whether debug statements should be printed to
+        the standard output
+    :param str local_folder: A path to a local folder containing extra repositories
+    :return: A list of tuples containing paired Repoistory objects and build
+        statuses
+    :rtype: list
+    """
+
+    return iter_remote_bundle_with_func(
+        gh_token,
+        [
+            (
+                rerun_workflow,
+                (user, branch, workflow_filename, rerun_level),
+                {"debug": debug},
+            )
+        ],
+        local_folder=local_folder,
     )
 
 
@@ -193,12 +329,52 @@ def save_build_statuses(
     parser.add_argument(
         "--debug", action="store_true", help="Print debug text during execution"
     )
+    parser.add_argument(
+        "--rerun-level",
+        metavar="R",
+        type=int,
+        dest="rerun_level",
+        default=0,
+        help="Level to rerun CI workflows (0 = none, 1 = failed, 2 = all)",
+    )
+    parser.add_argument(
+        "--local-folder",
+        metavar="L",
+        type=str,
+        dest="local_folder",
+        default="",
+        help="An additional folder to check and run",
+    )
 
     args = parser.parse_args()
 
+    if args.rerun_level:
+        if args.debug:
+            print("Rerunning workflows...")
+        rerun_workflows(
+            args.gh_token,
+            args.user,
+            args.branch,
+            args.workflow,
+            args.rerun_level,
+            debug=args.debug,
+            local_folder=args.local_folder,
+        )
+        if args.debug:
+            print("Waiting 10 minutes to allow workflows to finish running...")
+        time.sleep(600)
+
+    if args.debug:
+        print("Checking workflows statuses...")
     results = check_build_statuses(
-        args.gh_token, args.user, args.branch, args.workflow, debug=args.debug
+        args.gh_token,
+        args.user,
+        args.branch,
+        args.workflow,
+        debug=args.debug,
+        local_folder=args.local_folder,
     )
+
     fail_list = [
         repo_name.name for repo_name, repo_results in results if not repo_results[0]
     ]

tools/iterate_libraries.py

@@ -16,8 +16,9 @@
 
 import os
 import glob
+import pathlib
 from collections.abc import Sequence, Iterable
-from typing import TypeVar
+from typing import TypeVar, Any, Union, List
 from typing_extensions import TypeAlias
 import parse
 from github import Github
@@ -65,9 +66,25 @@
 _BUNDLE_BRANCHES = ("drivers", "helpers")
 
 
+def perform_func(
+    item: Any,
+    func_workflow: Union[RemoteLibFunc_IterInstruction, LocalLibFunc_IterInstruction],
+) -> Union[List[RemoteLibFunc_IterResult], List[LocalLibFunc_IterResult]]:
+    """
+    Perform the given function
+    """
+    func_results = []
+    for func, args, kwargs in func_workflow:
+        result = func(item, *args, **kwargs)
+        func_results.append(result)
+    return func_results
+
+
 def iter_local_bundle_with_func(
     bundle_path: StrPath,
     func_workflow: Iterable[LocalLibFunc_IterInstruction],
+    *,
+    local_folder: str = "",
 ) -> list[LocalLibFunc_IterResult]:
     """Iterate through the libraries and run a given function with the
     provided arguments
@@ -85,6 +102,9 @@ def iter_local_bundle_with_func(
     # Initialize list of results
     results = []
 
+    # Keep track of all libraries iterated
+    iterated = set()
+
     # Loop through each bundle branch
     for branch_name in _BUNDLE_BRANCHES:
 
@@ -94,20 +114,30 @@ def iter_local_bundle_with_func(
         # Enter each library in the bundle
         for library_path in libraries_path_list:
 
-            func_results = []
-
-            for func, args, kwargs in func_workflow:
-                result = func(library_path, *args, **kwargs)
-                func_results.append(result)
+            iterated.add(os.path.split(library_path)[1].lower())
+            func_results = perform_func(library_path, func_workflow)
 
             results.append((library_path, func_results))
 
+    if local_folder:
+        additional = {
+            os.path.split(pathname)[1].lower()
+            for pathname in glob.glob(os.path.join(local_folder, "*"))
+        }
+        diff = additional.difference(iterated)
+        for unused in diff:
+            unused_func_results = perform_func(unused, func_workflow)
+            results.append((unused, unused_func_results))
+
     return results
 
 
 # pylint: disable=too-many-locals
 def iter_remote_bundle_with_func(
-    gh_token: str, func_workflow: RemoteLibFunc_IterInstruction
+    gh_token: str,
+    func_workflow: RemoteLibFunc_IterInstruction,
+    *,
+    local_folder: str = "",
 ) -> list[RemoteLibFunc_IterResult]:
     """Iterate through the remote bundle, accessing each library's git repo
     using the GitHub RESTful API (specifically using ``PyGithub``)
@@ -129,6 +159,9 @@ def iter_remote_bundle_with_func(
     # Initialize list of results
     results = []
 
+    # Keep track of all libraries iterated
+    iterated = set()
+
     # Loop through each bundle branch
     for branch_name in _BUNDLE_BRANCHES:
 
@@ -144,13 +177,19 @@ def iter_remote_bundle_with_func(
             repo_name: str = repo_name_result.named["repo_name"]
 
             repo = github_client.get_repo(f"adafruit/{repo_name}")
+            iterated.add(repo_name.lower())
 
-            func_results = []
-
-            for func, args, kwargs in func_workflow:
-                result = func(repo, *args, **kwargs)
-                func_results.append(result)
-
+            func_results = perform_func(repo, func_workflow)
             results.append((repo, func_results))
 
+    if local_folder:
+        additional = {
+            path.name.lower() for path in pathlib.Path(local_folder).glob("*")
+        }
+        diff = additional.difference(iterated)
+        for unused in diff:
+            unused_repo = github_client.get_repo(f"adafruit/{unused}")
+            unused_func_results = perform_func(unused_repo, func_workflow)
+            results.append((unused_repo, unused_func_results))
+
     return results

tools/run_black.sh

@@ -0,0 +1,17 @@
+# SPDX-FileCopyrightText: 2023 Alec Delaney
+#
+# SPDX-License-Identifier: MIT
+
+rm -rf .gitlibs
+mkdir .gitlibs
+cd .libraries
+for repo in *; do
+    cd ../.gitlibs
+    git clone https://github.com/adafruit/$repo.git
+    cd $repo
+    pre-commit run --all-files
+    git add -A
+    git commit -m "Run pre-commit"
+    git push
+    cd ..
+done