Merge branch 'master' of github.com:adafruit/adabot

Author: ladyada

.github/workflows/bundle_cron.yml

@@ -49,6 +49,7 @@ jobs:
     - name: Run adabot.circuitpython_bundle
       env:
         ADABOT_EMAIL: ${{ secrets.ADABOT_EMAIL }}
+        ADABOT_GITHUB_USER: ${{ secrets.ADABOT_GITHUB_USER }}
         ADABOT_GITHUB_ACCESS_TOKEN: ${{ secrets.ADABOT_GITHUB_ACCESS_TOKEN }}
         REDIS_PORT: ${{ job.services.redis.ports[6379] }}
       run: |

.github/workflows/reports_cron.yml

@@ -23,6 +23,7 @@ jobs:
     # be limited (they run on all forks' default branches).
     if: startswith(github.repository, 'adafruit/')
     env:
+      ADABOT_GITHUB_USER: ${{ secrets.ADABOT_GITHUB_USER }}
       ADABOT_GITHUB_ACCESS_TOKEN: ${{ secrets.ADABOT_GITHUB_ACCESS_TOKEN }}
     steps:
       - name: Dump GitHub context

README.rst

@@ -115,7 +115,7 @@ Applying Patches To All CircuitPython Libraries
 ================================================
 To apply a patch to all CircuitPython libraries (only guaranteed for files shared
 among all libraries, such as those included in the cookiecutter (e.g. README.rst,
-.travis.yml, etc), do the following:
+ etc), do the following:
 
 1. Apply your update(s) to any library as normal, using ``git commit``. It is recommended to
 give a short, detailed description of the patch. This description will be used by the next

adabot/arduino_libraries.py

@@ -49,7 +49,7 @@ def list_repos():
     """
     repos = []
     result = github.get("/search/repositories",
-                        params={"q":"Arduino in:name in:description in:readme fork:true user:adafruit archived:false AND NOT PCB in:name AND NOT CircuitPython in:name",
+                        params={"q":"Arduino in:name in:description in:readme fork:true user:adafruit archived:false OR Library in:name in:description in:readme fork:true user:adafruit archived:false AND NOT PCB in:name AND NOT CircuitPython in:name",
                                 "per_page": 100,
                                 "sort": "updated",
                                 "order": "asc"})
@@ -160,11 +160,11 @@ def validate_release_state(repo):
 
     return
 
-def validate_travis(repo):
-    """Validate if a repo has .travis.yml.
+def validate_actions(repo):
+    """Validate if a repo has workflows/githubci.yml
     """
-    repo_has_travis = requests.get("https://raw.githubusercontent.com/adafruit/" + repo["name"] + "/master/.travis.yml")
-    return repo_has_travis.ok
+    repo_has_actions = requests.get("https://raw.githubusercontent.com/adafruit/" + repo["name"] + "/master/.github/workflows/githubci.yml")
+    return repo_has_actions.ok
 
 def validate_actions(repo):
     """Validate if a repo has actions githubci.yml
@@ -187,7 +187,7 @@ def run_arduino_lib_checks():
     failed_lib_prop = [["  Repo", "Release Tag", "library.properties Version"], ["  ----", "-----------", "--------------------------"]]
     needs_release_list = [["  Repo", "Latest Release", "Commits Behind"], ["  ----", "--------------", "--------------"]]
     needs_registration_list = [["  Repo"], ["  ----"]]
-    missing_ci_list = [["  Repo"], ["  ----"]]
+    missing_actions_list = [["  Repo"], ["  ----"]]
     missing_library_properties_list = [["  Repo"], ["  ----"]]
 
     for repo in repo_list:
@@ -223,10 +223,10 @@ def run_arduino_lib_checks():
         if needs_release:
             needs_release_list.append(["  " + str(repo["name"]), needs_release[0], needs_release[1]])
 
-        missing_ci = not validate_travis(repo) and not validate_actions(repo)
-        entry['needs_ci'] = missing_ci
-        if missing_ci:
-            missing_ci_list.append(["  " + str(repo["name"])])
+        missing_actions = not validate_actions(repo)
+        entry['needs_actions'] = missing_actions
+        if missing_actions:
+            missing_actions_list.append(["  " + str(repo["name"])])
 
         all_libraries.append(entry)
 
@@ -242,8 +242,8 @@ def run_arduino_lib_checks():
     if len(needs_release_list) > 2:
         print_list_output("Libraries have commits since last release: ({})", needs_release_list);
 
-    if len(missing_ci_list) > 2:
-        print_list_output("Libraries that is not configured with Travis or Actions: ({})", missing_ci_list)
+    if len(missing_actions_list) > 2:
+        print_list_output("Libraries that is not configured with Actions: ({})", missing_actions_list)
 
     if len(missing_library_properties_list) > 2:
         print_list_output("Libraries that is missing library.properties file: ({})", missing_library_properties_list)

adabot/circuitpython_libraries.py

@@ -32,7 +32,6 @@
 import requests
 
 from adabot import github_requests as github
-from adabot import travis_requests as travis
 from adabot import pypi_requests as pypi
 from adabot.lib import circuitpython_library_validators as cirpy_lib_vals
 from adabot.lib import common_funcs
@@ -68,12 +67,6 @@
     default=5,
     metavar="n"
 )
-cmd_line_parser.add_argument(
-    "-t", "--travis-github-token",
-    help="Prompt for the GitHub user's password in order to make a GitHub token to use on Travis.",
-    dest="gh_token",
-    action="store_true"
-)
 cmd_line_parser.add_argument(
     "-v", "--validator",
     help="Run validators with 'all', or only the validator(s) supplied in a string.",
@@ -103,7 +96,8 @@
     if vals[0].startswith("validate")
 ]
 
-pr_sort_re = re.compile("(?<=\(Open\s)(.+)(?=\sdays)")
+pr_sort_re = re.compile(r"(?<=\(Open\s)(.+)(?=\sdays)")
+close_pr_sort_re = re.compile(r"(?<=\(Days\sopen:\s)(.+)(?=\))")
 
 def run_library_checks(validators, bundle_submodules, latest_pylint, kw_args):
     """runs the various library checking functions"""
@@ -112,7 +106,8 @@ def run_library_checks(validators, bundle_submodules, latest_pylint, kw_args):
         latest_pylint = pylint_info.json()["info"]["version"]
     output_handler("Latest pylint is: {}".format(latest_pylint))
 
-    repos = common_funcs.list_repos(include_repos=('Adafruit_Blinka',))
+    repos = common_funcs.list_repos(include_repos=('Adafruit_Blinka',
+                                                   'CircuitPython_Community_Bundle'))
     output_handler("Found {} repos to check.".format(len(repos)))
     bundle_submodules = common_funcs.get_bundle_submodules()
     output_handler("Found {} submodules in the bundle.".format(len(bundle_submodules)))
@@ -165,7 +160,8 @@ def run_library_checks(validators, bundle_submodules, latest_pylint, kw_args):
                 insights = blinka_insights
             elif repo["name"] == "circuitpython":
                 insights = core_insights
-        errors = validator.gather_insights(repo, insights, since)
+        closed_metric = bool(insights == lib_insights)
+        errors = validator.gather_insights(repo, insights, since, show_closed_metric=closed_metric)
         if errors:
             print("insights error")
             for error in errors:
@@ -182,11 +178,11 @@ def run_library_checks(validators, bundle_submodules, latest_pylint, kw_args):
                 updated_libs[repo["name"]] = repo["html_url"]
 
     output_handler()
-    output_handler("State of CircuitPython + Libraries")
+    output_handler("State of CircuitPython + Libraries + Blinka")
 
     output_handler("Overall")
-    print_pr_overview(lib_insights, core_insights)
-    print_issue_overview(lib_insights, core_insights)
+    print_pr_overview(lib_insights, core_insights, blinka_insights)
+    print_issue_overview(lib_insights, core_insights, blinka_insights)
 
     output_handler()
     output_handler("Core")
@@ -208,20 +204,38 @@ def run_library_checks(validators, bundle_submodules, latest_pylint, kw_args):
                                                          core_insights["milestones"][milestone]))
     output_handler("  * {} issues not assigned a milestone".format(len(core_insights["open_issues"]) - ms_count))
     output_handler()
-    print_circuitpython_download_stats()
+
+    ## temporarily disabling core download stats:
+    #  - GitHub API has been broken, due to the number of release artifacts
+    #  - Release asset delivery is being moved to AWS CloudFront/S3
+    #print_circuitpython_download_stats()
+    output_handler(
+        "* Core download stats available at https://circuitpython.org/stats"
+    )
 
     output_handler()
     output_handler("Libraries")
     print_pr_overview(lib_insights)
-    output_handler("* {} open pull requests".format(len(lib_insights["open_prs"])))
-    sorted_prs = sorted(lib_insights["open_prs"],
-                        key=lambda days: int(pr_sort_re.search(days).group(1)),
+    output_handler("  * Merged pull requests:")
+    sorted_prs = sorted(lib_insights["merged_prs"],
+                        key=lambda days: int(close_pr_sort_re.search(days).group(1)),
                         reverse=True)
     for pr in sorted_prs:
-        output_handler("  * {}".format(pr))
+        output_handler("    * {}".format(pr))
     print_issue_overview(lib_insights)
-    output_handler("* {} open issues".format(len(lib_insights["open_issues"])))
-    output_handler("  * https://circuitpython.org/contributing")
+    output_handler("* https://circuitpython.org/contributing")
+    output_handler("  * {} open issues".format(len(lib_insights["open_issues"])))
+    open_pr_days = [
+        int(pr_sort_re.search(pr).group(1)) for pr in lib_insights["open_prs"]
+        if pr_sort_re.search(pr) is not None
+    ]
+    output_handler(
+        "  * {0} open pull requests (Oldest: {1}, Newest: {2})".format(
+            len(lib_insights["open_prs"]),
+            max(open_pr_days),
+            max((min(open_pr_days), 1)) # ensure the minumum is '1'
+        )
+    )
     output_handler("Library updates in the last seven days:")
     if len(new_libs) != 0:
         output_handler("**New Libraries**")
@@ -280,9 +294,25 @@ def output_handler(message="", quiet=False):
 
 def print_circuitpython_download_stats():
     """Gather and report analytics on the main CircuitPython repository."""
-    response = github.get("/repos/adafruit/circuitpython/releases")
+
+    # TODO: with the move of release assets to AWS CloudFront/S3, update
+    #       this to use AWS CloudWatch metrics to gather download stats.
+    #       AWS' Python SDK `boto3` has CloudWatch interfaces which should
+    #       enable this. https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/cloudwatch.html
+
+    try:
+        response = github.get("/repos/adafruit/circuitpython/releases")
+    except (ValueError, RuntimeError):
+        output_handler(
+            "Core CircuitPython GitHub download statistics request failed."
+        )
+        return
+
     if not response.ok:
-        output_handler("Core CircuitPython GitHub analytics request failed.")
+        output_handler(
+            "Core CircuitPython GitHub download statistics request failed."
+        )
+        return
     releases = response.json()
 
     found_unstable = False
@@ -416,7 +446,7 @@ def print_circuitpython_download_stats():
     output_handler()
 
 def print_pr_overview(*insights):
-    merged_prs = sum([x["merged_prs"] for x in insights])
+    merged_prs = sum([len(x["merged_prs"]) for x in insights])
     authors = set().union(*[x["pr_merged_authors"] for x in insights])
     reviewers = set().union(*[x["pr_reviewers"] for x in insights])
 
@@ -467,10 +497,6 @@ def print_issue_overview(*insights):
         error_depth = cmd_line_args.error_depth
         startup_message.append(" - Depth for listing libraries with errors: {}".format(error_depth))
 
-        github_token = cmd_line_args.gh_token
-        validator_kwarg_list["github_token"] = github_token
-        startup_message.append(" - Prompts for the GitHub Token are {}.".format(("enabled" if github_token else "disabled")))
-
         if cmd_line_args.validator != "all":
             validators = []
             for func in cmd_line_args.validator.split(","):

adabot/circuitpython_library_patches.py

@@ -173,6 +173,11 @@ def check_patches(repo, patches, flags, use_apply, dry_run):
             run_apply = False
             if (b"error" not in Err.stderr or
                 b"patch does not apply" in Err.stderr):
+                    parse_err = Err.stderr.decode()
+                    parse_err = parse_err[parse_err.rfind(":")+1:-1]
+                    print(
+                        "   . Skipping {}:{}".format(repo["name"], parse_err)
+                    )
                     skipped += 1
             else:
                 failed += 1

adabot/github_requests.py

@@ -35,6 +35,8 @@
 import time
 import traceback
 
+from base64 import b64encode
+
 TIMEOUT = 60
 
 def _fix_url(url):
@@ -52,11 +54,12 @@ def _fix_kwargs(kwargs):
     else:
         kwargs["headers"] = {"Accept": "application/vnd.github.hellcat-preview+json"}
     if "ADABOT_GITHUB_ACCESS_TOKEN" in os.environ and "auth" not in kwargs:
+        user = os.environ.get("ADABOT_GITHUB_USER", "")
         access_token = os.environ["ADABOT_GITHUB_ACCESS_TOKEN"]
-        if "params" in kwargs:
-            kwargs["params"]["access_token"] = access_token
-        else:
-            kwargs["params"] = {"access_token": access_token}
+        basic_encoded = b64encode(str(user + ":" + access_token).encode()).decode()
+        auth_header = "Basic {}".format(basic_encoded)
+
+        kwargs["headers"]["Authorization"] = auth_header
 
     return kwargs
 
@@ -79,15 +82,11 @@ def get(url, **kwargs):
             print("GitHub API Rate Limit reached. Pausing until Rate Limit reset.")
             while datetime.datetime.now() < rate_limit_reset:
                 print("Rate Limit will reset at: {}".format(rate_limit_reset))
-                if "TRAVIS" in os.environ:
-                    # only pause for 5 minutes so that Travis doesn't timeout
-                    # due to idle console output.
-                    time.sleep(300)
-                else:
-                    reset_diff = rate_limit_reset - datetime.datetime.now()
-
-                    print("Sleeping {} seconds".format(reset_diff.seconds))
-                    time.sleep(reset_diff.seconds + 1)
+                reset_diff = rate_limit_reset - datetime.datetime.now()
+
+                print("Sleeping {} seconds".format(reset_diff.seconds))
+                time.sleep(reset_diff.seconds + 1)
+
         if remaining % 100 == 0:
             print(remaining, "requests remaining this hour")
     return response

adabot/lib/blinka_funcs.py

@@ -20,38 +20,17 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 # THE SOFTWARE.
 
-import os
-import pathlib
-import re
-
-import sh
-from sh.contrib import git
-
+from adabot import github_requests as github
 
 def board_count():
     """ Retrieve the number of boards currently supported by Adafruit_Blinka,
-        via Adafruit_Python_PlatformDetect.
+        via the count of files in circuitpython-org/_blinka.
     """
-    platdetec_boards_re = re.compile(r'^[A-Z]\w+\s+\=\s[\'\"]\w+[\'\"]',
-                                     re.MULTILINE)
     board_count = 0
-    working_dir = os.getcwd()
-    blinka_dir = pathlib.Path(working_dir, '.blinka')
-    repo_url = 'https://github.com/adafruit/Adafruit_Python_PlatformDetect.git'
-
-    if not blinka_dir.exists():
-        try:
-            git.clone(repo_url, blinka_dir.resolve(), '--depth', '1')
-        except sh.ErrorReturnCode_128:
-            print("Failed to clone Adafruit_Blinka. Board count not determined.")
-            board_count = "Error"
-
-    src_board_path = blinka_dir / 'adafruit_platformdetect/board.py'
-    if src_board_path.exists():
-        board_content = ""
-        with open(src_board_path, 'r') as board_py:
-            board_content = board_py.read()
-        content_re = platdetec_boards_re.findall(board_content)
-        board_count = len(content_re)
+    cirpy_org_url = '/repos/adafruit/circuitpython-org/contents/_blinka'
+    response = github.get(cirpy_org_url)
+    if response.ok:
+        response_json = response.json()
+        board_count = len(response_json)
 
     return board_count