|
| 1 | +# The MIT License (MIT) |
| 2 | +# |
| 3 | +# Copyright (c) 2017 Scott Shawcroft for Adafruit Industries |
| 4 | +# |
| 5 | +# Permission is hereby granted, free of charge, to any person obtaining a copy |
| 6 | +# of this software and associated documentation files (the "Software"), to deal |
| 7 | +# in the Software without restriction, including without limitation the rights |
| 8 | +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell |
| 9 | +# copies of the Software, and to permit persons to whom the Software is |
| 10 | +# furnished to do so, subject to the following conditions: |
| 11 | +# |
| 12 | +# The above copyright notice and this permission notice shall be included in |
| 13 | +# all copies or substantial portions of the Software. |
| 14 | +# |
| 15 | +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
| 16 | +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
| 17 | +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE |
| 18 | +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER |
| 19 | +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, |
| 20 | +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN |
| 21 | +# THE SOFTWARE. |
| 22 | + |
| 23 | +# GitHub API Serch has stopped returning the core repo for some reason. Tried several |
| 24 | +# different search params, and came up emtpy. Hardcoding it as a failsafe. |
| 25 | + |
| 26 | +import re |
| 27 | +import requests |
| 28 | +from adabot import github_requests as github |
| 29 | +from adabot import pypi_requests as pypi |
| 30 | + |
| 31 | +core_repo_url = "/repos/adafruit/circuitpython" |
| 32 | + |
| 33 | +def parse_gitmodules(input_text): |
| 34 | + """Parse a .gitmodules file and return a list of all the git submodules |
| 35 | + defined inside of it. Each list item is 2-tuple with: |
| 36 | + - submodule name (string) |
| 37 | + - submodule variables (dictionary with variables as keys and their values) |
| 38 | + The input should be a string of text with the complete representation of |
| 39 | + the .gitmodules file. |
| 40 | +
|
| 41 | + See this for the format of the .gitmodules file, it follows the git config |
| 42 | + file format: |
| 43 | + https://www.kernel.org/pub/software/scm/git/docs/git-config.html |
| 44 | +
|
| 45 | + Note although the format appears to be like a ConfigParser-readable ini file |
| 46 | + it is NOT possible to parse with Python's built-in ConfigParser module. The |
| 47 | + use of tabs in the git config format breaks ConfigParser, and the subsection |
| 48 | + values in double quotes are completely lost. A very basic regular |
| 49 | + expression-based parsing logic is used here to parse the data. This parsing |
| 50 | + is far from perfect and does not handle escaping quotes, line continuations |
| 51 | + (when a line ends in '\;'), etc. Unfortunately the git config format is |
| 52 | + surprisingly complex and no mature parsing modules are available (outside |
| 53 | + the code in git itself). |
| 54 | + """ |
| 55 | + # Assume no results if invalid input. |
| 56 | + if input_text is None: |
| 57 | + return [] |
| 58 | + # Define a regular expression to match a basic submodule section line and |
| 59 | + # capture its subsection value. |
| 60 | + submodule_section_re = '^\[submodule "(.+)"\]$' |
| 61 | + # Define a regular expression to match a variable setting line and capture |
| 62 | + # the variable name and value. This does NOT handle multi-line or quote |
| 63 | + # escaping (far outside the abilities of a regular expression). |
| 64 | + variable_re = '^\s*([a-zA-Z0-9\-]+) =\s+(.+?)\s*$' |
| 65 | + # Process all the lines to parsing submodule sections and the variables |
| 66 | + # within them as they're found. |
| 67 | + results = [] |
| 68 | + submodule_name = None |
| 69 | + submodule_variables = {} |
| 70 | + for line in input_text.splitlines(): |
| 71 | + submodule_section_match = re.match(submodule_section_re, line, flags=re.IGNORECASE) |
| 72 | + variable_match = re.match(variable_re, line) |
| 73 | + if submodule_section_match: |
| 74 | + # Found a new section. End the current one if it had data and add |
| 75 | + # it to the results, then start parsing a new section. |
| 76 | + if submodule_name is not None: |
| 77 | + results.append((submodule_name, submodule_variables)) |
| 78 | + submodule_name = submodule_section_match.group(1) |
| 79 | + submodule_variables = {} |
| 80 | + elif variable_match: |
| 81 | + # Found a variable, add it to the current section variables. |
| 82 | + # Force the variable name to lower case as variable names are |
| 83 | + # case-insensitive in git config sections and this makes later |
| 84 | + # processing easier (can assume lower-case names to find values). |
| 85 | + submodule_variables[variable_match.group(1).lower()] = variable_match.group(2) |
| 86 | + # Add the last parsed section if it exists. |
| 87 | + if submodule_name is not None: |
| 88 | + results.append((submodule_name, submodule_variables)) |
| 89 | + return results |
| 90 | + |
| 91 | +def get_bundle_submodules(): |
| 92 | + """Query Adafruit_CircuitPython_Bundle repository for all the submodules |
| 93 | + (i.e. modules included inside) and return a list of the found submodules. |
| 94 | + Each list item is a 2-tuple of submodule name and a dict of submodule |
| 95 | + variables including 'path' (location of submodule in bundle) and |
| 96 | + 'url' (URL to git repository with submodule contents). |
| 97 | + """ |
| 98 | + # Assume the bundle repository is public and get the .gitmodules file |
| 99 | + # without any authentication or Github API usage. Also assumes the |
| 100 | + # master branch of the bundle is the canonical source of the bundle release. |
| 101 | + result = requests.get('https://raw.githubusercontent.com/adafruit/Adafruit_CircuitPython_Bundle/master/.gitmodules', |
| 102 | + timeout=15) |
| 103 | + if result.status_code != 200: |
| 104 | + #output_handler("Failed to access bundle .gitmodules file from GitHub!", quiet=True) |
| 105 | + raise RuntimeError('Failed to access bundle .gitmodules file from GitHub!') |
| 106 | + return parse_gitmodules(result.text) |
| 107 | + |
| 108 | +def sanitize_url(url): |
| 109 | + """Convert a Github repository URL into a format which can be compared for |
| 110 | + equality with simple string comparison. Will strip out any leading URL |
| 111 | + scheme, set consistent casing, and remove any optional .git suffix. The |
| 112 | + attempt is to turn a URL from Github (which can be one of many different |
| 113 | + schemes with and without suffxes) into canonical values for easy comparison. |
| 114 | + """ |
| 115 | + # Make the url lower case to perform case-insensitive comparisons. |
| 116 | + # This might not actually be correct if Github cares about case (assumption |
| 117 | + # is no Github does not, but this is unverified). |
| 118 | + url = url.lower() |
| 119 | + # Strip out any preceding http://, https:// or git:// from the URL to |
| 120 | + # make URL comparisons safe (probably better to explicitly parse using |
| 121 | + # a URL module in the future). |
| 122 | + scheme_end = url.find('://') |
| 123 | + if scheme_end >= 0: |
| 124 | + url = url[scheme_end:] |
| 125 | + # Strip out any .git suffix if it exists. |
| 126 | + if url.endswith('.git'): |
| 127 | + url = url[:-4] |
| 128 | + return url |
| 129 | + |
| 130 | +def is_repo_in_bundle(repo_clone_url, bundle_submodules): |
| 131 | + """Return a boolean indicating if the specified repository (the clone URL |
| 132 | + as a string) is in the bundle. Specify bundle_submodules as a dictionary |
| 133 | + of bundle submodule state returned by get_bundle_submodules. |
| 134 | + """ |
| 135 | + # Sanitize url for easy comparison. |
| 136 | + repo_clone_url = sanitize_url(repo_clone_url) |
| 137 | + # Search all the bundle submodules for any that have a URL which matches |
| 138 | + # this clone URL. Not the most efficient search but it's a handful of |
| 139 | + # items in the bundle. |
| 140 | + for submodule in bundle_submodules: |
| 141 | + name, variables = submodule |
| 142 | + submodule_url = variables.get('url', '') |
| 143 | + # Compare URLs and skip to the next submodule if it's not a match. |
| 144 | + # Right now this is a case sensitive compare, but perhaps it should |
| 145 | + # be insensitive in the future (unsure if Github repos are sensitive). |
| 146 | + if repo_clone_url != sanitize_url(submodule_url): |
| 147 | + continue |
| 148 | + # URLs matched so now check if the submodule is placed in the libraries |
| 149 | + # subfolder of the bundle. Just look at the path from the submodule |
| 150 | + # state. |
| 151 | + if variables.get('path', '').startswith('libraries/'): |
| 152 | + # Success! Found the repo as a submodule of the libraries folder |
| 153 | + # in the bundle. |
| 154 | + return True |
| 155 | + # Failed to find the repo as a submodule of the libraries folders. |
| 156 | + return False |
| 157 | + |
| 158 | +def list_repos(): |
| 159 | + """Return a list of all Adafruit repositories that start with |
| 160 | + Adafruit_CircuitPython. Each list item is a dictionary of GitHub API |
| 161 | + repository state. |
| 162 | + """ |
| 163 | + repos = [] |
| 164 | + result = github.get("/search/repositories", |
| 165 | + params={"q":"Adafruit_CircuitPython user:adafruit", |
| 166 | + "per_page": 100, |
| 167 | + "sort": "updated", |
| 168 | + "order": "asc"} |
| 169 | + ) |
| 170 | + while result.ok: |
| 171 | + links = result.headers["Link"] |
| 172 | + #repos.extend(result.json()["items"]) # uncomment and comment below, to include all forks |
| 173 | + repos.extend(repo for repo in result.json()["items"] if (repo["owner"]["login"] == "adafruit" and |
| 174 | + (repo["name"].startswith("Adafruit_CircuitPython") or repo["name"] == "circuitpython"))) |
| 175 | + |
| 176 | + next_url = None |
| 177 | + for link in links.split(","): |
| 178 | + link, rel = link.split(";") |
| 179 | + link = link.strip(" <>") |
| 180 | + rel = rel.strip() |
| 181 | + if rel == "rel=\"next\"": |
| 182 | + next_url = link |
| 183 | + break |
| 184 | + if not next_url: |
| 185 | + break |
| 186 | + # Subsequent links have our access token already so we use requests directly. |
| 187 | + result = requests.get(link, timeout=30) |
| 188 | + if "circuitpython" not in [repo["name"] for repo in repos]: |
| 189 | + core = github.get(core_repo_url) |
| 190 | + if core.ok: |
| 191 | + repos.append(core.json()) |
| 192 | + |
| 193 | + return repos |
| 194 | + |
| 195 | +def repo_is_on_pypi(repo): |
| 196 | + """returns True when the provided repository is in pypi""" |
| 197 | + is_on = False |
| 198 | + the_page = pypi.get("/pypi/"+repo["name"]+"/json") |
| 199 | + if the_page and the_page.status_code == 200: |
| 200 | + is_on = True |
| 201 | + |
| 202 | + return is_on |
0 commit comments