diff --git a/circuitpython_build_tools/build.py b/circuitpython_build_tools/build.py index 06bcfad..f884a1c 100644 --- a/circuitpython_build_tools/build.py +++ b/circuitpython_build_tools/build.py @@ -36,8 +36,48 @@ import subprocess import tempfile +# pyproject.toml `py_modules` values that are incorrect. These should all have PRs filed! +# and should be removed when the fixed version is incorporated in its respective bundle. + +pyproject_py_modules_blacklist = set(( + # adafruit bundle + "adafruit_colorsys", + + # community bundle + "at24mac_eeprom", + "circuitpython_Candlesticks", + "CircuitPython_Color_Picker", + "CircuitPython_Equalizer", + "CircuitPython_Scales", + "circuitPython_Slider", + "circuitpython_uboxplot", + "P1AM", + "p1am_200_helpers", +)) + +if sys.version_info >= (3, 11): + from tomllib import loads as load_toml +else: + from tomli import loads as load_toml + +def load_pyproject_toml(lib_path: pathlib.Path): + try: + return load_toml((lib_path / "pyproject.toml") .read_text(encoding="utf-8")) + except FileNotFoundError: + print(f"No pyproject.toml in {lib_path}") + return {} + +def get_nested(doc, *args, default=None): + for a in args: + if doc is None: return default + try: + doc = doc[a] + except (KeyError, IndexError) as e: + return default + return doc + IGNORE_PY = ["setup.py", "conf.py", "__init__.py"] -GLOB_PATTERNS = ["*.py", "font5x8.bin"] +GLOB_PATTERNS = ["*.py", "*.bin"] S3_MPY_PREFIX = "https://adafruit-circuit-python.s3.amazonaws.com/bin/mpy-cross" def version_string(path=None, *, valid_semver=False): @@ -131,17 +171,13 @@ def mpy_cross(mpy_cross_filename, circuitpython_tag, quiet=False): shutil.copy("build_deps/circuitpython/mpy-cross/mpy-cross", mpy_cross_filename) def _munge_to_temp(original_path, temp_file, library_version): - with open(original_path, "rb") as original_file: + with open(original_path, "r", encoding="utf-8") as original_file: for line in original_file: - if original_path.endswith(".bin"): - # this is solely for adafruit_framebuf/examples/font5x8.bin - temp_file.write(line) - else: - line = line.decode("utf-8").strip("\n") - if line.startswith("__version__"): - line = line.replace("0.0.0-auto.0", library_version) - line = line.replace("0.0.0+auto.0", library_version) - temp_file.write(line.encode("utf-8") + b"\r\n") + line = line.strip("\n") + if line.startswith("__version__"): + line = line.replace("0.0.0-auto.0", library_version) + line = line.replace("0.0.0+auto.0", library_version) + print(line, file=temp_file) temp_file.flush() def get_package_info(library_path, package_folder_prefix): @@ -154,61 +190,52 @@ def get_package_info(library_path, package_folder_prefix): for pattern in GLOB_PATTERNS: glob_search.extend(list(lib_path.rglob(pattern))) - package_info["is_package"] = False - for file in glob_search: - if file.parts[parent_idx] != "examples": - if len(file.parts) > parent_idx + 1: - for prefix in package_folder_prefix: - if file.parts[parent_idx].startswith(prefix): - package_info["is_package"] = True - if package_info["is_package"]: - package_files.append(file) - else: - if file.name in IGNORE_PY: - #print("Ignoring:", file.resolve()) - continue - if file.parent == lib_path: - py_files.append(file) - - if package_files: - package_info["module_name"] = package_files[0].relative_to(library_path).parent.name - elif py_files: - package_info["module_name"] = py_files[0].relative_to(library_path).name[:-3] - else: - package_info["module_name"] = None - - try: - package_info["version"] = version_string(library_path, valid_semver=True) - except ValueError as e: - package_info["version"] = version_string(library_path) - - return package_info - -def library(library_path, output_directory, package_folder_prefix, - mpy_cross=None, example_bundle=False): - py_files = [] - package_files = [] - example_files = [] - total_size = 512 - - lib_path = pathlib.Path(library_path) - parent_idx = len(lib_path.parts) - glob_search = [] - for pattern in GLOB_PATTERNS: - glob_search.extend(list(lib_path.rglob(pattern))) + pyproject_toml = load_pyproject_toml(lib_path) + py_modules = get_nested(pyproject_toml, "tool", "setuptools", "py-modules", default=[]) + packages = get_nested(pyproject_toml, "tool", "setuptools", "packages", default=[]) + + blacklisted = [name for name in py_modules if name in pyproject_py_modules_blacklist] + + if blacklisted: + print(f"{lib_path}/settings.toml:1: {blacklisted[0]} blacklisted: not using metadata from pyproject.toml") + py_modules = packages = () + + example_files = [sub_path for sub_path in (lib_path / "examples").rglob("*") + if sub_path.is_file()] + + if packages and py_modules: + raise ValueError("Cannot specify both tool.setuptools.py-modules and .packages") + + elif packages: + if len(packages) > 1: + raise ValueError("Only a single package is supported") + package_name = packages[0] + #print(f"Using package name from pyproject.toml: {package_name}") + package_info["is_package"] = True + package_info["module_name"] = package_name + package_files = [sub_path for sub_path in (lib_path / package_name).rglob("*") + if sub_path.is_file()] + + elif py_modules: + if len(py_modules) > 1: + raise ValueError("Only a single module is supported") + py_module = py_modules[0] + #print(f"Using module name from pyproject.toml: {py_module}") + package_name = py_module + package_info["is_package"] = False + package_info["module_name"] = py_module + py_files = [lib_path / f"{py_module}.py"] - for file in glob_search: - if file.parts[parent_idx] == "examples": - example_files.append(file) - else: - if not example_bundle: - is_package = False + else: + print(f"{lib_path}: Using legacy autodetection") + package_info["is_package"] = False + for file in glob_search: + if file.parts[parent_idx] != "examples": if len(file.parts) > parent_idx + 1: for prefix in package_folder_prefix: if file.parts[parent_idx].startswith(prefix): - is_package = True - - if is_package: + package_info["is_package"] = True + if package_info["is_package"]: package_files.append(file) else: if file.name in IGNORE_PY: @@ -217,91 +244,78 @@ def library(library_path, output_directory, package_folder_prefix, if file.parent == lib_path: py_files.append(file) + if package_files: + package_info["module_name"] = package_files[0].relative_to(library_path).parent.name + elif py_files: + package_info["module_name"] = py_files[0].relative_to(library_path).name[:-3] + else: + package_info["module_name"] = None + if len(py_files) > 1: raise ValueError("Multiple top level py files not allowed. Please put " "them in a package or combine them into a single file.") - if package_files: - module_name = package_files[0].relative_to(library_path).parent.name - elif py_files: - module_name = py_files[0].relative_to(library_path).name[:-3] - else: - module_name = None + package_info["package_files"] = package_files + package_info["py_files"] = py_files + package_info["example_files"] = example_files + + try: + package_info["version"] = version_string(library_path, valid_semver=True) + except ValueError as e: + print(library_path + " has version that doesn't follow SemVer (semver.org)") + print(e) + package_info["version"] = version_string(library_path) + + return package_info + +def library(library_path, output_directory, package_folder_prefix, + mpy_cross=None, example_bundle=False): + lib_path = pathlib.Path(library_path) + package_info = get_package_info(library_path, package_folder_prefix) + py_package_files = package_info["package_files"] + package_info["py_files"] + example_files = package_info["example_files"] + module_name = package_info["module_name"] for fn in example_files: base_dir = os.path.join(output_directory.replace("/lib", "/"), fn.relative_to(library_path).parent) if not os.path.isdir(base_dir): os.makedirs(base_dir) - total_size += 512 - for fn in package_files: + for fn in py_package_files: base_dir = os.path.join(output_directory, fn.relative_to(library_path).parent) if not os.path.isdir(base_dir): os.makedirs(base_dir) - total_size += 512 - new_extension = ".py" - if mpy_cross: - new_extension = ".mpy" + library_version = package_info['version'] - try: - library_version = version_string(library_path, valid_semver=True) - except ValueError as e: - print(library_path + " has version that doesn't follow SemVer (semver.org)") - print(e) - library_version = version_string(library_path) - - for filename in py_files: - full_path = os.path.join(library_path, filename) - output_file = os.path.join( - output_directory, - filename.relative_to(library_path).with_suffix(new_extension) - ) - with tempfile.NamedTemporaryFile(delete=False) as temp_file: - _munge_to_temp(full_path, temp_file, library_version) - temp_filename = temp_file.name - # Windows: close the temp file before it can be read or copied by name - if mpy_cross: - mpy_success = subprocess.call([ - mpy_cross, - "-o", output_file, - "-s", str(filename.relative_to(library_path)), - temp_filename - ]) - if mpy_success != 0: - raise RuntimeError("mpy-cross failed on", full_path) - else: - shutil.copyfile(temp_filename, output_file) - os.remove(temp_filename) - - for filename in package_files: - full_path = os.path.join(library_path, filename) - output_file = "" - with tempfile.NamedTemporaryFile(delete=False) as temp_file: - _munge_to_temp(full_path, temp_file, library_version) - temp_filename = temp_file.name - # Windows: close the temp file before it can be read or copied by name - if not mpy_cross or os.stat(full_path).st_size == 0: - output_file = os.path.join(output_directory, - filename.relative_to(library_path)) - shutil.copyfile(temp_filename, output_file) - else: - output_file = os.path.join( - output_directory, - filename.relative_to(library_path).with_suffix(new_extension) - ) - - mpy_success = subprocess.call([ - mpy_cross, - "-o", output_file, - "-s", str(filename.relative_to(library_path)), - temp_filename - ]) - if mpy_success != 0: - raise RuntimeError("mpy-cross failed on", full_path) - os.remove(temp_filename) + if not example_bundle: + for filename in py_package_files: + full_path = os.path.join(library_path, filename) + output_file = output_directory / filename.relative_to(library_path) + if filename.suffix == ".py": + with tempfile.NamedTemporaryFile(delete=False, mode="w+") as temp_file: + temp_file_name = temp_file.name + try: + _munge_to_temp(full_path, temp_file, library_version) + temp_file.close() + if mpy_cross and os.stat(temp_file.name).st_size != 0: + output_file = output_file.with_suffix(".mpy") + mpy_success = subprocess.call([ + mpy_cross, + "-o", output_file, + "-s", str(filename.relative_to(library_path)), + temp_file.name + ]) + if mpy_success != 0: + raise RuntimeError("mpy-cross failed on", full_path) + else: + shutil.copyfile(full_path, output_file) + finally: + os.remove(temp_file_name) + else: + shutil.copyfile(full_path, output_file) requirements_files = lib_path.glob("requirements.txt*") requirements_files = [f for f in requirements_files if f.stat().st_size > 0] @@ -314,11 +328,9 @@ def library(library_path, output_directory, package_folder_prefix, requirements_dir = pathlib.Path(output_directory).parent / "requirements" if not os.path.isdir(requirements_dir): os.makedirs(requirements_dir, exist_ok=True) - total_size += 512 requirements_subdir = f"{requirements_dir}/{module_name}" if not os.path.isdir(requirements_subdir): os.makedirs(requirements_subdir, exist_ok=True) - total_size += 512 for filename in requirements_files: full_path = os.path.join(library_path, filename) output_file = os.path.join(requirements_subdir, filename.name) @@ -328,9 +340,4 @@ def library(library_path, output_directory, package_folder_prefix, full_path = os.path.join(library_path, filename) output_file = os.path.join(output_directory.replace("/lib", "/"), filename.relative_to(library_path)) - temp_filename = "" - with tempfile.NamedTemporaryFile(delete=False) as temp_file: - _munge_to_temp(full_path, temp_file, library_version) - temp_filename = temp_file.name - shutil.copyfile(temp_filename, output_file) - os.remove(temp_filename) + shutil.copyfile(full_path, output_file) diff --git a/requirements.txt b/requirements.txt index 861b8da..8a3514c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,3 +2,4 @@ Click requests semver wheel +tomli; python_version < "3.11"