Merge pull request #4352 from agriyakhetarpal/maintenance/pyodide-build-unvendoring

Handle Pyodide and pyodide-build unvendoring, and update Pyodide CI job

Author: Zac-HD

.github/workflows/main.yml

@@ -223,17 +223,17 @@ jobs:
     - name: Run tests
       run: TASK=${{ matrix.task }} ./build.sh
 
-  # See https://pyodide.org/en/stable/development/building-and-testing-packages.html#testing-packages-against-pyodide
+  # See https://pyodide.org/en/stable/usage/building-and-testing-packages.html
   # and https://github.com/numpy/numpy/blob/9a650391651c8486d8cb8b27b0e75aed5d36033e/.github/workflows/emscripten.yml
   test-pyodide:
     runs-on: ubuntu-latest
     env:
-      NODE_VERSION: 18
-      # Note that the versions below must be updated in sync; we've automated
-      # that with `update_pyodide_versions()` in our weekly cronjob.
-      PYODIDE_VERSION: 0.27.3
+      NODE_VERSION: 22
+      # Note that the versions below are updated by `update_pyodide_versions()` in our weekly cronjob.
+      # The versions of pyodide-build and the Pyodide runtime may differ.
+      PYODIDE_VERSION: 0.27.5
+      PYODIDE_BUILD_VERSION: 0.30.0
       PYTHON_VERSION: 3.12.7
-      EMSCRIPTEN_VERSION: 3.1.58
     steps:
       - uses: actions/checkout@v3
         with:
@@ -246,17 +246,10 @@ jobs:
         uses: actions/setup-node@b39b52d1213e96004bfcb1c61a8a6fa8ab84f3e8 # v4.0.1
         with:
           node-version: ${{ env.NODE_VERSION }}
-      - name: Set up Emscripten
-        uses: mymindstorm/setup-emsdk@6ab9eb1bda2574c4ddb79809fc9247783eaf9021 # v14
-        with:
-          version: ${{ env.EMSCRIPTEN_VERSION }}
-      - name: Build
+      - name: Install pyodide-build and Pyodide cross-build environment
         run: |
-          # TODO remove https://github.com/pyodide/pyodide/issues/5585
-          pip install -U wheel==0.45.1
-          pip install pyodide-build==$PYODIDE_VERSION
-          cd hypothesis-python/
-          CFLAGS=-g2 LDFLAGS=-g2 pyodide build
+          pip install pyodide-build==$PYODIDE_BUILD_VERSION
+          pyodide xbuildenv install $PYODIDE_VERSION
       - name: Set up Pyodide venv and install dependencies
         run: |
           pip install --upgrade setuptools pip wheel==0.45.1 build

AUTHORS.rst

@@ -14,6 +14,7 @@ their individual contributions.
 * `Adam Sven Johnson <https://www.github.com/pkqk>`_
 * `Afrida Tabassum <https://github.com/oxfordhalfblood>`_ ([email protected])
 * `Afonso Silva <https://github.com/ajcerejeira>`_ ([email protected])
+* `Agriya Khetarpal <https://github.com/agriyakhetarpal>`_
 * `Agustín Covarrubias <https://github.com/agucova>`_ ([email protected])
 * `Akash Suresh <https://www.github.com/akash-suresh>`_ ([email protected])
 * `Alex Gaynor <https://github.com/alex>`_

tooling/src/hypothesistooling/__main__.py

@@ -403,31 +403,65 @@ def update_django_versions():
 
 
 def update_pyodide_versions():
+
+    def version_tuple(v: str) -> tuple[int, int, int]:
+        return tuple(int(x) for x in v.split("."))  # type: ignore
+
     vers_re = r"(\d+\.\d+\.\d+)"
-    all_versions = re.findall(
+    all_pyodide_build_versions = re.findall(
         f"pyodide_build-{vers_re}-py3-none-any.whl",  # excludes pre-releases
         requests.get("https://pypi.org/simple/pyodide-build/").text,
     )
-    for pyodide_version in sorted(
+    pyodide_build_version = max(
         # Don't just pick the most recent version; find the highest stable version.
-        set(all_versions),
-        key=lambda version: tuple(int(x) for x in version.split(".")),
-        reverse=True,
-    ):
-        makefile_url = f"https://raw.githubusercontent.com/pyodide/pyodide/{pyodide_version}/Makefile.envs"
-        match = re.search(
-            rf"export PYVERSION \?= {vers_re}\nexport PYODIDE_EMSCRIPTEN_VERSION \?= {vers_re}\n",
-            requests.get(makefile_url).text,
+        set(all_pyodide_build_versions),
+        key=version_tuple,
+    )
+
+    cross_build_environments_url = "https://raw.githubusercontent.com/pyodide/pyodide/refs/heads/main/pyodide-cross-build-environments.json"
+    cross_build_environments_data = requests.get(cross_build_environments_url).json()
+
+    # Find the latest stable release for the Pyodide runtime/xbuildenv that is compatible
+    # with the pyodide-build version we found
+    stable_releases = [
+        release
+        for release in cross_build_environments_data["releases"].values()
+        if re.fullmatch(vers_re, release["version"])
+    ]
+
+    compatible_releases = []
+    for release in stable_releases:  # sufficiently large values
+        min_build_version = release.get("min_pyodide_build_version", "0.0.0")
+        max_build_version = release.get("max_pyodide_build_version", "999.999.999")
+
+        # Perform version comparisons to avoid getting an incompatible pyodide-build version
+        # with the Pyodide runtime
+        if (
+            version_tuple(min_build_version)
+            <= version_tuple(pyodide_build_version)
+            <= version_tuple(max_build_version)
+        ):
+            compatible_releases.append(release)
+
+    if not compatible_releases:
+        raise RuntimeError(
+            f"No compatible Pyodide release found for pyodide-build {pyodide_build_version}"
         )
-        if match is not None:
-            python_version, emscripten_version = match.groups()
-            break
+
+    pyodide_release = max(
+        compatible_releases,
+        key=lambda release: version_tuple(release["version"]),
+    )
+
+    pyodide_version = pyodide_release["version"]
+    python_version = pyodide_release["python_version"]
+
     ci_file = tools.ROOT / ".github/workflows/main.yml"
     config = ci_file.read_text(encoding="utf-8")
     for name, var in [
         ("PYODIDE", pyodide_version),
+        ("PYODIDE_BUILD", pyodide_build_version),
         ("PYTHON", python_version),
-        ("EMSCRIPTEN", emscripten_version),
     ]:
         config = re.sub(f"{name}_VERSION: {vers_re}", f"{name}_VERSION: {var}", config)
     ci_file.write_text(config, encoding="utf-8")