Add support for installing pyodide Pythons

WIP

Author: hoodmane

crates/uv-python/download-metadata.json

@@ -8847,6 +8847,22 @@
     "sha256": "e5a904ecfb4061389773dd655d3b5665447c80cbf2948fcb1c07e92716eed955",
     "variant": null
   },
+  "cpython-3.13.2-emscripten-wasm32-musl": {
+    "name": "cpython",
+    "arch": {
+      "family": "wasm32",
+      "variant": null
+    },
+    "os": "emscripten",
+    "libc": "musl",
+    "major": 3,
+    "minor": 13,
+    "patch": 2,
+    "prerelease": "",
+    "url": "https://github.com/pyodide/pyodide/releases/download/0.28.0/xbuildenv-0.28.0.tar.bz2",
+    "sha256": null,
+    "variant": null
+  },
   "cpython-3.13.2-linux-aarch64-gnu": {
     "name": "cpython",
     "arch": {
@@ -13391,6 +13407,22 @@
     "sha256": "848405b92bda20fad1f9bba99234c7d3f11e0b31e46f89835d1cb3d735e932aa",
     "variant": null
   },
+  "cpython-3.12.7-emscripten-wasm32-musl": {
+    "name": "cpython",
+    "arch": {
+      "family": "wasm32",
+      "variant": null
+    },
+    "os": "emscripten",
+    "libc": "musl",
+    "major": 3,
+    "minor": 12,
+    "patch": 7,
+    "prerelease": "",
+    "url": "https://github.com/pyodide/pyodide/releases/download/0.27.7/xbuildenv-0.27.7.tar.bz2",
+    "sha256": null,
+    "variant": null
+  },
   "cpython-3.12.7-linux-aarch64-gnu": {
     "name": "cpython",
     "arch": {
@@ -15439,6 +15471,22 @@
     "sha256": "eca96158c1568dedd9a0b3425375637a83764d1fa74446438293089a8bfac1f8",
     "variant": null
   },
+  "cpython-3.12.1-emscripten-wasm32-musl": {
+    "name": "cpython",
+    "arch": {
+      "family": "wasm32",
+      "variant": null
+    },
+    "os": "emscripten",
+    "libc": "musl",
+    "major": 3,
+    "minor": 12,
+    "patch": 1,
+    "prerelease": "",
+    "url": "https://github.com/pyodide/pyodide/releases/download/0.26.4/xbuildenv-0.26.4.tar.bz2",
+    "sha256": null,
+    "variant": null
+  },
   "cpython-3.12.1-linux-aarch64-gnu": {
     "name": "cpython",
     "arch": {
@@ -19775,6 +19823,22 @@
     "sha256": "f710b8d60621308149c100d5175fec39274ed0b9c99645484fd93d1716ef4310",
     "variant": null
   },
+  "cpython-3.11.3-emscripten-wasm32-musl": {
+    "name": "cpython",
+    "arch": {
+      "family": "wasm32",
+      "variant": null
+    },
+    "os": "emscripten",
+    "libc": "musl",
+    "major": 3,
+    "minor": 11,
+    "patch": 3,
+    "prerelease": "",
+    "url": "https://github.com/pyodide/pyodide/releases/download/0.25.1/xbuildenv-0.25.1.tar.bz2",
+    "sha256": null,
+    "variant": null
+  },
   "cpython-3.11.3-linux-aarch64-gnu": {
     "name": "cpython",
     "arch": {

crates/uv-python/fetch-download-metadata.py

@@ -541,6 +541,96 @@ async def _fetch_checksums(self, downloads: list[PythonDownload]) -> None:
             download.sha256 = checksums.get(download.filename)
 
 
+class PyodideFinder(Finder):
+    implementation = ImplementationName.CPYTHON
+
+    RELEASE_URL = "https://api.github.com/repos/pyodide/pyodide/releases"
+    METADATA_URL = (
+        "https://pyodide.github.io/pyodide/api/pyodide-cross-build-environments.json"
+    )
+
+    TRIPLE = PlatformTriple(
+        platform="emscripten",
+        arch=Arch("wasm32"),
+        libc="musl",
+    )
+
+    def __init__(self, client: httpx.AsyncClient):
+        self.client = client
+
+    async def find(self) -> list[PythonDownload]:
+        downloads = await self._fetch_downloads()
+        await self._fetch_checksums(downloads, n=10)
+        return downloads
+
+    async def _fetch_downloads(self) -> list[PythonDownload]:
+        # This will only download the first page, i.e., ~30 releases
+        [release_resp, meta_resp] = await asyncio.gather(
+            self.client.get(self.RELEASE_URL), self.client.get(self.METADATA_URL)
+        )
+        release_resp.raise_for_status()
+        meta_resp.raise_for_status()
+        releases = release_resp.json()
+        metadata = meta_resp.json()["releases"]
+
+        maj_minor_seen = set()
+        results = []
+        for release in releases:
+            pyodide_version = release["tag_name"]
+            meta = metadata.get(pyodide_version, None)
+            if meta is None:
+                continue
+
+            maj_min = pyodide_version.rpartition(".")[0]
+            # Only keep latest
+            if maj_min in maj_minor_seen:
+                continue
+            maj_minor_seen.add(maj_min)
+
+            python_version = Version.from_str(meta["python_version"])
+            # Find xbuildenv asset
+            for asset in release["assets"]:
+                if asset["name"].startswith("xbuildenv"):
+                    break
+
+            url = asset["browser_download_url"]
+            results.append(
+                PythonDownload(
+                    release=0,
+                    version=python_version,
+                    triple=self.TRIPLE,
+                    flavor=pyodide_version,
+                    implementation=self.implementation,
+                    filename=asset["name"],
+                    url=url,
+                )
+            )
+
+        return results
+
+    def _normalize_arch(self, arch: str) -> Arch:
+        return Arch(self.ARCH_MAPPING.get(arch, arch), None)
+
+    def _normalize_os(self, os: str) -> str:
+        return self.PLATFORM_MAPPING.get(os, os)
+
+    async def _fetch_checksums(self, downloads: list[PythonDownload], n: int) -> None:
+        for idx, batch in enumerate(batched(downloads, n)):
+            logging.info("Fetching Pyodide checksums: %d/%d", idx * n, len(downloads))
+            checksum_requests = []
+            for download in batch:
+                url = download.url + ".sha256"
+                checksum_requests.append(self.client.get(url))
+            for download, resp in zip(batch, await asyncio.gather(*checksum_requests), strict=False):
+                try:
+                    resp.raise_for_status()
+                except httpx.HTTPStatusError as e:
+                    if e.response.status_code == 404:
+                        continue
+                    raise
+                download.sha256 = resp.text.strip()
+
+
 class GraalPyFinder(Finder):
     implementation = ImplementationName.GRAALPY
 
@@ -670,6 +760,7 @@ def sort_key(download: PythonDownload) -> tuple:
             ImplementationName.CPYTHON,
             ImplementationName.PYPY,
             ImplementationName.GRAALPY,
+            ImplementationName.PYODIDE,
         ]
         prerelease = prerelease_sort_key(download.version.prerelease)
         return (
@@ -737,6 +828,7 @@ async def find() -> None:
         CPythonFinder(client),
         PyPyFinder(client),
         GraalPyFinder(client),
+        PyodideFinder(client),
     ]
     downloads = []
 

crates/uv-python/src/downloads.rs

@@ -379,21 +379,31 @@ impl PythonDownloadRequest {
 
     /// Whether this request is satisfied by an installation key.
     pub fn satisfied_by_key(&self, key: &PythonInstallationKey) -> bool {
-        if let Some(os) = &self.os {
-            if key.os != *os {
-                return false;
-            }
+        let key_is_emscripten = *key.os() == Os(target_lexicon::OperatingSystem::Emscripten);
+        let target_is_windows = self.os == Some(Os(target_lexicon::OperatingSystem::Windows));
+        // Emscripten is not compatible with windows
+        if key_is_emscripten && target_is_windows {
+            return false;
         }
+        // Emscripten is compatible with all other platforms so skip platform
+        // check in this case.
+        if !key_is_emscripten {
+            if let Some(os) = &self.os {
+                if key.os != *os {
+                    return false;
+                }
+            }
 
-        if let Some(arch) = &self.arch {
-            if !arch.satisfied_by(key.arch) {
-                return false;
+            if let Some(arch) = &self.arch {
+                if !arch.satisfied_by(key.arch) {
+                    return false;
+                }
             }
-        }
 
-        if let Some(libc) = &self.libc {
-            if key.libc != *libc {
-                return false;
+            if let Some(libc) = &self.libc {
+                if key.libc != *libc {
+                    return false;
+                }
             }
         }
         if let Some(implementation) = &self.implementation {
@@ -453,24 +463,6 @@ impl PythonDownloadRequest {
                 return false;
             }
         }
-        if let Some(os) = self.os() {
-            let interpreter_os = Os::from(interpreter.platform().os());
-            if &interpreter_os != os {
-                debug!(
-                    "Skipping interpreter at `{executable}`: operating system `{interpreter_os}` does not match request `{os}`"
-                );
-                return false;
-            }
-        }
-        if let Some(arch) = self.arch() {
-            let interpreter_arch = Arch::from(&interpreter.platform().arch());
-            if !arch.satisfied_by(interpreter_arch) {
-                debug!(
-                    "Skipping interpreter at `{executable}`: architecture `{interpreter_arch}` does not match request `{arch}`"
-                );
-                return false;
-            }
-        }
         if let Some(implementation) = self.implementation() {
             let interpreter_implementation = interpreter.implementation_name();
             if LenientImplementationName::from(interpreter_implementation)
@@ -482,13 +474,41 @@ impl PythonDownloadRequest {
                 return false;
             }
         }
-        if let Some(libc) = self.libc() {
-            let interpreter_libc = Libc::from(interpreter.platform().os());
-            if &interpreter_libc != libc {
-                debug!(
-                    "Skipping interpreter at `{executable}`: libc `{interpreter_libc}` does not match request `{libc}`"
-                );
-                return false;
+        let interpreter_os = Os::from(interpreter.platform().os());
+        let interp_is_emscripten =
+            interpreter_os == Os(target_lexicon::OperatingSystem::Emscripten);
+        let target_is_windows = self.os() == Some(&Os(target_lexicon::OperatingSystem::Windows));
+        // Emscripten does not work on windows
+        if interp_is_emscripten && target_is_windows {
+            return false;
+        }
+        // Emscripten works on all other platforms
+        if !interp_is_emscripten {
+            if let Some(os) = self.os() {
+                if &interpreter_os != os {
+                    debug!(
+                        "Skipping interpreter at `{executable}`: operating system `{interpreter_os}` does not match request `{os}`"
+                    );
+                    return false;
+                }
+            }
+            if let Some(arch) = self.arch() {
+                let interpreter_arch = Arch::from(&interpreter.platform().arch());
+                if !arch.satisfied_by(interpreter_arch) {
+                    debug!(
+                        "Skipping interpreter at `{executable}`: architecture `{interpreter_arch}` does not match request `{arch}`"
+                    );
+                    return false;
+                }
+            }
+            if let Some(libc) = self.libc() {
+                let interpreter_libc = Libc::from(interpreter.platform().os());
+                if &interpreter_libc != libc {
+                    debug!(
+                        "Skipping interpreter at `{executable}`: libc `{interpreter_libc}` does not match request `{libc}`"
+                    );
+                    return false;
+                }
             }
         }
         true
@@ -790,7 +810,7 @@ impl ManagedPythonDownload {
         reporter: Option<&dyn Reporter>,
     ) -> Result<DownloadResult, Error> {
         let url = self.download_url(python_install_mirror, pypy_install_mirror)?;
-        let path = installation_dir.join(self.key().to_string());
+        let mut path = installation_dir.join(self.key().to_string());
 
         // If it is not a reinstall and the dir already exists, return it.
         if !reinstall && path.is_dir() {
@@ -916,18 +936,30 @@ impl ManagedPythonDownload {
             extracted = extracted.join("install");
         }
 
-        // If the distribution is missing a `python`-to-`pythonX.Y` symlink, add it. PEP 394 permits
-        // it, and python-build-standalone releases after `20240726` include it, but releases prior
-        // to that date do not.
+        let is_emscripten: bool = *self.os() == Os(target_lexicon::OperatingSystem::Emscripten);
+
         #[cfg(unix)]
         {
-            match fs_err::os::unix::fs::symlink(
-                format!("python{}.{}", self.key.major, self.key.minor),
-                extracted.join("bin").join("python"),
-            ) {
-                Ok(()) => {}
-                Err(err) if err.kind() == io::ErrorKind::AlreadyExists => {}
-                Err(err) => return Err(err.into()),
+            let discard_already_exists_error = |res: io::Result<()>| match res {
+                Err(err) if err.kind() == io::ErrorKind::AlreadyExists => Ok(()),
+                _ => res,
+            };
+            if is_emscripten {
+                // Emscripten has a "python" file but no pythonX.Y file.
+                discard_already_exists_error(fs_err::os::unix::fs::symlink(
+                    "python",
+                    extracted
+                        .join("pyodide-root/dist")
+                        .join(format!("python{}.{}", self.key.major, self.key.minor)),
+                ))?;
+            } else {
+                // If the distribution is missing a `python`-to-`pythonX.Y` symlink, add it. PEP 394 permits
+                // it, and python-build-standalone releases after `20240726` include it, but releases prior
+                // to that date do not.
+                discard_already_exists_error(fs_err::os::unix::fs::symlink(
+                    format!("python{}.{}", self.key.major, self.key.minor),
+                    extracted.join("bin").join("python"),
+                ))?;
             }
         }
 
@@ -937,6 +969,16 @@ impl ManagedPythonDownload {
             fs_err::tokio::remove_dir_all(&path).await?;
         }
 
+        if is_emscripten {
+            fs_err::create_dir(&path)?;
+            extracted.push("pyodide-root/dist");
+            path.push("bin");
+            fs_err::copy(
+                extracted.join("python"),
+                extracted.join(format!("python{}.{}", self.key.major, self.key.minor)),
+            )?;
+        }
+
         // Persist it to the target.
         debug!("Moving {} to {}", extracted.display(), path.user_display());
         rename_with_retry(extracted, &path)
@@ -946,6 +988,9 @@ impl ManagedPythonDownload {
                 err,
             })?;
 
+        if is_emscripten {
+            path.pop();
+        }
         Ok(DownloadResult::Fetched(path))
     }