From d38f819c49cd6b6d9549f36371fc38169b0dda77 Mon Sep 17 00:00:00 2001 From: Jimmy Date: Sun, 14 Jun 2026 22:34:34 +1200 Subject: [PATCH] Add browser_fetch helpers and async HTTP demos for Pyodide Ship pyfetch-based fetch utilities in lib/, run asyncio scripts via nest-asyncio in the worker, and add sample demos for HTTPS in the browser. Co-authored-by: Cursor --- lib/browser_fetch.py | 38 +++++ .../bundled-demos/demo/aiohttp_fetch_demo.py | 32 +++++ .../bundled-demos/demo/async_fetch_demo.py | 27 ++++ src/static/bundled-lib/browser_fetch.py | 38 +++++ src/static/pyodide-worker.js | 131 ++++++++++++++++-- 5 files changed, 255 insertions(+), 11 deletions(-) create mode 100644 lib/browser_fetch.py create mode 100644 src/static/bundled-demos/demo/aiohttp_fetch_demo.py create mode 100644 src/static/bundled-demos/demo/async_fetch_demo.py create mode 100644 src/static/bundled-lib/browser_fetch.py diff --git a/lib/browser_fetch.py b/lib/browser_fetch.py new file mode 100644 index 0000000..7a3bf2e --- /dev/null +++ b/lib/browser_fetch.py @@ -0,0 +1,38 @@ +"""Browser-friendly async HTTP for the Pyodide worker. + +These helpers use ``pyodide.http.pyfetch``, which maps to the browser +``fetch`` API (no Python TLS stack). Prefer them for ``https://`` in +Pyodide: ``aiohttp`` in the worker often raises +``RuntimeError('SSL is not supported.')`` for HTTPS even though the wheel +exists, because user-level SSL is not wired the same as on CPython. + +The browser's normal rules apply: the page is served over HTTPS so +``http://`` URLs are blocked as mixed content, and the response host must +send permissive CORS headers (e.g. ``Access-Control-Allow-Origin``) or the +browser hides the body even if the request succeeded. +""" + +from __future__ import annotations + +from typing import Any + + +async def fetch_text(url: str) -> str: + from pyodide.http import pyfetch + + r = await pyfetch(url) + return await r.text() + + +async def fetch_bytes(url: str) -> bytes: + from pyodide.http import pyfetch + + r = await pyfetch(url) + return await r.bytes() + + +async def fetch_json(url: str, **kwargs: Any) -> Any: + from pyodide.http import pyfetch + + r = await pyfetch(url) + return await r.json(**kwargs) diff --git a/src/static/bundled-demos/demo/aiohttp_fetch_demo.py b/src/static/bundled-demos/demo/aiohttp_fetch_demo.py new file mode 100644 index 0000000..afac97d --- /dev/null +++ b/src/static/bundled-demos/demo/aiohttp_fetch_demo.py @@ -0,0 +1,32 @@ +"""Async HTTPS GET of Pyodide’s lock file (browser-safe). + +Older Pyodide + ``aiohttp`` examples used ``ClientSession`` here, but the +Pyodide ``aiohttp`` wheel often raises ``RuntimeError('SSL is not supported.')`` +for ``https://`` — there is no real Python TLS stack in the worker; traffic must +go through the browser’s ``fetch``. + +This demo uses the shared ``browser_fetch`` helpers (``pyodide.http.pyfetch`` +under the hood), same idea as ``async_fetch_demo.py``. + +CORS still applies — jsDelivr allows cross-origin GETs. ``print(..., flush=True)`` +helps batched worker stdout appear before the run finishes. +""" + +import asyncio + +from browser_fetch import fetch_json + + +async def main(): + url = "https://cdn.jsdelivr.net/pyodide/v0.26.4/full/pyodide-lock.json" + print("Fetching", url, "via browser_fetch.fetch_json ...", flush=True) + data = await fetch_json(url) + info = data.get("info") or {} + packages = data.get("packages") or {} + print("pyodide-lock version:", info.get("version"), flush=True) + print("python runtime :", info.get("python"), flush=True) + print("indexed packages :", len(packages), flush=True) + print("aiohttp wheel in lock:", "aiohttp" in packages, flush=True) + + +asyncio.run(main()) diff --git a/src/static/bundled-demos/demo/async_fetch_demo.py b/src/static/bundled-demos/demo/async_fetch_demo.py new file mode 100644 index 0000000..d43b469 --- /dev/null +++ b/src/static/bundled-demos/demo/async_fetch_demo.py @@ -0,0 +1,27 @@ +"""Async HTTP in the browser (Pyodide). + +``aiohttp`` and similar clients use OS sockets, which Wasm does not provide, so +a ``session.get(...)`` can hang after ``print("Running")`` with no body. + +Use ``pyodide.http.pyfetch`` or the shared ``browser_fetch`` helpers. From an +HTTPS editor page, use ``https://`` URLs (mixed content blocks ``http://``). + +Many sites do not send CORS headers, so the browser blocks the response even +when the URL is valid. This demo uses jsDelivr JSON that allows cross-origin +GET (same host Pyodide loads from). +""" + +import asyncio + +from browser_fetch import fetch_json + + +async def main(): + print("Running") + url = "https://cdn.jsdelivr.net/pyodide/v0.26.4/full/repodata.json" + data = await fetch_json(url) + info = data.get("info") or {} + print("Fetched repodata; pyodide lock says:", info.get("version"), info.get("python")) + + +asyncio.run(main()) diff --git a/src/static/bundled-lib/browser_fetch.py b/src/static/bundled-lib/browser_fetch.py new file mode 100644 index 0000000..7a3bf2e --- /dev/null +++ b/src/static/bundled-lib/browser_fetch.py @@ -0,0 +1,38 @@ +"""Browser-friendly async HTTP for the Pyodide worker. + +These helpers use ``pyodide.http.pyfetch``, which maps to the browser +``fetch`` API (no Python TLS stack). Prefer them for ``https://`` in +Pyodide: ``aiohttp`` in the worker often raises +``RuntimeError('SSL is not supported.')`` for HTTPS even though the wheel +exists, because user-level SSL is not wired the same as on CPython. + +The browser's normal rules apply: the page is served over HTTPS so +``http://`` URLs are blocked as mixed content, and the response host must +send permissive CORS headers (e.g. ``Access-Control-Allow-Origin``) or the +browser hides the body even if the request succeeded. +""" + +from __future__ import annotations + +from typing import Any + + +async def fetch_text(url: str) -> str: + from pyodide.http import pyfetch + + r = await pyfetch(url) + return await r.text() + + +async def fetch_bytes(url: str) -> bytes: + from pyodide.http import pyfetch + + r = await pyfetch(url) + return await r.bytes() + + +async def fetch_json(url: str, **kwargs: Any) -> Any: + from pyodide.http import pyfetch + + r = await pyfetch(url) + return await r.json(**kwargs) diff --git a/src/static/pyodide-worker.js b/src/static/pyodide-worker.js index 72962e6..23f6b11 100644 --- a/src/static/pyodide-worker.js +++ b/src/static/pyodide-worker.js @@ -6,6 +6,26 @@ const PYODIDE_INDEX_URL = 'https://cdn.jsdelivr.net/pyodide/v0.26.4/full/'; let pyodide = null; let loadingPromise = null; +/** + * Install PyPI wheels into this Pyodide interpreter via micropip. + * Pure-Python wheels (or Pyodide-built packages) work; manylinux + * binary wheels that are not built for Emscripten will fail. + */ +async function micropipInstallSpecs(p, specs) { + const list = specs + .map((s) => String(s || '').trim()) + .filter(Boolean); + if (!list.length) { + return; + } + p.globals.set('__micropip_specs', p.toPy(list)); + await p.runPythonAsync(` +import micropip +specs = list(__micropip_specs) +await micropip.install(specs) +`); +} + async function ensurePyodide() { if (pyodide) { return pyodide; @@ -20,9 +40,17 @@ async function ensurePyodide() { batched: (txt) => self.postMessage({ type: 'io', stream: 'stderr', text: txt }), }); await p.loadPackage('micropip'); + /* Optional wheels (numpy, scipy, pandas, scikit-learn, …) are not preloaded — + first worker init stays small. Install from user code with + `await micropip.install("…", index_urls=[…])` (same host as `loadPyodide`). + HTTPS: use `pyodide.http.pyfetch` / `browser_fetch`, not `aiohttp`. */ await p.runPythonAsync(` import micropip await micropip.install("jedi") +try: + await micropip.install("nest-asyncio") +except Exception: + pass `); return p; })(); @@ -137,13 +165,47 @@ self.onmessage = async (event) => { self.__serial_in_capacity = 0; } } - await ensurePyodide(); - self.postMessage({ id, type: 'init', ok: true }); + const p = await ensurePyodide(); + const rawExtra = + payload && Array.isArray(payload.persistedMicropipPackages) + ? payload.persistedMicropipPackages + : []; + const extra = rawExtra.map((s) => String(s || '').trim()).filter(Boolean); + let micropipRestoreError = null; + if (extra.length) { + try { + await micropipInstallSpecs(p, extra); + } catch (err) { + micropipRestoreError = err && err.message ? String(err.message) : String(err); + } + } + const initReply = { id, type: 'init', ok: true }; + if (micropipRestoreError) { + initReply.micropipRestoreError = micropipRestoreError; + } + self.postMessage(initReply); return; } const p = await ensurePyodide(); + if (type === 'micropipInstall') { + const raw = + payload && payload.specs != null + ? payload.specs + : payload && payload.spec != null + ? [payload.spec] + : []; + const specs = (Array.isArray(raw) ? raw : [raw]).map((s) => String(s || '').trim()).filter(Boolean); + if (!specs.length) { + self.postMessage({ id, type: 'micropipInstall', ok: false, error: 'No package names given' }); + return; + } + await micropipInstallSpecs(p, specs); + self.postMessage({ id, type: 'micropipInstall', ok: true }); + return; + } + if (type === 'complete') { const rel = String(payload.path || 'scratch.py').replace(/^\/+/, ''); const vpath = `/workspace/${rel}`; @@ -168,12 +230,12 @@ for rel_path, body in extra.items(): os.makedirs(os.path.dirname(__cm_path), exist_ok=True) with open(__cm_path, "w", encoding="utf-8") as fh: fh.write(__cm_code) -for entry in ("/workspace/code", "/workspace/lib", "/workspace"): +for entry in ("/workspace/code", "/workspace/demos", "/workspace/lib", "/workspace"): if entry not in sys.path: - sys.path.insert(0, entry) + sys.path.append(entry) proj = jedi.Project( "/workspace", - added_sys_path=["/workspace/code", "/workspace/lib", "/workspace"], + added_sys_path=["/workspace/code", "/workspace/demos", "/workspace/lib", "/workspace"], ) s = jedi.Script(code=__cm_code, path=__cm_path, project=proj) items = s.complete(line=__cm_line, column=__cm_col) @@ -206,12 +268,12 @@ for rel_path, body in extra.items(): os.makedirs(os.path.dirname(__diag_path), exist_ok=True) with open(__diag_path, "w", encoding="utf-8") as fh: fh.write(__diag_code) -for entry in ("/workspace/code", "/workspace/lib", "/workspace"): +for entry in ("/workspace/code", "/workspace/demos", "/workspace/lib", "/workspace"): if entry not in sys.path: - sys.path.insert(0, entry) + sys.path.append(entry) proj = jedi.Project( "/workspace", - added_sys_path=["/workspace/code", "/workspace/lib", "/workspace"], + added_sys_path=["/workspace/code", "/workspace/demos", "/workspace/lib", "/workspace"], ) s = jedi.Script(code=__diag_code, path=__diag_path, project=proj) errs = s.get_syntax_errors() @@ -243,15 +305,62 @@ for rel, body in files.items(): with open(full, "w", encoding="utf-8") as fh: fh.write(str(body)) -for entry in ("/workspace/code", "/workspace/lib", "/workspace"): +for entry in ("/workspace/code", "/workspace/demos", "/workspace/lib", "/workspace"): if entry not in sys.path: - sys.path.insert(0, entry) + sys.path.append(entry) os.chdir("/workspace") main = __run_main sys.argv = [main] + list(__run_args) -runpy.run_path(main, run_name="__main__") +# runpy.run_path is synchronous; user asyncio.run() needs an *active* loop (nest_asyncio). +# Await an async wrapper so get_running_loop() works during the script. +import asyncio +import asyncio.events +import asyncio.base_events +def _patch_set_debug(cls): + _orig = cls.set_debug + def set_debug(self, enabled): + try: + return _orig(self, enabled) + except NotImplementedError: + pass + cls.set_debug = set_debug +_patch_set_debug(asyncio.events.AbstractEventLoop) +_patch_set_debug(asyncio.base_events.BaseEventLoop) +async def __pyodide_run_user_script(): + loop = asyncio.get_running_loop() + try: + import nest_asyncio + nest_asyncio.apply(loop) + except Exception: + try: + import nest_asyncio + nest_asyncio.apply() + except Exception: + pass + + def _pyodide_asyncio_run(coro, *, debug=False): + task = asyncio.ensure_future(coro, loop=loop) + return loop.run_until_complete(task) + + asyncio.run = _pyodide_asyncio_run + import runpy + runpy.run_path(main, run_name="__main__") + +await __pyodide_run_user_script() `); + try { + await p.runPythonAsync(` +import sys +for _s in (sys.stdout, sys.stderr): + try: + _s.flush() + except Exception: + pass +`); + } catch (_e) { + // ignore + } self.postMessage({ id, type: 'run', ok: true }); return; }