Support passing of index URLs to piplite (CLI, runtime configuration, and requirements files)

jupyterlite_pyodide_kernel/tests/test_piplite.py

@@ -135,6 +135,69 @@ def a_lite_config_file(request, an_empty_lite_dir):
     return an_empty_lite_dir / request.param
 
 
+@pytest.mark.parametrize(
+    "index_urls",
+    [
+        ["https://example.com/simple"],
+        ["https://example.com/simple", "https://pypi.org/simple"],
+    ],
+    ids=["single", "multiple"],
+)
+def test_validate_piplite_install_default_options_valid(
+    script_runner, a_lite_config_file, index_urls
+):
+    """valid index_urls (list of URLs) in pipliteInstallDefaultOptions passes check"""
+    lite_dir = a_lite_config_file.parent
+    output = lite_dir / "_output"
+
+    build = script_runner.run(["jupyter", "lite", "build"], cwd=str(lite_dir))
+    assert build.success
+    shutil.copy2(output / a_lite_config_file.name, a_lite_config_file)
+
+    whole_file = config_data = json.loads(a_lite_config_file.read_text(**UTF8))
+    if a_lite_config_file.name == JUPYTERLITE_IPYNB:
+        config_data = whole_file["metadata"][JUPYTERLITE_METADATA]
+
+    config_data[JUPYTER_CONFIG_DATA].setdefault(LITE_PLUGIN_SETTINGS, {}).setdefault(
+        PYODIDE_KERNEL_PLUGIN_ID, {}
+    )["pipliteInstallDefaultOptions"] = {"index_urls": index_urls}
+
+    a_lite_config_file.write_text(json.dumps(whole_file, **JSON_FMT), **UTF8)
+    rebuild = script_runner.run(["jupyter", "lite", "build"], cwd=str(lite_dir))
+    assert rebuild.success
+
+    check = script_runner.run(["jupyter", "lite", "check"], cwd=str(lite_dir))
+    assert check.success, f"check failed for index_urls={index_urls!r}"
+
+
+def test_validate_piplite_install_default_options_invalid(
+    script_runner, a_lite_config_file
+):
+    """a non-string, non-list index_urls in pipliteInstallDefaultOptions fails check"""
+    lite_dir = a_lite_config_file.parent
+    output = lite_dir / "_output"
+
+    build = script_runner.run(["jupyter", "lite", "build"], cwd=str(lite_dir))
+    assert build.success
+    shutil.copy2(output / a_lite_config_file.name, a_lite_config_file)
+
+    whole_file = config_data = json.loads(a_lite_config_file.read_text(**UTF8))
+    if a_lite_config_file.name == JUPYTERLITE_IPYNB:
+        config_data = whole_file["metadata"][JUPYTERLITE_METADATA]
+
+    config_data[JUPYTER_CONFIG_DATA].setdefault(LITE_PLUGIN_SETTINGS, {}).setdefault(
+        PYODIDE_KERNEL_PLUGIN_ID, {}
+    )["pipliteInstallDefaultOptions"] = {"index_urls": 42}
+
+    invalid_config = json.dumps(whole_file, **JSON_FMT)
+    a_lite_config_file.write_text(invalid_config, **UTF8)
+    rebuild = script_runner.run(["jupyter", "lite", "build"], cwd=str(lite_dir))
+    assert rebuild.success
+
+    recheck = script_runner.run(["jupyter", "lite", "check"], cwd=str(lite_dir))
+    assert not recheck.success, invalid_config
+
+
 def test_validate_config(script_runner, a_lite_config_file):
     lite_dir = a_lite_config_file.parent
     output = lite_dir / "_output"

packages/pyodide-kernel-extension/schema/kernel.v0.schema.json

@@ -25,6 +25,30 @@
       "default": [],
       "format": "uri"
     },
+    "pipliteIndexUrls": {
+      "description": "URL(s) of package indices to use as a site-wide default, passed as `index_urls` to micropip.install. Relative paths are resolved against the base URL of the JupyterLite deployment. Preferred over `pipliteInstallDefaultOptions.index_urls`.",
+      "type": "array",
+      "items": {
+        "type": "string",
+        "format": "uri-reference"
+      },
+      "default": []
+    },
+    "pipliteInstallDefaultOptions": {
+      "type": "object",
+      "description": "Additional default options to pass to piplite.install (forwarded as kwargs to micropip.install). The `index_urls` key is supported for backwards compatibility; prefer `pipliteIndexUrls` instead.",
+      "default": {},
+      "properties": {
+        "index_urls": {
+          "description": "URL(s) of package indices to use, passed as `index_urls` to micropip.install. Prefer the top-level `pipliteIndexUrls` key instead.",
+          "type": "array",
+          "items": {
+            "type": "string",
+            "format": "uri-reference"
+          }
+        }
+      }
+    },
     "loadPyodideOptions": {
       "type": "object",
       "description": "additional options to provide to `loadPyodide`, see https://pyodide.org/en/stable/usage/api/js-api.html#globalThis.loadPyodide",

packages/pyodide-kernel-extension/src/index.ts

@@ -59,8 +59,23 @@ const kernel: JupyterFrontEndPlugin<void> = {
       : undefined;
     const rawPipUrls = config.pipliteUrls || [];
     const pipliteUrls = rawPipUrls.map((pipUrl: string) => URLExt.parse(pipUrl).href);
+    // pipliteIndexUrls: config-utils.js in JupyterLite resolves relative paths automatically
+    // for top-level keys ending in `Urls`, so no further resolution is needed here.
+    const pipliteIndexUrls = (config.pipliteIndexUrls as string[] | undefined) || [];
     const disablePyPIFallback = !!config.disablePyPIFallback;
     const loadPyodideOptions = config.loadPyodideOptions || {};
+    // pipliteInstallDefaultOptions;
+    // config-utils.js does not recurse into nested objects, so relative
+    // index_urls are resolved manually here.
+    const pipliteInstallDefaultOptions = {
+      ...(config.pipliteInstallDefaultOptions || {}),
+    };
+    if (Array.isArray(pipliteInstallDefaultOptions.index_urls)) {
+      pipliteInstallDefaultOptions.index_urls =
+        pipliteInstallDefaultOptions.index_urls.map(
+          (u: string) => new URL(u, baseUrl).href,
+        );
+    }
 
     for (const [key, value] of Object.entries(loadPyodideOptions)) {
       if (key.endsWith('URL') && typeof value === 'string') {
@@ -112,10 +127,12 @@ const kernel: JupyterFrontEndPlugin<void> = {
           pyodideUrl,
           pipliteWheelUrl,
           pipliteUrls,
+          pipliteIndexUrls,
           disablePyPIFallback,
           mountDrive,
           loadPyodideOptions,
           contentsManager,
+          pipliteInstallDefaultOptions,
           browsingContextId: serviceWorkerManager?.browsingContextId,
           logger,
         });

packages/pyodide-kernel/py/piplite/piplite/cli.py

@@ -13,7 +13,7 @@ async def install(
         deps: bool = True,                              # --no-deps
         credentials: str | None = None,                 # no CLI alias
         pre: bool = False,                              # --pre
-        index_urls: list[str] | str | None = None,      # no CLI alias
+        index_urls: list[str] | str | None = None,      # -i and --index-url
         *,
         constraints: list[str] | None = None,           # --constraints
         reinstall: bool = False,                        # no CLI alias
@@ -27,6 +27,7 @@ async def install(
 from __future__ import annotations
 
 import re
+import shlex
 import sys
 from typing import Any, TYPE_CHECKING
 from argparse import ArgumentParser
@@ -37,6 +38,31 @@ async def install(
 
 REQ_FILE_SPEC = r"^(?P<flag>-r|--requirements)\s*=?\s*(?P<path_ref>.+)$"
 
+
+def _parse_index_url_line(raw: str) -> str | None:
+    """Return the index URL from a ``--index-url``/``-i`` directive line.
+
+    Handles the ``--flag URL``, ``--flag=URL``, and all other quoting forms
+    that ``shlex`` understands. Returns ``None`` if the line is not such a
+    directive.
+    """
+    try:
+        parts = shlex.split(raw)
+    except ValueError:
+        return None
+    if not parts:
+        return None
+    flag = parts[0]
+    # --index-url=URL / -i=URL
+    for prefix in ("--index-url=", "-i="):
+        if flag.startswith(prefix):
+            return flag[len(prefix) :] or None
+    # --index-url URL / -i URL
+    if flag in ("--index-url", "-i") and len(parts) >= 2:
+        return parts[1]
+    return None
+
+
 __all__ = ["get_transformed_code"]
 
 
@@ -95,6 +121,13 @@ def _get_parser() -> ArgumentParser:
         action="store_true",
         help="whether pre-release packages should be considered",
     )
+    parser.add_argument(
+        "--index-url",
+        "-i",
+        type=str,
+        default=None,
+        help="base URL of the package index to use for lookup",
+    )
     parser.add_argument(
         "--force-reinstall",
         action="store_true",
@@ -138,8 +171,7 @@ async def get_action_kwargs(argv: list[str]) -> tuple[str | None, dict[str, Any]
     except (Exception, SystemExit):
         return None, {}
 
-    kwargs = {}
-
+    kwargs: dict[str, Any] = {}
     action = args.action
 
     if action == "install":
@@ -154,33 +186,54 @@ async def get_action_kwargs(argv: list[str]) -> tuple[str | None, dict[str, Any]
         if args.verbose:
             kwargs["keep_going"] = True
 
+        if args.index_url:
+            kwargs["index_urls"] = args.index_url
+
         if args.force_reinstall:
             kwargs["reinstall"] = True
 
+        index_urls: list[str] = []
         for req_file in args.requirements or []:
-            async for spec in _specs_from_requirements_file(Path(req_file)):
+            async for spec in _specs_from_requirements_file(
+                Path(req_file), index_urls=index_urls
+            ):
                 kwargs["requirements"] += [spec]
 
         for const_file in args.constraints or []:
             async for spec in _specs_from_requirements_file(Path(const_file)):
-                kwargs["constraints"] += [spec]
+                kwargs.setdefault("constraints", []).append(spec)
+
+        # Apply index URLs from requirements files only if --index-url was not
+        # already given on the command line.
+        if index_urls and "index_urls" not in kwargs:
+            kwargs["index_urls"] = index_urls
 
     return action, kwargs
 
 
-async def _specs_from_requirements_file(spec_path: Path) -> AsyncIterator[str]:
+async def _specs_from_requirements_file(
+    spec_path: Path,
+    *,
+    index_urls: list[str] | None = None,
+) -> AsyncIterator[str]:
     """Extract package specs from a ``requirements.txt``-style file."""
     if not spec_path.exists():
         warn(f"piplite could not find requirements file {spec_path}")
         return
 
-    for line_no, line in enumerate(spec_path.read_text(encoding="utf").splitlines()):
-        async for spec in _specs_from_requirements_line(spec_path, line_no + 1, line):
+    for line_no, line in enumerate(spec_path.read_text(encoding="utf-8").splitlines()):
+        async for spec in _specs_from_requirements_line(
+            spec_path, line_no + 1, line, index_urls=index_urls
+        ):
             yield spec
 
 
 async def _specs_from_requirements_line(
-    spec_path: Path, line_no: int, line: str
+    spec_path: Path,
+    line_no: int,
+    line: str,
+    *,
+    index_urls: list[str] | None = None,
 ) -> AsyncIterator[str]:
     """Get package specs from a line of a ``requirements.txt``-style file.
 
@@ -195,13 +248,20 @@ async def _specs_from_requirements_line(
     if file_match:
         ref = file_match.groupdict()["path_ref"]
         ref_path = Path(ref if ref.startswith("/") else spec_path.parent / ref)
-        async for sub_spec in _specs_from_requirements_file(ref_path):
+        async for sub_spec in _specs_from_requirements_file(
+            ref_path, index_urls=index_urls
+        ):
             yield sub_spec
+        return
+
+    url = _parse_index_url_line(raw)
+    if url is not None:
+        if index_urls is not None:
+            index_urls.append(url)
+        return
     elif raw.startswith("-"):
         warn(f"{spec_path}:{line_no}: unrecognized spec: {raw}")
         return
-    else:
-        spec = raw
 
-    if spec:
-        yield spec
+    if raw:
+        yield raw

packages/pyodide-kernel/py/piplite/piplite/piplite.py

@@ -20,18 +20,37 @@
 logger = logging.getLogger(__name__)
 
 
-#: a list of Warehouse-like API endpoints or derived multi-package all.json
-_PIPLITE_URLS: list[str] = []
-
 #: a cache of available packages
 _PIPLITE_INDICES: dict[str, dict[str, Any]] = {}
 
-#: don't fall back to pypi.org if a package is not found in _PIPLITE_URLS
-_PIPLITE_DISABLE_PYPI = False
-
 #: a well-known file name respected by the rest of the build chain
 ALL_JSON = "/all.json"
 
+#: This is the runtime configuration set from the JS side via worker.ts, which acts as
+#: a single source of truth for all site-level piplite settings.
+#:
+#: Keys written by worker.ts on every kernel start:
+#:   piplite_urls  – list of local all.json warehouse index URLs
+#:   disable_pypi  – whether to block fallback to pypi.org
+#:
+#: Keys that are optionally written by worker.ts from pipliteIndexUrls (preferred)
+#: or pipliteInstallDefaultOptions.index_urls.
+#:   index_urls    – default index URL(s) that get forwarded to micropip.install
+_PIPLITE_DEFAULT_INSTALL_ARGS: dict[str, Any] = {
+    "piplite_urls": [],  # a list of Warehouse-like API endpoints or derived multi-package all.json
+    "disable_pypi": False,  # don't fall back to pypi.org if package not found in _PIPLITE_URLS
+}
+
+#: a list of Warehouse-like API endpoints or derived multi-package all.json
+#: N.B. this is kept as a live alias to ``_PIPLITE_DEFAULT_INSTALL_ARGS["piplite_urls"]``
+#: right now but should be deprecated later
+_PIPLITE_URLS: list[str] = _PIPLITE_DEFAULT_INSTALL_ARGS["piplite_urls"]
+
+#: don't fall back to pypi.org if a package is not found in ``_PIPLITE_URLS``.
+#: N.B. this reflects the initial value of ``_PIPLITE_DEFAULT_INSTALL_ARGS["disable_pypi"]`` at
+#: module load time, but is not kept in sync with it and should be deprecated later
+_PIPLITE_DISABLE_PYPI: bool = _PIPLITE_DEFAULT_INSTALL_ARGS["disable_pypi"]
+
 
 class PiplitePyPIDisabled(ValueError):
     """An error for when PyPI is disabled at the site level, but a download was
@@ -91,7 +110,7 @@ async def _query_package(
     fetch_kwargs: dict[str, Any] | None = None,
 ) -> ProjectInfo:
     """Fetch the warehouse API metadata for a specific ``pkgname``."""
-    for piplite_url in _PIPLITE_URLS:
+    for piplite_url in _PIPLITE_DEFAULT_INSTALL_ARGS.get("piplite_urls", []):
         if not piplite_url.split("?")[0].split("#")[0].endswith(ALL_JSON):
             logger.warning("Non-all.json piplite URL not supported %s", piplite_url)
             continue
@@ -105,7 +124,7 @@ async def _query_package(
         if pypi_json_from_index:
             return pypi_json_from_index
 
-    if _PIPLITE_DISABLE_PYPI:
+    if _PIPLITE_DEFAULT_INSTALL_ARGS.get("disable_pypi", False):
         raise PiplitePyPIDisabled(
             f"{name} could not be installed: PyPI fallback is disabled"
         )
@@ -130,7 +149,17 @@ async def _install(
     constraints: list[str] | None = None,
     reinstall: bool = False,
 ):
-    """Invoke micropip.install with a patch to get data from local indexes"""
+    """Invoke micropip.install with a patch to get data from local indexes.
+
+    If ``index_urls`` is not explicitly provided and a default has been
+    configured via ``_PIPLITE_DEFAULT_INSTALL_ARGS`` (e.g. from
+    ``pipliteInstallDefaultOptions`` in ``jupyter-lite.json``), the default
+    is used.
+    """
+    effective_index_urls = index_urls
+    if effective_index_urls is None:
+        effective_index_urls = _PIPLITE_DEFAULT_INSTALL_ARGS.get("index_urls")
+
     with patch("micropip.package_index.query_package", _query_package):
         return await micropip.install(
             requirements=requirements,
@@ -139,7 +168,7 @@ async def _install(
             credentials=credentials,
             pre=pre,
             constraints=constraints,
-            index_urls=index_urls,
+            index_urls=effective_index_urls,
             verbose=verbose,
             reinstall=reinstall,
         )