diff --git a/jupyterlite_pyodide_kernel/tests/test_piplite.py b/jupyterlite_pyodide_kernel/tests/test_piplite.py index a1571e4e..5569ab02 100644 --- a/jupyterlite_pyodide_kernel/tests/test_piplite.py +++ b/jupyterlite_pyodide_kernel/tests/test_piplite.py @@ -135,6 +135,69 @@ def a_lite_config_file(request, an_empty_lite_dir): return an_empty_lite_dir / request.param +@pytest.mark.parametrize( + "index_urls", + [ + ["https://example.com/simple"], + ["https://example.com/simple", "https://pypi.org/simple"], + ], + ids=["single", "multiple"], +) +def test_validate_piplite_install_default_options_valid( + script_runner, a_lite_config_file, index_urls +): + """valid index_urls (list of URLs) in pipliteInstallDefaultOptions passes check""" + lite_dir = a_lite_config_file.parent + output = lite_dir / "_output" + + build = script_runner.run(["jupyter", "lite", "build"], cwd=str(lite_dir)) + assert build.success + shutil.copy2(output / a_lite_config_file.name, a_lite_config_file) + + whole_file = config_data = json.loads(a_lite_config_file.read_text(**UTF8)) + if a_lite_config_file.name == JUPYTERLITE_IPYNB: + config_data = whole_file["metadata"][JUPYTERLITE_METADATA] + + config_data[JUPYTER_CONFIG_DATA].setdefault(LITE_PLUGIN_SETTINGS, {}).setdefault( + PYODIDE_KERNEL_PLUGIN_ID, {} + )["pipliteInstallDefaultOptions"] = {"index_urls": index_urls} + + a_lite_config_file.write_text(json.dumps(whole_file, **JSON_FMT), **UTF8) + rebuild = script_runner.run(["jupyter", "lite", "build"], cwd=str(lite_dir)) + assert rebuild.success + + check = script_runner.run(["jupyter", "lite", "check"], cwd=str(lite_dir)) + assert check.success, f"check failed for index_urls={index_urls!r}" + + +def test_validate_piplite_install_default_options_invalid( + script_runner, a_lite_config_file +): + """a non-string, non-list index_urls in pipliteInstallDefaultOptions fails check""" + lite_dir = a_lite_config_file.parent + output = lite_dir / "_output" + + build = script_runner.run(["jupyter", "lite", "build"], cwd=str(lite_dir)) + assert build.success + shutil.copy2(output / a_lite_config_file.name, a_lite_config_file) + + whole_file = config_data = json.loads(a_lite_config_file.read_text(**UTF8)) + if a_lite_config_file.name == JUPYTERLITE_IPYNB: + config_data = whole_file["metadata"][JUPYTERLITE_METADATA] + + config_data[JUPYTER_CONFIG_DATA].setdefault(LITE_PLUGIN_SETTINGS, {}).setdefault( + PYODIDE_KERNEL_PLUGIN_ID, {} + )["pipliteInstallDefaultOptions"] = {"index_urls": 42} + + invalid_config = json.dumps(whole_file, **JSON_FMT) + a_lite_config_file.write_text(invalid_config, **UTF8) + rebuild = script_runner.run(["jupyter", "lite", "build"], cwd=str(lite_dir)) + assert rebuild.success + + recheck = script_runner.run(["jupyter", "lite", "check"], cwd=str(lite_dir)) + assert not recheck.success, invalid_config + + def test_validate_config(script_runner, a_lite_config_file): lite_dir = a_lite_config_file.parent output = lite_dir / "_output" diff --git a/packages/pyodide-kernel-extension/schema/kernel.v0.schema.json b/packages/pyodide-kernel-extension/schema/kernel.v0.schema.json index 4c849977..a1e3f239 100644 --- a/packages/pyodide-kernel-extension/schema/kernel.v0.schema.json +++ b/packages/pyodide-kernel-extension/schema/kernel.v0.schema.json @@ -25,6 +25,30 @@ "default": [], "format": "uri" }, + "pipliteIndexUrls": { + "description": "URL(s) of package indices to use as a site-wide default, passed as `index_urls` to micropip.install. Relative paths are resolved against the base URL of the JupyterLite deployment. Preferred over `pipliteInstallDefaultOptions.index_urls`.", + "type": "array", + "items": { + "type": "string", + "format": "uri-reference" + }, + "default": [] + }, + "pipliteInstallDefaultOptions": { + "type": "object", + "description": "Additional default options to pass to piplite.install (forwarded as kwargs to micropip.install). The `index_urls` key is supported for backwards compatibility; prefer `pipliteIndexUrls` instead.", + "default": {}, + "properties": { + "index_urls": { + "description": "URL(s) of package indices to use, passed as `index_urls` to micropip.install. Prefer the top-level `pipliteIndexUrls` key instead.", + "type": "array", + "items": { + "type": "string", + "format": "uri-reference" + } + } + } + }, "loadPyodideOptions": { "type": "object", "description": "additional options to provide to `loadPyodide`, see https://pyodide.org/en/stable/usage/api/js-api.html#globalThis.loadPyodide", diff --git a/packages/pyodide-kernel-extension/src/index.ts b/packages/pyodide-kernel-extension/src/index.ts index 9ee2536c..a3e1ef09 100644 --- a/packages/pyodide-kernel-extension/src/index.ts +++ b/packages/pyodide-kernel-extension/src/index.ts @@ -59,8 +59,23 @@ const kernel: JupyterFrontEndPlugin = { : undefined; const rawPipUrls = config.pipliteUrls || []; const pipliteUrls = rawPipUrls.map((pipUrl: string) => URLExt.parse(pipUrl).href); + // pipliteIndexUrls: config-utils.js in JupyterLite resolves relative paths automatically + // for top-level keys ending in `Urls`, so no further resolution is needed here. + const pipliteIndexUrls = (config.pipliteIndexUrls as string[] | undefined) || []; const disablePyPIFallback = !!config.disablePyPIFallback; const loadPyodideOptions = config.loadPyodideOptions || {}; + // pipliteInstallDefaultOptions; + // config-utils.js does not recurse into nested objects, so relative + // index_urls are resolved manually here. + const pipliteInstallDefaultOptions = { + ...(config.pipliteInstallDefaultOptions || {}), + }; + if (Array.isArray(pipliteInstallDefaultOptions.index_urls)) { + pipliteInstallDefaultOptions.index_urls = + pipliteInstallDefaultOptions.index_urls.map( + (u: string) => new URL(u, baseUrl).href, + ); + } for (const [key, value] of Object.entries(loadPyodideOptions)) { if (key.endsWith('URL') && typeof value === 'string') { @@ -112,10 +127,12 @@ const kernel: JupyterFrontEndPlugin = { pyodideUrl, pipliteWheelUrl, pipliteUrls, + pipliteIndexUrls, disablePyPIFallback, mountDrive, loadPyodideOptions, contentsManager, + pipliteInstallDefaultOptions, browsingContextId: serviceWorkerManager?.browsingContextId, logger, }); diff --git a/packages/pyodide-kernel/py/piplite/piplite/cli.py b/packages/pyodide-kernel/py/piplite/piplite/cli.py index 93a80bba..f7b422bf 100644 --- a/packages/pyodide-kernel/py/piplite/piplite/cli.py +++ b/packages/pyodide-kernel/py/piplite/piplite/cli.py @@ -13,7 +13,7 @@ async def install( deps: bool = True, # --no-deps credentials: str | None = None, # no CLI alias pre: bool = False, # --pre - index_urls: list[str] | str | None = None, # no CLI alias + index_urls: list[str] | str | None = None, # -i and --index-url *, constraints: list[str] | None = None, # --constraints reinstall: bool = False, # no CLI alias @@ -27,6 +27,7 @@ async def install( from __future__ import annotations import re +import shlex import sys from typing import Any, TYPE_CHECKING from argparse import ArgumentParser @@ -37,6 +38,31 @@ async def install( REQ_FILE_SPEC = r"^(?P-r|--requirements)\s*=?\s*(?P.+)$" + +def _parse_index_url_line(raw: str) -> str | None: + """Return the index URL from a ``--index-url``/``-i`` directive line. + + Handles the ``--flag URL``, ``--flag=URL``, and all other quoting forms + that ``shlex`` understands. Returns ``None`` if the line is not such a + directive. + """ + try: + parts = shlex.split(raw) + except ValueError: + return None + if not parts: + return None + flag = parts[0] + # --index-url=URL / -i=URL + for prefix in ("--index-url=", "-i="): + if flag.startswith(prefix): + return flag[len(prefix) :] or None + # --index-url URL / -i URL + if flag in ("--index-url", "-i") and len(parts) >= 2: + return parts[1] + return None + + __all__ = ["get_transformed_code"] @@ -95,6 +121,13 @@ def _get_parser() -> ArgumentParser: action="store_true", help="whether pre-release packages should be considered", ) + parser.add_argument( + "--index-url", + "-i", + type=str, + default=None, + help="base URL of the package index to use for lookup", + ) parser.add_argument( "--force-reinstall", action="store_true", @@ -138,8 +171,7 @@ async def get_action_kwargs(argv: list[str]) -> tuple[str | None, dict[str, Any] except (Exception, SystemExit): return None, {} - kwargs = {} - + kwargs: dict[str, Any] = {} action = args.action if action == "install": @@ -154,33 +186,54 @@ async def get_action_kwargs(argv: list[str]) -> tuple[str | None, dict[str, Any] if args.verbose: kwargs["keep_going"] = True + if args.index_url: + kwargs["index_urls"] = args.index_url + if args.force_reinstall: kwargs["reinstall"] = True + index_urls: list[str] = [] for req_file in args.requirements or []: - async for spec in _specs_from_requirements_file(Path(req_file)): + async for spec in _specs_from_requirements_file( + Path(req_file), index_urls=index_urls + ): kwargs["requirements"] += [spec] for const_file in args.constraints or []: async for spec in _specs_from_requirements_file(Path(const_file)): - kwargs["constraints"] += [spec] + kwargs.setdefault("constraints", []).append(spec) + + # Apply index URLs from requirements files only if --index-url was not + # already given on the command line. + if index_urls and "index_urls" not in kwargs: + kwargs["index_urls"] = index_urls return action, kwargs -async def _specs_from_requirements_file(spec_path: Path) -> AsyncIterator[str]: +async def _specs_from_requirements_file( + spec_path: Path, + *, + index_urls: list[str] | None = None, +) -> AsyncIterator[str]: """Extract package specs from a ``requirements.txt``-style file.""" if not spec_path.exists(): warn(f"piplite could not find requirements file {spec_path}") return - for line_no, line in enumerate(spec_path.read_text(encoding="utf").splitlines()): - async for spec in _specs_from_requirements_line(spec_path, line_no + 1, line): + for line_no, line in enumerate(spec_path.read_text(encoding="utf-8").splitlines()): + async for spec in _specs_from_requirements_line( + spec_path, line_no + 1, line, index_urls=index_urls + ): yield spec async def _specs_from_requirements_line( - spec_path: Path, line_no: int, line: str + spec_path: Path, + line_no: int, + line: str, + *, + index_urls: list[str] | None = None, ) -> AsyncIterator[str]: """Get package specs from a line of a ``requirements.txt``-style file. @@ -195,13 +248,20 @@ async def _specs_from_requirements_line( if file_match: ref = file_match.groupdict()["path_ref"] ref_path = Path(ref if ref.startswith("/") else spec_path.parent / ref) - async for sub_spec in _specs_from_requirements_file(ref_path): + async for sub_spec in _specs_from_requirements_file( + ref_path, index_urls=index_urls + ): yield sub_spec + return + + url = _parse_index_url_line(raw) + if url is not None: + if index_urls is not None: + index_urls.append(url) + return elif raw.startswith("-"): warn(f"{spec_path}:{line_no}: unrecognized spec: {raw}") return - else: - spec = raw - if spec: - yield spec + if raw: + yield raw diff --git a/packages/pyodide-kernel/py/piplite/piplite/piplite.py b/packages/pyodide-kernel/py/piplite/piplite/piplite.py index d86a89a4..69579e04 100644 --- a/packages/pyodide-kernel/py/piplite/piplite/piplite.py +++ b/packages/pyodide-kernel/py/piplite/piplite/piplite.py @@ -20,18 +20,37 @@ logger = logging.getLogger(__name__) -#: a list of Warehouse-like API endpoints or derived multi-package all.json -_PIPLITE_URLS: list[str] = [] - #: a cache of available packages _PIPLITE_INDICES: dict[str, dict[str, Any]] = {} -#: don't fall back to pypi.org if a package is not found in _PIPLITE_URLS -_PIPLITE_DISABLE_PYPI = False - #: a well-known file name respected by the rest of the build chain ALL_JSON = "/all.json" +#: This is the runtime configuration set from the JS side via worker.ts, which acts as +#: a single source of truth for all site-level piplite settings. +#: +#: Keys written by worker.ts on every kernel start: +#: piplite_urls – list of local all.json warehouse index URLs +#: disable_pypi – whether to block fallback to pypi.org +#: +#: Keys that are optionally written by worker.ts from pipliteIndexUrls (preferred) +#: or pipliteInstallDefaultOptions.index_urls. +#: index_urls – default index URL(s) that get forwarded to micropip.install +_PIPLITE_DEFAULT_INSTALL_ARGS: dict[str, Any] = { + "piplite_urls": [], # a list of Warehouse-like API endpoints or derived multi-package all.json + "disable_pypi": False, # don't fall back to pypi.org if package not found in _PIPLITE_URLS +} + +#: a list of Warehouse-like API endpoints or derived multi-package all.json +#: N.B. this is kept as a live alias to ``_PIPLITE_DEFAULT_INSTALL_ARGS["piplite_urls"]`` +#: right now but should be deprecated later +_PIPLITE_URLS: list[str] = _PIPLITE_DEFAULT_INSTALL_ARGS["piplite_urls"] + +#: don't fall back to pypi.org if a package is not found in ``_PIPLITE_URLS``. +#: N.B. this reflects the initial value of ``_PIPLITE_DEFAULT_INSTALL_ARGS["disable_pypi"]`` at +#: module load time, but is not kept in sync with it and should be deprecated later +_PIPLITE_DISABLE_PYPI: bool = _PIPLITE_DEFAULT_INSTALL_ARGS["disable_pypi"] + class PiplitePyPIDisabled(ValueError): """An error for when PyPI is disabled at the site level, but a download was @@ -91,7 +110,7 @@ async def _query_package( fetch_kwargs: dict[str, Any] | None = None, ) -> ProjectInfo: """Fetch the warehouse API metadata for a specific ``pkgname``.""" - for piplite_url in _PIPLITE_URLS: + for piplite_url in _PIPLITE_DEFAULT_INSTALL_ARGS.get("piplite_urls", []): if not piplite_url.split("?")[0].split("#")[0].endswith(ALL_JSON): logger.warning("Non-all.json piplite URL not supported %s", piplite_url) continue @@ -105,7 +124,7 @@ async def _query_package( if pypi_json_from_index: return pypi_json_from_index - if _PIPLITE_DISABLE_PYPI: + if _PIPLITE_DEFAULT_INSTALL_ARGS.get("disable_pypi", False): raise PiplitePyPIDisabled( f"{name} could not be installed: PyPI fallback is disabled" ) @@ -130,7 +149,17 @@ async def _install( constraints: list[str] | None = None, reinstall: bool = False, ): - """Invoke micropip.install with a patch to get data from local indexes""" + """Invoke micropip.install with a patch to get data from local indexes. + + If ``index_urls`` is not explicitly provided and a default has been + configured via ``_PIPLITE_DEFAULT_INSTALL_ARGS`` (e.g. from + ``pipliteInstallDefaultOptions`` in ``jupyter-lite.json``), the default + is used. + """ + effective_index_urls = index_urls + if effective_index_urls is None: + effective_index_urls = _PIPLITE_DEFAULT_INSTALL_ARGS.get("index_urls") + with patch("micropip.package_index.query_package", _query_package): return await micropip.install( requirements=requirements, @@ -139,7 +168,7 @@ async def _install( credentials=credentials, pre=pre, constraints=constraints, - index_urls=index_urls, + index_urls=effective_index_urls, verbose=verbose, reinstall=reinstall, ) diff --git a/packages/pyodide-kernel/src/kernel.ts b/packages/pyodide-kernel/src/kernel.ts index 5ad42909..956fd04a 100644 --- a/packages/pyodide-kernel/src/kernel.ts +++ b/packages/pyodide-kernel/src/kernel.ts @@ -157,6 +157,8 @@ export class PyodideKernel extends BaseKernel implements IKernel { location: this.location, mountDrive: options.mountDrive, loadPyodideOptions: options.loadPyodideOptions || {}, + pipliteIndexUrls: options.pipliteIndexUrls, + pipliteInstallDefaultOptions: options.pipliteInstallDefaultOptions, browsingContextId: options.browsingContextId, kernelId: this.id, }; @@ -432,6 +434,18 @@ export namespace PyodideKernel { */ mountDrive: boolean; + /** + * Default index URLs to pass to piplite.install as `index_urls`. Takes + * precedence over `pipliteInstallDefaultOptions.index_urls`. + */ + pipliteIndexUrls?: string[]; + + /** + * Additional default options to pass to piplite.install. The `index_urls` + * key is supported, but prefer `pipliteIndexUrls`. + */ + pipliteInstallDefaultOptions?: IPyodideWorkerKernel.IPipliteInstallOptions; + /** * additional options to provide to `loadPyodide` * @see https://pyodide.org/en/stable/usage/api/js-api.html#globalThis.loadPyodide diff --git a/packages/pyodide-kernel/src/tokens.ts b/packages/pyodide-kernel/src/tokens.ts index 0ed7fcbe..ed99f1cd 100644 --- a/packages/pyodide-kernel/src/tokens.ts +++ b/packages/pyodide-kernel/src/tokens.ts @@ -83,9 +83,25 @@ export interface ICoincidentPyodideWorkerKernel extends IPyodideWorkerKernel { export type IRemotePyodideWorkerKernel = IPyodideWorkerKernel; /** - * An namespace for Pyodide workers. + * A namespace for Pyodide workers. */ export namespace IPyodideWorkerKernel { + /** + * Options for piplite installation. + */ + export interface IPipliteInstallOptions { + /** + * URL(s) of package indices to use, forwarded to micropip.install as + * ``index_urls``. + */ + index_urls?: string[]; + + /** + * Any additional piplite install options. + */ + [key: string]: any; + } + /** * Initialization options for a worker. */ @@ -125,6 +141,18 @@ export namespace IPyodideWorkerKernel { */ mountDrive: boolean; + /** + * Default index URLs to pass to piplite.install as `index_urls`. Takes + * precedence over `pipliteInstallDefaultOptions.index_urls`. + */ + pipliteIndexUrls?: string[]; + + /** + * Additional default options to pass to piplite.install. The `index_urls` + * key is supported, but prefer `pipliteIndexUrls`. + */ + pipliteInstallDefaultOptions?: IPipliteInstallOptions; + /** * A unique ID to identify the origin of this request. * This should be provided by `IServiceWorkerManager` and is used to diff --git a/packages/pyodide-kernel/src/worker.ts b/packages/pyodide-kernel/src/worker.ts index 92946bf9..3fca79c0 100644 --- a/packages/pyodide-kernel/src/worker.ts +++ b/packages/pyodide-kernel/src/worker.ts @@ -93,8 +93,14 @@ ${e.stack}`; throw new Error('Uninitialized'); } - const { pipliteWheelUrl, disablePyPIFallback, pipliteUrls, loadPyodideOptions } = - this._options; + const { + pipliteWheelUrl, + disablePyPIFallback, + pipliteUrls, + loadPyodideOptions, + pipliteIndexUrls, + pipliteInstallDefaultOptions, + } = this._options; const preloaded = (loadPyodideOptions || {}).packages || []; @@ -109,12 +115,23 @@ ${e.stack}`; `); } + const pyJson = JSON.stringify({ + piplite_urls: pipliteUrls, + disable_pypi: disablePyPIFallback, + // pipliteInstallDefaultOptions.index_urls is supported but + // pipliteIndexUrls takes precedence if set + ...(pipliteInstallDefaultOptions || {}), + ...(pipliteIndexUrls?.length ? { index_urls: pipliteIndexUrls } : {}), + }); + + const pythonConfig = [ + 'import piplite.piplite, json', + `_from_js = json.loads("""${pyJson}""")`, + 'piplite.piplite._PIPLITE_DEFAULT_INSTALL_ARGS.update(_from_js)', + ]; + // get piplite early enough to impact pyodide-kernel dependencies - await this._pyodide.runPythonAsync(` - import piplite.piplite - piplite.piplite._PIPLITE_DISABLE_PYPI = ${disablePyPIFallback ? 'True' : 'False'} - piplite.piplite._PIPLITE_URLS = ${JSON.stringify(pipliteUrls)} - `); + await this._pyodide.runPythonAsync(pythonConfig.join('\n')); } protected async initKernel(options: IPyodideWorkerKernel.IOptions): Promise {