diff --git a/docs/environment-variables.md b/docs/environment-variables.md index 983ae3cb5f..3770e79c5a 100644 --- a/docs/environment-variables.md +++ b/docs/environment-variables.md @@ -131,7 +131,24 @@ is not resolved. ::: -:::{envvar} RULES_PYTHON_REPO_DEBUG +:::{envvar} RULES_PYTHON_WHL_LIBRARY_OPTIMIZED + +When `1`, the whl_library will use optimized naming and target generation: +* Spoke repository names no longer include the Python version, allowing + wheel reuse across different Python versions. +* Per-extra targets (`pkg__extra`) are generated in the spoke repository + instead of bundling all extras into a single target. +* The hub repository creates explicit aliases mapping `pkg`, `pkg[]`, + and `pkg[extra]` to the appropriate spoke targets. + +Defaults to `0` if unset. + +:::{versionadded} VERSION_NEXT_FEATURE +::: + +::: + +::{envvar} RULES_PYTHON_REPO_DEBUG When `1`, repository rules will print debug information about what they're doing. This is mostly useful for development to debug errors. diff --git a/docs/readthedocs_build.sh b/docs/readthedocs_build.sh index cd60792a3f..d830ec4be8 100755 --- a/docs/readthedocs_build.sh +++ b/docs/readthedocs_build.sh @@ -1,5 +1,8 @@ #!/usr/bin/env bash +# Exercising the whl_library optimized mode to ensure it is working +export RULES_PYTHON_WHL_LIBRARY_OPTIMIZED=1 + set -eou pipefail declare -a extra_env diff --git a/plan.md b/plan.md new file mode 100644 index 0000000000..588c03fdf8 --- /dev/null +++ b/plan.md @@ -0,0 +1,131 @@ +# Single dep for spoke repos per `whl_name`. + +The goal is to have `whl_library` instances from wheels (where we pass `url` for a URL or we pass +a `whl_file` label which points to an actual wheel) to be reused across different python versions +and across different configurations where different extras are requested. This means that +if the user is using the same wheel in 2 different configurations, the wheel is reused, because +the extracted contents should not differ. This could be important for: +* platform-specific wheels for custom platforms defined by the user. Otherwise the + platform-specific wheel behaviour should remain unchanged. +* cross-platform wheels for are the main affected item here. + +The code should be gated by a feature flag, which can be flipped to be enabled using an +environmental variable via + +The file to modify `python/private/internal_config_repo.bzl`. Use the +`RULES_PYTHON_WHL_LIBRARY_OPTIMIZED=1` to enable the feature. Add it to the +`docs/readthedocs_build.sh` file so that we are exercising the code. And document this in +the `docs/environment-variables.md` file. +If the flag is off, then it is equivalent to the current +`main` branch behaviour, if it is on, then it is equivalent to turning all of the behaviour +implemented here on. + +Since this is a refactor for users not using private APIs, the env variable will be switched/removed +later together with old code. + +All of the code should be written in a TDD style where we add a failing test for a feature and then +we implement the new code. + +## Change the naming of the spoke repositories: + +> The hub repositories should be named `_. + +The names should be changed: +* from `+pip+dev_pip_314_requests_py3_none_any_2a0d60c1_linux_x86_64_linux_x86_64_freethreaded` + to `+pip+dev_pip_requests_py3_none_any_2a0d60c1` +* from `+pip+dev_pip_314_roman_numerals_py3_none_any_647ba99c` + to `+pip+dev_pip_roman_numerals_py3_none_any_647ba99c` +* from `+pip+dev_pip_314_markupsafe_cp314_cp314t_manylinux_2_17_x86_64_fed51ac4` + to `+pip+dev_pip_markupsafe_cp314_cp314t_manylinux_2_17_x86_64_fed51ac4` + +So the naming convention after the change is: +`_____` + +Where `` is the hub repository name and the rest of the segments come from the wheel name +itself. + +The sdist building `whl_library` instances should change the naming to: +`___` + +The file to modify `python/private/pypi/whl_repo_name.bzl` + +## Change the `whl_library_targets` + +Using the `METADATA` file that is parsed from `whl_metadata` function where the `Provides-Extra` is +retrieved from the file. Right now we generate a single target which includes all of the extras that +are required. Instead do the following target generation: +* `pkg` (no extras) - target with no extras, this is the main target that includes the python sources, + the other targets just include extra dependencies. +* `pkg__extra` - target for each extra in the provides-extra list. If the target depends on + `self[another_extra]`, then explode the nodes so that the target only depends on `pkg` target + + extra dependencies. Use `py_library` for this. Also be smart about generating targets: + - If the only extra is in the env marker, but the dependency is not in the hub repo, then skip + generating the whole `pkg__extra` target. + - Propose any extra ideas here. + +Related files: +* `python/private/pypi/whl_library.bzl` +* `python/private/pypi/whl_metadata.bzl` +* `python/private/pypi/whl_library_targets.bzl` + +## Change `hub_builder` and `hub_repository` + +Instead of passing the `requirement[extra1,extra2]` to the `whl_library`, pass it to the +`hub_repository` via `whl_config_setting` so that the `render_pkg_aliases` is using the information to alias to the right +extra target based on what is requested. This should retain the behaviour where different target +platforms are allowed to target `foo[baz]` and `foo[bar]` and we select the +`@dev_pip_spoke_repo//:pkg__baz` and `@dev_pip_spoke_repo//:pkg__bar` appropriately. + +We should also generate extra targets here: +* `pkg` should point to the target with all specified extras as passed to the `hub_repository`. This is to keep backwards compatibility with how it used to be done + previously. This also keeps compatibility with `rules_pycross`. Add this as a comment + when doing this. The spoke repos should not be used directly, so this difference in + behaviour is OK. + This reuses the targets declared in the hub repository described below. + It is backed by a `py_library` target if there are multiple extras, or an `alias` if it + is only a single extra. It is the same target as described below. +* `pkg[]` should point to the target in the spoke without extras +* `pkg[extra]` should point to the target in the spoke with particular extras. Do this for each + provided extra. So for `requirements[extra1,extra2]` passed to the hub repo, 2 extra targets will + be created. If the extras have not been specified for certain platforms via the + `whl_config_setting`, then the `select` statement should raise an `error` for no match. + Document the mapping explicitly: hub `pkg[]` → spoke `pkg`, hub `pkg[extra]` → spoke `pkg__extra`. + The fact that some of the targets in the spoke repos are unreachable is intentional - this is + because the hub repository contents are created before the `whl_library` downloads the whl and + inspects the METADATA. + The reverse direction is not a concern because the requirements file locking results in + consistent file. If something like this happens, then we should provide a message to the + user that something went wrong and that they should create a ticket in rules_python bug + tracker. +* If there is `requirement[extra1,extra2]` passed, that means that we should create a special alias + that includes both of the targets at once. For this use `py_library` instead of `alias` and pass + `:pkg[extra1]` and `:pkg[extra2]` to the `deps` of the `py_library`. This won't cause + a failure because for a particular platform that is requested this should work by + construction, because the dependency targets will be also defined for the particular platform. + +* If the `requirement` is passed without any extras, then hub `pkg` should alias to `pkg[]`. +* If the target is `py_library`, then we should name it without `[]` to avoid any aspects traversing + `py_library` targets changing behaviour. Only alias targets can have `[]`. + +Related files: +* `python/private/pypi/render_pkg_aliases.bzl` +* `python/private/pypi/hub_builder.bzl` +* `python/private/pypi/hub_repository.bzl` +* `python/private/pypi/whl_config_setting.bzl` - consider extending this struct to specify which + extras are requested for which platform. + +## Modify `pip_repository` + +Do similar changes to the `WORKSPACE` code to ensure easier maintenance of `whl_library` so that all +code paths to `whl_library` remain consistent. + +File: +* `python/private/pypi/pip_repository.bzl` + +## Modify `unified_hub_repo` + +Pass the extras to the `unified_hub_repo` as well so that the extra targets are created. The target +topology in the unified hub repo should correspond to the hub_repository. + +File: +* `python/private/pypi/unified_hub_repo.bzl` diff --git a/python/private/internal_config_repo.bzl b/python/private/internal_config_repo.bzl index 72970cf100..4fbb6106b4 100644 --- a/python/private/internal_config_repo.bzl +++ b/python/private/internal_config_repo.bzl @@ -24,12 +24,16 @@ load(":repo_utils.bzl", "repo_utils") _ENABLE_DEPRECATION_WARNINGS_ENVVAR_NAME = "RULES_PYTHON_DEPRECATION_WARNINGS" _ENABLE_DEPRECATION_WARNINGS_DEFAULT = "0" +_WHL_LIBRARY_OPTIMIZED_ENVVAR_NAME = "RULES_PYTHON_WHL_LIBRARY_OPTIMIZED" +_WHL_LIBRARY_OPTIMIZED_DEFAULT = "0" + _CONFIG_TEMPLATE = """ config = struct( build_python_zip_default = {build_python_zip_default}, supports_whl_extraction = {supports_whl_extraction}, enable_pystar = True, enable_deprecation_warnings = {enable_deprecation_warnings}, + whl_library_optimized = {whl_library_optimized}, bazel_8_or_later = {bazel_8_or_later}, bazel_9_or_later = {bazel_9_or_later}, bazel_10_or_later = {bazel_10_or_later}, @@ -102,6 +106,7 @@ def _internal_config_repo_impl(rctx): rctx.file("rules_python_config.bzl", _CONFIG_TEMPLATE.format( build_python_zip_default = repo_utils.get_platforms_os_name(rctx) == "windows", enable_deprecation_warnings = _bool_from_environ(rctx, _ENABLE_DEPRECATION_WARNINGS_ENVVAR_NAME, _ENABLE_DEPRECATION_WARNINGS_DEFAULT), + whl_library_optimized = _bool_from_environ(rctx, _WHL_LIBRARY_OPTIMIZED_ENVVAR_NAME, _WHL_LIBRARY_OPTIMIZED_DEFAULT), builtin_py_info_symbol = builtin_py_info_symbol, builtin_py_runtime_info_symbol = builtin_py_runtime_info_symbol, supports_whl_extraction = str(supports_whl_extraction), diff --git a/python/private/pypi/extension.bzl b/python/private/pypi/extension.bzl index 62d2c8de16..7c4dd1054f 100644 --- a/python/private/pypi/extension.bzl +++ b/python/private/pypi/extension.bzl @@ -194,13 +194,16 @@ def _configure(config, *, override = False, **kwargs): def build_config( *, module_ctx, - enable_pipstar_extract): + enable_pipstar_extract, + whl_library_optimized = False): """Parse 'configure' and 'default' extension tags Args: module_ctx: {type}`module_ctx` module context. enable_pipstar_extract: {type}`bool | None` a flag to also not pass Python interpreter to `whl_library` when possible. + whl_library_optimized: {type}`bool` If True, use optimized whl_library + naming and target generation. Returns: A struct with the configuration. @@ -261,6 +264,7 @@ def build_config( for name, values in defaults["platforms"].items() }, enable_pipstar_extract = enable_pipstar_extract, + whl_library_optimized = whl_library_optimized, toml_decode = toml.decode, ) @@ -269,6 +273,7 @@ def parse_modules( _fail = fail, simpleapi_download = simpleapi_download, enable_pipstar_extract = False, + whl_library_optimized = False, **kwargs): """Implementation of parsing the tag classes for the extension and return a struct for registering repositories. @@ -277,6 +282,8 @@ def parse_modules( simpleapi_download: Used for testing overrides enable_pipstar_extract: {type}`bool` a flag to enable dropping Python dependency for extracting wheels. + whl_library_optimized: {type}`bool` If True, use optimized whl_library + naming and target generation. _fail: {type}`function` the failure function, mainly for testing. **kwargs: Extra arguments passed to the hub_builder. @@ -314,7 +321,11 @@ You cannot use both the additive_build_content and additive_build_content_file a srcs_exclude_glob = whl_mod.srcs_exclude_glob, ) - config = build_config(module_ctx = module_ctx, enable_pipstar_extract = enable_pipstar_extract) + config = build_config( + module_ctx = module_ctx, + enable_pipstar_extract = enable_pipstar_extract, + whl_library_optimized = whl_library_optimized, + ) # TODO @aignas 2025-06-03: Merge override API with the builder? _overriden_whl_set = {} @@ -432,6 +443,7 @@ You cannot use both the additive_build_content and additive_build_content_file a exposed_packages = {} extra_aliases = {} whl_libraries = {} + hub_whl_extras = {} for hub in pip_hub_map.values(): out = hub.build() @@ -445,6 +457,7 @@ You cannot use both the additive_build_content and additive_build_content_file a extra_aliases[hub.name] = out.extra_aliases hub_group_map[hub.name] = out.group_map hub_whl_map[hub.name] = out.whl_map + hub_whl_extras[hub.name] = out.whl_extras return struct( config = config, @@ -454,6 +467,7 @@ You cannot use both the additive_build_content and additive_build_content_file a facts = simpleapi_cache.get_facts(), hub_group_map = hub_group_map, hub_whl_map = hub_whl_map, + hub_whl_extras = hub_whl_extras, whl_libraries = whl_libraries, whl_mods = whl_mods, platform_config_settings = { @@ -492,6 +506,34 @@ def _create_unified_hub_repo(mods): if hub_name not in extra_aliases[qual_alias]: extra_aliases[qual_alias].append(hub_name) + # Add optimized mode extras to extra_aliases for the unified hub. + for whl_name, extras_info in mods.hub_whl_extras.get(hub_name, {}).items(): + norm_pkg = normalize_name(whl_name) + for extra_names in extras_info.values(): + for extra in extra_names: + alias_name = "%s__%s" % (norm_pkg, extra) + qual_alias = "%s:%s" % (alias_name, "pkg") + if qual_alias not in extra_aliases: + extra_aliases[qual_alias] = [] + if hub_name not in extra_aliases[qual_alias]: + extra_aliases[qual_alias].append(hub_name) + + # Also add whl, data, dist_info aliases for the extra + for std_alias in ["whl", "data", "dist_info"]: + qual = "%s:%s" % (alias_name, std_alias) + if qual not in extra_aliases: + extra_aliases[qual] = [] + if hub_name not in extra_aliases[qual]: + extra_aliases[qual].append(hub_name) + + # Add pkg__ alias (no extras) + no_extras_alias = "%s__" % norm_pkg + qual_alias = "%s:%s" % (no_extras_alias, "pkg") + if qual_alias not in extra_aliases: + extra_aliases[qual_alias] = [] + if hub_name not in extra_aliases[qual_alias]: + extra_aliases[qual_alias].append(hub_name) + unified_hub_repo( name = "pypi", default_hub = mods.default_hub or (hubs[0] if hubs else ""), @@ -569,6 +611,7 @@ def _pip_impl(module_ctx): mods = parse_modules( module_ctx, enable_pipstar_extract = rp_config.bazel_8_or_later, + whl_library_optimized = rp_config.whl_library_optimized, ) # Build all of the wheel modifications if the tag class is called. @@ -589,6 +632,7 @@ def _pip_impl(module_ctx): packages = mods.exposed_packages.get(hub_name, []), platform_config_settings = mods.platform_config_settings.get(hub_name, {}), groups = mods.hub_group_map.get(hub_name), + whl_extras = mods.hub_whl_extras.get(hub_name, {}), ) _create_unified_hub_repo(mods) diff --git a/python/private/pypi/generate_whl_library_build_bazel.bzl b/python/private/pypi/generate_whl_library_build_bazel.bzl index a9a29081f7..35247d3d7e 100644 --- a/python/private/pypi/generate_whl_library_build_bazel.bzl +++ b/python/private/pypi/generate_whl_library_build_bazel.bzl @@ -27,6 +27,8 @@ _RENDER = { "extras": render.list, "group_deps": render.list, "include": str, + "optimized": repr, + "provides_extra": render.list, "requires_dist": render.list, "srcs_exclude": render.list, "tags": render.list, diff --git a/python/private/pypi/hub_builder.bzl b/python/private/pypi/hub_builder.bzl index 7717f36731..4d6ac6eb6a 100644 --- a/python/private/pypi/hub_builder.bzl +++ b/python/private/pypi/hub_builder.bzl @@ -75,6 +75,9 @@ def hub_builder( # setting originated from. # dict[str whl_name, dict[str config_setting, str repo_name]] _whl_map = {}, # modified by _add_whl_library + # Map of whl extras for optimized mode. + # dict[str whl_name, list[str]] + _whl_extras = {}, # modified by _add_whl_library # Internal @@ -142,6 +145,9 @@ def _build(self): # Mapping of whl_library repo names and their kwargs. # dict[str repo_name, dict[str, object] kwargs] whl_libraries = self._whl_libraries, + # Per-whl extras for optimized mode. + # dict[str whl_name, dict[str repo_name, list[str]]] + whl_extras = self._whl_extras, ) def _pip_parse(self, module_ctx, pip_attr): @@ -308,13 +314,10 @@ def _add_whl_library(self, *, python_version, whl, repo): # disallow building from sdist. return - # TODO @aignas 2025-06-29: we should not need the version in the repo_name if - # we are using pipstar and we are downloading the wheel using the downloader - # - # However, for that we should first have a different way to reference closures with - # extras. For example, if some package depends on `foo[extra]` and another depends on - # `foo`, we should have 2 py_library targets. - repo_name = "{}_{}_{}".format(self.name, version_label(python_version), repo.repo_name) + if self._config.whl_library_optimized: + repo_name = "{}_{}".format(self.name, repo.repo_name) + else: + repo_name = "{}_{}_{}".format(self.name, version_label(python_version), repo.repo_name) if repo_name in self._whl_libraries: diff = _diff_dict(self._whl_libraries[repo_name], repo.args) @@ -549,6 +552,7 @@ def _create_whl_repos( is_multiple_versions = whl.is_multiple_versions, interpreter = interpreter, enable_pipstar_extract = enable_pipstar_extract, + optimized = self._config.whl_library_optimized, ) _add_whl_library( self, @@ -557,6 +561,11 @@ def _create_whl_repos( repo = repo, ) + if self._config.whl_library_optimized: + extras = _parse_extras(src.requirement_line) + if extras: + self._whl_extras.setdefault(whl.name, {})[repo.repo_name] = extras + def _common_args(self, module_ctx, *, pip_attr): # Construct args separately so that the lock file can be smaller and does not include unused # attrs. @@ -623,7 +632,8 @@ def _whl_repo( python_version, use_downloader, interpreter, - enable_pipstar_extract = False): + enable_pipstar_extract = False, + optimized = False): args = dict(whl_library_args) args["requirement"] = src.requirement_line is_whl = src.filename.endswith(".whl") @@ -678,14 +688,14 @@ def _whl_repo( # TODO @aignas 2025-11-02: once we have pipstar enabled we can add extra # targets to each hub for each extra combination and solve this more cleanly as opposed to # duplicating whl_library repositories. - target_platforms = src.target_platforms if is_multiple_versions else [] + target_platforms = src.target_platforms if (is_multiple_versions and not optimized) else [] return struct( - repo_name = whl_repo_name(src.filename, src.sha256, *target_platforms), + repo_name = whl_repo_name(src.filename, src.sha256, target_platforms = target_platforms, optimized = optimized), args = args, config_setting = whl_config_setting( version = python_version, - target_platforms = src.target_platforms, + target_platforms = src.target_platforms if not optimized else None, ), ) @@ -694,3 +704,25 @@ def _use_downloader(self, python_version, whl_name): normalize_name(whl_name), self._get_index_urls.get(python_version) != None, ) + +def _parse_extras(requirement_line): + """Parse extras from a requirement line. + + Args: + requirement_line: {type}`str` A requirement line like + ``requests[security,socks]==2.28.0``. + + Returns: + A list of extra names, or empty list if none. + """ + bracket_start = requirement_line.find("[") + if bracket_start < 0: + return [] + bracket_end = requirement_line.find("]", bracket_start) + if bracket_end < 0: + return [] + return sorted([ + e.strip() + for e in requirement_line[bracket_start + 1:bracket_end].split(",") + if e.strip() + ]) diff --git a/python/private/pypi/hub_repository.bzl b/python/private/pypi/hub_repository.bzl index f915aa1c77..42a83314e6 100644 --- a/python/private/pypi/hub_repository.bzl +++ b/python/private/pypi/hub_repository.bzl @@ -35,6 +35,7 @@ def _impl(rctx): extra_hub_aliases = rctx.attr.extra_hub_aliases, requirement_cycles = rctx.attr.groups, platform_config_settings = rctx.attr.platform_config_settings, + whl_extras = rctx.attr.whl_extras, ) for path, contents in aliases.items(): rctx.file(path, contents) @@ -92,6 +93,13 @@ The list of packages that will be exposed via all_*requirements macros. Defaults mandatory = True, doc = "The apparent name of the repo. This is needed because in bzlmod, the name attribute becomes the canonical name.", ), + "whl_extras": attr.string_list_dict( + mandatory = False, + doc = """\ +Per-wheel extras for the optimized whl_library mode. Maps whl names to spoke +repo names and their extras. +""", + ), "whl_map": attr.string_dict( mandatory = True, doc = """\ diff --git a/python/private/pypi/render_pkg_aliases.bzl b/python/private/pypi/render_pkg_aliases.bzl index 0a1c328491..1a5f4c04df 100644 --- a/python/private/pypi/render_pkg_aliases.bzl +++ b/python/private/pypi/render_pkg_aliases.bzl @@ -87,7 +87,25 @@ package(default_visibility = ["//visibility:public"]) extra_loads = extra_loads, ) -def render_pkg_aliases(*, aliases, requirement_cycles = None, extra_hub_aliases = {}, **kwargs): +def _render_extra_alias(*, name, repo, target): + return """\ +package(default_visibility = ["//visibility:public"]) + +alias( + name = "pkg", + actual = "@{repo}//:{target}", +) + +alias( + name = "whl", + actual = "@{repo}//:{target}", +) +""".format( + repo = repo, + target = target, + ) + +def render_pkg_aliases(*, aliases, requirement_cycles = None, extra_hub_aliases = {}, whl_extras = {}, **kwargs): """Create alias declarations for each PyPI package. The aliases should be appended to the pip_repository BUILD.bazel file. These aliases @@ -100,6 +118,8 @@ def render_pkg_aliases(*, aliases, requirement_cycles = None, extra_hub_aliases requirement_cycles: any package groups to also add. extra_hub_aliases: The list of extra aliases for each whl to be added in addition to the default ones. + whl_extras: Per-wheel extras for optimized mode. Maps whl names to spoke + repo names and their extras. **kwargs: Extra kwargs to pass to the rules. Returns: @@ -135,6 +155,26 @@ def render_pkg_aliases(*, aliases, requirement_cycles = None, extra_hub_aliases for name, pkg_aliases in aliases.items() } + # Generate extra alias directories for optimized mode extras. + for name, extras_info in whl_extras.items(): + normalized = normalize_name(name) + for repo_name, extra_names in extras_info.items(): + for extra in extra_names: + extra_pkg = "{}_{}".format(normalized, extra) + files["{}/BUILD.bazel".format(extra_pkg)] = _render_extra_alias( + name = extra_pkg, + repo = repo_name, + target = "{}__{}".format(normalized, extra), + ) + + # Also create pkg__ alias (no extras) + no_extras_pkg = "{}__".format(normalized) + files["{}/BUILD.bazel".format(no_extras_pkg)] = _render_extra_alias( + name = no_extras_pkg, + repo = repo_name, + target = normalized, + ) + if requirement_cycles: files["_groups/BUILD.bazel"] = generate_group_library_build_bazel("", requirement_cycles) return files @@ -151,7 +191,7 @@ def _major_minor_versions(python_versions): # Use a dict as a simple set return sorted({_major_minor(v): None for v in python_versions}) -def render_multiplatform_pkg_aliases(*, aliases, platform_config_settings = {}, **kwargs): +def render_multiplatform_pkg_aliases(*, aliases, platform_config_settings = {}, whl_extras = {}, **kwargs): """Render the multi-platform pkg aliases. Args: @@ -159,6 +199,8 @@ def render_multiplatform_pkg_aliases(*, aliases, platform_config_settings = {}, transformed from ones having `filename` to ones having `config_setting`. platform_config_settings: {type}`dict[str, list[str]]` contains all of the target platforms and their appropriate `target_settings`. + whl_extras: {type}`dict[str, dict[str, list[str]]]` per-wheel extras for + the optimized whl_library mode. **kwargs: extra arguments passed to render_pkg_aliases. Returns: @@ -175,6 +217,7 @@ def render_multiplatform_pkg_aliases(*, aliases, platform_config_settings = {}, contents = render_pkg_aliases( aliases = aliases, + whl_extras = whl_extras, **kwargs ) contents["_config/BUILD.bazel"] = _render_config_settings( diff --git a/python/private/pypi/whl_library.bzl b/python/private/pypi/whl_library.bzl index 37cc36492e..b4254973fd 100644 --- a/python/private/pypi/whl_library.bzl +++ b/python/private/pypi/whl_library.bzl @@ -14,6 +14,7 @@ "" +load("@rules_python_internal//:rules_python_config.bzl", rp_config = "config") # buildifier: disable=bzl-visibility load("//python/private:auth.bzl", "AUTH_ATTRS", "get_auth") load("//python/private:envsubst.bzl", "envsubst") load("//python/private:is_standalone_interpreter.bzl", "is_standalone_interpreter") @@ -456,7 +457,9 @@ def _whl_library_impl(rctx): group_name = rctx.attr.group_name, namespace_package_files = namespace_package_files, extras = requirement(rctx.attr.requirement).extras, + provides_extra = metadata.provides_extra, entry_points = entry_points, + optimized = rp_config.whl_library_optimized, purl = _to_purl( index = rctx.attr.index_url, metadata = metadata, diff --git a/python/private/pypi/whl_library_targets.bzl b/python/private/pypi/whl_library_targets.bzl index 01a89aadcc..ae9bae5128 100644 --- a/python/private/pypi/whl_library_targets.bzl +++ b/python/private/pypi/whl_library_targets.bzl @@ -15,8 +15,9 @@ """Macro to generate all of the targets present in a {obj}`whl_library`.""" load("@bazel_skylib//rules:copy_file.bzl", "copy_file") +load("@rules_python//python:defs.bzl", "py_library") load("//python:py_binary.bzl", "py_binary") -load("//python:py_library.bzl", "py_library") +load("//python:py_library.bzl", py_library_rule = "py_library") load("//python/private:normalize_name.bzl", "normalize_name") load(":env_marker_setting.bzl", "env_marker_setting") load( @@ -54,6 +55,8 @@ def whl_library_targets_from_requires( metadata_version = "", requires_dist = [], extras = [], + provides_extra = [], + optimized = False, entry_points = {}, include = [], group_deps = [], @@ -71,23 +74,89 @@ def whl_library_targets_from_requires( requires_dist: {type}`list[str]` The list of `Requires-Dist` values from the whl `METADATA`. extras: {type}`list[str]` The list of requested extras. This essentially includes extra transitive dependencies in the final targets depending on the wheel `METADATA`. + provides_extra: {type}`list[str]` The list of extras that this package provides. + optimized: {type}`bool` If True, generate per-extra targets instead of + bundling all extras into a single target. entry_points: {type}`list[dict]` A list of parsed entry point definitions. include: {type}`list[str]` The list of packages to include. **kwargs: Extra args passed to the {obj}`whl_library_targets` """ - package_deps = _parse_requires_dist( + if optimized: + _whl_library_targets_from_requires_optimized( + name = name, + metadata_name = metadata_name, + metadata_version = metadata_version, + requires_dist = requires_dist, + provides_extra = provides_extra, + entry_points = entry_points, + include = include, + group_deps = group_deps, + **kwargs + ) + else: + package_deps = _parse_requires_dist( + name = metadata_name, + requires_dist = requires_dist, + excludes = group_deps, + extras = extras, + include = include, + ) + + whl_library_targets( + name = name, + dependencies = package_deps.deps, + dependencies_with_markers = package_deps.deps_select, + entry_points = entry_points, + tags = [ + "pypi_name={}".format(metadata_name), + "pypi_version={}".format(metadata_version), + ], + **kwargs + ) + +def _whl_library_targets_from_requires_optimized( + *, + name, + metadata_name, + metadata_version, + requires_dist, + provides_extra, + entry_points, + include, + group_deps, + **kwargs): + dep_template = kwargs.pop("dep_template", "") + dep_tmpl = dep_template.format(name = "{}", target = PY_LIBRARY_PUBLIC_LABEL) + + base_deps = _parse_requires_dist( name = metadata_name, requires_dist = requires_dist, excludes = group_deps, - extras = extras, + extras = [], include = include, ) + extra_deps_map = {} + for extra in (provides_extra or []): + extra_deps_map[extra] = _parse_requires_dist( + name = metadata_name, + requires_dist = requires_dist, + excludes = group_deps, + extras = [extra], + include = include, + ) + + all_deps_select = dict(base_deps.deps_select) + for extra_deps in extra_deps_map.values(): + for d, m in extra_deps.deps_select.items(): + all_deps_select.setdefault(d, m) + whl_library_targets( name = name, - dependencies = package_deps.deps, - dependencies_with_markers = package_deps.deps_select, + dependencies = base_deps.deps, + dependencies_with_markers = all_deps_select, entry_points = entry_points, + dep_template = dep_template, tags = [ "pypi_name={}".format(metadata_name), "pypi_version={}".format(metadata_version), @@ -95,6 +164,47 @@ def whl_library_targets_from_requires( **kwargs ) + base_deps_set = {d: True for d in base_deps.deps} + + for extra in (provides_extra or []): + extra_deps = extra_deps_map[extra] + extra_only_deps = sorted([ + d + for d in extra_deps.deps + if d not in base_deps_set + ]) + extra_only_deps_select = { + d: m + for d, m in extra_deps.deps_select.items() + if d not in base_deps_set + } + + if not extra_only_deps and not extra_only_deps_select: + continue + + deps_entries = [":pkg"] + [ + dep_tmpl.format(d) + for d in extra_only_deps + ] + + if extra_only_deps_select: + select_entries = { + ":is_include_{}_true".format(d): [dep_tmpl.format(d)] + for d in sorted(extra_only_deps_select) + } + select_entries.setdefault("//conditions:default", []) + deps_entries.append(select(select_entries)) + py_library( + name = "pkg__{}".format(extra), + deps = deps_entries, + tags = [ + "pypi_name={}".format(metadata_name), + "pypi_version={}".format(metadata_version), + "pypi_extra={}".format(extra), + ], + visibility = ["//visibility:public"], + ) + def _parse_requires_dist( *, name, @@ -134,7 +244,7 @@ def whl_library_targets( rules = struct( copy_file = copy_file, py_binary = py_binary, - py_library = py_library, + py_library = py_library_rule, venv_entry_point = venv_entry_point, venv_rewrite_shebang = venv_rewrite_shebang, env_marker_setting = env_marker_setting, diff --git a/python/private/pypi/whl_repo_name.bzl b/python/private/pypi/whl_repo_name.bzl index 29d774c361..5a28603614 100644 --- a/python/private/pypi/whl_repo_name.bzl +++ b/python/private/pypi/whl_repo_name.bzl @@ -18,14 +18,17 @@ load("//python/private:normalize_name.bzl", "normalize_name") load(":parse_whl_name.bzl", "parse_whl_name") -def whl_repo_name(filename, sha256, *target_platforms): +def whl_repo_name(filename, sha256, target_platforms = (), optimized = False): """Return a valid whl_library repo name given a distribution filename. Args: filename: {type}`str` the filename of the distribution. sha256: {type}`str` the sha256 of the distribution. - *target_platforms: {type}`list[str]` the extra suffixes to append. + target_platforms: {type}`tuple[str]` the extra suffixes to append. Only used when we need to support different extras per version. + Deprecated: only used in the non-optimized mode. + optimized: {type}`bool` If True, the target_platforms will not be + included in the name. When False, the non-optimized naming is used. Returns: a string that can be used in {obj}`whl_library`. @@ -61,7 +64,8 @@ def whl_repo_name(filename, sha256, *target_platforms): elif version: parts.insert(1, version) - parts.extend([p.partition("_")[-1] for p in target_platforms]) + if not optimized: + parts.extend([p.partition("_")[-1] for p in target_platforms]) return "_".join(parts) diff --git a/tests/pypi/hub_builder/hub_builder_tests.bzl b/tests/pypi/hub_builder/hub_builder_tests.bzl index 60017593fb..8d52c9068d 100644 --- a/tests/pypi/hub_builder/hub_builder_tests.bzl +++ b/tests/pypi/hub_builder/hub_builder_tests.bzl @@ -79,6 +79,7 @@ def hub_builder( }, netrc = None, auth_patterns = None, + whl_library_optimized = False, ), whl_overrides = whl_overrides, minor_mapping = minor_mapping or {"3.15": "3.15.19"}, @@ -550,6 +551,7 @@ def _test_torch_experimental_index_url(env): ("windows", "aarch64"): ["win_arm64"], # this should be ignored }.items() }, + whl_library_optimized = False, ), available_interpreters = { "python_3_12_host": "unit_test_interpreter_target", @@ -1322,6 +1324,7 @@ def _test_pipstar_platforms(env): ("osx", "aarch64"), ] }, + whl_library_optimized = False, ), ) builder.pip_parse( @@ -1406,6 +1409,7 @@ def _test_pipstar_platforms_limit(env): ("osx", "aarch64"), ] }, + whl_library_optimized = False, ), ) builder.pip_parse( diff --git a/tests/pypi/whl_repo_name/whl_repo_name_tests.bzl b/tests/pypi/whl_repo_name/whl_repo_name_tests.bzl index 35e6bcdf9f..9e12532fb1 100644 --- a/tests/pypi/whl_repo_name/whl_repo_name_tests.bzl +++ b/tests/pypi/whl_repo_name/whl_repo_name_tests.bzl @@ -25,6 +25,54 @@ def _test_simple(env): _tests.append(_test_simple) +def _test_simple_optimized(env): + got = whl_repo_name("foo-1.2.3-py3-none-any.whl", "deadbeef", optimized = True) + env.expect.that_str(got).equals("foo_py3_none_any_deadbeef") + +_tests.append(_test_simple_optimized) + +def _test_platform_whl_with_target_platforms(env): + got = whl_repo_name( + "foo-1.2.3-cp39-cp39-manylinux_2_17_x86_64.whl", + "deadbeef", + target_platforms = ("linux_x86_64",), + ) + env.expect.that_str(got).equals("foo_cp39_cp39_manylinux_2_17_x86_64_deadbeef_x86_64") + +_tests.append(_test_platform_whl_with_target_platforms) + +def _test_platform_whl_with_target_platforms_optimized(env): + got = whl_repo_name( + "foo-1.2.3-cp39-cp39-manylinux_2_17_x86_64.whl", + "deadbeef", + target_platforms = ("linux_x86_64",), + optimized = True, + ) + env.expect.that_str(got).equals("foo_cp39_cp39_manylinux_2_17_x86_64_deadbeef") + +_tests.append(_test_platform_whl_with_target_platforms_optimized) + +def _test_sdist_with_target_platforms(env): + got = whl_repo_name( + "foo-1.2.3.tar.gz", + "deadbeef", + target_platforms = ("linux_x86_64",), + ) + env.expect.that_str(got).equals("foo_sdist_deadbeef_x86_64") + +_tests.append(_test_sdist_with_target_platforms) + +def _test_sdist_with_target_platforms_optimized(env): + got = whl_repo_name( + "foo-1.2.3.tar.gz", + "deadbeef", + target_platforms = ("linux_x86_64",), + optimized = True, + ) + env.expect.that_str(got).equals("foo_sdist_deadbeef") + +_tests.append(_test_sdist_with_target_platforms_optimized) + def _test_simple_no_sha(env): got = whl_repo_name("foo-1.2.3-py3-none-any.whl", "") env.expect.that_str(got).equals("foo_1_2_3_py3_none_any")