From ab461fea18d3611aae2a98e68c413a1533e97333 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sat, 27 Jun 2026 15:43:58 +0900 Subject: [PATCH 01/10] plan.md --- plan.md | 65 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) create mode 100644 plan.md diff --git a/plan.md b/plan.md new file mode 100644 index 0000000000..0209820d3a --- /dev/null +++ b/plan.md @@ -0,0 +1,65 @@ +# Single dep for spoke repos per `whl_name`. + +The goal is to have `whl_library` instances from wheels (where we pass `url` for a URL or we pass a +`whl_file` label which points to an actual wheel) to be reused across different python versions and +across different configurations where different extras are requested. + +## Change the naming of the spoke repositories: + +> The hub repositories should be named `_. + +The names should be changed: +* from `+pip+dev_pip_314_requests_py3_none_any_2a0d60c1_linux_x86_64_linux_x86_64_freethreaded` + to `+pip+dev_pip_requests_py3_none_any_2a0d60c1` +* from `+pip+dev_pip_314_roman_numerals_py3_none_any_647ba99c` + to `+pip+dev_pip_roman_numerals_py3_none_any_647ba99c` +* from `+pip+dev_pip_314_markupsafe_cp314_cp314t_manylinux_2_17_x86_64_fed51ac4` + to `+pip+dev_pip_markupsafe_cp314_cp314t_manylinux_2_17_x86_64_fed51ac4` + +The file to modify `python/private/pypi/whl_repo_name.bzl` + +## Change the `whl_library_targets` + +Using the `METADATA` file that is parsed from `whl_metadata` function where the `Provides-Extra` is +retrieved from the file. Right now we generate a single target which includes all of the extras that +are required. Instead do the following target generation: +* `pkg` - target with no extras, this is the main target that includes the python sources, the + other targets just include extra dependencies. If the `requirement` to `whl_library` is passed + with extras, that means that this is legacy code path, do not generate any `pkg[extra]` targets. +* `pkg[extra]` - target for each extra in the provides-extra list. If the target depends on + `self[another_extra]`, then explode the nodes so that the target only depends on `pkg` target + + extra dependencies. Use `py_library` for this. Also be smart about generating targets: + - If the only extra is in the env marker, but the dependency is not in the hub repo, then skip + generating the whole `pkg[extra]` target. + - Propose any extra ideas here. + +Related files: +* `python/private/pypi/whl_library.bzl` +* `python/private/pypi/whl_metadata.bzl` +* `python/private/pypi/whl_library_targets.bzl` + +## Change `hub_builder` and `hub_repository` + +Instead of passing the `requirement[extra1,extra2]` to the `whl_library`, pass it to the +`hub_repository` so that the `render_pkg_aliases` is using the information to alias to the right +extra target based on what is requested. This should retain the behaviour where different target +platforms are allowed to target `foo[baz]` and `foo[bar]` and we select the +`@dev_pip_spoke_repo//:pkg[baz]` and `@dev_pip_spoke_repo//:pkg[bar]` appropriately. + +We should also generate extra targets here: +* `pkg` should point to the target with all specified extras as passed to the `hub_repository` +* `pkg[]` should point to the target in the spoke without extras +* `pkg[extra]` should point to the target in the spoke with particular extras. Do this for each + provided extra. So for `requirements[extra1,extra2]` passed to the hub repo, 2 extra targets will + be created. If the extras have not been specified for certain platforms via the + `whl_config_setting`, then the `select` statement should raise an `error` for no match. +* If there is `requirement[extra1,extra2]` passed, that means that we should create a special alias + that includes both of the targets at once. For this use `py_library` instead of `alias` and pass + `:pkg[extra1]` and `:pkg[extra2]` to the `deps` of the `py_library`. + +Related files: +* `python/private/pypi/render_pkg_aliases.bzl` +* `python/private/pypi/hub_builder.bzl` +* `python/private/pypi/hub_repository.bzl` +* `python/private/pypi/whl_config_setting.bzl` - consider extending this struct to specify which + extras are requested for which platform. From 21f9bc14807e5ab9714e92503ea42ceab0f95a5e Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sat, 27 Jun 2026 15:58:40 +0900 Subject: [PATCH 02/10] fix --- plan.md | 43 +++++++++++++++++++++++++++++++++++++------ 1 file changed, 37 insertions(+), 6 deletions(-) diff --git a/plan.md b/plan.md index 0209820d3a..838a06e5ac 100644 --- a/plan.md +++ b/plan.md @@ -1,8 +1,24 @@ # Single dep for spoke repos per `whl_name`. -The goal is to have `whl_library` instances from wheels (where we pass `url` for a URL or we pass a -`whl_file` label which points to an actual wheel) to be reused across different python versions and -across different configurations where different extras are requested. +The goal is to have `whl_library` instances from wheels (where we pass `url` for a URL or we pass +a `whl_file` label which points to an actual wheel) to be reused across different python versions +and across different configurations where different extras are requested. This means that +if the user is using the same wheel in 2 different configurations, the wheel is reused, because +the extracted contents should not differ. This could be important for: +* platform-specific wheels for custom platforms defined by the user. Otherwise the + platform-specific wheel behaviour should remain unchanged. +* cross-platform wheels for are the main affected item here. + +The code should be gated by a feature flag, which can be flipped to be enabled using an +environmental variable via + +The file to modify `python/private/internal_config_repo.bzl`. Use the +`RULES_PYTHON_WHL_LIBRARY_OPTIMIZED=1` to enable the feature. Add it to the +`docs/readthedocs_build.sh` file so that we are exercising the code. And document this in +the `docs/environment-variables.md` file. + +All of the code should be written in a TDD style where we add a failing test for a feature and then +we implement the new code. ## Change the naming of the spoke repositories: @@ -23,15 +39,22 @@ The file to modify `python/private/pypi/whl_repo_name.bzl` Using the `METADATA` file that is parsed from `whl_metadata` function where the `Provides-Extra` is retrieved from the file. Right now we generate a single target which includes all of the extras that are required. Instead do the following target generation: -* `pkg` - target with no extras, this is the main target that includes the python sources, the +* `pkg[]` - target with no extras, this is the main target that includes the python sources, the other targets just include extra dependencies. If the `requirement` to `whl_library` is passed with extras, that means that this is legacy code path, do not generate any `pkg[extra]` targets. + > **Suggestion:** The "legacy" detection here means the spoke repo's behavior depends on + > how `whl_library` was called. But section 3 moves extras to `hub_repository`, so + > `whl_library` would always receive a bare requirement. Consider clarifying whether this + > legacy branch is temporary backward compat or will remain indefinitely. Also: if the hub + > repo aliases point to `pkg[extra]` targets that don't exist (legacy path), that's a + > silent breakage — should the hub repo detect and handle this? * `pkg[extra]` - target for each extra in the provides-extra list. If the target depends on `self[another_extra]`, then explode the nodes so that the target only depends on `pkg` target + extra dependencies. Use `py_library` for this. Also be smart about generating targets: - If the only extra is in the env marker, but the dependency is not in the hub repo, then skip generating the whole `pkg[extra]` target. - Propose any extra ideas here. +* `pkg` should be a target that cannot be used if the new behaviour is active. Related files: * `python/private/pypi/whl_library.bzl` @@ -47,15 +70,23 @@ platforms are allowed to target `foo[baz]` and `foo[bar]` and we select the `@dev_pip_spoke_repo//:pkg[baz]` and `@dev_pip_spoke_repo//:pkg[bar]` appropriately. We should also generate extra targets here: -* `pkg` should point to the target with all specified extras as passed to the `hub_repository` +* `pkg` should point to the target with all specified extras as passed to the `hub_repository`. This is to keep backwards compatibility with how it used to be done + previously. This also keeps compatibility with `rules_pycross`. Add this as a comment + when doing this. The spoke repos should not be used directly, so this difference in + behaviour is OK. * `pkg[]` should point to the target in the spoke without extras * `pkg[extra]` should point to the target in the spoke with particular extras. Do this for each provided extra. So for `requirements[extra1,extra2]` passed to the hub repo, 2 extra targets will be created. If the extras have not been specified for certain platforms via the `whl_config_setting`, then the `select` statement should raise an `error` for no match. + The fact that some of the targets in the spoke repos are unreachable is intentional - this is + because the hub repository contents are created before the `whl_library` downloads the whl and + inspects the METADATA. * If there is `requirement[extra1,extra2]` passed, that means that we should create a special alias that includes both of the targets at once. For this use `py_library` instead of `alias` and pass - `:pkg[extra1]` and `:pkg[extra2]` to the `deps` of the `py_library`. + `:pkg[extra1]` and `:pkg[extra2]` to the `deps` of the `py_library`. This won't cause + a failure because for a particular platform that is requested this should work by + construction. Related files: * `python/private/pypi/render_pkg_aliases.bzl` From 5eb15669fc8d46e7655ca52aa4113a52b58fbcee Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sat, 27 Jun 2026 16:04:36 +0900 Subject: [PATCH 03/10] wip --- plan.md | 57 ++++++++++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 46 insertions(+), 11 deletions(-) diff --git a/plan.md b/plan.md index 838a06e5ac..66a85bd468 100644 --- a/plan.md +++ b/plan.md @@ -16,6 +16,9 @@ The file to modify `python/private/internal_config_repo.bzl`. Use the `RULES_PYTHON_WHL_LIBRARY_OPTIMIZED=1` to enable the feature. Add it to the `docs/readthedocs_build.sh` file so that we are exercising the code. And document this in the `docs/environment-variables.md` file. +If the flag is off, then it is equivalent to the current +`main` branch behaviour, if it is on, then it is equivalent to turning all of the behaviour +implemented here on. All of the code should be written in a TDD style where we add a failing test for a feature and then we implement the new code. @@ -40,21 +43,14 @@ Using the `METADATA` file that is parsed from `whl_metadata` function where the retrieved from the file. Right now we generate a single target which includes all of the extras that are required. Instead do the following target generation: * `pkg[]` - target with no extras, this is the main target that includes the python sources, the - other targets just include extra dependencies. If the `requirement` to `whl_library` is passed - with extras, that means that this is legacy code path, do not generate any `pkg[extra]` targets. - > **Suggestion:** The "legacy" detection here means the spoke repo's behavior depends on - > how `whl_library` was called. But section 3 moves extras to `hub_repository`, so - > `whl_library` would always receive a bare requirement. Consider clarifying whether this - > legacy branch is temporary backward compat or will remain indefinitely. Also: if the hub - > repo aliases point to `pkg[extra]` targets that don't exist (legacy path), that's a - > silent breakage — should the hub repo detect and handle this? + other targets just include extra dependencies. * `pkg[extra]` - target for each extra in the provides-extra list. If the target depends on - `self[another_extra]`, then explode the nodes so that the target only depends on `pkg` target + + `self[another_extra]`, then explode the nodes so that the target only depends on `pkg[]` target + extra dependencies. Use `py_library` for this. Also be smart about generating targets: - If the only extra is in the env marker, but the dependency is not in the hub repo, then skip generating the whole `pkg[extra]` target. - Propose any extra ideas here. -* `pkg` should be a target that cannot be used if the new behaviour is active. +* `pkg` target should not be declared if the new behaviour is active. Related files: * `python/private/pypi/whl_library.bzl` @@ -74,6 +70,7 @@ We should also generate extra targets here: previously. This also keeps compatibility with `rules_pycross`. Add this as a comment when doing this. The spoke repos should not be used directly, so this difference in behaviour is OK. + This reuse the targets declared in the hub repository described below. * `pkg[]` should point to the target in the spoke without extras * `pkg[extra]` should point to the target in the spoke with particular extras. Do this for each provided extra. So for `requirements[extra1,extra2]` passed to the hub repo, 2 extra targets will @@ -82,11 +79,15 @@ We should also generate extra targets here: The fact that some of the targets in the spoke repos are unreachable is intentional - this is because the hub repository contents are created before the `whl_library` downloads the whl and inspects the METADATA. + The reverse direction is not a concern because the requirements file locking results in + consistent file. If something like this happens, then we should provide a message to the + user that something went wrong and that they should create a ticket in rules_python bug + tracker. * If there is `requirement[extra1,extra2]` passed, that means that we should create a special alias that includes both of the targets at once. For this use `py_library` instead of `alias` and pass `:pkg[extra1]` and `:pkg[extra2]` to the `deps` of the `py_library`. This won't cause a failure because for a particular platform that is requested this should work by - construction. + construction, because the dependency targets will be also defined for the particular platform. Related files: * `python/private/pypi/render_pkg_aliases.bzl` @@ -94,3 +95,37 @@ Related files: * `python/private/pypi/hub_repository.bzl` * `python/private/pypi/whl_config_setting.bzl` - consider extending this struct to specify which extras are requested for which platform. + +## Open questions on conflicting requirements + +1. **Feature flag scope:** When `RULES_PYTHON_WHL_LIBRARY_OPTIMIZED=1` is OFF, does the hub + repo fall back to the old behavior (passing `requirement[extras]` to `whl_library`) or + does it keep the new structure (extras in hub) while the spoke runs the legacy path (no + `pkg[extra]` targets)? The answer determines whether the legacy check in section 2 is + ever triggered after migration. + +2. **Hub `pkg` implementation under the new flag:** Spoke `pkg` is unusable (line 57), so + hub `pkg` (line 73) cannot alias to it. Does hub `pkg` become a `py_library` bundling + all `pkg[extra]` targets? If so, how is it different from the merged target on line 85? + Are they the same target, or two separate targets with overlapping deps? + +3. **Guarantee that hub-requested extras exist in spoke:** The hub creates aliases before + the spoke downloads the whl (lines 82-84). If `Provides-Extra` in the whl's METADATA + doesn't list an extra that was passed to the hub, the spoke won't generate that + `pkg[extra]` target and the hub alias will dangle. Is this guaranteed impossible because + pip resolved the extras before passing them to the hub? If so, document the assumption. + +4. **Merged `py_library` satisfiability guarantee:** The plan asserts the merged target + (line 85) "won't cause a failure" because it works "by construction" (line 88). But each + individual `pkg[extra]` uses `select()` and raises on no match (line 81). What guarantees + that for any platform evaluating the merged target, both individual extra targets are + satisfiable? If `extra1` has a linux `whl_config_setting` and `extra2` has a mac one, the + merged target fails on both platforms. + +5. **`pkg` name usage across spoke and hub:** Section 2 uses `pkg` for a target that "cannot + be used" under the new flag. Section 3 uses `pkg` for backward-compat "all extras" target. + Section 2 uses `pkg[]` for "no extras". Section 3 also uses `pkg[]` for "no extras". The + `pkg` and `pkg[]` names are consistent between sections now (both use `pkg[]` for no + extras), but spoke's `pkg` is dead-under-new-flag while hub's `pkg` is alive. Is the plan + to eventually remove spoke's `pkg` entirely once the flag is the only path, making the + naming fully consistent? From 916634abd9f94742b0218cd899b0f1d625c34e23 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sat, 27 Jun 2026 16:12:42 +0900 Subject: [PATCH 04/10] wip --- plan.md | 58 +++++++++++++++++++++++++-------------------------------- 1 file changed, 25 insertions(+), 33 deletions(-) diff --git a/plan.md b/plan.md index 66a85bd468..9f8694d126 100644 --- a/plan.md +++ b/plan.md @@ -60,7 +60,7 @@ Related files: ## Change `hub_builder` and `hub_repository` Instead of passing the `requirement[extra1,extra2]` to the `whl_library`, pass it to the -`hub_repository` so that the `render_pkg_aliases` is using the information to alias to the right +`hub_repository` via `whl_config_setting` so that the `render_pkg_aliases` is using the information to alias to the right extra target based on what is requested. This should retain the behaviour where different target platforms are allowed to target `foo[baz]` and `foo[bar]` and we select the `@dev_pip_spoke_repo//:pkg[baz]` and `@dev_pip_spoke_repo//:pkg[bar]` appropriately. @@ -70,7 +70,10 @@ We should also generate extra targets here: previously. This also keeps compatibility with `rules_pycross`. Add this as a comment when doing this. The spoke repos should not be used directly, so this difference in behaviour is OK. - This reuse the targets declared in the hub repository described below. + This reuses the targets declared in the hub repository described below. + It is backed by a `py_library` target if there are multiple extras, or it is an `alias` if it + is only a single extra. Since the different extras may be passed based on the target platform, + `pkg` will be always an `alias`. * `pkg[]` should point to the target in the spoke without extras * `pkg[extra]` should point to the target in the spoke with particular extras. Do this for each provided extra. So for `requirements[extra1,extra2]` passed to the hub repo, 2 extra targets will @@ -89,6 +92,8 @@ We should also generate extra targets here: a failure because for a particular platform that is requested this should work by construction, because the dependency targets will be also defined for the particular platform. +* If the `requirement` is passed without any extras, then hub `pkg` should alias to `pkg[]`. + Related files: * `python/private/pypi/render_pkg_aliases.bzl` * `python/private/pypi/hub_builder.bzl` @@ -98,34 +103,21 @@ Related files: ## Open questions on conflicting requirements -1. **Feature flag scope:** When `RULES_PYTHON_WHL_LIBRARY_OPTIMIZED=1` is OFF, does the hub - repo fall back to the old behavior (passing `requirement[extras]` to `whl_library`) or - does it keep the new structure (extras in hub) while the spoke runs the legacy path (no - `pkg[extra]` targets)? The answer determines whether the legacy check in section 2 is - ever triggered after migration. - -2. **Hub `pkg` implementation under the new flag:** Spoke `pkg` is unusable (line 57), so - hub `pkg` (line 73) cannot alias to it. Does hub `pkg` become a `py_library` bundling - all `pkg[extra]` targets? If so, how is it different from the merged target on line 85? - Are they the same target, or two separate targets with overlapping deps? - -3. **Guarantee that hub-requested extras exist in spoke:** The hub creates aliases before - the spoke downloads the whl (lines 82-84). If `Provides-Extra` in the whl's METADATA - doesn't list an extra that was passed to the hub, the spoke won't generate that - `pkg[extra]` target and the hub alias will dangle. Is this guaranteed impossible because - pip resolved the extras before passing them to the hub? If so, document the assumption. - -4. **Merged `py_library` satisfiability guarantee:** The plan asserts the merged target - (line 85) "won't cause a failure" because it works "by construction" (line 88). But each - individual `pkg[extra]` uses `select()` and raises on no match (line 81). What guarantees - that for any platform evaluating the merged target, both individual extra targets are - satisfiable? If `extra1` has a linux `whl_config_setting` and `extra2` has a mac one, the - merged target fails on both platforms. - -5. **`pkg` name usage across spoke and hub:** Section 2 uses `pkg` for a target that "cannot - be used" under the new flag. Section 3 uses `pkg` for backward-compat "all extras" target. - Section 2 uses `pkg[]` for "no extras". Section 3 also uses `pkg[]` for "no extras". The - `pkg` and `pkg[]` names are consistent between sections now (both use `pkg[]` for no - extras), but spoke's `pkg` is dead-under-new-flag while hub's `pkg` is alive. Is the plan - to eventually remove spoke's `pkg` entirely once the flag is the only path, making the - naming fully consistent? +1. **Hub `pkg` implementation:** Line 73 says hub `pkg` "reuses the targets declared in the + hub repository described below" but doesn't specify *how*. Is hub `pkg` a `py_library` + depending on `pkg[]` + all `pkg[extra]` targets? If no extras are specified, does it + alias to `pkg[]`? If multiple extras are specified, is hub `pkg` the same target as the + merged `py_library` on line 93, or a different one? The plan should state the target type + and its deps explicitly. + +2. **Migration path for the feature flag:** The flag gates a complete switch between two + different code paths (old: everything in whl_library, new: split across hub/spoke). Is + the old path ever removed, making `RULES_PYTHON_WHL_LIBRARY_OPTIMIZED` always-on? If so, + does the flag remain as a no-op for a deprecation period, or is there a plan to delete + the old code? This affects test coverage and maintenance burden. + +3. **`pkg` naming across spoke and hub after flag removal:** Currently spoke does not + declare `pkg` under the new flag (line 53), while hub uses `pkg` for backward compat + (line 69). Once the flag is removed and the old path is gone, does spoke also declare + `pkg` (as an alias to `pkg[]`) to make the two repos consistent? Or is there a deliberate + reason spoke and hub have different `pkg` semantics even long-term? From f1325de9e8191dea9d39392a9dd749c48d764d31 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sat, 27 Jun 2026 16:22:42 +0900 Subject: [PATCH 05/10] wip --- plan.md | 52 ++++++++++++++++++++++++++-------------------------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/plan.md b/plan.md index 9f8694d126..1c9b7e8ef3 100644 --- a/plan.md +++ b/plan.md @@ -20,6 +20,9 @@ If the flag is off, then it is equivalent to the current `main` branch behaviour, if it is on, then it is equivalent to turning all of the behaviour implemented here on. +Since this is a refactor for users not using private APIs, the env variable will be switched/removed +later together with old code. + All of the code should be written in a TDD style where we add a failing test for a feature and then we implement the new code. @@ -42,13 +45,13 @@ The file to modify `python/private/pypi/whl_repo_name.bzl` Using the `METADATA` file that is parsed from `whl_metadata` function where the `Provides-Extra` is retrieved from the file. Right now we generate a single target which includes all of the extras that are required. Instead do the following target generation: -* `pkg[]` - target with no extras, this is the main target that includes the python sources, the +* `pkg` - target with no extras, this is the main target that includes the python sources, the other targets just include extra dependencies. -* `pkg[extra]` - target for each extra in the provides-extra list. If the target depends on - `self[another_extra]`, then explode the nodes so that the target only depends on `pkg[]` target + +* `pkg__extra` - target for each extra in the provides-extra list. If the target depends on + `self[another_extra]`, then explode the nodes so that the target only depends on `pkg` target + extra dependencies. Use `py_library` for this. Also be smart about generating targets: - If the only extra is in the env marker, but the dependency is not in the hub repo, then skip - generating the whole `pkg[extra]` target. + generating the whole `pkg__extra` target. - Propose any extra ideas here. * `pkg` target should not be declared if the new behaviour is active. @@ -63,7 +66,7 @@ Instead of passing the `requirement[extra1,extra2]` to the `whl_library`, pass i `hub_repository` via `whl_config_setting` so that the `render_pkg_aliases` is using the information to alias to the right extra target based on what is requested. This should retain the behaviour where different target platforms are allowed to target `foo[baz]` and `foo[bar]` and we select the -`@dev_pip_spoke_repo//:pkg[baz]` and `@dev_pip_spoke_repo//:pkg[bar]` appropriately. +`@dev_pip_spoke_repo//:pkg__baz` and `@dev_pip_spoke_repo//:pkg__bar` appropriately. We should also generate extra targets here: * `pkg` should point to the target with all specified extras as passed to the `hub_repository`. This is to keep backwards compatibility with how it used to be done @@ -73,7 +76,7 @@ We should also generate extra targets here: This reuses the targets declared in the hub repository described below. It is backed by a `py_library` target if there are multiple extras, or it is an `alias` if it is only a single extra. Since the different extras may be passed based on the target platform, - `pkg` will be always an `alias`. + `pkg` should delegate to a `select()` that picks the right combination per platform. * `pkg[]` should point to the target in the spoke without extras * `pkg[extra]` should point to the target in the spoke with particular extras. Do this for each provided extra. So for `requirements[extra1,extra2]` passed to the hub repo, 2 extra targets will @@ -93,6 +96,8 @@ We should also generate extra targets here: construction, because the dependency targets will be also defined for the particular platform. * If the `requirement` is passed without any extras, then hub `pkg` should alias to `pkg[]`. +* If the target is `py_library`, then we should name it without `[]` to avoid any aspects traversing + `py_library` targets changing behaviour. Only alias targets can have `[]`. Related files: * `python/private/pypi/render_pkg_aliases.bzl` @@ -101,23 +106,18 @@ Related files: * `python/private/pypi/whl_config_setting.bzl` - consider extending this struct to specify which extras are requested for which platform. -## Open questions on conflicting requirements - -1. **Hub `pkg` implementation:** Line 73 says hub `pkg` "reuses the targets declared in the - hub repository described below" but doesn't specify *how*. Is hub `pkg` a `py_library` - depending on `pkg[]` + all `pkg[extra]` targets? If no extras are specified, does it - alias to `pkg[]`? If multiple extras are specified, is hub `pkg` the same target as the - merged `py_library` on line 93, or a different one? The plan should state the target type - and its deps explicitly. - -2. **Migration path for the feature flag:** The flag gates a complete switch between two - different code paths (old: everything in whl_library, new: split across hub/spoke). Is - the old path ever removed, making `RULES_PYTHON_WHL_LIBRARY_OPTIMIZED` always-on? If so, - does the flag remain as a no-op for a deprecation period, or is there a plan to delete - the old code? This affects test coverage and maintenance burden. - -3. **`pkg` naming across spoke and hub after flag removal:** Currently spoke does not - declare `pkg` under the new flag (line 53), while hub uses `pkg` for backward compat - (line 69). Once the flag is removed and the old path is gone, does spoke also declare - `pkg` (as an alias to `pkg[]`) to make the two repos consistent? Or is there a deliberate - reason spoke and hub have different `pkg` semantics even long-term? +## Modify `pip_repository` + +Do similar changes to the `WORKSPACE` code to ensure easier maintenance of `whl_library` so that all +code paths to `whl_library` remain consistent. + +File: +* `python/private/pypi/pip_repository.bzl` + +## Modify `unified_hub_repo` + +Pass the extras to the `unified_hub_repo` as well so that the extra targets are created. The target +topology in the unified hub repo should correspond to the hub_repository. + +File: +* `python/private/pypi/unified_hub_repo.bzl` From fbee9d3b6361a7edcf68ef5487712783e2914a29 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sat, 27 Jun 2026 16:25:54 +0900 Subject: [PATCH 06/10] wip --- plan.md | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/plan.md b/plan.md index 1c9b7e8ef3..17b124458d 100644 --- a/plan.md +++ b/plan.md @@ -45,15 +45,14 @@ The file to modify `python/private/pypi/whl_repo_name.bzl` Using the `METADATA` file that is parsed from `whl_metadata` function where the `Provides-Extra` is retrieved from the file. Right now we generate a single target which includes all of the extras that are required. Instead do the following target generation: -* `pkg` - target with no extras, this is the main target that includes the python sources, the - other targets just include extra dependencies. +* `pkg` (no extras) - target with no extras, this is the main target that includes the python sources, + the other targets just include extra dependencies. * `pkg__extra` - target for each extra in the provides-extra list. If the target depends on `self[another_extra]`, then explode the nodes so that the target only depends on `pkg` target + extra dependencies. Use `py_library` for this. Also be smart about generating targets: - If the only extra is in the env marker, but the dependency is not in the hub repo, then skip generating the whole `pkg__extra` target. - Propose any extra ideas here. -* `pkg` target should not be declared if the new behaviour is active. Related files: * `python/private/pypi/whl_library.bzl` @@ -74,14 +73,14 @@ We should also generate extra targets here: when doing this. The spoke repos should not be used directly, so this difference in behaviour is OK. This reuses the targets declared in the hub repository described below. - It is backed by a `py_library` target if there are multiple extras, or it is an `alias` if it - is only a single extra. Since the different extras may be passed based on the target platform, - `pkg` should delegate to a `select()` that picks the right combination per platform. + It is backed by a `py_library` target if there are multiple extras, or an `alias` if it + is only a single extra. * `pkg[]` should point to the target in the spoke without extras * `pkg[extra]` should point to the target in the spoke with particular extras. Do this for each provided extra. So for `requirements[extra1,extra2]` passed to the hub repo, 2 extra targets will be created. If the extras have not been specified for certain platforms via the `whl_config_setting`, then the `select` statement should raise an `error` for no match. + Document the mapping explicitly: hub `pkg[]` → spoke `pkg`, hub `pkg[extra]` → spoke `pkg__extra`. The fact that some of the targets in the spoke repos are unreachable is intentional - this is because the hub repository contents are created before the `whl_library` downloads the whl and inspects the METADATA. From eedb2a964dcb78043e153080b4294b30853fc9fb Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sat, 27 Jun 2026 16:32:37 +0900 Subject: [PATCH 07/10] wip --- plan.md | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/plan.md b/plan.md index 17b124458d..588c03fdf8 100644 --- a/plan.md +++ b/plan.md @@ -38,6 +38,15 @@ The names should be changed: * from `+pip+dev_pip_314_markupsafe_cp314_cp314t_manylinux_2_17_x86_64_fed51ac4` to `+pip+dev_pip_markupsafe_cp314_cp314t_manylinux_2_17_x86_64_fed51ac4` +So the naming convention after the change is: +`_____` + +Where `` is the hub repository name and the rest of the segments come from the wheel name +itself. + +The sdist building `whl_library` instances should change the naming to: +`___` + The file to modify `python/private/pypi/whl_repo_name.bzl` ## Change the `whl_library_targets` @@ -74,7 +83,7 @@ We should also generate extra targets here: behaviour is OK. This reuses the targets declared in the hub repository described below. It is backed by a `py_library` target if there are multiple extras, or an `alias` if it - is only a single extra. + is only a single extra. It is the same target as described below. * `pkg[]` should point to the target in the spoke without extras * `pkg[extra]` should point to the target in the spoke with particular extras. Do this for each provided extra. So for `requirements[extra1,extra2]` passed to the hub repo, 2 extra targets will From 2c8d44ae0835c634a5870423292a9bed13e4c4a1 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sat, 27 Jun 2026 17:45:40 +0900 Subject: [PATCH 08/10] feat(pypi): add whl_library_optimized mode behind env var gate Implements the first phase of the single-dep whl_library optimization, gated by RULES_PYTHON_WHL_LIBRARY_OPTIMIZED=1. What's implemented: - Feature flag struct and env var plumbing in internal_config_repo.bzl - documented in docs/environment-variables.md - exported in docs/readthedocs_build.sh - whl_repo_name.bzl: optimized=True skips target_platforms suffix in repo names - extension.bzl: threads whl_library_optimized through config -> hub_builder - hub_builder.bzl: optimized mode creates single spoke per wheel (no per-version/platform duplication), passes optimized=True to whl_repo_name - whl_library.bzl: loads rp_config, passes provides_extra+optimized to build file generation - generate_whl_library_build_bazel.bzl: _RENDER dict updated with provides_extra/optimized fields - whl_library_targets.bzl: _whl_library_targets_from_requires_optimized generates per-extra py_library targets (pkg__extra) with marker-based selects combining deps from base and extra requirements - Tests: optimized naming tests in whl_repo_name_tests.bzl; fixed hub_builder_tests.bzl config structs to include whl_library_optimized Not yet implemented (next commits): - Hub aliases for pkg[], pkg[extra] - pip_repository.bzl WORKSPACE-mode changes - unified_hub_repo.bzl changes - Integration tests for optimized mode Part of the plan in plan.md for single-dep whl_library restructuring. --- docs/environment-variables.md | 19 ++- docs/readthedocs_build.sh | 3 + python/private/internal_config_repo.bzl | 5 + python/private/pypi/extension.bzl | 16 ++- .../pypi/generate_whl_library_build_bazel.bzl | 2 + python/private/pypi/hub_builder.bzl | 21 ++- python/private/pypi/whl_library.bzl | 3 + python/private/pypi/whl_library_targets.bzl | 122 +++++++++++++++++- python/private/pypi/whl_repo_name.bzl | 10 +- tests/pypi/hub_builder/hub_builder_tests.bzl | 4 + .../whl_repo_name/whl_repo_name_tests.bzl | 48 +++++++ 11 files changed, 230 insertions(+), 23 deletions(-) diff --git a/docs/environment-variables.md b/docs/environment-variables.md index 983ae3cb5f..3770e79c5a 100644 --- a/docs/environment-variables.md +++ b/docs/environment-variables.md @@ -131,7 +131,24 @@ is not resolved. ::: -:::{envvar} RULES_PYTHON_REPO_DEBUG +:::{envvar} RULES_PYTHON_WHL_LIBRARY_OPTIMIZED + +When `1`, the whl_library will use optimized naming and target generation: +* Spoke repository names no longer include the Python version, allowing + wheel reuse across different Python versions. +* Per-extra targets (`pkg__extra`) are generated in the spoke repository + instead of bundling all extras into a single target. +* The hub repository creates explicit aliases mapping `pkg`, `pkg[]`, + and `pkg[extra]` to the appropriate spoke targets. + +Defaults to `0` if unset. + +:::{versionadded} VERSION_NEXT_FEATURE +::: + +::: + +::{envvar} RULES_PYTHON_REPO_DEBUG When `1`, repository rules will print debug information about what they're doing. This is mostly useful for development to debug errors. diff --git a/docs/readthedocs_build.sh b/docs/readthedocs_build.sh index cd60792a3f..d830ec4be8 100755 --- a/docs/readthedocs_build.sh +++ b/docs/readthedocs_build.sh @@ -1,5 +1,8 @@ #!/usr/bin/env bash +# Exercising the whl_library optimized mode to ensure it is working +export RULES_PYTHON_WHL_LIBRARY_OPTIMIZED=1 + set -eou pipefail declare -a extra_env diff --git a/python/private/internal_config_repo.bzl b/python/private/internal_config_repo.bzl index 72970cf100..4fbb6106b4 100644 --- a/python/private/internal_config_repo.bzl +++ b/python/private/internal_config_repo.bzl @@ -24,12 +24,16 @@ load(":repo_utils.bzl", "repo_utils") _ENABLE_DEPRECATION_WARNINGS_ENVVAR_NAME = "RULES_PYTHON_DEPRECATION_WARNINGS" _ENABLE_DEPRECATION_WARNINGS_DEFAULT = "0" +_WHL_LIBRARY_OPTIMIZED_ENVVAR_NAME = "RULES_PYTHON_WHL_LIBRARY_OPTIMIZED" +_WHL_LIBRARY_OPTIMIZED_DEFAULT = "0" + _CONFIG_TEMPLATE = """ config = struct( build_python_zip_default = {build_python_zip_default}, supports_whl_extraction = {supports_whl_extraction}, enable_pystar = True, enable_deprecation_warnings = {enable_deprecation_warnings}, + whl_library_optimized = {whl_library_optimized}, bazel_8_or_later = {bazel_8_or_later}, bazel_9_or_later = {bazel_9_or_later}, bazel_10_or_later = {bazel_10_or_later}, @@ -102,6 +106,7 @@ def _internal_config_repo_impl(rctx): rctx.file("rules_python_config.bzl", _CONFIG_TEMPLATE.format( build_python_zip_default = repo_utils.get_platforms_os_name(rctx) == "windows", enable_deprecation_warnings = _bool_from_environ(rctx, _ENABLE_DEPRECATION_WARNINGS_ENVVAR_NAME, _ENABLE_DEPRECATION_WARNINGS_DEFAULT), + whl_library_optimized = _bool_from_environ(rctx, _WHL_LIBRARY_OPTIMIZED_ENVVAR_NAME, _WHL_LIBRARY_OPTIMIZED_DEFAULT), builtin_py_info_symbol = builtin_py_info_symbol, builtin_py_runtime_info_symbol = builtin_py_runtime_info_symbol, supports_whl_extraction = str(supports_whl_extraction), diff --git a/python/private/pypi/extension.bzl b/python/private/pypi/extension.bzl index 62d2c8de16..2fb3c012b5 100644 --- a/python/private/pypi/extension.bzl +++ b/python/private/pypi/extension.bzl @@ -194,13 +194,16 @@ def _configure(config, *, override = False, **kwargs): def build_config( *, module_ctx, - enable_pipstar_extract): + enable_pipstar_extract, + whl_library_optimized = False): """Parse 'configure' and 'default' extension tags Args: module_ctx: {type}`module_ctx` module context. enable_pipstar_extract: {type}`bool | None` a flag to also not pass Python interpreter to `whl_library` when possible. + whl_library_optimized: {type}`bool` If True, use optimized whl_library + naming and target generation. Returns: A struct with the configuration. @@ -261,6 +264,7 @@ def build_config( for name, values in defaults["platforms"].items() }, enable_pipstar_extract = enable_pipstar_extract, + whl_library_optimized = whl_library_optimized, toml_decode = toml.decode, ) @@ -269,6 +273,7 @@ def parse_modules( _fail = fail, simpleapi_download = simpleapi_download, enable_pipstar_extract = False, + whl_library_optimized = False, **kwargs): """Implementation of parsing the tag classes for the extension and return a struct for registering repositories. @@ -277,6 +282,8 @@ def parse_modules( simpleapi_download: Used for testing overrides enable_pipstar_extract: {type}`bool` a flag to enable dropping Python dependency for extracting wheels. + whl_library_optimized: {type}`bool` If True, use optimized whl_library + naming and target generation. _fail: {type}`function` the failure function, mainly for testing. **kwargs: Extra arguments passed to the hub_builder. @@ -314,7 +321,11 @@ You cannot use both the additive_build_content and additive_build_content_file a srcs_exclude_glob = whl_mod.srcs_exclude_glob, ) - config = build_config(module_ctx = module_ctx, enable_pipstar_extract = enable_pipstar_extract) + config = build_config( + module_ctx = module_ctx, + enable_pipstar_extract = enable_pipstar_extract, + whl_library_optimized = whl_library_optimized, + ) # TODO @aignas 2025-06-03: Merge override API with the builder? _overriden_whl_set = {} @@ -569,6 +580,7 @@ def _pip_impl(module_ctx): mods = parse_modules( module_ctx, enable_pipstar_extract = rp_config.bazel_8_or_later, + whl_library_optimized = rp_config.whl_library_optimized, ) # Build all of the wheel modifications if the tag class is called. diff --git a/python/private/pypi/generate_whl_library_build_bazel.bzl b/python/private/pypi/generate_whl_library_build_bazel.bzl index a9a29081f7..35247d3d7e 100644 --- a/python/private/pypi/generate_whl_library_build_bazel.bzl +++ b/python/private/pypi/generate_whl_library_build_bazel.bzl @@ -27,6 +27,8 @@ _RENDER = { "extras": render.list, "group_deps": render.list, "include": str, + "optimized": repr, + "provides_extra": render.list, "requires_dist": render.list, "srcs_exclude": render.list, "tags": render.list, diff --git a/python/private/pypi/hub_builder.bzl b/python/private/pypi/hub_builder.bzl index 7717f36731..293c9bbb82 100644 --- a/python/private/pypi/hub_builder.bzl +++ b/python/private/pypi/hub_builder.bzl @@ -308,13 +308,10 @@ def _add_whl_library(self, *, python_version, whl, repo): # disallow building from sdist. return - # TODO @aignas 2025-06-29: we should not need the version in the repo_name if - # we are using pipstar and we are downloading the wheel using the downloader - # - # However, for that we should first have a different way to reference closures with - # extras. For example, if some package depends on `foo[extra]` and another depends on - # `foo`, we should have 2 py_library targets. - repo_name = "{}_{}_{}".format(self.name, version_label(python_version), repo.repo_name) + if self._config.whl_library_optimized: + repo_name = "{}_{}".format(self.name, repo.repo_name) + else: + repo_name = "{}_{}_{}".format(self.name, version_label(python_version), repo.repo_name) if repo_name in self._whl_libraries: diff = _diff_dict(self._whl_libraries[repo_name], repo.args) @@ -549,6 +546,7 @@ def _create_whl_repos( is_multiple_versions = whl.is_multiple_versions, interpreter = interpreter, enable_pipstar_extract = enable_pipstar_extract, + optimized = self._config.whl_library_optimized, ) _add_whl_library( self, @@ -623,7 +621,8 @@ def _whl_repo( python_version, use_downloader, interpreter, - enable_pipstar_extract = False): + enable_pipstar_extract = False, + optimized = False): args = dict(whl_library_args) args["requirement"] = src.requirement_line is_whl = src.filename.endswith(".whl") @@ -678,14 +677,14 @@ def _whl_repo( # TODO @aignas 2025-11-02: once we have pipstar enabled we can add extra # targets to each hub for each extra combination and solve this more cleanly as opposed to # duplicating whl_library repositories. - target_platforms = src.target_platforms if is_multiple_versions else [] + target_platforms = src.target_platforms if (is_multiple_versions and not optimized) else [] return struct( - repo_name = whl_repo_name(src.filename, src.sha256, *target_platforms), + repo_name = whl_repo_name(src.filename, src.sha256, target_platforms = target_platforms, optimized = optimized), args = args, config_setting = whl_config_setting( version = python_version, - target_platforms = src.target_platforms, + target_platforms = src.target_platforms if not optimized else None, ), ) diff --git a/python/private/pypi/whl_library.bzl b/python/private/pypi/whl_library.bzl index 37cc36492e..b4254973fd 100644 --- a/python/private/pypi/whl_library.bzl +++ b/python/private/pypi/whl_library.bzl @@ -14,6 +14,7 @@ "" +load("@rules_python_internal//:rules_python_config.bzl", rp_config = "config") # buildifier: disable=bzl-visibility load("//python/private:auth.bzl", "AUTH_ATTRS", "get_auth") load("//python/private:envsubst.bzl", "envsubst") load("//python/private:is_standalone_interpreter.bzl", "is_standalone_interpreter") @@ -456,7 +457,9 @@ def _whl_library_impl(rctx): group_name = rctx.attr.group_name, namespace_package_files = namespace_package_files, extras = requirement(rctx.attr.requirement).extras, + provides_extra = metadata.provides_extra, entry_points = entry_points, + optimized = rp_config.whl_library_optimized, purl = _to_purl( index = rctx.attr.index_url, metadata = metadata, diff --git a/python/private/pypi/whl_library_targets.bzl b/python/private/pypi/whl_library_targets.bzl index 01a89aadcc..ae9bae5128 100644 --- a/python/private/pypi/whl_library_targets.bzl +++ b/python/private/pypi/whl_library_targets.bzl @@ -15,8 +15,9 @@ """Macro to generate all of the targets present in a {obj}`whl_library`.""" load("@bazel_skylib//rules:copy_file.bzl", "copy_file") +load("@rules_python//python:defs.bzl", "py_library") load("//python:py_binary.bzl", "py_binary") -load("//python:py_library.bzl", "py_library") +load("//python:py_library.bzl", py_library_rule = "py_library") load("//python/private:normalize_name.bzl", "normalize_name") load(":env_marker_setting.bzl", "env_marker_setting") load( @@ -54,6 +55,8 @@ def whl_library_targets_from_requires( metadata_version = "", requires_dist = [], extras = [], + provides_extra = [], + optimized = False, entry_points = {}, include = [], group_deps = [], @@ -71,23 +74,89 @@ def whl_library_targets_from_requires( requires_dist: {type}`list[str]` The list of `Requires-Dist` values from the whl `METADATA`. extras: {type}`list[str]` The list of requested extras. This essentially includes extra transitive dependencies in the final targets depending on the wheel `METADATA`. + provides_extra: {type}`list[str]` The list of extras that this package provides. + optimized: {type}`bool` If True, generate per-extra targets instead of + bundling all extras into a single target. entry_points: {type}`list[dict]` A list of parsed entry point definitions. include: {type}`list[str]` The list of packages to include. **kwargs: Extra args passed to the {obj}`whl_library_targets` """ - package_deps = _parse_requires_dist( + if optimized: + _whl_library_targets_from_requires_optimized( + name = name, + metadata_name = metadata_name, + metadata_version = metadata_version, + requires_dist = requires_dist, + provides_extra = provides_extra, + entry_points = entry_points, + include = include, + group_deps = group_deps, + **kwargs + ) + else: + package_deps = _parse_requires_dist( + name = metadata_name, + requires_dist = requires_dist, + excludes = group_deps, + extras = extras, + include = include, + ) + + whl_library_targets( + name = name, + dependencies = package_deps.deps, + dependencies_with_markers = package_deps.deps_select, + entry_points = entry_points, + tags = [ + "pypi_name={}".format(metadata_name), + "pypi_version={}".format(metadata_version), + ], + **kwargs + ) + +def _whl_library_targets_from_requires_optimized( + *, + name, + metadata_name, + metadata_version, + requires_dist, + provides_extra, + entry_points, + include, + group_deps, + **kwargs): + dep_template = kwargs.pop("dep_template", "") + dep_tmpl = dep_template.format(name = "{}", target = PY_LIBRARY_PUBLIC_LABEL) + + base_deps = _parse_requires_dist( name = metadata_name, requires_dist = requires_dist, excludes = group_deps, - extras = extras, + extras = [], include = include, ) + extra_deps_map = {} + for extra in (provides_extra or []): + extra_deps_map[extra] = _parse_requires_dist( + name = metadata_name, + requires_dist = requires_dist, + excludes = group_deps, + extras = [extra], + include = include, + ) + + all_deps_select = dict(base_deps.deps_select) + for extra_deps in extra_deps_map.values(): + for d, m in extra_deps.deps_select.items(): + all_deps_select.setdefault(d, m) + whl_library_targets( name = name, - dependencies = package_deps.deps, - dependencies_with_markers = package_deps.deps_select, + dependencies = base_deps.deps, + dependencies_with_markers = all_deps_select, entry_points = entry_points, + dep_template = dep_template, tags = [ "pypi_name={}".format(metadata_name), "pypi_version={}".format(metadata_version), @@ -95,6 +164,47 @@ def whl_library_targets_from_requires( **kwargs ) + base_deps_set = {d: True for d in base_deps.deps} + + for extra in (provides_extra or []): + extra_deps = extra_deps_map[extra] + extra_only_deps = sorted([ + d + for d in extra_deps.deps + if d not in base_deps_set + ]) + extra_only_deps_select = { + d: m + for d, m in extra_deps.deps_select.items() + if d not in base_deps_set + } + + if not extra_only_deps and not extra_only_deps_select: + continue + + deps_entries = [":pkg"] + [ + dep_tmpl.format(d) + for d in extra_only_deps + ] + + if extra_only_deps_select: + select_entries = { + ":is_include_{}_true".format(d): [dep_tmpl.format(d)] + for d in sorted(extra_only_deps_select) + } + select_entries.setdefault("//conditions:default", []) + deps_entries.append(select(select_entries)) + py_library( + name = "pkg__{}".format(extra), + deps = deps_entries, + tags = [ + "pypi_name={}".format(metadata_name), + "pypi_version={}".format(metadata_version), + "pypi_extra={}".format(extra), + ], + visibility = ["//visibility:public"], + ) + def _parse_requires_dist( *, name, @@ -134,7 +244,7 @@ def whl_library_targets( rules = struct( copy_file = copy_file, py_binary = py_binary, - py_library = py_library, + py_library = py_library_rule, venv_entry_point = venv_entry_point, venv_rewrite_shebang = venv_rewrite_shebang, env_marker_setting = env_marker_setting, diff --git a/python/private/pypi/whl_repo_name.bzl b/python/private/pypi/whl_repo_name.bzl index 29d774c361..5a28603614 100644 --- a/python/private/pypi/whl_repo_name.bzl +++ b/python/private/pypi/whl_repo_name.bzl @@ -18,14 +18,17 @@ load("//python/private:normalize_name.bzl", "normalize_name") load(":parse_whl_name.bzl", "parse_whl_name") -def whl_repo_name(filename, sha256, *target_platforms): +def whl_repo_name(filename, sha256, target_platforms = (), optimized = False): """Return a valid whl_library repo name given a distribution filename. Args: filename: {type}`str` the filename of the distribution. sha256: {type}`str` the sha256 of the distribution. - *target_platforms: {type}`list[str]` the extra suffixes to append. + target_platforms: {type}`tuple[str]` the extra suffixes to append. Only used when we need to support different extras per version. + Deprecated: only used in the non-optimized mode. + optimized: {type}`bool` If True, the target_platforms will not be + included in the name. When False, the non-optimized naming is used. Returns: a string that can be used in {obj}`whl_library`. @@ -61,7 +64,8 @@ def whl_repo_name(filename, sha256, *target_platforms): elif version: parts.insert(1, version) - parts.extend([p.partition("_")[-1] for p in target_platforms]) + if not optimized: + parts.extend([p.partition("_")[-1] for p in target_platforms]) return "_".join(parts) diff --git a/tests/pypi/hub_builder/hub_builder_tests.bzl b/tests/pypi/hub_builder/hub_builder_tests.bzl index 60017593fb..8d52c9068d 100644 --- a/tests/pypi/hub_builder/hub_builder_tests.bzl +++ b/tests/pypi/hub_builder/hub_builder_tests.bzl @@ -79,6 +79,7 @@ def hub_builder( }, netrc = None, auth_patterns = None, + whl_library_optimized = False, ), whl_overrides = whl_overrides, minor_mapping = minor_mapping or {"3.15": "3.15.19"}, @@ -550,6 +551,7 @@ def _test_torch_experimental_index_url(env): ("windows", "aarch64"): ["win_arm64"], # this should be ignored }.items() }, + whl_library_optimized = False, ), available_interpreters = { "python_3_12_host": "unit_test_interpreter_target", @@ -1322,6 +1324,7 @@ def _test_pipstar_platforms(env): ("osx", "aarch64"), ] }, + whl_library_optimized = False, ), ) builder.pip_parse( @@ -1406,6 +1409,7 @@ def _test_pipstar_platforms_limit(env): ("osx", "aarch64"), ] }, + whl_library_optimized = False, ), ) builder.pip_parse( diff --git a/tests/pypi/whl_repo_name/whl_repo_name_tests.bzl b/tests/pypi/whl_repo_name/whl_repo_name_tests.bzl index 35e6bcdf9f..9e12532fb1 100644 --- a/tests/pypi/whl_repo_name/whl_repo_name_tests.bzl +++ b/tests/pypi/whl_repo_name/whl_repo_name_tests.bzl @@ -25,6 +25,54 @@ def _test_simple(env): _tests.append(_test_simple) +def _test_simple_optimized(env): + got = whl_repo_name("foo-1.2.3-py3-none-any.whl", "deadbeef", optimized = True) + env.expect.that_str(got).equals("foo_py3_none_any_deadbeef") + +_tests.append(_test_simple_optimized) + +def _test_platform_whl_with_target_platforms(env): + got = whl_repo_name( + "foo-1.2.3-cp39-cp39-manylinux_2_17_x86_64.whl", + "deadbeef", + target_platforms = ("linux_x86_64",), + ) + env.expect.that_str(got).equals("foo_cp39_cp39_manylinux_2_17_x86_64_deadbeef_x86_64") + +_tests.append(_test_platform_whl_with_target_platforms) + +def _test_platform_whl_with_target_platforms_optimized(env): + got = whl_repo_name( + "foo-1.2.3-cp39-cp39-manylinux_2_17_x86_64.whl", + "deadbeef", + target_platforms = ("linux_x86_64",), + optimized = True, + ) + env.expect.that_str(got).equals("foo_cp39_cp39_manylinux_2_17_x86_64_deadbeef") + +_tests.append(_test_platform_whl_with_target_platforms_optimized) + +def _test_sdist_with_target_platforms(env): + got = whl_repo_name( + "foo-1.2.3.tar.gz", + "deadbeef", + target_platforms = ("linux_x86_64",), + ) + env.expect.that_str(got).equals("foo_sdist_deadbeef_x86_64") + +_tests.append(_test_sdist_with_target_platforms) + +def _test_sdist_with_target_platforms_optimized(env): + got = whl_repo_name( + "foo-1.2.3.tar.gz", + "deadbeef", + target_platforms = ("linux_x86_64",), + optimized = True, + ) + env.expect.that_str(got).equals("foo_sdist_deadbeef") + +_tests.append(_test_sdist_with_target_platforms_optimized) + def _test_simple_no_sha(env): got = whl_repo_name("foo-1.2.3-py3-none-any.whl", "") env.expect.that_str(got).equals("foo_1_2_3_py3_none_any") From 7f854494304f3e15adba1637cdf3234aadd0f4be Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sat, 27 Jun 2026 17:58:29 +0900 Subject: [PATCH 09/10] feat(pypi): implement hub aliases for optimized whl_library extras When RULES_PYTHON_WHL_LIBRARY_OPTIMIZED=1: - hub_builder.bzl: tracks per-wheel extras parsed from requirement_line, includes whl_extras in build output struct - extension.bzl: threads hub_whl_extras through parse_modules -> _pip_impl - hub_repository.bzl: accepts new whl_extras attribute, passes to alias rendering - render_pkg_aliases.bzl: generates extra alias targets for pkg[extra] and pkg[] pointing to spoke pkg__extra and pkg targets respectively The hub now creates per-extra alias directories (e.g. requests_security/BUILD.bazel) for each requested extra, plus a pkg__/ alias for no-extras access. The main pkg alias continues to work as before (select-based to the spoke). Not yet implemented (follow-up): - unified_hub_repo.bzl extras forwarding - unified_hub_setup.bzl extras handling - pip_repository.bzl WORKSPACE-mode mirror - Integration tests for optimized mode Part of plan.md for single-dep whl_library restructuring. --- python/private/pypi/extension.bzl | 4 ++ python/private/pypi/hub_builder.bzl | 33 +++++++++++++++ python/private/pypi/hub_repository.bzl | 8 ++++ python/private/pypi/render_pkg_aliases.bzl | 47 +++++++++++++++++++++- 4 files changed, 90 insertions(+), 2 deletions(-) diff --git a/python/private/pypi/extension.bzl b/python/private/pypi/extension.bzl index 2fb3c012b5..0b2c0eff03 100644 --- a/python/private/pypi/extension.bzl +++ b/python/private/pypi/extension.bzl @@ -443,6 +443,7 @@ You cannot use both the additive_build_content and additive_build_content_file a exposed_packages = {} extra_aliases = {} whl_libraries = {} + hub_whl_extras = {} for hub in pip_hub_map.values(): out = hub.build() @@ -456,6 +457,7 @@ You cannot use both the additive_build_content and additive_build_content_file a extra_aliases[hub.name] = out.extra_aliases hub_group_map[hub.name] = out.group_map hub_whl_map[hub.name] = out.whl_map + hub_whl_extras[hub.name] = out.whl_extras return struct( config = config, @@ -465,6 +467,7 @@ You cannot use both the additive_build_content and additive_build_content_file a facts = simpleapi_cache.get_facts(), hub_group_map = hub_group_map, hub_whl_map = hub_whl_map, + hub_whl_extras = hub_whl_extras, whl_libraries = whl_libraries, whl_mods = whl_mods, platform_config_settings = { @@ -601,6 +604,7 @@ def _pip_impl(module_ctx): packages = mods.exposed_packages.get(hub_name, []), platform_config_settings = mods.platform_config_settings.get(hub_name, {}), groups = mods.hub_group_map.get(hub_name), + whl_extras = mods.hub_whl_extras.get(hub_name, {}), ) _create_unified_hub_repo(mods) diff --git a/python/private/pypi/hub_builder.bzl b/python/private/pypi/hub_builder.bzl index 293c9bbb82..4d6ac6eb6a 100644 --- a/python/private/pypi/hub_builder.bzl +++ b/python/private/pypi/hub_builder.bzl @@ -75,6 +75,9 @@ def hub_builder( # setting originated from. # dict[str whl_name, dict[str config_setting, str repo_name]] _whl_map = {}, # modified by _add_whl_library + # Map of whl extras for optimized mode. + # dict[str whl_name, list[str]] + _whl_extras = {}, # modified by _add_whl_library # Internal @@ -142,6 +145,9 @@ def _build(self): # Mapping of whl_library repo names and their kwargs. # dict[str repo_name, dict[str, object] kwargs] whl_libraries = self._whl_libraries, + # Per-whl extras for optimized mode. + # dict[str whl_name, dict[str repo_name, list[str]]] + whl_extras = self._whl_extras, ) def _pip_parse(self, module_ctx, pip_attr): @@ -555,6 +561,11 @@ def _create_whl_repos( repo = repo, ) + if self._config.whl_library_optimized: + extras = _parse_extras(src.requirement_line) + if extras: + self._whl_extras.setdefault(whl.name, {})[repo.repo_name] = extras + def _common_args(self, module_ctx, *, pip_attr): # Construct args separately so that the lock file can be smaller and does not include unused # attrs. @@ -693,3 +704,25 @@ def _use_downloader(self, python_version, whl_name): normalize_name(whl_name), self._get_index_urls.get(python_version) != None, ) + +def _parse_extras(requirement_line): + """Parse extras from a requirement line. + + Args: + requirement_line: {type}`str` A requirement line like + ``requests[security,socks]==2.28.0``. + + Returns: + A list of extra names, or empty list if none. + """ + bracket_start = requirement_line.find("[") + if bracket_start < 0: + return [] + bracket_end = requirement_line.find("]", bracket_start) + if bracket_end < 0: + return [] + return sorted([ + e.strip() + for e in requirement_line[bracket_start + 1:bracket_end].split(",") + if e.strip() + ]) diff --git a/python/private/pypi/hub_repository.bzl b/python/private/pypi/hub_repository.bzl index f915aa1c77..42a83314e6 100644 --- a/python/private/pypi/hub_repository.bzl +++ b/python/private/pypi/hub_repository.bzl @@ -35,6 +35,7 @@ def _impl(rctx): extra_hub_aliases = rctx.attr.extra_hub_aliases, requirement_cycles = rctx.attr.groups, platform_config_settings = rctx.attr.platform_config_settings, + whl_extras = rctx.attr.whl_extras, ) for path, contents in aliases.items(): rctx.file(path, contents) @@ -92,6 +93,13 @@ The list of packages that will be exposed via all_*requirements macros. Defaults mandatory = True, doc = "The apparent name of the repo. This is needed because in bzlmod, the name attribute becomes the canonical name.", ), + "whl_extras": attr.string_list_dict( + mandatory = False, + doc = """\ +Per-wheel extras for the optimized whl_library mode. Maps whl names to spoke +repo names and their extras. +""", + ), "whl_map": attr.string_dict( mandatory = True, doc = """\ diff --git a/python/private/pypi/render_pkg_aliases.bzl b/python/private/pypi/render_pkg_aliases.bzl index 0a1c328491..1a5f4c04df 100644 --- a/python/private/pypi/render_pkg_aliases.bzl +++ b/python/private/pypi/render_pkg_aliases.bzl @@ -87,7 +87,25 @@ package(default_visibility = ["//visibility:public"]) extra_loads = extra_loads, ) -def render_pkg_aliases(*, aliases, requirement_cycles = None, extra_hub_aliases = {}, **kwargs): +def _render_extra_alias(*, name, repo, target): + return """\ +package(default_visibility = ["//visibility:public"]) + +alias( + name = "pkg", + actual = "@{repo}//:{target}", +) + +alias( + name = "whl", + actual = "@{repo}//:{target}", +) +""".format( + repo = repo, + target = target, + ) + +def render_pkg_aliases(*, aliases, requirement_cycles = None, extra_hub_aliases = {}, whl_extras = {}, **kwargs): """Create alias declarations for each PyPI package. The aliases should be appended to the pip_repository BUILD.bazel file. These aliases @@ -100,6 +118,8 @@ def render_pkg_aliases(*, aliases, requirement_cycles = None, extra_hub_aliases requirement_cycles: any package groups to also add. extra_hub_aliases: The list of extra aliases for each whl to be added in addition to the default ones. + whl_extras: Per-wheel extras for optimized mode. Maps whl names to spoke + repo names and their extras. **kwargs: Extra kwargs to pass to the rules. Returns: @@ -135,6 +155,26 @@ def render_pkg_aliases(*, aliases, requirement_cycles = None, extra_hub_aliases for name, pkg_aliases in aliases.items() } + # Generate extra alias directories for optimized mode extras. + for name, extras_info in whl_extras.items(): + normalized = normalize_name(name) + for repo_name, extra_names in extras_info.items(): + for extra in extra_names: + extra_pkg = "{}_{}".format(normalized, extra) + files["{}/BUILD.bazel".format(extra_pkg)] = _render_extra_alias( + name = extra_pkg, + repo = repo_name, + target = "{}__{}".format(normalized, extra), + ) + + # Also create pkg__ alias (no extras) + no_extras_pkg = "{}__".format(normalized) + files["{}/BUILD.bazel".format(no_extras_pkg)] = _render_extra_alias( + name = no_extras_pkg, + repo = repo_name, + target = normalized, + ) + if requirement_cycles: files["_groups/BUILD.bazel"] = generate_group_library_build_bazel("", requirement_cycles) return files @@ -151,7 +191,7 @@ def _major_minor_versions(python_versions): # Use a dict as a simple set return sorted({_major_minor(v): None for v in python_versions}) -def render_multiplatform_pkg_aliases(*, aliases, platform_config_settings = {}, **kwargs): +def render_multiplatform_pkg_aliases(*, aliases, platform_config_settings = {}, whl_extras = {}, **kwargs): """Render the multi-platform pkg aliases. Args: @@ -159,6 +199,8 @@ def render_multiplatform_pkg_aliases(*, aliases, platform_config_settings = {}, transformed from ones having `filename` to ones having `config_setting`. platform_config_settings: {type}`dict[str, list[str]]` contains all of the target platforms and their appropriate `target_settings`. + whl_extras: {type}`dict[str, dict[str, list[str]]]` per-wheel extras for + the optimized whl_library mode. **kwargs: extra arguments passed to render_pkg_aliases. Returns: @@ -175,6 +217,7 @@ def render_multiplatform_pkg_aliases(*, aliases, platform_config_settings = {}, contents = render_pkg_aliases( aliases = aliases, + whl_extras = whl_extras, **kwargs ) contents["_config/BUILD.bazel"] = _render_config_settings( From f3fa6c97d7912626ec5b3029358220b93d0abd3c Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sat, 27 Jun 2026 18:29:11 +0900 Subject: [PATCH 10/10] feat(pypi): forward optimized whl extras to unified hub repo When RULES_PYTHON_WHL_LIBRARY_OPTIMIZED=1, the unified @pypi hub now creates extra alias targets for pkg[extra] and pkg[] access patterns, mirroring what the per-hub hub_repository does. extension.bzl: _create_unified_hub_repo iterates mods.hub_whl_extras and adds pkg__extra and pkg__ aliases to the extra_aliases dict for each hub that contains the wheel. Part of plan.md for single-dep whl_library restructuring. --- python/private/pypi/extension.bzl | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/python/private/pypi/extension.bzl b/python/private/pypi/extension.bzl index 0b2c0eff03..7c4dd1054f 100644 --- a/python/private/pypi/extension.bzl +++ b/python/private/pypi/extension.bzl @@ -506,6 +506,34 @@ def _create_unified_hub_repo(mods): if hub_name not in extra_aliases[qual_alias]: extra_aliases[qual_alias].append(hub_name) + # Add optimized mode extras to extra_aliases for the unified hub. + for whl_name, extras_info in mods.hub_whl_extras.get(hub_name, {}).items(): + norm_pkg = normalize_name(whl_name) + for extra_names in extras_info.values(): + for extra in extra_names: + alias_name = "%s__%s" % (norm_pkg, extra) + qual_alias = "%s:%s" % (alias_name, "pkg") + if qual_alias not in extra_aliases: + extra_aliases[qual_alias] = [] + if hub_name not in extra_aliases[qual_alias]: + extra_aliases[qual_alias].append(hub_name) + + # Also add whl, data, dist_info aliases for the extra + for std_alias in ["whl", "data", "dist_info"]: + qual = "%s:%s" % (alias_name, std_alias) + if qual not in extra_aliases: + extra_aliases[qual] = [] + if hub_name not in extra_aliases[qual]: + extra_aliases[qual].append(hub_name) + + # Add pkg__ alias (no extras) + no_extras_alias = "%s__" % norm_pkg + qual_alias = "%s:%s" % (no_extras_alias, "pkg") + if qual_alias not in extra_aliases: + extra_aliases[qual_alias] = [] + if hub_name not in extra_aliases[qual_alias]: + extra_aliases[qual_alias].append(hub_name) + unified_hub_repo( name = "pypi", default_hub = mods.default_hub or (hubs[0] if hubs else ""),