From 5cc0602a4da0403fb1e6b5f117a44082fc63e7f6 Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Fri, 26 Jun 2026 05:20:25 +0000 Subject: [PATCH 1/3] feat(pypi): add pip.dep to declare abstract pypi dependencies Introduce the pip.dep tag class to allow modules to declare abstract PyPI dependencies. These dependencies are fed directly into the unified hub repository, ensuring their target structures exist and automatically routing any unimplemented declarations to analysis-time errors. --- .agents/rules/bzl.md | 8 ++++ docs/pypi/download.md | 35 ++++++++++++++ news/pip-dep-tag-class.added.md | 4 ++ python/private/pypi/extension.bzl | 46 +++++++++++++++++++ tests/integration/unified_pypi/BUILD.bazel | 13 ++++++ tests/integration/unified_pypi/MODULE.bazel | 4 ++ .../unified_pypi/bin_declared_only.py | 2 + tests/integration/unified_pypi_test.py | 21 +++++++++ tests/pypi/extension/extension_tests.bzl | 39 +++++++++++++++- 9 files changed, 171 insertions(+), 1 deletion(-) create mode 100644 .agents/rules/bzl.md create mode 100644 news/pip-dep-tag-class.added.md create mode 100644 tests/integration/unified_pypi/bin_declared_only.py diff --git a/.agents/rules/bzl.md b/.agents/rules/bzl.md new file mode 100644 index 0000000000..104c52c590 --- /dev/null +++ b/.agents/rules/bzl.md @@ -0,0 +1,8 @@ +--- +trigger: glob +description: Starlark / Bazel .bzl file coding style rules +globs: *.bzl +--- + +* Use triple-quoted strings for multi-line rule doc args. +* Don't use backslash line continuation in rule doc args. diff --git a/docs/pypi/download.md b/docs/pypi/download.md index 6705df1f3a..ebdb63bf5f 100644 --- a/docs/pypi/download.md +++ b/docs/pypi/download.md @@ -121,6 +121,41 @@ Shared library targets can simply depend on the unified hub (e.g., `@pypi//numpy`), and the dependency will automatically resolve to the correct wheel version from the active hub during the build. +### Declaring Abstract Dependencies (pip.dep) + +:::{versionadded} VERSION_NEXT_FEATURE +Declaring abstract PyPI dependencies via `pip.dep` tags. +::: + +Sometimes a shared library target or a ruleset needs to depend on a PyPI +package (e.g., `@pypi//numpy`), but does not want to force a specific package +version or a concrete `requirements.txt` lock file on its consumers. + +Instead of calling `pip.parse()`, the module can declare its dependency using +the `pip.dep` tag: + +```starlark +pip = use_extension("@rules_python//python/extensions:pip.bzl", "pip") + +# Declare an abstract dependency on 'numpy' and specify extra targets that +# are expected to be available in the package. +pip.dep( + name = "numpy", + extra_targets = ["extra-alias"], +) +``` + +This ensures that the target structure `@pypi//numpy` (and +`@pypi//numpy:extra-alias`) exists in the unified `@pypi` hub repository, so the +declaring module can compile and analyze successfully without needing any local +requirements file. + +The actual concrete implementation and version of the package must be provided +by a downstream module calling `pip.parse`. + +If a downstream module attempts to build a target that depends on an abstract +dependency, but has not provided a concrete implementation for it via any +`pip.parse` call, the build will fail at execution time. As with any repository rule or extension, if you would like to ensure that `pip_parse` is diff --git a/news/pip-dep-tag-class.added.md b/news/pip-dep-tag-class.added.md new file mode 100644 index 0000000000..5306186081 --- /dev/null +++ b/news/pip-dep-tag-class.added.md @@ -0,0 +1,4 @@ +(pypi) Added a `dep` tag class to the `pip` bzlmod extension. This allows +modules to declare abstract PyPI dependencies, ensuring target structures +exist in the unified hub, while allowing other modules to provide the +concrete implementation via `pip.parse`. diff --git a/python/private/pypi/extension.bzl b/python/private/pypi/extension.bzl index 62d2c8de16..a3e775e08b 100644 --- a/python/private/pypi/extension.bzl +++ b/python/private/pypi/extension.bzl @@ -424,6 +424,15 @@ You cannot use both the additive_build_content and additive_build_content_file a pip_attr = pip_attr, ) + # dict[str package, dict[str, None] extra_targets] + declared_deps = {} + for mod in module_ctx.modules: + for dep_attr in mod.tags.dep: + name = normalize_name(dep_attr.name) + targets = declared_deps.setdefault(name, {}) + for target in dep_attr.extra_targets: + targets[target] = None + # Keeps track of all the hub's whl repos across the different versions. # dict[hub, dict[whl, dict[version, str pip]]] # Where hub, whl, and pip are the repo names @@ -448,6 +457,7 @@ You cannot use both the additive_build_content and additive_build_content_file a return struct( config = config, + declared_deps = declared_deps, default_hub = config.default_hub or renamed_default_hub, exposed_packages = exposed_packages, extra_aliases = extra_aliases, @@ -492,6 +502,16 @@ def _create_unified_hub_repo(mods): if hub_name not in extra_aliases[qual_alias]: extra_aliases[qual_alias].append(hub_name) + for dep_name, extra_targets in mods.declared_deps.items(): + norm_pkg = normalize_name(dep_name) + if norm_pkg not in packages: + packages[norm_pkg] = [] + + for target_name in extra_targets: + qual_alias = "%s:%s" % (norm_pkg, target_name) + if qual_alias not in extra_aliases: + extra_aliases[qual_alias] = [] + unified_hub_repo( name = "pypi", default_hub = mods.default_hub or (hubs[0] if hubs else ""), @@ -1015,6 +1035,31 @@ Apply any overrides (e.g. patches) to a given Python distribution defined by other tags in this extension.""", ) +_dep_tag = tag_class( + attrs = { + "extra_targets": attr.string_list( + doc = """\ +A list of extra target names in the package that are expected to be available. +See {obj}`pip.parse.extra_hub_aliases`. +""", + default = [], + ), + "name": attr.string( + doc = "The name of a pypi package. Note that the name is normalized.", + mandatory = True, + ), + }, + doc = """\ +Declare an abstract PyPI dependency to ensure its target structure exists in the unified hub. + +This is useful for targets or rules that need to depend on a package (e.g., `@pypi//numpy`) +but do not want to force a specific version or concrete requirements lock file on their +consumers. The concrete version and implementation must be provided by downstreams calling +`pip.parse`. If they are not, the target will still be defined, but it will result in an +execution-phase error when built. +""", +) + pypi = module_extension( environ = ["RULES_PYTHON_PYPI_HUB_RESERVED"], doc = """\ @@ -1065,6 +1110,7 @@ terms used in this extension. ::: """, ), + "dep": _dep_tag, "override": _override_tag, "parse": tag_class( attrs = _pip_parse_ext_attrs(), diff --git a/tests/integration/unified_pypi/BUILD.bazel b/tests/integration/unified_pypi/BUILD.bazel index 8a37d4b153..c0b9905fa6 100644 --- a/tests/integration/unified_pypi/BUILD.bazel +++ b/tests/integration/unified_pypi/BUILD.bazel @@ -46,3 +46,16 @@ py_binary( }, deps = ["@pypi//six"], ) + +py_binary( + name = "bin_declared_only", + srcs = ["bin_declared_only.py"], + deps = ["@pypi//declared_only_pkg"], +) + +py_binary( + name = "bin_declared_only_alias", + srcs = ["bin_declared_only.py"], + main = "bin_declared_only.py", + deps = ["@pypi//declared_only_pkg:declared-only-alias"], +) diff --git a/tests/integration/unified_pypi/MODULE.bazel b/tests/integration/unified_pypi/MODULE.bazel index 0d0f44f61c..6a4b87e015 100644 --- a/tests/integration/unified_pypi/MODULE.bazel +++ b/tests/integration/unified_pypi/MODULE.bazel @@ -45,4 +45,8 @@ pip.parse( use_repo(pip, "pypi_b") pip.default(default_hub = "pypi_b") +pip.dep( + name = "declared-only-pkg", + extra_targets = ["declared-only-alias"], +) use_repo(pip, "pypi") diff --git a/tests/integration/unified_pypi/bin_declared_only.py b/tests/integration/unified_pypi/bin_declared_only.py new file mode 100644 index 0000000000..0b03607e77 --- /dev/null +++ b/tests/integration/unified_pypi/bin_declared_only.py @@ -0,0 +1,2 @@ +# Dummy file for integration test +print("declared_only") diff --git a/tests/integration/unified_pypi_test.py b/tests/integration/unified_pypi_test.py index 707ab13444..1b4aee4bef 100644 --- a/tests/integration/unified_pypi_test.py +++ b/tests/integration/unified_pypi_test.py @@ -74,6 +74,27 @@ def test_invalid_default_hub_fails_evaluation(self): "default_hub 'invalid_hub' is not a defined PyPI hub", ) + def test_unimplemented_declared_dep_fails_build(self): + # Even though cquery succeeds: + self.run_bazel("cquery", "//:bin_declared_only") + + # Build must fail because the package is not implemented by any concrete hub + result = self.run_bazel("build", "//:bin_declared_only", check=False) + self.assertNotEqual(result.exit_code, 0) + self.assert_result_matches( + result, + 'ERROR: PyPI package "declared_only_pkg" is not available when building under PyPI hub "pypi_b".', + ) + + def test_unimplemented_declared_dep_alias_fails_build(self): + # Build must fail for alias too + result = self.run_bazel("build", "//:bin_declared_only_alias", check=False) + self.assertNotEqual(result.exit_code, 0) + self.assert_result_matches( + result, + 'ERROR: PyPI package "declared_only_pkg:declared-only-alias" is not available when building under PyPI hub "pypi_b".', + ) + if __name__ == "__main__": unittest.main() diff --git a/tests/pypi/extension/extension_tests.bzl b/tests/pypi/extension/extension_tests.bzl index bc4c0bcb5b..143f06a471 100644 --- a/tests/pypi/extension/extension_tests.bzl +++ b/tests/pypi/extension/extension_tests.bzl @@ -90,7 +90,13 @@ _default_tags_default = [ }.items() ] -def _mod(*, name, default = _default_tags_default, parse = [], override = [], whl_mods = [], is_root = True): +def _dep(*, name, extra_targets = []): + return struct( + name = name, + extra_targets = extra_targets, + ) + +def _mod(*, name, default = _default_tags_default, parse = [], override = [], whl_mods = [], dep = [], is_root = True): return struct( name = name, tags = struct( @@ -98,6 +104,7 @@ def _mod(*, name, default = _default_tags_default, parse = [], override = [], wh override = override, whl_mods = whl_mods, default = default, + dep = dep, ), is_root = is_root, ) @@ -106,6 +113,7 @@ def _parse_modules(env, **kwargs): return env.expect.that_struct( parse_modules(**kwargs), attrs = dict( + declared_deps = subjects.dict, default_hub = subjects.str, exposed_packages = subjects.dict, hub_group_map = subjects.dict, @@ -435,6 +443,35 @@ def _test_default_hub_precedence(env): _tests.append(_test_default_hub_precedence) +def _test_extension_dep(env): + pypi = _parse_modules( + env, + module_ctx = _pypi_mock_mctx( + _mod( + name = "my_module", + dep = [ + _dep( + name = "declared-pkg", + extra_targets = ["declared-alias"], + ), + ], + ), + os_name = "linux", + arch_name = "x86_64", + ), + available_interpreters = {}, + minor_mapping = {}, + ) + + pypi.declared_deps().contains_exactly({"declared_pkg": {"declared-alias": None}}) + pypi.exposed_packages().contains_exactly({}) + pypi.hub_group_map().contains_exactly({}) + pypi.hub_whl_map().contains_exactly({}) + pypi.whl_libraries().contains_exactly({}) + pypi.whl_mods().contains_exactly({}) + +_tests.append(_test_extension_dep) + def extension_test_suite(name): """Create the test suite. From 32ff95b5be45144b56c7bcac7edda2bbe2e70a24 Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Sun, 28 Jun 2026 22:16:48 +0000 Subject: [PATCH 2/3] test(pypi): add unit test for pip.dep coexisting with concrete hub Simplify the declared_deps loop by removing a redundant normalization call. Add a unit test to verify that when a package is declared via pip.dep and also provided by a concrete hub, the concrete hub implementation coexists correctly and takes precedence. --- python/private/pypi/extension.bzl | 3 +- tests/pypi/extension/extension_tests.bzl | 54 ++++++++++++++++++++++++ 2 files changed, 55 insertions(+), 2 deletions(-) diff --git a/python/private/pypi/extension.bzl b/python/private/pypi/extension.bzl index a3e775e08b..90bb8804ad 100644 --- a/python/private/pypi/extension.bzl +++ b/python/private/pypi/extension.bzl @@ -502,8 +502,7 @@ def _create_unified_hub_repo(mods): if hub_name not in extra_aliases[qual_alias]: extra_aliases[qual_alias].append(hub_name) - for dep_name, extra_targets in mods.declared_deps.items(): - norm_pkg = normalize_name(dep_name) + for norm_pkg, extra_targets in mods.declared_deps.items(): if norm_pkg not in packages: packages[norm_pkg] = [] diff --git a/tests/pypi/extension/extension_tests.bzl b/tests/pypi/extension/extension_tests.bzl index 143f06a471..c2e0c8b60f 100644 --- a/tests/pypi/extension/extension_tests.bzl +++ b/tests/pypi/extension/extension_tests.bzl @@ -472,6 +472,60 @@ def _test_extension_dep(env): _tests.append(_test_extension_dep) +def _test_extension_dep_coexists_with_concrete_hub(env): + pypi = _parse_modules( + env, + module_ctx = _pypi_mock_mctx( + _mod( + name = "my_module", + parse = [ + _parse( + hub_name = "pypi_a", + python_version = "3.15", + simpleapi_skip = ["simple"], + requirements_lock = "requirements.txt", + ), + ], + dep = [ + _dep( + name = "simple", + extra_targets = ["extra-target"], + ), + ], + ), + os_name = "linux", + arch_name = "x86_64", + ), + available_interpreters = { + "python_3_15_host": "unit_test_interpreter_target", + }, + minor_mapping = {"3.15": "3.15.19"}, + ) + + pypi.declared_deps().contains_exactly({"simple": {"extra-target": None}}) + pypi.exposed_packages().contains_exactly({"pypi_a": ["simple"]}) + pypi.hub_group_map().contains_exactly({"pypi_a": {}}) + pypi.hub_whl_map().contains_exactly({"pypi_a": { + "simple": { + "pypi_a_315_simple": [ + whl_config_setting( + version = "3.15", + ), + ], + }, + }}) + pypi.whl_libraries().contains_exactly({ + "pypi_a_315_simple": { + "config_load": "@pypi_a//:config.bzl", + "dep_template": "@pypi_a//{name}:{target}", + "python_interpreter_target": "unit_test_interpreter_target", + "requirement": "simple==0.0.1 --hash=sha256:deadbeef --hash=sha256:deadbaaf", + }, + }) + pypi.whl_mods().contains_exactly({}) + +_tests.append(_test_extension_dep_coexists_with_concrete_hub) + def extension_test_suite(name): """Create the test suite. From 2e5ecf634d5ad0b02eefe8fe20b87541689f5041 Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Sun, 28 Jun 2026 22:21:46 +0000 Subject: [PATCH 3/3] chore(agents): add title to Starlark rules file Add the missing H1 title to the Starlark rules file to ensure it conforms to the workspace rule formatting guidelines. --- .agents/rules/bzl.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.agents/rules/bzl.md b/.agents/rules/bzl.md index 104c52c590..10aca72184 100644 --- a/.agents/rules/bzl.md +++ b/.agents/rules/bzl.md @@ -4,5 +4,7 @@ description: Starlark / Bazel .bzl file coding style rules globs: *.bzl --- +# Starlark Rules + * Use triple-quoted strings for multi-line rule doc args. * Don't use backslash line continuation in rule doc args.