From 3c816eb140f8781630456f90f7e6814673219c5d Mon Sep 17 00:00:00 2001 From: Ming-Jer Lee Date: Sun, 2 Aug 2026 04:01:19 -0700 Subject: [PATCH] fix: import clgraph without PyYAML installed `clgraph/orchestrators/kestra.py` imported yaml at module scope, and `clgraph/orchestrators/__init__.py` imports every backend eagerly, so a bare `pip install clgraph` followed by `import clgraph` raised: ModuleNotFoundError: No module named 'yaml' PyYAML was never a declared dependency. It only ever arrived transitively in development environments, which is why the full suite and every CI job stayed green - all of them install ".[dev]". Bisected across published releases: 0.0.3 imports fine, 0.0.5 and 0.0.6 do not. PyYAML is now imported at point of use via `_require_yaml()`. KestraOrchestrator imports and constructs without it; only to_flow(), to_flow_with_triggers() and to_flow_dict() need it, and they raise an ImportError naming the package and the install command. Airflow, Dagster, Prefect and Mage emit code as text and were never affected. Also adds: - a `clgraph[kestra]` extra, so the error message's install hint is real - a `bare-install` CI job that installs the built wheel into a clean environment with no extras and imports it - nothing in the pipeline would have caught this otherwise - regression tests that run in a subprocess with yaml made unimportable, since the dev environment has PyYAML and the failure only reproduces without it Bumps to 0.0.7. --- .github/workflows/ci.yml | 66 +++++++++++++ CHANGELOG.md | 33 +++++++ README.md | 8 ++ pyproject.toml | 9 +- src/clgraph/orchestrators/kestra.py | 26 ++++- tests/test_optional_orchestrator_deps.py | 120 +++++++++++++++++++++++ uv.lock | 8 +- 7 files changed, 263 insertions(+), 7 deletions(-) create mode 100644 tests/test_optional_orchestrator_deps.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f4176e2..26488e8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -59,6 +59,72 @@ jobs: - name: Run tests run: uv run pytest tests/ -v --cov=src/clgraph --cov-report=term-missing + bare-install: + # Every other job installs ".[dev]", so an optional dependency that leaks + # into an import path is invisible to them. This installs the built wheel + # into a clean environment with no extras - what `pip install clgraph` + # actually gives a user. A missing PyYAML broke `import clgraph` outright + # in 0.0.5 and 0.0.6 while CI stayed green. + name: Bare install (no extras) + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ['3.10', '3.13'] + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Install uv + run: pip install uv + + - name: Build the wheel + run: uv build --wheel + + - name: Install the wheel with no extras + run: | + python -m venv /tmp/bare + /tmp/bare/bin/pip install --upgrade pip + /tmp/bare/bin/pip install dist/*.whl + + - name: Import clgraph and build a pipeline + run: | + /tmp/bare/bin/python - <<'PY' + import clgraph + from clgraph import Pipeline + + print("clgraph", clgraph.__version__) + pipeline = Pipeline( + [("q", "CREATE TABLE mart_orders AS SELECT id, amount FROM raw_orders")], + dialect="bigquery", + ) + assert pipeline.columns, "pipeline produced no columns" + print("columns:", len(pipeline.columns)) + PY + + - name: Optional dependencies must fail only at point of use + run: | + /tmp/bare/bin/python - <<'PY' + from clgraph import Pipeline + from clgraph.orchestrators import KestraOrchestrator + + pipeline = Pipeline( + [("q", "CREATE TABLE mart_orders AS SELECT id FROM raw_orders")], + dialect="bigquery", + ) + try: + KestraOrchestrator(pipeline).to_flow(flow_id="f", namespace="n") + except ImportError as exc: + assert "pyyaml" in str(exc).lower(), f"unhelpful message: {exc}" + print("Kestra without PyYAML raised as expected:", exc) + else: + raise SystemExit("expected ImportError naming PyYAML") + PY + notebooks: name: Example Notebooks runs-on: ubuntu-latest diff --git a/CHANGELOG.md b/CHANGELOG.md index 5eb7b01..b81c77e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,39 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.0.7] - 2026-08-02 + +### Fixed + +- **`import clgraph` failed on a clean install.** `clgraph.orchestrators.kestra` + imported `yaml` at module scope and `clgraph/orchestrators/__init__.py` imports + every backend eagerly, so `import clgraph` raised + `ModuleNotFoundError: No module named 'yaml'` for anyone who installed clgraph + without extras. PyYAML was never a declared dependency - it only ever arrived + transitively in development environments, which is why the full test suite and + CI stayed green. Regression introduced in 0.0.5 and also present in 0.0.6; + 0.0.3 was unaffected. + + PyYAML is now imported at point of use. `KestraOrchestrator` imports and + constructs without it; only `to_flow()`, `to_flow_with_triggers()` and + `to_flow_dict()` need it, and they raise an `ImportError` naming the package + and how to install it. The other orchestrators (Airflow, Dagster, Prefect, + Mage) emit code as text and were never affected. + +### Added + +- `clgraph[kestra]` extra, which installs PyYAML. +- CI job `bare-install`, which installs the built wheel into a clean environment + with no extras and imports it. Every other job installs `.[dev]`, so nothing + in the pipeline would have caught this class of bug. +- Regression tests in `tests/test_optional_orchestrator_deps.py` that run in a + subprocess with `yaml` made unimportable, since the development environment + has PyYAML installed and the failure only reproduces without it. + +### Compatibility + +No API changes. Anyone who already has PyYAML installed sees identical behavior. + ## [0.0.6] - 2026-08-01 ### Added diff --git a/README.md b/README.md index 4be0432..401d401 100644 --- a/README.md +++ b/README.md @@ -74,6 +74,14 @@ Or with uv: uv pip install clgraph ``` +Optional extras, for features with extra dependencies: + +```bash +pip install 'clgraph[llm]' # LLM-powered descriptions, text-to-SQL, agent +pip install 'clgraph[mcp]' # MCP server for Claude Desktop +pip install 'clgraph[kestra]' # Kestra flow generation (PyYAML) +``` + ## Quick Start ### Single Query Column Lineage diff --git a/pyproject.toml b/pyproject.toml index f6ead9c..dd7be68 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "clgraph" -version = "0.0.6" +version = "0.0.7" description = "Column lineage and pipeline dependency analysis for SQL" readme = "README.md" requires-python = ">=3.10" @@ -48,6 +48,13 @@ llm = [ mcp = [ "fastmcp>=3.2.0", ] +# Kestra orchestrator support (KestraOrchestrator / Pipeline.to_kestra_flow). +# The other orchestrators emit code as text and need nothing extra; Kestra is +# the only one that serializes YAML, so PyYAML lives here rather than in the +# core dependencies. +kestra = [ + "pyyaml>=6.0", +] dev = [ "pytest>=7.0.0", "pytest-cov>=4.0.0", diff --git a/src/clgraph/orchestrators/kestra.py b/src/clgraph/orchestrators/kestra.py index 259aa8e..192a5d8 100644 --- a/src/clgraph/orchestrators/kestra.py +++ b/src/clgraph/orchestrators/kestra.py @@ -7,14 +7,31 @@ from typing import TYPE_CHECKING, Any, Dict, Optional -import yaml - from .base import BaseOrchestrator if TYPE_CHECKING: pass +def _require_yaml(): + """Import PyYAML on demand, with an actionable message when it is absent. + + PyYAML is an optional dependency: only this orchestrator needs it, and + ``clgraph.orchestrators`` imports every backend eagerly, so importing it at + module scope would make ``import clgraph`` fail outright for anyone who + installed clgraph without extras. Importing it here keeps the cost on the + people who actually generate Kestra flows. + """ + try: + import yaml + except ImportError as exc: # pragma: no cover - exercised in a subprocess + raise ImportError( + "KestraOrchestrator requires PyYAML, which is not installed. " + "Install it with `pip install 'clgraph[kestra]'` or `pip install pyyaml`." + ) from exc + return yaml + + class KestraOrchestrator(BaseOrchestrator): """ Converts clgraph pipelines to Kestra YAML flows. @@ -149,7 +166,7 @@ def to_flow( # Add any additional kwargs flow.update(kwargs) - return yaml.dump(flow, default_flow_style=False, sort_keys=False) + return _require_yaml().dump(flow, default_flow_style=False, sort_keys=False) def to_flow_with_triggers( self, @@ -185,6 +202,7 @@ def to_flow_with_triggers( cron="0 * * * *" ) """ + yaml = _require_yaml() flow_yaml = self.to_flow(flow_id=flow_id, namespace=namespace, **kwargs) flow_dict = yaml.safe_load(flow_yaml) @@ -220,7 +238,7 @@ def to_flow_dict( Dictionary representing Kestra flow structure """ yaml_content = self.to_flow(flow_id=flow_id, namespace=namespace, **kwargs) - return yaml.safe_load(yaml_content) + return _require_yaml().safe_load(yaml_content) __all__ = ["KestraOrchestrator"] diff --git a/tests/test_optional_orchestrator_deps.py b/tests/test_optional_orchestrator_deps.py new file mode 100644 index 0000000..ebe0903 --- /dev/null +++ b/tests/test_optional_orchestrator_deps.py @@ -0,0 +1,120 @@ +"""Importing clgraph must not require optional orchestrator dependencies. + +Regression tests for a bug shipped in 0.0.5: ``clgraph.orchestrators.kestra`` +did ``import yaml`` at module scope, and ``orchestrators/__init__.py`` imports +every backend eagerly, so a bare ``pip install clgraph`` followed by +``import clgraph`` raised ``ModuleNotFoundError: No module named 'yaml'``. +PyYAML is not a declared dependency; it only ever arrived transitively in +development environments, which is why the whole test suite stayed green. + +These tests run in a subprocess with ``yaml`` made unimportable, because the +development environment genuinely has PyYAML installed - the failure only +reproduces when the module is absent. +""" + +import subprocess +import sys + +import pytest + +# Installed ahead of every import so `yaml` is unavailable even to modules that +# have not been loaded yet. A meta_path finder is used rather than deleting +# sys.modules entries, which a later import would simply repopulate. +_BLOCK_YAML = """ +import sys + + +class _BlockYaml: + def find_spec(self, name, path=None, target=None): + if name == "yaml" or name.startswith("yaml."): + raise ImportError("No module named 'yaml'") + return None + + +sys.meta_path.insert(0, _BlockYaml()) +for _mod in [m for m in list(sys.modules) if m == "yaml" or m.startswith("yaml.")]: + del sys.modules[_mod] +""" + + +def _run_without_yaml(code: str) -> subprocess.CompletedProcess: + return subprocess.run( + [sys.executable, "-c", _BLOCK_YAML + code], + capture_output=True, + text=True, + timeout=120, + ) + + +def test_yaml_blocker_actually_blocks(): + """Guard the guard: if the blocker stopped working these tests would pass + vacuously against a dev environment that has PyYAML installed.""" + result = _run_without_yaml("import yaml") + assert result.returncode != 0 + assert "No module named 'yaml'" in result.stderr + + +def test_import_clgraph_without_pyyaml(): + result = _run_without_yaml("import clgraph; print(clgraph.__version__)") + assert result.returncode == 0, f"importing clgraph needs PyYAML:\n{result.stderr}" + + +def test_import_orchestrators_package_without_pyyaml(): + result = _run_without_yaml( + "from clgraph.orchestrators import AirflowOrchestrator, DagsterOrchestrator; print('ok')" + ) + assert result.returncode == 0, result.stderr + assert "ok" in result.stdout + + +def test_kestra_orchestrator_is_importable_without_pyyaml(): + """The class must import; only *using* it should need PyYAML.""" + result = _run_without_yaml( + "from clgraph.orchestrators import KestraOrchestrator; print(KestraOrchestrator.__name__)" + ) + assert result.returncode == 0, result.stderr + assert "KestraOrchestrator" in result.stdout + + +@pytest.mark.parametrize( + "method_call", + [ + 'k.to_flow(flow_id="f", namespace="n")', + 'k.to_flow_with_triggers(flow_id="f", namespace="n", cron="0 0 * * *")', + "k.to_flow_dict(flow_id='f', namespace='n')", + ], +) +def test_using_kestra_without_pyyaml_raises_actionable_error(method_call): + """A missing optional dependency must name itself and how to install it, + rather than surfacing a bare ModuleNotFoundError from deep in the stack.""" + result = _run_without_yaml(f""" +from clgraph import Pipeline +from clgraph.orchestrators import KestraOrchestrator + +p = Pipeline([("q", "CREATE TABLE mart_orders AS SELECT id FROM raw_orders")], dialect="bigquery") +k = KestraOrchestrator(p) +try: + {method_call} +except ImportError as exc: + print("RAISED:", exc) +else: + print("NO ERROR RAISED") +""") + assert result.returncode == 0, result.stderr + assert "RAISED:" in result.stdout, result.stdout + message = result.stdout.split("RAISED:", 1)[1].lower() + assert "pyyaml" in message + assert "kestra" in message + + +def test_pipeline_works_end_to_end_without_pyyaml(): + """The core product - lineage - must not be collateral damage.""" + result = _run_without_yaml(""" +from clgraph import Pipeline + +p = Pipeline([("q", "CREATE TABLE mart_orders AS SELECT id, amount FROM raw_orders")], dialect="bigquery") +print("COLUMNS:", len(p.columns)) +""") + assert result.returncode == 0, result.stderr + assert "COLUMNS:" in result.stdout + assert int(result.stdout.split("COLUMNS:")[1].strip()) > 0 diff --git a/uv.lock b/uv.lock index de20b20..82a8993 100644 --- a/uv.lock +++ b/uv.lock @@ -811,7 +811,7 @@ wheels = [ [[package]] name = "clgraph" -version = "0.0.6" +version = "0.0.7" source = { editable = "." } dependencies = [ { name = "graphviz" }, @@ -859,6 +859,9 @@ examples = [ { name = "pandas" }, { name = "streamlit" }, ] +kestra = [ + { name = "pyyaml" }, +] llm = [ { name = "langchain" }, { name = "langchain-core" }, @@ -899,6 +902,7 @@ requires-dist = [ { name = "pandas", marker = "extra == 'examples'", specifier = ">=2.0.0" }, { name = "pytest", marker = "extra == 'dev'", specifier = ">=7.0.0" }, { name = "pytest-cov", marker = "extra == 'dev'", specifier = ">=4.0.0" }, + { name = "pyyaml", marker = "extra == 'kestra'", specifier = ">=6.0" }, { name = "rich", specifier = ">=13.0.0" }, { name = "ruff", marker = "extra == 'dev'", specifier = ">=0.16.1,<0.17" }, { name = "sqlglot", specifier = ">=28.0.0,<31.0.0" }, @@ -907,7 +911,7 @@ requires-dist = [ { name = "ty", marker = "extra == 'dev'", specifier = ">=0.0.1a0" }, { name = "typer", specifier = ">=0.12.0" }, ] -provides-extras = ["llm", "mcp", "dev", "build", "examples", "templates", "airflow", "all"] +provides-extras = ["llm", "mcp", "kestra", "dev", "build", "examples", "templates", "airflow", "all"] [[package]] name = "click"