feat: public description-prompt API and explicit generate_description behavior (0.0.6)

CHANGELOG.md

@@ -7,6 +7,52 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [0.0.6] - 2026-08-01
+
+### Added
+
+- `build_description_prompt(column, pipeline)` is now public API, exported from
+  the package root. It builds clgraph's lineage-aware column-description prompt
+  (including the column's SQL expression and upstream sources) so callers who
+  want to drive the LLM themselves - to control error handling, batching, or
+  model choice - can reuse it instead of reimplementing it. The former private
+  name `_build_description_prompt` remains as an alias.
+- `generate_description(...)` gained two keyword-only parameters:
+  - `overwrite=False` - by default a description that came from a SQL comment
+    (`description_source` is `SOURCE`) is left alone and the LLM is not called.
+    Pass `True` to describe the column anyway, which is what you want when
+    capturing a model's opinion *alongside* the authored text.
+  - `on_error="fallback"` - `"raise"` raises the new `DescriptionGenerationError`
+    instead of silently writing a rule-based description derived from the column
+    name. Use it when a silent fallback would be mistaken for real model output.
+- `generate_description(...)` now returns `bool`: `True` only when the LLM
+  produced the stored description, `False` when the column was skipped or a
+  fallback was written. Previously it returned `None`, so a caller could not
+  tell a successful generation from a fallback.
+- `DescriptionGenerationError`, raised by `on_error="raise"`.
+- `Pipeline.generate_all_descriptions()` and
+  `MetadataManager.generate_all_descriptions()` accept and forward the same
+  `overwrite` and `on_error` keyword-only parameters.
+- `generate_description` and `DescriptionGenerationError` are now exported from
+  the package root alongside `build_description_prompt`.
+
+### Fixed
+
+- Callers had no way to distinguish "the LLM wrote this description" from "the
+  LLM call failed and clgraph substituted the humanized column name" - both left
+  `description_source` set to `GENERATED`. Any tool attributing the result to a
+  model could therefore label a rule-based fallback, or a column's own
+  hand-authored SQL comment, as model-generated output. The new return value and
+  `on_error="raise"` make both cases detectable.
+
+### Compatibility
+
+No breaking changes. All new parameters are keyword-only with defaults that
+preserve the previous behavior exactly, the new return value replaces `None`
+(falsy either way), and the private prompt-builder name still resolves.
+
+## [0.0.5] - 2026-07-31
+
 ### Security
 
 - `Pipeline.from_sql_files()` and `Pipeline.from_json_file()` now validate paths:

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "clgraph"
-version = "0.0.5"
+version = "0.0.6"
 description = "Column lineage and pipeline dependency analysis for SQL"
 readme = "README.md"
 requires-python = ">=3.10"

src/clgraph/__init__.py

@@ -16,6 +16,11 @@
 # Import diff functionality
 # Import lineage intelligence components
 from .agent import AgentResult, LineageAgent, QuestionType
+from .column import (
+    DescriptionGenerationError,
+    build_description_prompt,
+    generate_description,
+)
 from .diff import ColumnDiff, PipelineDiff
 
 # Import execution functionality
@@ -126,6 +131,10 @@
     "TableNode",
     "TableDependencyGraph",
     "TemplateTokenizer",
+    # Description generation
+    "build_description_prompt",
+    "generate_description",
+    "DescriptionGenerationError",
     # Metadata
     "DescriptionSource",
     "PipelineDiff",

src/clgraph/column.py

@@ -30,21 +30,57 @@
 # ============================================================================
 
 
-def generate_description(column: ColumnNode, llm: Any, pipeline: "Pipeline"):
+class DescriptionGenerationError(RuntimeError):
+    """Raised when ``generate_description(..., on_error="raise")`` cannot obtain
+    a usable LLM description - because the call failed, or because the model's
+    output was rejected by output validation."""
+
+
+def generate_description(
+    column: ColumnNode,
+    llm: Any,
+    pipeline: "Pipeline",
+    *,
+    overwrite: bool = False,
+    on_error: str = "fallback",
+) -> bool:
     """
     Generate description using LLM based on SQL expression and source columns.
 
     Args:
         column: The column node to generate description for
         llm: LangChain LLM instance (BaseChatModel)
         pipeline: The pipeline for source lookup
+        overwrite: By default a description that came from a SQL comment
+            (``description_source`` is ``SOURCE``) is left alone and the LLM is
+            not called. Pass ``True`` to describe the column anyway - useful
+            when you want a model's opinion *alongside* the authored text and
+            will store it separately.
+        on_error: ``"fallback"`` (default) keeps the historical behavior: if the
+            LLM call fails or its output is rejected by validation, a rule-based
+            description derived from the column name is written instead.
+            ``"raise"`` raises :class:`DescriptionGenerationError` and leaves the
+            column untouched - use it when a silent fallback would be worse than
+            a visible failure, e.g. when the result is attributed to the model.
+
+    Returns:
+        ``True`` if the LLM produced the description now stored on the column.
+        ``False`` if the column was skipped, or a rule-based fallback was used.
+        A ``False`` return is the signal that the text is not model-authored.
+
+    Raises:
+        ValueError: If ``on_error`` is not ``"fallback"`` or ``"raise"``.
+        DescriptionGenerationError: If generation failed and ``on_error="raise"``.
     """
-    # Don't overwrite source descriptions
-    if column.description_source == DescriptionSource.SOURCE:
-        return
+    if on_error not in ("fallback", "raise"):
+        raise ValueError(f"on_error must be 'fallback' or 'raise', got {on_error!r}")
+
+    # Don't overwrite source descriptions unless explicitly asked to.
+    if not overwrite and column.description_source == DescriptionSource.SOURCE:
+        return False
 
     # Build prompt
-    prompt = _build_description_prompt(column, pipeline)
+    prompt = build_description_prompt(column, pipeline)
 
     # Call LLM
     try:
@@ -68,18 +104,41 @@ def generate_description(column: ColumnNode, llm: Any, pipeline: "Pipeline"):
 
         validated = _validate_description_output(raw, column.column_name, column.table_name)
         if validated is None:
+            if on_error == "raise":
+                raise DescriptionGenerationError(
+                    f"Generated description for {column.full_name} was rejected by "
+                    f"output validation"
+                )
             _generate_fallback_description(column)
-        else:
-            column.description = validated
-            column.description_source = DescriptionSource.GENERATED
+            return False
+        column.description = validated
+        column.description_source = DescriptionSource.GENERATED
+        return True
     except (ImportError, ValueError, AttributeError, RuntimeError) as e:
+        if on_error == "raise":
+            if isinstance(e, DescriptionGenerationError):
+                raise
+            raise DescriptionGenerationError(
+                f"Description generation failed for {column.full_name}: {e}"
+            ) from e
         # Fallback to simple rule-based description if LLM fails
         logger.debug("LLM description generation failed: %s", e)
         _generate_fallback_description(column)
+        return False
+
 
+def build_description_prompt(column: ColumnNode, pipeline: "Pipeline") -> str:
+    """Build the LLM prompt for a column description (sanitized + delimited).
 
-def _build_description_prompt(column: ColumnNode, pipeline: "Pipeline") -> str:
-    """Build LLM prompt for description generation (sanitized + delimited)."""
+    Public so callers that want to drive the model themselves - to control
+    error handling, batching, or which model is used - can reuse clgraph's
+    lineage-aware prompt instead of reimplementing it. The prompt includes the
+    column's SQL expression and its upstream source columns.
+
+    Pair it with your own LLM call when :func:`generate_description`'s
+    behavior does not fit; the returned string is ready to send as the user
+    message.
+    """
     from .prompt_sanitization import sanitize_for_prompt, sanitize_sql_for_prompt
 
     data_lines = [
@@ -120,6 +179,11 @@ def _build_description_prompt(column: ColumnNode, pipeline: "Pipeline") -> str:
     return "\n".join(data_lines + instructions)
 
 
+# Backwards-compatible alias: this function was private until it was promoted,
+# and downstream code imported the underscore name. Keep it working.
+_build_description_prompt = build_description_prompt
+
+
 def _generate_fallback_description(column: ColumnNode):
     """Generate simple fallback description without LLM"""
     # Humanize column name
@@ -461,7 +525,9 @@ def to_simplified(self) -> "PipelineLineageGraph":
 
 
 __all__ = [
+    "DescriptionGenerationError",
     "PipelineLineageGraph",
+    "build_description_prompt",
     "generate_description",
     "propagate_metadata",
 ]

src/clgraph/metadata_manager.py

@@ -56,7 +56,14 @@ def __init__(self, pipeline: "Pipeline"):
         """
         self._pipeline = pipeline
 
-    def generate_all_descriptions(self, batch_size: int = 10, verbose: bool = True):
+    def generate_all_descriptions(
+        self,
+        batch_size: int = 10,
+        verbose: bool = True,
+        *,
+        overwrite: bool = False,
+        on_error: str = "fallback",
+    ):
         """
         Generate descriptions for all columns using LLM.
 
@@ -65,6 +72,14 @@ def generate_all_descriptions(self, batch_size: int = 10, verbose: bool = True):
         Args:
             batch_size: Number of columns per batch (currently processes sequentially)
             verbose: If True, print progress messages
+            overwrite: By default only columns that have no description yet are
+                processed. Pass ``True`` to also re-describe columns that already
+                have one, including descriptions authored as SQL comments.
+            on_error: ``"fallback"`` (default) writes a rule-based description
+                when the LLM fails; ``"raise"`` propagates
+                :class:`~clgraph.column.DescriptionGenerationError` instead. Use
+                ``"raise"`` when a silent fallback would be mistaken for a real
+                model-generated description.
         """
         if not self._pipeline.llm:
             raise ValueError("LLM not configured. Set pipeline.llm before calling.")
@@ -79,7 +94,7 @@ def generate_all_descriptions(self, batch_size: int = 10, verbose: bool = True):
                 for col in self._pipeline.columns.values():
                     if (
                         col.table_name == query.destination_table
-                        and not col.description
+                        and (overwrite or not col.description)
                         and col.is_computed()
                     ):
                         columns_to_process.append(col)
@@ -91,7 +106,13 @@ def generate_all_descriptions(self, batch_size: int = 10, verbose: bool = True):
             if (i + 1) % batch_size == 0:
                 logger.info("Processed %d/%d columns...", i + 1, len(columns_to_process))
 
-            generate_description(col, self._pipeline.llm, self._pipeline)
+            generate_description(
+                col,
+                self._pipeline.llm,
+                self._pipeline,
+                overwrite=overwrite,
+                on_error=on_error,
+            )
 
         logger.info("Done! Generated %d descriptions", len(columns_to_process))
 

src/clgraph/pipeline.py

@@ -786,7 +786,14 @@ def get_lineage_path(
             to_column,
         )
 
-    def generate_all_descriptions(self, batch_size: int = 10, verbose: bool = True):
+    def generate_all_descriptions(
+        self,
+        batch_size: int = 10,
+        verbose: bool = True,
+        *,
+        overwrite: bool = False,
+        on_error: str = "fallback",
+    ):
         """
         Generate descriptions for all columns using LLM.
 
@@ -795,8 +802,16 @@ def generate_all_descriptions(self, batch_size: int = 10, verbose: bool = True):
         Args:
             batch_size: Number of columns per batch (currently processes sequentially)
             verbose: If True, print progress messages
-        """
-        return self._metadata_manager.generate_all_descriptions(batch_size, verbose)
+            overwrite: By default only columns that have no description yet are
+                processed. Pass ``True`` to also re-describe columns that already
+                have one, including descriptions authored as SQL comments.
+            on_error: ``"fallback"`` (default) writes a rule-based description
+                when the LLM fails; ``"raise"`` propagates
+                :class:`~clgraph.column.DescriptionGenerationError` instead.
+        """
+        return self._metadata_manager.generate_all_descriptions(
+            batch_size, verbose, overwrite=overwrite, on_error=on_error
+        )
 
     def propagate_all_metadata(self, verbose: bool = True):
         """