feat: let any package ship extension migrations

docs/changelog.rst

@@ -9,6 +9,39 @@ important operational fixes.
 Recent Updates
 ==============
 
+v0.57.0
+------------------------------------------------------------------------------
+
+**Added:**
+
+* Packages distributed separately from SQLSpec can now ship migrations. Set
+  ``migrations_path`` in an ``extension_config`` entry to point at a directory or
+  a ``'<dotted.module>:<subdir>'`` specification, and the extension is discovered
+  and auto-included without appearing in ``include_extensions``.
+* Added ``add_extension_migrations(name, migrations_path, settings=None)`` on
+  database configurations, for packages that register migrations at runtime
+  rather than declaratively.
+
+**Fixed:**
+
+* Extension names containing underscores no longer lose their version. A
+  migration such as ``ext_my_extension_0001_init.sql`` previously resolved to the
+  version ``ext_my_extension``, dropping ``0001`` and recording a malformed
+  version in the migration tracking table.
+* A configured extension that cannot be resolved now reports which module was
+  tried and that no migrations were registered, instead of the ambiguous
+  ``Extension <name> not found``.
+* Extensions that ship no migrations directory no longer log a warning. Six of
+  the bundled extensions have no migrations by design, so the warning was noise.
+* :func:`sqlspec.utils.module_loader.module_to_os_path` resolves namespace
+  packages to their search location instead of returning a path named ``None``.
+
+**Changed:**
+
+* :func:`sqlspec.utils.module_loader.module_to_os_path` raises
+  :class:`ModuleNotFoundError` rather than :class:`TypeError` when a module
+  cannot be found, so callers can catch the real condition.
+
 v0.56.2
 ------------------------------------------------------------------------------
 

docs/usage/migrations.rst

@@ -15,6 +15,7 @@ Core Concepts
 - Migrations are SQL or Python files stored in a migrations directory.
 - Each database configuration can include its own migration settings.
 - Extension migrations (ADK, events, Litestar sessions) are opt-in and versioned.
+- Any installed package can ship migrations, not just those under ``sqlspec.extensions``.
 
 Common Commands
 ---------------
@@ -74,6 +75,65 @@ specific extension, ``migration_config["include_extensions"]`` to opt in
 explicitly by extension name, or ``migration_config["enabled"] = False`` to
 disable migrations entirely for a database config.
 
+Third-Party Extension Migrations
+--------------------------------
+
+Extension names resolve against the ``sqlspec.extensions.<name>`` namespace by
+default. A package distributed separately from SQLSpec points at its own
+migrations directory with ``migrations_path``:
+
+.. code-block:: python
+
+    config = AsyncpgConfig(
+        connection_config={"dsn": "postgresql://localhost/app"},
+        extension_config={
+            "litestar_queues": {
+                "migrations_path": "litestar_queues.backends.sqlspec:migrations",
+                "table_name": "queue_tasks",
+            }
+        },
+    )
+
+Declaring ``migrations_path`` auto-includes the extension, so it does not also
+need to appear in ``include_extensions``. ``exclude_extensions`` still opts it
+back out.
+
+``migrations_path`` accepts either form:
+
+- A ``'<dotted.module>:<subdir>'`` specification, resolved against the installed
+  package. Portable across machines, so this is the form to use in
+  ``[tool.sqlspec]`` configuration.
+- A filesystem path, absolute or relative to the working directory, matching how
+  ``script_location`` resolves.
+
+Packages that register migrations at runtime can call
+``add_extension_migrations`` instead of declaring the key:
+
+.. code-block:: python
+
+    from pathlib import Path
+
+    config.add_extension_migrations(
+        "litestar_queues",
+        Path(__file__).parent / "migrations",
+        settings={"table_name": "queue_tasks"},
+    )
+
+Both forms record the extension under ``extension_config`` and opt it into
+``include_extensions``. Call ``add_extension_migrations`` before
+``get_migration_commands()``; mutating ``extension_config`` directly after the
+configuration is built does not re-run discovery.
+
+.. note::
+
+   Migrations are versioned under an ``ext_{name}_`` prefix, and that prefix is
+   written to the migration tracking table. Keep the extension name stable once
+   migrations have been applied — renaming it orphans the applied records.
+
+   A package shipping migrations must include the directory as package data. If
+   it compiles its own modules, the migration sources must remain on disk, since
+   Python migrations are read and compiled at runtime.
+
 Configuring a Default Schema
 ----------------------------
 

sqlspec/config.py

@@ -11,7 +11,7 @@
 import asyncio
 import threading
 from abc import ABC, abstractmethod
-from collections.abc import Callable
+from collections.abc import Callable, Mapping
 from inspect import Signature, signature
 from pathlib import Path
 from typing import TYPE_CHECKING, Any, ClassVar, Generic, Literal, TypeAlias, TypeVar, cast
@@ -145,7 +145,8 @@ class MigrationConfig(TypedDict):
     """List of extension names whose migrations should be included. Extension migrations maintain separate versioning and are prefixed with 'ext_{name}_'.
 
     Note: Extensions with migration support (litestar, adk, events) are auto-included when
-    their settings are present in ``extension_config``. Use ``exclude_extensions`` to opt out.
+    their settings are present in ``extension_config``, as is any extension whose settings
+    declare ``migrations_path``. Use ``exclude_extensions`` to opt out.
     """
 
     exclude_extensions: NotRequired["list[str]"]
@@ -238,6 +239,13 @@ class LitestarConfig(TypedDict):
     All fields are optional with sensible defaults.
     """
 
+    migrations_path: NotRequired[str | Path]
+    """Directory containing this extension's migrations, or a ``'<dotted.module>:<subdir>'`` specification.
+
+    Overrides the default ``sqlspec.extensions.<name>`` lookup. Setting this auto-includes the
+    extension in ``migration_config["include_extensions"]``.
+    """
+
     session_table: NotRequired["bool | str"]
     """Enable session table for server-side session storage.
 
@@ -476,6 +484,13 @@ class ADKConfig(TypedDict):
         3. Selective features (sessions OR memory, not both)
     """
 
+    migrations_path: NotRequired[str | Path]
+    """Directory containing this extension's migrations, or a ``'<dotted.module>:<subdir>'`` specification.
+
+    Overrides the default ``sqlspec.extensions.<name>`` lookup. Setting this auto-includes the
+    extension in ``migration_config["include_extensions"]``.
+    """
+
     manage_schema: NotRequired[bool]
     """Apply additive target-schema reconciliation. Default: True."""
 
@@ -591,6 +606,13 @@ class EventsConfig(TypedDict):
     Use in ``extension_config["events"]``.
     """
 
+    migrations_path: NotRequired[str | Path]
+    """Directory containing this extension's migrations, or a ``'<dotted.module>:<subdir>'`` specification.
+
+    Overrides the default ``sqlspec.extensions.<name>`` lookup. Setting this auto-includes the
+    extension in ``migration_config["include_extensions"]``.
+    """
+
     manage_schema: NotRequired[bool]
     """Apply additive target-schema reconciliation. Default: True."""
 
@@ -944,6 +966,40 @@ def get_migration_commands(self) -> "SyncMigrationCommands[Any] | AsyncMigration
         """
         return self._ensure_migration_commands()
 
+    def add_extension_migrations(
+        self, name: str, migrations_path: "str | Path", settings: "dict[str, Any] | None" = None
+    ) -> None:
+        """Register migrations shipped by a package outside the ``sqlspec.extensions`` namespace.
+
+        Records the extension under ``extension_config``, opts it into
+        ``migration_config["include_extensions"]``, and rebuilds the cached migration
+        commands so the extension is discovered. Migrations are versioned under the
+        ``ext_{name}_`` prefix, so ``name`` must stay stable once migrations are applied.
+
+        Args:
+            name: Extension name, used as the ``ext_{name}_`` version prefix.
+            migrations_path: Directory containing the migrations, or a
+                ``'<dotted.module>:<subdir>'`` specification.
+            settings: Extension settings passed to its migrations. Merged into any
+                settings already registered under ``name``.
+        """
+        extension_config = cast("dict[str, Any]", self.extension_config)
+        existing = extension_config.get(name)
+        merged: dict[str, Any] = dict(existing) if isinstance(existing, Mapping) else {}
+        if settings:
+            merged.update(settings)
+        merged["migrations_path"] = migrations_path
+        extension_config[name] = merged
+
+        migration_config = cast("dict[str, Any]", self.migration_config)
+        include_extensions = migration_config.get("include_extensions")
+        include_list = list(include_extensions) if include_extensions else []
+        if name not in include_list:
+            include_list.append(name)
+        migration_config["include_extensions"] = include_list
+
+        self._rebuild_migration_commands()
+
     @abstractmethod
     def migrate_up(
         self,
@@ -1074,6 +1130,9 @@ def _ensure_extension_migrations(self) -> None:
         - **adk**: When any adk settings are present
         - **events**: When any events settings are present
 
+        Any other extension is auto-included when its settings declare ``migrations_path``,
+        which is how packages outside the ``sqlspec.extensions`` namespace ship migrations.
+
         Use ``exclude_extensions`` to opt out of auto-inclusion.
         """
         extension_settings = cast("dict[str, Any]", self.extension_config)
@@ -1087,7 +1146,7 @@ def _ensure_extension_migrations(self) -> None:
 
         litestar_settings = extension_settings.get("litestar")
         if (
-            litestar_settings is not None
+            isinstance(litestar_settings, Mapping)
             and "session_table" in litestar_settings
             and "litestar" not in exclude_extensions
         ):
@@ -1104,22 +1163,21 @@ def _ensure_extension_migrations(self) -> None:
         if events_settings is not None and "events" not in exclude_extensions:
             extensions_to_add.append("events")
 
+        for ext_name, ext_settings in extension_settings.items():
+            if ext_name in extensions_to_add or ext_name in exclude_extensions:
+                continue
+            if isinstance(ext_settings, Mapping) and "migrations_path" in ext_settings:
+                extensions_to_add.append(ext_name)
+
         if not extensions_to_add:
             return
 
         include_extensions = migration_config.get("include_extensions")
-        if include_extensions is None:
-            include_list: list[str] = []
-            migration_config["include_extensions"] = include_list
-        elif isinstance(include_extensions, tuple):
-            include_list = list(include_extensions)  # pyright: ignore
-            migration_config["include_extensions"] = include_list
-        else:
-            include_list = cast("list[str]", include_extensions)
-
+        include_list = list(include_extensions) if include_extensions else []
         for ext in extensions_to_add:
             if ext not in include_list:
                 include_list.append(ext)
+        migration_config["include_extensions"] = include_list
 
     def _build_storage_capabilities(self) -> "StorageCapabilities":
         arrow_dependency_needed = self.supports_native_arrow_export or self.supports_native_arrow_import
@@ -1250,6 +1308,10 @@ def _initialize_migration_components(self) -> None:
         """Initialize migration loader and migration command helpers."""
         runtime = self.get_observability_runtime()
         self._migration_loader = SQLFileLoader(runtime=runtime)
+        self._rebuild_migration_commands()
+
+    def _rebuild_migration_commands(self) -> None:
+        """Rebuild the cached migration commands against current configuration."""
         self._migration_commands = create_migration_commands(self)  # pyright: ignore
 
     def _ensure_migration_loader(self) -> "SQLFileLoader":

sqlspec/extensions/events/_store.py

@@ -40,6 +40,7 @@ class BaseEventQueueStore(ABC, Generic[ConfigT]):
         "event_poll_interval",
         "lease_seconds",
         "manage_schema",
+        "migrations_path",
         "poll_interval",
         "queue_table",
         "retention_seconds",

sqlspec/extensions/litestar/store.py

@@ -63,6 +63,7 @@ class BaseSQLSpecStore(ABC, Generic[ConfigT]):
         "extra_commit_statuses",
         "extra_rollback_statuses",
         "manage_schema",
+        "migrations_path",
         "pool_key",
         "run_migrations",
         "session_key",

sqlspec/migrations/base.py

@@ -13,6 +13,7 @@
 from sqlspec.exceptions import MigrationError
 from sqlspec.migrations.templates import MigrationTemplateSettings, build_template_settings
 from sqlspec.migrations.utils import resolve_default_schema as _resolve_default_schema
+from sqlspec.migrations.utils import resolve_extension_migrations_path
 from sqlspec.migrations.utils import resolve_tracker_schema as _resolve_tracker_schema
 from sqlspec.migrations.version import parse_version
 from sqlspec.utils.logging import get_logger
@@ -553,26 +554,52 @@ def _parse_extension_configs(self) -> "dict[str, dict[str, Any]]":
     def _discover_extension_migrations(self) -> "dict[str, Path]":
         """Discover migration paths for configured extensions.
 
+        An extension may declare ``migrations_path`` in its ``extension_config`` entry to
+        ship migrations from any package. Extensions without it resolve against the
+        ``sqlspec.extensions.<name>`` namespace.
+
         Returns:
             Dictionary mapping extension names to their migration paths.
         """
 
         extension_migrations = {}
 
-        for ext_name in self.extension_configs:
-            module_name = "sqlspec.extensions.litestar" if ext_name == "litestar" else f"sqlspec.extensions.{ext_name}"
+        for ext_name, ext_options in self.extension_configs.items():
+            spec = ext_options.get("migrations_path")
+            if spec is not None:
+                resolved = resolve_extension_migrations_path(ext_name, spec)
+                if resolved is None:
+                    logger.warning(
+                        "Extension '%s' migrations_path %r is not an existing directory; "
+                        "no migrations were registered for it.",
+                        ext_name,
+                        spec,
+                    )
+                else:
+                    extension_migrations[ext_name] = resolved
+                    logger.debug("Found migrations for extension %s at %s", ext_name, resolved)
+                continue
 
+            module_name = f"sqlspec.extensions.{ext_name}"
             try:
                 module_path = module_to_os_path(module_name)
-                migrations_dir = module_path / "migrations"
+            except ImportError:
+                logger.warning(
+                    "Extension '%s' is configured but '%s' is not importable; no migrations were "
+                    "registered for it. Set extension_config['%s']['migrations_path'] to register "
+                    "migrations shipped by another package.",
+                    ext_name,
+                    module_name,
+                    ext_name,
+                )
+                continue
 
-                if migrations_dir.exists():
-                    extension_migrations[ext_name] = migrations_dir
-                    logger.debug("Found migrations for extension %s at %s", ext_name, migrations_dir)
-                else:
-                    logger.warning("No migrations directory found for extension %s", ext_name)
-            except TypeError:
-                logger.warning("Extension %s not found", ext_name)
+            migrations_dir = module_path / "migrations"
+            if migrations_dir.exists():
+                extension_migrations[ext_name] = migrations_dir
+                logger.debug("Found migrations for extension %s at %s", ext_name, migrations_dir)
+            else:
+                logger.debug("Extension %s ships no migrations directory", ext_name)
 
         return extension_migrations
 

sqlspec/migrations/loaders.py

@@ -14,7 +14,7 @@
 
 from sqlspec.loader import SQLFileLoader as CoreSQLFileLoader
 from sqlspec.migrations.context import MigrationContext
-from sqlspec.migrations.version import _format_sequential_version
+from sqlspec.migrations.version import _format_sequential_version, parse_extension_stem
 from sqlspec.utils.sync_tools import await_
 
 __all__ = ("BaseMigrationLoader", "MigrationLoadError", "PythonFileLoader", "SQLFileLoader", "get_migration_loader")
@@ -169,16 +169,16 @@ def _extract_version(self, filename: str) -> str:
         Returns:
             Version string or empty string if invalid.
         """
-        extension_version_parts = 3
         timestamp_min_length = 4
 
         name_without_ext = filename.rsplit(".", 1)[0]
 
         if name_without_ext.startswith("ext_"):
-            parts = name_without_ext.split("_", 3)
-            if len(parts) >= extension_version_parts:
-                return f"{parts[0]}_{parts[1]}_{parts[2]}"
-            return ""
+            extension = parse_extension_stem(name_without_ext)
+            if extension is None:
+                return ""
+            ext_name, ext_version = extension
+            return f"ext_{ext_name}_{ext_version}"
 
         parts = name_without_ext.split("_", 1)
         if parts and parts[0].isdigit():