diff --git a/docs/source/user-guide/graph-and-links.md b/docs/source/user-guide/graph-and-links.md index 9e7029353..5df25a173 100644 --- a/docs/source/user-guide/graph-and-links.md +++ b/docs/source/user-guide/graph-and-links.md @@ -2,19 +2,19 @@ The Graph of Links feature is experimental. The APIs may change. # Explore the Entity/Link Graph with GraphQL -Tiled can optionally serve a graph of entities and links alongside a +Tiled can optionally serve a graph of entities connected by links, alongside a catalog-backed tree, queryable through a GraphQL API. See {doc}`../explanations/graphs` for background on what this feature is and why it exists. -This guide walks through starting a demo server with the graph enabled and +This guide walks you through the process of starting a demo server with the graph enabled and exploring it interactively in the browser. ## Enable the graph feature The graph is available automatically whenever a server is serving a catalog-backed tree (see {doc}`example-server-config`)---there is no separate -configuration flag. A ready-to-run demo lives in +configuration flag. A ready-to-run demo can be found in `example_configs/graphs/` in the Tiled source repository: ``` diff --git a/example_configs/graphs/serve_with_config.py b/example_configs/graphs/serve_with_config.py index 19fdb1b69..f8ef90297 100644 --- a/example_configs/graphs/serve_with_config.py +++ b/example_configs/graphs/serve_with_config.py @@ -3,10 +3,8 @@ import argparse import uvicorn -from sqlalchemy import Column, Integer, Table from tiled.config import parse_configs -from tiled.graph.store import _metadata from tiled.server.app import build_app_from_config @@ -19,14 +17,6 @@ def main() -> None: parser.add_argument("catalog", nargs="?", default=None) args = parser.parse_args() - # Ensure graph metadata can resolve entities.node_id foreign key. - Table( - "nodes", - _metadata, - Column("id", Integer, primary_key=True), - extend_existing=True, - ) - config = parse_configs(args.config) app = build_app_from_config(config) diff --git a/share/tiled/templates/index.html b/share/tiled/templates/index.html index b670b06b9..03697dee3 100644 --- a/share/tiled/templates/index.html +++ b/share/tiled/templates/index.html @@ -50,7 +50,7 @@

Explore the Graph API

Use the interactive GraphQL playground to query the links graph.

- + diff --git a/tests/test_graph_access_control.py b/tests/test_graph_access_control.py index 7aa187b04..c69e7d6df 100644 --- a/tests/test_graph_access_control.py +++ b/tests/test_graph_access_control.py @@ -1,11 +1,11 @@ import pytest -from sqlalchemy import Column, Integer, Table from starlette.testclient import TestClient from tiled.catalog import in_memory +from tiled.catalog.core import initialize_database from tiled.config import Database from tiled.graph.schema import schema -from tiled.graph.store import GraphSQLAlchemyStore, _metadata +from tiled.graph.store import GraphSQLAlchemyStore from tiled.queries import AccessBlobFilter from tiled.server.app import build_app from tiled.server.authentication import ( @@ -109,14 +109,12 @@ async def filters( @pytest.fixture async def store(): - Table( - "nodes", - _metadata, - Column("id", Integer, primary_key=True), - extend_existing=True, - ) database_settings = DatabaseSettings(uri="sqlite:///:memory:") s = await GraphSQLAlchemyStore.from_database_settings(database_settings) + # The store no longer creates its own tables; provision the catalog schema + # (which now includes the graph tables and the `nodes` table that + # entities.node_id references) on the shared in-memory database. + await initialize_database(s._engine) yield s # Tear down the shared pool entry (rather than just `s.close()`, which is # a no-op here) so each test gets an isolated in-memory database instead @@ -600,14 +598,6 @@ async def test_graphql_expands_and_compacts_curies(store, policy): def test_graphql_http_route_access_control_integration(tmp_path, policy): """Validate HTTP GraphQL route wiring with auth dependencies and policy checks.""" - # Ensure graph metadata can resolve the entities.node_id foreign key. - Table( - "nodes", - _metadata, - Column("id", Integer, primary_key=True), - extend_existing=True, - ) - catalog = in_memory(writable_storage=str(tmp_path / "storage")) app = build_app( catalog, diff --git a/tiled/catalog/core.py b/tiled/catalog/core.py index 38714a093..ffb1232f2 100644 --- a/tiled/catalog/core.py +++ b/tiled/catalog/core.py @@ -35,6 +35,11 @@ async def initialize_database(engine: AsyncEngine): # The definitions in .orm alter Base.metadata. + # The graph (splash-links) tables also live in the catalog database and + # attach to Base.metadata, so importing them here ensures create_all + # provisions them on fresh databases (existing databases get them via the + # Alembic migration c31f6a1d7e20). + from ..graph import orm as graph_orm # noqa: F401 from . import orm # noqa: F401 async with engine.connect() as connection: diff --git a/tiled/catalog/migrations/versions/c31f6a1d7e20_add_graph_entities_and_links_tables.py b/tiled/catalog/migrations/versions/c31f6a1d7e20_add_graph_entities_and_links_tables.py index 6ed107942..27d6c15b1 100644 --- a/tiled/catalog/migrations/versions/c31f6a1d7e20_add_graph_entities_and_links_tables.py +++ b/tiled/catalog/migrations/versions/c31f6a1d7e20_add_graph_entities_and_links_tables.py @@ -1,4 +1,4 @@ -"""Add graph entities and links tables +"""Add graph entities, links, and namespaces tables Revision ID: c31f6a1d7e20 Revises: 9bc9b57294b9 @@ -71,8 +71,18 @@ def upgrade(): unique=False, ) + op.create_table( + "namespaces", + sa.Column("prefix", sa.String(), nullable=False), + sa.Column("uri", sa.String(), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False), + sa.PrimaryKeyConstraint("prefix"), + ) + def downgrade(): + op.drop_table("namespaces") + op.drop_index("links_triple_idx", table_name="links") op.drop_index("links_predicate_object_idx", table_name="links") op.drop_index("links_subject_predicate_idx", table_name="links") diff --git a/tiled/graph/core.py b/tiled/graph/core.py deleted file mode 100644 index de5747488..000000000 --- a/tiled/graph/core.py +++ /dev/null @@ -1,11 +0,0 @@ -from sqlalchemy.ext.asyncio import AsyncEngine - -from .store import _metadata - -ALL_REVISIONS = ["7f3a9d1c0b25"] -REQUIRED_REVISION = ALL_REVISIONS[0] - - -async def initialize_database(engine: AsyncEngine) -> None: - async with engine.begin() as conn: - await conn.run_sync(_metadata.create_all) diff --git a/tiled/graph/curie.py b/tiled/graph/curie.py index 4a9c151d0..3aea80c24 100644 --- a/tiled/graph/curie.py +++ b/tiled/graph/curie.py @@ -1,9 +1,8 @@ """ CURIE (Compact URI) expansion/compaction against the namespace registry. -Shared between the GraphQL schema (which expands terms written through -mutations and compacts terms read back out) and the JSON-LD REST import/ -export routes, so both interfaces resolve prefixes the same way. +Used by the GraphQL schema to expand terms written through mutations and +compact terms read back out, so prefixes resolve consistently. """ from __future__ import annotations diff --git a/tiled/graph/orm.py b/tiled/graph/orm.py new file mode 100644 index 000000000..fc67848c7 --- /dev/null +++ b/tiled/graph/orm.py @@ -0,0 +1,79 @@ +""" +SQLAlchemy Core table definitions for the graph (splash-links) feature. + +These tables live in the catalog database alongside the catalog's own tables +(``entities.node_id`` is a foreign key into the catalog ``nodes`` table). They +are attached to the catalog's ``Base.metadata`` so that the two supported ways +of provisioning a catalog database both include them: + +* a fresh database created by ``tiled.catalog.core.initialize_database`` + (which runs ``Base.metadata.create_all``), and +* an existing database upgraded through the Alembic migration + ``c31f6a1d7e20``. + +The store (``tiled.graph.store``) uses these ``Table`` objects to read and +write rows; it does not create them itself. This mirrors how ``metadata_fts5`` +is declared as a Core table on ``Base.metadata`` in ``tiled.catalog.orm``. +""" + +from __future__ import annotations + +from sqlalchemy import JSON, Column, DateTime, ForeignKey, Index, Integer, String, Table + +from ..catalog.base import Base + +metadata = Base.metadata + +entities = Table( + "entities", + metadata, + Column("id", String, primary_key=True), + Column( + "node_id", + Integer, + ForeignKey("nodes.id", ondelete="SET NULL"), + nullable=True, + ), + Column("entity_type", String, nullable=False), + Column("name", String, nullable=False), + Column("uri", String, nullable=True), + Column("properties", JSON, nullable=False), + Column("access_blob", JSON, nullable=False), + Column("created_at", DateTime(timezone=True), nullable=False), + Index("entities_node_id_idx", "node_id"), + Index("entities_type_created_idx", "entity_type", "created_at"), + Index("entities_uri_idx", "uri"), +) + +links = Table( + "links", + metadata, + Column("id", String, primary_key=True), + Column( + "subject_id", + String, + ForeignKey("entities.id", ondelete="CASCADE"), + nullable=False, + ), + Column("predicate", String, nullable=False), + Column( + "object_id", + String, + ForeignKey("entities.id", ondelete="CASCADE"), + nullable=False, + ), + Column("properties", JSON, nullable=False), + Column("access_blob", JSON, nullable=False), + Column("created_at", DateTime(timezone=True), nullable=False), + Index("links_subject_predicate_idx", "subject_id", "predicate"), + Index("links_predicate_object_idx", "predicate", "object_id"), + Index("links_triple_idx", "subject_id", "predicate", "object_id"), +) + +namespaces = Table( + "namespaces", + metadata, + Column("prefix", String, primary_key=True), + Column("uri", String, nullable=False), + Column("created_at", DateTime(timezone=True), nullable=False), +) diff --git a/tiled/graph/router.py b/tiled/graph/router.py index 58a97df1a..17e490351 100644 --- a/tiled/graph/router.py +++ b/tiled/graph/router.py @@ -15,6 +15,7 @@ from __future__ import annotations import logging +import re from typing import Callable from fastapi import APIRouter, Depends, Request @@ -31,6 +32,69 @@ logger = logging.getLogger(__name__) +# The query pre-loaded into the GraphiQL editor. It orients newcomers to the +# entity/link graph rather than showing GraphiQL's generic welcome text. Keep +# this free of backticks and `${...}` so it stays a valid JavaScript template +# literal when injected into the IDE HTML below. +DEFAULT_GRAPHIQL_QUERY = """# Tiled — Entity/Link Graph explorer +# +# This endpoint serves the graph of entities (nodes) and the links (edges) +# between them, alongside the catalog tree. +# +# Access is controlled, so most queries need an API key. Open the "Headers" +# tab below and add your key: +# +# { "Authorization": "Apikey YOUR_API_KEY" } +# +# Without it, queries do not error — they just return empty results. +# +# Run a query with Ctrl-Enter (or the play button). Browse the schema in the +# "Docs" and "Explorer" panels on the left. + +query ExploreGraph { + entities(limit: 10) { + id + name + entityType + uri + outgoingLinks(limit: 5) { + predicate + object { + id + name + } + } + } + namespaces { + prefix + uri + } +} +""" + +# Matches Strawberry's bundled `const EXAMPLE_QUERY = ` ... ` ;` assignment. +_EXAMPLE_QUERY_RE = re.compile(r"const EXAMPLE_QUERY = `.*?`;", re.DOTALL) + + +class _TiledGraphQLRouter(GraphQLRouter): + """GraphQLRouter that preloads a Tiled-specific default query. + + Strawberry bundles a static GraphiQL page whose editor opens with a + generic welcome message. We reuse that page but swap the default query for + one tailored to the entity/link graph. If Strawberry ever changes the + template and the marker is not found, the original HTML is served + unchanged. + """ + + @property + def graphql_ide_html(self) -> str: + html = super().graphql_ide_html + return _EXAMPLE_QUERY_RE.sub( + lambda _: f"const EXAMPLE_QUERY = `{DEFAULT_GRAPHIQL_QUERY}`;", + html, + count=1, + ) + def create_router(get_database_settings: Callable[[], DatabaseSettings]) -> APIRouter: store: list[GraphSQLAlchemyStore] = [] # mutable cell — populated on startup @@ -59,7 +123,7 @@ async def get_context( "access_policy": getattr(request.app.state, "access_policy", None), } - graphql_router = GraphQLRouter( + graphql_router = _TiledGraphQLRouter( schema, context_getter=get_context, graphql_ide="graphiql", diff --git a/tiled/graph/schema.py b/tiled/graph/schema.py index f97a5c021..d7f147318 100644 --- a/tiled/graph/schema.py +++ b/tiled/graph/schema.py @@ -5,7 +5,7 @@ - Entity — a named node with a type and arbitrary JSON properties - Link — a directed, predicate-labeled edge between two entities - Namespace — a CURIE prefix -> URI mapping used to expand/compact terms - (property keys and link predicates) for JSON-LD import/export. + (property keys and link predicates). Query highlights: - entity / entities — fetch nodes @@ -20,8 +20,7 @@ Property keys and link predicates are expanded against the namespace registry when written and compacted back to CURIEs when read, so a -prefix registered through `upsertNamespace` (or through JSON-LD import) -is resolved consistently regardless of which interface wrote the data. +prefix registered through `upsertNamespace` is resolved consistently. """ from __future__ import annotations @@ -31,6 +30,7 @@ import strawberry from graphql import GraphQLError +from strawberry.extensions import QueryDepthLimiter from strawberry.scalars import JSON as StrawberryJSON from strawberry.types import Info from strawberry.types.unset import UNSET, UnsetType @@ -44,6 +44,13 @@ logger = logging.getLogger(__name__) +# Maximum nesting depth allowed in a single GraphQL query. The entity/link +# graph is recursively traversable (Entity.outgoingLinks -> Link.object -> +# Entity.outgoingLinks -> ...), so an unbounded query could force arbitrarily +# deep and expensive resolution. Introspection queries are exempt (the limiter +# ignores them by default), so the GraphiQL "Docs" panel is unaffected. +MAX_QUERY_DEPTH = 10 + # --------------------------------------------------------------------------- # JSON scalar — pass arbitrary dicts / lists / primitives through GraphQL # --------------------------------------------------------------------------- @@ -575,4 +582,8 @@ async def delete_namespace(self, info: Info, prefix: str) -> bool: # Schema # --------------------------------------------------------------------------- -schema = strawberry.Schema(query=Query, mutation=Mutation) +schema = strawberry.Schema( + query=Query, + mutation=Mutation, + extensions=[lambda: QueryDepthLimiter(max_depth=MAX_QUERY_DEPTH)], +) diff --git a/tiled/graph/store.py b/tiled/graph/store.py index 2636817f5..49152b35f 100644 --- a/tiled/graph/store.py +++ b/tiled/graph/store.py @@ -6,6 +6,11 @@ ``tiled.server.connection_pool``), so the graph tables and the catalog tables are always served from a single shared pool rather than opening a second connection pool to the same database. + +The graph tables themselves are defined in ``tiled.graph.orm`` (attached to +the catalog's ``Base.metadata``) and provisioned by the catalog's database +initialization / Alembic migrations. This store only reads and writes rows; it +does not create tables. """ from __future__ import annotations @@ -16,15 +21,6 @@ from pydantic import BaseModel, ConfigDict from sqlalchemy import ( - JSON, - Column, - DateTime, - ForeignKey, - Index, - Integer, - MetaData, - String, - Table, and_, delete, false, @@ -36,16 +32,24 @@ update, ) from sqlalchemy.dialects.postgresql import ARRAY, JSONB, TEXT +from sqlalchemy.exc import IntegrityError from sqlalchemy.ext.asyncio import AsyncEngine from sqlalchemy.sql.expression import cast as sql_cast +from ..catalog.orm import Node from ..queries import AccessBlobFilter from ..server.connection_pool import get_database_engine from ..server.settings import DatabaseSettings from ..utils import UnsupportedQueryType +from .orm import entities as _entities +from .orm import links as _links +from .orm import namespaces as _namespaces UNSET = object() +# The catalog ``nodes`` table, used to resolve entities.node_id by catalog path. +_nodes = Node.__table__ + # --------------------------------------------------------------------------- # Data records # --------------------------------------------------------------------------- @@ -76,81 +80,6 @@ class LinkRecord(BaseModel): created_at: datetime -# --------------------------------------------------------------------------- -# SQLAlchemy schema -# --------------------------------------------------------------------------- - -_metadata = MetaData() - -# Register the catalog nodes table so entities.node_id can resolve -# ForeignKey("nodes.id") when SQLAlchemy sorts DDL dependencies, and so -# resolve_node_id() below can look up a node's id by its catalog path -# without importing tiled.catalog (this table always already exists--- -# it is created by the catalog's own migrations). -_nodes = Table( - "nodes", - _metadata, - Column("id", Integer, primary_key=True), - Column("parent", Integer, ForeignKey("nodes.id"), nullable=True), - Column("key", String, nullable=False), - extend_existing=True, -) - -_entities = Table( - "entities", - _metadata, - Column("id", String, primary_key=True), - Column( - "node_id", - Integer, - ForeignKey("nodes.id", ondelete="SET NULL"), - nullable=True, - ), - Column("entity_type", String, nullable=False), - Column("name", String, nullable=False), - Column("uri", String, nullable=True), - Column("properties", JSON, nullable=False), - Column("access_blob", JSON, nullable=False), - Column("created_at", DateTime(timezone=True), nullable=False), - Index("entities_node_id_idx", "node_id"), - Index("entities_type_created_idx", "entity_type", "created_at"), - Index("entities_uri_idx", "uri"), -) - -_links = Table( - "links", - _metadata, - Column("id", String, primary_key=True), - Column( - "subject_id", - String, - ForeignKey("entities.id", ondelete="CASCADE"), - nullable=False, - ), - Column("predicate", String, nullable=False), - Column( - "object_id", - String, - ForeignKey("entities.id", ondelete="CASCADE"), - nullable=False, - ), - Column("properties", JSON, nullable=False), - Column("access_blob", JSON, nullable=False), - Column("created_at", DateTime(timezone=True), nullable=False), - Index("links_subject_predicate_idx", "subject_id", "predicate"), - Index("links_predicate_object_idx", "predicate", "object_id"), - Index("links_triple_idx", "subject_id", "predicate", "object_id"), -) - -_namespaces = Table( - "namespaces", - _metadata, - Column("prefix", String, primary_key=True), - Column("uri", String, nullable=False), - Column("created_at", DateTime(timezone=True), nullable=False), -) - - def _access_blob_condition( dialect_name: str, access_blob_column, query: AccessBlobFilter ): @@ -208,7 +137,9 @@ class GraphSQLAlchemyStore: Async SQLAlchemy-backed store that can reuse Tiled's shared DB pool. Use ``from_database_settings`` to attach to the same async engine registry - used by the rest of the server. + used by the rest of the server. The graph tables are provisioned by the + catalog database (see ``tiled.graph.orm``); this store does not create + them. """ def __init__(self, engine: AsyncEngine, owns_engine: bool = False) -> None: @@ -221,9 +152,7 @@ async def from_database_settings( database_settings: DatabaseSettings, ) -> "GraphSQLAlchemyStore": engine = get_database_engine(database_settings) - store = cls(engine, owns_engine=False) - await store._initialize_schema() - return store + return cls(engine, owns_engine=False) @staticmethod def _to_entity(row) -> EntityRecord: @@ -250,10 +179,6 @@ def _to_link(row) -> LinkRecord: created_at=row.created_at, ) - async def _initialize_schema(self) -> None: - async with self._engine.begin() as conn: - await conn.run_sync(_metadata.create_all) - async def create_entity( self, entity_type: str, @@ -355,26 +280,38 @@ async def create_link( properties: Optional[dict] = None, access_blob: Optional[dict] = None, ) -> LinkRecord: - if not await self.get_entity(subject_id): - raise ValueError(f"Subject entity '{subject_id}' not found") - if not await self.get_entity(object_id): - raise ValueError(f"Object entity '{object_id}' not found") - id_ = str(uuid.uuid4()) now = datetime.now(timezone.utc) - async with self._engine.begin() as conn: - await conn.execute( - insert(_links).values( - id=id_, - subject_id=subject_id, - predicate=predicate, - object_id=object_id, - properties=properties or {}, - access_blob=access_blob or {}, - created_at=now, + # The subject_id/object_id foreign keys reference entities.id, so the + # database rejects a link to a nonexistent entity (SQLite enforces this + # too: the shared pool sets PRAGMA foreign_keys=ON). Insert directly and + # let the constraint do the checking, rather than pre-querying both + # endpoints on every create. + try: + async with self._engine.begin() as conn: + await conn.execute( + insert(_links).values( + id=id_, + subject_id=subject_id, + predicate=predicate, + object_id=object_id, + properties=properties or {}, + access_blob=access_blob or {}, + created_at=now, + ) ) - ) - row = (await conn.execute(select(_links).where(_links.c.id == id_))).one() + row = ( + await conn.execute(select(_links).where(_links.c.id == id_)) + ).one() + except IntegrityError as exc: + # A foreign-key violation means one of the endpoints is missing. + # Resolve which one only on this failure path so the success path + # stays a single INSERT. + if not await self.get_entity(subject_id): + raise ValueError(f"Subject entity '{subject_id}' not found") from exc + if not await self.get_entity(object_id): + raise ValueError(f"Object entity '{object_id}' not found") from exc + raise return self._to_link(row) async def get_link(self, id: str) -> Optional[LinkRecord]: