From 718f789b9b4b12bfb65760676a14262c9666cc3e Mon Sep 17 00:00:00 2001
From: Chandrasekharan M <chandrasekharan@zipstack.com>
Date: Fri, 24 Jul 2026 15:43:57 +0530
Subject: [PATCH 01/10] feat(clone): follow DRF pagination on all list
 endpoints
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Every list_* helper unwrapped a paginated envelope but never sent ?page
and never followed `next` — it read page one and stopped. Endpoints that
already paginate (tags/, pipeline/, api/deployment/) have therefore been
silently truncating at 50 rows, and the same would hit adapters,
connectors and prompt-studio once UN-3770 makes their pagination
unconditional. A clone that copies a subset without erroring is worse
than one that fails.

Add PlatformClient._paginate(), which short-circuits on a bare list so
the client keeps working against deployments where an endpoint is not
paginated, otherwise walks `next` to exhaustion. It refuses to return a
short read: the collected row count is checked against the reported
count, and a cyclic `next` raises instead of looping forever.

_request() is split so the absolute `next` URLs DRF emits can be issued
without going through org-relative path composition. Its signature is
unchanged, so every existing caller is untouched.

All 23 list helpers now route through it. list_lookup_versions keeps its
bespoke unwrap — that endpoint returns {"versions": [...]}, not a DRF
envelope.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_018HXYFVCZGi7YN9qjDGyJCQ
---
 src/unstract/clone/client.py | 177 +++++++++++++++++++----------------
 tests/clone/test_client.py   |  41 ++++++++
 2 files changed, 136 insertions(+), 82 deletions(-)

diff --git a/src/unstract/clone/client.py b/src/unstract/clone/client.py
index 3f79499..fb60e97 100644
--- a/src/unstract/clone/client.py
+++ b/src/unstract/clone/client.py
@@ -70,7 +70,32 @@ def _request(
         files: dict[str, Any] | None = None,
         data: dict[str, Any] | None = None,
     ) -> Any:
-        url = self._url(path)
+        return self._send(
+            method,
+            self._url(path),
+            path,
+            params=params,
+            json=json,
+            files=files,
+            data=data,
+        )
+
+    def _send(
+        self,
+        method: str,
+        url: str,
+        label: str,
+        *,
+        params: dict[str, Any] | None = None,
+        json: Any = None,
+        files: dict[str, Any] | None = None,
+        data: dict[str, Any] | None = None,
+    ) -> Any:
+        """Issue a request against an already-built URL.
+
+        Split out from ``_request`` so pagination can follow the absolute
+        ``next`` links DRF returns, which are not org-path-relative.
+        """
         # Redact secrets from logs: only entity path + method, never body.
         logger.debug("%s %s", method, url)
         resp = self._session.request(
@@ -85,7 +110,7 @@ def _request(
         )
         if not 200 <= resp.status_code < 300:
             raise PlatformAPIError(
-                f"{method} {path} returned {resp.status_code}",
+                f"{method} {label} returned {resp.status_code}",
                 status_code=resp.status_code,
                 body=resp.text[:2000],
             )
@@ -93,6 +118,41 @@ def _request(
             return None
         return resp.json()
 
+    def _paginate(
+        self, path: str, params: dict[str, Any] | None = None
+    ) -> list[dict[str, Any]]:
+        """GET a list endpoint, following DRF pagination to exhaustion.
+
+        A bare list is returned unchanged, so the same client works against
+        deployments where the endpoint is not paginated. A short read is
+        raised rather than returned: a clone that silently copies the first
+        page is worse than one that fails.
+        """
+        result = self._request("GET", path, params=dict(params or {}))
+        if isinstance(result, list):
+            return result
+        if not isinstance(result, dict) or "results" not in result:
+            raise PlatformAPIError(f"GET {path} returned an unrecognised list payload")
+
+        rows: list[dict[str, Any]] = []
+        expected = result.get("count")
+        seen: set[str] = set()
+        while True:
+            rows.extend(result.get("results") or [])
+            next_url = result.get("next")
+            if not next_url:
+                break
+            if next_url in seen:
+                raise PlatformAPIError(f"GET {path} pagination looped at {next_url}")
+            seen.add(next_url)
+            result = self._send("GET", next_url, path)
+
+        if expected is not None and len(rows) != expected:
+            raise PlatformAPIError(
+                f"GET {path} returned {len(rows)} rows but reported count={expected}"
+            )
+        return rows
+
     def get_post_schema(self, entity_path: str) -> frozenset[str]:
         """Return the set of fields the backend's POST accepts.
 
@@ -137,8 +197,7 @@ def list_users(self) -> list[dict[str, Any]]:
 
     def list_groups(self) -> list[dict[str, Any]]:
         """List org groups; no server-side name filter — callers match in memory."""
-        result = self._request("GET", "groups/")
-        return result if isinstance(result, list) else (result or {}).get("results", [])
+        return self._paginate("groups/")
 
     def create_group(self, payload: dict[str, Any]) -> dict[str, Any]:
         """Create a group; response has no ``id`` — re-list to learn the pk."""
@@ -146,8 +205,7 @@ def create_group(self, payload: dict[str, Any]) -> dict[str, Any]:
 
     def list_group_members(self, group_id: Any) -> list[dict[str, Any]]:
         """List a group's member rows (each carries ``email``)."""
-        result = self._request("GET", f"groups/{group_id}/members/")
-        return result if isinstance(result, list) else result.get("results", [])
+        return self._paginate(f"groups/{group_id}/members/")
 
     def add_group_members(self, group_id: Any, user_ids: list[int]) -> Any:
         """Bulk-add members by user pk; idempotent server-side."""
@@ -175,9 +233,7 @@ def list_adapters(
             params["adapter_name"] = name
         if adapter_type is not None:
             params["adapter_type"] = adapter_type
-        result = self._request("GET", "adapter/", params=params)
-        # DRF ModelViewSet.list returns a bare list (no pagination on this endpoint).
-        return result if isinstance(result, list) else result.get("results", [])
+        return self._paginate("adapter/", params)
 
     def get_adapter(self, adapter_pk: str) -> dict[str, Any]:
         return self._request("GET", f"adapter/{adapter_pk}/")
@@ -199,8 +255,7 @@ def list_connectors(
             params["connector_name"] = name
         if connector_type is not None:
             params["connector_type"] = connector_type
-        result = self._request("GET", "connector/", params=params)
-        return result if isinstance(result, list) else result.get("results", [])
+        return self._paginate("connector/", params)
 
     def get_connector(self, connector_pk: str) -> dict[str, Any]:
         return self._request("GET", f"connector/{connector_pk}/")
@@ -215,9 +270,7 @@ def list_tags(self, *, name: str | None = None) -> list[dict[str, Any]]:
         params: dict[str, Any] = {}
         if name is not None:
             params["name"] = name
-        result = self._request("GET", "tags/", params=params)
-        # Tags endpoint uses pagination — accept either bare list or paginated envelope.
-        return result if isinstance(result, list) else result.get("results", [])
+        return self._paginate("tags/", params)
 
     def create_tag(self, payload: dict[str, Any]) -> dict[str, Any]:
         return self._request("POST", "tags/", json=payload)
@@ -226,8 +279,7 @@ def create_tag(self, payload: dict[str, Any]) -> dict[str, Any]:
 
     def list_custom_tools(self) -> list[dict[str, Any]]:
         """List all prompt-studio projects in this org. No name filter."""
-        result = self._request("GET", "prompt-studio/")
-        return result if isinstance(result, list) else result.get("results", [])
+        return self._paginate("prompt-studio/")
 
     def get_custom_tool(self, tool_id: str) -> dict[str, Any]:
         """Fetch a single prompt-studio project.
@@ -249,8 +301,7 @@ def list_profiles(self, tool_id: str) -> list[dict[str, Any]]:
         default profile's adapter UUIDs so they can be remapped to
         target adapter ids for ``import_project``.
         """
-        result = self._request("GET", f"prompt-studio/prompt-studio-profile/{tool_id}/")
-        return result if isinstance(result, list) else result.get("results", [])
+        return self._paginate(f"prompt-studio/prompt-studio-profile/{tool_id}/")
 
     def list_prompts(self, tool_id: str) -> list[dict[str, Any]]:
         """List a tool's prompts (``prompt_id`` + ``prompt_key`` per row).
@@ -259,10 +310,7 @@ def list_prompts(self, tool_id: str) -> list[dict[str, Any]]:
         ``import_project`` / ``sync_prompts`` (matched by ``prompt_key``),
         so prompt-scoped cloud config can remap its FKs.
         """
-        result = self._request(
-            "GET", "prompt-studio/prompt/", params={"tool_id": tool_id}
-        )
-        return result if isinstance(result, list) else result.get("results", [])
+        return self._paginate("prompt-studio/prompt/", {"tool_id": tool_id})
 
     def export_project(self, tool_id: str) -> dict[str, Any]:
         """Export a prompt-studio project as a portable JSON blob.
@@ -338,10 +386,7 @@ def list_prompt_documents(self, tool_id: str) -> list[dict[str, Any]]:
         enumeration. Response items carry ``document_id``,
         ``document_name``, and ``tool``.
         """
-        result = self._request(
-            "GET", "prompt-studio/prompt-document/", params={"tool_id": tool_id}
-        )
-        return result if isinstance(result, list) else result.get("results", [])
+        return self._paginate("prompt-studio/prompt-document/", {"tool_id": tool_id})
 
     def download_prompt_file(self, tool_id: str, document_id: str) -> dict[str, Any]:
         """GET a Prompt Studio document by tool + document id.
@@ -398,8 +443,7 @@ def list_workflows(self, *, name: str | None = None) -> list[dict[str, Any]]:
         params: dict[str, Any] = {}
         if name is not None:
             params["workflow_name"] = name
-        result = self._request("GET", "workflow/", params=params)
-        return result if isinstance(result, list) else result.get("results", [])
+        return self._paginate("workflow/", params)
 
     def get_workflow(self, workflow_id: str) -> dict[str, Any]:
         return self._request("GET", f"workflow/{workflow_id}/")
@@ -420,8 +464,7 @@ def list_registries(
         params: dict[str, Any] = {}
         if custom_tool is not None:
             params["custom_tool"] = custom_tool
-        result = self._request("GET", "prompt-studio/registry/", params=params)
-        return result if isinstance(result, list) else result.get("results", [])
+        return self._paginate("prompt-studio/registry/", params)
 
     # ----- tool instances -----
 
@@ -432,8 +475,7 @@ def list_tool_instances(
         params: dict[str, Any] = {}
         if workflow_id is not None:
             params["workflow"] = workflow_id
-        result = self._request("GET", "tool_instance/", params=params)
-        return result if isinstance(result, list) else result.get("results", [])
+        return self._paginate("tool_instance/", params)
 
     def create_tool_instance(self, payload: dict[str, Any]) -> dict[str, Any]:
         """Create a tool instance (max 1 per workflow). The created row comes
@@ -466,8 +508,7 @@ def list_workflow_endpoints(
         params: dict[str, Any] = {}
         if workflow_id is not None:
             params["workflow"] = workflow_id
-        result = self._request("GET", "workflow/endpoint/", params=params)
-        return result if isinstance(result, list) else result.get("results", [])
+        return self._paginate("workflow/endpoint/", params)
 
     def update_workflow_endpoint(
         self, endpoint_id: str, payload: dict[str, Any]
@@ -490,8 +531,7 @@ def list_pipelines(
             params["pipeline_name"] = name
         if pipeline_type is not None:
             params["type"] = pipeline_type
-        result = self._request("GET", "pipeline/", params=params)
-        return result if isinstance(result, list) else result.get("results", [])
+        return self._paginate("pipeline/", params)
 
     def get_pipeline(self, pipeline_id: str) -> dict[str, Any]:
         return self._request("GET", f"pipeline/{pipeline_id}/")
@@ -518,8 +558,7 @@ def list_api_deployments(
         params: dict[str, Any] = {}
         if api_name is not None:
             params["api_name"] = api_name
-        result = self._request("GET", "api/deployment/", params=params)
-        return result if isinstance(result, list) else result.get("results", [])
+        return self._paginate("api/deployment/", params)
 
     def get_api_deployment(self, deployment_id: str) -> dict[str, Any]:
         return self._request("GET", f"api/deployment/{deployment_id}/")
@@ -539,13 +578,11 @@ def update_api_deployment(
 
     def list_pipeline_keys(self, pipeline_id: str) -> list[dict[str, Any]]:
         """List API keys belonging to a pipeline."""
-        result = self._request("GET", f"api/keys/pipeline/{pipeline_id}/")
-        return result if isinstance(result, list) else result.get("results", [])
+        return self._paginate(f"api/keys/pipeline/{pipeline_id}/")
 
     def list_api_deployment_keys(self, deployment_id: str) -> list[dict[str, Any]]:
         """List API keys belonging to an API deployment."""
-        result = self._request("GET", f"api/keys/api/{deployment_id}/")
-        return result if isinstance(result, list) else result.get("results", [])
+        return self._paginate(f"api/keys/api/{deployment_id}/")
 
     def create_api_key(self, payload: dict[str, Any]) -> dict[str, Any]:
         """Create an extra API key tied to a pipeline or deployment.
@@ -560,8 +597,7 @@ def create_api_key(self, payload: dict[str, Any]) -> dict[str, Any]:
 
     def list_lookup_definitions(self) -> list[dict[str, Any]]:
         """List lookup definitions in this org. Also the capability-probe path."""
-        result = self._request("GET", "lookups/definitions/")
-        return result if isinstance(result, list) else (result or {}).get("results", [])
+        return self._paginate("lookups/definitions/")
 
     def get_lookup_definition(self, lookup_id: str) -> dict[str, Any]:
         """Fetch a lookup definition's detail.
@@ -607,8 +643,7 @@ def list_lookup_files(self, lookup_id: str) -> list[dict[str, Any]]:
         """List a lookup's draft reference files (rows carry ``file_id``,
         ``file_name``, ``file_size``).
         """
-        result = self._request("GET", f"lookups/definitions/{lookup_id}/files/")
-        return result if isinstance(result, list) else (result or {}).get("results", [])
+        return self._paginate(f"lookups/definitions/{lookup_id}/files/")
 
     def download_lookup_file(self, lookup_id: str, file_id: str) -> bytes:
         """Download a reference file's original bytes.
@@ -648,8 +683,7 @@ def list_lookup_assignments(self) -> list[dict[str, Any]]:
         ``version`` (source lookup-version uuid), ``lookup_definition``
         (source lookup_id), ``is_draft_version``, and ``variable_mappings``.
         """
-        result = self._request("GET", "lookups/assignments/")
-        return result if isinstance(result, list) else (result or {}).get("results", [])
+        return self._paginate("lookups/assignments/")
 
     def create_lookup_assignment(self, payload: dict[str, Any]) -> dict[str, Any]:
         """Create a prompt-lookup assignment.
@@ -668,9 +702,7 @@ def update_lookup_share(
         ``payload`` carries ``shared_to_org`` + ``shared_users`` (target user
         pks). Lookups expose no group-sharing axis, so no ``shared_groups``.
         """
-        return self._request(
-            "PATCH", f"lookups/definitions/{lookup_id}/", json=payload
-        )
+        return self._request("PATCH", f"lookups/definitions/{lookup_id}/", json=payload)
 
     def list_lookup_versions(self, lookup_id: str) -> list[dict[str, Any]]:
         """List a lookup's versions (draft + published).
@@ -678,17 +710,13 @@ def list_lookup_versions(self, lookup_id: str) -> list[dict[str, Any]]:
         Rows carry ``version_id``, ``is_draft``, ``version_number``,
         ``version_name``; the detail (``get_lookup_version``) inlines content.
         """
-        result = self._request(
-            "GET", f"lookups/definitions/{lookup_id}/versions/"
-        )
+        result = self._request("GET", f"lookups/definitions/{lookup_id}/versions/")
         if isinstance(result, list):
             return result
         # This endpoint wraps rows as {"versions": [...], "next_version_number"}.
         return (result or {}).get("versions", (result or {}).get("results", []))
 
-    def get_lookup_version(
-        self, lookup_id: str, version_id: str
-    ) -> dict[str, Any]:
+    def get_lookup_version(self, lookup_id: str, version_id: str) -> dict[str, Any]:
         """Fetch a version's detail (``prompt_template``, adapters, files)."""
         return self._request(
             "GET", f"lookups/definitions/{lookup_id}/versions/{version_id}/"
@@ -796,12 +824,9 @@ def list_auto_approval_settings(self) -> list[dict[str, Any]]:
         Returns 200 bare with no query params, so it doubles as the
         manual-review capability probe path.
         """
-        result = self._request("GET", "manual_review/auto_approval_settings/")
-        return result if isinstance(result, list) else (result or {}).get("results", [])
+        return self._paginate("manual_review/auto_approval_settings/")
 
-    def create_auto_approval_settings(
-        self, payload: dict[str, Any]
-    ) -> dict[str, Any]:
+    def create_auto_approval_settings(self, payload: dict[str, Any]) -> dict[str, Any]:
         """Create org-level auto-approval settings.
 
         Writable: ``auto_approved_document_classes``, ``auto_approved_users``.
@@ -813,8 +838,7 @@ def create_auto_approval_settings(
 
     def list_review_api_keys(self) -> list[dict[str, Any]]:
         """List review API keys in this org."""
-        result = self._request("GET", "manual_review/api/keys/")
-        return result if isinstance(result, list) else (result or {}).get("results", [])
+        return self._paginate("manual_review/api/keys/")
 
     def create_review_api_key(self, payload: dict[str, Any]) -> dict[str, Any]:
         """Create a review API key. The ``api_key`` secret is server-minted
@@ -833,8 +857,7 @@ def list_agentic_projects(self) -> list[dict[str, Any]]:
         ``lightweight_llm_connector_id`` / ``text_extractor_connector_id``),
         and ``canary_fields``.
         """
-        result = self._request("GET", "agentic/projects/")
-        return result if isinstance(result, list) else (result or {}).get("results", [])
+        return self._paginate("agentic/projects/")
 
     def create_agentic_project(self, payload: dict[str, Any]) -> dict[str, Any]:
         """Create an agentic project. Returns the created row (carries ``id``)."""
@@ -851,8 +874,7 @@ def list_agentic_prompt_versions(
         params: dict[str, Any] = {}
         if project_id is not None:
             params["project_id"] = project_id
-        result = self._request("GET", "agentic/prompt-versions/", params=params)
-        return result if isinstance(result, list) else (result or {}).get("results", [])
+        return self._paginate("agentic/prompt-versions/", params)
 
     def create_agentic_prompt_version(self, payload: dict[str, Any]) -> dict[str, Any]:
         """Create an agentic prompt version (flat endpoint, ``project`` in body)."""
@@ -869,8 +891,7 @@ def list_agentic_schemas(
         params: dict[str, Any] = {}
         if project_id is not None:
             params["project_id"] = project_id
-        result = self._request("GET", "agentic/schemas/", params=params)
-        return result if isinstance(result, list) else (result or {}).get("results", [])
+        return self._paginate("agentic/schemas/", params)
 
     def create_agentic_schema(self, payload: dict[str, Any]) -> dict[str, Any]:
         """Create an agentic schema (flat endpoint, ``project`` in body)."""
@@ -878,8 +899,7 @@ def create_agentic_schema(self, payload: dict[str, Any]) -> dict[str, Any]:
 
     def list_agentic_settings(self) -> list[dict[str, Any]]:
         """List agentic settings. Org-wide key/value rows (no project FK)."""
-        result = self._request("GET", "agentic/settings/")
-        return result if isinstance(result, list) else (result or {}).get("results", [])
+        return self._paginate("agentic/settings/")
 
     def create_agentic_setting(self, payload: dict[str, Any]) -> dict[str, Any]:
         """Create an org-wide agentic setting."""
@@ -919,18 +939,14 @@ def list_agentic_registries(
         params: dict[str, Any] = {}
         if agentic_project is not None:
             params["agentic_project"] = agentic_project
-        result = self._request("GET", "agentic-studio-registry/", params=params)
-        return result if isinstance(result, list) else (result or {}).get("results", [])
+        return self._paginate("agentic-studio-registry/", params)
 
     def list_agentic_documents(self, project_id: str) -> list[dict[str, Any]]:
         """List a project's uploaded documents. Rows carry ``id`` and
         ``original_filename``. Agentic docs are a store of their own, distinct
         from Prompt Studio ``prompt-document`` rows.
         """
-        result = self._request(
-            "GET", "agentic/documents/", params={"project_id": project_id}
-        )
-        return result if isinstance(result, list) else (result or {}).get("results", [])
+        return self._paginate("agentic/documents/", {"project_id": project_id})
 
     def download_agentic_document(self, document_id: str) -> bytes:
         """Download an agentic document's original bytes.
@@ -968,10 +984,7 @@ def list_agentic_verified_data(self, project_id: str) -> list[dict[str, Any]]:
         """List a project's verified (ground-truth) data rows. Each carries
         ``document_name``, ``document``, and ``data`` (the curated JSON).
         """
-        result = self._request(
-            "GET", "agentic/verified-data/", params={"project_id": project_id}
-        )
-        return result if isinstance(result, list) else (result or {}).get("results", [])
+        return self._paginate("agentic/verified-data/", {"project_id": project_id})
 
     def create_agentic_verified_data(self, payload: dict[str, Any]) -> dict[str, Any]:
         """Create a verified-data row (``project``, ``document``, ``data``).
diff --git a/tests/clone/test_client.py b/tests/clone/test_client.py
index 561c91a..c11f12f 100644
--- a/tests/clone/test_client.py
+++ b/tests/clone/test_client.py
@@ -7,6 +7,7 @@
 - 204 / empty body returns ``None`` instead of raising on .json().
 - ``get_post_schema`` parses DRF ``actions.POST`` and caches per path.
 - ``close()`` shuts the underlying session; context manager works.
+- ``_paginate`` follows ``next`` to exhaustion and refuses to return a short read.
 """
 
 from __future__ import annotations
@@ -157,3 +158,43 @@ def test_get_review_settings_reraises_non_500():
     with pytest.raises(PlatformAPIError) as exc_info:
         client.get_review_settings("wf-1")
     assert exc_info.value.status_code == 403
+
+
+def _client_with_pages(*payloads) -> tuple[PlatformClient, MagicMock]:
+    """Client whose session returns each payload in turn, one per request."""
+    client = PlatformClient(_endpoint())
+    mock_request = MagicMock(
+        side_effect=[_fake_response(200, p) for p in payloads],
+    )
+    client._session.request = mock_request
+    return client, mock_request
+
+
+def test_paginate_follows_next_across_pages():
+    page1 = {
+        "count": 3,
+        "next": "https://api.example.com/next?page=2",
+        "results": [1, 2],
+    }
+    page2 = {"count": 3, "next": None, "results": [3]}
+    client, mock_request = _client_with_pages(page1, page2)
+
+    assert client.list_tags() == [1, 2, 3]
+    # Second hop must GET the absolute ``next`` URL verbatim, not an org path.
+    assert mock_request.call_args.args[1] == "https://api.example.com/next?page=2"
+
+
+def test_paginate_raises_on_short_read():
+    # A page set that doesn't add up means rows were dropped; a clone that
+    # silently copies a subset is worse than one that fails.
+    truncated = {"count": 9, "next": None, "results": [1, 2]}
+    client, _ = _client_with_pages(truncated)
+    with pytest.raises(PlatformAPIError, match="count=9"):
+        client.list_tags()
+
+
+def test_paginate_raises_on_cyclic_next():
+    looping = {"count": 2, "next": "https://api.example.com/loop", "results": [1]}
+    client, _ = _client_with_pages(looping, looping, looping)
+    with pytest.raises(PlatformAPIError, match="looped"):
+        client.list_tags()

From 64d4cb53b268b2d95807746c9895bd886fcf5911 Mon Sep 17 00:00:00 2001
From: Chandrasekharan M <chandrasekharan@zipstack.com>
Date: Wed, 29 Jul 2026 14:42:06 +0530
Subject: [PATCH 02/10] ci: add focused clone test workflow (UN-3770)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Adds a paths-scoped GitHub Actions workflow that runs the clone test
suite whenever clone code, its tests, or the dependency set changes.
Gives a fast, dedicated signal for the pagination page-following logic
in client._paginate — a silent-truncation regression there is worse
than a hard failure, so it must stay guarded on every clone change.

The full suite in test.yml still runs on every PR; this narrows the
trigger and the run to tests/clone/ for quicker feedback.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
---
 .github/workflows/clone-tests.yml | 52 +++++++++++++++++++++++++++++++
 1 file changed, 52 insertions(+)
 create mode 100644 .github/workflows/clone-tests.yml

diff --git a/.github/workflows/clone-tests.yml b/.github/workflows/clone-tests.yml
new file mode 100644
index 0000000..4b4e1f4
--- /dev/null
+++ b/.github/workflows/clone-tests.yml
@@ -0,0 +1,52 @@
+name: Clone Tests
+
+# Focused, fast signal for the clone tooling — runs only the clone suite when
+# clone code, its tests, or the dependency set changes. The full suite in
+# test.yml still runs on every PR; this gates the pagination page-following
+# logic (client._paginate) that silently truncates a migration if it regresses.
+on:
+  pull_request:
+    branches: [main, "feat/**", "fix/**"]
+    paths:
+      - "src/unstract/clone/**"
+      - "tests/clone/**"
+      - "pyproject.toml"
+      - "uv.lock"
+      - ".github/workflows/clone-tests.yml"
+  push:
+    branches: [main]
+    paths:
+      - "src/unstract/clone/**"
+      - "tests/clone/**"
+      - "pyproject.toml"
+      - "uv.lock"
+      - ".github/workflows/clone-tests.yml"
+
+jobs:
+  clone-tests:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.11", "3.12"]
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v6
+        with:
+          version: "0.6.14"
+          enable-cache: true
+
+      - name: Install dependencies
+        run: uv sync --dev --all-extras
+
+      - name: Create test env
+        run: cp tests/sample.env tests/.env
+
+      - name: Clone tests (pytest)
+        run: uv run pytest tests/clone/ -v

From 4904331c97bf88e646e36ce4a11f2f97e6a72ae3 Mon Sep 17 00:00:00 2001
From: Chandrasekharan M <chandrasekharan@zipstack.com>
Date: Thu, 30 Jul 2026 13:51:39 +0530
Subject: [PATCH 03/10] Harden _paginate: validate every page + reject
 off-origin next links
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Addresses Greptile review on #24:
- Validate the DRF envelope of every page, not just the first — a later
  page that is a bare list / non-envelope now raises PlatformAPIError
  instead of an incidental AttributeError on the next loop turn.
- Reject a `next` link whose origin differs from the configured platform
  endpoint before following it, so a compromised/misconfigured response
  cannot forward the bearer key to another host.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_018HXYFVCZGi7YN9qjDGyJCQ
---
 src/unstract/clone/client.py | 20 ++++++++++++++++++++
 tests/clone/test_client.py   | 18 ++++++++++++++++++
 2 files changed, 38 insertions(+)

diff --git a/src/unstract/clone/client.py b/src/unstract/clone/client.py
index fb60e97..fb2b011 100644
--- a/src/unstract/clone/client.py
+++ b/src/unstract/clone/client.py
@@ -13,6 +13,7 @@
 import json as json_lib
 import logging
 from typing import Any
+from urllib.parse import urlparse
 
 import requests
 
@@ -118,6 +119,20 @@ def _send(
             return None
         return resp.json()
 
+    def _assert_same_origin(self, url: str, label: str) -> None:
+        """Reject a pagination ``next`` link that leaves the platform origin.
+
+        DRF builds ``next`` from the request host, but a compromised or
+        misconfigured response must not redirect the bearer key elsewhere.
+        """
+        base = urlparse(self.endpoint.base_url)
+        target = urlparse(url)
+        if (target.scheme, target.netloc) != (base.scheme, base.netloc):
+            raise PlatformAPIError(
+                f"GET {label} pagination 'next' left the platform origin: "
+                f"{target.scheme}://{target.netloc}"
+            )
+
     def _paginate(
         self, path: str, params: dict[str, Any] | None = None
     ) -> list[dict[str, Any]]:
@@ -145,7 +160,12 @@ def _paginate(
             if next_url in seen:
                 raise PlatformAPIError(f"GET {path} pagination looped at {next_url}")
             seen.add(next_url)
+            self._assert_same_origin(next_url, path)
             result = self._send("GET", next_url, path)
+            if not isinstance(result, dict) or "results" not in result:
+                raise PlatformAPIError(
+                    f"GET {path} returned an unrecognised list payload"
+                )
 
         if expected is not None and len(rows) != expected:
             raise PlatformAPIError(
diff --git a/tests/clone/test_client.py b/tests/clone/test_client.py
index c11f12f..63d7c71 100644
--- a/tests/clone/test_client.py
+++ b/tests/clone/test_client.py
@@ -198,3 +198,21 @@ def test_paginate_raises_on_cyclic_next():
     client, _ = _client_with_pages(looping, looping, looping)
     with pytest.raises(PlatformAPIError, match="looped"):
         client.list_tags()
+
+
+def test_paginate_rejects_offsite_next():
+    # A ``next`` pointing at another host must not receive the bearer key.
+    page1 = {"count": 3, "next": "https://evil.example.com/next", "results": [1, 2]}
+    client, _ = _client_with_pages(page1)
+    with pytest.raises(PlatformAPIError, match="left the platform origin"):
+        client.list_tags()
+
+
+def test_paginate_raises_on_malformed_later_page():
+    # A later page that isn't a DRF envelope must fail loudly, not raise an
+    # incidental AttributeError on the next loop turn.
+    page1 = {"count": 3, "next": "https://api.example.com/next", "results": [1, 2]}
+    page2 = [3]  # bare list where an envelope was expected
+    client, _ = _client_with_pages(page1, page2)
+    with pytest.raises(PlatformAPIError, match="unrecognised list payload"):
+        client.list_tags()

From c38d1efc282842025380ca072ad326e9cdb2b6f4 Mon Sep 17 00:00:00 2001
From: Chandrasekharan M <chandrasekharan@zipstack.com>
Date: Thu, 30 Jul 2026 14:50:12 +0530
Subject: [PATCH 04/10] UN-3770 [FIX] Address Greptile: normalise pagination
 origin, validate every page

- _assert_same_origin compares normalised (scheme, host, port) so equivalent
  hosts (case, explicit default port) aren't rejected as off-site.
- _results_or_raise validates results is a list on every page, so a non-list
  results value fails loudly instead of corrupting rows via extend.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_018HXYFVCZGi7YN9qjDGyJCQ
---
 src/unstract/clone/client.py | 39 +++++++++++++++++++++++++-----------
 tests/clone/test_client.py   | 22 ++++++++++++++++++++
 2 files changed, 49 insertions(+), 12 deletions(-)

diff --git a/src/unstract/clone/client.py b/src/unstract/clone/client.py
index fb2b011..f77b7a7 100644
--- a/src/unstract/clone/client.py
+++ b/src/unstract/clone/client.py
@@ -119,20 +119,41 @@ def _send(
             return None
         return resp.json()
 
+    @staticmethod
+    def _origin(url: str) -> tuple[str, str, int | None]:
+        """Normalised (scheme, host, port) — case- and default-port-insensitive."""
+        parsed = urlparse(url)
+        scheme = parsed.scheme.lower()
+        host = (parsed.hostname or "").lower()
+        port = parsed.port or {"http": 80, "https": 443}.get(scheme)
+        return (scheme, host, port)
+
     def _assert_same_origin(self, url: str, label: str) -> None:
         """Reject a pagination ``next`` link that leaves the platform origin.
 
         DRF builds ``next`` from the request host, but a compromised or
         misconfigured response must not redirect the bearer key elsewhere.
+        Compared on normalised origin so equivalent hosts (case, default port)
+        aren't rejected as off-site.
         """
-        base = urlparse(self.endpoint.base_url)
-        target = urlparse(url)
-        if (target.scheme, target.netloc) != (base.scheme, base.netloc):
+        if self._origin(url) != self._origin(self.endpoint.base_url):
+            scheme, host, _ = self._origin(url)
             raise PlatformAPIError(
                 f"GET {label} pagination 'next' left the platform origin: "
-                f"{target.scheme}://{target.netloc}"
+                f"{scheme}://{host}"
             )
 
+    @staticmethod
+    def _results_or_raise(result: Any, path: str) -> list[Any]:
+        """Return the DRF envelope's ``results`` list or raise on any other shape.
+
+        A non-dict, a missing ``results`` or a ``results`` that isn't a list all
+        fail loudly here rather than corrupting rows via ``extend`` downstream.
+        """
+        if not isinstance(result, dict) or not isinstance(result.get("results"), list):
+            raise PlatformAPIError(f"GET {path} returned an unrecognised list payload")
+        return result["results"]
+
     def _paginate(
         self, path: str, params: dict[str, Any] | None = None
     ) -> list[dict[str, Any]]:
@@ -146,14 +167,12 @@ def _paginate(
         result = self._request("GET", path, params=dict(params or {}))
         if isinstance(result, list):
             return result
-        if not isinstance(result, dict) or "results" not in result:
-            raise PlatformAPIError(f"GET {path} returned an unrecognised list payload")
 
         rows: list[dict[str, Any]] = []
-        expected = result.get("count")
+        expected = result.get("count") if isinstance(result, dict) else None
         seen: set[str] = set()
         while True:
-            rows.extend(result.get("results") or [])
+            rows.extend(self._results_or_raise(result, path))
             next_url = result.get("next")
             if not next_url:
                 break
@@ -162,10 +181,6 @@ def _paginate(
             seen.add(next_url)
             self._assert_same_origin(next_url, path)
             result = self._send("GET", next_url, path)
-            if not isinstance(result, dict) or "results" not in result:
-                raise PlatformAPIError(
-                    f"GET {path} returned an unrecognised list payload"
-                )
 
         if expected is not None and len(rows) != expected:
             raise PlatformAPIError(
diff --git a/tests/clone/test_client.py b/tests/clone/test_client.py
index 63d7c71..2c40b9c 100644
--- a/tests/clone/test_client.py
+++ b/tests/clone/test_client.py
@@ -208,6 +208,28 @@ def test_paginate_rejects_offsite_next():
         client.list_tags()
 
 
+def test_paginate_follows_equivalent_origin_next():
+    # Same origin with uppercase host + explicit default port must be followed,
+    # not rejected as off-site.
+    page1 = {
+        "count": 3,
+        "next": "https://API.EXAMPLE.COM:443/next?page=2",
+        "results": [1, 2],
+    }
+    page2 = {"count": 3, "next": None, "results": [3]}
+    client, _ = _client_with_pages(page1, page2)
+    assert client.list_tags() == [1, 2, 3]
+
+
+def test_paginate_raises_on_nonlist_results():
+    # `results` present but not a list must fail loudly, not corrupt rows via
+    # extend (character-by-character for a string, TypeError for an int).
+    bad = {"count": 1, "next": None, "results": "oops"}
+    client, _ = _client_with_pages(bad)
+    with pytest.raises(PlatformAPIError, match="unrecognised list payload"):
+        client.list_tags()
+
+
 def test_paginate_raises_on_malformed_later_page():
     # A later page that isn't a DRF envelope must fail loudly, not raise an
     # incidental AttributeError on the next loop turn.

From cc0786daab1abf2a406e7f9a3e34e2fb06448b01 Mon Sep 17 00:00:00 2001
From: Chandrasekharan M <chandrasekharan@zipstack.com>
Date: Thu, 30 Jul 2026 15:02:25 +0530
Subject: [PATCH 05/10] UN-3770 [FIX] Guard malformed port in pagination 'next'
 URL

urlparse defers port parsing to attribute access, so a `next` link with a
non-numeric or out-of-range port leaked a ValueError from `_origin` instead
of the actionable PlatformAPIError used for every other malformed link.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_018HXYFVCZGi7YN9qjDGyJCQ
---
 src/unstract/clone/client.py | 12 ++++++++++--
 tests/clone/test_client.py   | 13 +++++++++++++
 2 files changed, 23 insertions(+), 2 deletions(-)

diff --git a/src/unstract/clone/client.py b/src/unstract/clone/client.py
index f77b7a7..24b6bdd 100644
--- a/src/unstract/clone/client.py
+++ b/src/unstract/clone/client.py
@@ -136,8 +136,16 @@ def _assert_same_origin(self, url: str, label: str) -> None:
         Compared on normalised origin so equivalent hosts (case, default port)
         aren't rejected as off-site.
         """
-        if self._origin(url) != self._origin(self.endpoint.base_url):
-            scheme, host, _ = self._origin(url)
+        try:
+            link_origin = self._origin(url)
+        except ValueError as e:
+            # urlparse raises on a non-numeric / out-of-range port only when
+            # ``.port`` is read, so a malformed ``next`` surfaces here.
+            raise PlatformAPIError(
+                f"GET {label} pagination 'next' is a malformed URL: {e}"
+            ) from e
+        if link_origin != self._origin(self.endpoint.base_url):
+            scheme, host, _ = link_origin
             raise PlatformAPIError(
                 f"GET {label} pagination 'next' left the platform origin: "
                 f"{scheme}://{host}"
diff --git a/tests/clone/test_client.py b/tests/clone/test_client.py
index 2c40b9c..2209f85 100644
--- a/tests/clone/test_client.py
+++ b/tests/clone/test_client.py
@@ -221,6 +221,19 @@ def test_paginate_follows_equivalent_origin_next():
     assert client.list_tags() == [1, 2, 3]
 
 
+def test_paginate_raises_on_malformed_port_in_next():
+    # A `next` URL with a non-numeric port makes urlparse raise ValueError on
+    # `.port`; it must surface as PlatformAPIError, not an incidental traceback.
+    page1 = {
+        "count": 3,
+        "next": "https://api.example.com:notaport/next",
+        "results": [1, 2],
+    }
+    client, _ = _client_with_pages(page1)
+    with pytest.raises(PlatformAPIError, match="malformed URL"):
+        client.list_tags()
+
+
 def test_paginate_raises_on_nonlist_results():
     # `results` present but not a list must fail loudly, not corrupt rows via
     # extend (character-by-character for a string, TypeError for an int).

From c9f635757afdf8c563db963ccafaffa0fb30f76e Mon Sep 17 00:00:00 2001
From: Chandrasekharan M <chandrasekharan@zipstack.com>
Date: Thu, 30 Jul 2026 17:20:06 +0530
Subject: [PATCH 06/10] UN-3770 [CI] Pin clone-tests actions to commit SHAs

Addresses Greptile: mutable major-version tags (checkout@v4, setup-python@v5,
setup-uv@v6) could execute unreviewed code on an upstream tag move. Pinned to
full commit SHAs with the version tracked in a trailing comment.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_018HXYFVCZGi7YN9qjDGyJCQ
---
 .github/workflows/clone-tests.yml | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/clone-tests.yml b/.github/workflows/clone-tests.yml
index 4b4e1f4..aeef5be 100644
--- a/.github/workflows/clone-tests.yml
+++ b/.github/workflows/clone-tests.yml
@@ -30,14 +30,16 @@ jobs:
       matrix:
         python-version: ["3.11", "3.12"]
     steps:
-      - uses: actions/checkout@v4
+      # Actions pinned to full commit SHAs (immutable) — a moved major-version
+      # tag can't inject unreviewed code into CI. Comment tracks the version.
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
 
-      - uses: actions/setup-python@v5
+      - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
         with:
           python-version: ${{ matrix.python-version }}
 
       - name: Install uv
-        uses: astral-sh/setup-uv@v6
+        uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6.8.0
         with:
           version: "0.6.14"
           enable-cache: true

From 3dacd587e9a27e35febafdffe4aa526f5385cb23 Mon Sep 17 00:00:00 2001
From: Chandrasekharan M <chandrasekharan@zipstack.com>
Date: Fri, 31 Jul 2026 11:30:05 +0530
Subject: [PATCH 07/10] Address pagination review: tolerate proxy origins +
 empty bodies

Two blockers from review:

- _paginate raised "unrecognised list payload" on a 204/empty body
  because None falls through to _results_or_raise. Restore the old
  (result or {}).get("results", []) behaviour: an empty body returns [].
  A next link that yields an empty body ends pagination; the count guard
  still flags a genuine short read.

- The same-origin check compared scheme+host+port, so a TLS-terminating
  proxy emitting http:// (or off-port) next links for an https:// client
  aborted every paginated list. Compare host only -- the boundary the
  bearer key is actually scoped to -- so the key still can't leak to
  another host while legitimate proxy setups keep working.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_018HXYFVCZGi7YN9qjDGyJCQ
---
 src/unstract/clone/client.py | 29 +++++++++++++++++++----------
 tests/clone/test_client.py   | 28 ++++++++++++++++++++++++++--
 2 files changed, 45 insertions(+), 12 deletions(-)

diff --git a/src/unstract/clone/client.py b/src/unstract/clone/client.py
index 24b6bdd..57860c8 100644
--- a/src/unstract/clone/client.py
+++ b/src/unstract/clone/client.py
@@ -128,27 +128,28 @@ def _origin(url: str) -> tuple[str, str, int | None]:
         port = parsed.port or {"http": 80, "https": 443}.get(scheme)
         return (scheme, host, port)
 
-    def _assert_same_origin(self, url: str, label: str) -> None:
-        """Reject a pagination ``next`` link that leaves the platform origin.
+    def _assert_same_host(self, url: str, label: str) -> None:
+        """Reject a pagination ``next`` link that points at a different host.
 
         DRF builds ``next`` from the request host, but a compromised or
         misconfigured response must not redirect the bearer key elsewhere.
-        Compared on normalised origin so equivalent hosts (case, default port)
-        aren't rejected as off-site.
+        Only the host is compared: a TLS-terminating proxy legitimately emits
+        ``http://`` next links (or a non-default port) for an ``https://``
+        client, and rejecting those would break paginated lists on a backend
+        misconfiguration the client can neither see nor fix. The host is the
+        security boundary the bearer key is scoped to.
         """
         try:
-            link_origin = self._origin(url)
+            link_host = self._origin(url)[1]
         except ValueError as e:
             # urlparse raises on a non-numeric / out-of-range port only when
             # ``.port`` is read, so a malformed ``next`` surfaces here.
             raise PlatformAPIError(
                 f"GET {label} pagination 'next' is a malformed URL: {e}"
             ) from e
-        if link_origin != self._origin(self.endpoint.base_url):
-            scheme, host, _ = link_origin
+        if link_host != self._origin(self.endpoint.base_url)[1]:
             raise PlatformAPIError(
-                f"GET {label} pagination 'next' left the platform origin: "
-                f"{scheme}://{host}"
+                f"GET {label} pagination 'next' left the platform host: {link_host}"
             )
 
     @staticmethod
@@ -173,6 +174,10 @@ def _paginate(
         page is worse than one that fails.
         """
         result = self._request("GET", path, params=dict(params or {}))
+        # A 204 / empty body means "no rows", matching the pre-pagination
+        # ``(result or {}).get("results", [])`` guard the call sites relied on.
+        if result is None:
+            return []
         if isinstance(result, list):
             return result
 
@@ -187,8 +192,12 @@ def _paginate(
             if next_url in seen:
                 raise PlatformAPIError(f"GET {path} pagination looped at {next_url}")
             seen.add(next_url)
-            self._assert_same_origin(next_url, path)
+            self._assert_same_host(next_url, path)
             result = self._send("GET", next_url, path)
+            # A ``next`` link that yields an empty body ends pagination; the
+            # count guard below still flags it as a short read.
+            if result is None:
+                break
 
         if expected is not None and len(rows) != expected:
             raise PlatformAPIError(
diff --git a/tests/clone/test_client.py b/tests/clone/test_client.py
index 2209f85..d9af39f 100644
--- a/tests/clone/test_client.py
+++ b/tests/clone/test_client.py
@@ -204,12 +204,12 @@ def test_paginate_rejects_offsite_next():
     # A ``next`` pointing at another host must not receive the bearer key.
     page1 = {"count": 3, "next": "https://evil.example.com/next", "results": [1, 2]}
     client, _ = _client_with_pages(page1)
-    with pytest.raises(PlatformAPIError, match="left the platform origin"):
+    with pytest.raises(PlatformAPIError, match="left the platform host"):
         client.list_tags()
 
 
 def test_paginate_follows_equivalent_origin_next():
-    # Same origin with uppercase host + explicit default port must be followed,
+    # Same host with uppercase + explicit default port must be followed,
     # not rejected as off-site.
     page1 = {
         "count": 3,
@@ -221,6 +221,30 @@ def test_paginate_follows_equivalent_origin_next():
     assert client.list_tags() == [1, 2, 3]
 
 
+def test_paginate_follows_next_with_different_scheme_or_port():
+    # A TLS-terminating proxy emits an http:// (and/or off-port) next link for
+    # an https:// client. Same host → must be followed, not rejected.
+    page1 = {
+        "count": 3,
+        "next": "http://api.example.com:8080/next?page=2",
+        "results": [1, 2],
+    }
+    page2 = {"count": 3, "next": None, "results": [3]}
+    client, _ = _client_with_pages(page1, page2)
+    assert client.list_tags() == [1, 2, 3]
+
+
+def test_paginate_empty_body_returns_empty_list():
+    # A 204 / empty first page means "no rows", not a malformed payload — it
+    # must return [] like the pre-pagination ``(result or {}).get`` guard did.
+    client = PlatformClient(_endpoint())
+    empty = MagicMock()
+    empty.status_code = 204
+    empty.content = b""
+    client._session.request = MagicMock(return_value=empty)
+    assert client.list_tags() == []
+
+
 def test_paginate_raises_on_malformed_port_in_next():
     # A `next` URL with a non-numeric port makes urlparse raise ValueError on
     # `.port`; it must surface as PlatformAPIError, not an incidental traceback.

From e8ea582880e671493d5b902e5492dfd166798900 Mon Sep 17 00:00:00 2001
From: Chandrasekharan M
 <117059509+chandrasekharan-zipstack@users.noreply.github.com>
Date: Fri, 31 Jul 2026 11:43:09 +0530
Subject: [PATCH 08/10] Update src/unstract/clone/client.py

Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com>
Signed-off-by: Chandrasekharan M <117059509+chandrasekharan-zipstack@users.noreply.github.com>
---
 src/unstract/clone/client.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/unstract/clone/client.py b/src/unstract/clone/client.py
index 57860c8..4b169ce 100644
--- a/src/unstract/clone/client.py
+++ b/src/unstract/clone/client.py
@@ -189,6 +189,10 @@ def _paginate(
             next_url = result.get("next")
             if not next_url:
                 break
+            if not isinstance(next_url, str):
+                raise PlatformAPIError(
+                    f"GET {path} pagination 'next' is not a URL string"
+                )
             if next_url in seen:
                 raise PlatformAPIError(f"GET {path} pagination looped at {next_url}")
             seen.add(next_url)

From 587317b2c3b96afe2196685ded8cd9dbbae9f380 Mon Sep 17 00:00:00 2001
From: Chandrasekharan M <chandrasekharan@zipstack.com>
Date: Fri, 31 Jul 2026 11:58:23 +0530
Subject: [PATCH 09/10] Pin pagination next links to the configured origin

Two Greptile P1s on the previous review round:

- Host-only origin check let an https client follow an http:// (or
  off-port) next link on the same host, putting the bearer key on the
  wire in plaintext or at an unrelated service. Follow the link but pin
  scheme+host+port to the configured base_url, keeping only the server's
  path+query, so the key only ever reaches the configured origin. An
  off-host next is still rejected. Replaces _origin/_assert_same_host
  with _same_origin_url.

- A truthy non-string next (int/list) blew up in seen.add / urlparse
  with an incidental TypeError; guard it and raise PlatformAPIError.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_018HXYFVCZGi7YN9qjDGyJCQ
---
 src/unstract/clone/client.py | 49 +++++++++++++++++-------------------
 tests/clone/test_client.py   | 20 ++++++++++++---
 2 files changed, 40 insertions(+), 29 deletions(-)

diff --git a/src/unstract/clone/client.py b/src/unstract/clone/client.py
index 4b169ce..424ba8a 100644
--- a/src/unstract/clone/client.py
+++ b/src/unstract/clone/client.py
@@ -13,7 +13,7 @@
 import json as json_lib
 import logging
 from typing import Any
-from urllib.parse import urlparse
+from urllib.parse import urlparse, urlunparse
 
 import requests
 
@@ -119,38 +119,36 @@ def _send(
             return None
         return resp.json()
 
-    @staticmethod
-    def _origin(url: str) -> tuple[str, str, int | None]:
-        """Normalised (scheme, host, port) — case- and default-port-insensitive."""
-        parsed = urlparse(url)
-        scheme = parsed.scheme.lower()
-        host = (parsed.hostname or "").lower()
-        port = parsed.port or {"http": 80, "https": 443}.get(scheme)
-        return (scheme, host, port)
-
-    def _assert_same_host(self, url: str, label: str) -> None:
-        """Reject a pagination ``next`` link that points at a different host.
-
-        DRF builds ``next`` from the request host, but a compromised or
-        misconfigured response must not redirect the bearer key elsewhere.
-        Only the host is compared: a TLS-terminating proxy legitimately emits
-        ``http://`` next links (or a non-default port) for an ``https://``
-        client, and rejecting those would break paginated lists on a backend
-        misconfiguration the client can neither see nor fix. The host is the
-        security boundary the bearer key is scoped to.
+    def _same_origin_url(self, url: str, label: str) -> str:
+        """Pin a pagination ``next`` link to the configured platform origin.
+
+        The bearer key must never be sent over a scheme/port the server picked:
+        a TLS-terminating proxy without ``SECURE_PROXY_SSL_HEADER`` emits
+        ``http://`` (or off-port) ``next`` links for an ``https://`` client, and
+        following those verbatim would put the credential on the wire in
+        plaintext or at an unrelated service. So only the host is trusted for
+        equality (a differing host is a compromised/misconfigured response and
+        is rejected outright), and the scheme, host and port of the followed
+        request are always taken from the configured ``base_url`` — keeping only
+        the server's path and query. The key therefore only ever reaches the
+        origin the client was configured with.
         """
         try:
-            link_host = self._origin(url)[1]
+            link = urlparse(url)
+            link_host = (link.hostname or "").lower()
+            link.port  # noqa: B018 -- forces the ValueError on a malformed port
         except ValueError as e:
-            # urlparse raises on a non-numeric / out-of-range port only when
-            # ``.port`` is read, so a malformed ``next`` surfaces here.
             raise PlatformAPIError(
                 f"GET {label} pagination 'next' is a malformed URL: {e}"
             ) from e
-        if link_host != self._origin(self.endpoint.base_url)[1]:
+        base = urlparse(self.endpoint.base_url)
+        if link_host != (base.hostname or "").lower():
             raise PlatformAPIError(
                 f"GET {label} pagination 'next' left the platform host: {link_host}"
             )
+        return urlunparse(
+            (base.scheme, base.netloc, link.path, link.params, link.query, link.fragment)
+        )
 
     @staticmethod
     def _results_or_raise(result: Any, path: str) -> list[Any]:
@@ -196,8 +194,7 @@ def _paginate(
             if next_url in seen:
                 raise PlatformAPIError(f"GET {path} pagination looped at {next_url}")
             seen.add(next_url)
-            self._assert_same_host(next_url, path)
-            result = self._send("GET", next_url, path)
+            result = self._send("GET", self._same_origin_url(next_url, path), path)
             # A ``next`` link that yields an empty body ends pagination; the
             # count guard below still flags it as a short read.
             if result is None:
diff --git a/tests/clone/test_client.py b/tests/clone/test_client.py
index d9af39f..1c0a1ff 100644
--- a/tests/clone/test_client.py
+++ b/tests/clone/test_client.py
@@ -221,17 +221,31 @@ def test_paginate_follows_equivalent_origin_next():
     assert client.list_tags() == [1, 2, 3]
 
 
-def test_paginate_follows_next_with_different_scheme_or_port():
+def test_paginate_pins_next_to_configured_origin():
     # A TLS-terminating proxy emits an http:// (and/or off-port) next link for
-    # an https:// client. Same host → must be followed, not rejected.
+    # an https:// client. Same host → followed, but the request is pinned back
+    # to the configured https origin so the bearer never goes over plaintext or
+    # an unrelated port. Only the path + query are taken from the server.
     page1 = {
         "count": 3,
         "next": "http://api.example.com:8080/next?page=2",
         "results": [1, 2],
     }
     page2 = {"count": 3, "next": None, "results": [3]}
-    client, _ = _client_with_pages(page1, page2)
+    client, mock_request = _client_with_pages(page1, page2)
     assert client.list_tags() == [1, 2, 3]
+    # Second hop must go to the configured https origin, not the http:8080 the
+    # server returned.
+    assert mock_request.call_args.args[1] == "https://api.example.com/next?page=2"
+
+
+def test_paginate_raises_on_non_string_next():
+    # A truthy non-string `next` must fail loudly, not blow up in seen.add /
+    # urlparse with an incidental TypeError.
+    page1 = {"count": 3, "next": 12345, "results": [1, 2]}
+    client, _ = _client_with_pages(page1)
+    with pytest.raises(PlatformAPIError, match="not a URL string"):
+        client.list_tags()
 
 
 def test_paginate_empty_body_returns_empty_list():

From 2f7956560abeeea39b8875de6f012ed8232e087e Mon Sep 17 00:00:00 2001
From: Chandrasekharan M <chandrasekharan@zipstack.com>
Date: Fri, 31 Jul 2026 12:01:21 +0530
Subject: [PATCH 10/10] Wrap urlunparse args to satisfy ruff E501

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_018HXYFVCZGi7YN9qjDGyJCQ
---
 src/unstract/clone/client.py | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/src/unstract/clone/client.py b/src/unstract/clone/client.py
index 424ba8a..0103106 100644
--- a/src/unstract/clone/client.py
+++ b/src/unstract/clone/client.py
@@ -147,7 +147,14 @@ def _same_origin_url(self, url: str, label: str) -> str:
                 f"GET {label} pagination 'next' left the platform host: {link_host}"
             )
         return urlunparse(
-            (base.scheme, base.netloc, link.path, link.params, link.query, link.fragment)
+            (
+                base.scheme,
+                base.netloc,
+                link.path,
+                link.params,
+                link.query,
+                link.fragment,
+            )
         )
 
     @staticmethod