diff --git a/cmd/relayfile-cli/main.go b/cmd/relayfile-cli/main.go index d292b94c..0e04cd6a 100644 --- a/cmd/relayfile-cli/main.go +++ b/cmd/relayfile-cli/main.go @@ -425,10 +425,17 @@ type syncStateEventListener struct { // syncStateBootstrap is the CLI-surface mirror of mountsync's public // bootstrap status block. type syncStateBootstrap struct { - Phase string `json:"phase"` - FilesSynced int `json:"filesSynced"` - FilesTotal int `json:"filesTotal,omitempty"` - StartedAt string `json:"startedAt,omitempty"` + Phase string `json:"phase"` + FilesSynced int `json:"filesSynced"` + FilesTotal int `json:"filesTotal,omitempty"` + StartedAt string `json:"startedAt,omitempty"` + CurrentPath string `json:"currentPath,omitempty"` + PageOffset int `json:"pageOffset,omitempty"` + DirectoriesPending int `json:"directoriesPending,omitempty"` + DirectoriesDiscovered int `json:"directoriesDiscovered,omitempty"` + StallCycles int `json:"stallCycles,omitempty"` + StallLimit int `json:"stallLimit,omitempty"` + Reason string `json:"reason,omitempty"` } // syncStateGuards mirrors mountsync.telemetryCounters and the circuit @@ -6741,6 +6748,9 @@ func buildWorkspaceViewHealth(record workspaceRecord, state syncStateFile) []wor func canonicalViewStatus(state syncStateFile) string { if state.Bootstrap != nil { + if phase := strings.TrimSpace(state.Bootstrap.Phase); phase != "" { + return phase + } return "bootstrapping" } if strings.TrimSpace(state.Status) == "" { @@ -9083,6 +9093,9 @@ func runStatus(args []string, stdout io.Writer) error { workspaceLabel = fmt.Sprintf("%s (%s)", workspaceID, record.Name) } fmt.Fprintf(stdout, "workspace %s mode: %s lag: %s\n", workspaceLabel, snapshot.Mode, formatLag(maxLagSeconds(status.Providers))) + if mountStatus := operatorMountStatus(snapshot.Status); mountStatus != "" { + fmt.Fprintf(stdout, "mount: %s\n", mountStatus) + } if authLine := statusAuthLine(record.LocalDir, time.Now().UTC()); authLine != "" { fmt.Fprintln(stdout, authLine) } @@ -9125,17 +9138,28 @@ func runStatus(args []string, stdout io.Writer) error { fmt.Fprintln(stdout, daemonStatusLine(record)) } if snapshot.Bootstrap != nil { - // Initial mirror in progress: show progress instead of a - // misleading generic stall. - line := fmt.Sprintf("\nbootstrapping: %d", snapshot.Bootstrap.FilesSynced) - if snapshot.Bootstrap.FilesTotal > 0 { - line += fmt.Sprintf("/%d", snapshot.Bootstrap.FilesTotal) + phase := strings.ToLower(strings.TrimSpace(snapshot.Bootstrap.Phase)) + prefix := "bootstrapping" + if phase == "stalled" { + prefix = "bootstrap stalled" } - line += " files" + line := "\n" + prefix + ": " + formatBootstrapFileProgress(snapshot.Bootstrap) if started := strings.TrimSpace(snapshot.Bootstrap.StartedAt); started != "" { line += " (started " + humanizeRecentTime(started) + ")" } fmt.Fprintln(stdout, line) + if snapshot.Bootstrap.DirectoriesPending > 0 { + fmt.Fprintf(stdout, " directories pending: %d\n", snapshot.Bootstrap.DirectoriesPending) + } + if currentPath := strings.TrimSpace(snapshot.Bootstrap.CurrentPath); currentPath != "" { + fmt.Fprintf(stdout, " current path: %s (page offset %d)\n", currentPath, snapshot.Bootstrap.PageOffset) + } + if phase == "stalled" { + reason := firstNonBlank(snapshot.Bootstrap.Reason, persistedStallReason) + if reason != "" { + fmt.Fprintf(stdout, " reason: %s\n", reason) + } + } } else if persistedStallReason != "" { fmt.Fprintf(stdout, "\nstall: %s\n", persistedStallReason) } @@ -9143,6 +9167,24 @@ func runStatus(args []string, stdout io.Writer) error { return nil } +func operatorMountStatus(status string) string { + status = strings.ToLower(strings.TrimSpace(status)) + if status == "ready" { + return "healthy" + } + return status +} + +func formatBootstrapFileProgress(bootstrap *syncStateBootstrap) string { + if bootstrap == nil { + return "0 files synced" + } + if bootstrap.FilesTotal > 0 { + return fmt.Sprintf("%d/%d files", bootstrap.FilesSynced, bootstrap.FilesTotal) + } + return fmt.Sprintf("%d files synced (authoritative total unavailable)", bootstrap.FilesSynced) +} + func readPersistedStallReason(localDir string) string { if localDir == "" { return "" @@ -11531,16 +11573,28 @@ func providerReadyForMirror(client *apiClient, workspaceID, provider string, sta } func buildSyncStateSnapshot(status syncStatusResponse, workspaceID, mode string, interval time.Duration, localDir string, pid int, stallReason string) syncStateFile { + var localState syncStateFile + var pendingConflicts, deniedPaths int + if strings.TrimSpace(localDir) != "" { + localState = readWritebackStateBestEffort(localDir) + pendingConflicts = countFilesInDir(filepath.Join(localDir, ".relay", "conflicts")) + deniedPaths = countLines(filepath.Join(localDir, ".relay", "permissions-denied.log")) + } snapshot := syncStateFile{ - WorkspaceID: workspaceID, - RemoteRoot: readMountRemoteRoot(localDir), - Mode: defaultIfBlank(mode, defaultMountMode), - IntervalMs: interval.Milliseconds(), - PendingWriteback: countDirtyTrackedFiles(localDir), - PendingConflicts: countFilesInDir(filepath.Join(localDir, ".relay", "conflicts")), - DeniedPaths: countLines(filepath.Join(localDir, ".relay", "permissions-denied.log")), - FailedWritebacks: readPersistedFailedWritebacks(localDir), - StallReason: stallReason, + WorkspaceID: workspaceID, + RemoteRoot: readMountRemoteRoot(localDir), + Mode: defaultIfBlank(mode, defaultMountMode), + Status: strings.TrimSpace(localState.Status), + IntervalMs: interval.Milliseconds(), + LastReconcileAt: strings.TrimSpace(localState.LastReconcileAt), + LastSuccessfulReconcileAt: strings.TrimSpace(localState.LastSuccessfulReconcileAt), + PendingWriteback: countDirtyTrackedFiles(localDir), + PendingConflicts: pendingConflicts, + DeniedPaths: deniedPaths, + FailedWritebacks: readPersistedFailedWritebacks(localDir), + StallReason: stallReason, + LastError: localState.LastError, + IncrementalReadNotReadySince: localState.IncrementalReadNotReadySince, } if pid != 0 { snapshot.Daemon = &syncStateDaemon{ @@ -11577,8 +11631,16 @@ func buildSyncStateSnapshot(status syncStatusResponse, workspaceID, mode string, snapshot.Providers = providers snapshot.LastEventAt = lastEvent snapshot.Guards = readGuardCounters(localDir) - snapshot.Bootstrap = readBootstrapStatus(localDir) - snapshot.EventListener = readEventListenerHealth(localDir) + snapshot.Bootstrap = localState.Bootstrap + snapshot.EventListener = localState.EventListener + switch { + case snapshot.Bootstrap != nil && strings.EqualFold(snapshot.Bootstrap.Phase, "stalled"): + snapshot.Status = "stalled" + case snapshot.Bootstrap != nil: + snapshot.Status = "bootstrapping" + case strings.TrimSpace(stallReason) != "": + snapshot.Status = "stalled" + } return snapshot } @@ -11603,23 +11665,6 @@ func readBootstrapStatus(localDir string) *syncStateBootstrap { return view.Bootstrap } -func readEventListenerHealth(localDir string) *syncStateEventListener { - if localDir == "" { - return nil - } - payload, err := os.ReadFile(filepath.Join(localDir, ".relay", "state.json")) - if err != nil { - return nil - } - var view struct { - EventListener *syncStateEventListener `json:"eventListener"` - } - if json.Unmarshal(payload, &view) != nil { - return nil - } - return view.EventListener -} - // readGuardCounters reads the mountsync public state file under // .relay/state.json and copies the telemetry counters + circuit snapshot // into the CLI-surface shape. Returns nil if the state file is missing @@ -13848,6 +13893,20 @@ func runMountLoopWithAuthLock(rootCtx context.Context, syncer *mountsync.Syncer, writeSnapshot() return err } + if mountsync.IsBootstrapTerminalError(err) { + // The CLI daemon used to treat this typed hard stop as one ordinary + // failed cycle, which let supervisors and the poll ticker retry the + // identical checkpoint forever. Persist the actionable reason and + // let it escape the runner. + reason := err.Error() + if bs := readBootstrapStatus(localDir); bs != nil && strings.TrimSpace(bs.Reason) != "" { + reason = strings.TrimSpace(bs.Reason) + } + setStallReason(reason) + log.Printf("mount bootstrap terminal failure: %v", err) + writeSnapshot() + return err + } // Mid-bootstrap, a per-cycle deadline exceeded is expected // progress, not a stall: the rootCtx-derived bootstrap // context keeps the heavy pull alive across cycles and @@ -13856,7 +13915,7 @@ func runMountLoopWithAuthLock(rootCtx context.Context, syncer *mountsync.Syncer, if errors.Is(err, context.DeadlineExceeded) { if bs := readBootstrapStatus(localDir); bs != nil { setStallReason("") - log.Printf("mount bootstrapping: %d/%d files (in progress)", bs.FilesSynced, bs.FilesTotal) + log.Printf("mount bootstrapping: %s (in progress)", formatBootstrapFileProgress(bs)) writeSnapshot() return err } @@ -13915,6 +13974,9 @@ func runMountLoopWithAuthLock(rootCtx context.Context, syncer *mountsync.Syncer, log.Print(mountStartBanner(localDir, interval, intervalJitter)) initialErr := runCycle(true) logStuckEventSummary(syncer, initialErr) + if mountsync.IsBootstrapTerminalError(initialErr) { + return initialErr + } if once { return initialErr } @@ -13942,14 +14004,22 @@ func runMountLoopWithAuthLock(rootCtx context.Context, syncer *mountsync.Syncer, cycle++ reconcile := shouldReconcileMountCycle(websocketEnabled && watcherActive, cycle) if reconcile { - _ = runCycle(true) + if err := runCycle(true); mountsync.IsBootstrapTerminalError(err) { + return err + } } if !isDegraded() && time.Since(lastSuccessAt()) >= 10*time.Minute { if bs := readBootstrapStatus(localDir); bs != nil { - // Long-running initial mirror is making progress - // across cycles — not a stall. - setStallReason("") - log.Printf("mount bootstrapping: %d/%d files (in progress)", bs.FilesSynced, bs.FilesTotal) + if strings.EqualFold(bs.Phase, "stalled") { + reason := firstNonBlank(bs.Reason, currentStallReason()) + setStallReason(reason) + log.Printf("mount bootstrap stalled: %s; path=%s directories_pending=%d reason=%s", formatBootstrapFileProgress(bs), bs.CurrentPath, bs.DirectoriesPending, reason) + } else { + // Long-running initial mirror is making progress + // across cycles — not a stall. + setStallReason("") + log.Printf("mount bootstrapping: %s (in progress)", formatBootstrapFileProgress(bs)) + } writeSnapshot() } else { setStallReason("no successful reconcile for 10m") diff --git a/cmd/relayfile-cli/main_test.go b/cmd/relayfile-cli/main_test.go index 44a9aec1..9056d6ff 100644 --- a/cmd/relayfile-cli/main_test.go +++ b/cmd/relayfile-cli/main_test.go @@ -5804,6 +5804,101 @@ func TestStatusSurfacesDegradedStallReason(t *testing.T) { } } +func TestStatusDistinguishesStalledBootstrapFromHealthyProvider(t *testing.T) { + t.Setenv("HOME", t.TempDir()) + clearRelayfileEnv(t) + + localDir := t.TempDir() + if err := ensureMirrorLayout(localDir); err != nil { + t.Fatalf("ensureMirrorLayout failed: %v", err) + } + stallReason := `bootstrap stalled for 20 consecutive checkpoint-stable cycles (limit 20, path "/neon/advisors/by-project", page cursor "", page offset 0, 14822 directories pending)` + if err := writeMirrorStateFile(localDir, syncStateFile{ + WorkspaceID: "rw_7ccfea89", + Mode: defaultMountMode, + Status: "stalled", + StallReason: stallReason, + LastError: &statusError{ + Kind: "bootstrap_stalled", + Code: "bootstrap_stall_cycle_limit", + Message: stallReason, + }, + Bootstrap: &syncStateBootstrap{ + Phase: "stalled", + FilesSynced: 27392, + FilesTotal: 40000, + StartedAt: time.Now().UTC().Add(-164 * time.Hour).Format(time.RFC3339Nano), + CurrentPath: "/neon/advisors/by-project", + DirectoriesPending: 14822, + StallCycles: 20, + StallLimit: 20, + Reason: stallReason, + }, + }); err != nil { + t.Fatalf("writeMirrorStateFile failed: %v", err) + } + if _, err := upsertWorkspaceDetails(workspaceRecord{ + Name: "default", + ID: "rw_7ccfea89", + LocalDir: localDir, + CreatedAt: time.Now().UTC().Format(time.RFC3339), + LastUsedAt: time.Now().UTC().Format(time.RFC3339), + }); err != nil { + t.Fatalf("upsertWorkspaceDetails failed: %v", err) + } + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, `{"workspaceId":"rw_7ccfea89","providers":[{"provider":"github","status":"healthy","lagSeconds":0}]}`) + })) + defer server.Close() + writeDelegatedCredentialsForTest(t, delegatedauth.Bundle{ + RelayfileURL: server.URL, + RelayfileWorkspaceID: "rw_7ccfea89", + AccessToken: "delegated_token", + }) + + var stdout bytes.Buffer + if err := run([]string{"status", "default"}, strings.NewReader(""), &stdout, &stdout); err != nil { + t.Fatalf("run status failed: %v", err) + } + got := stdout.String() + for _, want := range []string{ + "mount: stalled", + "github healthy", + "bootstrap stalled: 27392/40000 files", + "directories pending: 14822", + "current path: /neon/advisors/by-project", + "reason: bootstrap stalled", + } { + if !strings.Contains(got, want) { + t.Fatalf("status output missing %q: %q", want, got) + } + } + if strings.Contains(got, "\nbootstrapping:") { + t.Fatalf("stalled bootstrap was rendered as in-progress: %q", got) + } + + var jsonOutput bytes.Buffer + if err := run([]string{"status", "default", "--json"}, strings.NewReader(""), &jsonOutput, &jsonOutput); err != nil { + t.Fatalf("run status --json failed: %v", err) + } + var snapshot syncStateFile + if err := json.Unmarshal(jsonOutput.Bytes(), &snapshot); err != nil { + t.Fatalf("decode status JSON: %v\n%s", err, jsonOutput.String()) + } + if snapshot.Status != "stalled" || snapshot.Bootstrap == nil || snapshot.Bootstrap.Phase != "stalled" { + t.Fatalf("status JSON hid stalled bootstrap: status=%q bootstrap=%+v", snapshot.Status, snapshot.Bootstrap) + } +} + +func TestBootstrapProgressNeverRendersZeroDenominator(t *testing.T) { + got := formatBootstrapFileProgress(&syncStateBootstrap{FilesSynced: 27392}) + if strings.Contains(got, "/0") || !strings.Contains(got, "total unavailable") { + t.Fatalf("unknown denominator rendered as fake completion ratio: %q", got) + } +} + func TestEnsureMirrorLayoutDoesNotRewriteUnchangedSkill(t *testing.T) { localDir := t.TempDir() if err := ensureMirrorLayout(localDir); err != nil { diff --git a/cmd/relayfile-cli/mount_up_path_priority_test.go b/cmd/relayfile-cli/mount_up_path_priority_test.go index 6e758891..97c8456c 100644 --- a/cmd/relayfile-cli/mount_up_path_priority_test.go +++ b/cmd/relayfile-cli/mount_up_path_priority_test.go @@ -2,10 +2,15 @@ package main import ( "context" + "encoding/json" + "errors" "io" "log" + "net/http" + "net/http/httptest" "os" "path/filepath" + "strings" "sync" "testing" "time" @@ -14,6 +19,87 @@ import ( "github.com/agentworkforce/relayfile/internal/relayfile" ) +func TestCLIMountLoopStopsAndPersistsStatusOnBootstrapStall(t *testing.T) { + t.Setenv("RELAYFILE_MOUNT_FORCE_RECURSIVE_WATCHER", "1") + const workspaceID = "ws_cli_bootstrap_stall" + var treeCalls int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + switch { + case strings.HasSuffix(r.URL.Path, "/sync/status"): + _, _ = io.WriteString(w, `{"workspaceId":"ws_cli_bootstrap_stall","providers":[{"provider":"github","status":"healthy","lagSeconds":0}]}`) + case strings.HasSuffix(r.URL.Path, "/fs/export"): + w.WriteHeader(http.StatusBadRequest) + _, _ = io.WriteString(w, `{"code":"bad_request","message":"use paginated tree"}`) + case strings.HasSuffix(r.URL.Path, "/fs/tree"): + treeCalls++ + w.WriteHeader(http.StatusBadRequest) + _, _ = io.WriteString(w, `{"code":"bad_request","message":"stuck subtree"}`) + default: + http.NotFound(w, r) + } + })) + defer server.Close() + + localDir := t.TempDir() + client := mountsync.NewHTTPClient(server.URL, "test-token", server.Client()) + syncer, err := mountsync.NewSyncer(client, mountsync.SyncerOptions{ + WorkspaceID: workspaceID, + RemoteRoot: "/neon/advisors/by-project", + LocalRoot: localDir, + StateDir: t.TempDir(), + RootCtx: context.Background(), + WebSocket: boolPtr(false), + BootstrapStallCycles: 1, + Logger: log.New(io.Discard, "", 0), + }) + if err != nil { + t.Fatalf("NewSyncer: %v", err) + } + + previousLogWriter := log.Writer() + log.SetOutput(io.Discard) + defer log.SetOutput(previousLogWriter) + err = runMountLoop( + context.Background(), + syncer, + localDir, + workspaceID, + server.URL, + "", + time.Second, + time.Hour, + 0, + false, + false, + false, + mountPIDFile(localDir), + mountLogFile(localDir), + ) + var stalled *mountsync.BootstrapStalledError + if !errors.As(err, &stalled) { + t.Fatalf("CLI mount loop must return terminal bootstrap stall, got %v", err) + } + if treeCalls != 1 { + t.Fatalf("tree calls = %d, want one terminal attempt", treeCalls) + } + + payload, readErr := os.ReadFile(filepath.Join(localDir, ".relay", "state.json")) + if readErr != nil { + t.Fatalf("read public state: %v", readErr) + } + var state syncStateFile + if err := json.Unmarshal(payload, &state); err != nil { + t.Fatalf("decode public state: %v", err) + } + if state.Status != "stalled" || state.Bootstrap == nil || state.Bootstrap.Phase != "stalled" { + t.Fatalf("CLI status writer hid hard stall: status=%q bootstrap=%+v", state.Status, state.Bootstrap) + } + if state.Bootstrap.CurrentPath != "/neon/advisors/by-project" || !strings.Contains(state.StallReason, "stuck subtree") { + t.Fatalf("CLI status lacks actionable subtree/reason: %+v stall=%q", state.Bootstrap, state.StallReason) + } +} + // blockingBootstrapClient embeds the full RemoteClient contract and overrides // only the calls this regression reaches. The nil promoted methods are a // deliberate tripwire: if the bootstrap flow unexpectedly leaves the export diff --git a/cmd/relayfile-mount/main.go b/cmd/relayfile-mount/main.go index 491a345d..2636a991 100644 --- a/cmd/relayfile-mount/main.go +++ b/cmd/relayfile-mount/main.go @@ -484,8 +484,7 @@ func runSinglePollingMount(rootCtx context.Context, cfg mountConfig) error { err = syncer.SyncOnce(ctx) } if err != nil { - var stalled *mountsync.BootstrapStalledError - if errors.As(err, &stalled) { + if mountsync.IsBootstrapTerminalError(err) { // This is an operator-actionable hard stop, not a transient // cycle failure. Returning it terminates this runner (and, for // scoped layouts, cancels sibling runners) instead of letting @@ -494,7 +493,7 @@ func runSinglePollingMount(rootCtx context.Context, cfg mountConfig) error { } if errors.Is(err, context.DeadlineExceeded) { if synced, total, ok := readBootstrapProgress(cfg.localDir); ok { - log.Printf("mount bootstrapping: %d/%d files (in progress)", synced, total) + log.Printf("mount bootstrapping: %s (in progress)", formatBootstrapProgress(synced, total)) return nil } } @@ -694,6 +693,13 @@ func readBootstrapProgress(localDir string) (synced, total int, ok bool) { return view.Bootstrap.FilesSynced, view.Bootstrap.FilesTotal, true } +func formatBootstrapProgress(synced, total int) string { + if total > 0 { + return fmt.Sprintf("%d/%d files", synced, total) + } + return fmt.Sprintf("%d files synced (authoritative total unavailable)", synced) +} + func envOrDefault(name, fallback string) string { value := strings.TrimSpace(os.Getenv(name)) if value == "" { diff --git a/docs/environment-variables.md b/docs/environment-variables.md index d3545000..8bc457f0 100644 --- a/docs/environment-variables.md +++ b/docs/environment-variables.md @@ -138,6 +138,8 @@ Queue backend precedence is: | `RELAYFILE_MOUNT_INTERVAL_JITTER` | float | `0.2` | No | Clamped into the `0..1` range | | `RELAYFILE_MOUNT_TIMEOUT` | duration | `15s` | No | Per-sync timeout | | `RELAYFILE_MOUNT_WEBSOCKET` | bool | `true` | No | Enables WebSocket streaming when available | +| `RELAYFILE_BOOTSTRAP_STALL_CYCLES` | int | `20` | No | Consecutive checkpoint-stable bootstrap cycles before the mount fails terminally | +| `RELAYFILE_BOOTSTRAP_MAX_DIRECTORIES` | int | `50000` | No | Maximum distinct bounded-tree directories; raise only after inspecting the stalled path for cyclic/expanding aliases | ## CLI Package: `cmd/relayfile-cli` @@ -165,6 +167,8 @@ the `workspace_id`/`wks` claim in the active token, then the default stored by | `RELAYFILE_MOUNT_INTERVAL_JITTER` | float | `0.2` | Mount command jitter default | | `RELAYFILE_MOUNT_TIMEOUT` | duration | `15s` | Mount command timeout default | | `RELAYFILE_MOUNT_WEBSOCKET` | bool | `true` | Mount command WebSocket default | +| `RELAYFILE_BOOTSTRAP_STALL_CYCLES` | int | `20` | Consecutive checkpoint-stable bootstrap cycles before the CLI daemon exits non-zero | +| `RELAYFILE_BOOTSTRAP_MAX_DIRECTORIES` | int | `50000` | Maximum distinct bootstrap traversal directories; raising it re-arms the persisted checkpoint after operator inspection | Server resolution order in the CLI is: diff --git a/internal/httpapi/server.go b/internal/httpapi/server.go index e6f8f9de..20982392 100644 --- a/internal/httpapi/server.go +++ b/internal/httpapi/server.go @@ -1515,7 +1515,7 @@ func (s *Server) handleTree(w http.ResponseWriter, r *http.Request, workspaceID, writeForkAwareError(w, err, correlationID) return } - if len(resp.Entries) > 0 { + if len(resp.Entries) > 0 || resp.TotalFiles > 0 { base := normalizeRoutePath(resp.Path) visibleFiles := map[string]struct{}{} visibleDirs := map[string]struct{}{} @@ -1586,6 +1586,10 @@ func (s *Server) handleTree(w http.ResponseWriter, r *http.Request, workspaceID, } } resp.Entries = filtered + // Store-level totals include every file below the requested path. The + // HTTP contract must expose only files this caller can see, while still + // keeping the total stable across pagination pages. + resp.TotalFiles = len(visibleFiles) } writeJSON(w, http.StatusOK, resp) } diff --git a/internal/httpapi/server_test.go b/internal/httpapi/server_test.go index 3185cd8f..85a340a7 100644 --- a/internal/httpapi/server_test.go +++ b/internal/httpapi/server_test.go @@ -2338,6 +2338,9 @@ func TestTreeEndpointPaginatesBoundedEntries(t *testing.T) { if len(pageOne.Entries) != 1000 { t.Fatalf("expected 1000 page-one entries, got %d", len(pageOne.Entries)) } + if pageOne.TotalFiles != 1002 { + t.Fatalf("page-one totalFiles = %d, want 1002", pageOne.TotalFiles) + } if pageOne.NextCursor == nil || *pageOne.NextCursor != "/notion/Paged/File0999.md" { t.Fatalf("unexpected page-one next cursor: %v", pageOne.NextCursor) } @@ -2360,6 +2363,9 @@ func TestTreeEndpointPaginatesBoundedEntries(t *testing.T) { if len(pageTwo.Entries) != 2 { t.Fatalf("expected 2 page-two entries, got %d", len(pageTwo.Entries)) } + if pageTwo.TotalFiles != 1002 { + t.Fatalf("page-two totalFiles = %d, want stable total 1002", pageTwo.TotalFiles) + } if pageTwo.NextCursor != nil { t.Fatalf("expected nil page-two next cursor, got %q", *pageTwo.NextCursor) } @@ -2779,6 +2785,9 @@ func TestTreeEndpointFiltersUnauthorizedFiles(t *testing.T) { if _, ok := paths["/notion/private"]; ok { t.Fatalf("expected restricted dir to be hidden when no accessible descendants") } + if tree.TotalFiles != 1 { + t.Fatalf("permission-filtered totalFiles = %d, want 1 caller-visible file", tree.TotalFiles) + } } func TestFilePermissionPolicyDenyOverridesAllowAndPublic(t *testing.T) { diff --git a/internal/mountsync/bootstrap_test.go b/internal/mountsync/bootstrap_test.go index fb36bd09..483e83f0 100644 --- a/internal/mountsync/bootstrap_test.go +++ b/internal/mountsync/bootstrap_test.go @@ -111,7 +111,7 @@ func (c *bootstrapClient) ListTree(ctx context.Context, workspaceID, path string f := c.files[p] entries = append(entries, TreeEntry{Path: p, Type: "file", Revision: f.Revision, ContentHash: f.ContentHash}) } - resp := TreeResponse{Path: normalizeRemotePath(path), Entries: entries} + resp := TreeResponse{Path: normalizeRemotePath(path), Entries: entries, TotalFiles: len(paths)} if end < len(paths) { next := paths[end-1] resp.NextCursor = &next @@ -731,6 +731,101 @@ func TestBootstrapStallCycleGuardPersistsAndFailsHard(t *testing.T) { if st.LastError == nil || st.LastError.Kind != "bootstrap_stalled" || st.LastError.Code != "bootstrap_stall_cycle_limit" { t.Fatalf("expected structured persisted bootstrap stall error, got %#v", st.LastError) } + if !strings.Contains(st.LastError.Message, `path "/"`) || !strings.Contains(st.LastError.Message, "page cursor") { + t.Fatalf("stall error must identify the full directory/page checkpoint, got %q", st.LastError.Message) + } + pub := readPublicStateFile(t, localDir) + if pub.Status != "stalled" || pub.Bootstrap == nil || pub.Bootstrap.Phase != "stalled" { + t.Fatalf("hard stall must override bootstrapping health, got status=%q bootstrap=%+v", pub.Status, pub.Bootstrap) + } + if pub.Bootstrap.CurrentPath != "/" || pub.Bootstrap.FilesTotal != 30 || pub.Bootstrap.FilesSynced > pub.Bootstrap.FilesTotal { + t.Fatalf("unexpected actionable stalled bootstrap progress: %+v", pub.Bootstrap) + } + + // A supervisor restart must not spend another request at a checkpoint that + // already exhausted the configured limit. The private cursor/directory + // checkpoint remains intact and raising the limit deliberately re-arms it. + callsBeforeRestart := client.listTreeCalls.Load() + s = newBootstrapSyncer(t, client, localDir, opts) + err = s.Reconcile(context.Background()) + if !errors.As(err, &stalled) { + t.Fatalf("expected persisted terminal stall after restart, got %v", err) + } + if got := client.listTreeCalls.Load(); got != callsBeforeRestart { + t.Fatalf("terminal restart made %d new tree requests, want 0", got-callsBeforeRestart) + } +} + +func TestBootstrapDirectoryTraversalLimitFailsLoudlyAndPersistsPath(t *testing.T) { + client := &hierarchicalTreeClient{fakeClient: &fakeClient{files: map[string]RemoteFile{ + "/a/one/two/three/file.md": {Path: "/a/one/two/three/file.md", Revision: "rev_a", Content: "a"}, + "/b/one/two/three/file.md": {Path: "/b/one/two/three/file.md", Revision: "rev_b", Content: "b"}, + }}} + localDir := t.TempDir() + s, err := NewSyncer(client, SyncerOptions{ + WorkspaceID: "ws_directory_limit", + RemoteRoot: "/", + LocalRoot: localDir, + RootCtx: context.Background(), + BootstrapMaxDirectories: 2, + }) + if err != nil { + t.Fatalf("NewSyncer: %v", err) + } + + err = s.Reconcile(context.Background()) + var limitErr *BootstrapTraversalLimitError + if !errors.As(err, &limitErr) { + t.Fatalf("expected terminal directory traversal limit, got %v", err) + } + if limitErr.Path != "/b/one/two" || limitErr.DirectoriesDiscovered != 2 || limitErr.Limit != 2 { + t.Fatalf("unexpected traversal limit detail: %#v", limitErr) + } + st := loadPersistedState(t, localDir) + if st.LastError == nil || st.LastError.Kind != "bootstrap_stalled" || st.LastError.Code != "bootstrap_traversal_limit" { + t.Fatalf("expected persisted traversal-limit status, got %#v", st.LastError) + } + pub := readPublicStateFile(t, localDir) + if pub.Status != "stalled" || pub.Bootstrap == nil || pub.Bootstrap.CurrentPath != "/b/one/two" { + t.Fatalf("traversal limit must expose the blocked path, got status=%q bootstrap=%+v", pub.Status, pub.Bootstrap) + } + + callsBeforeRestart := len(client.calls) + restarted, err := NewSyncer(client, SyncerOptions{ + WorkspaceID: "ws_directory_limit", + RemoteRoot: "/", + LocalRoot: localDir, + RootCtx: context.Background(), + BootstrapMaxDirectories: 2, + }) + if err != nil { + t.Fatalf("restart NewSyncer: %v", err) + } + if err := restarted.Reconcile(context.Background()); !errors.As(err, &limitErr) { + t.Fatalf("expected persisted traversal limit on restart, got %v", err) + } + if len(client.calls) != callsBeforeRestart { + t.Fatalf("terminal traversal-limit restart issued %d new tree calls", len(client.calls)-callsBeforeRestart) + } + + // Raising the bound after operator inspection re-arms the same persisted + // directory checkpoint; no bootstrap state is discarded. + rearmed, err := NewSyncer(client, SyncerOptions{ + WorkspaceID: "ws_directory_limit", + RemoteRoot: "/", + LocalRoot: localDir, + RootCtx: context.Background(), + BootstrapMaxDirectories: 3, + }) + if err != nil { + t.Fatalf("rearm NewSyncer: %v", err) + } + if err := rearmed.Reconcile(context.Background()); err != nil { + t.Fatalf("raised traversal bound did not resume saved checkpoint: %v", err) + } + if st := loadPersistedState(t, localDir); !st.BootstrapComplete { + t.Fatalf("re-armed traversal did not complete: %#v", st) + } } func TestBootstrapStallCycleGuardIgnoresCanceledContext(t *testing.T) { @@ -831,6 +926,24 @@ func TestBootstrapStallCycleLimitUsesOptionThenEnvThenDefault(t *testing.T) { } } +func TestBootstrapDirectoryLimitUsesOptionThenEnvThenDefault(t *testing.T) { + newSyncer := func(t *testing.T, opts SyncerOptions) *Syncer { + t.Helper() + return newBootstrapSyncer(t, newBootstrapClient(1, 1), t.TempDir(), opts) + } + t.Setenv("RELAYFILE_BOOTSTRAP_MAX_DIRECTORIES", "123") + if got := newSyncer(t, SyncerOptions{}).bootstrapMaxDirectories; got != 123 { + t.Fatalf("env bootstrap directory limit = %d, want 123", got) + } + if got := newSyncer(t, SyncerOptions{BootstrapMaxDirectories: 7}).bootstrapMaxDirectories; got != 7 { + t.Fatalf("explicit bootstrap directory limit = %d, want 7", got) + } + t.Setenv("RELAYFILE_BOOTSTRAP_MAX_DIRECTORIES", "invalid") + if got := newSyncer(t, SyncerOptions{}).bootstrapMaxDirectories; got != defaultBootstrapMaxDirectories { + t.Fatalf("invalid env bootstrap directory limit = %d, want default %d", got, defaultBootstrapMaxDirectories) + } +} + // TestBootstrapCompleteGatesFastPath: a hand-seeded state with Files + // LastEventAt but BootstrapComplete=false MUST NOT short-circuit; the // full pull runs and the mirror converges (the rw_517d60b6 repro). diff --git a/internal/mountsync/syncer.go b/internal/mountsync/syncer.go index 8991c936..ec5ff654 100644 --- a/internal/mountsync/syncer.go +++ b/internal/mountsync/syncer.go @@ -98,6 +98,12 @@ const ( // response-size and timeout bounds; the tree traversal persists its cursor // and resumes on the next poll cycle. defaultBootstrapMaxFilesPerCycle = 2000 + // defaultBootstrapMaxDirectories bounds the number of distinct directory + // checkpoints a bounded-tree bootstrap may discover. The queue is + // de-duplicated, so exceeding this limit indicates either a pathologically + // broad workspace or a server namespace that expands cyclically through + // ever-new paths. Both require operator action rather than unbounded growth. + defaultBootstrapMaxDirectories = 50000 // defaultFullPullMinInterval rate-limits the expensive trust-but-verify // audit. Incremental events remain the normal reconciliation path. defaultFullPullMinInterval = 24 * time.Hour @@ -197,6 +203,24 @@ func resolveBootstrapMaxFilesPerCycle(opt int, logger Logger) int { return defaultBootstrapMaxFilesPerCycle } +func resolveBootstrapMaxDirectories(opt int, logger Logger) int { + if opt > 0 { + return opt + } + if opt < 0 && logger != nil { + logger.Printf("ignoring invalid BootstrapMaxDirectories=%d; using env/default", opt) + } + const env = "RELAYFILE_BOOTSTRAP_MAX_DIRECTORIES" + if raw := strings.TrimSpace(os.Getenv(env)); raw != "" { + if parsed, err := strconv.Atoi(raw); err == nil && parsed > 0 { + return parsed + } else if logger != nil { + logger.Printf("ignoring invalid %s=%q; expected a positive integer", env, raw) + } + } + return defaultBootstrapMaxDirectories +} + var providerLayoutAliasSegments = []string{ "by-title", "by-id", @@ -253,14 +277,25 @@ func (e *IncrementalReadNotReadyError) Error() string { // so callers can distinguish an operator-actionable hard stop from a normal // transient cloud failure. type BootstrapStalledError struct { - Cycles int - Limit int - Cursor string - Cause error + Cycles int + Limit int + Path string + Cursor string + PageOffset int + DirectoriesPending int + Cause error } func (e *BootstrapStalledError) Error() string { - message := fmt.Sprintf("bootstrap stalled for %d consecutive checkpoint-stable cycles (limit %d, cursor %q)", e.Cycles, e.Limit, e.Cursor) + message := fmt.Sprintf( + "bootstrap stalled for %d consecutive checkpoint-stable cycles (limit %d, path %q, page cursor %q, page offset %d, %d directories pending)", + e.Cycles, + e.Limit, + normalizeRemotePath(e.Path), + e.Cursor, + e.PageOffset, + e.DirectoriesPending, + ) if e.Cause != nil { return message + ": " + e.Cause.Error() } @@ -269,6 +304,36 @@ func (e *BootstrapStalledError) Error() string { func (e *BootstrapStalledError) Unwrap() error { return e.Cause } +// BootstrapTraversalLimitError is a terminal bootstrap failure raised before +// a pathological/cyclic namespace can grow the persisted directory frontier +// without bound. Raising RELAYFILE_BOOTSTRAP_MAX_DIRECTORIES after inspecting +// and fixing the named subtree deliberately re-arms the saved checkpoint. +type BootstrapTraversalLimitError struct { + Path string + DirectoriesDiscovered int + Limit int +} + +func (e *BootstrapTraversalLimitError) Error() string { + return fmt.Sprintf( + "bootstrap directory traversal limit reached at %q (%d distinct directories discovered, limit %d); inspect the subtree for cyclic/expanding aliases, then raise RELAYFILE_BOOTSTRAP_MAX_DIRECTORIES to resume the persisted checkpoint", + normalizeRemotePath(e.Path), + e.DirectoriesDiscovered, + e.Limit, + ) +} + +// IsBootstrapTerminalError identifies bootstrap failures that polling runners +// must return instead of treating as one more retryable cycle failure. +func IsBootstrapTerminalError(err error) bool { + var stalled *BootstrapStalledError + if errors.As(err, &stalled) { + return true + } + var traversalLimit *BootstrapTraversalLimitError + return errors.As(err, &traversalLimit) +} + type TreeEntry struct { Path string `json:"path"` Type string `json:"type"` @@ -282,6 +347,7 @@ type TreeResponse struct { Path string `json:"path"` Entries []TreeEntry `json:"entries"` NextCursor *string `json:"nextCursor"` + TotalFiles int `json:"totalFiles,omitempty"` } type FilesystemEvent struct { @@ -950,6 +1016,10 @@ type SyncerOptions struct { // cycles that may leave the persisted traversal checkpoint unchanged. 0 // falls back to RELAYFILE_BOOTSTRAP_STALL_CYCLES, then the default (20). BootstrapStallCycles int + // BootstrapMaxDirectories bounds distinct directory checkpoints discovered + // by the bounded-tree fallback. 0 uses + // RELAYFILE_BOOTSTRAP_MAX_DIRECTORIES, then the default (50000). + BootstrapMaxDirectories int // CursorTimeout bounds each resolveLatestEventCursor attempt with its OWN // deadline derived from RootCtx. Timeout-class failures are retried with // backoff before the caller decides whether a full pull is safe. @@ -1159,6 +1229,7 @@ type Syncer struct { bootstrapTimeout time.Duration bootstrapIdleTimeout time.Duration bootstrapStallCycles int + bootstrapMaxDirectories int readNotReadyTTL time.Duration forceFullReconcile bool incrementalCycles int @@ -1306,10 +1377,20 @@ type mountState struct { // BootstrapPageOffset resumes within a server page that exceeds the // per-cycle file budget. It is scoped by BootstrapDirectories[0] and // BootstrapCursor and resets whenever either advances. - BootstrapPageOffset int `json:"bootstrapPageOffset,omitempty"` - BootstrapFilesSynced int `json:"bootstrapFilesSynced,omitempty"` - BootstrapFilesTotal int `json:"bootstrapFilesTotal,omitempty"` - BootstrapStartedAt string `json:"bootstrapStartedAt,omitempty"` + BootstrapPageOffset int `json:"bootstrapPageOffset,omitempty"` + BootstrapFilesSynced int `json:"bootstrapFilesSynced,omitempty"` + BootstrapFilesTotal int `json:"bootstrapFilesTotal,omitempty"` + // BootstrapFilesTotalUnavailable is persisted once traversal prunes a + // reserved runtime subtree. The server's total includes those descendants, + // but the mount intentionally never enumerates them, so retaining that + // denominator would make N/M unreachable across page and process resumes. + BootstrapFilesTotalUnavailable bool `json:"bootstrapFilesTotalUnavailable,omitempty"` + BootstrapStartedAt string `json:"bootstrapStartedAt,omitempty"` + // BootstrapDirectoriesDiscovered is monotonic for one bootstrap and backs + // the runaway-directory guard across process restarts. BootstrapBlockedPath + // names the first directory refused by that guard. + BootstrapDirectoriesDiscovered int `json:"bootstrapDirectoriesDiscovered,omitempty"` + BootstrapBlockedPath string `json:"bootstrapBlockedPath,omitempty"` // LastFullPullAt persists the last completed authoritative full-tree audit // so process restarts cannot accidentally re-arm an expensive audit storm. LastFullPullAt string `json:"lastFullPullAt,omitempty"` @@ -1605,10 +1686,17 @@ type eventListenerHealth struct { // bootstrapStatus is the public, cursor-free view of bootstrap progress. type bootstrapStatus struct { - Phase string `json:"phase"` - FilesSynced int `json:"filesSynced"` - FilesTotal int `json:"filesTotal,omitempty"` - StartedAt string `json:"startedAt,omitempty"` + Phase string `json:"phase"` + FilesSynced int `json:"filesSynced"` + FilesTotal int `json:"filesTotal,omitempty"` + StartedAt string `json:"startedAt,omitempty"` + CurrentPath string `json:"currentPath,omitempty"` + PageOffset int `json:"pageOffset,omitempty"` + DirectoriesPending int `json:"directoriesPending,omitempty"` + DirectoriesDiscovered int `json:"directoriesDiscovered,omitempty"` + StallCycles int `json:"stallCycles,omitempty"` + StallLimit int `json:"stallLimit,omitempty"` + Reason string `json:"reason,omitempty"` } type publicStateFlags struct { @@ -1771,6 +1859,7 @@ func NewSyncer(client RemoteClient, opts SyncerOptions) (*Syncer, error) { bootstrapIdleTimeout = defaultBootstrapIdleTimeout } bootstrapStallCycles := resolveBootstrapStallCycles(opts.BootstrapStallCycles, opts.Logger) + bootstrapMaxDirectories := resolveBootstrapMaxDirectories(opts.BootstrapMaxDirectories, opts.Logger) exportTimeout := resolveDurationEnv(opts.ExportTimeout, "RELAYFILE_EXPORT_TIMEOUT", defaultExportTimeout, opts.Logger) if exportTimeout <= 0 { exportTimeout = defaultExportTimeout @@ -1908,6 +1997,7 @@ func NewSyncer(client RemoteClient, opts SyncerOptions) (*Syncer, error) { bootstrapTimeout: bootstrapTimeout, bootstrapIdleTimeout: bootstrapIdleTimeout, bootstrapStallCycles: bootstrapStallCycles, + bootstrapMaxDirectories: bootstrapMaxDirectories, readNotReadyTTL: readNotReadyTTL, forceFullReconcile: forceFullReconcile, oversizedLogged: map[string]struct{}{}, @@ -3234,6 +3324,14 @@ func (s *Syncer) syncReserved(ctx context.Context, forcePoll bool) error { s.mu.Unlock() return err } + if terminalErr := s.persistedBootstrapTerminalError(); terminalErr != nil { + // A process/supervisor restart must not silently re-arm a checkpoint + // that already exhausted its bounded retry policy. Re-publish the saved + // structured status and return before making any cloud request. + _ = s.saveState() + s.mu.Unlock() + return terminalErr + } s.mu.Unlock() @@ -3310,6 +3408,50 @@ func (s *Syncer) syncReserved(ctx context.Context, forcePoll bool) error { return s.saveState() } +func (s *Syncer) persistedBootstrapTerminalError() error { + if s.state.BootstrapComplete || s.state.LastError == nil { + return nil + } + switch s.state.LastError.Code { + case "bootstrap_stall_cycle_limit": + limit := s.bootstrapStallCycles + if limit <= 0 { + limit = defaultBootstrapStallCycles + } + if s.state.BootstrapStallCycles < limit { + // The operator raised the bounded retry limit after addressing the + // checkpoint. Re-arm in place without discarding cursor progress. + s.state.LastError = nil + return nil + } + return &BootstrapStalledError{ + Cycles: s.state.BootstrapStallCycles, + Limit: limit, + Path: s.bootstrapCurrentPath(), + Cursor: strings.TrimSpace(s.state.BootstrapCursor), + PageOffset: s.state.BootstrapPageOffset, + DirectoriesPending: len(s.state.BootstrapDirectories), + } + case "bootstrap_traversal_limit": + limit := s.bootstrapMaxDirectories + if limit <= 0 { + limit = defaultBootstrapMaxDirectories + } + if s.state.BootstrapDirectoriesDiscovered < limit { + s.state.LastError = nil + s.state.BootstrapBlockedPath = "" + return nil + } + return &BootstrapTraversalLimitError{ + Path: s.bootstrapCurrentPath(), + DirectoriesDiscovered: s.state.BootstrapDirectoriesDiscovered, + Limit: limit, + } + default: + return nil + } +} + func (s *Syncer) runClosingDigestJobsLocked(ctx context.Context) error { if s.closeScheduler == nil { return nil @@ -3715,6 +3857,9 @@ func (p bootstrapProgress) touch() { // also tears the watchdog goroutine down (no leak). func (s *Syncer) bootstrapContext(parent context.Context) (context.Context, context.CancelFunc, bootstrapProgress, error) { _ = parent // intentionally derive from rootCtx, not the per-cycle ctx + if !s.state.BootstrapComplete && strings.TrimSpace(s.state.BootstrapStartedAt) == "" { + s.state.BootstrapStartedAt = s.now().UTC().Format(time.RFC3339Nano) + } if err := s.ensureBootstrapProgressStateFile(); err != nil { return nil, nil, bootstrapProgress{}, fmt.Errorf("initialize bootstrap progress state file: %w", err) } @@ -4300,6 +4445,18 @@ func (s *Syncer) pullRemoteFullExport(ctx context.Context, client exportSnapshot sort.Slice(files, func(i, j int) bool { return normalizeRemotePath(files[i].Path) < normalizeRemotePath(files[j].Path) }) + if !s.state.BootstrapComplete { + s.state.BootstrapFilesSynced = 0 + if s.lazyRepos { + // The export total includes intentionally-unhydrated GitHub repo + // contents, so it is not a valid materialization denominator. + s.state.BootstrapFilesTotal = 0 + s.state.BootstrapFilesTotalUnavailable = true + } else { + s.state.BootstrapFilesTotal = len(files) + s.state.BootstrapFilesTotalUnavailable = false + } + } remotePaths := map[string]struct{}{} maxObservedRevision := "" for i := range files { @@ -4324,6 +4481,9 @@ func (s *Syncer) pullRemoteFullExport(ctx context.Context, client exportSnapshot s.yieldFullPullStateLock() prog.touch() remotePaths[remotePath] = struct{}{} + if !s.state.BootstrapComplete { + s.state.BootstrapFilesSynced++ + } files[i].Content = "" } @@ -4718,7 +4878,26 @@ func isLazyGithubRepoSubtreePath(path string) bool { return false } +// markBootstrapTotalUnavailable persistently suppresses the bootstrap file +// total once a reserved runtime subtree is discovered anywhere in the +// traversal. totalFiles counts every caller-visible remote file, including +// reserved mount runtime descendants, but this traversal prunes those +// subtrees before enumeration, so their exact contribution is unknown; an +// already-suppressed total is left alone rather than logged again. +func (s *Syncer) markBootstrapTotalUnavailable(runtimeRoot string) { + if s.state.BootstrapFilesTotalUnavailable { + return + } + s.state.BootstrapFilesTotal = 0 + s.state.BootstrapFilesTotalUnavailable = true + s.logf("bootstrap file total unavailable after pruning reserved runtime subtree %s", runtimeRoot) +} + func (s *Syncer) pullRemoteFullTree(ctx context.Context, conflicted map[string]struct{}, prog bootstrapProgress) (returnErr error) { + maxDirectories := s.bootstrapMaxDirectories + if maxDirectories <= 0 { + maxDirectories = defaultBootstrapMaxDirectories + } metrics := fullTreeTraversalMetrics{startedAt: time.Now()} defer func() { s.logf( @@ -4774,8 +4953,11 @@ func (s *Syncer) pullRemoteFullTree(ctx context.Context, conflicted map[string]s for _, directory := range directories { queuedDirectories[directory] = struct{}{} } + if s.state.BootstrapDirectoriesDiscovered < len(queuedDirectories) { + s.state.BootstrapDirectoriesDiscovered = len(queuedDirectories) + } if s.state.BootstrapStartedAt == "" { - s.state.BootstrapStartedAt = time.Now().UTC().Format(time.RFC3339Nano) + s.state.BootstrapStartedAt = s.now().UTC().Format(time.RFC3339Nano) } persistTraversal := func(filesThisPage int) error { if s.state.BootstrapComplete { @@ -4815,6 +4997,37 @@ func (s *Syncer) pullRemoteFullTree(ctx context.Context, conflicted map[string]s } s.recordCloudSuccess() prog.touch() + if !s.state.BootstrapFilesTotalUnavailable { + // The per-cycle file budget below only walks a chunk of page.Entries, + // so a reserved runtime subtree can sit past that chunk on this very + // page. Scan the full page up front: trusting totalFiles before this + // check would persist an unreachable denominator for every cycle + // between now and whichever later cycle happens to walk that entry. + for _, entry := range page.Entries { + if runtimeRoot := mountRuntimeRemoteRoot(normalizeRemotePath(entry.Path)); runtimeRoot != "" { + s.markBootstrapTotalUnavailable(runtimeRoot) + break + } + } + } + if currentDirectory == s.remoteRoot && !s.lazyRepos && !s.state.BootstrapFilesTotalUnavailable && page.TotalFiles > 0 { + // totalFiles is stable across server pagination and counts the full + // caller-visible subtree, not just this page. Persist it on the root + // frontier so N/M is an actual completion denominator. Older servers + // omit it (zero), in which case status intentionally renders N only. + // + // A runtime subtree that only appears on a later page of the root + // frontier's own pagination can still leave this total transiently + // wrong until that later page is walked and markBootstrapTotalUnavailable + // self-corrects it (both scans above run on every page). Deferring + // publication until the root frontier's pagination fully drains would + // close that window, but it also delays the very early-progress signal + // this field exists for during a large, slow, multi-page bootstrap + // (TestBootstrapStallCycleGuardPersistsAndFailsHard depends on the + // eager value surfacing across repeatedly-retried, never-draining + // cycles). Eager-and-self-correcting is the intentional tradeoff. + s.state.BootstrapFilesTotal = page.TotalFiles + } entryStart := pageOffset if entryStart < 0 || entryStart > len(page.Entries) { s.logf("discarding invalid bootstrap page offset %d for page with %d entries", entryStart, len(page.Entries)) @@ -4861,6 +5074,7 @@ func (s *Syncer) pullRemoteFullTree(ctx context.Context, conflicted map[string]s runtimeRoot := mountRuntimeRemoteRoot(remotePath) if runtimeRoot != "" { metrics.runtimeEntriesSeen++ + s.markBootstrapTotalUnavailable(runtimeRoot) } if !isUnderRemoteRoot(s.remoteRoot, remotePath) { continue @@ -4882,7 +5096,19 @@ func (s *Syncer) pullRemoteFullTree(ctx context.Context, conflicted map[string]s if _, exists := queuedDirectories[remotePath]; exists { continue } + if s.state.BootstrapDirectoriesDiscovered >= maxDirectories { + s.state.BootstrapBlockedPath = remotePath + s.state.BootstrapDirectories = append([]string(nil), directories...) + s.state.BootstrapCursor = cursor + s.state.BootstrapPageOffset = pageOffset + return &BootstrapTraversalLimitError{ + Path: remotePath, + DirectoriesDiscovered: s.state.BootstrapDirectoriesDiscovered, + Limit: maxDirectories, + } + } queuedDirectories[remotePath] = struct{}{} + s.state.BootstrapDirectoriesDiscovered++ directories = append(directories, remotePath) } continue @@ -4954,6 +5180,7 @@ func (s *Syncer) pullRemoteFullTree(ctx context.Context, conflicted map[string]s if markErr := s.markReadDenied(result.RemotePath); markErr != nil { return markErr } + filesThisPage++ continue } // Transient HTTP error (503, 429, etc.): stop the current @@ -4987,7 +5214,11 @@ func (s *Syncer) pullRemoteFullTree(ctx context.Context, conflicted map[string]s // cursor so the same page (including the failing path) is retried next // cycle. Save state up to the last successfully-committed cursor. if transientBootstrapAbort { - if err := persistTraversal(filesThisPage); err != nil { + // The page checkpoint intentionally does not advance when a transient + // read fails. Do not add its successfully-applied prefix to the durable + // progress counter either: the next cycle will revisit that prefix, and + // counting it twice can make FilesSynced exceed the authoritative total. + if err := persistTraversal(0); err != nil { return err } break @@ -5224,7 +5455,10 @@ func (s *Syncer) markBootstrapComplete() { s.state.BootstrapStartedAt = "" s.state.BootstrapFilesSynced = 0 s.state.BootstrapFilesTotal = 0 + s.state.BootstrapFilesTotalUnavailable = false s.state.BootstrapStallCycles = 0 + s.state.BootstrapDirectoriesDiscovered = 0 + s.state.BootstrapBlockedPath = "" // Clear persisted quarantine so a fixed adapter gets a clean slate. s.state.QuarantinedPaths = nil s.clearAllIncrementalReadNotReady() @@ -5262,8 +5496,16 @@ func (s *Syncer) bootstrapCheckpointAdvanced(previousCursor string, previousDire // same cursor. At the configured limit it returns a typed hard failure and // intentionally leaves BootstrapComplete false. func (s *Syncer) recordBootstrapCycle(previousCursor string, previousDirectories []string, previousPageOffset int, cause error) error { - if s.state.BootstrapComplete || s.bootstrapCheckpointAdvanced(previousCursor, previousDirectories, previousPageOffset) { + if s.state.BootstrapComplete { s.state.BootstrapStallCycles = 0 + s.state.BootstrapBlockedPath = "" + return nil + } + if s.bootstrapCheckpointAdvanced(previousCursor, previousDirectories, previousPageOffset) { + s.state.BootstrapStallCycles = 0 + if !IsBootstrapTerminalError(cause) { + s.state.BootstrapBlockedPath = "" + } return nil } // Root-context cancellation means this runner is intentionally stopping, @@ -5282,13 +5524,26 @@ func (s *Syncer) recordBootstrapCycle(previousCursor string, previousDirectories return nil } return &BootstrapStalledError{ - Cycles: s.state.BootstrapStallCycles, - Limit: limit, - Cursor: strings.TrimSpace(s.state.BootstrapCursor), - Cause: cause, + Cycles: s.state.BootstrapStallCycles, + Limit: limit, + Path: s.bootstrapCurrentPath(), + Cursor: strings.TrimSpace(s.state.BootstrapCursor), + PageOffset: s.state.BootstrapPageOffset, + DirectoriesPending: len(s.state.BootstrapDirectories), + Cause: cause, } } +func (s *Syncer) bootstrapCurrentPath() string { + if blocked := strings.TrimSpace(s.state.BootstrapBlockedPath); blocked != "" { + return normalizeRemotePath(blocked) + } + if len(s.state.BootstrapDirectories) > 0 { + return normalizeRemotePath(s.state.BootstrapDirectories[0]) + } + return s.remoteRoot +} + // snapshotDeleteUnsafe reports whether running snapshot-driven deletes is // unsafe given how many files the fresh remote listing returned versus how // many we currently track. It guards against a degraded cloud response @@ -7310,7 +7565,10 @@ func (s *Syncer) loadState() error { s.state.BootstrapStartedAt = "" s.state.BootstrapFilesSynced = 0 s.state.BootstrapFilesTotal = 0 + s.state.BootstrapFilesTotalUnavailable = false s.state.BootstrapStallCycles = 0 + s.state.BootstrapDirectoriesDiscovered = 0 + s.state.BootstrapBlockedPath = "" } if s.state.BootstrapComplete && s.fullPullMinInterval > 0 && strings.TrimSpace(s.state.LastFullPullAt) == "" { // Completed state written by an older version has no audit timestamp. @@ -7496,18 +7754,33 @@ func (s *Syncer) savePublicState() error { status = "stale" } - // Bootstrap-in-progress overrides "stale"/"ready": surface explicit - // progress so operators (and the CLI status surface) see - // "bootstrapping N/M" instead of a misleading stall while a large - // initial mirror is still running. + // Bootstrap state overrides "stale"/"ready". A terminal bootstrap error + // takes precedence over generic in-progress reporting; otherwise the exact + // production wedge would keep reading green/bootstrapping after the hard + // stall guard fired. var bootstrap *bootstrapStatus if !s.state.BootstrapComplete && strings.TrimSpace(s.state.BootstrapStartedAt) != "" { - status = "bootstrapping" + phase := "bootstrapping" + reason := "" + if s.state.LastError != nil && s.state.LastError.Kind == "bootstrap_stalled" { + phase = "stalled" + status = "stalled" + reason = s.state.LastError.Message + } else { + status = "bootstrapping" + } bootstrap = &bootstrapStatus{ - Phase: "bootstrapping", - FilesSynced: s.state.BootstrapFilesSynced, - FilesTotal: s.state.BootstrapFilesTotal, - StartedAt: s.state.BootstrapStartedAt, + Phase: phase, + FilesSynced: s.state.BootstrapFilesSynced, + FilesTotal: s.state.BootstrapFilesTotal, + StartedAt: s.state.BootstrapStartedAt, + CurrentPath: s.bootstrapCurrentPath(), + PageOffset: s.state.BootstrapPageOffset, + DirectoriesPending: len(s.state.BootstrapDirectories), + DirectoriesDiscovered: s.state.BootstrapDirectoriesDiscovered, + StallCycles: s.state.BootstrapStallCycles, + StallLimit: s.bootstrapStallCycles, + Reason: reason, } } @@ -7929,6 +8202,12 @@ func classifyStatusError(err error) *statusError { status.Code = "bootstrap_stall_cycle_limit" return status } + var traversalLimit *BootstrapTraversalLimitError + if errors.As(err, &traversalLimit) { + status.Kind = "bootstrap_stalled" + status.Code = "bootstrap_traversal_limit" + return status + } var httpErr *HTTPError if errors.As(err, &httpErr) { status.StatusCode = httpErr.StatusCode diff --git a/internal/mountsync/syncer_test.go b/internal/mountsync/syncer_test.go index a90d5ffc..854b8628 100644 --- a/internal/mountsync/syncer_test.go +++ b/internal/mountsync/syncer_test.go @@ -8131,6 +8131,10 @@ type hierarchicalTreeCall struct { type pagedTreeClient struct { *fakeClient pageSize int + // totalFiles, when nonzero, is reported on every page as TreeResponse.TotalFiles, + // mirroring the production contract that the total is stable across + // server pagination. + totalFiles int } func (c *pagedTreeClient) ListTree(_ context.Context, _ string, path string, _ int, cursor string) (TreeResponse, error) { @@ -8171,7 +8175,7 @@ func (c *pagedTreeClient) ListTree(_ context.Context, _ string, path string, _ i value := strconv.Itoa(end) next = &value } - return TreeResponse{Path: base, Entries: entries, NextCursor: next}, nil + return TreeResponse{Path: base, Entries: entries, NextCursor: next, TotalFiles: c.totalFiles}, nil } // hierarchicalTreeClient mirrors the production tree contract: a request @@ -8210,11 +8214,13 @@ func (c *hierarchicalTreeClient) ListTree(ctx context.Context, _ string, path st return TreeResponse{}, errors.New("synthetic ListTree failure") } entriesByPath := map[string]TreeEntry{} + totalFiles := 0 for remotePath, file := range c.files { remotePath = normalizeRemotePath(remotePath) if !isUnderRemoteRoot(base, remotePath) || remotePath == base { continue } + totalFiles++ relative := strings.TrimPrefix(strings.TrimPrefix(remotePath, base), "/") parts := strings.Split(relative, "/") levels := depth @@ -8258,7 +8264,7 @@ func (c *hierarchicalTreeClient) ListTree(ctx context.Context, _ string, path st for _, entry := range entries { c.returnedPaths = append(c.returnedPaths, entry.Path) } - return TreeResponse{Path: base, Entries: entries}, nil + return TreeResponse{Path: base, Entries: entries, TotalFiles: totalFiles}, nil } func (c *fakeClient) LatestEventID(ctx context.Context, workspaceID, provider string) (string, error) { @@ -10807,6 +10813,182 @@ func TestSkipMountRuntimeRemotePathPreservesActiveRootRuntime(t *testing.T) { } } +func TestBootstrapProgressSuppressesRuntimeInclusiveTotalAcrossRestart(t *testing.T) { + files := map[string]RemoteFile{ + "/workspace/.relay/state.json": { + Path: "/workspace/.relay/state.json", Revision: "rev_runtime", Content: `{"runtime":true}`, + }, + "/workspace/a.txt": {Path: "/workspace/a.txt", Revision: "rev_a", Content: "a"}, + "/workspace/b.txt": {Path: "/workspace/b.txt", Revision: "rev_b", Content: "b"}, + } + localDir := t.TempDir() + newSyncer := func(client RemoteClient) *Syncer { + t.Helper() + syncer, err := NewSyncer(client, SyncerOptions{ + WorkspaceID: "ws_runtime_total", + RemoteRoot: "/workspace", + LocalRoot: localDir, + BootstrapMaxFilesPerCycle: 1, + BootstrapStallCycles: 5, + }) + if err != nil { + t.Fatalf("new syncer failed: %v", err) + } + return syncer + } + + first := newSyncer(&hierarchicalTreeClient{fakeClient: &fakeClient{files: files}}) + if err := first.SyncOnce(context.Background()); err != nil { + t.Fatalf("first bootstrap cycle failed: %v", err) + } + state := loadPersistedState(t, localDir) + if state.BootstrapComplete { + t.Fatalf("first bounded cycle unexpectedly completed bootstrap") + } + if state.BootstrapFilesTotal != 0 || !state.BootstrapFilesTotalUnavailable { + t.Fatalf("runtime-inclusive total remained publishable: total=%d unavailable=%t", state.BootstrapFilesTotal, state.BootstrapFilesTotalUnavailable) + } + + // A fresh process revisits the root page with the server's raw total, but + // must honor the persisted invalidation instead of restoring an unreachable + // denominator after the saved page offset. + second := newSyncer(&hierarchicalTreeClient{fakeClient: &fakeClient{files: files}}) + if err := second.SyncOnce(context.Background()); err != nil { + t.Fatalf("resumed bootstrap cycle failed: %v", err) + } + state = loadPersistedState(t, localDir) + if state.BootstrapFilesSynced != 1 { + t.Fatalf("resumed files synced = %d, want 1", state.BootstrapFilesSynced) + } + if state.BootstrapFilesTotal != 0 || !state.BootstrapFilesTotalUnavailable { + t.Fatalf("process restart restored runtime-inclusive total: total=%d unavailable=%t", state.BootstrapFilesTotal, state.BootstrapFilesTotalUnavailable) + } + + publicBytes, err := os.ReadFile(filepath.Join(localDir, ".relay", "state.json")) + if err != nil { + t.Fatalf("read public state: %v", err) + } + var public struct { + Bootstrap map[string]json.RawMessage `json:"bootstrap"` + } + if err := json.Unmarshal(publicBytes, &public); err != nil { + t.Fatalf("decode public state: %v", err) + } + if _, published := public.Bootstrap["filesTotal"]; published { + t.Fatalf("public state published unreachable runtime-inclusive total: %s", publicBytes) + } +} + +// TestBootstrapProgressSuppressesTotalWhenRuntimeSubtreeOutlivesFileBudget +// covers a runtime subtree whose entries sort after enough real files to +// fall outside the per-cycle file budget's processed chunk on the very page +// that already reported an authoritative-looking page.TotalFiles. The +// runtime dir/file never enter s.state via the budget-limited entry loop +// this cycle, so only a full-page scan ahead of persisting the total catches +// it; persisting page.TotalFiles here renders an unreachable N/M until some +// later cycle happens to walk that entry. +func TestBootstrapProgressSuppressesTotalWhenRuntimeSubtreeOutlivesFileBudget(t *testing.T) { + files := map[string]RemoteFile{ + "/workspace/a.txt": {Path: "/workspace/a.txt", Revision: "rev_a", Content: "a"}, + "/workspace/b.txt": {Path: "/workspace/b.txt", Revision: "rev_b", Content: "b"}, + "/workspace/z/.relay/state.json": { + Path: "/workspace/z/.relay/state.json", Revision: "rev_runtime", Content: `{"runtime":true}`, + }, + } + localDir := t.TempDir() + syncer, err := NewSyncer(&hierarchicalTreeClient{fakeClient: &fakeClient{files: files}}, SyncerOptions{ + WorkspaceID: "ws_runtime_total_budget", + RemoteRoot: "/workspace", + LocalRoot: localDir, + BootstrapMaxFilesPerCycle: 1, + BootstrapStallCycles: 5, + }) + if err != nil { + t.Fatalf("new syncer failed: %v", err) + } + + // The single-file budget's chunk this cycle covers only a.txt: b.txt and + // the nested runtime subtree sort after it and stay outside the loop that + // would otherwise discover the runtime path and suppress the total. + if err := syncer.SyncOnce(context.Background()); err != nil { + t.Fatalf("first bootstrap cycle failed: %v", err) + } + state := loadPersistedState(t, localDir) + if state.BootstrapFilesSynced != 1 { + t.Fatalf("files synced = %d, want 1 (only a.txt processed this cycle)", state.BootstrapFilesSynced) + } + if state.BootstrapFilesTotal != 0 || !state.BootstrapFilesTotalUnavailable { + t.Fatalf("runtime subtree outside the budgeted chunk left an unreachable total publishable: total=%d unavailable=%t", state.BootstrapFilesTotal, state.BootstrapFilesTotalUnavailable) + } +} + +// TestBootstrapProgressSelfCorrectsWhenRuntimeSubtreeIsOnALaterRootPage +// covers a pruned runtime subtree that only appears on a later cursor page +// of the remote root's own listing. The first root page reports a positive +// totalFiles and contains no runtime entries, so it is published eagerly +// (matching TestBootstrapStallCycleGuardPersistsAndFailsHard's expectation +// that a large, slow, multi-page bootstrap gets an early progress signal +// rather than none at all while its own pagination is still in flight). The +// total is transiently optimistic until the runtime-bearing page is reached, +// at which point it must self-correct to unavailable rather than staying +// published as an unreachable denominator forever. +func TestBootstrapProgressSelfCorrectsWhenRuntimeSubtreeIsOnALaterRootPage(t *testing.T) { + files := map[string]RemoteFile{ + "/workspace/a.txt": {Path: "/workspace/a.txt", Revision: "rev_a", Content: "a"}, + "/workspace/b.txt": {Path: "/workspace/b.txt", Revision: "rev_b", Content: "b"}, + "/workspace/z/.relay/state.json": { + Path: "/workspace/z/.relay/state.json", Revision: "rev_runtime", Content: `{"runtime":true}`, + }, + // Real work still pending after the runtime page keeps bootstrap + // incomplete past cycle 2, so the self-corrected flag persists long + // enough to assert on rather than being cleared by completion. + "/workspace/zz_extra1.txt": {Path: "/workspace/zz_extra1.txt", Revision: "rev_e1", Content: "e1"}, + "/workspace/zz_extra2.txt": {Path: "/workspace/zz_extra2.txt", Revision: "rev_e2", Content: "e2"}, + } + localDir := t.TempDir() + client := &pagedTreeClient{fakeClient: &fakeClient{files: files}, pageSize: 2, totalFiles: 5} + syncer, err := NewSyncer(client, SyncerOptions{ + WorkspaceID: "ws_runtime_total_paginated_root", + RemoteRoot: "/workspace", + LocalRoot: localDir, + BootstrapMaxFilesPerCycle: 2, + BootstrapStallCycles: 5, + FullPullEvery: -1, + }) + if err != nil { + t.Fatalf("new syncer failed: %v", err) + } + + // Root page 1 (a.txt, b.txt) exactly fills this cycle's budget and + // reports totalFiles=5; the runtime subtree is not on this page, so the + // total is published eagerly as the early progress signal. + if err := syncer.pullRemoteFullTree(context.Background(), nil, bootstrapProgress{}); err != nil { + t.Fatalf("first bootstrap cycle failed: %v", err) + } + if got := syncer.state.BootstrapFilesSynced; got != 2 { + t.Fatalf("files synced = %d, want 2 (root page 1 only)", got) + } + if syncer.state.BootstrapFilesTotal != 5 { + t.Fatalf("root page 1's totalFiles was not published eagerly: total=%d", syncer.state.BootstrapFilesTotal) + } + if syncer.state.BootstrapCursor == "" { + t.Fatalf("expected a persisted cursor into root page 2, got none") + } + + // Root page 2, reached by cursor, carries the pruned runtime subtree. + // The previously-published total must self-correct to unavailable + // instead of staying published as an unreachable denominator. + if err := syncer.pullRemoteFullTree(context.Background(), nil, bootstrapProgress{}); err != nil { + t.Fatalf("second bootstrap cycle failed: %v", err) + } + if syncer.state.BootstrapComplete { + t.Fatalf("bootstrap unexpectedly completed with root pagination still pending") + } + if syncer.state.BootstrapFilesTotal != 0 || !syncer.state.BootstrapFilesTotalUnavailable { + t.Fatalf("runtime subtree on a later root pagination page left an unreachable total published: total=%d unavailable=%t", syncer.state.BootstrapFilesTotal, syncer.state.BootstrapFilesTotalUnavailable) + } +} + func TestPullRemoteFullTreePrunesNestedMountRuntimeBeforeDescendantEnumeration(t *testing.T) { files := map[string]RemoteFile{ "/slack/channels/C123/messages/1780145510_376649.json": { diff --git a/internal/relayfile/store.go b/internal/relayfile/store.go index 3c9141ad..44206298 100644 --- a/internal/relayfile/store.go +++ b/internal/relayfile/store.go @@ -81,6 +81,11 @@ type TreeResponse struct { Path string `json:"path"` Entries []TreeEntry `json:"entries"` NextCursor *string `json:"nextCursor"` + // TotalFiles is the authoritative number of files below Path before + // pagination. HTTP handlers replace it with the caller-visible count after + // permission filtering. Clients can therefore render real N/M bootstrap + // progress instead of treating a missing total as zero. + TotalFiles int `json:"totalFiles,omitempty"` } type FileSemantics struct { @@ -1160,6 +1165,7 @@ func (s *Store) ListTree(workspaceID, path string, depth int, cursor string) (Tr } entryMap := map[string]TreeEntry{} + totalFiles := 0 for filePath, file := range ws.Files { if !withinBase(base, filePath) { continue @@ -1169,6 +1175,7 @@ func (s *Store) ListTree(workspaceID, path string, depth int, cursor string) (Tr if rest == "" { continue } + totalFiles++ parts := strings.Split(rest, "/") if len(parts) == 0 { continue @@ -1212,7 +1219,7 @@ func (s *Store) ListTree(workspaceID, path string, depth int, cursor string) (Tr return TreeResponse{}, err } - return TreeResponse{Path: base, Entries: entries, NextCursor: nextCursor}, nil + return TreeResponse{Path: base, Entries: entries, NextCursor: nextCursor, TotalFiles: totalFiles}, nil } func (s *Store) ReadFile(workspaceID, path string) (File, error) { @@ -4916,6 +4923,7 @@ func listTreeFromFiles(files map[string]File, path string, depth int, cursor str } entryMap := map[string]TreeEntry{} + totalFiles := 0 for filePath, file := range files { if !withinBase(base, filePath) { continue @@ -4925,6 +4933,7 @@ func listTreeFromFiles(files map[string]File, path string, depth int, cursor str if rest == "" { continue } + totalFiles++ parts := strings.Split(rest, "/") if len(parts) == 0 { continue @@ -4968,7 +4977,7 @@ func listTreeFromFiles(files map[string]File, path string, depth int, cursor str return TreeResponse{}, err } - return TreeResponse{Path: base, Entries: entries, NextCursor: nextCursor}, nil + return TreeResponse{Path: base, Entries: entries, NextCursor: nextCursor, TotalFiles: totalFiles}, nil } // paginateTreeEntries slices the supplied entries with the supplied cursor. diff --git a/internal/relayfile/store_test.go b/internal/relayfile/store_test.go index 20ec3034..c67369bb 100644 --- a/internal/relayfile/store_test.go +++ b/internal/relayfile/store_test.go @@ -1164,6 +1164,9 @@ func TestListTreePaginatesBoundedEntries(t *testing.T) { if len(pageOne.Entries) != maxTreeEntriesPerPage { t.Fatalf("expected %d page-one entries, got %d", maxTreeEntriesPerPage, len(pageOne.Entries)) } + if pageOne.TotalFiles != maxTreeEntriesPerPage+2 { + t.Fatalf("page-one totalFiles = %d, want %d", pageOne.TotalFiles, maxTreeEntriesPerPage+2) + } if pageOne.NextCursor == nil { t.Fatalf("expected page-one next cursor") } @@ -1178,6 +1181,9 @@ func TestListTreePaginatesBoundedEntries(t *testing.T) { if len(pageTwo.Entries) != 2 { t.Fatalf("expected 2 page-two entries, got %d", len(pageTwo.Entries)) } + if pageTwo.TotalFiles != maxTreeEntriesPerPage+2 { + t.Fatalf("page-two totalFiles = %d, want stable total %d", pageTwo.TotalFiles, maxTreeEntriesPerPage+2) + } if pageTwo.Entries[0].Path != "/external/File1000.md" || pageTwo.Entries[1].Path != "/external/File1001.md" { t.Fatalf("unexpected page-two entries: %+v", pageTwo.Entries) } diff --git a/openapi/relayfile-v1.openapi.yaml b/openapi/relayfile-v1.openapi.yaml index c10d7052..6afe6f3f 100644 --- a/openapi/relayfile-v1.openapi.yaml +++ b/openapi/relayfile-v1.openapi.yaml @@ -2400,6 +2400,10 @@ components: nextCursor: type: string nullable: true + totalFiles: + type: integer + minimum: 0 + description: Total caller-visible files below path before pagination; omitted by older servers. FileReadResponse: type: object diff --git a/package-lock.json b/package-lock.json index 26f9cd16..254b9298 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2194,6 +2194,19 @@ "linux" ] }, + "node_modules/@relayfile/mount-linux-x64": { + "version": "0.10.42", + "resolved": "https://registry.npmjs.org/@relayfile/mount-linux-x64/-/mount-linux-x64-0.10.42.tgz", + "integrity": "sha512-pFUp3cDisB2uS5MHTdUaWvWqmt8aePb2SrnhPT9OLp/VwIGnAuDd27lJ74/iAYsQKjMmdwwUndxhyxhIY/lmrg==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ] + }, "node_modules/@relayfile/sdk": { "resolved": "packages/sdk/typescript", "link": true diff --git a/packages/cli/CHANGELOG.md b/packages/cli/CHANGELOG.md index 74fcdbd2..7d45304c 100644 --- a/packages/cli/CHANGELOG.md +++ b/packages/cli/CHANGELOG.md @@ -8,6 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- Mount bootstrap stalls now terminate the CLI daemon after the persisted retry limit, survive supervisor restarts without reissuing the same request, and surface as `mount: stalled` with the blocked path. Bootstrap progress uses the tree API's authoritative file total when safe, suppresses totals that include pruned runtime subtrees, and never renders a synthetic `/0` denominator. - Cloud session auth no longer shells out to `agent-relay cloud session`. The CLI reads the canonical credential file `agent-relay cloud login` writes (`~/.agentworkforce/relay/cloud-auth.json`), or the `CLOUD_API_*` environment, and refreshes preemptively — an access token within five minutes of expiry, or a refresh token within 24 hours of its own expiry — through Cloud's `/api/v1/auth/token/refresh` endpoint, writing the rotated pair back under the same lock `agent-relay` uses. This restores auto-recovery from a routine token expiry. A session supplied through `CLOUD_API_*` is refreshed in memory only and never written to disk; an access token supplied with no refresh token is used as-is and never refreshed. - Relayfile no longer reads `AGENT_RELAY_BIN` to locate the `agent-relay` CLI. Agent Relay uses that variable for the *broker* binary, so every relay-spawned agent pointed Relayfile at `agent-relay-broker` — which has no `cloud` or `workspace` subcommand — and a routine expiry surfaced as `agent-relay CLI >= 8.7.0 required`. Use `RELAYFILE_AGENT_RELAY_BIN` to override the CLI path; otherwise `agent-relay` is resolved from `PATH`. - The `agent-relay` CLI compatibility probe now names the exact argv it ran, the binary it ran it with, and how that binary was resolved. It no longer probes `cloud session`, since Relayfile does not use it. diff --git a/packages/sdk/python/src/relayfile/types.py b/packages/sdk/python/src/relayfile/types.py index c882e0bd..47c57a5f 100644 --- a/packages/sdk/python/src/relayfile/types.py +++ b/packages/sdk/python/src/relayfile/types.py @@ -79,6 +79,7 @@ class TreeResponse: path: str entries: list[TreeEntry] next_cursor: str | None = None + total_files: int | None = None @dataclass diff --git a/packages/sdk/python/tests/test_client.py b/packages/sdk/python/tests/test_client.py index bddde9d2..4562415c 100644 --- a/packages/sdk/python/tests/test_client.py +++ b/packages/sdk/python/tests/test_client.py @@ -46,6 +46,7 @@ def test_list_tree(self) -> None: "path": "/", "entries": [{"path": "/zendesk", "type": "dir", "revision": "rev_1"}], "nextCursor": None, + "totalFiles": 1, } respx.get(f"{BASE}/v1/workspaces/ws_acme/fs/tree").mock( return_value=httpx.Response(200, json=payload) @@ -54,6 +55,7 @@ def test_list_tree(self) -> None: res = client.list_tree("ws_acme") assert len(res["entries"]) == 1 assert res["entries"][0]["path"] == "/zendesk" + assert res["totalFiles"] == 1 @respx.mock def test_list_tree_params(self) -> None: diff --git a/packages/sdk/typescript/CHANGELOG.md b/packages/sdk/typescript/CHANGELOG.md index b9a76dd2..7043e849 100644 --- a/packages/sdk/typescript/CHANGELOG.md +++ b/packages/sdk/typescript/CHANGELOG.md @@ -6,7 +6,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] -_No unreleased changes._ +- Added optional `TreeResponse.totalFiles`, the caller-visible file count below the requested path before pagination. ## [0.10.42] - 2026-08-14 diff --git a/packages/sdk/typescript/src/client.test.ts b/packages/sdk/typescript/src/client.test.ts index 27cbfde0..0094e21d 100644 --- a/packages/sdk/typescript/src/client.test.ts +++ b/packages/sdk/typescript/src/client.test.ts @@ -1153,12 +1153,14 @@ describe("RelayFileClient — existing methods", () => { { path: "/readme.md", type: "file", revision: "rev_2" }, ], nextCursor: null, + totalFiles: 2, }; const f = mockFetch(payload); const client = makeClient(f); const res = await client.listTree("ws_acme"); expect(res.entries).toHaveLength(2); expect(res.entries[0]!.path).toBe("/zendesk"); + expect(res.totalFiles).toBe(2); const url = f.mock.calls[0]![0] as string; expect(url).toContain("/v1/workspaces/ws_acme/fs/tree"); }); diff --git a/packages/sdk/typescript/src/types.ts b/packages/sdk/typescript/src/types.ts index 0f854def..b9f1505e 100644 --- a/packages/sdk/typescript/src/types.ts +++ b/packages/sdk/typescript/src/types.ts @@ -40,6 +40,8 @@ export interface TreeResponse { path: string; entries: TreeEntry[]; nextCursor: string | null; + /** Total caller-visible files below `path`, before pagination. */ + totalFiles?: number; } export interface FileSemantics {