feat: Update All albums button + fast, accurate index scans

photomap/backend/config.py

@@ -126,6 +126,17 @@ class Album(BaseModel):
             "little to semantic search."
         ),
     )
+    min_image_bytes: int = Field(
+        default=8192,
+        ge=0,
+        description=(
+            "Minimum file size (in bytes) for an image to be indexed; "
+            "smaller files are rejected without being opened, which keeps "
+            "scans fast on thumbnail-heavy or network-mounted libraries. "
+            "0 disables the check. The 8192 default suits the default 256px "
+            "min_image_dimension; lower both together."
+        ),
+    )
 
     @model_validator(mode="before")
     @classmethod
@@ -203,6 +214,7 @@ def to_dict(self) -> dict[str, Any]:
             "max_search_results": self.max_search_results,
             "use_query_optimization": self.use_query_optimization,
             "min_image_dimension": self.min_image_dimension,
+            "min_image_bytes": self.min_image_bytes,
         }
         # Keep directory-album YAML free of irrelevant InvokeAI keys.
         if self.source_type == "invokeai_board":
@@ -235,6 +247,7 @@ def from_dict(cls, key: str, data: dict[str, Any]) -> "Album":
             max_search_results=data.get("max_search_results", 100),
             use_query_optimization=data.get("use_query_optimization", True),
             min_image_dimension=data.get("min_image_dimension", 256),
+            min_image_bytes=data.get("min_image_bytes", 8192),
             invokeai_url=data.get("invokeai_url"),
             invokeai_username=data.get("invokeai_username"),
             invokeai_password=data.get("invokeai_password"),
@@ -663,6 +676,7 @@ def create_album(
     max_search_results: int | None = None,
     use_query_optimization: bool | None = None,
     min_image_dimension: int | None = None,
+    min_image_bytes: int | None = None,
     source_type: str = "directory",
     invokeai_url: str | None = None,
     invokeai_username: str | None = None,
@@ -703,6 +717,8 @@ def create_album(
         fields["use_query_optimization"] = use_query_optimization
     if min_image_dimension is not None:
         fields["min_image_dimension"] = min_image_dimension
+    if min_image_bytes is not None:
+        fields["min_image_bytes"] = min_image_bytes
     return Album(**fields)
 
 

photomap/backend/routers/album.py

@@ -104,6 +104,7 @@ def get_embeddings_for_album(album_key: str) -> Embeddings:
         embeddings_path=Path(album_config.index),
         encoder_spec=album_config.encoder_spec,
         min_image_dimension=album_config.min_image_dimension,
+        min_image_bytes=album_config.min_image_bytes,
     )
 
 
@@ -211,6 +212,7 @@ def _album_public_dict(album: Album) -> dict[str, Any]:
         "max_search_results": album.max_search_results,
         "use_query_optimization": album.use_query_optimization,
         "min_image_dimension": album.min_image_dimension,
+        "min_image_bytes": album.min_image_bytes,
         "invokeai_url": album.invokeai_url,
         "invokeai_username": album.invokeai_username,
         "invokeai_root": album.invokeai_root,
@@ -310,6 +312,7 @@ async def update_album(album_data: dict) -> JSONResponse:
             max_search_results=album_data.get("max_search_results"),
             use_query_optimization=album_data.get("use_query_optimization"),
             min_image_dimension=album_data.get("min_image_dimension"),
+            min_image_bytes=album_data.get("min_image_bytes"),
             source_type=album_data.get("source_type", "directory"),
             invokeai_url=album_data.get("invokeai_url"),
             invokeai_username=album_data.get("invokeai_username"),

photomap/backend/routers/index.py

@@ -16,7 +16,7 @@
 
 from .. import invokeai_client
 from ..config import get_config_manager
-from ..embeddings import Embeddings, peek_encoder_spec
+from ..embeddings import LAST_UPDATED_FILENAME, Embeddings, peek_encoder_spec
 from ..progress import IndexingCancelled, progress_tracker
 from .album import (
     AlbumDep,
@@ -252,8 +252,14 @@ async def index_metadata(album_config: AlbumDep) -> EmbeddingsIndexMetadata:
     if not index_path.exists():
         raise HTTPException(status_code=404, detail="Index file does not exist")
 
-    # Get file metadata
+    # "Last updated" is when an update operation last COMPLETED, not when the
+    # .npz content last changed — a no-change update skips the save, and the
+    # card shouldn't look stale right after a successful refresh. The marker
+    # is absent for indexes predating it; fall back to the .npz mtime.
     last_modified = index_path.stat().st_mtime
+    marker = index_path.parent / LAST_UPDATED_FILENAME
+    if marker.exists():
+        last_modified = max(last_modified, marker.stat().st_mtime)
     filename_count = len(Embeddings.open_cached_embeddings(index_path)["filenames"])
 
     return EmbeddingsIndexMetadata(
@@ -579,6 +585,7 @@ async def _update_index_background_async(album_key: str, album_config):
             embeddings_path=index_path,
             encoder_spec=album_config.encoder_spec,
             min_image_dimension=album_config.min_image_dimension,
+            min_image_bytes=getattr(album_config, "min_image_bytes", 8192),
         )
 
         if index_path.exists():

photomap/frontend/static/css/album-manager.css

@@ -356,6 +356,40 @@
   font-size: 14px;
 }
 
+/* Byte-size + pixel-dimension gates on one line in the album editor. */
+.min-image-gates {
+  display: flex;
+  align-items: center;
+  gap: 0.5em;
+}
+
+.min-image-gates input {
+  width: 5em;
+}
+
+.min-image-gate-unit {
+  color: #aaa;
+  font-size: 0.9em;
+}
+
+/* Header "Update All" button: shaped like .btn-primary, coloured like the
+   per-album .btn-index buttons it fans out to. */
+.btn-update-all {
+  background: #2196f3;
+  border: none;
+  color: #fff;
+  padding: 0.5em 1em;
+  border-radius: 6px;
+  cursor: pointer;
+  font-size: 14px;
+  white-space: nowrap;
+}
+
+.btn-update-all:disabled {
+  opacity: 0.6;
+  cursor: default;
+}
+
 .btn-secondary {
   background: #666;
   border: none;
@@ -394,6 +428,7 @@
   border-radius: 4px;
   cursor: pointer;
   font-size: 0.85em;
+  white-space: nowrap;
 }
 
 .btn-cancel {

photomap/frontend/static/javascript/album-manager.js

@@ -78,6 +78,16 @@ export class AlbumManager {
   // Constants
   static POLL_INTERVAL = 1000;
   static PROGRESS_HIDE_DELAY = 3000;
+  // "Update All" pipeline depth. The backend serializes both the file scan
+  // and the GPU-heavy embedding stage across albums (process-wide semaphores
+  // in embeddings.py), so two in-flight updates let one album scan its files
+  // or build its UMAP while another encodes; more would only queue on those
+  // gates while holding extra embedding arrays in memory.
+  static MAX_CONCURRENT_UPDATES = 2;
+  // Backend statuses that mean an index operation is still in flight.
+  static RUNNING_STATUSES = ["scanning", "downloading", "indexing", "mapping"];
+  // Consecutive index_progress failures before Update All gives up on an album.
+  static MAX_PROGRESS_POLL_FAILURES = 5;
   static AUTO_INDEXING_DELAY = 500;
   static SETUP_EXIT_DELAY = 10000;
   static FORM_ANIMATION_DELAY = 300;
@@ -124,6 +134,10 @@ export class AlbumManager {
     };
 
     this.progressPollers = new Map();
+    // Non-null while "Update All" is working through the albums:
+    // { finished, total }. Instance state (not just button text) so the
+    // button can be repainted after the dialog is closed and reopened.
+    this.updateAllProgress = null;
     this.isSetupMode = false;
     this.autoIndexingAlbums = new Set();
     // Per-album non-fatal indexing notice (e.g. board images missing on disk),
@@ -164,6 +178,14 @@ export class AlbumManager {
       this.showAddAlbumForm();
     });
 
+    // Update the index of every album
+    const updateAllBtn = document.getElementById("updateAllBtn");
+    if (updateAllBtn) {
+      updateAllBtn.addEventListener("click", () => {
+        this.updateAllAlbums();
+      });
+    }
+
     // Cancel add album buttons (both X and Cancel button)
     document.getElementById("cancelAddAlbumBtn").addEventListener("click", () => {
       this.hideAddAlbumForm();
@@ -805,6 +827,9 @@ export class AlbumManager {
   async show() {
     this.overlay.classList.add("visible");
     hideSpinner();
+    // Repaint the Update All button in case a run is still in flight from
+    // before the dialog was closed.
+    this._refreshUpdateAllButton();
     await this.loadAlbums();
     await this.checkForOngoingIndexing(); // <-- Move this after loadAlbums
 
@@ -978,6 +1003,8 @@ export class AlbumManager {
       albums.forEach((album) => {
         this.createAlbumCard(album);
       });
+      // Card count just changed — show/hide the Update All button to match.
+      this._refreshUpdateAllButton();
     } catch (error) {
       console.error("Failed to load albums:", error);
     }
@@ -1349,6 +1376,13 @@ export class AlbumManager {
       minDimInput.value = album.min_image_dimension ?? 256;
     }
 
+    // Minimum-byte-size gate: stored in bytes, edited in kb. Fall back to
+    // 8192 bytes (Album.min_image_bytes default) for albums predating it.
+    const minBytesInput = editForm.querySelector(".edit-album-min-image-bytes");
+    if (minBytesInput) {
+      minBytesInput.value = Math.round((album.min_image_bytes ?? 8192) / 1024);
+    }
+
     // Initialize the encoder dropdown for THIS specific card
     populateEncoderSelect(editForm.querySelector(".edit-album-encoder"), album.encoder_spec);
 
@@ -1469,12 +1503,19 @@ export class AlbumManager {
     const minDim =
       Number.isFinite(minDimParsed) && minDimParsed >= 1 ? minDimParsed : (album.min_image_dimension ?? 256);
 
+    // Byte-size gate is edited in kb but stored in bytes; 0 disables it.
+    const minBytesRaw = editForm.querySelector(".edit-album-min-image-bytes")?.value;
+    const minBytesParsed = Number.parseInt(minBytesRaw, 10);
+    const minBytes =
+      Number.isFinite(minBytesParsed) && minBytesParsed >= 0 ? minBytesParsed * 1024 : (album.min_image_bytes ?? 8192);
+
     const updatedAlbum = {
       key: album.key,
       name: editForm.querySelector(".edit-album-name").value,
       description: editForm.querySelector(".edit-album-description").value,
       encoder_spec: editForm.querySelector(".edit-album-encoder")?.value || album.encoder_spec || DEFAULT_ENCODER_SPEC,
       min_image_dimension: minDim,
+      min_image_bytes: minBytes,
     };
 
     let sourceChanged = false;
@@ -1561,7 +1602,7 @@ export class AlbumManager {
     // Backend guard: check if indexing is already running
     try {
       const progress = await fetchJson(`index_progress/${albumKey}`);
-      if (progress.status === "indexing" || progress.status === "scanning" || progress.status === "mapping") {
+      if (AlbumManager.RUNNING_STATUSES.includes(progress.status)) {
         console.log(`Backend reports indexing already in progress for album: ${albumKey}`);
         this.showProgressUIWithoutScroll(cardElement, progress);
         this.startProgressPolling(albumKey, cardElement);
@@ -1598,6 +1639,92 @@ export class AlbumManager {
     this.startProgressPolling(albumKey, cardElement);
   }
 
+  // Polls the backend until the album's indexing run leaves a running state.
+  // Deliberately independent of the card progress pollers: hide() tears those
+  // down when the dialog closes, and "Update All" must keep working through
+  // its queue whether or not the dialog is open.
+  async _waitForIndexingToFinish(albumKey) {
+    let consecutiveFailures = 0;
+    for (;;) {
+      await new Promise((resolve) => setTimeout(resolve, AlbumManager.POLL_INTERVAL));
+      try {
+        const progress = await fetchJson(`index_progress/${albumKey}`);
+        consecutiveFailures = 0;
+        if (!AlbumManager.RUNNING_STATUSES.includes(progress.status)) {
+          return;
+        }
+      } catch (error) {
+        // Tolerate transient failures (the server can be busy mid-scan);
+        // give up only after several in a row so the queue can't hang.
+        consecutiveFailures += 1;
+        if (consecutiveFailures >= AlbumManager.MAX_PROGRESS_POLL_FAILURES) {
+          console.error(`Giving up waiting on indexing progress for album ${albumKey}:`, error);
+          return;
+        }
+      }
+    }
+  }
+
+  // Repaint the Update All button from instance state. The button is looked
+  // up fresh and this is also called from show() and loadAlbums(), so the
+  // label survives the dialog being closed and reopened mid-run, and the
+  // button hides entirely when no albums exist (fresh install / setup mode).
+  _refreshUpdateAllButton() {
+    const btn = document.getElementById("updateAllBtn");
+    if (!btn) {
+      return;
+    }
+    const hasAlbums = !!this.albumsList?.querySelector(".album-card");
+    btn.style.display = hasAlbums ? "" : "none";
+    if (this.updateAllProgress) {
+      const { finished, total } = this.updateAllProgress;
+      btn.disabled = true;
+      btn.textContent = `Updating ${finished}/${total}…`;
+    } else {
+      btn.disabled = false;
+      btn.textContent = "Update All";
+    }
+  }
+
+  // Update the index of every album, MAX_CONCURRENT_UPDATES at a time (see
+  // the constant for why the pipeline is kept shallow). Albums already
+  // indexing are awaited rather than restarted; one album failing doesn't
+  // stop the rest.
+  async updateAllAlbums() {
+    if (this.updateAllProgress) {
+      return;
+    }
+    const cards = [...this.albumsList.querySelectorAll(".album-card[data-album-key]")];
+    if (cards.length === 0) {
+      return;
+    }
+
+    this.updateAllProgress = { finished: 0, total: cards.length };
+    this._refreshUpdateAllButton();
+
+    const queue = [...cards];
+    const worker = async () => {
+      let card;
+      while ((card = queue.shift())) {
+        const albumKey = card.dataset.albumKey;
+        try {
+          await this.startIndexing(albumKey, this._liveCardFor(albumKey, card));
+          await this._waitForIndexingToFinish(albumKey);
+        } catch (error) {
+          console.error(`Update All: indexing failed for album ${albumKey}:`, error);
+        }
+        this.updateAllProgress.finished += 1;
+        this._refreshUpdateAllButton();
+      }
+    };
+    await Promise.all(
+      Array.from({ length: Math.min(AlbumManager.MAX_CONCURRENT_UPDATES, cards.length) }, () => worker())
+    );
+
+    this.updateAllProgress = null;
+    this._refreshUpdateAllButton();
+  }
+
   showProgressUI(cardElement) {
     this.showProgressUIWithoutScroll(cardElement);
 
@@ -1876,7 +2003,7 @@ export class AlbumManager {
       try {
         const progress = await fetchJson(`index_progress/${albumKey}`);
 
-        if (progress.status === "indexing" || progress.status === "scanning") {
+        if (AlbumManager.RUNNING_STATUSES.includes(progress.status)) {
           console.log(`Restoring progress UI for ongoing operation: ${albumKey} (${progress.status})`);
 
           this.showProgressUIWithoutScroll(cardElement, progress);

photomap/frontend/templates/modules/album-manager.html

@@ -6,6 +6,7 @@
       <button id="backToSettingsBtn" class="back-button" title="Back to Settings">← Settings</button>
       <h2>Album Management</h2>
       <div class="header-buttons">
+        <button id="updateAllBtn" class="btn-update-all" title="Update the index of every album">Update All</button>
         <button id="showAddAlbumBtn" class="btn-primary">+ Add Album</button>
         <button id="closeAlbumManagementBtn" class="modal-close" title="Close">✖</button>
       </div>
@@ -205,8 +206,13 @@ <h4 class="edit-album-title">Edit Album</h4>
         </div>
       </div>
       <div class="form-group">
-        <label>Exclude thumbnails and other images below this size (px)</label>
-        <input class="edit-album-min-image-dimension" type="number" min="1" step="1" value="256" />
+        <label>Exclude thumbnails and other images below this size:</label>
+        <div class="min-image-gates">
+          <input class="edit-album-min-image-bytes" type="number" min="0" step="1" value="8" />
+          <span class="min-image-gate-unit">kb</span>
+          <input class="edit-album-min-image-dimension" type="number" min="1" step="1" value="256" />
+          <span class="min-image-gate-unit">pixels</span>
+        </div>
       </div>
       <div class="form-group">
         <label>Encoder</label>