diff --git a/photomap/backend/config.py b/photomap/backend/config.py index 240480a2..cb297700 100644 --- a/photomap/backend/config.py +++ b/photomap/backend/config.py @@ -126,6 +126,17 @@ class Album(BaseModel): "little to semantic search." ), ) + min_image_bytes: int = Field( + default=8192, + ge=0, + description=( + "Minimum file size (in bytes) for an image to be indexed; " + "smaller files are rejected without being opened, which keeps " + "scans fast on thumbnail-heavy or network-mounted libraries. " + "0 disables the check. The 8192 default suits the default 256px " + "min_image_dimension; lower both together." + ), + ) @model_validator(mode="before") @classmethod @@ -203,6 +214,7 @@ def to_dict(self) -> dict[str, Any]: "max_search_results": self.max_search_results, "use_query_optimization": self.use_query_optimization, "min_image_dimension": self.min_image_dimension, + "min_image_bytes": self.min_image_bytes, } # Keep directory-album YAML free of irrelevant InvokeAI keys. if self.source_type == "invokeai_board": @@ -235,6 +247,7 @@ def from_dict(cls, key: str, data: dict[str, Any]) -> "Album": max_search_results=data.get("max_search_results", 100), use_query_optimization=data.get("use_query_optimization", True), min_image_dimension=data.get("min_image_dimension", 256), + min_image_bytes=data.get("min_image_bytes", 8192), invokeai_url=data.get("invokeai_url"), invokeai_username=data.get("invokeai_username"), invokeai_password=data.get("invokeai_password"), @@ -663,6 +676,7 @@ def create_album( max_search_results: int | None = None, use_query_optimization: bool | None = None, min_image_dimension: int | None = None, + min_image_bytes: int | None = None, source_type: str = "directory", invokeai_url: str | None = None, invokeai_username: str | None = None, @@ -703,6 +717,8 @@ def create_album( fields["use_query_optimization"] = use_query_optimization if min_image_dimension is not None: fields["min_image_dimension"] = min_image_dimension + if min_image_bytes is not None: + fields["min_image_bytes"] = min_image_bytes return Album(**fields) diff --git a/photomap/backend/embeddings.py b/photomap/backend/embeddings.py index 3918d9ca..6b62c7ee 100644 --- a/photomap/backend/embeddings.py +++ b/photomap/backend/embeddings.py @@ -54,6 +54,55 @@ DEFAULT_BATCH_SIZE = 8 DEFAULT_NUM_WORKERS = 4 +# Files larger than this pass the pixel-dimension gate on byte size alone, +# skipping the per-file header open (see _passes_dimension_gate). Tuned for +# the default 256px minimum: real photos over 500 KB are essentially never +# that small in pixels. (Validated by sampling a 122k-file real library, +# 2026-07-04: 0 of 1,175 sampled files over 500 KB failed the pixel gate.) +DIMENSION_PROBE_MAX_BYTES = 500 * 1024 + +# Default for the per-album ``min_image_bytes`` setting: files smaller than +# this are REJECTED by the gate on byte size alone, again skipping the header +# open. From the same 2026-07-04 sampling (5,000 files probed): at 8 KB the +# false-negative rate — real >=256px photos wrongly rejected — measured 0.15% +# (they exist: old heavily-compressed JPEGs go down to 6.5 KB), while 75% of +# the sub-256px thumbnail population falls below it. Raising this buys almost +# nothing and loses photos fast (16 KB -> 3.4% FN, 100 KB -> ~49%, which is +# the bug PR #269 fixed). The default is calibrated to the default 256px +# pixel gate; albums with a smaller pixel gate should lower it (both are +# editable side by side in the album editor). +DIMENSION_REJECT_MIN_BYTES = 8 * 1024 + +# Directory names (matched case-insensitively) that hold derived previews, +# never originals — pruned from the scan alongside our own photomap_index. +# NAS and desktop indexers can stash as many thumbnails as there are photos, +# and every one of them would be probed and rejected on each scan otherwise. +# Hidden (dot-prefixed) directories are pruned wholesale in the walk itself: +# on a photo library they are app caches (.shotwell/thumbs, .thumbnails, +# .dtrash, ...), and some hold thumbnails big enough to pass BOTH gates — +# Shotwell's 360px thumbs would otherwise be indexed as photos. +EXCLUDED_SCAN_DIRS = { + "photomap_index", + "@eadir", # Synology + "__macosx", # AppleDouble cruft from unzipped archives +} + +# Marker file next to embeddings.npz whose mtime records when an index +# create/update operation last COMPLETED — as opposed to the .npz mtime, +# which records when the index content last changed. A no-change update +# deliberately skips rewriting the .npz (and touching the .npz instead would +# spuriously invalidate the umap.npz staleness check), but the UI's "Index +# updated " line should still show that the album was just refreshed. +LAST_UPDATED_FILENAME = "last_updated" + +# Sidecar next to embeddings.npz remembering files the dimension gate +# rejected, keyed the same way as the index diff, with the size/mtime seen +# at rejection time. Lets update scans skip re-probing (re-opening) files +# that were already checked and turned away — on a library with a large +# thumbnail population that is as many header opens as the album has photos, +# on every single update. +SCAN_REJECTS_FILENAME = "scan_rejects.npz" + # Process-wide gate around the GPU-using portion of indexing. Two concurrent # albums each spinning up a CLIP/SigLIP encoder will OOM a typical 8-12 GiB # card; this serializes them so the second album waits its turn. Created @@ -69,6 +118,21 @@ def _get_indexing_semaphore() -> asyncio.Semaphore: return _indexing_semaphore +# Process-wide gate around the file-traversal stage. The scan opens every +# candidate file's header for the dimension gate, so it is seek-bound (and +# GIL-bound); two albums scanning at once run far slower than back-to-back. +# Kept separate from _indexing_semaphore so one album's scan can still +# overlap another album's GPU encoding — that pairing is the useful pipeline. +_scan_semaphore: asyncio.Semaphore | None = None + + +def _get_scan_semaphore() -> asyncio.Semaphore: + global _scan_semaphore + if _scan_semaphore is None: + _scan_semaphore = asyncio.Semaphore(1) + return _scan_semaphore + + def _l2_normalize(x: np.ndarray, axis: int = -1, eps: float = 1e-12) -> np.ndarray: """L2-normalize ``x`` along ``axis`` with an epsilon guard against zero vectors. @@ -378,6 +442,7 @@ class Embeddings(BaseModel): # smallest patch grid the bundled CLIP/SigLIP variants encode without # heavy upscaling. Default mirrors the Album field default in config.py. min_image_dimension: int = 256 + min_image_bytes: int = DIMENSION_REJECT_MIN_BYTES def __init__(self, **data): """Ensure embeddings_path is always resolved to prevent cache key mismatches.""" @@ -428,18 +493,46 @@ def _cleanup_cuda_memory(device: str) -> None: # Log but don't crash if CUDA operations fail logger.warning(f"CUDA cleanup failed: {e}") - def _passes_dimension_gate(self, path: Path) -> bool: + def _passes_dimension_gate(self, path: Path, st: os.stat_result | None = None) -> bool: """Return True if ``path``'s pixel dimensions are >= ``min_image_dimension``. - Reads only the image header via ``Image.open(...).size`` — PIL does - not decode pixels until they're accessed, so this is a few-KB read - per file. Unreadable / corrupt files return False and are logged at - debug level (they'd fail at encoding time anyway and would land in - ``bad_files`` there; here we just keep the scan log quiet). + The gate has three bands, decided from a stat where possible because + a stat is far cheaper than a header open (~200ms per open on a + network mount): + + - smaller than the album's ``min_image_bytes`` (0 disables) — + reject on byte size alone. At the 8 KB default this measured a + ~0.15% false-negative rate against a real library. + - larger than :data:`DIMENSION_PROBE_MAX_BYTES` — pass on byte size + alone; files that big are effectively never below the (default + 256px) pixel gate, and a rare false accept just indexes a small + image, which is harmless. + - in between — read the image header via ``Image.open(...).size``; + PIL does not decode pixels until they're accessed, so this is a + few-KB read per file. Unreadable / corrupt files return False and + are logged at debug level (they'd fail at encoding time anyway and + would land in ``bad_files`` there; here we just keep the scan log + quiet). + + The byte and pixel thresholds are independent album settings, + surfaced side by side in the album editor; either can be disabled. + Callers that already stat'ed the file pass the result as ``st``. """ min_dim = self.min_image_dimension - if min_dim <= 1: + byte_floor = self.min_image_bytes + pixel_gate_active = min_dim > 1 + if not pixel_gate_active and byte_floor <= 0: return True + try: + if st is None: + st = path.stat() + if byte_floor > 0 and st.st_size < byte_floor: + return False + if not pixel_gate_active or st.st_size > DIMENSION_PROBE_MAX_BYTES: + return True + except OSError as e: + logger.debug(f"Skipping unstattable image during scan: {path}: {e}") + return False try: with Image.open(path) as im: width, height = im.size @@ -448,45 +541,134 @@ def _passes_dimension_gate(self, path: Path) -> bool: return False return width >= min_dim and height >= min_dim + def last_updated_path(self) -> Path: + return self.embeddings_path.parent / LAST_UPDATED_FILENAME + + def _touch_last_updated(self) -> None: + """Record that an index create/update operation completed (see + :data:`LAST_UPDATED_FILENAME`). Failures only cost UI accuracy.""" + try: + path = self.last_updated_path() + path.parent.mkdir(parents=True, exist_ok=True) + path.touch() + except OSError as e: + logger.warning(f"Could not record index update time: {e}") + + def _scan_rejects_path(self) -> Path: + return self.embeddings_path.parent / SCAN_REJECTS_FILENAME + + def _load_scan_rejects(self) -> dict[str, tuple[int, float]]: + """Load the gate-rejection cache: compare-key -> (size, mtime). + + Returns an empty dict when the sidecar is absent, unreadable, or was + written under different gate thresholds (the verdicts it memoizes + would no longer hold). + """ + path = self._scan_rejects_path() + if not path.exists(): + return {} + try: + with np.load(path) as data: + if ( + int(data["min_dim"]) != self.min_image_dimension + or int(data["min_bytes"]) != self.min_image_bytes + ): + return {} + return { + str(key): (int(size), float(mtime)) + for key, size, mtime in zip( + data["keys"], data["sizes"], data["mtimes"], strict=True + ) + } + except Exception as e: + logger.warning(f"Ignoring unreadable scan-reject cache {path}: {e}") + return {} + + def _save_scan_rejects(self, rejects: dict[str, tuple[int, float]]) -> None: + """Persist the gate-rejection cache; failures only cost speed, not + correctness, so they're logged and swallowed.""" + path = self._scan_rejects_path() + try: + path.parent.mkdir(parents=True, exist_ok=True) + atomic_savez( + path, + keys=np.array(list(rejects.keys()), dtype=str), + sizes=np.array([v[0] for v in rejects.values()], dtype=np.int64), + mtimes=np.array([v[1] for v in rejects.values()], dtype=np.float64), + min_dim=np.int64(self.min_image_dimension), + min_bytes=np.int64(self.min_image_bytes), + ) + except OSError as e: + logger.warning(f"Could not save scan-reject cache {path}: {e}") + def get_image_files_from_directory( self, directory: Path, exts: set[str] = SUPPORTED_EXTENSIONS, progress_callback: Callable | None = None, update_interval: int = 100, + apply_dimension_gate: bool = True, + reject_sink: dict[str, tuple[int, float]] | None = None, ) -> list[Path]: """ Recursively collect all image files from a directory. - Each candidate file's header is opened to read pixel dimensions; - images with either dimension below ``self.min_image_dimension`` are - skipped. The header read is a few KB per file, so a scan of 10k - files typically adds 10-30s on SSD — small next to encoding time. + Hidden (dot-prefixed) subdirectories and those named in + :data:`EXCLUDED_SCAN_DIRS` (thumbnail caches and our own + ``photomap_index``) are pruned from the walk entirely. The album + root itself is exempt, so an album may point at a hidden directory. + + With ``apply_dimension_gate`` (the default), each candidate file + under :data:`DIMENSION_PROBE_MAX_BYTES` has its header opened to + read pixel dimensions; images with either dimension below + ``self.min_image_dimension`` are skipped. Pass + ``apply_dimension_gate=False`` to collect by extension only — the + update path does this and probes just the files not already indexed. Args: directory: Directory to scan exts: File extensions to include progress_callback: Optional callback function(count, message) for progress updates update_interval: How often to call progress_callback (every N files found) + apply_dimension_gate: Probe pixel dimensions during the walk + reject_sink: When given, gate-rejected files are recorded here as + compare-key -> (size, mtime) so they can be persisted to the + scan-reject cache and skipped on later scans """ logger.info(f"Scanning directory {directory} for image files...") image_files = [] files_checked = 0 skipped_too_small = 0 + gate_active = apply_dimension_gate and (self.min_image_dimension > 1 or self.min_image_bytes > 0) for root, dirs, files in os.walk(directory): - # Remove 'photomap_index' from dirs so os.walk skips it and its subdirs - dirs[:] = [d for d in dirs if d != "photomap_index"] + # Prune hidden dirs (app caches — see EXCLUDED_SCAN_DIRS note), + # thumbnail caches, and our own index dir from the walk. The + # album root itself is never pruned, only subdirectories. + dirs[:] = [d for d in dirs if not d.startswith(".") and d.lower() not in EXCLUDED_SCAN_DIRS] for file in [Path(x) for x in files]: files_checked += 1 if file.suffix.lower() not in exts: continue full = Path(root, file) - if self._passes_dimension_gate(full): + if not gate_active: image_files.append(full.resolve()) else: - skipped_too_small += 1 + st = None + try: + st = full.stat() + except OSError as e: + logger.debug(f"Skipping unstattable image during scan: {full}: {e}") + if st is not None and self._passes_dimension_gate(full, st): + image_files.append(full.resolve()) + else: + skipped_too_small += 1 + if reject_sink is not None and st is not None: + reject_sink[self._path_compare_key(full.resolve())] = ( + st.st_size, + st.st_mtime, + ) # Provide progress updates at regular intervals if progress_callback and files_checked % update_interval == 0: @@ -515,6 +697,8 @@ def get_image_files( image_paths_or_dir: list[Path] | Path, exts: set[str] = SUPPORTED_EXTENSIONS, progress_callback: Callable | None = None, + apply_dimension_gate: bool = True, + reject_sink: dict[str, tuple[int, float]] | None = None, ) -> list[Path]: """ Get a list of image file paths from a directory or a list of image paths. @@ -522,6 +706,10 @@ def get_image_files( Args: image_paths_or_dir (list of str or str): List of image paths or a directory path. progress_callback: Optional callback function for progress updates + apply_dimension_gate: Probe pixel dimensions during the scan (see + ``get_image_files_from_directory``) + reject_sink: Collects gate-rejected files (see + ``get_image_files_from_directory``) Returns: list of Path: List of image file paths. @@ -530,7 +718,11 @@ def get_image_files( if isinstance(image_paths_or_dir, Path): # If it's a single Path object, treat it as a directory images = self.get_image_files_from_directory( - image_paths_or_dir, exts, progress_callback + image_paths_or_dir, + exts, + progress_callback, + apply_dimension_gate=apply_dimension_gate, + reject_sink=reject_sink, ) elif isinstance(image_paths_or_dir, list): images = [] @@ -538,10 +730,16 @@ def get_image_files( for p in image_paths_or_dir: if p.is_dir(): images.extend( - self.get_image_files_from_directory(p, exts, progress_callback) + self.get_image_files_from_directory( + p, + exts, + progress_callback, + apply_dimension_gate=apply_dimension_gate, + reject_sink=reject_sink, + ) ) elif p.suffix.lower() in exts: - if self._passes_dimension_gate(p): + if not apply_dimension_gate or self._passes_dimension_gate(p): images.append(p) else: skipped_too_small += 1 @@ -853,10 +1051,30 @@ def _get_new_and_missing_images( image_paths_or_dir: list[Path] | Path, existing_filenames: np.ndarray, progress_callback: Callable | None = None, + check_progress_callback: Callable | None = None, ) -> tuple[set[Path], set[Path]]: - """Determine which images are new and which are missing.""" + """Determine which images are new and which are missing. + + The traversal collects candidates by extension only; the pixel + dimension gate runs afterwards, and only on files not already in the + index. Already-indexed files passed the gate when first indexed, so + re-probing all of them on every update paid the scan's dominant cost + (a per-file header open) for zero information. + + Files the gate has already rejected are remembered in the scan-reject + cache with their size/mtime; while those match, later scans dismiss + them on the stat alone. Without this, a rejected file (e.g. a NAS + thumbnail) would look "new" — and be re-opened — on every update. + + ``check_progress_callback(checked, total)``, when given, is invoked + periodically through the gate pass so callers can drive a real + progress bar — ``total`` is known up front here, unlike during the + traversal. + """ live_paths = self.get_image_files( - image_paths_or_dir, progress_callback=progress_callback + image_paths_or_dir, + progress_callback=progress_callback, + apply_dimension_gate=False, ) # Build map from casefolded posix key to the *original-case* Path. # The casefolded keys drive the set diff; the original Paths flow @@ -872,7 +1090,45 @@ def _get_new_and_missing_images( new_keys = set(live_by_key) - set(existing_by_key) missing_keys = set(existing_by_key) - set(live_by_key) - new_image_paths = {live_by_key[k] for k in new_keys} + gate_active = self.min_image_dimension > 1 or self.min_image_bytes > 0 + rejects = self._load_scan_rejects() if gate_active else {} + new_rejects: dict[str, tuple[int, float]] = {} + new_image_paths: set[Path] = set() + skipped_too_small = 0 + total_new = len(new_keys) + for i, key in enumerate(new_keys, start=1): + path = live_by_key[key] + if not gate_active: + new_image_paths.add(path) + continue + st = None + try: + st = path.stat() + except OSError as e: + logger.debug(f"Skipping unstattable image during scan: {path}: {e}") + if st is None: + skipped_too_small += 1 + elif rejects.get(key) == (st.st_size, st.st_mtime): + # Rejected before and unchanged since — no need to re-probe. + new_rejects[key] = rejects[key] + skipped_too_small += 1 + elif self._passes_dimension_gate(path, st): + new_image_paths.add(path) + else: + new_rejects[key] = (st.st_size, st.st_mtime) + skipped_too_small += 1 + if check_progress_callback and (i % 100 == 0 or i == total_new): + check_progress_callback(i, total_new) + if skipped_too_small: + logger.info( + f"Skipped {skipped_too_small} new image(s) under " + f"{self.min_image_dimension}px in either dimension." + ) + # Rebuilding from this run's rejects self-prunes entries for files + # that were deleted or changed; skip the write when nothing moved. + if gate_active and new_rejects != rejects: + self._save_scan_rejects(new_rejects) + missing_image_paths = {existing_by_key[k] for k in missing_keys} return new_image_paths, missing_image_paths @@ -962,7 +1218,9 @@ def create_index( num_workers: int = DEFAULT_NUM_WORKERS, ) -> IndexResult: """Index images using CLIP and save their embeddings.""" - image_paths = self.get_image_files(image_paths_or_dir) + reject_sink: dict[str, tuple[int, float]] = {} + image_paths = self.get_image_files(image_paths_or_dir, reject_sink=reject_sink) + self._save_scan_rejects(reject_sink) total_images = len(image_paths) progress_callback = tqdm_progress_callback(total_images) @@ -985,6 +1243,7 @@ def create_index( f"Created UMAP index with shape: {result.umap_embeddings.shape}" ) + self._touch_last_updated() return result async def create_index_async( @@ -1003,12 +1262,24 @@ def traversal_callback(count, message): progress_tracker.update_total_images(album_key, max(count, 0)) progress_tracker.update_progress(album_key, count, message) - # Offload the blocking traversal to a thread - image_paths = await asyncio.to_thread( - self.get_image_files, - image_paths_or_dir, - progress_callback=traversal_callback, - ) + # Offload the blocking traversal to a thread, one album at a time + # (see _scan_semaphore). + scan_semaphore = _get_scan_semaphore() + if scan_semaphore.locked(): + progress_tracker.update_progress( + album_key, 0, "Waiting for another album's file scan to finish…" + ) + reject_sink: dict[str, tuple[int, float]] = {} + async with scan_semaphore: + image_paths = await asyncio.to_thread( + self.get_image_files, + image_paths_or_dir, + progress_callback=traversal_callback, + reject_sink=reject_sink, + ) + # Seed the reject cache so the first *update* doesn't have to + # re-probe everything the gate just turned away. + self._save_scan_rejects(reject_sink) total_images = len(image_paths) logger.info( f"Found {total_images} image files in {describe_image_source(image_paths_or_dir)}" @@ -1044,6 +1315,7 @@ def traversal_callback(count, message): progress_tracker.complete_operation( album_key, "Indexing completed successfully" ) + self._touch_last_updated() return result except Exception as e: progress_tracker.set_error(album_key, str(e)) @@ -1203,6 +1475,7 @@ def update_index( f"UMAP index created with shape: {result.umap_embeddings.shape}" ) + self._touch_last_updated() return result except Exception as e: @@ -1233,12 +1506,30 @@ def traversal_callback(count, message): progress_tracker.update_total_images(album_key, max(count, 0)) progress_tracker.update_progress(album_key, count, message) - new_image_paths, missing_image_paths = await asyncio.to_thread( - self._get_new_and_missing_images, - image_paths_or_dir, - existing.filenames, - progress_callback=traversal_callback, - ) + def check_progress_callback(checked, total): + # Unlike the open-ended traversal, the gate pass knows its + # total up front, so it can drive a real percentage. + progress_tracker.update_total_images(album_key, total) + progress_tracker.update_progress( + album_key, + checked, + f"Checking new image files ({checked:,} of {total:,})...", + ) + + # One album traverses at a time (see _scan_semaphore). + scan_semaphore = _get_scan_semaphore() + if scan_semaphore.locked(): + progress_tracker.update_progress( + album_key, 0, "Waiting for another album's file scan to finish…" + ) + async with scan_semaphore: + new_image_paths, missing_image_paths = await asyncio.to_thread( + self._get_new_and_missing_images, + image_paths_or_dir, + existing.filenames, + progress_callback=traversal_callback, + check_progress_callback=check_progress_callback, + ) filtered_existing = self._filter_missing_images( missing_image_paths, @@ -1306,6 +1597,7 @@ def _on_save_start() -> None: f"Successfully indexed {len(result.embeddings)} new images", ) + self._touch_last_updated() return result except Exception as e: diff --git a/photomap/backend/routers/album.py b/photomap/backend/routers/album.py index 0a8e5ab8..cb93f0b8 100644 --- a/photomap/backend/routers/album.py +++ b/photomap/backend/routers/album.py @@ -104,6 +104,7 @@ def get_embeddings_for_album(album_key: str) -> Embeddings: embeddings_path=Path(album_config.index), encoder_spec=album_config.encoder_spec, min_image_dimension=album_config.min_image_dimension, + min_image_bytes=album_config.min_image_bytes, ) @@ -211,6 +212,7 @@ def _album_public_dict(album: Album) -> dict[str, Any]: "max_search_results": album.max_search_results, "use_query_optimization": album.use_query_optimization, "min_image_dimension": album.min_image_dimension, + "min_image_bytes": album.min_image_bytes, "invokeai_url": album.invokeai_url, "invokeai_username": album.invokeai_username, "invokeai_root": album.invokeai_root, @@ -310,6 +312,7 @@ async def update_album(album_data: dict) -> JSONResponse: max_search_results=album_data.get("max_search_results"), use_query_optimization=album_data.get("use_query_optimization"), min_image_dimension=album_data.get("min_image_dimension"), + min_image_bytes=album_data.get("min_image_bytes"), source_type=album_data.get("source_type", "directory"), invokeai_url=album_data.get("invokeai_url"), invokeai_username=album_data.get("invokeai_username"), diff --git a/photomap/backend/routers/index.py b/photomap/backend/routers/index.py index e4ed35c4..a2353f09 100644 --- a/photomap/backend/routers/index.py +++ b/photomap/backend/routers/index.py @@ -16,7 +16,7 @@ from .. import invokeai_client from ..config import get_config_manager -from ..embeddings import Embeddings, peek_encoder_spec +from ..embeddings import LAST_UPDATED_FILENAME, Embeddings, peek_encoder_spec from ..progress import IndexingCancelled, progress_tracker from .album import ( AlbumDep, @@ -252,8 +252,14 @@ async def index_metadata(album_config: AlbumDep) -> EmbeddingsIndexMetadata: if not index_path.exists(): raise HTTPException(status_code=404, detail="Index file does not exist") - # Get file metadata + # "Last updated" is when an update operation last COMPLETED, not when the + # .npz content last changed — a no-change update skips the save, and the + # card shouldn't look stale right after a successful refresh. The marker + # is absent for indexes predating it; fall back to the .npz mtime. last_modified = index_path.stat().st_mtime + marker = index_path.parent / LAST_UPDATED_FILENAME + if marker.exists(): + last_modified = max(last_modified, marker.stat().st_mtime) filename_count = len(Embeddings.open_cached_embeddings(index_path)["filenames"]) return EmbeddingsIndexMetadata( @@ -579,6 +585,7 @@ async def _update_index_background_async(album_key: str, album_config): embeddings_path=index_path, encoder_spec=album_config.encoder_spec, min_image_dimension=album_config.min_image_dimension, + min_image_bytes=getattr(album_config, "min_image_bytes", 8192), ) if index_path.exists(): diff --git a/photomap/frontend/static/css/album-manager.css b/photomap/frontend/static/css/album-manager.css index aaa70164..ac908ed1 100644 --- a/photomap/frontend/static/css/album-manager.css +++ b/photomap/frontend/static/css/album-manager.css @@ -356,6 +356,40 @@ font-size: 14px; } +/* Byte-size + pixel-dimension gates on one line in the album editor. */ +.min-image-gates { + display: flex; + align-items: center; + gap: 0.5em; +} + +.min-image-gates input { + width: 5em; +} + +.min-image-gate-unit { + color: #aaa; + font-size: 0.9em; +} + +/* Header "Update All" button: shaped like .btn-primary, coloured like the + per-album .btn-index buttons it fans out to. */ +.btn-update-all { + background: #2196f3; + border: none; + color: #fff; + padding: 0.5em 1em; + border-radius: 6px; + cursor: pointer; + font-size: 14px; + white-space: nowrap; +} + +.btn-update-all:disabled { + opacity: 0.6; + cursor: default; +} + .btn-secondary { background: #666; border: none; @@ -394,6 +428,7 @@ border-radius: 4px; cursor: pointer; font-size: 0.85em; + white-space: nowrap; } .btn-cancel { diff --git a/photomap/frontend/static/javascript/album-manager.js b/photomap/frontend/static/javascript/album-manager.js index f47c5b1a..8ba63850 100644 --- a/photomap/frontend/static/javascript/album-manager.js +++ b/photomap/frontend/static/javascript/album-manager.js @@ -78,6 +78,16 @@ export class AlbumManager { // Constants static POLL_INTERVAL = 1000; static PROGRESS_HIDE_DELAY = 3000; + // "Update All" pipeline depth. The backend serializes both the file scan + // and the GPU-heavy embedding stage across albums (process-wide semaphores + // in embeddings.py), so two in-flight updates let one album scan its files + // or build its UMAP while another encodes; more would only queue on those + // gates while holding extra embedding arrays in memory. + static MAX_CONCURRENT_UPDATES = 2; + // Backend statuses that mean an index operation is still in flight. + static RUNNING_STATUSES = ["scanning", "downloading", "indexing", "mapping"]; + // Consecutive index_progress failures before Update All gives up on an album. + static MAX_PROGRESS_POLL_FAILURES = 5; static AUTO_INDEXING_DELAY = 500; static SETUP_EXIT_DELAY = 10000; static FORM_ANIMATION_DELAY = 300; @@ -124,6 +134,10 @@ export class AlbumManager { }; this.progressPollers = new Map(); + // Non-null while "Update All" is working through the albums: + // { finished, total }. Instance state (not just button text) so the + // button can be repainted after the dialog is closed and reopened. + this.updateAllProgress = null; this.isSetupMode = false; this.autoIndexingAlbums = new Set(); // Per-album non-fatal indexing notice (e.g. board images missing on disk), @@ -164,6 +178,14 @@ export class AlbumManager { this.showAddAlbumForm(); }); + // Update the index of every album + const updateAllBtn = document.getElementById("updateAllBtn"); + if (updateAllBtn) { + updateAllBtn.addEventListener("click", () => { + this.updateAllAlbums(); + }); + } + // Cancel add album buttons (both X and Cancel button) document.getElementById("cancelAddAlbumBtn").addEventListener("click", () => { this.hideAddAlbumForm(); @@ -805,6 +827,9 @@ export class AlbumManager { async show() { this.overlay.classList.add("visible"); hideSpinner(); + // Repaint the Update All button in case a run is still in flight from + // before the dialog was closed. + this._refreshUpdateAllButton(); await this.loadAlbums(); await this.checkForOngoingIndexing(); // <-- Move this after loadAlbums @@ -978,6 +1003,8 @@ export class AlbumManager { albums.forEach((album) => { this.createAlbumCard(album); }); + // Card count just changed — show/hide the Update All button to match. + this._refreshUpdateAllButton(); } catch (error) { console.error("Failed to load albums:", error); } @@ -1349,6 +1376,13 @@ export class AlbumManager { minDimInput.value = album.min_image_dimension ?? 256; } + // Minimum-byte-size gate: stored in bytes, edited in kb. Fall back to + // 8192 bytes (Album.min_image_bytes default) for albums predating it. + const minBytesInput = editForm.querySelector(".edit-album-min-image-bytes"); + if (minBytesInput) { + minBytesInput.value = Math.round((album.min_image_bytes ?? 8192) / 1024); + } + // Initialize the encoder dropdown for THIS specific card populateEncoderSelect(editForm.querySelector(".edit-album-encoder"), album.encoder_spec); @@ -1469,12 +1503,19 @@ export class AlbumManager { const minDim = Number.isFinite(minDimParsed) && minDimParsed >= 1 ? minDimParsed : (album.min_image_dimension ?? 256); + // Byte-size gate is edited in kb but stored in bytes; 0 disables it. + const minBytesRaw = editForm.querySelector(".edit-album-min-image-bytes")?.value; + const minBytesParsed = Number.parseInt(minBytesRaw, 10); + const minBytes = + Number.isFinite(minBytesParsed) && minBytesParsed >= 0 ? minBytesParsed * 1024 : (album.min_image_bytes ?? 8192); + const updatedAlbum = { key: album.key, name: editForm.querySelector(".edit-album-name").value, description: editForm.querySelector(".edit-album-description").value, encoder_spec: editForm.querySelector(".edit-album-encoder")?.value || album.encoder_spec || DEFAULT_ENCODER_SPEC, min_image_dimension: minDim, + min_image_bytes: minBytes, }; let sourceChanged = false; @@ -1561,7 +1602,7 @@ export class AlbumManager { // Backend guard: check if indexing is already running try { const progress = await fetchJson(`index_progress/${albumKey}`); - if (progress.status === "indexing" || progress.status === "scanning" || progress.status === "mapping") { + if (AlbumManager.RUNNING_STATUSES.includes(progress.status)) { console.log(`Backend reports indexing already in progress for album: ${albumKey}`); this.showProgressUIWithoutScroll(cardElement, progress); this.startProgressPolling(albumKey, cardElement); @@ -1598,6 +1639,92 @@ export class AlbumManager { this.startProgressPolling(albumKey, cardElement); } + // Polls the backend until the album's indexing run leaves a running state. + // Deliberately independent of the card progress pollers: hide() tears those + // down when the dialog closes, and "Update All" must keep working through + // its queue whether or not the dialog is open. + async _waitForIndexingToFinish(albumKey) { + let consecutiveFailures = 0; + for (;;) { + await new Promise((resolve) => setTimeout(resolve, AlbumManager.POLL_INTERVAL)); + try { + const progress = await fetchJson(`index_progress/${albumKey}`); + consecutiveFailures = 0; + if (!AlbumManager.RUNNING_STATUSES.includes(progress.status)) { + return; + } + } catch (error) { + // Tolerate transient failures (the server can be busy mid-scan); + // give up only after several in a row so the queue can't hang. + consecutiveFailures += 1; + if (consecutiveFailures >= AlbumManager.MAX_PROGRESS_POLL_FAILURES) { + console.error(`Giving up waiting on indexing progress for album ${albumKey}:`, error); + return; + } + } + } + } + + // Repaint the Update All button from instance state. The button is looked + // up fresh and this is also called from show() and loadAlbums(), so the + // label survives the dialog being closed and reopened mid-run, and the + // button hides entirely when no albums exist (fresh install / setup mode). + _refreshUpdateAllButton() { + const btn = document.getElementById("updateAllBtn"); + if (!btn) { + return; + } + const hasAlbums = !!this.albumsList?.querySelector(".album-card"); + btn.style.display = hasAlbums ? "" : "none"; + if (this.updateAllProgress) { + const { finished, total } = this.updateAllProgress; + btn.disabled = true; + btn.textContent = `Updating ${finished}/${total}…`; + } else { + btn.disabled = false; + btn.textContent = "Update All"; + } + } + + // Update the index of every album, MAX_CONCURRENT_UPDATES at a time (see + // the constant for why the pipeline is kept shallow). Albums already + // indexing are awaited rather than restarted; one album failing doesn't + // stop the rest. + async updateAllAlbums() { + if (this.updateAllProgress) { + return; + } + const cards = [...this.albumsList.querySelectorAll(".album-card[data-album-key]")]; + if (cards.length === 0) { + return; + } + + this.updateAllProgress = { finished: 0, total: cards.length }; + this._refreshUpdateAllButton(); + + const queue = [...cards]; + const worker = async () => { + let card; + while ((card = queue.shift())) { + const albumKey = card.dataset.albumKey; + try { + await this.startIndexing(albumKey, this._liveCardFor(albumKey, card)); + await this._waitForIndexingToFinish(albumKey); + } catch (error) { + console.error(`Update All: indexing failed for album ${albumKey}:`, error); + } + this.updateAllProgress.finished += 1; + this._refreshUpdateAllButton(); + } + }; + await Promise.all( + Array.from({ length: Math.min(AlbumManager.MAX_CONCURRENT_UPDATES, cards.length) }, () => worker()) + ); + + this.updateAllProgress = null; + this._refreshUpdateAllButton(); + } + showProgressUI(cardElement) { this.showProgressUIWithoutScroll(cardElement); @@ -1876,7 +2003,7 @@ export class AlbumManager { try { const progress = await fetchJson(`index_progress/${albumKey}`); - if (progress.status === "indexing" || progress.status === "scanning") { + if (AlbumManager.RUNNING_STATUSES.includes(progress.status)) { console.log(`Restoring progress UI for ongoing operation: ${albumKey} (${progress.status})`); this.showProgressUIWithoutScroll(cardElement, progress); diff --git a/photomap/frontend/templates/modules/album-manager.html b/photomap/frontend/templates/modules/album-manager.html index 18f14f92..0e0e8ae2 100644 --- a/photomap/frontend/templates/modules/album-manager.html +++ b/photomap/frontend/templates/modules/album-manager.html @@ -6,6 +6,7 @@

Album Management

+
@@ -205,8 +206,13 @@

Edit Album

- - + +
+ + kb + + pixels +
diff --git a/tests/backend/test_albums.py b/tests/backend/test_albums.py index 0cc844e7..155159ae 100644 --- a/tests/backend/test_albums.py +++ b/tests/backend/test_albums.py @@ -533,3 +533,44 @@ def test_legacy_album_dict_loads_as_directory_album(): assert album.invokeai_board_ids == [] # And directory albums keep their YAML free of InvokeAI keys. assert not any(k.startswith("invokeai") for k in album.to_dict()) + + +def test_min_image_bytes_round_trips(client, tmp_path): + """The Edit Album dialogue's byte-size gate must round-trip through + add_album → /available_albums → /update_album, defaulting to 8192, and + 0 (gate disabled) must be accepted.""" + img_dir = tmp_path / "imgs" + img_dir.mkdir() + + response = client.post( + "/add_album/", + json={ + "key": "bytes_default", + "name": "Default bytes", + "image_paths": [str(img_dir)], + "index": str(tmp_path / "b.npz"), + "umap_eps": 0.1, + "encoder_spec": "openai-clip:ViT-B/32", + }, + ) + assert response.status_code == 201 + + listing = {a["key"]: a for a in client.get("/available_albums/").json()} + assert listing["bytes_default"]["min_image_bytes"] == 8192 + + # Explicit value (16 kb) and the 0 = disabled sentinel must both persist. + for value in (16 * 1024, 0): + response = client.post( + "/update_album/", + json={ + "key": "bytes_default", + "name": "Default bytes", + "image_paths": [str(img_dir)], + "index": str(tmp_path / "b.npz"), + "encoder_spec": "openai-clip:ViT-B/32", + "min_image_bytes": value, + }, + ) + assert response.status_code == 200 + listing = {a["key"]: a for a in client.get("/available_albums/").json()} + assert listing["bytes_default"]["min_image_bytes"] == value diff --git a/tests/backend/test_embeddings_diff.py b/tests/backend/test_embeddings_diff.py index b6355727..33371fda 100644 --- a/tests/backend/test_embeddings_diff.py +++ b/tests/backend/test_embeddings_diff.py @@ -56,8 +56,15 @@ def test_windows_style_path_normalised_to_posix(self): def _embeddings_stub(tmp_path: Path) -> Embeddings: """Construct an ``Embeddings`` pointed at a path inside ``tmp_path`` so the encoder spec and clip root are valid but no model is ever loaded — - we only exercise the pure-Python diff method here.""" - return Embeddings(embeddings_path=tmp_path / "stub.npz") + we only exercise the pure-Python diff method here. + + ``min_image_dimension=0`` and ``min_image_bytes=0`` disable both gate + bands: the diff would otherwise stat/probe new files on disk, and these + tests use fabricated paths that don't exist (the gate itself is covered + in ``test_index.py``).""" + return Embeddings( + embeddings_path=tmp_path / "stub.npz", min_image_dimension=0, min_image_bytes=0 + ) class TestNewAndMissingDiff: diff --git a/tests/backend/test_index.py b/tests/backend/test_index.py index 2e2a87ad..59b81206 100644 --- a/tests/backend/test_index.py +++ b/tests/backend/test_index.py @@ -14,6 +14,18 @@ TEST_IMAGE_COUNT = count_test_images() +def _save_noise_jpeg(path, width, height, seed=0): + """Random-noise JPEG: compresses poorly, so even smallish dimensions stay + above the gate's byte-size reject floor. Solid-color images do NOT — a + 300x300 single-color JPEG is ~3 KB and gets floor-rejected regardless of + its pixel dimensions, which is exactly what the floor is for.""" + from PIL import Image + + rng = np.random.default_rng(seed) + arr = rng.integers(0, 256, (height, width, 3), dtype=np.uint8) + Image.fromarray(arr).save(path) + + def test_index_creation( client: TestClient, new_album: dict, monkeypatch: pytest.MonkeyPatch ): @@ -446,18 +458,17 @@ def test_min_image_dimension_filters_small_images(tmp_path): Mirrors the original bug where a hard-coded 100KB byte-size filter was silently dropping ~25% of a real user's photo library. """ - from PIL import Image img_dir = tmp_path / "imgs" img_dir.mkdir() # 100x100 — below the default 256 threshold; should be skipped. - Image.new("RGB", (100, 100), color="red").save(img_dir / "tiny.jpg") + _save_noise_jpeg(img_dir / "tiny.jpg", 100, 100) # 200x300 — height passes 256 but width does not; should be skipped. - Image.new("RGB", (200, 300), color="blue").save(img_dir / "narrow.jpg") + _save_noise_jpeg(img_dir / "narrow.jpg", 200, 300) # 300x300 — both dimensions pass; should be kept. - Image.new("RGB", (300, 300), color="green").save(img_dir / "ok.jpg") + _save_noise_jpeg(img_dir / "ok.jpg", 300, 300) # 256x256 — exactly on the boundary; >= passes; should be kept. - Image.new("RGB", (256, 256), color="yellow").save(img_dir / "exact.jpg") + _save_noise_jpeg(img_dir / "exact.jpg", 256, 256) # Non-image extension, should never get to the dimension check. (img_dir / "notes.txt").write_text("not an image") @@ -468,9 +479,265 @@ def test_min_image_dimension_filters_small_images(tmp_path): f"only 256+ images should be indexed, got {names}" ) - # Lowering the threshold makes the smaller ones eligible too. + # Lowering the threshold makes the smaller ones eligible too. The byte + # floor is disabled here to exercise the pixel gate in isolation — the + # small noise JPEGs sit below the default 8 KB floor. emb_low = Embeddings( - embeddings_path=tmp_path / "ignored.npz", min_image_dimension=100 + embeddings_path=tmp_path / "ignored.npz", + min_image_dimension=100, + min_image_bytes=0, ) names_low = sorted(Path(p).name for p in emb_low.get_image_files_from_directory(img_dir)) assert names_low == ["exact.jpg", "narrow.jpg", "ok.jpg", "tiny.jpg"] + + +def test_dimension_gate_size_shortcircuit(tmp_path): + """Files over DIMENSION_PROBE_MAX_BYTES pass the gate on byte size alone, + without the per-file header open. Proven with junk bytes PIL can't parse: + the big file passes (never opened), the small one is probed and dropped. + """ + from photomap.backend.embeddings import DIMENSION_PROBE_MAX_BYTES + + img_dir = tmp_path / "imgs" + img_dir.mkdir() + (img_dir / "big_junk.jpg").write_bytes(b"x" * (DIMENSION_PROBE_MAX_BYTES + 1)) + (img_dir / "small_junk.jpg").write_bytes(b"x" * 1024) + + emb = Embeddings(embeddings_path=tmp_path / "ignored.npz") + names = [Path(p).name for p in emb.get_image_files_from_directory(img_dir)] + assert names == ["big_junk.jpg"] + + +def test_update_scan_probes_only_new_files(tmp_path, monkeypatch): + """The update-path diff must dimension-probe only files that are not + already in the index — re-probing the whole library on every update was + the dominant scan cost. + """ + + img_dir = tmp_path / "imgs" + img_dir.mkdir() + for name in ["a.jpg", "b.jpg", "c.jpg", "d.jpg"]: + _save_noise_jpeg(img_dir / name, 300, 300) + # A new file that must be probed and rejected as too small. + _save_noise_jpeg(img_dir / "new_tiny.jpg", 100, 100) + + # a-c are already indexed (stored as resolved posix strings, matching + # what _process_images_batch writes); d and new_tiny are new. + existing = np.array( + [(img_dir / n).resolve().as_posix() for n in ["a.jpg", "b.jpg", "c.jpg"]] + ) + + probed: list[str] = [] + original_gate = Embeddings._passes_dimension_gate + + def spy(self, path, st=None): + probed.append(path.name) + return original_gate(self, path, st) + + monkeypatch.setattr(Embeddings, "_passes_dimension_gate", spy) + + emb = Embeddings(embeddings_path=tmp_path / "ignored.npz") + new_paths, missing_paths = emb._get_new_and_missing_images(img_dir, existing) + + assert sorted(probed) == ["d.jpg", "new_tiny.jpg"] + assert {p.name for p in new_paths} == {"d.jpg"} + assert missing_paths == set() + + +def _make_probe_spy(monkeypatch): + """Monkeypatch _passes_dimension_gate to record which files get probed.""" + probed: list[str] = [] + original_gate = Embeddings._passes_dimension_gate + + def spy(self, path, st=None): + probed.append(path.name) + return original_gate(self, path, st) + + monkeypatch.setattr(Embeddings, "_passes_dimension_gate", spy) + return probed + + +def test_scan_reject_cache_skips_reprobing(tmp_path, monkeypatch): + """A file the gate rejected is remembered (with size/mtime) in the + scan-reject cache; the next update dismisses it on the stat alone + instead of re-opening it. + """ + + from photomap.backend.embeddings import SCAN_REJECTS_FILENAME + + img_dir = tmp_path / "imgs" + img_dir.mkdir() + _save_noise_jpeg(img_dir / "ok.jpg", 300, 300) + _save_noise_jpeg(img_dir / "tiny.jpg", 100, 100) + existing = np.array([(img_dir / "ok.jpg").resolve().as_posix()]) + + emb = Embeddings(embeddings_path=tmp_path / "idx" / "embeddings.npz") + + # First update: tiny.jpg is new, gets probed, is rejected and cached. + new1, _ = emb._get_new_and_missing_images(img_dir, existing) + assert new1 == set() + assert (tmp_path / "idx" / SCAN_REJECTS_FILENAME).exists() + + # Second update: nothing gets probed at all. + probed = _make_probe_spy(monkeypatch) + new2, _ = emb._get_new_and_missing_images(img_dir, existing) + assert new2 == set() + assert probed == [] + + +def test_scan_reject_cache_revalidates_changed_files(tmp_path, monkeypatch): + """A cached rejection is keyed to size/mtime: replacing the file with a + large-enough image must be noticed and the file indexed.""" + + img_dir = tmp_path / "imgs" + img_dir.mkdir() + _save_noise_jpeg(img_dir / "photo.jpg", 100, 100) + existing = np.array([]) + + emb = Embeddings(embeddings_path=tmp_path / "idx" / "embeddings.npz") + new1, _ = emb._get_new_and_missing_images(img_dir, existing) + assert new1 == set() + + # Same name, new content (different size and mtime), now big enough. + _save_noise_jpeg(img_dir / "photo.jpg", 400, 400) + + probed = _make_probe_spy(monkeypatch) + new2, _ = emb._get_new_and_missing_images(img_dir, existing) + assert probed == ["photo.jpg"] + assert {p.name for p in new2} == {"photo.jpg"} + + +def test_scan_reject_cache_invalidated_by_min_dim_change(tmp_path): + """The cache memoizes verdicts for one min_image_dimension; changing the + threshold must discard it rather than keep stale rejections.""" + + img_dir = tmp_path / "imgs" + img_dir.mkdir() + _save_noise_jpeg(img_dir / "small.jpg", 100, 100) + existing = np.array([]) + index_path = tmp_path / "idx" / "embeddings.npz" + + # Byte floor off in both instances so the pixel-gate change is the only + # variable this test exercises. + emb = Embeddings(embeddings_path=index_path, min_image_bytes=0) # 256px gate + new1, _ = emb._get_new_and_missing_images(img_dir, existing) + assert new1 == set() + + emb_low = Embeddings( + embeddings_path=index_path, min_image_dimension=50, min_image_bytes=0 + ) + new2, _ = emb_low._get_new_and_missing_images(img_dir, existing) + assert {p.name for p in new2} == {"small.jpg"} + + +def test_thumbnail_dirs_pruned_from_traversal(tmp_path): + """Hidden directories and thumbnail-cache directories (and + photomap_index) are pruned from the walk entirely — their contents are + never candidates, gated or not.""" + + img_dir = tmp_path / "imgs" + for sub in ["@eaDir", ".thumbnails", ".@__thumb", "__MACOSX", "photomap_index", "vacation"]: + (img_dir / sub).mkdir(parents=True) + _save_noise_jpeg(img_dir / sub / "pic.jpg", 300, 300) + # Nested hidden caches (the Shotwell case): a 360px thumb passes both + # gates, so only the hidden-dir pruning keeps it out of the index. + shotwell = img_dir / ".shotwell" / "thumbs" / "thumbs360" + shotwell.mkdir(parents=True) + _save_noise_jpeg(shotwell / "thumb0000000000004edd.jpg", 360, 360) + _save_noise_jpeg(img_dir / "top.jpg", 300, 300) + + emb = Embeddings(embeddings_path=tmp_path / "ignored.npz") + # Gated and ungated traversals must prune identically. + for gate in (True, False): + found = { + p.relative_to(img_dir.resolve()).as_posix() + for p in emb.get_image_files_from_directory(img_dir, apply_dimension_gate=gate) + } + assert found == {"top.jpg", "vacation/pic.jpg"} + + +def test_check_progress_callback_reports_gate_progress(tmp_path): + """The gate pass drives a (checked, total) progress callback, ending on + (total, total) so the UI bar completes.""" + + img_dir = tmp_path / "imgs" + img_dir.mkdir() + for i in range(5): + _save_noise_jpeg(img_dir / f"p{i}.jpg", 300, 300) + existing = np.array([]) + + calls: list[tuple[int, int]] = [] + emb = Embeddings(embeddings_path=tmp_path / "idx" / "embeddings.npz") + new, _ = emb._get_new_and_missing_images( + img_dir, existing, check_progress_callback=lambda checked, total: calls.append((checked, total)) + ) + + assert len(new) == 5 + assert calls[-1] == (5, 5) + + +def test_dimension_gate_byte_reject_floor(tmp_path, monkeypatch): + """Files under the album's min_image_bytes are rejected without a header + open; the floor is an independent per-album setting that can be lowered + or disabled (0) for libraries with legitimately small photos.""" + from PIL import Image + + from photomap.backend import embeddings as embeddings_module + from photomap.backend.embeddings import DIMENSION_REJECT_MIN_BYTES + + img_dir = tmp_path / "imgs" + img_dir.mkdir() + # A solid-color 300x300 JPEG: passes the pixel gate but compresses far + # below the byte floor — the accepted ~0.15% false-negative case. + Image.new("RGB", (300, 300), color="red").save(img_dir / "solid.jpg") + solid_size = (img_dir / "solid.jpg").stat().st_size + assert solid_size < DIMENSION_REJECT_MIN_BYTES, "premise: solid jpg must be sub-floor" + + opens: list[str] = [] + real_open = embeddings_module.Image.open + + def counting_open(fp, *args, **kwargs): + opens.append(str(fp)) + return real_open(fp, *args, **kwargs) + + monkeypatch.setattr(embeddings_module.Image, "open", counting_open) + + # Default 8 KB floor: rejected on the stat alone — no open. + emb = Embeddings(embeddings_path=tmp_path / "ignored.npz") + assert emb.get_image_files_from_directory(img_dir) == [] + assert opens == [] + + # Lowered floor (1 kb): the file is probed and kept. + emb_low_floor = Embeddings( + embeddings_path=tmp_path / "ignored.npz", min_image_bytes=1024 + ) + found = emb_low_floor.get_image_files_from_directory(img_dir) + assert [Path(f).name for f in found] == ["solid.jpg"] + assert len(opens) == 1 + + # Floor disabled entirely (0): same outcome via the probe. + emb_no_floor = Embeddings( + embeddings_path=tmp_path / "ignored.npz", min_image_bytes=0 + ) + found = emb_no_floor.get_image_files_from_directory(img_dir) + assert [Path(f).name for f in found] == ["solid.jpg"] + assert len(opens) == 2 + + +def test_index_metadata_reflects_last_update_operation(client, new_album): + """The "Index updated " timestamp must advance after every + successful update operation, including a no-change one — the .npz is + deliberately not rewritten then, so its mtime alone would make a + freshly-refreshed album look stale.""" + import time + + build_index(client, new_album) + key = new_album["key"] + meta1 = client.get(f"/index_metadata/{key}").json() + + time.sleep(0.05) + build_index(client, new_album) # nothing new — the noop update path + meta2 = client.get(f"/index_metadata/{key}").json() + + assert meta2["filename_count"] == meta1["filename_count"] + assert meta2["last_modified"] > meta1["last_modified"] diff --git a/tests/backend/test_scan_serialization.py b/tests/backend/test_scan_serialization.py new file mode 100644 index 00000000..3bda78fe --- /dev/null +++ b/tests/backend/test_scan_serialization.py @@ -0,0 +1,50 @@ +"""Concurrent index operations must not traverse the filesystem at the same +time. The scan opens every candidate file's header for the dimension gate, so +it is seek- and GIL-bound: two overlapping traversals run far slower than the +same two back-to-back (see ``_scan_semaphore`` in ``embeddings.py``). +""" + +import asyncio +import threading +import time +from pathlib import Path + +from photomap.backend import embeddings as embeddings_module +from photomap.backend.embeddings import Embeddings + + +def test_concurrent_scans_are_serialized(tmp_path: Path, monkeypatch) -> None: + active = 0 + max_active = 0 + lock = threading.Lock() + + def fake_scan(self, image_paths_or_dir, exts=None, progress_callback=None, **kwargs): + nonlocal active, max_active + with lock: + active += 1 + max_active = max(max_active, active) + # Long enough that unserialized scans would reliably overlap. + time.sleep(0.05) + with lock: + active -= 1 + return [] + + monkeypatch.setattr(Embeddings, "get_image_files", fake_scan) + # The semaphore is created lazily and may be bound to a previous test's + # event loop; reset so this test's asyncio.run() gets a fresh one. + monkeypatch.setattr(embeddings_module, "_scan_semaphore", None) + + async def run_both() -> None: + one = Embeddings(embeddings_path=tmp_path / "a" / "embeddings.npz") + two = Embeddings(embeddings_path=tmp_path / "b" / "embeddings.npz") + # The empty scan result makes create_index_async record a "no images" + # error and return before any encoder work — the traversal is all + # that runs, which is exactly the stage under test. + await asyncio.gather( + one.create_index_async(tmp_path / "imgs_a", "scan_serialization_a"), + two.create_index_async(tmp_path / "imgs_b", "scan_serialization_b"), + ) + + asyncio.run(run_both()) + + assert max_active == 1, "two album scans overlapped despite _scan_semaphore" diff --git a/tests/frontend/album-manager-update-all.test.js b/tests/frontend/album-manager-update-all.test.js new file mode 100644 index 00000000..46e23582 --- /dev/null +++ b/tests/frontend/album-manager-update-all.test.js @@ -0,0 +1,253 @@ +/** + * Tests for the album-manager "Update All" button: the backend-driven wait + * that lets it march through the queue even when the dialog (and its UI + * pollers) is closed, the bounded worker pool, and the button repaint that + * survives a close/reopen of the dialog. + * + * album-manager.js pulls in a large sibling graph (index.js -> umap.js, etc.) + * whose modules touch the DOM at import time, so we mock the direct imports and + * dynamically load the module under test — the same pattern used by + * album-manager-progress.test.js. + */ +import { afterEach, beforeAll, describe, expect, jest, test } from "@jest/globals"; + +const M = "../../photomap/frontend/static/javascript"; + +jest.unstable_mockModule(`${M}/filetree.js`, () => ({ + createSimpleDirectoryPicker: jest.fn(), +})); +jest.unstable_mockModule(`${M}/index.js`, () => ({ + getIndexMetadata: jest.fn(), + removeIndex: jest.fn(), + updateIndex: jest.fn(), +})); +jest.unstable_mockModule(`${M}/search-ui.js`, () => ({ + exitSearchMode: jest.fn(), +})); +jest.unstable_mockModule(`${M}/settings.js`, () => ({ + closeSettingsModal: jest.fn(), + loadAvailableAlbums: jest.fn(), + openSettingsModal: jest.fn(), +})); +jest.unstable_mockModule(`${M}/state.js`, () => ({ + setAlbum: jest.fn(), + state: {}, +})); +jest.unstable_mockModule(`${M}/utils.js`, () => ({ + fetchJson: jest.fn(() => Promise.resolve({})), + hideSpinner: jest.fn(), + showSpinner: jest.fn(), +})); + +let AlbumManager; +let fetchJson; + +beforeAll(async () => { + // album-manager.js instantiates `new AlbumManager()` at module load, and its + // constructor wires click handlers on these buttons without null-guards, so + // they must exist before import or the module eval throws. + document.body.innerHTML = + `
` + + ["addAlbumBtn", "cancelAddAlbumBtn", "cancelAddAlbumBtn2", "closeAlbumManagementBtn", "showAddAlbumBtn"] + .map((id) => ``) + .join(""); + + ({ fetchJson } = await import(`${M}/utils.js`)); + ({ AlbumManager } = await import(`${M}/album-manager.js`)); + // The wait loop sleeps POLL_INTERVAL between backend polls; keep tests fast. + AlbumManager.POLL_INTERVAL = 5; +}); + +afterEach(() => { + fetchJson.mockReset(); + fetchJson.mockImplementation(() => Promise.resolve({})); +}); + +const flush = () => new Promise((resolve) => setTimeout(resolve, 20)); + +function makeAlbumsList(keys) { + const albumsList = document.createElement("div"); + for (const key of keys) { + const card = document.createElement("div"); + card.className = "album-card"; + card.dataset.albumKey = key; + albumsList.appendChild(card); + } + return albumsList; +} + +// A fake manager exposing just what updateAllAlbums touches. startIndexing is +// recorded, and each album's "run" finishes only when the test resolves its +// deferred — so tests control how many updates are in flight. +function makeManager(keys) { + const deferreds = new Map(); + const mgr = { + updateAllProgress: null, + albumsList: makeAlbumsList(keys), + startIndexing: jest.fn(() => Promise.resolve()), + _liveCardFor: (albumKey, fallback) => fallback, + _refreshUpdateAllButton: AlbumManager.prototype._refreshUpdateAllButton, + _waitForIndexingToFinish: jest.fn( + (albumKey) => + new Promise((resolve) => { + deferreds.set(albumKey, resolve); + }) + ), + }; + return { mgr, deferreds }; +} + +describe("_waitForIndexingToFinish", () => { + test("keeps waiting while the backend reports a running status, then returns", async () => { + const statuses = ["scanning", "indexing", "mapping", "completed"]; + fetchJson.mockImplementation(() => Promise.resolve({ status: statuses.shift() })); + + await AlbumManager.prototype._waitForIndexingToFinish.call({}, "alb"); + + expect(statuses).toHaveLength(0); // consumed through "completed" + expect(fetchJson).toHaveBeenCalledWith("index_progress/alb"); + }); + + test("returns when the backend reports idle (job never started)", async () => { + fetchJson.mockImplementation(() => Promise.resolve({ status: "idle" })); + await AlbumManager.prototype._waitForIndexingToFinish.call({}, "alb"); + expect(fetchJson).toHaveBeenCalledTimes(1); + }); + + test("survives transient poll failures but gives up after the limit", async () => { + let calls = 0; + fetchJson.mockImplementation(() => { + calls += 1; + if (calls === 1) { + return Promise.reject(new Error("busy")); + } + if (calls === 2) { + return Promise.resolve({ status: "indexing" }); + } + return Promise.reject(new Error("down")); + }); + + await AlbumManager.prototype._waitForIndexingToFinish.call({}, "alb"); + + // 1 failure (tolerated) + 1 running + MAX_PROGRESS_POLL_FAILURES failures. + expect(calls).toBe(2 + AlbumManager.MAX_PROGRESS_POLL_FAILURES); + }); +}); + +describe("_refreshUpdateAllButton", () => { + test("repaints the live button from instance state (survives reopen)", () => { + const btn = document.createElement("button"); + btn.id = "updateAllBtn"; + btn.textContent = "Update All"; + document.body.appendChild(btn); + try { + const mgr = { updateAllProgress: { finished: 2, total: 5 } }; + AlbumManager.prototype._refreshUpdateAllButton.call(mgr); + expect(btn.disabled).toBe(true); + expect(btn.textContent).toBe("Updating 2/5…"); + + mgr.updateAllProgress = null; + AlbumManager.prototype._refreshUpdateAllButton.call(mgr); + expect(btn.disabled).toBe(false); + expect(btn.textContent).toBe("Update All"); + } finally { + btn.remove(); + } + }); +}); + +describe("updateAllAlbums", () => { + test("updates every album but keeps at most MAX_CONCURRENT_UPDATES in flight", async () => { + const keys = ["a", "b", "c", "d"]; + const { mgr, deferreds } = makeManager(keys); + + const run = AlbumManager.prototype.updateAllAlbums.call(mgr); + await flush(); + + expect(mgr.startIndexing).toHaveBeenCalledTimes(AlbumManager.MAX_CONCURRENT_UPDATES); + + deferreds.get("a")(); + await flush(); + expect(mgr.startIndexing).toHaveBeenCalledTimes(3); + + deferreds.get("b")(); + await flush(); + expect(mgr.startIndexing).toHaveBeenCalledTimes(4); + + deferreds.get("c")(); + deferreds.get("d")(); + await run; + + expect(mgr.startIndexing.mock.calls.map(([key]) => key)).toEqual(keys); + expect(mgr.updateAllProgress).toBeNull(); + }); + + test("a failing album does not stop the remaining updates", async () => { + const { mgr } = makeManager(["a", "b", "c"]); + mgr.startIndexing = jest.fn((albumKey) => + albumKey === "a" ? Promise.reject(new Error("boom")) : Promise.resolve() + ); + mgr._waitForIndexingToFinish = jest.fn(() => Promise.resolve()); + + await AlbumManager.prototype.updateAllAlbums.call(mgr); + + expect(mgr.startIndexing.mock.calls.map(([key]) => key)).toEqual(["a", "b", "c"]); + expect(mgr.updateAllProgress).toBeNull(); + }); + + test("disables the button while running and restores it afterwards", async () => { + const btn = document.createElement("button"); + btn.id = "updateAllBtn"; + btn.textContent = "Update All"; + document.body.appendChild(btn); + try { + const { mgr, deferreds } = makeManager(["a", "b"]); + + const run = AlbumManager.prototype.updateAllAlbums.call(mgr); + await flush(); + expect(btn.disabled).toBe(true); + expect(btn.textContent).toBe("Updating 0/2…"); + + deferreds.get("a")(); + await flush(); + expect(btn.textContent).toBe("Updating 1/2…"); + + deferreds.get("b")(); + await run; + expect(btn.disabled).toBe(false); + expect(btn.textContent).toBe("Update All"); + } finally { + btn.remove(); + } + }); + + test("re-entry and empty album list are no-ops", async () => { + const { mgr } = makeManager([]); + await AlbumManager.prototype.updateAllAlbums.call(mgr); + expect(mgr.startIndexing).not.toHaveBeenCalled(); + + const { mgr: busy } = makeManager(["a"]); + busy.updateAllProgress = { finished: 0, total: 3 }; + await AlbumManager.prototype.updateAllAlbums.call(busy); + expect(busy.startIndexing).not.toHaveBeenCalled(); + }); +}); + +describe("Update All visibility", () => { + test("hidden when no albums exist, shown once albums are present", () => { + const btn = document.createElement("button"); + btn.id = "updateAllBtn"; + document.body.appendChild(btn); + try { + const empty = { updateAllProgress: null, albumsList: makeAlbumsList([]) }; + AlbumManager.prototype._refreshUpdateAllButton.call(empty); + expect(btn.style.display).toBe("none"); + + const populated = { updateAllProgress: null, albumsList: makeAlbumsList(["a"]) }; + AlbumManager.prototype._refreshUpdateAllButton.call(populated); + expect(btn.style.display).toBe(""); + } finally { + btn.remove(); + } + }); +});