fix: report plaintext size/etag for multipart objects in listings

s3proxy/handlers/buckets.py

@@ -15,7 +15,14 @@
 from .. import xml_responses
 from ..client import S3Credentials
 from ..errors import S3Error
-from ..state import INTERNAL_PREFIX, META_SUFFIX_LEGACY, delete_multipart_metadata
+from ..state import (
+    INTERNAL_PREFIX,
+    META_SUFFIX_LEGACY,
+    delete_multipart_metadata,
+    load_multipart_metadata,
+    plaintext_attr_cache,
+    synthetic_multipart_etag,
+)
 from ..xml_utils import find_element, find_elements
 from .base import BaseHandler
 
@@ -211,24 +218,46 @@ async def _process_list_objects(self, client, bucket: str, contents: list[dict])
         sem = asyncio.Semaphore(LIST_HEAD_CONCURRENCY)
 
         async def resolve(obj: dict) -> dict:
+            backend_etag = str(obj.get("ETag", "")).strip('"')
+            cached = plaintext_attr_cache.get(bucket, obj["Key"], backend_etag)
+            if cached is not None:
+                size, etag = cached
+                return self._list_entry(obj, size, etag)
             async with sem:
                 try:
                     head = await client.head_object(bucket, obj["Key"])
                     meta = head.get("Metadata", {})
-                    size = self._get_plaintext_size(meta, obj.get("Size", 0))
-                    etag = self._get_effective_etag(meta, obj.get("ETag", ""))
+                    if "plaintext-size" in meta:
+                        size = self._get_plaintext_size(meta, obj.get("Size", 0))
+                        etag = self._get_effective_etag(meta, obj.get("ETag", ""))
+                    elif mp_meta := await load_multipart_metadata(client, bucket, obj["Key"]):
+                        # Multipart objects can't carry plaintext-size in user
+                        # metadata (it is fixed at CreateMultipartUpload); the
+                        # size lives in the .meta sidecar. Reporting the backend
+                        # Size here would leak the ciphertext size and make sync
+                        # clients re-upload every multipart object on each pass.
+                        size = mp_meta.total_plaintext_size
+                        etag = synthetic_multipart_etag(size)
+                    else:
+                        size = self._get_plaintext_size(meta, obj.get("Size", 0))
+                        etag = self._get_effective_etag(meta, obj.get("ETag", ""))
+                    plaintext_attr_cache.put(bucket, obj["Key"], backend_etag, size, etag)
                 except Exception:
-                    size, etag = obj.get("Size", 0), obj.get("ETag", "").strip('"')
-            return {
-                "key": obj["Key"],
-                "last_modified": _s3_timestamp(obj["LastModified"]),
-                "etag": etag,
-                "size": size,
-                "storage_class": obj.get("StorageClass", "STANDARD"),
-            }
+                    size, etag = obj.get("Size", 0), backend_etag
+            return self._list_entry(obj, size, etag)
 
         return await asyncio.gather(*[resolve(obj) for obj in listable])
 
+    @staticmethod
+    def _list_entry(obj: dict, size: int, etag: str) -> dict:
+        return {
+            "key": obj["Key"],
+            "last_modified": _s3_timestamp(obj["LastModified"]),
+            "etag": etag,
+            "size": size,
+            "storage_class": obj.get("StorageClass", "STANDARD"),
+        }
+
     async def handle_create_bucket(self, request: Request, creds: S3Credentials) -> Response:
         bucket = self._parse_bucket(request.url.path)
         async with self._client(creds) as client:

s3proxy/handlers/multipart/lifecycle.py

@@ -23,7 +23,9 @@
     PartMetadata,
     delete_upload_state,
     persist_upload_state,
+    plaintext_attr_cache,
     save_multipart_metadata,
+    synthetic_multipart_etag,
 )
 from ...xml_utils import find_elements, get_element_text
 from ..base import BaseHandler
@@ -179,11 +181,21 @@ async def handle_complete_multipart_upload(
 
             # Complete in S3
             try:
-                await client.complete_multipart_upload(bucket, key, upload_id, s3_parts)
+                complete_resp = await client.complete_multipart_upload(
+                    bucket, key, upload_id, s3_parts
+                )
             except ClientError as e:
                 await self._handle_complete_error(
                     e, client, bucket, key, upload_id, s3_parts, completed_parts, total_plaintext
                 )
+            else:
+                plaintext_attr_cache.put(
+                    bucket,
+                    key,
+                    str(complete_resp.get("ETag", "")).strip('"'),
+                    total_plaintext,
+                    synthetic_multipart_etag(total_plaintext),
+                )
 
             # Save metadata first, then delete state.
             # Order matters: if metadata save fails, state is preserved

s3proxy/state/__init__.py

@@ -8,6 +8,9 @@
 - Recovery logic for lost Redis state
 """
 
+# Plaintext attribute cache for listings
+from .attr_cache import PlaintextAttrCache, plaintext_attr_cache, synthetic_multipart_etag
+
 # State manager and storage
 from .manager import MAX_INTERNAL_PARTS_PER_CLIENT, MultipartStateManager
 
@@ -81,6 +84,10 @@
     "load_upload_state",
     "persist_upload_state",
     "save_multipart_metadata",
+    # Plaintext attribute cache
+    "PlaintextAttrCache",
+    "plaintext_attr_cache",
+    "synthetic_multipart_etag",
     # Recovery
     "reconstruct_upload_state_from_s3",
     # Serialization

s3proxy/state/attr_cache.py

@@ -0,0 +1,60 @@
+"""Process-local cache of plaintext attributes for completed objects.
+
+Multipart objects can't carry their plaintext size in backend user metadata
+(user metadata is fixed at CreateMultipartUpload, before the final size is
+known), so list pages must resolve it from the .meta sidecar. This cache
+remembers the resolved (size, etag) per (bucket, key, backend_etag) so
+repeated listings — e.g. a backup client walking the same directories on
+every sync pass — skip the per-object round-trips. Keying by backend ETag
+scopes an entry to one object version: overwriting the key changes the
+backend ETag, which invalidates the cached attributes without coordination.
+"""
+
+import hashlib
+from collections import OrderedDict
+
+# ~100k entries of (bucket, key, etag) -> (size, etag) stays in the tens of
+# MB even with long backup keys — well inside the pod memory limit.
+_DEFAULT_MAXSIZE = 100_000
+
+
+def synthetic_multipart_etag(plaintext_size: int) -> str:
+    """ETag reported for multipart-encrypted objects.
+
+    Must stay in sync with what CompleteMultipartUpload and HEAD return for
+    these objects, so listings, HEADs, and upload responses all agree.
+    """
+    return hashlib.md5(str(plaintext_size).encode(), usedforsecurity=False).hexdigest()
+
+
+class PlaintextAttrCache:
+    def __init__(self, maxsize: int = _DEFAULT_MAXSIZE) -> None:
+        self._maxsize = maxsize
+        self._entries: OrderedDict[tuple[str, str, str], tuple[int, str]] = OrderedDict()
+
+    def get(self, bucket: str, key: str, backend_etag: str) -> tuple[int, str] | None:
+        if not backend_etag:
+            return None
+        entry_key = (bucket, key, backend_etag)
+        entry = self._entries.get(entry_key)
+        if entry is not None:
+            self._entries.move_to_end(entry_key)
+        return entry
+
+    def put(self, bucket: str, key: str, backend_etag: str, size: int, etag: str) -> None:
+        if not backend_etag:
+            return
+        entry_key = (bucket, key, backend_etag)
+        self._entries[entry_key] = (size, etag)
+        self._entries.move_to_end(entry_key)
+        while len(self._entries) > self._maxsize:
+            self._entries.popitem(last=False)
+
+    def clear(self) -> None:
+        self._entries.clear()
+
+    def __len__(self) -> int:
+        return len(self._entries)
+
+
+plaintext_attr_cache = PlaintextAttrCache()