From 5ff2b1aeebde68cea02ae004a02204c17efe6889 Mon Sep 17 00:00:00 2001 From: TX-RX Date: Wed, 8 Jul 2026 12:09:15 -0500 Subject: [PATCH 1/2] fix(data-package): upsert on filename to survive ATAK re-uploads MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `data_packages.filename` has a UNIQUE constraint. ATAK reuses the same filename (photo capture timestamp) whenever it re-uploads a data package — e.g. re-sending the same photo to a different recipient. On the second and later attempts, `save_data_package_to_db` raised `IntegrityError` on the filename constraint, caught it, and returned a Flask response tuple that its caller ignored. The upload endpoint returned 200 anyway, with a URL containing the new hash. ATAK then sent a CoT referencing that URL, and every recipient's `/Marti/sync/content?hash=` lookup returned 404: the file was on disk (keyed by hash) but no `data_packages` row existed for it, so the download handler's DB-first query missed. Silent delivery failure to every recipient after the first. Fix: SELECT-then-INSERT-or-UPDATE. Look up the existing row by filename; if it exists, update its hash and metadata in place; if not, INSERT as before. Latest upload wins, which matches ATAK's mental model when a user re-shares the same photo. The remaining IntegrityError catch handles the narrow race where a concurrent upload of the same filename slips between our SELECT and COMMIT. Reproduced on the operator's Azure OTS install: photo `20260707_214621.jpg.zip` uploaded four times with distinct hashes; DB only stored the first, recipients' `sync/content` requests for attempts 2–4 all 404'd. Also removes the misleading `"This data package has already been uploaded"` response — the endpoint was already returning 200 anyway, so the error tuple was unreachable in practice. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../marti_api/data_package_marti_api.py | 57 +++++++++++-------- 1 file changed, 33 insertions(+), 24 deletions(-) diff --git a/opentakserver/blueprints/marti_api/data_package_marti_api.py b/opentakserver/blueprints/marti_api/data_package_marti_api.py index 09ef52c1..2cb9d0e7 100644 --- a/opentakserver/blueprints/marti_api/data_package_marti_api.py +++ b/opentakserver/blueprints/marti_api/data_package_marti_api.py @@ -64,35 +64,44 @@ def save_data_package_to_db( username: str = None, eud_uid: str = None, ): + # data_packages.filename has a UNIQUE constraint. ATAK reuses the same + # filename (photo capture timestamp) whenever it re-uploads a data package + # — e.g. re-sending the same photo to a different recipient. A naive + # INSERT would raise IntegrityError on the second attempt; the caller + # ignored that error and returned 200 anyway, so the follow-up CoT + # referenced a hash the DB never learned about and every downstream + # /Marti/sync/content?hash=... lookup returned 404 (the file was on disk + # but not queryable). Upsert on filename so the latest hash wins. + existing = db.session.execute( + db.select(DataPackage).filter_by(filename=filename) + ).scalar_one_or_none() + + data_package = existing if existing is not None else DataPackage() + data_package.filename = filename + data_package.hash = sha256_hash + data_package.creator_uid = request.args.get("CreatorUid") # iTAK + data_package.creator_uid = request.args.get("creatorUid") # All other TAK clients + data_package.submission_user = current_user.id if current_user.is_authenticated else None + data_package.submission_time = datetime.now(timezone.utc) + data_package.mime_type = mimetype + data_package.size = file_size + data_package.creator_uid = eud_uid + + if username: + user = app.security.datastore.find_user(username=username) + if user: + data_package.submission_user = user.id + try: - data_package = DataPackage() - data_package.filename = filename - data_package.hash = sha256_hash - data_package.creator_uid = request.args.get("CreatorUid") # iTAK - data_package.creator_uid = request.args.get("creatorUid") # All other TAK clients - data_package.submission_user = current_user.id if current_user.is_authenticated else None - data_package.submission_time = datetime.now(timezone.utc) - data_package.mime_type = mimetype - data_package.size = file_size - data_package.creator_uid = eud_uid - - if username: - user = app.security.datastore.find_user(username=username) - if user: - data_package.submission_user = user.id - - db.session.add(data_package) + if existing is None: + db.session.add(data_package) db.session.commit() except sqlalchemy.exc.IntegrityError as e: + # Only reachable via a concurrent insert of the same filename between + # our SELECT and COMMIT. Log and give up rather than loop. db.session.rollback() - logger.error("Failed to save data package: {}".format(e)) + logger.error("Failed to save data package '{}': {}".format(filename, e)) logger.debug(traceback.format_exc()) - return ( - jsonify( - {"success": False, "error": gettext("This data package has already been uploaded")} - ), - 400, - ) def create_data_package_zip(file: FileStorage | str) -> str: From 1b8d7090ca06d92ce4795fe6a079318a5af8e5a4 Mon Sep 17 00:00:00 2001 From: TX-RX Date: Sat, 11 Jul 2026 12:39:03 -0500 Subject: [PATCH 2/2] fix(data-package): honor client sha256 in create_data_package_zip ATAK computes the sha256 of the raw file it uploads and passes it as the ?hash= query param on POST /Marti/sync/missionupload. It then looks up the wrapped package via PUT /Marti/api/sync/metadata//tool and GET /Marti/sync/content?hash= using that same client hash. The server was computing its own sha256 of the server-wrapped zip, saving the file and DB row under that server hash. Every client lookup then 404'd because the client hash was never on disk. Fileshare CoT reached the recipient but the follow-up download stalled with a 502/404. Honor request.args.get('hash') whenever present (which is always, in the request-driven code path); fall back to computing one only for non-request callers (there are none today, but the function still accepts a str path). Also emit a debug log with the filename/hash pair so future upload traces show the invariant at a glance. Verified synthetically on Azure OTS 20.114.34.147 at 2026-07-11 17:38 UTC: raw 2 KB JPG posted with a known sha256 => server saved the wrapped zip under exactly that hash (log, on-disk file, and HTTP response body all agreed). Real ATAK 5.7 and iTAK clients pre-wrap and therefore hit save_data_package_file instead; this branch protects the legacy raw-file upload path and any third-party tooling that hits missionupload with a non-zip file. --- .../marti_api/data_package_marti_api.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/opentakserver/blueprints/marti_api/data_package_marti_api.py b/opentakserver/blueprints/marti_api/data_package_marti_api.py index 2cb9d0e7..60314909 100644 --- a/opentakserver/blueprints/marti_api/data_package_marti_api.py +++ b/opentakserver/blueprints/marti_api/data_package_marti_api.py @@ -164,19 +164,29 @@ def create_data_package_zip(file: FileStorage | str) -> str: zipf.writestr("MANIFEST/manifest.xml", tostring(manifest)) zipf.close() - # Get the sha256 hash of the data package zip for its file name on disk and for the data_packages table zip_file = open(os.path.join(app.config.get("UPLOAD_FOLDER"), f"{filename}.zip"), "rb") zip_file_bytes = zip_file.read() zip_file.close() - sha256 = hashlib.sha256() - sha256.update(zip_file_bytes) - data_package_hash = sha256.hexdigest() + # ATAK looks up the data package by the sha256 IT computed of the raw file + # it uploaded — the same value it sent as ?hash=… on the POST — via + # PUT /Marti/api/sync/metadata//tool and later via sync/content?hash=… + # If we key the row and on-disk file by a server-computed hash of the + # server-wrapped zip, every client lookup 404s and the fileshare stalls. + # Honor the client hash whenever present; fall back to computing one only + # for non-request callers (there are none today, but the function still + # accepts a str path). + data_package_hash = request.args.get("hash") if request else None + if not data_package_hash: + sha256 = hashlib.sha256() + sha256.update(zip_file_bytes) + data_package_hash = sha256.hexdigest() os.rename( os.path.join(app.config.get("UPLOAD_FOLDER"), f"{filename}.zip"), os.path.join(app.config.get("UPLOAD_FOLDER"), f"{data_package_hash}.zip"), ) + logger.debug("Wrapped and saved data package: {} - {}".format(filename, data_package_hash)) save_data_package_to_db( f"{filename}.zip", data_package_hash, "application/zip", len(zip_file_bytes) )