From d838305cd3f2decbaa72120dff81ade891ce7693 Mon Sep 17 00:00:00 2001 From: daralynnrhode Date: Fri, 24 Jul 2026 09:59:16 -0600 Subject: [PATCH 1/6] Raise an exception and test for it --- imap_processing/cli.py | 1 + imap_processing/tests/test_cli.py | 49 +++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+) diff --git a/imap_processing/cli.py b/imap_processing/cli.py index 6bacdde06..eec215a10 100644 --- a/imap_processing/cli.py +++ b/imap_processing/cli.py @@ -489,6 +489,7 @@ def upload_products(self, products: list[Path]) -> None: continue else: logger.error(f"Upload failed with error: {message}") + raise except Exception as e: logger.error(f"Upload failed unknown error: {e}") diff --git a/imap_processing/tests/test_cli.py b/imap_processing/tests/test_cli.py index 82e8a51fe..1d28091da 100644 --- a/imap_processing/tests/test_cli.py +++ b/imap_processing/tests/test_cli.py @@ -908,3 +908,52 @@ def test_post_processing( "naif0012.tls", "imap_sclk_0001.tsc", ] + +@mock.patch("imap_processing.cli.filter_day_boundary_data") +@mock.patch("imap_processing.cli.swe_l1a") + +def test_post_processing_upload_503_error( + mock_swe_l1a, + mock_filter, + mock_instrument_dependencies, +): + """Test coverage for post processing when the upload fails with 503 error""" + + mocks = mock_instrument_dependencies + mocks["mock_download"].return_value = "dependency0" + mocks["mock_write_cdf"].side_effect = [ + "/path/to/imap_swe_l1a_test_20100105_v001.cdf" + ] + mocks[ + "mock_write_cdf" + ].return_value = "/path/to/imap_swe_l1a_test_20100105_v001.cdf" + mocks["mock_query"].return_value = [] + + #Mocks a 503 error received from the upload API + mocks["mock_upload"].side_effect = imap_data_access.io.IMAPDataAccessError( + '503 Service Unavailable: {"error": "ServiceUnavailable", ' + '"message": "The API is too busy."}' + ) + + test_ds = xr.Dataset() + mock_swe_l1a.return_value = [test_ds] + mock_filter.side_effect = lambda ds, _: ds + input_collection = ProcessingInputCollection( + ScienceInput("imap_swe_l0_raw_20100105_v001.pkts"), + SPICEInput("naif0012.tls", "imap_sclk_0001.tsc"), + ) + mocks["mock_pre_processing"].return_value = input_collection + + dependency_str = ( + '[{"type": "science","files": ["imap_swe_l0_raw_20100105_v001.pkts"]}, ' + '{"type": "spice", "files": ["naif0012.tls", "imap_sclk_0001.tsc"]}]' + ) + instrument = Swe("l1a", "raw", dependency_str, "20100105", None, "v001", True) + + # Checks that the upload failed and logs an error and raises an exception + with mock.patch("logging.Logger.error") as mock_error: + with pytest.raises(imap_data_access.io.IMAPDataAccessError): + instrument.process() + + # Check the upload failure was logged + assert any( "Upload failed with error" in str(call) for call in mock_error.call_args_list) From bc0c4c7a51d1d300cca3dcdaf25d7145124b8282 Mon Sep 17 00:00:00 2001 From: daralynnrhode Date: Fri, 24 Jul 2026 10:43:49 -0600 Subject: [PATCH 2/6] forgot to run pre-check --- imap_processing/tests/test_cli.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/imap_processing/tests/test_cli.py b/imap_processing/tests/test_cli.py index 822046c68..50cde5a24 100644 --- a/imap_processing/tests/test_cli.py +++ b/imap_processing/tests/test_cli.py @@ -938,9 +938,9 @@ def test_post_processing( "imap_sclk_0001.tsc", ] + @mock.patch("imap_processing.cli.filter_day_boundary_data") @mock.patch("imap_processing.cli.swe_l1a") - def test_post_processing_upload_503_error( mock_swe_l1a, mock_filter, @@ -958,7 +958,7 @@ def test_post_processing_upload_503_error( ].return_value = "/path/to/imap_swe_l1a_test_20100105_v001.cdf" mocks["mock_query"].return_value = [] - #Mocks a 503 error received from the upload API + # Mocks a 503 error received from the upload API mocks["mock_upload"].side_effect = imap_data_access.io.IMAPDataAccessError( '503 Service Unavailable: {"error": "ServiceUnavailable", ' '"message": "The API is too busy."}' @@ -985,4 +985,7 @@ def test_post_processing_upload_503_error( instrument.process() # Check the upload failure was logged - assert any( "Upload failed with error" in str(call) for call in mock_error.call_args_list) + assert any( + "Upload failed with error" in str(call) + for call in mock_error.call_args_list + ) From e612ed227ed5126278e42d467ef123698b40587e Mon Sep 17 00:00:00 2001 From: daralynnrhode Date: Fri, 24 Jul 2026 10:48:26 -0600 Subject: [PATCH 3/6] forgot to run pre-check --- imap_processing/tests/test_cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/imap_processing/tests/test_cli.py b/imap_processing/tests/test_cli.py index 50cde5a24..47652096b 100644 --- a/imap_processing/tests/test_cli.py +++ b/imap_processing/tests/test_cli.py @@ -984,7 +984,7 @@ def test_post_processing_upload_503_error( with pytest.raises(imap_data_access.io.IMAPDataAccessError): instrument.process() - # Check the upload failure was logged + # Checks the upload failure was logged assert any( "Upload failed with error" in str(call) for call in mock_error.call_args_list From 8b9b013c8e1a022673ff165164a1ff7672f72abd Mon Sep 17 00:00:00 2001 From: daralynnrhode Date: Tue, 28 Jul 2026 09:25:10 -0600 Subject: [PATCH 4/6] adding a sleep cycle --- imap_processing/cli.py | 42 +++++++++++++++++++++++-------- imap_processing/tests/test_cli.py | 23 +++++++++++------ 2 files changed, 47 insertions(+), 18 deletions(-) diff --git a/imap_processing/cli.py b/imap_processing/cli.py index 158a12c14..d7bd916fd 100644 --- a/imap_processing/cli.py +++ b/imap_processing/cli.py @@ -18,6 +18,7 @@ import sys from abc import ABC, abstractmethod from pathlib import Path +from time import sleep from typing import final import imap_data_access @@ -472,6 +473,8 @@ def upload_products(self, products: list[Path]) -> None: """ Upload data products to the IMAP SDC. + If a 503 SlowDown error is reported from the Upload API, a retry will be made + Parameters ---------- products : list[Path] @@ -483,19 +486,36 @@ def upload_products(self, products: list[Path]) -> None: return for filename in products: - try: - logger.info(f"Uploading file: {filename}") - imap_data_access.upload(filename) - except IMAPDataAccessError as e: - message = str(e) - if "FileAlreadyExists" in message and "409" in message: - logger.warning("Skipping upload of existing file, %s", filename) - continue - else: + max_retries = 3 + + for attempt in range(max_retries): + try: + logger.info(f"Uploading file: {filename}") + imap_data_access.upload(filename) + break + + except IMAPDataAccessError as e: + message = str(e) + + if "FileAlreadyExists" in message and "409" in message: + logger.warning( + "Skipping upload of existing file, %s", filename + ) + break + elif "503" in message and "SlowDown" in message: + if attempt < max_retries - 1: + logger.warning( + "Upload busy. Waiting 5 seconds before retrying..." + ) + sleep(5) + continue + logger.error(f"Upload failed with error: {message}") raise - except Exception as e: - logger.error(f"Upload failed unknown error: {e}") + + except Exception as e: + logger.error(f"Upload failed unknown error: {e}") + raise @final def process(self) -> None: diff --git a/imap_processing/tests/test_cli.py b/imap_processing/tests/test_cli.py index 47652096b..73b5bc735 100644 --- a/imap_processing/tests/test_cli.py +++ b/imap_processing/tests/test_cli.py @@ -939,11 +939,13 @@ def test_post_processing( ] +@mock.patch("imap_processing.cli.sleep") @mock.patch("imap_processing.cli.filter_day_boundary_data") @mock.patch("imap_processing.cli.swe_l1a") def test_post_processing_upload_503_error( mock_swe_l1a, mock_filter, + mock_sleep, mock_instrument_dependencies, ): """Test coverage for post processing when the upload fails with 503 error""" @@ -960,8 +962,10 @@ def test_post_processing_upload_503_error( # Mocks a 503 error received from the upload API mocks["mock_upload"].side_effect = imap_data_access.io.IMAPDataAccessError( - '503 Service Unavailable: {"error": "ServiceUnavailable", ' - '"message": "The API is too busy."}' + "503 Service Unavailable: " + "503 Slow Down" + "Code: SlowDown" + "Message: Please reduce your request rate." ) test_ds = xr.Dataset() @@ -984,8 +988,13 @@ def test_post_processing_upload_503_error( with pytest.raises(imap_data_access.io.IMAPDataAccessError): instrument.process() - # Checks the upload failure was logged - assert any( - "Upload failed with error" in str(call) - for call in mock_error.call_args_list - ) + # Upload should attempt 3 times + assert mocks["mock_upload"].call_count == 3 + + # Sleep should be called 2 times after first two failures + assert mock_sleep.call_count == 2 + + # Checks the upload failure was logged + assert any( + "Upload failed with error" in str(call) for call in mock_error.call_args_list + ) From 4c0ab996af1951b74665e67a356ee071d68737a1 Mon Sep 17 00:00:00 2001 From: daralynnrhode Date: Tue, 28 Jul 2026 10:04:34 -0600 Subject: [PATCH 5/6] adding test for unknown error from upload --- imap_processing/tests/test_cli.py | 49 +++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/imap_processing/tests/test_cli.py b/imap_processing/tests/test_cli.py index 73b5bc735..47c1afc57 100644 --- a/imap_processing/tests/test_cli.py +++ b/imap_processing/tests/test_cli.py @@ -998,3 +998,52 @@ def test_post_processing_upload_503_error( assert any( "Upload failed with error" in str(call) for call in mock_error.call_args_list ) + + +@mock.patch("imap_processing.cli.filter_day_boundary_data") +@mock.patch("imap_processing.cli.swe_l1a") +def test_post_processing_upload_unknown_error( + mock_swe_l1a, + mock_filter, + mock_instrument_dependencies, +): + """Test coverage for post processing when the upload fails with unknown error""" + + mocks = mock_instrument_dependencies + mocks["mock_download"].return_value = "dependency0" + mocks["mock_write_cdf"].side_effect = [ + "/path/to/imap_swe_l1a_test_20100105_v001.cdf" + ] + mocks[ + "mock_write_cdf" + ].return_value = "/path/to/imap_swe_l1a_test_20100105_v001.cdf" + mocks["mock_query"].return_value = [] + + # Mocks an unknown error received from the upload API + mocks["mock_upload"].side_effect = RuntimeError("Unexpected failure") + + test_ds = xr.Dataset() + mock_swe_l1a.return_value = [test_ds] + mock_filter.side_effect = lambda ds, _: ds + input_collection = ProcessingInputCollection( + ScienceInput("imap_swe_l0_raw_20100105_v001.pkts"), + SPICEInput("naif0012.tls", "imap_sclk_0001.tsc"), + ) + mocks["mock_pre_processing"].return_value = input_collection + + dependency_str = ( + '[{"type": "science","files": ["imap_swe_l0_raw_20100105_v001.pkts"]}, ' + '{"type": "spice", "files": ["naif0012.tls", "imap_sclk_0001.tsc"]}]' + ) + instrument = Swe("l1a", "raw", dependency_str, "20100105", None, "v001", True) + + # Checks that the upload failed and logs an error and raises an exception + with mock.patch("logging.Logger.error") as mock_error: + with pytest.raises(RuntimeError): + instrument.process() + + # Checks the upload failure was logged + assert any( + "Upload failed unknown error" in str(call) + for call in mock_error.call_args_list + ) From 0cc20516bd9a315920b97b6161901c10536aff2e Mon Sep 17 00:00:00 2001 From: daralynnrhode Date: Tue, 28 Jul 2026 10:17:18 -0600 Subject: [PATCH 6/6] precommit fail fix --- imap_processing/tests/test_cli.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/imap_processing/tests/test_cli.py b/imap_processing/tests/test_cli.py index 47c1afc57..9b902c4f3 100644 --- a/imap_processing/tests/test_cli.py +++ b/imap_processing/tests/test_cli.py @@ -1044,6 +1044,5 @@ def test_post_processing_upload_unknown_error( # Checks the upload failure was logged assert any( - "Upload failed unknown error" in str(call) - for call in mock_error.call_args_list + "Upload failed unknown error" in str(call) for call in mock_error.call_args_list )