From 725aac78cb84a8d178f9fc3f720f493647ae524d Mon Sep 17 00:00:00 2001 From: Anuj Date: Thu, 23 Jul 2026 18:46:43 +0530 Subject: [PATCH 1/2] Update python-osw-validation to 0.4.5 and include warnings in response ## Dev Board Ticket - https://dev.azure.com/TDEI-UW/TDEI/_workitems/edit/4047 - https://dev.azure.com/TDEI-UW/TDEI/_workitems/edit/4048 ## Changes - Updated `python-osw-validation` from `0.4.4` to `0.4.5` - Added `warning` to the validation response payload - Propagated `warnings` from the `python-osw-validation` package result - Updated sample validation response JSON - Added unit test coverage for warning propagation ## Testing ``` Ran 59 tests in 3.194s OK ``` --- requirements.txt | 2 +- src/assets/osw-validation.json | 5 ++-- src/models/queue_message_content.py | 6 +++-- src/osw_validator.py | 2 ++ src/validation.py | 3 +-- .../models/test_queue_message_content.py | 1 + tests/unit_tests/test_service.py | 23 +++++++++++++++++++ tests/unit_tests/test_validation.py | 18 +++++++++++++++ 8 files changed, 53 insertions(+), 7 deletions(-) diff --git a/requirements.txt b/requirements.txt index 13918e7..b0038db 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,4 +4,4 @@ python-ms-core==0.0.25 uvicorn==0.20.0 html_testRunner==1.2.1 geopandas==0.14.4 -python-osw-validation==0.4.4 +python-osw-validation==0.4.5 diff --git a/src/assets/osw-validation.json b/src/assets/osw-validation.json index 1a72f45..4b997b4 100644 --- a/src/assets/osw-validation.json +++ b/src/assets/osw-validation.json @@ -6,6 +6,7 @@ "user_id": "c59d29b6-a063-4249-943f-d320d15ac9ab", "tdei_project_group_id": "0b41ebc5-350c-42d3-90af-3af4ad3628fb", "success": true, - "message": "" + "message": "", + "warning": "" } -} \ No newline at end of file +} diff --git a/src/models/queue_message_content.py b/src/models/queue_message_content.py index aad3f7d..e1d7ed4 100644 --- a/src/models/queue_message_content.py +++ b/src/models/queue_message_content.py @@ -2,8 +2,10 @@ class ValidationResult: - is_valid: bool - validation_message: str + def __init__(self, is_valid: bool = False, validation_message: str = '', warning: str = ''): + self.is_valid = is_valid + self.validation_message = validation_message + self.warning = warning or '' class Upload: diff --git a/src/osw_validator.py b/src/osw_validator.py index 6b8397e..7d1401f 100644 --- a/src/osw_validator.py +++ b/src/osw_validator.py @@ -85,6 +85,7 @@ def validate(self, received_message: Upload): result = ValidationResult() result.is_valid = False result.validation_message = f'Error occurred while validating OSW request {e}' + result.warning = '' self.send_status(result=result, upload_message=received_message) status_sent = True finally: @@ -97,6 +98,7 @@ def send_status(self, result: ValidationResult, upload_message: Upload): upload_message.data.success = result.is_valid upload_message.data.message = result.validation_message resp_data = upload_message.data.to_json() + resp_data['warning'] = result.warning resp_data['package'] = { 'python-ms-core': Core.__version__, 'python-osw-validation': python_osw_validation.__version__ diff --git a/src/validation.py b/src/validation.py index fd04b27..14f9413 100644 --- a/src/validation.py +++ b/src/validation.py @@ -45,8 +45,6 @@ def validate(self, max_errors=20) -> ValidationResult: def is_osw_valid(self, max_errors) -> ValidationResult: start_time = time.time() result = ValidationResult() - result.is_valid = False - result.validation_message = '' root, ext = os.path.splitext(self.file_relative_path) if ext and ext.lower() == '.zip': downloaded_file_path = self.download_single_file(self.file_path) @@ -55,6 +53,7 @@ def is_osw_valid(self, max_errors) -> ValidationResult: validator = OSWValidation(zipfile_path=downloaded_file_path) validation_result = validator.validate(max_errors) result.is_valid = validation_result.is_valid + result.warning = validation_result.warnings if not result.is_valid: result.validation_message = json.dumps(validation_result.issues) logger.error(f' Error While Validating File: {json.dumps(validation_result.issues)}') diff --git a/tests/unit_tests/models/test_queue_message_content.py b/tests/unit_tests/models/test_queue_message_content.py index 3316f63..330f11b 100644 --- a/tests/unit_tests/models/test_queue_message_content.py +++ b/tests/unit_tests/models/test_queue_message_content.py @@ -92,6 +92,7 @@ def test_validation_result_init(self): result.validation_message = 'Validated' self.assertTrue(result.is_valid) self.assertEqual(result.validation_message, 'Validated') + self.assertEqual(result.warning, '') if __name__ == '__main__': diff --git a/tests/unit_tests/test_service.py b/tests/unit_tests/test_service.py index 29c16fa..5ac08db 100644 --- a/tests/unit_tests/test_service.py +++ b/tests/unit_tests/test_service.py @@ -263,6 +263,7 @@ def test_send_status_success(self): validation_result = ValidationResult() validation_result.is_valid = True validation_result.validation_message = '' + validation_result.warning = 'Coordinate precision exceeds 7 decimal places.' mock_message = Upload(data={ 'data': { @@ -281,6 +282,28 @@ def test_send_status_success(self): mock_publish.assert_called_once() + @patch('src.osw_validator.QueueMessage') + def test_send_status_includes_warning_in_response(self, mock_queue_message): + validation_result = ValidationResult( + is_valid=True, + validation_message='', + warning='Coordinate precision exceeds 7 decimal places.' + ) + mock_message = Upload(data={ + 'data': { + 'user_id': '1233', + 'tdei_project_group_id': '444444' + }, + 'message': 'test_message', + 'messageType': 'message_type', + 'messageId': '123' + }) + + self.service.send_status(result=validation_result, upload_message=mock_message) + + response_payload = mock_queue_message.data_from.call_args[0][0] + self.assertEqual(response_payload['data']['warning'], validation_result.warning) + def test_send_status_failure(self): validation_result = ValidationResult() validation_result.is_valid = False diff --git a/tests/unit_tests/test_validation.py b/tests/unit_tests/test_validation.py index 493adc8..1c727f5 100644 --- a/tests/unit_tests/test_validation.py +++ b/tests/unit_tests/test_validation.py @@ -65,6 +65,24 @@ def test_validate_valid_zip(self, mock_download_file, mock_clean_up): # Ensure clean_up is called twice (once for the file, once for the folder) self.assertEqual(mock_clean_up.call_count, 2) + @patch('src.validation.OSWValidation') + @patch('src.validation.Validation.clean_up') + @patch('src.validation.Validation.download_single_file') + def test_validate_includes_warning(self, mock_download_file, mock_clean_up, mock_osw_validation): + """Test the validate method includes warnings from the OSW validator.""" + expected_warning = 'Coordinate precision exceeds 7 decimal places.' + mock_download_file.return_value = f'{SAVED_FILE_PATH}/{SUCCESS_FILE_NAME}' + mock_validation_result = MagicMock() + mock_validation_result.is_valid = True + mock_validation_result.warnings = expected_warning + mock_osw_validation.return_value.validate.return_value = mock_validation_result + + result = self.validation.validate(max_errors=10) + + self.assertTrue(result.is_valid) + self.assertEqual(result.warning, expected_warning) + self.assertEqual(mock_clean_up.call_count, 2) + @patch('src.validation.Validation.clean_up') @patch('src.validation.Validation.download_single_file') From 9a1d6c52e2ea9ef2e846e1d310d6b8cf27d708f8 Mon Sep 17 00:00:00 2001 From: Anuj Date: Thu, 23 Jul 2026 18:53:23 +0530 Subject: [PATCH 2/2] Added config changes --- README.md | 7 ++++++- src/config.py | 3 +++ src/validation.py | 8 +++++++- tests/unit_tests/test_config.py | 9 +++++++++ tests/unit_tests/test_validation.py | 7 +++++++ 5 files changed, 32 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 09053b0..dcde238 100644 --- a/README.md +++ b/README.md @@ -37,6 +37,9 @@ VALIDATION_RES_TOPIC=xxxx CONTAINER_NAME=xxxx AUTH_PERMISSION_URL=xxx # This is the URL to get the token MAX_CONCURRENT_MESSAGES=xxx # Optional if not provided defaults to 2 +MAX_GEOMETRY_VERTICES=xxx # Optional if not provided defaults to 2000 +COORDINATE_PRECISION=xxx # Optional if not provided defaults to 7 +ALLOW_ZERO_LENGTH_LINES=xxx # Optional if not provided defaults to False AUTH_SIMULATE=xxx # Optional if not provided defaults to False ``` @@ -44,6 +47,7 @@ The application connect with the `STORAGECONNECTION` string provided in `.env` f `QUEUECONNECTION` is used to send out the messages and listen to messages. `MAX_CONCURRENT_MESSAGES` is the maximum number of concurrent messages that the service can handle. If not provided, defaults to 2 +`MAX_GEOMETRY_VERTICES`, `COORDINATE_PRECISION`, and `ALLOW_ZERO_LENGTH_LINES` configure `python-osw-validation` behavior. If not provided, the package defaults are used. ### How to Set up and Build Follow the steps to install the python packages required for both building and running the application @@ -96,7 +100,8 @@ Follow the steps to install the python packages required for both building and r "user_id": "user_id", "tdei_project_group_id": "tdei_project_group_id", "success": true/false, - "message": "message" // if false the error string else empty string + "message": "message", // if false the error string else empty string + "warning": "warning" // non-blocking validation warning else empty string }, "publishedDate": "published date" } diff --git a/src/config.py b/src/config.py index f9d84ad..6165da6 100644 --- a/src/config.py +++ b/src/config.py @@ -19,6 +19,9 @@ class Settings(BaseSettings): auth_permission_url: str = os.environ.get('AUTH_PERMISSION_URL', None) max_concurrent_messages: int = os.environ.get('MAX_CONCURRENT_MESSAGES', 1) max_receivable_messages: int = os.environ.get('MAX_RECEIVABLE_MESSAGES',-1) # -1 means no limit + max_geometry_vertices: int = os.environ.get('MAX_GEOMETRY_VERTICES', 2000) + coordinate_precision: int = os.environ.get('COORDINATE_PRECISION', 7) + allow_zero_length_lines: bool = os.environ.get('ALLOW_ZERO_LENGTH_LINES', False) @property def auth_provider(self) -> str: diff --git a/src/validation.py b/src/validation.py index 14f9413..5cd21e9 100644 --- a/src/validation.py +++ b/src/validation.py @@ -7,6 +7,7 @@ from pathlib import Path from .config import Settings from python_osw_validation import OSWValidation +from python_osw_validation.config import ValidationConfig from .models.queue_message_content import ValidationResult import uuid import json @@ -27,6 +28,11 @@ def __init__(self, file_path=None, storage_client=None): self.storage_client = storage_client self.file_path = file_path self.file_relative_path = file_path.split('/')[-1] + self.validation_config = ValidationConfig( + max_geometry_vertices=settings.max_geometry_vertices, + coordinate_precision=settings.coordinate_precision, + allow_zero_length_lines=settings.allow_zero_length_lines, + ) self.client = self.storage_client.get_container(container_name=self.container_name) is_exists = os.path.exists(DOWNLOAD_DIR) unique_id = self.get_unique_id() @@ -50,7 +56,7 @@ def is_osw_valid(self, max_errors) -> ValidationResult: downloaded_file_path = self.download_single_file(self.file_path) if downloaded_file_path: logger.info(f' Downloaded file path: {downloaded_file_path}') - validator = OSWValidation(zipfile_path=downloaded_file_path) + validator = OSWValidation(zipfile_path=downloaded_file_path, config=self.validation_config) validation_result = validator.validate(max_errors) result.is_valid = validation_result.is_valid result.warning = validation_result.warnings diff --git a/tests/unit_tests/test_config.py b/tests/unit_tests/test_config.py index fe2659f..ba40780 100644 --- a/tests/unit_tests/test_config.py +++ b/tests/unit_tests/test_config.py @@ -10,6 +10,9 @@ class TestSettings(unittest.TestCase): @patch.dict(os.environ, { 'AUTH_PERMISSION_URL': 'http://auth-url.com', 'MAX_CONCURRENT_MESSAGES': '5', + 'MAX_GEOMETRY_VERTICES': '1000', + 'COORDINATE_PRECISION': '6', + 'ALLOW_ZERO_LENGTH_LINES': 'True', 'AUTH_SIMULATE': 'True' }, clear=True) def test_settings_with_simulated_auth(self): @@ -17,6 +20,9 @@ def test_settings_with_simulated_auth(self): self.assertEqual(settings.app_name, 'python-osw-validation') self.assertEqual(settings.auth_permission_url, 'http://auth-url.com') self.assertEqual(settings.max_concurrent_messages, 5) + self.assertEqual(settings.max_geometry_vertices, 1000) + self.assertEqual(settings.coordinate_precision, 6) + self.assertTrue(settings.allow_zero_length_lines) self.assertEqual(settings.auth_provider, 'Simulated') @patch.dict(os.environ, { @@ -45,6 +51,9 @@ def test_default_settings(self): self.assertEqual(settings.event_bus.container_name, 'osw') self.assertIsNone(settings.auth_permission_url) self.assertEqual(settings.max_concurrent_messages, 1) + self.assertEqual(settings.max_geometry_vertices, 2000) + self.assertEqual(settings.coordinate_precision, 7) + self.assertFalse(settings.allow_zero_length_lines) if __name__ == '__main__': diff --git a/tests/unit_tests/test_validation.py b/tests/unit_tests/test_validation.py index 1c727f5..126660f 100644 --- a/tests/unit_tests/test_validation.py +++ b/tests/unit_tests/test_validation.py @@ -27,6 +27,9 @@ class TestValidation(unittest.TestCase): def setUp(self, mock_settings): # Mock Settings and storage client to avoid actual dependencies mock_settings.return_value.event_bus.container_name = 'test_container' + mock_settings.return_value.max_geometry_vertices = 2000 + mock_settings.return_value.coordinate_precision = 7 + mock_settings.return_value.allow_zero_length_lines = False self.mock_storage_client = MagicMock() @@ -81,6 +84,10 @@ def test_validate_includes_warning(self, mock_download_file, mock_clean_up, mock self.assertTrue(result.is_valid) self.assertEqual(result.warning, expected_warning) + config = mock_osw_validation.call_args[1]['config'] + self.assertEqual(config.max_geometry_vertices, 2000) + self.assertEqual(config.coordinate_precision, 7) + self.assertFalse(config.allow_zero_length_lines) self.assertEqual(mock_clean_up.call_count, 2)