From b2b11f3229fea6ddf567993e52b89795dc87b1b8 Mon Sep 17 00:00:00 2001 From: Jeremiah Kang Date: Mon, 20 Jul 2026 15:59:07 +0000 Subject: [PATCH 1/6] native.py code formatting --- .gitignore | 1 + src/hecdss/native.py | 107 +++++++++++++++++++++++++------------------ 2 files changed, 63 insertions(+), 45 deletions(-) diff --git a/.gitignore b/.gitignore index b464af1..b97254c 100644 --- a/.gitignore +++ b/.gitignore @@ -17,3 +17,4 @@ libhecdss.so .venv/ tests/csv_testing/ +src/hecdss/lib/libhecdss_backup.so diff --git a/src/hecdss/native.py b/src/hecdss/native.py index a85a8b9..885c4fc 100644 --- a/src/hecdss/native.py +++ b/src/hecdss/native.py @@ -1,15 +1,25 @@ import ctypes -from ctypes import c_float, c_double, c_char_p, c_int, c_void_p, POINTER -from ctypes import c_int32 -from ctypes import byref, create_string_buffer -from ctypes.util import find_library -import numpy as np -from importlib import resources import io import os import sys +from ctypes import ( + POINTER, + byref, + c_char_p, + c_double, + c_float, + c_int, + c_int32, + c_void_p, + create_string_buffer, +) +from ctypes.util import find_library +from importlib import resources from typing import List +import numpy as np + + # from hecdss.location_info import LocationInfo @@ -32,7 +42,6 @@ def load_hecdss_library(self, libname): raise FileNotFoundError(f"{libname} not found Paths searched: {paths_to_try}") return ctypes.CDLL(found_libs[0]) - def __init__(self): """Loads the hecdss shared library from disk""" @@ -239,7 +248,8 @@ def hec_dss_gridRetrieve(self, pathname: str, c_data = (c_float * 0)() result = self.dll.hec_dss_gridRetrieve(self.handle, pathname.encode("utf-8"), False, - ctypes.byref(type_pointer), ctypes.byref(dataType_pointer), + ctypes.byref(type_pointer), ctypes.byref( + dataType_pointer), c_lowerLeftCellX, c_lowerLeftCellY, c_numberOfCellsX, c_numberOfCellsY, c_numberOfRanges, c_srsDefinitionType, @@ -250,9 +260,12 @@ def hec_dss_gridRetrieve(self, pathname: str, c_srsName, srsNameLength, c_srsDefinition, srsDefinitionLength, c_timeZoneID, timeZoneIDLength, - ctypes.byref(c_cellSize), ctypes.byref(c_xCoordOfGridCellZero), - ctypes.byref(c_yCoordOfGridCellZero), ctypes.byref(c_nullValue), - ctypes.byref(c_maxDataValue), ctypes.byref(c_minDataValue), + ctypes.byref(c_cellSize), ctypes.byref( + c_xCoordOfGridCellZero), + ctypes.byref(c_yCoordOfGridCellZero), ctypes.byref( + c_nullValue), + ctypes.byref(c_maxDataValue), ctypes.byref( + c_minDataValue), ctypes.byref(c_meanDataValue), c_rangeLimitTable, rangeTablesLength, c_numberEqualOrExceedingRangeLimit, @@ -269,7 +282,8 @@ def hec_dss_gridRetrieve(self, pathname: str, c_data = (c_float * dataLength)() result = self.dll.hec_dss_gridRetrieve(self.handle, pathname.encode("utf-8"), True, - ctypes.byref(type_pointer), ctypes.byref(dataType_pointer), + ctypes.byref(type_pointer), ctypes.byref( + dataType_pointer), c_lowerLeftCellX, c_lowerLeftCellY, c_numberOfCellsX, c_numberOfCellsY, c_numberOfRanges, c_srsDefinitionType, @@ -280,9 +294,12 @@ def hec_dss_gridRetrieve(self, pathname: str, c_srsName, srsNameLength, c_srsDefinition, srsDefinitionLength, c_timeZoneID, timeZoneIDLength, - ctypes.byref(c_cellSize), ctypes.byref(c_xCoordOfGridCellZero), - ctypes.byref(c_yCoordOfGridCellZero), ctypes.byref(c_nullValue), - ctypes.byref(c_maxDataValue), ctypes.byref(c_minDataValue), + ctypes.byref(c_cellSize), ctypes.byref( + c_xCoordOfGridCellZero), + ctypes.byref(c_yCoordOfGridCellZero), ctypes.byref( + c_nullValue), + ctypes.byref(c_maxDataValue), ctypes.byref( + c_minDataValue), ctypes.byref(c_meanDataValue), c_rangeLimitTable, rangeTablesLength, c_numberEqualOrExceedingRangeLimit, @@ -573,7 +590,8 @@ def hec_dss_pdStore( if numberCurves > 1: _values = pd.values.tolist() if len(_values[0]) > 1: - _values = [[_values[i][j] for i in range(len(_values))] for j in range(len(_values[0]))] + _values = [[_values[i][j] + for i in range(len(_values))] for j in range(len(_values[0]))] values2 = np.array(_values) else: values2 = pd.values @@ -739,7 +757,7 @@ def hec_dss_tsRetrieve( times: List[int], values: List[float], arraySize: str, - numberValuesRead, + numberValuesRead: List[int], quality: List[int], qualityLength: int, julianBaseDate: List[int], @@ -923,7 +941,7 @@ def hec_dss_tsStoreIrregular( c_char_p, # units (const char*) c_char_p, # type (const char*) c_char_p, # timeZoneName (const char*) - c_int, # storageFlag (int) + c_int, # storageFlag (int) ] c_pathname = c_char_p(pathname.encode("utf-8")) @@ -1042,7 +1060,6 @@ def hec_dss_arrayRetrieve(self, pathname, intValues: List[int], floatValues: Lis if len(doubleValues) > 0: doubleValues[:] = np.ctypeslib.as_array(c_doubleValues, shape=(len(doubleValues),)) - else: print("Error reading array status = {status}") @@ -1159,28 +1176,28 @@ def hec_dss_locationStore(self, location_info, replace: int) -> int: return result def hec_dss_delete(self, pathname: str) -> int: - """ - Deletes a record from the DSS file. + """ + Deletes a record from the DSS file. - Args: - pathname (str): The pathname of the record to delete. + Args: + pathname (str): The pathname of the record to delete. - Returns: - int: Status of zero when successful, non-zero on error. - """ - f = self.dll.hec_dss_delete - f.argtypes = [ - c_void_p, # dss_file* dss - c_char_p # const char* pathname - ] - f.restype = c_int + Returns: + int: Status of zero when successful, non-zero on error. + """ + f = self.dll.hec_dss_delete + f.argtypes = [ + c_void_p, # dss_file* dss + c_char_p # const char* pathname + ] + f.restype = c_int - result = f(self.handle, pathname.encode("utf-8")) + result = f(self.handle, pathname.encode("utf-8")) - if result != 0: - print("Function call failed with result:", result) + if result != 0: + print("Function call failed with result:", result) - return result + return result def hec_dss_textStore(self, pathname, text, length=None): """ @@ -1200,12 +1217,12 @@ def hec_dss_textStore(self, pathname, text, length=None): f.restype = c_int result = f(self.handle, pathname.encode("utf-8"), - text.encode("utf-8"), - length if length is not None else len(text)) - + text.encode("utf-8"), + length if length is not None else len(text)) + return result - - def hec_dss_textRetrieve(self, pathname, buffer :List[str], buff_size: int) -> int: + + def hec_dss_textRetrieve(self, pathname, buffer: List[str], buff_size: int) -> int: """ Store text data in a DSS file. Args: @@ -1224,8 +1241,8 @@ def hec_dss_textRetrieve(self, pathname, buffer :List[str], buff_size: int) -> i c_buffer = create_string_buffer(buff_size) result = f(self.handle, pathname.encode("utf-8"), - c_buffer, - buff_size) - + c_buffer, + buff_size) + buffer.append(c_buffer.value.decode("utf-8")) - return result \ No newline at end of file + return result From ce2a6803a8d2377c301426520ed57798f3221657 Mon Sep 17 00:00:00 2001 From: Jeremiah Kang Date: Mon, 20 Jul 2026 16:34:29 +0000 Subject: [PATCH 2/6] formatted hecdss.py --- src/hecdss/hecdss.py | 88 +++++++++++++++++++++++++------------------- 1 file changed, 51 insertions(+), 37 deletions(-) diff --git a/src/hecdss/hecdss.py b/src/hecdss/hecdss.py index 51df75c..0144c6a 100644 --- a/src/hecdss/hecdss.py +++ b/src/hecdss/hecdss.py @@ -6,17 +6,18 @@ import hecdss.record_type from hecdss.array_container import ArrayContainer +from hecdss.catalog import Catalog +from hecdss.dateconverter import DateConverter +from hecdss.dsspath import DssPath +from hecdss.gridded_data import GriddedData +from hecdss.irregular_timeseries import IrregularTimeSeries from hecdss.location_info import LocationInfo -from hecdss.text import Text -from hecdss.paired_data import PairedData from hecdss.native import _Native -from hecdss.dateconverter import DateConverter +from hecdss.paired_data import PairedData from hecdss.record_type import RecordType from hecdss.regular_timeseries import RegularTimeSeries -from hecdss.irregular_timeseries import IrregularTimeSeries -from hecdss.catalog import Catalog -from hecdss.gridded_data import GriddedData -from hecdss.dsspath import DssPath +from hecdss.text import Text + DSS_UNDEFINED_VALUE = -340282346638528859811704183484516925440.000000 @@ -24,7 +25,8 @@ class HecDss: """ Main class for working with DSS files """ - def __init__(self, filename:str): + + def __init__(self, filename: str): """constructor for HecDSS Args: @@ -56,6 +58,7 @@ def __exit__(self, exc_type, exc_val, exc_tb): exc_tb (traceback): The traceback object. """ self.close() + @staticmethod def set_global_debug_level(level: int) -> None: """ @@ -73,6 +76,7 @@ def set_global_debug_level(level: int) -> None: """ # Set the native DLL level (controls what gets written) _Native().hec_dss_set_debug_level(level) + def close(self): """closes the DSS file and releases any locks """ @@ -115,7 +119,7 @@ def get(self, pathname: str, startdatetime=None, enddatetime=None, trim=False): """ if type == RecordType.RegularTimeSeries or type == RecordType.IrregularTimeSeries: new_pathname = pathname - if(DssPath(pathname).D.lower() != "ts-pattern"): + if (DssPath(pathname).D.lower() != "ts-pattern"): new_pathname = DssPath(pathname).path_without_date().__str__() elif type == RecordType.IrregularTimeSeries: raise ValueError("ts-pattern is not fully supported for irregular time series") @@ -137,16 +141,15 @@ def get(self, pathname: str, startdatetime=None, enddatetime=None, trim=False): def _get_text(self, pathname: str): textLength = 1024 - BUFFER_TOO_SMALL = -17 textArray = [] status = self._native.hec_dss_textRetrieve(pathname, textArray, textLength) while status == BUFFER_TOO_SMALL: textLength *= 2 - if textLength > 2*1048576: # 2 MB + if textLength > 2 * 1048576: # 2 MB print(f"Text record too large to read from '{pathname}'") return None - textArray = [] # otherwise we get an entry for each attempt + textArray = [] # otherwise we get an entry for each attempt status = self._native.hec_dss_textRetrieve(pathname, textArray, textLength) if status != 0: @@ -162,7 +165,8 @@ def _get_array(self, pathname: str): floatValuesCount = [0] doubleValuesCount = [0] - self._native.hec_dss_arrayRetrieveInfo(pathname, intValuesCount, floatValuesCount, doubleValuesCount) + self._native.hec_dss_arrayRetrieveInfo( + pathname, intValuesCount, floatValuesCount, doubleValuesCount) intValues = [] floatValues = [] @@ -177,7 +181,8 @@ def _get_array(self, pathname: str): status = self._native.hec_dss_arrayRetrieve(pathname, intValues, floatValues, doubleValues) location_info = self._get_location_info(pathname) - rval = ArrayContainer.create_array_container(intValues, floatValues, doubleValues, path=pathname, location_info=location_info) + rval = ArrayContainer.create_array_container( + intValues, floatValues, doubleValues, path=pathname, location_info=location_info) return rval def _get_gridded_data(self, pathname): @@ -340,7 +345,7 @@ def _get_paired_data(self, pathname): # ---------------------------------------------------------------------------------------- # # rearrange from consecutive values for each curve to consecutive curves for each ordinate # # ---------------------------------------------------------------------------------------- # - groups = [doubleValues[i*ordinateCount:(i+1)*ordinateCount] for i in range(n)] + groups = [doubleValues[i * ordinateCount:(i + 1) * ordinateCount] for i in range(n)] doubleValues = list(map(list, zip(*groups))) pd.values = np.array(doubleValues).reshape((len(doubleOrdinates), n)) # pd.values = [doubleValues[i:i+n] for i in range(0, len(doubleValues), n)] @@ -373,26 +378,29 @@ def _get_julian_time_range(self, pathname, boolFullSet): def _get_date_time_range(self, pathname, boolFullSet): - firstValidJulian, firstSeconds, lastValidJulian, lastSeconds = self._get_julian_time_range(pathname, boolFullSet) + firstValidJulian, firstSeconds, lastValidJulian, lastSeconds = self._get_julian_time_range( + pathname, boolFullSet) first = DateConverter.date_times_from_julian_array(firstSeconds, 1, firstValidJulian[0])[0] last = DateConverter.date_times_from_julian_array(lastSeconds, 1, lastValidJulian[0])[0] return (first, last) - def _get_timeseries(self, pathname, startDateTime, endDateTime, trim): # get sizes - firstValidJulian, firstSeconds, lastValidJulian, lastSeconds = self._get_julian_time_range(pathname, 1) + firstValidJulian, firstSeconds, lastValidJulian, lastSeconds = self._get_julian_time_range( + pathname, 1) if startDateTime is None: - _startDateTime = DateConverter.date_time_from_julian_second(firstValidJulian[0], firstSeconds[0]) + _startDateTime = DateConverter.date_time_from_julian_second( + firstValidJulian[0], firstSeconds[0]) firstSeconds, firstJulian = firstSeconds, firstValidJulian else: _startDateTime = startDateTime minutes = DateConverter.julian_array_from_date_times([startDateTime])[0] firstSeconds, firstJulian = [minutes % 1440 * 60], [minutes // 1440] if endDateTime is None: - _endDateTime = DateConverter.date_time_from_julian_second(lastValidJulian[0], lastSeconds[0]) + _endDateTime = DateConverter.date_time_from_julian_second( + lastValidJulian[0], lastSeconds[0]) lastSeconds, lastJulian = lastSeconds, lastValidJulian else: _endDateTime = endDateTime @@ -452,7 +460,7 @@ def _get_timeseries(self, pathname, startDateTime, endDateTime, trim): endTime, times, values, - number_periods+1, + number_periods + 1, numberValuesRead, quality, qualityElementSize[0], @@ -478,14 +486,15 @@ def _get_timeseries(self, pathname, startDateTime, endDateTime, trim): else: ts = RegularTimeSeries() if trim or not startDateTime or not endDateTime: - trimmed_indices = [i for i, v in enumerate(values) if values[i] != DSS_UNDEFINED_VALUE] + trimmed_indices = [i for i, v in enumerate( + values) if values[i] != DSS_UNDEFINED_VALUE] if not trimmed_indices: times = [] values = [] quality = [] else: start = 0 if startDateTime and not trim else trimmed_indices[0] - end = len(times) if endDateTime and not trim else trimmed_indices[-1]+1 + end = len(times) if endDateTime and not trim else trimmed_indices[-1] + 1 times = times[start:end] values = values[start:end] if quality != []: @@ -495,7 +504,8 @@ def _get_timeseries(self, pathname, startDateTime, endDateTime, trim): ) arr = np.array(values) if RecordType.IrregularTimeSeries == type(ts): - indices = np.where(np.isclose(values, DSS_UNDEFINED_VALUE, rtol=0, atol=0, equal_nan=True))[0] + indices = np.where(np.isclose(values, DSS_UNDEFINED_VALUE, + rtol=0, atol=0, equal_nan=True))[0] arr = np.delete(arr, indices) new_times = [new_times[i] for i in range(len(new_times)) if not np.isin(i, indices)] quality = [quality[i] for i in range(len(quality)) if not np.isin(i, indices)] @@ -507,10 +517,10 @@ def _get_timeseries(self, pathname, startDateTime, endDateTime, trim): time_granularity_seconds = timeGranularitySeconds[0] julian_base_date = julianBaseDate[0] timeZoneName = timeZoneName[0] - if(timeZoneName): + if (timeZoneName): try: new_times = [i.replace(tzinfo=ZoneInfo(timeZoneName)) for i in new_times] - except ZoneInfoNotFoundError as e: + except ZoneInfoNotFoundError as e: print(f"Warning: {e}. Using no zone instead.") timeZoneName = False elif (DssPath(pathname).D.lower() == "ts-pattern"): @@ -518,7 +528,8 @@ def _get_timeseries(self, pathname, startDateTime, endDateTime, trim): start_date = _startDateTime - timedelta(seconds=interval_seconds) location_info = self._get_location_info(pathname) - ts = ts.create(values=values, times=new_times, quality=quality, units=units, data_type=data_type, start_date=start_date, time_granularity_seconds=time_granularity_seconds, julian_base_date=julian_base_date, time_zone_name=timeZoneName, path=pathname, location_info=location_info) + ts = ts.create(values=values, times=new_times, quality=quality, units=units, data_type=data_type, start_date=start_date, + time_granularity_seconds=time_granularity_seconds, julian_base_date=julian_base_date, time_zone_name=timeZoneName, path=pathname, location_info=location_info) if (DssPath(pathname).D.lower() == "ts-pattern"): new_interval = ts._get_interval_path() @@ -623,10 +634,11 @@ def put(self, container) -> int: raise ValueError("ts-pattern is not fully supported for irregular time series") # def hec_dss_tsStoreRegular(dss, pathname, startDate, startTime, valueArray, qualityArray, # saveAsFloat, units, type): - start_date_base = (datetime(1900, 1, 1)+timedelta(days=its.julian_base_date)) + start_date_base = (datetime(1900, 1, 1) + timedelta(days=its.julian_base_date)) startDate, startTime = DateConverter.dss_datetime_strings_from_datetime(start_date_base) quality = container.quality - julian_times = DateConverter.julian_array_from_date_times(its.times, its.time_granularity_seconds, start_date_base) + julian_times = DateConverter.julian_array_from_date_times( + its.times, its.time_granularity_seconds, start_date_base) if max(julian_times) >= 2147483647: raise Exception("Julian times contains value larger than 2147483647, increase granularity or change " "start_date_base to fix.") @@ -654,20 +666,22 @@ def put(self, container) -> int: status = self._native.hec_dss_gridStore(gd) self._catalog = None elif type(container) is ArrayContainer: - status = self._native.hec_dss_arrayStore(container.id, container.int_values, container.float_values, container.double_values) + status = self._native.hec_dss_arrayStore( + container.id, container.int_values, container.float_values, container.double_values) self._catalog = None elif type(container) is LocationInfo: - status = self._native.hec_dss_locationStore(container,1) + status = self._native.hec_dss_locationStore(container, 1) self._catalog = None elif type(container) is Text: text = container status = self._native.hec_dss_textStore(text.id, text.text, len(text.text)) self._catalog = None else: - raise NotImplementedError(f"unsupported record_type: {type(container)}. Expected types are: {RecordType.SUPPORTED_RECORD_TYPES.value}") + raise NotImplementedError( + f"unsupported record_type: {type(container)}. Expected types are: {RecordType.SUPPORTED_RECORD_TYPES.value}") if hasattr(container, "location_info") and container.location_info is not None: - status = self._native.hec_dss_locationStore(container.location_info,1) + status = self._native.hec_dss_locationStore(container.location_info, 1) # TODO -- instead of invalidating catalog,with _catalog=None # can we be smart? @@ -679,7 +693,6 @@ def put(self, container) -> int: return status - def writePrecompressedGrid(self, gd, compressedData, CompressionSize): """ puts pre-compressed gridded data into the DSS file @@ -723,7 +736,7 @@ def delete(self, pathname: str, allrecords: bool = False, startdatetime=None, en if enddatetime: newEndDateTime = enddatetime - interval_seconds=DateConverter.intervalString_to_sec(delete_path.E) + interval_seconds = DateConverter.intervalString_to_sec(delete_path.E) firstMinutes = DateConverter.julian_array_from_date_times([newStartDateTime])[0] firstSeconds, firstJulian = [firstMinutes % 1440 * 60], [firstMinutes // 1440] @@ -740,7 +753,7 @@ def delete(self, pathname: str, allrecords: bool = False, startdatetime=None, en ) RTS = RegularTimeSeries() - RTS.values = [DSS_UNDEFINED_VALUE]*number_periods + RTS.values = [DSS_UNDEFINED_VALUE] * number_periods RTS.times = [newStartDateTime, newEndDateTime] RTS.start_date = startdatetime RTS.id = pathname @@ -779,7 +792,8 @@ def delete(self, pathname: str, allrecords: bool = False, startdatetime=None, en if status == 0: self._catalog = None else: - print(f"Error deleting record from '{pathname}', Record does not exist or timeseries path must be uncondensed") + print( + f"Error deleting record from '{pathname}', Record does not exist or timeseries path must be uncondensed") return status def get_catalog(self) -> Catalog: From 5a468140996a1bbc0f1ac38e41e3987070fc2456 Mon Sep 17 00:00:00 2001 From: Jeremiah Kang Date: Mon, 20 Jul 2026 20:07:55 +0000 Subject: [PATCH 3/6] integrate cnotes into hec-dss-python --- src/hecdss/hecdss.py | 20 ++++- src/hecdss/irregular_timeseries.py | 22 +++-- src/hecdss/native.py | 58 ++++++++++++- src/hecdss/regular_timeseries.py | 31 +++++-- tests/test_basics.py | 63 +++++++++++--- tests/test_irregular_timeseries.py | 134 ++++++++++++++++++++++------- 6 files changed, 268 insertions(+), 60 deletions(-) diff --git a/src/hecdss/hecdss.py b/src/hecdss/hecdss.py index 0144c6a..87a3394 100644 --- a/src/hecdss/hecdss.py +++ b/src/hecdss/hecdss.py @@ -20,6 +20,7 @@ DSS_UNDEFINED_VALUE = -340282346638528859811704183484516925440.000000 +MAX_NOTE_LENGTH: int = 40 class HecDss: @@ -445,6 +446,8 @@ def _get_timeseries(self, pathname, startDateTime, endDateTime, trim): values = [] numberValuesRead = [0] quality = [] + notes = [] + noteLength = MAX_NOTE_LENGTH julianBaseDate = [0] timeGranularitySeconds = [0] units = [""] @@ -461,6 +464,9 @@ def _get_timeseries(self, pathname, startDateTime, endDateTime, trim): times, values, number_periods + 1, + None, + noteLength, + notes, numberValuesRead, quality, qualityElementSize[0], @@ -492,6 +498,7 @@ def _get_timeseries(self, pathname, startDateTime, endDateTime, trim): times = [] values = [] quality = [] + notes = [] else: start = 0 if startDateTime and not trim else trimmed_indices[0] end = len(times) if endDateTime and not trim else trimmed_indices[-1] + 1 @@ -499,6 +506,8 @@ def _get_timeseries(self, pathname, startDateTime, endDateTime, trim): values = values[start:end] if quality != []: quality = quality[start:end] + if notes != []: + notes = notes[start:end] new_times = DateConverter.date_times_from_julian_array( times, timeGranularitySeconds[0], julianBaseDate[0] ) @@ -509,6 +518,7 @@ def _get_timeseries(self, pathname, startDateTime, endDateTime, trim): arr = np.delete(arr, indices) new_times = [new_times[i] for i in range(len(new_times)) if not np.isin(i, indices)] quality = [quality[i] for i in range(len(quality)) if not np.isin(i, indices)] + notes = [notes[i] for i in range(len(notes)) if not np.isin(i, indices)] values = arr units = units[0] @@ -528,7 +538,7 @@ def _get_timeseries(self, pathname, startDateTime, endDateTime, trim): start_date = _startDateTime - timedelta(seconds=interval_seconds) location_info = self._get_location_info(pathname) - ts = ts.create(values=values, times=new_times, quality=quality, units=units, data_type=data_type, start_date=start_date, + ts = ts.create(values=values, times=new_times, quality=quality, notes=notes, units=units, data_type=data_type, start_date=start_date, time_granularity_seconds=time_granularity_seconds, julian_base_date=julian_base_date, time_zone_name=timeZoneName, path=pathname, location_info=location_info) if (DssPath(pathname).D.lower() == "ts-pattern"): @@ -614,6 +624,7 @@ def put(self, container) -> int: startDate, startTime = DateConverter.dss_datetime_strings_from_datetime(ts.times[0]) quality = container.quality + noteLength = MAX_NOTE_LENGTH status = self._native.hec_dss_tsStoreRegular( ts.id, @@ -621,6 +632,8 @@ def put(self, container) -> int: startTime, ts.values, quality, + ts.notes, + noteLength, False, ts.units, ts.data_type, @@ -637,6 +650,7 @@ def put(self, container) -> int: start_date_base = (datetime(1900, 1, 1) + timedelta(days=its.julian_base_date)) startDate, startTime = DateConverter.dss_datetime_strings_from_datetime(start_date_base) quality = container.quality + noteLength = MAX_NOTE_LENGTH julian_times = DateConverter.julian_array_from_date_times( its.times, its.time_granularity_seconds, start_date_base) if max(julian_times) >= 2147483647: @@ -649,6 +663,8 @@ def put(self, container) -> int: its.time_granularity_seconds, its.values, quality, + its.notes, + noteLength, False, its.units, its.data_type, @@ -766,6 +782,8 @@ def delete(self, pathname: str, allrecords: bool = False, startdatetime=None, en startTime, RTS.values, RTS.quality, + RTS.notes, + MAX_NOTE_LENGTH, False, RTS.units, RTS.data_type, diff --git a/src/hecdss/irregular_timeseries.py b/src/hecdss/irregular_timeseries.py index 328bc2c..d7ebdb4 100644 --- a/src/hecdss/irregular_timeseries.py +++ b/src/hecdss/irregular_timeseries.py @@ -7,6 +7,7 @@ class IrregularTimeSeries: """ container for time-series data that is not at a consistent interval. data is stored internally as a numpy array """ + def __init__(self): """ Initialize an IrregularTimeSeries object with default values. @@ -15,6 +16,7 @@ def __init__(self): self.times = [] self.values = np.empty(0) self.quality = [] + self.notes: list[str] = [] self.units = "" self.data_type = "" self.interval = 0 @@ -81,10 +83,17 @@ def print_to_console(self): Print the time-series data to the console in a readable format. """ print("dsspath='" + self.id + "'") - print("units='"+self.units+"'") + print("units='" + self.units + "'") print("dataType='" + self.data_type + "'") - for time, value in zip(self.times, self.values): - print(f"Time: {time}, Value: {value}") + has_quality = len(self.quality) > 0 + has_notes = len(self.notes) > 0 + for i, (time, value) in enumerate(zip(self.times, self.values)): + line = f"Time: {time}, Value: {value}" + if has_quality: + line += f", Flag: {self.quality[i]}" + if has_notes: + line += f", Note: {self.notes[i]}" + print(line) def to_csv(self, file_path: str, with_metadata: bool = True) -> None: """ @@ -102,7 +111,7 @@ def to_csv(self, file_path: str, with_metadata: bool = True) -> None: def read_csv(file_path: str) -> "IrregularTimeSeries": """ Reads a .csv file and creates an IrregularTimeSeries instance from the data. - + Parameters: file_path (str): The path to the .csv file to read @@ -113,7 +122,7 @@ def read_csv(file_path: str) -> "IrregularTimeSeries": return timeseries_read_csv(IrregularTimeSeries, file_path) @staticmethod - def create(values, times, quality=[], units="", data_type="", interval=0, start_date="", time_granularity_seconds=1, julian_base_date=None, time_zone_name="", path=None, location_info=None): + def create(values, times, quality=[], notes=[], units="", data_type="", interval=0, start_date="", time_granularity_seconds=1, julian_base_date=None, time_zone_name="", path=None, location_info=None): """ Retrieve the value at a specific date in the time-series. @@ -127,6 +136,7 @@ def create(values, times, quality=[], units="", data_type="", interval=0, start_ irts.times = times irts.values = np.array(values) irts.quality = quality + irts.notes = notes irts.units = units irts.data_type = data_type irts.interval = interval @@ -134,7 +144,7 @@ def create(values, times, quality=[], units="", data_type="", interval=0, start_ irts.time_granularity_seconds = time_granularity_seconds irts.julian_base_date = 0 if julian_base_date is None and len(times): - irts.julian_base_date = (times[0]-datetime(1900, 1, 1)).days + irts.julian_base_date = (times[0] - datetime(1900, 1, 1)).days irts.time_zone_name = time_zone_name irts.id = path irts.location_info = location_info diff --git a/src/hecdss/native.py b/src/hecdss/native.py index 885c4fc..7f47f03 100644 --- a/src/hecdss/native.py +++ b/src/hecdss/native.py @@ -757,6 +757,9 @@ def hec_dss_tsRetrieve( times: List[int], values: List[float], arraySize: str, + cnotesBuffer: str, + cnoteSize: int, + notes: List[str], numberValuesRead: List[int], quality: List[int], qualityLength: int, @@ -781,6 +784,8 @@ def hec_dss_tsRetrieve( POINTER(c_int), # timeArray POINTER(c_double), # valueArray c_int, # arraySize + c_char_p, # cnotesBuffer + c_int, # cnoteSize POINTER(c_int), # numberValuesRead POINTER(c_int), # quality c_int, # qualityLength @@ -801,6 +806,8 @@ def hec_dss_tsRetrieve( c_numberValuesRead = c_int(0) size = qualityLength * arraySize + c_cnotesBuffer = create_string_buffer(arraySize * cnoteSize) + c_quality = (c_int * size)() c_julianBaseDate = c_int(0) c_timeGranularitySeconds = c_int(0) @@ -827,6 +834,8 @@ def hec_dss_tsRetrieve( c_times, c_values, c_arraySize, + c_cnotesBuffer, + cnoteSize, byref(c_numberValuesRead), c_quality, qualityLength, @@ -846,6 +855,11 @@ def hec_dss_tsRetrieve( values.extend(list(c_values[: c_numberValuesRead.value])) quality.clear() quality.extend(list(c_quality[: c_numberValuesRead.value])) + notes.clear() + raw = c_cnotesBuffer.raw + for i in range(c_numberValuesRead.value): + chunk = raw[i * cnoteSize: (i + 1) * cnoteSize] + notes.append(chunk.split(b"\x00", 1)[0].decode("utf-8")) units[0] = c_units.value.decode("utf-8") dataType[0] = c_dataType.value.decode("utf-8") julianBaseDate[0] = c_julianBaseDate.value @@ -861,6 +875,8 @@ def hec_dss_tsStoreRegular( startTime, valueArray, qualityArray, + notes, + cnoteSize, saveAsFloat, units, dataType, @@ -878,6 +894,8 @@ def hec_dss_tsStoreRegular( c_int, # valueArraySize (int) POINTER(c_int), # qualityArray (int*) c_int, # qualityArraySize (int) + c_char_p, # cnotesBuffer + c_int, # cnoteSize c_int, # saveAsFloat (int) c_char_p, # units (const char*) c_char_p, # type (const char*) @@ -895,6 +913,19 @@ def hec_dss_tsStoreRegular( c_valueArray = (c_double * len(valueArray))(*valueArray) c_qualityArray = (c_int * len(qualityArray))(*qualityArray) + if notes: + c_cnotesBuffer = create_string_buffer(len(valueArray) * cnoteSize) + for i, note in enumerate(notes): + # encoded defends that note doesn't break if None, helps convert to C raw bytes, and truncates to cnoteSize - 1 to reserve space for null byte \x00 + encoded = (note or "").encode("utf-8")[: cnoteSize - 1] + offset = i * cnoteSize + # write encoded bytes of encoded starting at offset + c_cnotesBuffer[offset: offset + len(encoded)] = encoded + c_cnoteSize = cnoteSize + else: + c_cnotesBuffer = None + c_cnoteSize = 0 + return self.dll.hec_dss_tsStoreRegular( self.handle, c_pathname, @@ -904,6 +935,8 @@ def hec_dss_tsStoreRegular( len(valueArray), c_qualityArray, len(qualityArray), + c_cnotesBuffer, + c_cnoteSize, int(saveAsFloat), c_units, c_type, @@ -919,6 +952,8 @@ def hec_dss_tsStoreIrregular( timeGranularitySeconds, valueArray, qualityArray, + notes, + cnoteSize, saveAsFloat, units, dataType, @@ -926,8 +961,8 @@ def hec_dss_tsStoreIrregular( storageFlag, ): - self.dll.hec_dss_tsStoreIregular.restype = c_int - self.dll.hec_dss_tsStoreIregular.argtypes = [ + self.dll.hec_dss_tsStoreIrregular.restype = c_int + self.dll.hec_dss_tsStoreIrregular.argtypes = [ c_void_p, # dss (void*) c_char_p, # pathname (const char*) c_char_p, # startDateBase (const char*) @@ -937,6 +972,8 @@ def hec_dss_tsStoreIrregular( c_int, # valueArraySize (int) POINTER(c_int), # qualityArray (int*) c_int, # qualityArraySize (int) + c_char_p, # cnotesBuffer (char*) + c_int, # cnoteSize (int) c_int, # saveAsFloat (int) c_char_p, # units (const char*) c_char_p, # type (const char*) @@ -954,7 +991,20 @@ def hec_dss_tsStoreIrregular( c_times = (c_int * len(times))(*times) c_qualityArray = (c_int * len(qualityArray))(*qualityArray) - return self.dll.hec_dss_tsStoreIregular( + if notes: + c_cnotesBuffer = create_string_buffer(len(valueArray) * cnoteSize) + for i, note in enumerate(notes): + # encoded defends that note doesn't break if None, helps convert to C raw bytes, and truncates to cnoteSize - 1 to reserve space for null byte \x00 + encoded = (note or "").encode("utf-8")[: cnoteSize - 1] + offset = i * cnoteSize + # write encoded bytes of encoded starting at offset + c_cnotesBuffer[offset: offset + len(encoded)] = encoded + c_cnoteSize = cnoteSize + else: + c_cnotesBuffer = None + c_cnoteSize = 0 + + return self.dll.hec_dss_tsStoreIrregular( self.handle, c_pathname, c_startDateBase, @@ -964,6 +1014,8 @@ def hec_dss_tsStoreIrregular( len(valueArray), c_qualityArray, len(qualityArray), + c_cnotesBuffer, + c_cnoteSize, int(saveAsFloat), c_units, c_type, diff --git a/src/hecdss/regular_timeseries.py b/src/hecdss/regular_timeseries.py index 0591612..38bbae5 100644 --- a/src/hecdss/regular_timeseries.py +++ b/src/hecdss/regular_timeseries.py @@ -15,6 +15,7 @@ def __init__(self): self.times = [] self.values = np.empty(0) self.quality = [] + self.notes: list[str] = [] self.units = "" self.data_type = "" self.interval = "" @@ -25,7 +26,7 @@ def __init__(self): self.id = "" self.location_info = None - def add_data_point(self, date, value, flag=None): + def add_data_point(self, date: datetime, value: float, flag: int = None, note: str = None): """ Adds a data point to the time series. @@ -34,10 +35,13 @@ def add_data_point(self, date, value, flag=None): value (float): The value of the data point. flag (int, optional): The quality flag of the data point. Defaults to None. """ + # TODO: Does this let you add a mismatched datetime?? self.times.append(date) self.values.append(value) if flag is not None: self.quality.append(flag) + if note is not None: + self.notes.append(note) def get_value_at(self, date): """ @@ -89,13 +93,21 @@ def print_to_console(self): print("dsspath='" + self.id + "'") print("units='" + self.units + "'") print("dataType='" + self.data_type + "'") - print("Time,Value,Flag") - if not len(self.quality) > 0: - for time, value in zip(self.times, self.values): - print(f"{time}, {value}") - else: - for time, value, flag in zip(self.times, self.values, self.quality): - print(f"{time}, {value}, {flag}") + has_quality = len(self.quality) > 0 + has_notes = len(self.notes) > 0 + header = "Time,Value" + if has_quality: + header += ",Flag" + if has_notes: + header += ",Note" + print(header) + for i, (time, value) in enumerate(zip(self.times, self.values)): + row = f"{time}, {value}" + if has_quality: + row += f", {self.quality[i]}" + if has_notes: + row += f", {self.notes[i]}" + print(row) def to_csv(self, file_path: str, with_metadata: bool = True) -> None: """ @@ -254,7 +266,7 @@ def read_csv(file_path: str) -> "RegularTimeSeries": return timeseries_read_csv(RegularTimeSeries, file_path) @staticmethod - def create(values, times=[], quality=[], units="", data_type="", interval="", start_date="", time_granularity_seconds=1, julian_base_date=0, time_zone_name="", path=None, location_info=None): + def create(values, times=[], quality=[], notes=[], units="", data_type="", interval="", start_date="", time_granularity_seconds=1, julian_base_date=0, time_zone_name="", path=None, location_info=None): """ Creates a new instance of the RegularTimeSeries class with the specified parameters. @@ -278,6 +290,7 @@ def create(values, times=[], quality=[], units="", data_type="", interval="", st rts.times = [i.replace(microsecond=0) for i in times] rts.values = np.array(values) rts.quality = quality + rts.notes = notes rts.units = units rts.data_type = data_type rts.interval = interval diff --git a/tests/test_basics.py b/tests/test_basics.py index e379f33..88fdb75 100644 --- a/tests/test_basics.py +++ b/tests/test_basics.py @@ -55,22 +55,17 @@ def test_catalog(self): for ds in catalog: print(ds.recType, ds) - def test_catalog_get(self): with HecDss(self.test_files.get_copy("sample7.dss")) as dss: catalog = dss.get_catalog() ts = dss.get(catalog.items[0]) - def test_with_block(self): with HecDss(self.test_files.get_copy("sample7.dss")) as dss: catalog = dss.get_catalog() ts = dss.get(catalog.items[0]) - - - def test_new_catalog(self): rawPaths = [ "//SACRAMENTO/TEMP-MIN/01Jan1989/1Day/OBS/", @@ -83,7 +78,6 @@ def test_new_catalog(self): c = Catalog(rawPaths, recordType) c.print() - def test_paired_data(self): """ read record from disk, add labels save to new path @@ -122,7 +116,7 @@ def test_missing_values(self): with HecDss(self.test_files.get_copy("missing_data.dss")) as dss: print("record count = " + str(dss.record_count())) tsc = dss.get("/CUMBERLAND RIVER/CUMBERLAND FALLS/FLOW//30Minute/MISSING/", datetime(2020, 1, 3, 11, 0), - datetime(2020, 1, 13, 0, 0)) + datetime(2020, 1, 13, 0, 0)) tsc.print_to_console() # values = ts.values @@ -204,13 +198,13 @@ def test_delete_range(self): t1 = datetime(1879, 1, 1) t2 = datetime(1885, 1, 1) dss.delete(path, False, t1, t2) - assert (not("//SACRAMENTO/PRECIP-INC/01Jan1882/1Day/OBS/" in dss.get_catalog().uncondensed_paths)) + assert (not ("//SACRAMENTO/PRECIP-INC/01Jan1882/1Day/OBS/" in dss.get_catalog().uncondensed_paths)) def test_Read_TS_Pattern_Regular(self): with HecDss(self.test_files.get_copy("Depth_Area_01.dss")) as dss: path = "//010010-B/FLOW-UNIT GRAPH/TS-Pattern/5Minute/DAA:Depth-Area 01>010005-C/" tsc = dss.get(path) - assert(len(tsc.values) > 0) + assert (len(tsc.values) > 0) def test_Write_TS_Pattern_Regular(self): with HecDss(self.test_files.get_copy("Depth_Area_01.dss")) as dss: @@ -221,7 +215,55 @@ def test_Write_TS_Pattern_Regular(self): tsc.start_date = tsc.start_date.replace(year=2023) dss.put(tsc) tsc2 = dss.get(tsc.id) - assert(len(tsc2.values) > 0) + assert (len(tsc2.values) > 0) + + def test_write_regular_timeseries_with_notes(self): + with HecDss(self.test_files.get_copy("sample7.dss")) as dss: + t1 = datetime(2005, 1, 1) + t2 = datetime(2005, 1, 4) + tsc = dss.get("//SACRAMENTO/PRECIP-INC//1Day/OBS/", t1, t2) + tsc.id = "//SACRAMENTO/PRECIP-INC/01Jan2005/1Day/OBS-notes/" + tsc.notes = ["" for _ in tsc.values] + tsc.notes[1] = "manual override" + + dss.put(tsc) + tsc2 = dss.get(tsc.id, t1, t2) + + self.assertEqual(tsc.notes, tsc2.notes) + + def test_write_regular_timeseries_with_quality(self): + """ + Quality flags written on a RegularTimeSeries survive a put/get round trip. + """ + with HecDss(self.test_files.get_copy("sample7.dss")) as dss: + t1 = datetime(2005, 1, 1) + t2 = datetime(2005, 1, 4) + tsc = dss.get("//SACRAMENTO/PRECIP-INC//1Day/OBS/", t1, t2) + tsc.id = "//SACRAMENTO/PRECIP-INC/01Jan2005/1Day/OBS-quality/" + tsc.quality = [0, 1, 2, 3][:len(tsc.values)] + + dss.put(tsc) + tsc2 = dss.get(tsc.id, t1, t2) + + self.assertEqual(tsc.quality, tsc2.quality) + + def test_write_regular_timeseries_with_quality_and_notes(self): + """ + Quality flags and notes written together on a RegularTimeSeries survive a put/get round trip. + """ + with HecDss(self.test_files.get_copy("sample7.dss")) as dss: + t1 = datetime(2005, 1, 1) + t2 = datetime(2005, 1, 4) + tsc = dss.get("//SACRAMENTO/PRECIP-INC//1Day/OBS/", t1, t2) + tsc.id = "//SACRAMENTO/PRECIP-INC/01Jan2005/1Day/OBS-quality-notes/" + tsc.quality = [0, 1, 2, 3][:len(tsc.values)] + tsc.notes = ["", "manual override", "", "estimated"][:len(tsc.values)] + + dss.put(tsc) + tsc2 = dss.get(tsc.id, t1, t2) + + self.assertEqual(tsc.quality, tsc2.quality) + self.assertEqual(tsc.notes, tsc2.notes) def test_path_empty_parts(self): with HecDss(self.test_files.get_copy("Depth_Area_01.dss")) as dss: @@ -229,6 +271,7 @@ def test_path_empty_parts(self): tsc = dss.get(path) assert (len(tsc.values) > 0) + if __name__ == "__main__": unittest.main() # test_catalog() diff --git a/tests/test_irregular_timeseries.py b/tests/test_irregular_timeseries.py index bb4aede..541b688 100644 --- a/tests/test_irregular_timeseries.py +++ b/tests/test_irregular_timeseries.py @@ -16,7 +16,7 @@ class TestRegularTimeSeries(unittest.TestCase): def setUp(self) -> None: self.test_files = FileManager() - + def tearDown(self) -> None: self.test_files.cleanup() @@ -26,12 +26,15 @@ def test_irregular_timeseries_read(self): """ path = "/irregular-time-series/FAIR OAKS CA/FLOW-ANNUAL PEAK/01Jan1900/IR-Century/USGS/" with HecDss(self.test_files.get_copy("examples-all-data-types.dss")) as dss: - irts = dss.get(path) - assert (113 == irts.get_length()), f"irts.get_length() should be 113. is {irts.get_length()}" + irts = dss.get(path) + assert (113 == irts.get_length() + ), f"irts.get_length() should be 113. is {irts.get_length()}" assert (39000 == irts.values[18]), f"irts.values[18] should be 39000. is {irts.values[18]}" - assert ("1924-02-09 00:00:00" == str(irts.times[19])), f"irts.times[19] should be '1924-02-09 00:00:00'. is {irts.times[19]}" + assert ("1924-02-09 00:00:00" + == str(irts.times[19])), f"irts.times[19] should be '1924-02-09 00:00:00'. is {irts.times[19]}" assert ("CFS" == irts.units), f"irts.units should be 'CFS'. is {irts.units}" - assert ("INST-VAL" == irts.data_type), f"irts.data_type should be 'INST-VAL'. is {irts.data_type}" + assert ("INST-VAL" + == irts.data_type), f"irts.data_type should be 'INST-VAL'. is {irts.data_type}" assert (0 == irts.interval), f"irts.interval should be 0. is {irts.interval}" def test_is_irregular_timeseries_type(self): @@ -41,7 +44,8 @@ def test_is_irregular_timeseries_type(self): path = "/irregular-time-series/FAIR OAKS CA/FLOW-ANNUAL PEAK/01Jan1900/IR-Century/USGS/" with HecDss(self.test_files.get_copy("examples-all-data-types.dss")) as dss: irts = dss.get(path) - assert (type(irts) is IrregularTimeSeries), f"irts should be type IrregularTimeSeries. is {type(irts)}" + assert ( + type(irts) is IrregularTimeSeries), f"irts should be type IrregularTimeSeries. is {type(irts)}" def test_irregular_timeseries_create_store(self): """ @@ -52,13 +56,14 @@ def test_irregular_timeseries_create_store(self): irpath = "/irregular-time-series/GAPT/FLOW//IR-Day/forecast6/" irts = IrregularTimeSeries() - dates = [datetime.today().replace(second=0, microsecond=0)+(i * timedelta(hours=2)) for i in range(15)] + dates = [datetime.today().replace(second=0, microsecond=0) + + (i * timedelta(hours=2)) for i in range(15)] dates[1] = dates[1] - timedelta(seconds=60) - irts = IrregularTimeSeries.create(times=dates, values=list(range(15)), data_type="INST-VAL", path=irpath) + irts = IrregularTimeSeries.create(times=dates, values=list( + range(15)), data_type="INST-VAL", path=irpath) dss.put(irts) - def test_irregular_timeseries_create_store_read(self): """ Generates a IrregularTimeSeries object then stores data on disk and read result @@ -67,17 +72,84 @@ def test_irregular_timeseries_create_store_read(self): with HecDss(self.test_files.get_copy(file)) as dss: irpath = "/irregular-time-series/GAPT/FLOW//IR-Day/forecast6/" - dates = [datetime.today().replace(microsecond=0) + (i * timedelta(hours=2)) for i in range(15)] + dates = [datetime.today().replace(microsecond=0) + (i * timedelta(hours=2)) + for i in range(15)] dates[1] = dates[1] - timedelta(seconds=60) - irts = IrregularTimeSeries.create(times=dates, values=list(range(15)), data_type="INST-VAL", path=irpath) + irts = IrregularTimeSeries.create(times=dates, values=list( + range(15)), data_type="INST-VAL", path=irpath) dss.put(irts) read_irts = dss.get(irpath) - assert (read_irts.times == irts.times), f"saved and read times should be identical saved times are" \ - f" \n{irts.times}\n and read times are \n{read_irts.times}" + f" \n{irts.times}\n and read times are \n{read_irts.times}" + + def test_irregular_timeseries_create_store_read_with_notes(self): + """ + Notes written on an IrregularTimeSeries survive a put/get round trip. + """ + file = "examples-all-data-types.dss" + with HecDss(self.test_files.get_copy(file)) as dss: + + irpath = "/irregular-time-series/GAPT/FLOW//IR-Day/forecast7/" + dates = [datetime.today().replace(microsecond=0) + (i * timedelta(hours=2)) + for i in range(5)] + notes = ["", "storm event", "", "gage malfunction", ""] + irts = IrregularTimeSeries.create(times=dates, values=list(range(5)), data_type="INST-VAL", + path=irpath, notes=notes) + + dss.put(irts) + + read_irts = dss.get(irpath) + + assert (read_irts.notes == notes), f"saved and read notes should be identical. saved notes are" \ + f" \n{notes}\n and read notes are \n{read_irts.notes}" + + def test_irregular_timeseries_create_store_read_with_quality(self): + """ + Quality flags written on an IrregularTimeSeries survive a put/get round trip. + """ + file = "examples-all-data-types.dss" + with HecDss(self.test_files.get_copy(file)) as dss: + + irpath = "/irregular-time-series/GAPT/FLOW//IR-Day/forecast8/" + dates = [datetime.today().replace(microsecond=0) + (i * timedelta(hours=2)) + for i in range(5)] + quality = [0, 1, 2, 3, 4] + irts = IrregularTimeSeries.create(times=dates, values=list(range(5)), data_type="INST-VAL", + path=irpath, quality=quality) + + dss.put(irts) + + read_irts = dss.get(irpath) + + assert (read_irts.quality == quality), f"saved and read quality should be identical. saved quality is" \ + f" \n{quality}\n and read quality is \n{read_irts.quality}" + + def test_irregular_timeseries_create_store_read_with_quality_and_notes(self): + """ + Quality flags and notes written together on an IrregularTimeSeries survive a put/get round trip. + """ + file = "examples-all-data-types.dss" + with HecDss(self.test_files.get_copy(file)) as dss: + + irpath = "/irregular-time-series/GAPT/FLOW//IR-Day/forecast9/" + dates = [datetime.today().replace(microsecond=0) + (i * timedelta(hours=2)) + for i in range(5)] + quality = [0, 1, 2, 3, 4] + notes = ["", "storm event", "", "gage malfunction", ""] + irts = IrregularTimeSeries.create(times=dates, values=list(range(5)), data_type="INST-VAL", + path=irpath, quality=quality, notes=notes) + + dss.put(irts) + + read_irts = dss.get(irpath) + + assert (read_irts.quality == quality), f"saved and read quality should be identical. saved quality is" \ + f" \n{quality}\n and read quality is \n{read_irts.quality}" + assert (read_irts.notes == notes), f"saved and read notes should be identical. saved notes are" \ + f" \n{notes}\n and read notes are \n{read_irts.notes}" def test_irregular_timeseries_read_store_read(self): """ @@ -96,17 +168,17 @@ def test_irregular_timeseries_read_store_read(self): irts_modified = dss.get(path_modified) np.set_printoptions(suppress=True) assert (irts.get_length() == irts_modified.get_length()), f"irts.get_length() is not equal to irts_modified.get_length()." \ - f" irts.get_length() is {irts.get_length()}, irts_modified.get_length() is {irts_modified.get_length()}" + f" irts.get_length() is {irts.get_length()}, irts_modified.get_length() is {irts_modified.get_length()}" assert (np.array_equal(irts.values, irts_modified.values)), f"irts.values is not equal to irts_modified.values." \ - f" irts.values is {irts.values}, irts_modified.values is {irts_modified.values}" + f" irts.values is {irts.values}, irts_modified.values is {irts_modified.values}" assert (np.array_equal(irts.times, irts_modified.times)), f"irts.times is not equal to irts_modified.times." \ - f" irts.times is {irts.times}, irts_modified.times is {irts_modified.times}" + f" irts.times is {irts.times}, irts_modified.times is {irts_modified.times}" assert (irts.units == irts_modified.units), f"irts.units is not equal to irts_modified.units." \ - f" irts.units is {irts.units}, irts_modified.units is {irts_modified.units}" + f" irts.units is {irts.units}, irts_modified.units is {irts_modified.units}" assert (irts.data_type == irts_modified.data_type), f"irts.data_type is not equal to irts_modified.data_type." \ - f" irts.data_type is {irts.data_type}, irts_modified.data_type is {irts_modified.data_type}" + f" irts.data_type is {irts.data_type}, irts_modified.data_type is {irts_modified.data_type}" assert (irts.interval == irts_modified.interval), f"irts.interval is not equal to irts_modified.interval." \ - f" irts.interval is {irts.interval}, irts_modified.interval is {irts_modified.interval}" + f" irts.interval is {irts.interval}, irts_modified.interval is {irts_modified.interval}" def test_irregular_timeseries_read_modify_store_modify_read(self): """ @@ -129,17 +201,17 @@ def test_irregular_timeseries_read_modify_store_modify_read(self): irts_modified = dss.get(path_modified) np.set_printoptions(suppress=True) assert (irts.get_length() == irts_modified.get_length()), f"irts.get_length() is not equal to irts_modified.get_length()." \ - f" irts.get_length() is {irts.get_length()}, irts_modified.get_length() is {irts_modified.get_length()}" + f" irts.get_length() is {irts.get_length()}, irts_modified.get_length() is {irts_modified.get_length()}" assert (np.array_equal(irts.values, irts_modified.values)), f"irts.values is not equal to irts_modified.values." \ - f" irts.values is {irts.values}, irts_modified.values is {irts_modified.values}" + f" irts.values is {irts.values}, irts_modified.values is {irts_modified.values}" assert (np.array_equal(irts.times, irts_modified.times)), f"irts.times is not equal to irts_modified.times." \ - f" irts.times is {irts.times}, irts_modified.times is {irts_modified.times}" + f" irts.times is {irts.times}, irts_modified.times is {irts_modified.times}" assert (irts.units == irts_modified.units), f"irts.units is not equal to irts_modified.units." \ - f" irts.units is {irts.units}, irts_modified.units is {irts_modified.units}" + f" irts.units is {irts.units}, irts_modified.units is {irts_modified.units}" assert (irts.data_type == irts_modified.data_type), f"irts.data_type is not equal to irts_modified.data_type." \ - f" irts.data_type is {irts.data_type}, irts_modified.data_type is {irts_modified.data_type}" + f" irts.data_type is {irts.data_type}, irts_modified.data_type is {irts_modified.data_type}" assert (irts.interval == irts_modified.interval), f"irts.interval is not equal to irts_modified.interval." \ - f" irts.interval is {irts.interval}, irts_modified.interval is {irts_modified.interval}" + f" irts.interval is {irts.interval}, irts_modified.interval is {irts_modified.interval}" def test_irregular_timeseries_read_modify_store_read(self): """ @@ -156,17 +228,17 @@ def test_irregular_timeseries_read_modify_store_read(self): irts_modified = dss.get(path) np.set_printoptions(suppress=True) assert (irts.get_length() == irts_modified.get_length()), f"irts.get_length() is not equal to irts_modified.get_length()." \ - f" irts.get_length() is {irts.get_length()}, irts_modified.get_length() is {irts_modified.get_length()}" + f" irts.get_length() is {irts.get_length()}, irts_modified.get_length() is {irts_modified.get_length()}" assert (np.array_equal(irts.values, irts_modified.values)), f"irts.values is not equal to irts_modified.values." \ - f" irts.values is {irts.values}, irts_modified.values is {irts_modified.values}" + f" irts.values is {irts.values}, irts_modified.values is {irts_modified.values}" assert (np.array_equal(irts.times, irts_modified.times)), f"irts.times is not equal to irts_modified.times." \ - f" irts.times is {irts.times}, irts_modified.times is {irts_modified.times}" + f" irts.times is {irts.times}, irts_modified.times is {irts_modified.times}" assert (irts.units == irts_modified.units), f"irts.units is not equal to irts_modified.units." \ - f" irts.units is {irts.units}, irts_modified.units is {irts_modified.units}" + f" irts.units is {irts.units}, irts_modified.units is {irts_modified.units}" assert (irts.data_type == irts_modified.data_type), f"irts.data_type is not equal to irts_modified.data_type." \ - f" irts.data_type is {irts.data_type}, irts_modified.data_type is {irts_modified.data_type}" + f" irts.data_type is {irts.data_type}, irts_modified.data_type is {irts_modified.data_type}" assert (irts.interval == irts_modified.interval), f"irts.interval is not equal to irts_modified.interval." \ - f" irts.interval is {irts.interval}, irts_modified.interval is {irts_modified.interval}" + f" irts.interval is {irts.interval}, irts_modified.interval is {irts_modified.interval}" if __name__ == "__main__": From c7120ceb68b30d010ba60d53259a8d94708f4374 Mon Sep 17 00:00:00 2001 From: Jeremiah Kang Date: Mon, 20 Jul 2026 21:20:31 +0000 Subject: [PATCH 4/6] implemented cnotes into csv functions --- src/hecdss/dss_csv.py | 68 +++--- src/hecdss/irregular_timeseries.py | 6 +- src/hecdss/native.py | 8 +- tests/test_ts_csv.py | 352 ++++++++++++++++++++++++++--- 4 files changed, 378 insertions(+), 56 deletions(-) diff --git a/src/hecdss/dss_csv.py b/src/hecdss/dss_csv.py index 72ef558..6386268 100644 --- a/src/hecdss/dss_csv.py +++ b/src/hecdss/dss_csv.py @@ -39,26 +39,37 @@ def timeseries_to_csv( continue writer.writerow([letter, "", "", value]) # Writing metadata rows writer.writerow(["Units", "", "", series.units]) - if len(series.quality) > 0: - # Write column names with quality - writer.writerow(["Type", "Date/Time", series.data_type, "Quality"]) - else: - # Write column names without quality - writer.writerow(["Type", "Date/Time", series.data_type]) + + header: list[str] = ["Type", "Date/Time", series.data_type] + + has_quality: bool = len(series.quality) > 0 + has_notes: bool = len(series.notes) > 0 + + if has_quality: + header.append("Quality") + if has_notes: + header.append("Notes") + + writer.writerow(header) time_format: str = ("%d%b%Y %H%M%S" if _needs_second_precision(series) else "%d%b%Y %H%M") - ordinate: int = 1 - if len(series.quality) > 0: - for time, value, quality in zip(series.times, series.values, series.quality): - formatted_time: str = time.strftime(time_format) - writer.writerow([ordinate, formatted_time, value, quality]) - ordinate += 1 - else: - for time, value in zip(series.times, series.values): - formatted_time: str = time.strftime(time_format) - writer.writerow([ordinate, formatted_time, value]) - ordinate += 1 + for i, (time, value) in enumerate(zip(series.times, series.values)): + ordinate: str = str(i + 1) + formatted_time: str = time.strftime(time_format) + row: list[str] = [ordinate, formatted_time, value] + + if has_quality: + try: + row.append(series.quality[i]) + except IndexError: + row.append(DEFAULT_MISSING_VALUE) + if has_notes: + try: + row.append(series.notes[i]) + except IndexError: + row.append(DEFAULT_MISSING_VALUE) + writer.writerow(row) def timeseries_read_csv(cls: type[RegularTimeSeries] | type[IrregularTimeSeries], path: str) -> RegularTimeSeries | IrregularTimeSeries: @@ -75,9 +86,9 @@ def timeseries_read_csv(cls: type[RegularTimeSeries] | type[IrregularTimeSeries] if cls not in (RegularTimeSeries, IrregularTimeSeries): raise TypeError("cls must be RegularTimeSeries or IrregularTimeSeries") - times, values, quality, units, data_type = [], [], [], "", "" + times, values, quality, notes, units, data_type = [], [], [], [], "", "" path_parts: dict[str, str] = _empty_path_parts() - has_quality: bool = False # flags + column_index: dict[str, int] = {} with open(path, "r", newline="", encoding="utf-8") as f: reader = csv.reader(f) @@ -94,21 +105,19 @@ def timeseries_read_csv(cls: type[RegularTimeSeries] | type[IrregularTimeSeries] elif first_column_item == "Type": # reached the header if len(row) >= 3: # ['Type', 'Date/Time', data_type, ...potentially more] data_type = row[2].strip() - # ['Type', 'Date/Time', data_type, 'Quality', ...potentially more] - if len(row) >= 4 and row[3].strip() == "Quality": - has_quality = True + # ['Type', 'Date/Time', data_type, 'Quality', 'Notes'] + column_index = { + name.strip(): i for i, name in enumerate(row) if i >= 3} else: # Data row if len(row) < 3: continue # csv is malformed, something is missing raw_time: str = row[1].strip() - # Time format is determined by length of raw_time time_format: str = _get_time_format(raw_time) if time_format is None: # Time format is unrecognized continue - # Do we need to roll over the day date? Yes if time is 2400 roll_day: bool = _need_roll_day(time_format, raw_time) if roll_day: # 2400 isn't a valid hour, so roll it to 0000 before parsing and add a day after raw_time = raw_time.replace(" 2400", " 0000") @@ -129,13 +138,19 @@ def timeseries_read_csv(cls: type[RegularTimeSeries] | type[IrregularTimeSeries] times.append(time) values.append(value) - if has_quality: # Always keep quality index-aligned with values, defaulting a missing cell to 0 - quality_str: str = row[3].strip() if len(row) >= 4 else "" + + quality_idx = column_index.get("Quality") + if quality_idx is not None: + quality_str = row[quality_idx].strip() if len(row) > quality_idx else "" try: quality.append(int(quality_str) if quality_str else 0) except ValueError: quality.append(0) + notes_idx = column_index.get("Notes") + if notes_idx is not None: + notes.append(row[notes_idx].strip() if len(row) > notes_idx else "") + id_path: str = _path_parts_to_id(path_parts) interval: str | int = path_parts["E"] @@ -143,6 +158,7 @@ def timeseries_read_csv(cls: type[RegularTimeSeries] | type[IrregularTimeSeries] values=values, times=times, quality=quality, + notes=notes, units=units, data_type=data_type, interval=interval, diff --git a/src/hecdss/irregular_timeseries.py b/src/hecdss/irregular_timeseries.py index d7ebdb4..d660c9d 100644 --- a/src/hecdss/irregular_timeseries.py +++ b/src/hecdss/irregular_timeseries.py @@ -27,13 +27,17 @@ def __init__(self): self.id = "" self.location_info = None - def add_data_point(self, date, value): + def add_data_point(self, date, value, flag=None, note=None): """ append a date,value to this time-series """ self.times.append(date) self.values.append(value) + if flag is not None: + self.quality.append(flag) + if note is not None: + self.notes.append(note) def get_value_at(self, date): """ diff --git a/src/hecdss/native.py b/src/hecdss/native.py index 7f47f03..0f11052 100644 --- a/src/hecdss/native.py +++ b/src/hecdss/native.py @@ -915,7 +915,9 @@ def hec_dss_tsStoreRegular( if notes: c_cnotesBuffer = create_string_buffer(len(valueArray) * cnoteSize) - for i, note in enumerate(notes): + # notes[:len(valueArray)] guards against notes being longer than + # valueArray, which would otherwise write past the end of the buffer + for i, note in enumerate(notes[:len(valueArray)]): # encoded defends that note doesn't break if None, helps convert to C raw bytes, and truncates to cnoteSize - 1 to reserve space for null byte \x00 encoded = (note or "").encode("utf-8")[: cnoteSize - 1] offset = i * cnoteSize @@ -993,7 +995,9 @@ def hec_dss_tsStoreIrregular( if notes: c_cnotesBuffer = create_string_buffer(len(valueArray) * cnoteSize) - for i, note in enumerate(notes): + # notes[:len(valueArray)] guards against notes being longer than + # valueArray, which would otherwise write past the end of the buffer + for i, note in enumerate(notes[:len(valueArray)]): # encoded defends that note doesn't break if None, helps convert to C raw bytes, and truncates to cnoteSize - 1 to reserve space for null byte \x00 encoded = (note or "").encode("utf-8")[: cnoteSize - 1] offset = i * cnoteSize diff --git a/tests/test_ts_csv.py b/tests/test_ts_csv.py index 29886d5..f0d9035 100644 --- a/tests/test_ts_csv.py +++ b/tests/test_ts_csv.py @@ -3,7 +3,6 @@ from unittest.mock import mock_open, patch import numpy as np - from file_manager import FileManager from hecdss import HecDss @@ -587,32 +586,6 @@ def test_to_csv_writes_empty_cell_for_missing_value(self): self.assertIn("2,01Sep2021 1200,\r\n", written) self.assertNotIn("None", written) - def test_to_csv_quality_shorter_than_values_truncates(self): - """FOOTGUN: to_csv takes the with-quality branch whenever quality is - non-empty, then zips (times, values, quality). A quality list shorter than - values makes zip stop at the shortest input, so trailing data points are - silently dropped from the CSV.""" - rts = RegularTimeSeries.create( - values=[1.0, 2.0, 3.0], - times=[ - datetime(2021, 9, 1, 6, 0), - datetime(2021, 9, 1, 12, 0), - datetime(2021, 9, 1, 18, 0), - ], - quality=[0], # only one flag for three values - units="CFS", - data_type="INST-VAL", - path="/A/B/C//6Hour/F/", - ) - mock_file = mock_open() - with patch("builtins.open", mock_file): - rts.to_csv("fake.csv", with_metadata=False) - handle = mock_file() - written = "".join(call.args[0] for call in handle.write.call_args_list) - self.assertIn("1,01Sep2021 0600,1.0,0", written) - self.assertNotIn("2,01Sep2021 1200", written) # silently dropped - self.assertNotIn("3,01Sep2021 1800", written) # silently dropped - def test_read_csv_skips_short_data_row(self): """A data row with fewer than 3 columns is malformed and skipped without raising (parity with the paired-data reader's short-row handling).""" @@ -662,6 +635,331 @@ def test_round_trip_irregular(self): self.assertEqual(result.data_type, "INST-VAL") self.assertEqual(result.id, "/A/B/C//IR-Year/F/") + # ================================================================== # + # NOTES TESTS + # ================================================================== # + + def test_to_csv_with_notes(self): + """When notes are present (no quality), header gets a 'Notes' col + and rows get the note text.""" + rts = RegularTimeSeries.create( + values=[1.0, 2.0], + times=[datetime(2021, 9, 1, 6, 0), datetime(2021, 9, 1, 12, 0)], + notes=["", "manual override"], + units="CFS", + data_type="INST-VAL", + path="/A/B/C//6Hour/F/", + ) + mock_file = mock_open() + with patch("builtins.open", mock_file): + rts.to_csv("fake.csv", with_metadata=True) + handle = mock_file() + written = "".join(call.args[0] for call in handle.write.call_args_list) + self.assertIn("Type,Date/Time,INST-VAL,Notes", written) + self.assertIn("1,01Sep2021 0600,1.0,", written) + self.assertIn("2,01Sep2021 1200,2.0,manual override", written) + + def test_read_csv_with_notes(self): + """A CSV with a Notes column (no Quality) populates rts.notes, + index-aligned with values.""" + content = ( + "Type,Date/Time,INST-VAL,Notes\n" + "1,01Sep2021 0600,10.5,\n" + "2,01Sep2021 1200,20.0,manual override\n" + ) + rts = self.read_rts_from_string(content) + self.assertEqual(rts.values.tolist(), [10.5, 20.0]) + self.assertEqual(rts.notes, ["", "manual override"]) + + def test_round_trip_with_notes(self): + """Full write->read round trip for RegularTimeSeries notes on a real + temp file.""" + path = self.test_files.create_test_file(".csv") + rts = RegularTimeSeries.create( + values=[1.0, 2.0, 3.0], + times=[ + datetime(2021, 9, 1, 6, 0), + datetime(2021, 9, 1, 12, 0), + datetime(2021, 9, 1, 18, 0), + ], + notes=["", "storm event", ""], + units="CFS", + data_type="INST-VAL", + path="/A/B/C//6Hour/F/", + ) + rts.to_csv(path, with_metadata=True) + result = RegularTimeSeries.read_csv(path) + + self.assertEqual(result.values.tolist(), [1.0, 2.0, 3.0]) + self.assertEqual(result.notes, ["", "storm event", ""]) + + def test_to_csv_with_notes_without_metadata(self): + """Like Quality, the Notes column is driven purely by len(series.notes), + independent of with_metadata -- it must still appear when + with_metadata=False.""" + rts = RegularTimeSeries.create( + values=[1.0, 2.0], + times=[datetime(2021, 9, 1, 6, 0), datetime(2021, 9, 1, 12, 0)], + notes=["", "manual override"], + units="CFS", + data_type="INST-VAL", + path="/A/B/C//6Hour/F/", + ) + mock_file = mock_open() + with patch("builtins.open", mock_file): + rts.to_csv("fake.csv", with_metadata=False) + handle = mock_file() + written = "".join(call.args[0] for call in handle.write.call_args_list) + self.assertNotIn("Units", written) + self.assertIn("Type,Date/Time,INST-VAL,Notes", written) + self.assertIn("2,01Sep2021 1200,2.0,manual override", written) + + def test_round_trip_with_notes_without_metadata(self): + """Without metadata, units/data_type come back empty (existing + behavior) but notes still survive the round trip.""" + path = self.test_files.create_test_file(".csv") + rts = RegularTimeSeries.create( + values=[1.0, 2.0], + times=[datetime(2021, 9, 1, 6, 0), datetime(2021, 9, 1, 12, 0)], + notes=["", "manual override"], + units="CFS", + data_type="INST-VAL", + path="/A/B/C//6Hour/F/", + ) + rts.to_csv(path, with_metadata=False) + result = RegularTimeSeries.read_csv(path) + + self.assertEqual(result.units, "") + self.assertEqual(result.values.tolist(), [1.0, 2.0]) + self.assertEqual(result.notes, ["", "manual override"]) + + def test_to_csv_with_quality_and_notes(self): + """When both quality and notes are present, the header/rows carry + both columns with Quality before Notes.""" + rts = RegularTimeSeries.create( + values=[1.0, 2.0], + times=[datetime(2021, 9, 1, 6, 0), datetime(2021, 9, 1, 12, 0)], + quality=[0, 5], + notes=["", "manual override"], + units="CFS", + data_type="INST-VAL", + path="/A/B/C//6Hour/F/", + ) + mock_file = mock_open() + with patch("builtins.open", mock_file): + rts.to_csv("fake.csv", with_metadata=True) + handle = mock_file() + written = "".join(call.args[0] for call in handle.write.call_args_list) + self.assertIn("Type,Date/Time,INST-VAL,Quality,Notes", written) + self.assertIn("1,01Sep2021 0600,1.0,0,", written) + self.assertIn("2,01Sep2021 1200,2.0,5,manual override", written) + + def test_round_trip_with_quality_and_notes(self): + """Quality and notes both survive a real write->read round trip + together, independently of each other.""" + path = self.test_files.create_test_file(".csv") + rts = RegularTimeSeries.create( + values=[1.0, 2.0, 3.0], + times=[ + datetime(2021, 9, 1, 6, 0), + datetime(2021, 9, 1, 12, 0), + datetime(2021, 9, 1, 18, 0), + ], + quality=[0, 5, 10], + notes=["", "storm event", ""], + units="CFS", + data_type="INST-VAL", + path="/A/B/C//6Hour/F/", + ) + rts.to_csv(path, with_metadata=True) + result = RegularTimeSeries.read_csv(path) + + self.assertEqual(result.values.tolist(), [1.0, 2.0, 3.0]) + self.assertEqual(result.quality, [0, 5, 10]) + self.assertEqual(result.notes, ["", "storm event", ""]) + + def test_to_csv_with_notes_but_no_quality_omits_quality_column(self): + """When notes are present but quality is empty, only the Notes + column should appear -- no stray 'Quality' header or column.""" + rts = RegularTimeSeries.create( + values=[1.0], + times=[datetime(2021, 9, 1, 6, 0)], + notes=["manual override"], + units="CFS", + data_type="INST-VAL", + path="/A/B/C//6Hour/F/", + ) + mock_file = mock_open() + with patch("builtins.open", mock_file): + rts.to_csv("fake.csv", with_metadata=False) + handle = mock_file() + written = "".join(call.args[0] for call in handle.write.call_args_list) + self.assertNotIn("Quality", written) + self.assertIn("Type,Date/Time,INST-VAL,Notes", written) + self.assertIn("1,01Sep2021 0600,1.0,manual override", written) + + def test_read_csv_notes_and_quality_together(self): + """Hand-typed CSV with both Quality and Notes columns present + parses both independently and correctly, in the assumed + Quality-then-Notes column order.""" + content = ( + "Type,Date/Time,INST-VAL,Quality,Notes\n" + "1,05Nov2004 0200,8,0,\n" + "2,05Nov2004 0300,9,1,estimated\n" + ) + rts = self.read_rts_from_string(content) + self.assertEqual(rts.values.tolist(), [8, 9]) + self.assertEqual(rts.quality, [0, 1]) + self.assertEqual(rts.notes, ["", "estimated"]) + + def test_read_csv_with_partial_notes(self): + """A data row missing its trailing Notes cell (even though the header + declares a Notes column) should default that entry to an empty + string rather than raising -- parity with how a missing Quality + cell already defaults to 0 in test_read_csv_with_partial_quality.""" + content = ( + "Type,Date/Time,INST-VAL,Notes\n" + "1,05Nov2004 0200,8,manual override\n" + "2,05Nov2004 0300,9\n" # missing notes cell! + "3,05Nov2004 0400,10,estimated\n" + ) + rts = self.read_rts_from_string(content) + self.assertEqual(rts.values.tolist(), [8, 9, 10]) + self.assertEqual(rts.notes[0], "manual override") + self.assertEqual(rts.notes[1], "") + self.assertEqual(rts.notes[2], "estimated") + + def test_read_csv_notes_containing_comma_round_trips(self): + """A note containing a comma is CSV-quoted on write and comes back + intact on read (parity with PairedData's quoted-label test).""" + path = self.test_files.create_test_file(".csv") + rts = RegularTimeSeries.create( + values=[1.0], + times=[datetime(2021, 9, 1, 6, 0)], + notes=["flow, cfs, estimated"], + units="CFS", + data_type="INST-VAL", + path="/A/B/C//6Hour/F/", + ) + rts.to_csv(path, with_metadata=True) + result = RegularTimeSeries.read_csv(path) + self.assertEqual(result.notes, ["flow, cfs, estimated"]) + + def test_to_csv_writes_empty_cell_for_empty_note(self): + """An empty-string note writes as an empty cell -- never the literal + text 'None' -- even though other entries have real note text.""" + rts = RegularTimeSeries.create( + values=[1.0, 2.0], + times=[datetime(2021, 9, 1, 6, 0), datetime(2021, 9, 1, 12, 0)], + notes=["", "manual override"], + units="CFS", + data_type="INST-VAL", + path="/A/B/C//6Hour/F/", + ) + mock_file = mock_open() + with patch("builtins.open", mock_file): + rts.to_csv("fake.csv", with_metadata=False) + handle = mock_file() + written = "".join(call.args[0] for call in handle.write.call_args_list) + self.assertIn("1,01Sep2021 0600,1.0,\r\n", written) + self.assertNotIn("None", written) + + # ---- IRREGULAR TIME SERIES NOTES TESTS ---- # + + def test_irregular_to_csv_with_notes(self): + """Basic structure test for IrregularTimeSeries to_csv with notes.""" + its = IrregularTimeSeries.create( + values=[10.5, 20.0, 42.0], + times=[ + datetime(2021, 9, 1, 0, 0), + datetime(2021, 9, 2, 0, 0), + datetime(2021, 9, 4, 0, 0), + ], + notes=["", "storm event", ""], + units="CFS", + data_type="INST-VAL", + path="/A/B/C//E/F/", + ) + mock_file = mock_open() + with patch("builtins.open", mock_file): + its.to_csv("fake_path.csv", with_metadata=True) + handle = mock_file() + written_data = "".join(call.args[0] for call in handle.write.call_args_list) + self.assertIn("Type,Date/Time,INST-VAL,Notes", written_data) + self.assertIn("1,01Sep2021 0000,10.5,", written_data) + self.assertIn("2,02Sep2021 0000,20.0,storm event", written_data) + self.assertIn("3,04Sep2021 0000,42.0,", written_data) + + def test_irregular_read_csv_with_notes(self): + """Hand-typed CSV with a Notes column populates its.notes for + IrregularTimeSeries.""" + content = ( + "E,,,IR-Year\n" + "Type,Date/Time,INST-VAL,Notes\n" + "1,01Sep2021 0000,10.5,\n" + "2,02Sep2021 0000,20.0,storm event\n" + "3,04Sep2021 0000,20.0,\n" + ) + its = self.read_its_from_string(content) + self.assertEqual(its.values.tolist(), [10.5, 20.0, 20.0]) + self.assertEqual(its.notes, ["", "storm event", ""]) + + def test_round_trip_irregular_with_notes(self): + """Full write->read round trip for IrregularTimeSeries notes on a + real temp file, with irregular (non-uniform) time gaps.""" + path = self.test_files.create_test_file(".csv") + its = IrregularTimeSeries.create( + values=[10.5, 20.0, 42.0], + times=[datetime(2021, 9, 1), datetime(2021, 9, 5), datetime(2021, 9, 20)], + notes=["", "storm event", ""], + units="CFS", + data_type="INST-VAL", + path="/A/B/C//IR-Year/F/", + ) + its.to_csv(path, with_metadata=True) + result = IrregularTimeSeries.read_csv(path) + self.assertEqual(result.values.tolist(), [10.5, 20.0, 42.0]) + self.assertEqual(result.notes, ["", "storm event", ""]) + + def test_irregular_to_csv_with_notes_without_metadata(self): + """Notes column appears for IrregularTimeSeries regardless of + with_metadata, same as RegularTimeSeries.""" + its = IrregularTimeSeries.create( + values=[10.5, 20.0], + times=[datetime(2021, 9, 1, 0, 0), datetime(2021, 9, 2, 0, 0)], + notes=["", "storm event"], + units="CFS", + data_type="INST-VAL", + path="/A/B/C//E/F/", + ) + mock_file = mock_open() + with patch("builtins.open", mock_file): + its.to_csv("fake.csv", with_metadata=False) + handle = mock_file() + written = "".join(call.args[0] for call in handle.write.call_args_list) + self.assertNotIn("Units", written) + self.assertIn("Type,Date/Time,INST-VAL,Notes", written) + self.assertIn("2,02Sep2021 0000,20.0,storm event", written) + + def test_round_trip_irregular_with_quality_and_notes(self): + """Quality and notes both survive an IrregularTimeSeries write->read + round trip together.""" + path = self.test_files.create_test_file(".csv") + its = IrregularTimeSeries.create( + values=[10.5, 20.0, 42.0], + times=[datetime(2021, 9, 1), datetime(2021, 9, 5), datetime(2021, 9, 20)], + quality=[0, 5, 10], + notes=["", "storm event", ""], + units="CFS", + data_type="INST-VAL", + path="/A/B/C//IR-Year/F/", + ) + its.to_csv(path, with_metadata=True) + result = IrregularTimeSeries.read_csv(path) + self.assertEqual(result.values.tolist(), [10.5, 20.0, 42.0]) + self.assertEqual(result.quality, [0, 5, 10]) + self.assertEqual(result.notes, ["", "storm event", ""]) + if __name__ == "__main__": unittest.main() From 219e8c7873d84b14c38ed2b0e372fea11522397d Mon Sep 17 00:00:00 2001 From: Jeremiah Kang Date: Thu, 23 Jul 2026 16:14:11 +0000 Subject: [PATCH 5/6] followed c api refactor to simplify cnotes str format --- src/hecdss/hecdss.py | 5 --- src/hecdss/native.py | 73 +++++++++++++++++++++----------------------- 2 files changed, 35 insertions(+), 43 deletions(-) diff --git a/src/hecdss/hecdss.py b/src/hecdss/hecdss.py index 87a3394..fecaaac 100644 --- a/src/hecdss/hecdss.py +++ b/src/hecdss/hecdss.py @@ -624,7 +624,6 @@ def put(self, container) -> int: startDate, startTime = DateConverter.dss_datetime_strings_from_datetime(ts.times[0]) quality = container.quality - noteLength = MAX_NOTE_LENGTH status = self._native.hec_dss_tsStoreRegular( ts.id, @@ -633,7 +632,6 @@ def put(self, container) -> int: ts.values, quality, ts.notes, - noteLength, False, ts.units, ts.data_type, @@ -650,7 +648,6 @@ def put(self, container) -> int: start_date_base = (datetime(1900, 1, 1) + timedelta(days=its.julian_base_date)) startDate, startTime = DateConverter.dss_datetime_strings_from_datetime(start_date_base) quality = container.quality - noteLength = MAX_NOTE_LENGTH julian_times = DateConverter.julian_array_from_date_times( its.times, its.time_granularity_seconds, start_date_base) if max(julian_times) >= 2147483647: @@ -664,7 +661,6 @@ def put(self, container) -> int: its.values, quality, its.notes, - noteLength, False, its.units, its.data_type, @@ -783,7 +779,6 @@ def delete(self, pathname: str, allrecords: bool = False, startdatetime=None, en RTS.values, RTS.quality, RTS.notes, - MAX_NOTE_LENGTH, False, RTS.units, RTS.data_type, diff --git a/src/hecdss/native.py b/src/hecdss/native.py index 0f11052..9d91e6e 100644 --- a/src/hecdss/native.py +++ b/src/hecdss/native.py @@ -23,6 +23,31 @@ # from hecdss.location_info import LocationInfo +def _pack_notes(notes, valueCount): + """Packs a list of note strings into the layout hec_dss_tsStore* expects: + one note per value, each terminated by a single null byte, no padding. + Pads with "" or drops extras so the note count always matches valueCount. + + Returns (packed_bytes, total_length), or (None, 0) when there is nothing + to store. + """ + if not notes or valueCount <= 0: + return None, 0 + + packed = bytearray() + for i in range(valueCount): + # Take the note if it exists, otherwise pad with an empty note + # so the note count always matches valueCount. + note = notes[i] if i < len(notes) else "" + if note is None: + note = "" + + packed += note.encode("utf-8") + packed += b"\x00" + + return bytes(packed), len(packed) + + class _Native: """Wrapper for Native method calls to hecdss.dll or libhecdss.so _Native should not be used directly; Use HecDss @@ -876,7 +901,6 @@ def hec_dss_tsStoreRegular( valueArray, qualityArray, notes, - cnoteSize, saveAsFloat, units, dataType, @@ -894,8 +918,8 @@ def hec_dss_tsStoreRegular( c_int, # valueArraySize (int) POINTER(c_int), # qualityArray (int*) c_int, # qualityArraySize (int) - c_char_p, # cnotesBuffer - c_int, # cnoteSize + c_char_p, # cnotes (packed, one '\0' terminated note per value) + c_int, # cnotesLengthTotal c_int, # saveAsFloat (int) c_char_p, # units (const char*) c_char_p, # type (const char*) @@ -913,20 +937,7 @@ def hec_dss_tsStoreRegular( c_valueArray = (c_double * len(valueArray))(*valueArray) c_qualityArray = (c_int * len(qualityArray))(*qualityArray) - if notes: - c_cnotesBuffer = create_string_buffer(len(valueArray) * cnoteSize) - # notes[:len(valueArray)] guards against notes being longer than - # valueArray, which would otherwise write past the end of the buffer - for i, note in enumerate(notes[:len(valueArray)]): - # encoded defends that note doesn't break if None, helps convert to C raw bytes, and truncates to cnoteSize - 1 to reserve space for null byte \x00 - encoded = (note or "").encode("utf-8")[: cnoteSize - 1] - offset = i * cnoteSize - # write encoded bytes of encoded starting at offset - c_cnotesBuffer[offset: offset + len(encoded)] = encoded - c_cnoteSize = cnoteSize - else: - c_cnotesBuffer = None - c_cnoteSize = 0 + c_cnotes, c_cnotesLengthTotal = _pack_notes(notes, len(valueArray)) return self.dll.hec_dss_tsStoreRegular( self.handle, @@ -937,8 +948,8 @@ def hec_dss_tsStoreRegular( len(valueArray), c_qualityArray, len(qualityArray), - c_cnotesBuffer, - c_cnoteSize, + c_cnotes, + c_cnotesLengthTotal, int(saveAsFloat), c_units, c_type, @@ -955,7 +966,6 @@ def hec_dss_tsStoreIrregular( valueArray, qualityArray, notes, - cnoteSize, saveAsFloat, units, dataType, @@ -974,8 +984,8 @@ def hec_dss_tsStoreIrregular( c_int, # valueArraySize (int) POINTER(c_int), # qualityArray (int*) c_int, # qualityArraySize (int) - c_char_p, # cnotesBuffer (char*) - c_int, # cnoteSize (int) + c_char_p, # cnotes (packed, one '\0' terminated note per value) + c_int, # cnotesLengthTotal c_int, # saveAsFloat (int) c_char_p, # units (const char*) c_char_p, # type (const char*) @@ -993,20 +1003,7 @@ def hec_dss_tsStoreIrregular( c_times = (c_int * len(times))(*times) c_qualityArray = (c_int * len(qualityArray))(*qualityArray) - if notes: - c_cnotesBuffer = create_string_buffer(len(valueArray) * cnoteSize) - # notes[:len(valueArray)] guards against notes being longer than - # valueArray, which would otherwise write past the end of the buffer - for i, note in enumerate(notes[:len(valueArray)]): - # encoded defends that note doesn't break if None, helps convert to C raw bytes, and truncates to cnoteSize - 1 to reserve space for null byte \x00 - encoded = (note or "").encode("utf-8")[: cnoteSize - 1] - offset = i * cnoteSize - # write encoded bytes of encoded starting at offset - c_cnotesBuffer[offset: offset + len(encoded)] = encoded - c_cnoteSize = cnoteSize - else: - c_cnotesBuffer = None - c_cnoteSize = 0 + c_cnotes, c_cnotesLengthTotal = _pack_notes(notes, len(valueArray)) return self.dll.hec_dss_tsStoreIrregular( self.handle, @@ -1018,8 +1015,8 @@ def hec_dss_tsStoreIrregular( len(valueArray), c_qualityArray, len(qualityArray), - c_cnotesBuffer, - c_cnoteSize, + c_cnotes, + c_cnotesLengthTotal, int(saveAsFloat), c_units, c_type, From 46e81e6e16298f05b356c04cd15dabd9716bbdf8 Mon Sep 17 00:00:00 2001 From: Jeremiah Kang Date: Thu, 23 Jul 2026 18:11:05 +0000 Subject: [PATCH 6/6] update version number --- src/hecdss/download_hecdss.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hecdss/download_hecdss.py b/src/hecdss/download_hecdss.py index 0ecc0d7..a385256 100644 --- a/src/hecdss/download_hecdss.py +++ b/src/hecdss/download_hecdss.py @@ -36,7 +36,7 @@ def download_and_unzip(url, zip_file, destination_dir): print(f"Failed to download zip file. Status code: {response.status_code}") base_url = "https://www.hec.usace.army.mil/nexus/repository/maven-public/mil/army/usace/hec/hecdss/" -version = "7-JA-4" +version = "7-JA-8" destination_dir = Path(__file__).parent.joinpath("lib") zip_url = f"{base_url}{version}-win-x86_64/hecdss-{version}-win-x86_64.zip"