diff --git a/.gitignore b/.gitignore index b464af1..b97254c 100644 --- a/.gitignore +++ b/.gitignore @@ -17,3 +17,4 @@ libhecdss.so .venv/ tests/csv_testing/ +src/hecdss/lib/libhecdss_backup.so diff --git a/src/hecdss/download_hecdss.py b/src/hecdss/download_hecdss.py index 0ecc0d7..a385256 100644 --- a/src/hecdss/download_hecdss.py +++ b/src/hecdss/download_hecdss.py @@ -36,7 +36,7 @@ def download_and_unzip(url, zip_file, destination_dir): print(f"Failed to download zip file. Status code: {response.status_code}") base_url = "https://www.hec.usace.army.mil/nexus/repository/maven-public/mil/army/usace/hec/hecdss/" -version = "7-JA-4" +version = "7-JA-8" destination_dir = Path(__file__).parent.joinpath("lib") zip_url = f"{base_url}{version}-win-x86_64/hecdss-{version}-win-x86_64.zip" diff --git a/src/hecdss/dss_csv.py b/src/hecdss/dss_csv.py index 72ef558..6386268 100644 --- a/src/hecdss/dss_csv.py +++ b/src/hecdss/dss_csv.py @@ -39,26 +39,37 @@ def timeseries_to_csv( continue writer.writerow([letter, "", "", value]) # Writing metadata rows writer.writerow(["Units", "", "", series.units]) - if len(series.quality) > 0: - # Write column names with quality - writer.writerow(["Type", "Date/Time", series.data_type, "Quality"]) - else: - # Write column names without quality - writer.writerow(["Type", "Date/Time", series.data_type]) + + header: list[str] = ["Type", "Date/Time", series.data_type] + + has_quality: bool = len(series.quality) > 0 + has_notes: bool = len(series.notes) > 0 + + if has_quality: + header.append("Quality") + if has_notes: + header.append("Notes") + + writer.writerow(header) time_format: str = ("%d%b%Y %H%M%S" if _needs_second_precision(series) else "%d%b%Y %H%M") - ordinate: int = 1 - if len(series.quality) > 0: - for time, value, quality in zip(series.times, series.values, series.quality): - formatted_time: str = time.strftime(time_format) - writer.writerow([ordinate, formatted_time, value, quality]) - ordinate += 1 - else: - for time, value in zip(series.times, series.values): - formatted_time: str = time.strftime(time_format) - writer.writerow([ordinate, formatted_time, value]) - ordinate += 1 + for i, (time, value) in enumerate(zip(series.times, series.values)): + ordinate: str = str(i + 1) + formatted_time: str = time.strftime(time_format) + row: list[str] = [ordinate, formatted_time, value] + + if has_quality: + try: + row.append(series.quality[i]) + except IndexError: + row.append(DEFAULT_MISSING_VALUE) + if has_notes: + try: + row.append(series.notes[i]) + except IndexError: + row.append(DEFAULT_MISSING_VALUE) + writer.writerow(row) def timeseries_read_csv(cls: type[RegularTimeSeries] | type[IrregularTimeSeries], path: str) -> RegularTimeSeries | IrregularTimeSeries: @@ -75,9 +86,9 @@ def timeseries_read_csv(cls: type[RegularTimeSeries] | type[IrregularTimeSeries] if cls not in (RegularTimeSeries, IrregularTimeSeries): raise TypeError("cls must be RegularTimeSeries or IrregularTimeSeries") - times, values, quality, units, data_type = [], [], [], "", "" + times, values, quality, notes, units, data_type = [], [], [], [], "", "" path_parts: dict[str, str] = _empty_path_parts() - has_quality: bool = False # flags + column_index: dict[str, int] = {} with open(path, "r", newline="", encoding="utf-8") as f: reader = csv.reader(f) @@ -94,21 +105,19 @@ def timeseries_read_csv(cls: type[RegularTimeSeries] | type[IrregularTimeSeries] elif first_column_item == "Type": # reached the header if len(row) >= 3: # ['Type', 'Date/Time', data_type, ...potentially more] data_type = row[2].strip() - # ['Type', 'Date/Time', data_type, 'Quality', ...potentially more] - if len(row) >= 4 and row[3].strip() == "Quality": - has_quality = True + # ['Type', 'Date/Time', data_type, 'Quality', 'Notes'] + column_index = { + name.strip(): i for i, name in enumerate(row) if i >= 3} else: # Data row if len(row) < 3: continue # csv is malformed, something is missing raw_time: str = row[1].strip() - # Time format is determined by length of raw_time time_format: str = _get_time_format(raw_time) if time_format is None: # Time format is unrecognized continue - # Do we need to roll over the day date? Yes if time is 2400 roll_day: bool = _need_roll_day(time_format, raw_time) if roll_day: # 2400 isn't a valid hour, so roll it to 0000 before parsing and add a day after raw_time = raw_time.replace(" 2400", " 0000") @@ -129,13 +138,19 @@ def timeseries_read_csv(cls: type[RegularTimeSeries] | type[IrregularTimeSeries] times.append(time) values.append(value) - if has_quality: # Always keep quality index-aligned with values, defaulting a missing cell to 0 - quality_str: str = row[3].strip() if len(row) >= 4 else "" + + quality_idx = column_index.get("Quality") + if quality_idx is not None: + quality_str = row[quality_idx].strip() if len(row) > quality_idx else "" try: quality.append(int(quality_str) if quality_str else 0) except ValueError: quality.append(0) + notes_idx = column_index.get("Notes") + if notes_idx is not None: + notes.append(row[notes_idx].strip() if len(row) > notes_idx else "") + id_path: str = _path_parts_to_id(path_parts) interval: str | int = path_parts["E"] @@ -143,6 +158,7 @@ def timeseries_read_csv(cls: type[RegularTimeSeries] | type[IrregularTimeSeries] values=values, times=times, quality=quality, + notes=notes, units=units, data_type=data_type, interval=interval, diff --git a/src/hecdss/hecdss.py b/src/hecdss/hecdss.py index 51df75c..fecaaac 100644 --- a/src/hecdss/hecdss.py +++ b/src/hecdss/hecdss.py @@ -6,25 +6,28 @@ import hecdss.record_type from hecdss.array_container import ArrayContainer +from hecdss.catalog import Catalog +from hecdss.dateconverter import DateConverter +from hecdss.dsspath import DssPath +from hecdss.gridded_data import GriddedData +from hecdss.irregular_timeseries import IrregularTimeSeries from hecdss.location_info import LocationInfo -from hecdss.text import Text -from hecdss.paired_data import PairedData from hecdss.native import _Native -from hecdss.dateconverter import DateConverter +from hecdss.paired_data import PairedData from hecdss.record_type import RecordType from hecdss.regular_timeseries import RegularTimeSeries -from hecdss.irregular_timeseries import IrregularTimeSeries -from hecdss.catalog import Catalog -from hecdss.gridded_data import GriddedData -from hecdss.dsspath import DssPath +from hecdss.text import Text + DSS_UNDEFINED_VALUE = -340282346638528859811704183484516925440.000000 +MAX_NOTE_LENGTH: int = 40 class HecDss: """ Main class for working with DSS files """ - def __init__(self, filename:str): + + def __init__(self, filename: str): """constructor for HecDSS Args: @@ -56,6 +59,7 @@ def __exit__(self, exc_type, exc_val, exc_tb): exc_tb (traceback): The traceback object. """ self.close() + @staticmethod def set_global_debug_level(level: int) -> None: """ @@ -73,6 +77,7 @@ def set_global_debug_level(level: int) -> None: """ # Set the native DLL level (controls what gets written) _Native().hec_dss_set_debug_level(level) + def close(self): """closes the DSS file and releases any locks """ @@ -115,7 +120,7 @@ def get(self, pathname: str, startdatetime=None, enddatetime=None, trim=False): """ if type == RecordType.RegularTimeSeries or type == RecordType.IrregularTimeSeries: new_pathname = pathname - if(DssPath(pathname).D.lower() != "ts-pattern"): + if (DssPath(pathname).D.lower() != "ts-pattern"): new_pathname = DssPath(pathname).path_without_date().__str__() elif type == RecordType.IrregularTimeSeries: raise ValueError("ts-pattern is not fully supported for irregular time series") @@ -137,16 +142,15 @@ def get(self, pathname: str, startdatetime=None, enddatetime=None, trim=False): def _get_text(self, pathname: str): textLength = 1024 - BUFFER_TOO_SMALL = -17 textArray = [] status = self._native.hec_dss_textRetrieve(pathname, textArray, textLength) while status == BUFFER_TOO_SMALL: textLength *= 2 - if textLength > 2*1048576: # 2 MB + if textLength > 2 * 1048576: # 2 MB print(f"Text record too large to read from '{pathname}'") return None - textArray = [] # otherwise we get an entry for each attempt + textArray = [] # otherwise we get an entry for each attempt status = self._native.hec_dss_textRetrieve(pathname, textArray, textLength) if status != 0: @@ -162,7 +166,8 @@ def _get_array(self, pathname: str): floatValuesCount = [0] doubleValuesCount = [0] - self._native.hec_dss_arrayRetrieveInfo(pathname, intValuesCount, floatValuesCount, doubleValuesCount) + self._native.hec_dss_arrayRetrieveInfo( + pathname, intValuesCount, floatValuesCount, doubleValuesCount) intValues = [] floatValues = [] @@ -177,7 +182,8 @@ def _get_array(self, pathname: str): status = self._native.hec_dss_arrayRetrieve(pathname, intValues, floatValues, doubleValues) location_info = self._get_location_info(pathname) - rval = ArrayContainer.create_array_container(intValues, floatValues, doubleValues, path=pathname, location_info=location_info) + rval = ArrayContainer.create_array_container( + intValues, floatValues, doubleValues, path=pathname, location_info=location_info) return rval def _get_gridded_data(self, pathname): @@ -340,7 +346,7 @@ def _get_paired_data(self, pathname): # ---------------------------------------------------------------------------------------- # # rearrange from consecutive values for each curve to consecutive curves for each ordinate # # ---------------------------------------------------------------------------------------- # - groups = [doubleValues[i*ordinateCount:(i+1)*ordinateCount] for i in range(n)] + groups = [doubleValues[i * ordinateCount:(i + 1) * ordinateCount] for i in range(n)] doubleValues = list(map(list, zip(*groups))) pd.values = np.array(doubleValues).reshape((len(doubleOrdinates), n)) # pd.values = [doubleValues[i:i+n] for i in range(0, len(doubleValues), n)] @@ -373,26 +379,29 @@ def _get_julian_time_range(self, pathname, boolFullSet): def _get_date_time_range(self, pathname, boolFullSet): - firstValidJulian, firstSeconds, lastValidJulian, lastSeconds = self._get_julian_time_range(pathname, boolFullSet) + firstValidJulian, firstSeconds, lastValidJulian, lastSeconds = self._get_julian_time_range( + pathname, boolFullSet) first = DateConverter.date_times_from_julian_array(firstSeconds, 1, firstValidJulian[0])[0] last = DateConverter.date_times_from_julian_array(lastSeconds, 1, lastValidJulian[0])[0] return (first, last) - def _get_timeseries(self, pathname, startDateTime, endDateTime, trim): # get sizes - firstValidJulian, firstSeconds, lastValidJulian, lastSeconds = self._get_julian_time_range(pathname, 1) + firstValidJulian, firstSeconds, lastValidJulian, lastSeconds = self._get_julian_time_range( + pathname, 1) if startDateTime is None: - _startDateTime = DateConverter.date_time_from_julian_second(firstValidJulian[0], firstSeconds[0]) + _startDateTime = DateConverter.date_time_from_julian_second( + firstValidJulian[0], firstSeconds[0]) firstSeconds, firstJulian = firstSeconds, firstValidJulian else: _startDateTime = startDateTime minutes = DateConverter.julian_array_from_date_times([startDateTime])[0] firstSeconds, firstJulian = [minutes % 1440 * 60], [minutes // 1440] if endDateTime is None: - _endDateTime = DateConverter.date_time_from_julian_second(lastValidJulian[0], lastSeconds[0]) + _endDateTime = DateConverter.date_time_from_julian_second( + lastValidJulian[0], lastSeconds[0]) lastSeconds, lastJulian = lastSeconds, lastValidJulian else: _endDateTime = endDateTime @@ -437,6 +446,8 @@ def _get_timeseries(self, pathname, startDateTime, endDateTime, trim): values = [] numberValuesRead = [0] quality = [] + notes = [] + noteLength = MAX_NOTE_LENGTH julianBaseDate = [0] timeGranularitySeconds = [0] units = [""] @@ -452,7 +463,10 @@ def _get_timeseries(self, pathname, startDateTime, endDateTime, trim): endTime, times, values, - number_periods+1, + number_periods + 1, + None, + noteLength, + notes, numberValuesRead, quality, qualityElementSize[0], @@ -478,27 +492,33 @@ def _get_timeseries(self, pathname, startDateTime, endDateTime, trim): else: ts = RegularTimeSeries() if trim or not startDateTime or not endDateTime: - trimmed_indices = [i for i, v in enumerate(values) if values[i] != DSS_UNDEFINED_VALUE] + trimmed_indices = [i for i, v in enumerate( + values) if values[i] != DSS_UNDEFINED_VALUE] if not trimmed_indices: times = [] values = [] quality = [] + notes = [] else: start = 0 if startDateTime and not trim else trimmed_indices[0] - end = len(times) if endDateTime and not trim else trimmed_indices[-1]+1 + end = len(times) if endDateTime and not trim else trimmed_indices[-1] + 1 times = times[start:end] values = values[start:end] if quality != []: quality = quality[start:end] + if notes != []: + notes = notes[start:end] new_times = DateConverter.date_times_from_julian_array( times, timeGranularitySeconds[0], julianBaseDate[0] ) arr = np.array(values) if RecordType.IrregularTimeSeries == type(ts): - indices = np.where(np.isclose(values, DSS_UNDEFINED_VALUE, rtol=0, atol=0, equal_nan=True))[0] + indices = np.where(np.isclose(values, DSS_UNDEFINED_VALUE, + rtol=0, atol=0, equal_nan=True))[0] arr = np.delete(arr, indices) new_times = [new_times[i] for i in range(len(new_times)) if not np.isin(i, indices)] quality = [quality[i] for i in range(len(quality)) if not np.isin(i, indices)] + notes = [notes[i] for i in range(len(notes)) if not np.isin(i, indices)] values = arr units = units[0] @@ -507,10 +527,10 @@ def _get_timeseries(self, pathname, startDateTime, endDateTime, trim): time_granularity_seconds = timeGranularitySeconds[0] julian_base_date = julianBaseDate[0] timeZoneName = timeZoneName[0] - if(timeZoneName): + if (timeZoneName): try: new_times = [i.replace(tzinfo=ZoneInfo(timeZoneName)) for i in new_times] - except ZoneInfoNotFoundError as e: + except ZoneInfoNotFoundError as e: print(f"Warning: {e}. Using no zone instead.") timeZoneName = False elif (DssPath(pathname).D.lower() == "ts-pattern"): @@ -518,7 +538,8 @@ def _get_timeseries(self, pathname, startDateTime, endDateTime, trim): start_date = _startDateTime - timedelta(seconds=interval_seconds) location_info = self._get_location_info(pathname) - ts = ts.create(values=values, times=new_times, quality=quality, units=units, data_type=data_type, start_date=start_date, time_granularity_seconds=time_granularity_seconds, julian_base_date=julian_base_date, time_zone_name=timeZoneName, path=pathname, location_info=location_info) + ts = ts.create(values=values, times=new_times, quality=quality, notes=notes, units=units, data_type=data_type, start_date=start_date, + time_granularity_seconds=time_granularity_seconds, julian_base_date=julian_base_date, time_zone_name=timeZoneName, path=pathname, location_info=location_info) if (DssPath(pathname).D.lower() == "ts-pattern"): new_interval = ts._get_interval_path() @@ -610,6 +631,7 @@ def put(self, container) -> int: startTime, ts.values, quality, + ts.notes, False, ts.units, ts.data_type, @@ -623,10 +645,11 @@ def put(self, container) -> int: raise ValueError("ts-pattern is not fully supported for irregular time series") # def hec_dss_tsStoreRegular(dss, pathname, startDate, startTime, valueArray, qualityArray, # saveAsFloat, units, type): - start_date_base = (datetime(1900, 1, 1)+timedelta(days=its.julian_base_date)) + start_date_base = (datetime(1900, 1, 1) + timedelta(days=its.julian_base_date)) startDate, startTime = DateConverter.dss_datetime_strings_from_datetime(start_date_base) quality = container.quality - julian_times = DateConverter.julian_array_from_date_times(its.times, its.time_granularity_seconds, start_date_base) + julian_times = DateConverter.julian_array_from_date_times( + its.times, its.time_granularity_seconds, start_date_base) if max(julian_times) >= 2147483647: raise Exception("Julian times contains value larger than 2147483647, increase granularity or change " "start_date_base to fix.") @@ -637,6 +660,7 @@ def put(self, container) -> int: its.time_granularity_seconds, its.values, quality, + its.notes, False, its.units, its.data_type, @@ -654,20 +678,22 @@ def put(self, container) -> int: status = self._native.hec_dss_gridStore(gd) self._catalog = None elif type(container) is ArrayContainer: - status = self._native.hec_dss_arrayStore(container.id, container.int_values, container.float_values, container.double_values) + status = self._native.hec_dss_arrayStore( + container.id, container.int_values, container.float_values, container.double_values) self._catalog = None elif type(container) is LocationInfo: - status = self._native.hec_dss_locationStore(container,1) + status = self._native.hec_dss_locationStore(container, 1) self._catalog = None elif type(container) is Text: text = container status = self._native.hec_dss_textStore(text.id, text.text, len(text.text)) self._catalog = None else: - raise NotImplementedError(f"unsupported record_type: {type(container)}. Expected types are: {RecordType.SUPPORTED_RECORD_TYPES.value}") + raise NotImplementedError( + f"unsupported record_type: {type(container)}. Expected types are: {RecordType.SUPPORTED_RECORD_TYPES.value}") if hasattr(container, "location_info") and container.location_info is not None: - status = self._native.hec_dss_locationStore(container.location_info,1) + status = self._native.hec_dss_locationStore(container.location_info, 1) # TODO -- instead of invalidating catalog,with _catalog=None # can we be smart? @@ -679,7 +705,6 @@ def put(self, container) -> int: return status - def writePrecompressedGrid(self, gd, compressedData, CompressionSize): """ puts pre-compressed gridded data into the DSS file @@ -723,7 +748,7 @@ def delete(self, pathname: str, allrecords: bool = False, startdatetime=None, en if enddatetime: newEndDateTime = enddatetime - interval_seconds=DateConverter.intervalString_to_sec(delete_path.E) + interval_seconds = DateConverter.intervalString_to_sec(delete_path.E) firstMinutes = DateConverter.julian_array_from_date_times([newStartDateTime])[0] firstSeconds, firstJulian = [firstMinutes % 1440 * 60], [firstMinutes // 1440] @@ -740,7 +765,7 @@ def delete(self, pathname: str, allrecords: bool = False, startdatetime=None, en ) RTS = RegularTimeSeries() - RTS.values = [DSS_UNDEFINED_VALUE]*number_periods + RTS.values = [DSS_UNDEFINED_VALUE] * number_periods RTS.times = [newStartDateTime, newEndDateTime] RTS.start_date = startdatetime RTS.id = pathname @@ -753,6 +778,7 @@ def delete(self, pathname: str, allrecords: bool = False, startdatetime=None, en startTime, RTS.values, RTS.quality, + RTS.notes, False, RTS.units, RTS.data_type, @@ -779,7 +805,8 @@ def delete(self, pathname: str, allrecords: bool = False, startdatetime=None, en if status == 0: self._catalog = None else: - print(f"Error deleting record from '{pathname}', Record does not exist or timeseries path must be uncondensed") + print( + f"Error deleting record from '{pathname}', Record does not exist or timeseries path must be uncondensed") return status def get_catalog(self) -> Catalog: diff --git a/src/hecdss/irregular_timeseries.py b/src/hecdss/irregular_timeseries.py index 328bc2c..d660c9d 100644 --- a/src/hecdss/irregular_timeseries.py +++ b/src/hecdss/irregular_timeseries.py @@ -7,6 +7,7 @@ class IrregularTimeSeries: """ container for time-series data that is not at a consistent interval. data is stored internally as a numpy array """ + def __init__(self): """ Initialize an IrregularTimeSeries object with default values. @@ -15,6 +16,7 @@ def __init__(self): self.times = [] self.values = np.empty(0) self.quality = [] + self.notes: list[str] = [] self.units = "" self.data_type = "" self.interval = 0 @@ -25,13 +27,17 @@ def __init__(self): self.id = "" self.location_info = None - def add_data_point(self, date, value): + def add_data_point(self, date, value, flag=None, note=None): """ append a date,value to this time-series """ self.times.append(date) self.values.append(value) + if flag is not None: + self.quality.append(flag) + if note is not None: + self.notes.append(note) def get_value_at(self, date): """ @@ -81,10 +87,17 @@ def print_to_console(self): Print the time-series data to the console in a readable format. """ print("dsspath='" + self.id + "'") - print("units='"+self.units+"'") + print("units='" + self.units + "'") print("dataType='" + self.data_type + "'") - for time, value in zip(self.times, self.values): - print(f"Time: {time}, Value: {value}") + has_quality = len(self.quality) > 0 + has_notes = len(self.notes) > 0 + for i, (time, value) in enumerate(zip(self.times, self.values)): + line = f"Time: {time}, Value: {value}" + if has_quality: + line += f", Flag: {self.quality[i]}" + if has_notes: + line += f", Note: {self.notes[i]}" + print(line) def to_csv(self, file_path: str, with_metadata: bool = True) -> None: """ @@ -102,7 +115,7 @@ def to_csv(self, file_path: str, with_metadata: bool = True) -> None: def read_csv(file_path: str) -> "IrregularTimeSeries": """ Reads a .csv file and creates an IrregularTimeSeries instance from the data. - + Parameters: file_path (str): The path to the .csv file to read @@ -113,7 +126,7 @@ def read_csv(file_path: str) -> "IrregularTimeSeries": return timeseries_read_csv(IrregularTimeSeries, file_path) @staticmethod - def create(values, times, quality=[], units="", data_type="", interval=0, start_date="", time_granularity_seconds=1, julian_base_date=None, time_zone_name="", path=None, location_info=None): + def create(values, times, quality=[], notes=[], units="", data_type="", interval=0, start_date="", time_granularity_seconds=1, julian_base_date=None, time_zone_name="", path=None, location_info=None): """ Retrieve the value at a specific date in the time-series. @@ -127,6 +140,7 @@ def create(values, times, quality=[], units="", data_type="", interval=0, start_ irts.times = times irts.values = np.array(values) irts.quality = quality + irts.notes = notes irts.units = units irts.data_type = data_type irts.interval = interval @@ -134,7 +148,7 @@ def create(values, times, quality=[], units="", data_type="", interval=0, start_ irts.time_granularity_seconds = time_granularity_seconds irts.julian_base_date = 0 if julian_base_date is None and len(times): - irts.julian_base_date = (times[0]-datetime(1900, 1, 1)).days + irts.julian_base_date = (times[0] - datetime(1900, 1, 1)).days irts.time_zone_name = time_zone_name irts.id = path irts.location_info = location_info diff --git a/src/hecdss/native.py b/src/hecdss/native.py index a85a8b9..9d91e6e 100644 --- a/src/hecdss/native.py +++ b/src/hecdss/native.py @@ -1,18 +1,53 @@ import ctypes -from ctypes import c_float, c_double, c_char_p, c_int, c_void_p, POINTER -from ctypes import c_int32 -from ctypes import byref, create_string_buffer -from ctypes.util import find_library -import numpy as np -from importlib import resources import io import os import sys +from ctypes import ( + POINTER, + byref, + c_char_p, + c_double, + c_float, + c_int, + c_int32, + c_void_p, + create_string_buffer, +) +from ctypes.util import find_library +from importlib import resources from typing import List +import numpy as np + + # from hecdss.location_info import LocationInfo +def _pack_notes(notes, valueCount): + """Packs a list of note strings into the layout hec_dss_tsStore* expects: + one note per value, each terminated by a single null byte, no padding. + Pads with "" or drops extras so the note count always matches valueCount. + + Returns (packed_bytes, total_length), or (None, 0) when there is nothing + to store. + """ + if not notes or valueCount <= 0: + return None, 0 + + packed = bytearray() + for i in range(valueCount): + # Take the note if it exists, otherwise pad with an empty note + # so the note count always matches valueCount. + note = notes[i] if i < len(notes) else "" + if note is None: + note = "" + + packed += note.encode("utf-8") + packed += b"\x00" + + return bytes(packed), len(packed) + + class _Native: """Wrapper for Native method calls to hecdss.dll or libhecdss.so _Native should not be used directly; Use HecDss @@ -32,7 +67,6 @@ def load_hecdss_library(self, libname): raise FileNotFoundError(f"{libname} not found Paths searched: {paths_to_try}") return ctypes.CDLL(found_libs[0]) - def __init__(self): """Loads the hecdss shared library from disk""" @@ -239,7 +273,8 @@ def hec_dss_gridRetrieve(self, pathname: str, c_data = (c_float * 0)() result = self.dll.hec_dss_gridRetrieve(self.handle, pathname.encode("utf-8"), False, - ctypes.byref(type_pointer), ctypes.byref(dataType_pointer), + ctypes.byref(type_pointer), ctypes.byref( + dataType_pointer), c_lowerLeftCellX, c_lowerLeftCellY, c_numberOfCellsX, c_numberOfCellsY, c_numberOfRanges, c_srsDefinitionType, @@ -250,9 +285,12 @@ def hec_dss_gridRetrieve(self, pathname: str, c_srsName, srsNameLength, c_srsDefinition, srsDefinitionLength, c_timeZoneID, timeZoneIDLength, - ctypes.byref(c_cellSize), ctypes.byref(c_xCoordOfGridCellZero), - ctypes.byref(c_yCoordOfGridCellZero), ctypes.byref(c_nullValue), - ctypes.byref(c_maxDataValue), ctypes.byref(c_minDataValue), + ctypes.byref(c_cellSize), ctypes.byref( + c_xCoordOfGridCellZero), + ctypes.byref(c_yCoordOfGridCellZero), ctypes.byref( + c_nullValue), + ctypes.byref(c_maxDataValue), ctypes.byref( + c_minDataValue), ctypes.byref(c_meanDataValue), c_rangeLimitTable, rangeTablesLength, c_numberEqualOrExceedingRangeLimit, @@ -269,7 +307,8 @@ def hec_dss_gridRetrieve(self, pathname: str, c_data = (c_float * dataLength)() result = self.dll.hec_dss_gridRetrieve(self.handle, pathname.encode("utf-8"), True, - ctypes.byref(type_pointer), ctypes.byref(dataType_pointer), + ctypes.byref(type_pointer), ctypes.byref( + dataType_pointer), c_lowerLeftCellX, c_lowerLeftCellY, c_numberOfCellsX, c_numberOfCellsY, c_numberOfRanges, c_srsDefinitionType, @@ -280,9 +319,12 @@ def hec_dss_gridRetrieve(self, pathname: str, c_srsName, srsNameLength, c_srsDefinition, srsDefinitionLength, c_timeZoneID, timeZoneIDLength, - ctypes.byref(c_cellSize), ctypes.byref(c_xCoordOfGridCellZero), - ctypes.byref(c_yCoordOfGridCellZero), ctypes.byref(c_nullValue), - ctypes.byref(c_maxDataValue), ctypes.byref(c_minDataValue), + ctypes.byref(c_cellSize), ctypes.byref( + c_xCoordOfGridCellZero), + ctypes.byref(c_yCoordOfGridCellZero), ctypes.byref( + c_nullValue), + ctypes.byref(c_maxDataValue), ctypes.byref( + c_minDataValue), ctypes.byref(c_meanDataValue), c_rangeLimitTable, rangeTablesLength, c_numberEqualOrExceedingRangeLimit, @@ -573,7 +615,8 @@ def hec_dss_pdStore( if numberCurves > 1: _values = pd.values.tolist() if len(_values[0]) > 1: - _values = [[_values[i][j] for i in range(len(_values))] for j in range(len(_values[0]))] + _values = [[_values[i][j] + for i in range(len(_values))] for j in range(len(_values[0]))] values2 = np.array(_values) else: values2 = pd.values @@ -739,7 +782,10 @@ def hec_dss_tsRetrieve( times: List[int], values: List[float], arraySize: str, - numberValuesRead, + cnotesBuffer: str, + cnoteSize: int, + notes: List[str], + numberValuesRead: List[int], quality: List[int], qualityLength: int, julianBaseDate: List[int], @@ -763,6 +809,8 @@ def hec_dss_tsRetrieve( POINTER(c_int), # timeArray POINTER(c_double), # valueArray c_int, # arraySize + c_char_p, # cnotesBuffer + c_int, # cnoteSize POINTER(c_int), # numberValuesRead POINTER(c_int), # quality c_int, # qualityLength @@ -783,6 +831,8 @@ def hec_dss_tsRetrieve( c_numberValuesRead = c_int(0) size = qualityLength * arraySize + c_cnotesBuffer = create_string_buffer(arraySize * cnoteSize) + c_quality = (c_int * size)() c_julianBaseDate = c_int(0) c_timeGranularitySeconds = c_int(0) @@ -809,6 +859,8 @@ def hec_dss_tsRetrieve( c_times, c_values, c_arraySize, + c_cnotesBuffer, + cnoteSize, byref(c_numberValuesRead), c_quality, qualityLength, @@ -828,6 +880,11 @@ def hec_dss_tsRetrieve( values.extend(list(c_values[: c_numberValuesRead.value])) quality.clear() quality.extend(list(c_quality[: c_numberValuesRead.value])) + notes.clear() + raw = c_cnotesBuffer.raw + for i in range(c_numberValuesRead.value): + chunk = raw[i * cnoteSize: (i + 1) * cnoteSize] + notes.append(chunk.split(b"\x00", 1)[0].decode("utf-8")) units[0] = c_units.value.decode("utf-8") dataType[0] = c_dataType.value.decode("utf-8") julianBaseDate[0] = c_julianBaseDate.value @@ -843,6 +900,7 @@ def hec_dss_tsStoreRegular( startTime, valueArray, qualityArray, + notes, saveAsFloat, units, dataType, @@ -860,6 +918,8 @@ def hec_dss_tsStoreRegular( c_int, # valueArraySize (int) POINTER(c_int), # qualityArray (int*) c_int, # qualityArraySize (int) + c_char_p, # cnotes (packed, one '\0' terminated note per value) + c_int, # cnotesLengthTotal c_int, # saveAsFloat (int) c_char_p, # units (const char*) c_char_p, # type (const char*) @@ -877,6 +937,8 @@ def hec_dss_tsStoreRegular( c_valueArray = (c_double * len(valueArray))(*valueArray) c_qualityArray = (c_int * len(qualityArray))(*qualityArray) + c_cnotes, c_cnotesLengthTotal = _pack_notes(notes, len(valueArray)) + return self.dll.hec_dss_tsStoreRegular( self.handle, c_pathname, @@ -886,6 +948,8 @@ def hec_dss_tsStoreRegular( len(valueArray), c_qualityArray, len(qualityArray), + c_cnotes, + c_cnotesLengthTotal, int(saveAsFloat), c_units, c_type, @@ -901,6 +965,7 @@ def hec_dss_tsStoreIrregular( timeGranularitySeconds, valueArray, qualityArray, + notes, saveAsFloat, units, dataType, @@ -908,8 +973,8 @@ def hec_dss_tsStoreIrregular( storageFlag, ): - self.dll.hec_dss_tsStoreIregular.restype = c_int - self.dll.hec_dss_tsStoreIregular.argtypes = [ + self.dll.hec_dss_tsStoreIrregular.restype = c_int + self.dll.hec_dss_tsStoreIrregular.argtypes = [ c_void_p, # dss (void*) c_char_p, # pathname (const char*) c_char_p, # startDateBase (const char*) @@ -919,11 +984,13 @@ def hec_dss_tsStoreIrregular( c_int, # valueArraySize (int) POINTER(c_int), # qualityArray (int*) c_int, # qualityArraySize (int) + c_char_p, # cnotes (packed, one '\0' terminated note per value) + c_int, # cnotesLengthTotal c_int, # saveAsFloat (int) c_char_p, # units (const char*) c_char_p, # type (const char*) c_char_p, # timeZoneName (const char*) - c_int, # storageFlag (int) + c_int, # storageFlag (int) ] c_pathname = c_char_p(pathname.encode("utf-8")) @@ -936,7 +1003,9 @@ def hec_dss_tsStoreIrregular( c_times = (c_int * len(times))(*times) c_qualityArray = (c_int * len(qualityArray))(*qualityArray) - return self.dll.hec_dss_tsStoreIregular( + c_cnotes, c_cnotesLengthTotal = _pack_notes(notes, len(valueArray)) + + return self.dll.hec_dss_tsStoreIrregular( self.handle, c_pathname, c_startDateBase, @@ -946,6 +1015,8 @@ def hec_dss_tsStoreIrregular( len(valueArray), c_qualityArray, len(qualityArray), + c_cnotes, + c_cnotesLengthTotal, int(saveAsFloat), c_units, c_type, @@ -1042,7 +1113,6 @@ def hec_dss_arrayRetrieve(self, pathname, intValues: List[int], floatValues: Lis if len(doubleValues) > 0: doubleValues[:] = np.ctypeslib.as_array(c_doubleValues, shape=(len(doubleValues),)) - else: print("Error reading array status = {status}") @@ -1159,28 +1229,28 @@ def hec_dss_locationStore(self, location_info, replace: int) -> int: return result def hec_dss_delete(self, pathname: str) -> int: - """ - Deletes a record from the DSS file. + """ + Deletes a record from the DSS file. - Args: - pathname (str): The pathname of the record to delete. + Args: + pathname (str): The pathname of the record to delete. - Returns: - int: Status of zero when successful, non-zero on error. - """ - f = self.dll.hec_dss_delete - f.argtypes = [ - c_void_p, # dss_file* dss - c_char_p # const char* pathname - ] - f.restype = c_int + Returns: + int: Status of zero when successful, non-zero on error. + """ + f = self.dll.hec_dss_delete + f.argtypes = [ + c_void_p, # dss_file* dss + c_char_p # const char* pathname + ] + f.restype = c_int - result = f(self.handle, pathname.encode("utf-8")) + result = f(self.handle, pathname.encode("utf-8")) - if result != 0: - print("Function call failed with result:", result) + if result != 0: + print("Function call failed with result:", result) - return result + return result def hec_dss_textStore(self, pathname, text, length=None): """ @@ -1200,12 +1270,12 @@ def hec_dss_textStore(self, pathname, text, length=None): f.restype = c_int result = f(self.handle, pathname.encode("utf-8"), - text.encode("utf-8"), - length if length is not None else len(text)) - + text.encode("utf-8"), + length if length is not None else len(text)) + return result - - def hec_dss_textRetrieve(self, pathname, buffer :List[str], buff_size: int) -> int: + + def hec_dss_textRetrieve(self, pathname, buffer: List[str], buff_size: int) -> int: """ Store text data in a DSS file. Args: @@ -1224,8 +1294,8 @@ def hec_dss_textRetrieve(self, pathname, buffer :List[str], buff_size: int) -> i c_buffer = create_string_buffer(buff_size) result = f(self.handle, pathname.encode("utf-8"), - c_buffer, - buff_size) - + c_buffer, + buff_size) + buffer.append(c_buffer.value.decode("utf-8")) - return result \ No newline at end of file + return result diff --git a/src/hecdss/regular_timeseries.py b/src/hecdss/regular_timeseries.py index 0591612..38bbae5 100644 --- a/src/hecdss/regular_timeseries.py +++ b/src/hecdss/regular_timeseries.py @@ -15,6 +15,7 @@ def __init__(self): self.times = [] self.values = np.empty(0) self.quality = [] + self.notes: list[str] = [] self.units = "" self.data_type = "" self.interval = "" @@ -25,7 +26,7 @@ def __init__(self): self.id = "" self.location_info = None - def add_data_point(self, date, value, flag=None): + def add_data_point(self, date: datetime, value: float, flag: int = None, note: str = None): """ Adds a data point to the time series. @@ -34,10 +35,13 @@ def add_data_point(self, date, value, flag=None): value (float): The value of the data point. flag (int, optional): The quality flag of the data point. Defaults to None. """ + # TODO: Does this let you add a mismatched datetime?? self.times.append(date) self.values.append(value) if flag is not None: self.quality.append(flag) + if note is not None: + self.notes.append(note) def get_value_at(self, date): """ @@ -89,13 +93,21 @@ def print_to_console(self): print("dsspath='" + self.id + "'") print("units='" + self.units + "'") print("dataType='" + self.data_type + "'") - print("Time,Value,Flag") - if not len(self.quality) > 0: - for time, value in zip(self.times, self.values): - print(f"{time}, {value}") - else: - for time, value, flag in zip(self.times, self.values, self.quality): - print(f"{time}, {value}, {flag}") + has_quality = len(self.quality) > 0 + has_notes = len(self.notes) > 0 + header = "Time,Value" + if has_quality: + header += ",Flag" + if has_notes: + header += ",Note" + print(header) + for i, (time, value) in enumerate(zip(self.times, self.values)): + row = f"{time}, {value}" + if has_quality: + row += f", {self.quality[i]}" + if has_notes: + row += f", {self.notes[i]}" + print(row) def to_csv(self, file_path: str, with_metadata: bool = True) -> None: """ @@ -254,7 +266,7 @@ def read_csv(file_path: str) -> "RegularTimeSeries": return timeseries_read_csv(RegularTimeSeries, file_path) @staticmethod - def create(values, times=[], quality=[], units="", data_type="", interval="", start_date="", time_granularity_seconds=1, julian_base_date=0, time_zone_name="", path=None, location_info=None): + def create(values, times=[], quality=[], notes=[], units="", data_type="", interval="", start_date="", time_granularity_seconds=1, julian_base_date=0, time_zone_name="", path=None, location_info=None): """ Creates a new instance of the RegularTimeSeries class with the specified parameters. @@ -278,6 +290,7 @@ def create(values, times=[], quality=[], units="", data_type="", interval="", st rts.times = [i.replace(microsecond=0) for i in times] rts.values = np.array(values) rts.quality = quality + rts.notes = notes rts.units = units rts.data_type = data_type rts.interval = interval diff --git a/tests/test_basics.py b/tests/test_basics.py index e379f33..88fdb75 100644 --- a/tests/test_basics.py +++ b/tests/test_basics.py @@ -55,22 +55,17 @@ def test_catalog(self): for ds in catalog: print(ds.recType, ds) - def test_catalog_get(self): with HecDss(self.test_files.get_copy("sample7.dss")) as dss: catalog = dss.get_catalog() ts = dss.get(catalog.items[0]) - def test_with_block(self): with HecDss(self.test_files.get_copy("sample7.dss")) as dss: catalog = dss.get_catalog() ts = dss.get(catalog.items[0]) - - - def test_new_catalog(self): rawPaths = [ "//SACRAMENTO/TEMP-MIN/01Jan1989/1Day/OBS/", @@ -83,7 +78,6 @@ def test_new_catalog(self): c = Catalog(rawPaths, recordType) c.print() - def test_paired_data(self): """ read record from disk, add labels save to new path @@ -122,7 +116,7 @@ def test_missing_values(self): with HecDss(self.test_files.get_copy("missing_data.dss")) as dss: print("record count = " + str(dss.record_count())) tsc = dss.get("/CUMBERLAND RIVER/CUMBERLAND FALLS/FLOW//30Minute/MISSING/", datetime(2020, 1, 3, 11, 0), - datetime(2020, 1, 13, 0, 0)) + datetime(2020, 1, 13, 0, 0)) tsc.print_to_console() # values = ts.values @@ -204,13 +198,13 @@ def test_delete_range(self): t1 = datetime(1879, 1, 1) t2 = datetime(1885, 1, 1) dss.delete(path, False, t1, t2) - assert (not("//SACRAMENTO/PRECIP-INC/01Jan1882/1Day/OBS/" in dss.get_catalog().uncondensed_paths)) + assert (not ("//SACRAMENTO/PRECIP-INC/01Jan1882/1Day/OBS/" in dss.get_catalog().uncondensed_paths)) def test_Read_TS_Pattern_Regular(self): with HecDss(self.test_files.get_copy("Depth_Area_01.dss")) as dss: path = "//010010-B/FLOW-UNIT GRAPH/TS-Pattern/5Minute/DAA:Depth-Area 01>010005-C/" tsc = dss.get(path) - assert(len(tsc.values) > 0) + assert (len(tsc.values) > 0) def test_Write_TS_Pattern_Regular(self): with HecDss(self.test_files.get_copy("Depth_Area_01.dss")) as dss: @@ -221,7 +215,55 @@ def test_Write_TS_Pattern_Regular(self): tsc.start_date = tsc.start_date.replace(year=2023) dss.put(tsc) tsc2 = dss.get(tsc.id) - assert(len(tsc2.values) > 0) + assert (len(tsc2.values) > 0) + + def test_write_regular_timeseries_with_notes(self): + with HecDss(self.test_files.get_copy("sample7.dss")) as dss: + t1 = datetime(2005, 1, 1) + t2 = datetime(2005, 1, 4) + tsc = dss.get("//SACRAMENTO/PRECIP-INC//1Day/OBS/", t1, t2) + tsc.id = "//SACRAMENTO/PRECIP-INC/01Jan2005/1Day/OBS-notes/" + tsc.notes = ["" for _ in tsc.values] + tsc.notes[1] = "manual override" + + dss.put(tsc) + tsc2 = dss.get(tsc.id, t1, t2) + + self.assertEqual(tsc.notes, tsc2.notes) + + def test_write_regular_timeseries_with_quality(self): + """ + Quality flags written on a RegularTimeSeries survive a put/get round trip. + """ + with HecDss(self.test_files.get_copy("sample7.dss")) as dss: + t1 = datetime(2005, 1, 1) + t2 = datetime(2005, 1, 4) + tsc = dss.get("//SACRAMENTO/PRECIP-INC//1Day/OBS/", t1, t2) + tsc.id = "//SACRAMENTO/PRECIP-INC/01Jan2005/1Day/OBS-quality/" + tsc.quality = [0, 1, 2, 3][:len(tsc.values)] + + dss.put(tsc) + tsc2 = dss.get(tsc.id, t1, t2) + + self.assertEqual(tsc.quality, tsc2.quality) + + def test_write_regular_timeseries_with_quality_and_notes(self): + """ + Quality flags and notes written together on a RegularTimeSeries survive a put/get round trip. + """ + with HecDss(self.test_files.get_copy("sample7.dss")) as dss: + t1 = datetime(2005, 1, 1) + t2 = datetime(2005, 1, 4) + tsc = dss.get("//SACRAMENTO/PRECIP-INC//1Day/OBS/", t1, t2) + tsc.id = "//SACRAMENTO/PRECIP-INC/01Jan2005/1Day/OBS-quality-notes/" + tsc.quality = [0, 1, 2, 3][:len(tsc.values)] + tsc.notes = ["", "manual override", "", "estimated"][:len(tsc.values)] + + dss.put(tsc) + tsc2 = dss.get(tsc.id, t1, t2) + + self.assertEqual(tsc.quality, tsc2.quality) + self.assertEqual(tsc.notes, tsc2.notes) def test_path_empty_parts(self): with HecDss(self.test_files.get_copy("Depth_Area_01.dss")) as dss: @@ -229,6 +271,7 @@ def test_path_empty_parts(self): tsc = dss.get(path) assert (len(tsc.values) > 0) + if __name__ == "__main__": unittest.main() # test_catalog() diff --git a/tests/test_irregular_timeseries.py b/tests/test_irregular_timeseries.py index bb4aede..541b688 100644 --- a/tests/test_irregular_timeseries.py +++ b/tests/test_irregular_timeseries.py @@ -16,7 +16,7 @@ class TestRegularTimeSeries(unittest.TestCase): def setUp(self) -> None: self.test_files = FileManager() - + def tearDown(self) -> None: self.test_files.cleanup() @@ -26,12 +26,15 @@ def test_irregular_timeseries_read(self): """ path = "/irregular-time-series/FAIR OAKS CA/FLOW-ANNUAL PEAK/01Jan1900/IR-Century/USGS/" with HecDss(self.test_files.get_copy("examples-all-data-types.dss")) as dss: - irts = dss.get(path) - assert (113 == irts.get_length()), f"irts.get_length() should be 113. is {irts.get_length()}" + irts = dss.get(path) + assert (113 == irts.get_length() + ), f"irts.get_length() should be 113. is {irts.get_length()}" assert (39000 == irts.values[18]), f"irts.values[18] should be 39000. is {irts.values[18]}" - assert ("1924-02-09 00:00:00" == str(irts.times[19])), f"irts.times[19] should be '1924-02-09 00:00:00'. is {irts.times[19]}" + assert ("1924-02-09 00:00:00" + == str(irts.times[19])), f"irts.times[19] should be '1924-02-09 00:00:00'. is {irts.times[19]}" assert ("CFS" == irts.units), f"irts.units should be 'CFS'. is {irts.units}" - assert ("INST-VAL" == irts.data_type), f"irts.data_type should be 'INST-VAL'. is {irts.data_type}" + assert ("INST-VAL" + == irts.data_type), f"irts.data_type should be 'INST-VAL'. is {irts.data_type}" assert (0 == irts.interval), f"irts.interval should be 0. is {irts.interval}" def test_is_irregular_timeseries_type(self): @@ -41,7 +44,8 @@ def test_is_irregular_timeseries_type(self): path = "/irregular-time-series/FAIR OAKS CA/FLOW-ANNUAL PEAK/01Jan1900/IR-Century/USGS/" with HecDss(self.test_files.get_copy("examples-all-data-types.dss")) as dss: irts = dss.get(path) - assert (type(irts) is IrregularTimeSeries), f"irts should be type IrregularTimeSeries. is {type(irts)}" + assert ( + type(irts) is IrregularTimeSeries), f"irts should be type IrregularTimeSeries. is {type(irts)}" def test_irregular_timeseries_create_store(self): """ @@ -52,13 +56,14 @@ def test_irregular_timeseries_create_store(self): irpath = "/irregular-time-series/GAPT/FLOW//IR-Day/forecast6/" irts = IrregularTimeSeries() - dates = [datetime.today().replace(second=0, microsecond=0)+(i * timedelta(hours=2)) for i in range(15)] + dates = [datetime.today().replace(second=0, microsecond=0) + + (i * timedelta(hours=2)) for i in range(15)] dates[1] = dates[1] - timedelta(seconds=60) - irts = IrregularTimeSeries.create(times=dates, values=list(range(15)), data_type="INST-VAL", path=irpath) + irts = IrregularTimeSeries.create(times=dates, values=list( + range(15)), data_type="INST-VAL", path=irpath) dss.put(irts) - def test_irregular_timeseries_create_store_read(self): """ Generates a IrregularTimeSeries object then stores data on disk and read result @@ -67,17 +72,84 @@ def test_irregular_timeseries_create_store_read(self): with HecDss(self.test_files.get_copy(file)) as dss: irpath = "/irregular-time-series/GAPT/FLOW//IR-Day/forecast6/" - dates = [datetime.today().replace(microsecond=0) + (i * timedelta(hours=2)) for i in range(15)] + dates = [datetime.today().replace(microsecond=0) + (i * timedelta(hours=2)) + for i in range(15)] dates[1] = dates[1] - timedelta(seconds=60) - irts = IrregularTimeSeries.create(times=dates, values=list(range(15)), data_type="INST-VAL", path=irpath) + irts = IrregularTimeSeries.create(times=dates, values=list( + range(15)), data_type="INST-VAL", path=irpath) dss.put(irts) read_irts = dss.get(irpath) - assert (read_irts.times == irts.times), f"saved and read times should be identical saved times are" \ - f" \n{irts.times}\n and read times are \n{read_irts.times}" + f" \n{irts.times}\n and read times are \n{read_irts.times}" + + def test_irregular_timeseries_create_store_read_with_notes(self): + """ + Notes written on an IrregularTimeSeries survive a put/get round trip. + """ + file = "examples-all-data-types.dss" + with HecDss(self.test_files.get_copy(file)) as dss: + + irpath = "/irregular-time-series/GAPT/FLOW//IR-Day/forecast7/" + dates = [datetime.today().replace(microsecond=0) + (i * timedelta(hours=2)) + for i in range(5)] + notes = ["", "storm event", "", "gage malfunction", ""] + irts = IrregularTimeSeries.create(times=dates, values=list(range(5)), data_type="INST-VAL", + path=irpath, notes=notes) + + dss.put(irts) + + read_irts = dss.get(irpath) + + assert (read_irts.notes == notes), f"saved and read notes should be identical. saved notes are" \ + f" \n{notes}\n and read notes are \n{read_irts.notes}" + + def test_irregular_timeseries_create_store_read_with_quality(self): + """ + Quality flags written on an IrregularTimeSeries survive a put/get round trip. + """ + file = "examples-all-data-types.dss" + with HecDss(self.test_files.get_copy(file)) as dss: + + irpath = "/irregular-time-series/GAPT/FLOW//IR-Day/forecast8/" + dates = [datetime.today().replace(microsecond=0) + (i * timedelta(hours=2)) + for i in range(5)] + quality = [0, 1, 2, 3, 4] + irts = IrregularTimeSeries.create(times=dates, values=list(range(5)), data_type="INST-VAL", + path=irpath, quality=quality) + + dss.put(irts) + + read_irts = dss.get(irpath) + + assert (read_irts.quality == quality), f"saved and read quality should be identical. saved quality is" \ + f" \n{quality}\n and read quality is \n{read_irts.quality}" + + def test_irregular_timeseries_create_store_read_with_quality_and_notes(self): + """ + Quality flags and notes written together on an IrregularTimeSeries survive a put/get round trip. + """ + file = "examples-all-data-types.dss" + with HecDss(self.test_files.get_copy(file)) as dss: + + irpath = "/irregular-time-series/GAPT/FLOW//IR-Day/forecast9/" + dates = [datetime.today().replace(microsecond=0) + (i * timedelta(hours=2)) + for i in range(5)] + quality = [0, 1, 2, 3, 4] + notes = ["", "storm event", "", "gage malfunction", ""] + irts = IrregularTimeSeries.create(times=dates, values=list(range(5)), data_type="INST-VAL", + path=irpath, quality=quality, notes=notes) + + dss.put(irts) + + read_irts = dss.get(irpath) + + assert (read_irts.quality == quality), f"saved and read quality should be identical. saved quality is" \ + f" \n{quality}\n and read quality is \n{read_irts.quality}" + assert (read_irts.notes == notes), f"saved and read notes should be identical. saved notes are" \ + f" \n{notes}\n and read notes are \n{read_irts.notes}" def test_irregular_timeseries_read_store_read(self): """ @@ -96,17 +168,17 @@ def test_irregular_timeseries_read_store_read(self): irts_modified = dss.get(path_modified) np.set_printoptions(suppress=True) assert (irts.get_length() == irts_modified.get_length()), f"irts.get_length() is not equal to irts_modified.get_length()." \ - f" irts.get_length() is {irts.get_length()}, irts_modified.get_length() is {irts_modified.get_length()}" + f" irts.get_length() is {irts.get_length()}, irts_modified.get_length() is {irts_modified.get_length()}" assert (np.array_equal(irts.values, irts_modified.values)), f"irts.values is not equal to irts_modified.values." \ - f" irts.values is {irts.values}, irts_modified.values is {irts_modified.values}" + f" irts.values is {irts.values}, irts_modified.values is {irts_modified.values}" assert (np.array_equal(irts.times, irts_modified.times)), f"irts.times is not equal to irts_modified.times." \ - f" irts.times is {irts.times}, irts_modified.times is {irts_modified.times}" + f" irts.times is {irts.times}, irts_modified.times is {irts_modified.times}" assert (irts.units == irts_modified.units), f"irts.units is not equal to irts_modified.units." \ - f" irts.units is {irts.units}, irts_modified.units is {irts_modified.units}" + f" irts.units is {irts.units}, irts_modified.units is {irts_modified.units}" assert (irts.data_type == irts_modified.data_type), f"irts.data_type is not equal to irts_modified.data_type." \ - f" irts.data_type is {irts.data_type}, irts_modified.data_type is {irts_modified.data_type}" + f" irts.data_type is {irts.data_type}, irts_modified.data_type is {irts_modified.data_type}" assert (irts.interval == irts_modified.interval), f"irts.interval is not equal to irts_modified.interval." \ - f" irts.interval is {irts.interval}, irts_modified.interval is {irts_modified.interval}" + f" irts.interval is {irts.interval}, irts_modified.interval is {irts_modified.interval}" def test_irregular_timeseries_read_modify_store_modify_read(self): """ @@ -129,17 +201,17 @@ def test_irregular_timeseries_read_modify_store_modify_read(self): irts_modified = dss.get(path_modified) np.set_printoptions(suppress=True) assert (irts.get_length() == irts_modified.get_length()), f"irts.get_length() is not equal to irts_modified.get_length()." \ - f" irts.get_length() is {irts.get_length()}, irts_modified.get_length() is {irts_modified.get_length()}" + f" irts.get_length() is {irts.get_length()}, irts_modified.get_length() is {irts_modified.get_length()}" assert (np.array_equal(irts.values, irts_modified.values)), f"irts.values is not equal to irts_modified.values." \ - f" irts.values is {irts.values}, irts_modified.values is {irts_modified.values}" + f" irts.values is {irts.values}, irts_modified.values is {irts_modified.values}" assert (np.array_equal(irts.times, irts_modified.times)), f"irts.times is not equal to irts_modified.times." \ - f" irts.times is {irts.times}, irts_modified.times is {irts_modified.times}" + f" irts.times is {irts.times}, irts_modified.times is {irts_modified.times}" assert (irts.units == irts_modified.units), f"irts.units is not equal to irts_modified.units." \ - f" irts.units is {irts.units}, irts_modified.units is {irts_modified.units}" + f" irts.units is {irts.units}, irts_modified.units is {irts_modified.units}" assert (irts.data_type == irts_modified.data_type), f"irts.data_type is not equal to irts_modified.data_type." \ - f" irts.data_type is {irts.data_type}, irts_modified.data_type is {irts_modified.data_type}" + f" irts.data_type is {irts.data_type}, irts_modified.data_type is {irts_modified.data_type}" assert (irts.interval == irts_modified.interval), f"irts.interval is not equal to irts_modified.interval." \ - f" irts.interval is {irts.interval}, irts_modified.interval is {irts_modified.interval}" + f" irts.interval is {irts.interval}, irts_modified.interval is {irts_modified.interval}" def test_irregular_timeseries_read_modify_store_read(self): """ @@ -156,17 +228,17 @@ def test_irregular_timeseries_read_modify_store_read(self): irts_modified = dss.get(path) np.set_printoptions(suppress=True) assert (irts.get_length() == irts_modified.get_length()), f"irts.get_length() is not equal to irts_modified.get_length()." \ - f" irts.get_length() is {irts.get_length()}, irts_modified.get_length() is {irts_modified.get_length()}" + f" irts.get_length() is {irts.get_length()}, irts_modified.get_length() is {irts_modified.get_length()}" assert (np.array_equal(irts.values, irts_modified.values)), f"irts.values is not equal to irts_modified.values." \ - f" irts.values is {irts.values}, irts_modified.values is {irts_modified.values}" + f" irts.values is {irts.values}, irts_modified.values is {irts_modified.values}" assert (np.array_equal(irts.times, irts_modified.times)), f"irts.times is not equal to irts_modified.times." \ - f" irts.times is {irts.times}, irts_modified.times is {irts_modified.times}" + f" irts.times is {irts.times}, irts_modified.times is {irts_modified.times}" assert (irts.units == irts_modified.units), f"irts.units is not equal to irts_modified.units." \ - f" irts.units is {irts.units}, irts_modified.units is {irts_modified.units}" + f" irts.units is {irts.units}, irts_modified.units is {irts_modified.units}" assert (irts.data_type == irts_modified.data_type), f"irts.data_type is not equal to irts_modified.data_type." \ - f" irts.data_type is {irts.data_type}, irts_modified.data_type is {irts_modified.data_type}" + f" irts.data_type is {irts.data_type}, irts_modified.data_type is {irts_modified.data_type}" assert (irts.interval == irts_modified.interval), f"irts.interval is not equal to irts_modified.interval." \ - f" irts.interval is {irts.interval}, irts_modified.interval is {irts_modified.interval}" + f" irts.interval is {irts.interval}, irts_modified.interval is {irts_modified.interval}" if __name__ == "__main__": diff --git a/tests/test_ts_csv.py b/tests/test_ts_csv.py index 29886d5..f0d9035 100644 --- a/tests/test_ts_csv.py +++ b/tests/test_ts_csv.py @@ -3,7 +3,6 @@ from unittest.mock import mock_open, patch import numpy as np - from file_manager import FileManager from hecdss import HecDss @@ -587,32 +586,6 @@ def test_to_csv_writes_empty_cell_for_missing_value(self): self.assertIn("2,01Sep2021 1200,\r\n", written) self.assertNotIn("None", written) - def test_to_csv_quality_shorter_than_values_truncates(self): - """FOOTGUN: to_csv takes the with-quality branch whenever quality is - non-empty, then zips (times, values, quality). A quality list shorter than - values makes zip stop at the shortest input, so trailing data points are - silently dropped from the CSV.""" - rts = RegularTimeSeries.create( - values=[1.0, 2.0, 3.0], - times=[ - datetime(2021, 9, 1, 6, 0), - datetime(2021, 9, 1, 12, 0), - datetime(2021, 9, 1, 18, 0), - ], - quality=[0], # only one flag for three values - units="CFS", - data_type="INST-VAL", - path="/A/B/C//6Hour/F/", - ) - mock_file = mock_open() - with patch("builtins.open", mock_file): - rts.to_csv("fake.csv", with_metadata=False) - handle = mock_file() - written = "".join(call.args[0] for call in handle.write.call_args_list) - self.assertIn("1,01Sep2021 0600,1.0,0", written) - self.assertNotIn("2,01Sep2021 1200", written) # silently dropped - self.assertNotIn("3,01Sep2021 1800", written) # silently dropped - def test_read_csv_skips_short_data_row(self): """A data row with fewer than 3 columns is malformed and skipped without raising (parity with the paired-data reader's short-row handling).""" @@ -662,6 +635,331 @@ def test_round_trip_irregular(self): self.assertEqual(result.data_type, "INST-VAL") self.assertEqual(result.id, "/A/B/C//IR-Year/F/") + # ================================================================== # + # NOTES TESTS + # ================================================================== # + + def test_to_csv_with_notes(self): + """When notes are present (no quality), header gets a 'Notes' col + and rows get the note text.""" + rts = RegularTimeSeries.create( + values=[1.0, 2.0], + times=[datetime(2021, 9, 1, 6, 0), datetime(2021, 9, 1, 12, 0)], + notes=["", "manual override"], + units="CFS", + data_type="INST-VAL", + path="/A/B/C//6Hour/F/", + ) + mock_file = mock_open() + with patch("builtins.open", mock_file): + rts.to_csv("fake.csv", with_metadata=True) + handle = mock_file() + written = "".join(call.args[0] for call in handle.write.call_args_list) + self.assertIn("Type,Date/Time,INST-VAL,Notes", written) + self.assertIn("1,01Sep2021 0600,1.0,", written) + self.assertIn("2,01Sep2021 1200,2.0,manual override", written) + + def test_read_csv_with_notes(self): + """A CSV with a Notes column (no Quality) populates rts.notes, + index-aligned with values.""" + content = ( + "Type,Date/Time,INST-VAL,Notes\n" + "1,01Sep2021 0600,10.5,\n" + "2,01Sep2021 1200,20.0,manual override\n" + ) + rts = self.read_rts_from_string(content) + self.assertEqual(rts.values.tolist(), [10.5, 20.0]) + self.assertEqual(rts.notes, ["", "manual override"]) + + def test_round_trip_with_notes(self): + """Full write->read round trip for RegularTimeSeries notes on a real + temp file.""" + path = self.test_files.create_test_file(".csv") + rts = RegularTimeSeries.create( + values=[1.0, 2.0, 3.0], + times=[ + datetime(2021, 9, 1, 6, 0), + datetime(2021, 9, 1, 12, 0), + datetime(2021, 9, 1, 18, 0), + ], + notes=["", "storm event", ""], + units="CFS", + data_type="INST-VAL", + path="/A/B/C//6Hour/F/", + ) + rts.to_csv(path, with_metadata=True) + result = RegularTimeSeries.read_csv(path) + + self.assertEqual(result.values.tolist(), [1.0, 2.0, 3.0]) + self.assertEqual(result.notes, ["", "storm event", ""]) + + def test_to_csv_with_notes_without_metadata(self): + """Like Quality, the Notes column is driven purely by len(series.notes), + independent of with_metadata -- it must still appear when + with_metadata=False.""" + rts = RegularTimeSeries.create( + values=[1.0, 2.0], + times=[datetime(2021, 9, 1, 6, 0), datetime(2021, 9, 1, 12, 0)], + notes=["", "manual override"], + units="CFS", + data_type="INST-VAL", + path="/A/B/C//6Hour/F/", + ) + mock_file = mock_open() + with patch("builtins.open", mock_file): + rts.to_csv("fake.csv", with_metadata=False) + handle = mock_file() + written = "".join(call.args[0] for call in handle.write.call_args_list) + self.assertNotIn("Units", written) + self.assertIn("Type,Date/Time,INST-VAL,Notes", written) + self.assertIn("2,01Sep2021 1200,2.0,manual override", written) + + def test_round_trip_with_notes_without_metadata(self): + """Without metadata, units/data_type come back empty (existing + behavior) but notes still survive the round trip.""" + path = self.test_files.create_test_file(".csv") + rts = RegularTimeSeries.create( + values=[1.0, 2.0], + times=[datetime(2021, 9, 1, 6, 0), datetime(2021, 9, 1, 12, 0)], + notes=["", "manual override"], + units="CFS", + data_type="INST-VAL", + path="/A/B/C//6Hour/F/", + ) + rts.to_csv(path, with_metadata=False) + result = RegularTimeSeries.read_csv(path) + + self.assertEqual(result.units, "") + self.assertEqual(result.values.tolist(), [1.0, 2.0]) + self.assertEqual(result.notes, ["", "manual override"]) + + def test_to_csv_with_quality_and_notes(self): + """When both quality and notes are present, the header/rows carry + both columns with Quality before Notes.""" + rts = RegularTimeSeries.create( + values=[1.0, 2.0], + times=[datetime(2021, 9, 1, 6, 0), datetime(2021, 9, 1, 12, 0)], + quality=[0, 5], + notes=["", "manual override"], + units="CFS", + data_type="INST-VAL", + path="/A/B/C//6Hour/F/", + ) + mock_file = mock_open() + with patch("builtins.open", mock_file): + rts.to_csv("fake.csv", with_metadata=True) + handle = mock_file() + written = "".join(call.args[0] for call in handle.write.call_args_list) + self.assertIn("Type,Date/Time,INST-VAL,Quality,Notes", written) + self.assertIn("1,01Sep2021 0600,1.0,0,", written) + self.assertIn("2,01Sep2021 1200,2.0,5,manual override", written) + + def test_round_trip_with_quality_and_notes(self): + """Quality and notes both survive a real write->read round trip + together, independently of each other.""" + path = self.test_files.create_test_file(".csv") + rts = RegularTimeSeries.create( + values=[1.0, 2.0, 3.0], + times=[ + datetime(2021, 9, 1, 6, 0), + datetime(2021, 9, 1, 12, 0), + datetime(2021, 9, 1, 18, 0), + ], + quality=[0, 5, 10], + notes=["", "storm event", ""], + units="CFS", + data_type="INST-VAL", + path="/A/B/C//6Hour/F/", + ) + rts.to_csv(path, with_metadata=True) + result = RegularTimeSeries.read_csv(path) + + self.assertEqual(result.values.tolist(), [1.0, 2.0, 3.0]) + self.assertEqual(result.quality, [0, 5, 10]) + self.assertEqual(result.notes, ["", "storm event", ""]) + + def test_to_csv_with_notes_but_no_quality_omits_quality_column(self): + """When notes are present but quality is empty, only the Notes + column should appear -- no stray 'Quality' header or column.""" + rts = RegularTimeSeries.create( + values=[1.0], + times=[datetime(2021, 9, 1, 6, 0)], + notes=["manual override"], + units="CFS", + data_type="INST-VAL", + path="/A/B/C//6Hour/F/", + ) + mock_file = mock_open() + with patch("builtins.open", mock_file): + rts.to_csv("fake.csv", with_metadata=False) + handle = mock_file() + written = "".join(call.args[0] for call in handle.write.call_args_list) + self.assertNotIn("Quality", written) + self.assertIn("Type,Date/Time,INST-VAL,Notes", written) + self.assertIn("1,01Sep2021 0600,1.0,manual override", written) + + def test_read_csv_notes_and_quality_together(self): + """Hand-typed CSV with both Quality and Notes columns present + parses both independently and correctly, in the assumed + Quality-then-Notes column order.""" + content = ( + "Type,Date/Time,INST-VAL,Quality,Notes\n" + "1,05Nov2004 0200,8,0,\n" + "2,05Nov2004 0300,9,1,estimated\n" + ) + rts = self.read_rts_from_string(content) + self.assertEqual(rts.values.tolist(), [8, 9]) + self.assertEqual(rts.quality, [0, 1]) + self.assertEqual(rts.notes, ["", "estimated"]) + + def test_read_csv_with_partial_notes(self): + """A data row missing its trailing Notes cell (even though the header + declares a Notes column) should default that entry to an empty + string rather than raising -- parity with how a missing Quality + cell already defaults to 0 in test_read_csv_with_partial_quality.""" + content = ( + "Type,Date/Time,INST-VAL,Notes\n" + "1,05Nov2004 0200,8,manual override\n" + "2,05Nov2004 0300,9\n" # missing notes cell! + "3,05Nov2004 0400,10,estimated\n" + ) + rts = self.read_rts_from_string(content) + self.assertEqual(rts.values.tolist(), [8, 9, 10]) + self.assertEqual(rts.notes[0], "manual override") + self.assertEqual(rts.notes[1], "") + self.assertEqual(rts.notes[2], "estimated") + + def test_read_csv_notes_containing_comma_round_trips(self): + """A note containing a comma is CSV-quoted on write and comes back + intact on read (parity with PairedData's quoted-label test).""" + path = self.test_files.create_test_file(".csv") + rts = RegularTimeSeries.create( + values=[1.0], + times=[datetime(2021, 9, 1, 6, 0)], + notes=["flow, cfs, estimated"], + units="CFS", + data_type="INST-VAL", + path="/A/B/C//6Hour/F/", + ) + rts.to_csv(path, with_metadata=True) + result = RegularTimeSeries.read_csv(path) + self.assertEqual(result.notes, ["flow, cfs, estimated"]) + + def test_to_csv_writes_empty_cell_for_empty_note(self): + """An empty-string note writes as an empty cell -- never the literal + text 'None' -- even though other entries have real note text.""" + rts = RegularTimeSeries.create( + values=[1.0, 2.0], + times=[datetime(2021, 9, 1, 6, 0), datetime(2021, 9, 1, 12, 0)], + notes=["", "manual override"], + units="CFS", + data_type="INST-VAL", + path="/A/B/C//6Hour/F/", + ) + mock_file = mock_open() + with patch("builtins.open", mock_file): + rts.to_csv("fake.csv", with_metadata=False) + handle = mock_file() + written = "".join(call.args[0] for call in handle.write.call_args_list) + self.assertIn("1,01Sep2021 0600,1.0,\r\n", written) + self.assertNotIn("None", written) + + # ---- IRREGULAR TIME SERIES NOTES TESTS ---- # + + def test_irregular_to_csv_with_notes(self): + """Basic structure test for IrregularTimeSeries to_csv with notes.""" + its = IrregularTimeSeries.create( + values=[10.5, 20.0, 42.0], + times=[ + datetime(2021, 9, 1, 0, 0), + datetime(2021, 9, 2, 0, 0), + datetime(2021, 9, 4, 0, 0), + ], + notes=["", "storm event", ""], + units="CFS", + data_type="INST-VAL", + path="/A/B/C//E/F/", + ) + mock_file = mock_open() + with patch("builtins.open", mock_file): + its.to_csv("fake_path.csv", with_metadata=True) + handle = mock_file() + written_data = "".join(call.args[0] for call in handle.write.call_args_list) + self.assertIn("Type,Date/Time,INST-VAL,Notes", written_data) + self.assertIn("1,01Sep2021 0000,10.5,", written_data) + self.assertIn("2,02Sep2021 0000,20.0,storm event", written_data) + self.assertIn("3,04Sep2021 0000,42.0,", written_data) + + def test_irregular_read_csv_with_notes(self): + """Hand-typed CSV with a Notes column populates its.notes for + IrregularTimeSeries.""" + content = ( + "E,,,IR-Year\n" + "Type,Date/Time,INST-VAL,Notes\n" + "1,01Sep2021 0000,10.5,\n" + "2,02Sep2021 0000,20.0,storm event\n" + "3,04Sep2021 0000,20.0,\n" + ) + its = self.read_its_from_string(content) + self.assertEqual(its.values.tolist(), [10.5, 20.0, 20.0]) + self.assertEqual(its.notes, ["", "storm event", ""]) + + def test_round_trip_irregular_with_notes(self): + """Full write->read round trip for IrregularTimeSeries notes on a + real temp file, with irregular (non-uniform) time gaps.""" + path = self.test_files.create_test_file(".csv") + its = IrregularTimeSeries.create( + values=[10.5, 20.0, 42.0], + times=[datetime(2021, 9, 1), datetime(2021, 9, 5), datetime(2021, 9, 20)], + notes=["", "storm event", ""], + units="CFS", + data_type="INST-VAL", + path="/A/B/C//IR-Year/F/", + ) + its.to_csv(path, with_metadata=True) + result = IrregularTimeSeries.read_csv(path) + self.assertEqual(result.values.tolist(), [10.5, 20.0, 42.0]) + self.assertEqual(result.notes, ["", "storm event", ""]) + + def test_irregular_to_csv_with_notes_without_metadata(self): + """Notes column appears for IrregularTimeSeries regardless of + with_metadata, same as RegularTimeSeries.""" + its = IrregularTimeSeries.create( + values=[10.5, 20.0], + times=[datetime(2021, 9, 1, 0, 0), datetime(2021, 9, 2, 0, 0)], + notes=["", "storm event"], + units="CFS", + data_type="INST-VAL", + path="/A/B/C//E/F/", + ) + mock_file = mock_open() + with patch("builtins.open", mock_file): + its.to_csv("fake.csv", with_metadata=False) + handle = mock_file() + written = "".join(call.args[0] for call in handle.write.call_args_list) + self.assertNotIn("Units", written) + self.assertIn("Type,Date/Time,INST-VAL,Notes", written) + self.assertIn("2,02Sep2021 0000,20.0,storm event", written) + + def test_round_trip_irregular_with_quality_and_notes(self): + """Quality and notes both survive an IrregularTimeSeries write->read + round trip together.""" + path = self.test_files.create_test_file(".csv") + its = IrregularTimeSeries.create( + values=[10.5, 20.0, 42.0], + times=[datetime(2021, 9, 1), datetime(2021, 9, 5), datetime(2021, 9, 20)], + quality=[0, 5, 10], + notes=["", "storm event", ""], + units="CFS", + data_type="INST-VAL", + path="/A/B/C//IR-Year/F/", + ) + its.to_csv(path, with_metadata=True) + result = IrregularTimeSeries.read_csv(path) + self.assertEqual(result.values.tolist(), [10.5, 20.0, 42.0]) + self.assertEqual(result.quality, [0, 5, 10]) + self.assertEqual(result.notes, ["", "storm event", ""]) + if __name__ == "__main__": unittest.main()