From 50889cc67765c2c1b6f01298ce8ca245dc426837 Mon Sep 17 00:00:00 2001 From: Melissa Linkert Date: Tue, 12 Aug 2025 16:23:03 -0500 Subject: [PATCH 01/21] Add case for timestamps with timezone offset --- isyntax2raw/__init__.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/isyntax2raw/__init__.py b/isyntax2raw/__init__.py index c2ed935..3d36762 100644 --- a/isyntax2raw/__init__.py +++ b/isyntax2raw/__init__.py @@ -327,12 +327,17 @@ def acquisition_datetime(self): timestamp = str(pe_in.DICOM_ACQUISITION_DATETIME).strip() else: timestamp = pe_in.acquisition_datetime.strip() - # older files store the date time in YYYYmmddHHMMSS.ffffff format + # older files store the date time in YYYYmmddHHMMSS.ffffff format, + # optionally with a timezone offset appended # newer files use ISO 8601, i.e. YYYY-mm-ddTHH:mm:ss # other timestamp formats may be used in the future try: - # Handle "special" isyntax date/time format - return datetime.strptime(timestamp, "%Y%m%d%H%M%S.%f") + try: + # Handle "special" isyntax date/time format + return datetime.strptime(timestamp, "%Y%m%d%H%M%S.%f") + except ValueError: + # Handle "special" isyntax date/time format with timezone + return datetime.strptime(timestamp, "%Y%m%d%H%M%S.%f%z") except ValueError: # Handle other date/time formats (such as ISO 8601) return parse(timestamp) From bfb9d680a665d5eb760b9eb02b07f3a7b8b605ee Mon Sep 17 00:00:00 2001 From: Melissa Linkert Date: Tue, 2 Sep 2025 15:42:47 -0500 Subject: [PATCH 02/21] Allow lossy image compression method and ratio to be missing or empty --- isyntax2raw/__init__.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/isyntax2raw/__init__.py b/isyntax2raw/__init__.py index 3d36762..2c618a9 100644 --- a/isyntax2raw/__init__.py +++ b/isyntax2raw/__init__.py @@ -262,13 +262,26 @@ def get_image_metadata_sdk_v2(self, image_no): image_type = self.image_type(image_no) view = img.source_view image_scale_factor = view.scale + + compression_method = None + try: + compression_method = img.lossy_image_compression_method + except RuntimeError: + log.warn("could not read lossy_image_compression_method") + + compression_ratio = None + try: + compression_ratio = img.lossy_image_compression_ratio + except RuntimeError: + log.warn("could not read lossy_image_compression_ratio") + image_metadata = { "Image type": image_type, "Lossy image compression method": - img.lossy_image_compression_method, + compression_method, "Lossy image compression ratio": - img.lossy_image_compression_ratio, + compression_ratio, "Image dimension names": view.dimension_names, "Image dimension types": From 7b617cc9157b1145be3c519003320956b2273e4f Mon Sep 17 00:00:00 2001 From: Melissa Linkert Date: Thu, 11 Sep 2025 17:30:09 -0500 Subject: [PATCH 03/21] First pass at 16-bit support --- isyntax2raw/__init__.py | 38 ++++++++++++++++++++++---- isyntax2raw/resources/ome_template.xml | 4 +-- 2 files changed, 34 insertions(+), 8 deletions(-) diff --git a/isyntax2raw/__init__.py b/isyntax2raw/__init__.py index 2c618a9..9240840 100644 --- a/isyntax2raw/__init__.py +++ b/isyntax2raw/__init__.py @@ -225,8 +225,10 @@ def get_image_metadata_sdk_v1(self, image_no): self.pixel_size_y = img.IMAGE_SCALE_FACTOR[1] view = pe_in.SourceView() - image_metadata["Bits allocated"] = view.bitsAllocated() - image_metadata["Bits stored"] = view.bitsStored() + self.bits_per_pixel = view.bitsAllocated() + image_metadata["Bits allocated"] = self.bits_per_pixel + self.bits_stored = view.bitsStored() + image_metadata["Bits stored"] = self.bits_stored image_metadata["High bit"] = view.highBit() image_metadata["Pixel representation"] = \ view.pixelRepresentation() @@ -303,8 +305,10 @@ def get_image_metadata_sdk_v2(self, image_no): self.pixel_size_x = image_scale_factor[0] self.pixel_size_y = image_scale_factor[1] - image_metadata["Bits allocated"] = view.bits_allocated - image_metadata["Bits stored"] = view.bits_stored + self.bits_per_pixel = view.bits_allocated + image_metadata["Bits allocated"] = self.bits_per_pixel + self.bits_stored = view.bits_stored + image_metadata["Bits stored"] = self.bits_stored image_metadata["High bit"] = view.high_bit image_metadata["Pixel representation"] = \ view.pixel_representation @@ -483,6 +487,8 @@ def write_metadata_xml(self, metadata_file): 'pixels': { 'sizeX': int(self.size_x), 'sizeY': int(self.size_y), + 'type': get_pixel_type(int(self.bits_per_pixel)), + 'bits_stored': int(self.significant_bits), 'physicalSizeX': self.pixel_size_x, 'physicalSizeY': self.pixel_size_y } @@ -519,6 +525,22 @@ def write_metadata(self): ) self.write_metadata_xml(metadata_file) + def get_pixel_type(bits): + '''get the OME-XML pixel type string from the bits per pixel''' + if bits == 8: + return 'uint8' + elif bits == 16: + return 'uint16' + raise ValueError("Allocated bits not supported: %d" % bits) + + def get_data_type(bits): + '''get numpy data type from bits per pixel''' + if bits == 8: + return numpy.uint8 + elif bits == 16: + return numpy.uint16 + raise ValueError("Allocated bits not supported: %d" % bits) + def get_size(self, dim_range): '''calculate the length in pixels of a dimension''' v = (dim_range[2] - dim_range[0]) / dim_range[1] @@ -589,7 +611,8 @@ def create_tile_directory(self, series, resolution, width, height): self.zarr_group.create_dataset( "%s/%s" % (str(series), str(resolution)), shape=(1, 3, 1, height, width), - chunks=(1, 1, 1, self.tile_height, self.tile_width), dtype='B' + chunks=(1, 1, 1, self.tile_height, self.tile_width), + dtype=get_data_type(self.bits_per_pixel) ) def make_planar(self, pixels, tile_width, tile_height): @@ -715,7 +738,10 @@ def write_tile( ) height = int(height) pixel_buffer_size = width * height * 3 - pixels = np.empty(pixel_buffer_size, dtype='B') + pixels = np.empty( + pixel_buffer_size, + dtype=get_data_type(self.bits_per_pixel) + ) patch_id = patch_ids.pop(regions.index(region)) x_start, y_start = patch_id x_start *= self.tile_width diff --git a/isyntax2raw/resources/ome_template.xml b/isyntax2raw/resources/ome_template.xml index 53a84c0..1f733e7 100644 --- a/isyntax2raw/resources/ome_template.xml +++ b/isyntax2raw/resources/ome_template.xml @@ -12,11 +12,11 @@ PhysicalSizeX="${image['pixels']['physicalSizeX']}" PhysicalSizeXUnit="µm" PhysicalSizeY="${image['pixels']['physicalSizeY']}" - PhysicalSizeYUnit="µm" SignificantBits="8" + PhysicalSizeYUnit="µm" SignificantBits="${image['pixels']['bits_stored']}" SizeC="3" SizeT="1" SizeX="${image['pixels']['sizeX']}" SizeY="${image['pixels']['sizeY']}" - SizeZ="1" Type="uint8"> + SizeZ="1" Type="${image['pixels']['type']}"> From 78af889bd66209be2fd6623fafc5d43d4ec0a47f Mon Sep 17 00:00:00 2001 From: Melissa Linkert Date: Thu, 11 Sep 2025 17:36:44 -0500 Subject: [PATCH 04/21] Fix build errors --- isyntax2raw/__init__.py | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/isyntax2raw/__init__.py b/isyntax2raw/__init__.py index 9240840..be635d6 100644 --- a/isyntax2raw/__init__.py +++ b/isyntax2raw/__init__.py @@ -414,6 +414,22 @@ def num_images(self): else: return pe_in.num_images + def get_pixel_type(self, bits): + '''get the OME-XML pixel type string from the bits per pixel''' + if bits == 8: + return 'uint8' + elif bits == 16: + return 'uint16' + raise ValueError("Allocated bits not supported: %d" % bits) + + def get_data_type(self, bits): + '''get numpy data type from bits per pixel''' + if bits == 8: + return np.uint8 + elif bits == 16: + return np.uint16 + raise ValueError("Allocated bits not supported: %d" % bits) + def wait_any(self, regions): if self.sdk_v1: return self.pixel_engine.waitAny(regions) @@ -487,7 +503,7 @@ def write_metadata_xml(self, metadata_file): 'pixels': { 'sizeX': int(self.size_x), 'sizeY': int(self.size_y), - 'type': get_pixel_type(int(self.bits_per_pixel)), + 'type': self.get_pixel_type(int(self.bits_per_pixel)), 'bits_stored': int(self.significant_bits), 'physicalSizeX': self.pixel_size_x, 'physicalSizeY': self.pixel_size_y @@ -525,22 +541,6 @@ def write_metadata(self): ) self.write_metadata_xml(metadata_file) - def get_pixel_type(bits): - '''get the OME-XML pixel type string from the bits per pixel''' - if bits == 8: - return 'uint8' - elif bits == 16: - return 'uint16' - raise ValueError("Allocated bits not supported: %d" % bits) - - def get_data_type(bits): - '''get numpy data type from bits per pixel''' - if bits == 8: - return numpy.uint8 - elif bits == 16: - return numpy.uint16 - raise ValueError("Allocated bits not supported: %d" % bits) - def get_size(self, dim_range): '''calculate the length in pixels of a dimension''' v = (dim_range[2] - dim_range[0]) / dim_range[1] @@ -612,7 +612,7 @@ def create_tile_directory(self, series, resolution, width, height): "%s/%s" % (str(series), str(resolution)), shape=(1, 3, 1, height, width), chunks=(1, 1, 1, self.tile_height, self.tile_width), - dtype=get_data_type(self.bits_per_pixel) + dtype=self.get_data_type(self.bits_per_pixel) ) def make_planar(self, pixels, tile_width, tile_height): @@ -740,7 +740,7 @@ def write_tile( pixel_buffer_size = width * height * 3 pixels = np.empty( pixel_buffer_size, - dtype=get_data_type(self.bits_per_pixel) + dtype=self.get_data_type(self.bits_per_pixel) ) patch_id = patch_ids.pop(regions.index(region)) x_start, y_start = patch_id From e60a177b6580af1187b34ad9c9127ac5cc5035b4 Mon Sep 17 00:00:00 2001 From: Melissa Linkert Date: Thu, 11 Sep 2025 18:16:24 -0500 Subject: [PATCH 05/21] Pin pillow for testing --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 86b8171..cc44210 100644 --- a/setup.py +++ b/setup.py @@ -81,7 +81,7 @@ def read(fname): setup_requires=['flake8'], install_requires=[ 'click==7.0', - 'pillow>=7.1.0', + 'pillow==7.1.0', 'numpy==1.17.3', 'zarr==2.8.1', 'kajiki==0.8.2', From 1d857eb9b0e42a73e10343d447e0509089f8fda1 Mon Sep 17 00:00:00 2001 From: Melissa Linkert Date: Thu, 11 Sep 2025 18:22:35 -0500 Subject: [PATCH 06/21] Fix variable name --- isyntax2raw/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/isyntax2raw/__init__.py b/isyntax2raw/__init__.py index be635d6..b9a3b41 100644 --- a/isyntax2raw/__init__.py +++ b/isyntax2raw/__init__.py @@ -504,7 +504,7 @@ def write_metadata_xml(self, metadata_file): 'sizeX': int(self.size_x), 'sizeY': int(self.size_y), 'type': self.get_pixel_type(int(self.bits_per_pixel)), - 'bits_stored': int(self.significant_bits), + 'bits_stored': int(self.bits_stored), 'physicalSizeX': self.pixel_size_x, 'physicalSizeY': self.pixel_size_y } From 1ea748a4c1d4b228bcdbe398514de0c046e62250 Mon Sep 17 00:00:00 2001 From: Melissa Linkert Date: Thu, 11 Sep 2025 18:22:52 -0500 Subject: [PATCH 07/21] Increase buffer size and use byte type Keeping the same size but with the correct type doesn't seem to work. --- isyntax2raw/__init__.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/isyntax2raw/__init__.py b/isyntax2raw/__init__.py index b9a3b41..b4681d4 100644 --- a/isyntax2raw/__init__.py +++ b/isyntax2raw/__init__.py @@ -738,9 +738,12 @@ def write_tile( ) height = int(height) pixel_buffer_size = width * height * 3 + if self.bits_per_pixel is not None: + pixel_buffer_size = pixel_buffer_size * (self.bits_per_pixel / 8) + pixels = np.empty( pixel_buffer_size, - dtype=self.get_data_type(self.bits_per_pixel) + dtype='B' ) patch_id = patch_ids.pop(regions.index(region)) x_start, y_start = patch_id From 60879160920330d1a21322dfd440733ff00d997c Mon Sep 17 00:00:00 2001 From: Melissa Linkert Date: Thu, 11 Sep 2025 18:45:51 -0500 Subject: [PATCH 08/21] Fix array reshaping --- isyntax2raw/__init__.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/isyntax2raw/__init__.py b/isyntax2raw/__init__.py index b4681d4..80e3a5e 100644 --- a/isyntax2raw/__init__.py +++ b/isyntax2raw/__init__.py @@ -616,6 +616,9 @@ def create_tile_directory(self, series, resolution, width, height): ) def make_planar(self, pixels, tile_width, tile_height): + if self.bits_per_pixel is not None: + pixels.dtype = self.get_data_type(self.bits_per_pixel) + r = pixels[0::3] g = pixels[1::3] b = pixels[2::3] @@ -742,7 +745,7 @@ def write_tile( pixel_buffer_size = pixel_buffer_size * (self.bits_per_pixel / 8) pixels = np.empty( - pixel_buffer_size, + int(pixel_buffer_size), dtype='B' ) patch_id = patch_ids.pop(regions.index(region)) From 737510681b86771ac6873ce6f6290a8f4e5aaa5d Mon Sep 17 00:00:00 2001 From: Melissa Linkert Date: Thu, 11 Sep 2025 21:02:01 -0500 Subject: [PATCH 09/21] Fix flake8 error --- isyntax2raw/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/isyntax2raw/__init__.py b/isyntax2raw/__init__.py index 80e3a5e..ddaca67 100644 --- a/isyntax2raw/__init__.py +++ b/isyntax2raw/__init__.py @@ -742,7 +742,8 @@ def write_tile( height = int(height) pixel_buffer_size = width * height * 3 if self.bits_per_pixel is not None: - pixel_buffer_size = pixel_buffer_size * (self.bits_per_pixel / 8) + bpp = self.bits_per_pixel / 8 + pixel_buffer_size = pixel_buffer_size * bpp pixels = np.empty( int(pixel_buffer_size), From eb6e650d36dc6357418775620980fd91ace51ec1 Mon Sep 17 00:00:00 2001 From: Melissa Linkert Date: Fri, 12 Sep 2025 10:53:58 -0500 Subject: [PATCH 10/21] Attempt to handle mismatch in image size vs metadata dimensions for labels --- isyntax2raw/__init__.py | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/isyntax2raw/__init__.py b/isyntax2raw/__init__.py index ddaca67..512e7a6 100644 --- a/isyntax2raw/__init__.py +++ b/isyntax2raw/__init__.py @@ -556,11 +556,13 @@ def get_size(self, dim_range): def write_label_image(self): '''write the label image (if present) as a JPEG file''' - self.write_image_type("LABELIMAGE", 1) + self.write_image_type("LABELIMAGE", 1, + self.label_x, self.label_y) def write_macro_image(self): '''write the macro image (if present) as a JPEG file''' - self.write_image_type("MACROIMAGE", 2) + self.write_image_type("MACROIMAGE", 2, + self.macro_x, self.macro_y) def find_image_type(self, image_type): '''look up a given image type in the pixel engine''' @@ -570,7 +572,7 @@ def find_image_type(self, image_type): return pe_in[index] return None - def write_image_type(self, image_type, series): + def write_image_type(self, image_type, series, img_x, img_y): '''write an image of the specified type''' image = self.find_image_type(image_type) if image is not None: @@ -578,15 +580,26 @@ def write_image_type(self, image_type, series): # pixels are JPEG compressed, need to decompress first img = Image.open(BytesIO(pixels)) - width = img.width - height = img.height - self.create_tile_directory(series, 0, width, height) + # dimensions may be different + # use the metadata dimensions, but warn on mismatch + # the label image in particular may encounter this case, + # with the actual stored image being much smaller than + # all metadata indicates (possibly due to deidentification) + if img.width != img_x: + log.warn("width %d does not match metadata %d" % + (img.width, img_x)) + + if img.height != img_y: + log.warn("height %d does not match metadata %d" % + (img.height, img_y)) + + self.create_tile_directory(series, 0, img_x, img_y) tile = self.zarr_group["%d/0" % series] tile.attrs['image type'] = image_type for channel in range(0, 3): band = np.array(img.getdata(band=channel)) - band.shape = (height, width) + band.shape = (img.height, img.width) tile[0, channel, 0] = band self.write_image_metadata(range(1), series) From 8bbf7ea4eeee07ea42cf0362442b7845c78c9f00 Mon Sep 17 00:00:00 2001 From: Melissa Linkert Date: Fri, 12 Sep 2025 11:13:13 -0500 Subject: [PATCH 11/21] Fix array padding, don't allow label images larger than metadata dims --- isyntax2raw/__init__.py | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/isyntax2raw/__init__.py b/isyntax2raw/__init__.py index 512e7a6..5b1651c 100644 --- a/isyntax2raw/__init__.py +++ b/isyntax2raw/__init__.py @@ -582,24 +582,41 @@ def write_image_type(self, image_type, series, img_x, img_y): img = Image.open(BytesIO(pixels)) # dimensions may be different - # use the metadata dimensions, but warn on mismatch + # if image is smaller, use the metadata dimensions and warn # the label image in particular may encounter this case, # with the actual stored image being much smaller than # all metadata indicates (possibly due to deidentification) - if img.width != img_x: + # + # if the image is larger, error, as this is unexpected + if img.width < img_x: log.warn("width %d does not match metadata %d" % (img.width, img_x)) - if img.height != img_y: + if img.height < img_y: log.warn("height %d does not match metadata %d" % (img.height, img_y)) + if img.width > img_x: + raise ValueError("width %d does not match metadata %d" % + (img.width, img_x)) + + if img.height > img_y: + raise ValueError("height %d does not match metadata %d" % + (img.height, img_y)) + self.create_tile_directory(series, 0, img_x, img_y) tile = self.zarr_group["%d/0" % series] tile.attrs['image type'] = image_type for channel in range(0, 3): band = np.array(img.getdata(band=channel)) band.shape = (img.height, img.width) + + # if the image is smaller than metadata dimensions, + # pad to the correct size + height_diff = img_y - img.height + width_diff = img_x - img.width + band = np.pad(band, [(0, height_diff), (0, width_diff)]) + tile[0, channel, 0] = band self.write_image_metadata(range(1), series) From 794feaa981b6db5663dbd2fb3f074fd7e57930f9 Mon Sep 17 00:00:00 2001 From: Melissa Linkert Date: Fri, 12 Sep 2025 16:24:46 -0500 Subject: [PATCH 12/21] Fix pad width types --- isyntax2raw/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/isyntax2raw/__init__.py b/isyntax2raw/__init__.py index 5b1651c..71ac065 100644 --- a/isyntax2raw/__init__.py +++ b/isyntax2raw/__init__.py @@ -613,8 +613,8 @@ def write_image_type(self, image_type, series, img_x, img_y): # if the image is smaller than metadata dimensions, # pad to the correct size - height_diff = img_y - img.height - width_diff = img_x - img.width + height_diff = int(img_y - img.height) + width_diff = int(img_x - img.width) band = np.pad(band, [(0, height_diff), (0, width_diff)]) tile[0, channel, 0] = band From 8dfea83c11dd2c3ea949ac3c327374d1a56b887d Mon Sep 17 00:00:00 2001 From: Melissa Linkert Date: Fri, 12 Sep 2025 17:01:19 -0500 Subject: [PATCH 13/21] First try at filtering 16 bit data --- isyntax2raw/__init__.py | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/isyntax2raw/__init__.py b/isyntax2raw/__init__.py index 71ac065..79db339 100644 --- a/isyntax2raw/__init__.py +++ b/isyntax2raw/__init__.py @@ -35,6 +35,9 @@ # version of the Zarr layout LAYOUT_VERSION = 3 +# name of filter to convert 16 bit to 8 bit +FILTER_16_TO_8 = "Linear16ToSRGB8" + class MaxQueuePool(object): """This Class wraps a concurrent.futures.Executor @@ -258,11 +261,17 @@ def get_image_metadata_sdk_v1(self, image_no): self.macro_y = self.get_size(img.IMAGE_DIMENSION_RANGES[1]) + 1 return image_metadata + def get_view(self, img): + view = img.source_view + user_view = view.add_user_view() + user_view.add_filter(FILTER_16_TO_8) + return user_view + def get_image_metadata_sdk_v2(self, image_no): pe_in = self.pixel_engine["in"] img = pe_in[image_no] image_type = self.image_type(image_no) - view = img.source_view + view = self.get_view(img) image_scale_factor = view.scale compression_method = None @@ -371,7 +380,7 @@ def data_envelopes(self, image, resolution): if self.sdk_v1: return pe_in.SourceView().dataEnvelopes(resolution) else: - return image.source_view.data_envelopes(resolution) + return self.get_view(image).data_envelopes(resolution) def derivation_description(self): pe_in = self.pixel_engine["in"] @@ -385,7 +394,7 @@ def dimension_ranges(self, image, resolution): if self.sdk_v1: return pe_in.SourceView().dimensionRanges(resolution) else: - return image.source_view.dimension_ranges(resolution) + return self.get_view(image).dimension_ranges(resolution) def image_data(self, image): if self.sdk_v1: @@ -405,7 +414,7 @@ def num_derived_levels(self, image): if self.sdk_v1: return pe_in.numLevels() else: - return image.source_view.num_derived_levels + return self.get_view(image).num_derived_levels def num_images(self): pe_in = self.pixel_engine["in"] @@ -730,7 +739,8 @@ def write_tile( if self.sdk_v1: request_regions = pe_in.SourceView().requestRegions else: - request_regions = image.source_view.request_regions + request_regions = self.get_view(image).request_regions + regions = request_regions( patches[i:i + self.batch_size], envelopes, True, [self.fill_color] * 3 From 85b220c00d0af6c7732e11eec0e373e4c1d767a8 Mon Sep 17 00:00:00 2001 From: Melissa Linkert Date: Fri, 12 Sep 2025 17:12:35 -0500 Subject: [PATCH 14/21] Restrict filtering to WSI images (not label/macro) --- isyntax2raw/__init__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/isyntax2raw/__init__.py b/isyntax2raw/__init__.py index 79db339..3163cdc 100644 --- a/isyntax2raw/__init__.py +++ b/isyntax2raw/__init__.py @@ -263,6 +263,9 @@ def get_image_metadata_sdk_v1(self, image_no): def get_view(self, img): view = img.source_view + if img.image_type != "WSI": + return view + user_view = view.add_user_view() user_view.add_filter(FILTER_16_TO_8) return user_view From 67b00c8b90148bb3e8df96ebf125ee945954dadf Mon Sep 17 00:00:00 2001 From: Melissa Linkert Date: Mon, 15 Sep 2025 11:04:05 -0500 Subject: [PATCH 15/21] Save initialized user view --- isyntax2raw/__init__.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/isyntax2raw/__init__.py b/isyntax2raw/__init__.py index 3163cdc..c521a3c 100644 --- a/isyntax2raw/__init__.py +++ b/isyntax2raw/__init__.py @@ -106,6 +106,7 @@ def __init__( ) self.pixel_engine["in"].open(input_path, "ficom") self.sdk_v1 = hasattr(self.pixel_engine["in"], "BARCODE") + self.user_view = None def __enter__(self): return self @@ -266,9 +267,11 @@ def get_view(self, img): if img.image_type != "WSI": return view - user_view = view.add_user_view() - user_view.add_filter(FILTER_16_TO_8) - return user_view + if self.user_view is None: + self.user_view = view.add_user_view() + self.user_view.add_filter(FILTER_16_TO_8) + + return self.user_view def get_image_metadata_sdk_v2(self, image_no): pe_in = self.pixel_engine["in"] From 79a5e910cccb7f8170d380fd17937530769cf689 Mon Sep 17 00:00:00 2001 From: Melissa Linkert Date: Mon, 15 Sep 2025 17:29:42 -0500 Subject: [PATCH 16/21] Reset envelopes before requesting a batch This prevents a segfault when filtering 16 to 8 bit data. --- isyntax2raw/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/isyntax2raw/__init__.py b/isyntax2raw/__init__.py index c521a3c..e3f48ff 100644 --- a/isyntax2raw/__init__.py +++ b/isyntax2raw/__init__.py @@ -729,10 +729,10 @@ def write_tile( [self.tile_width, self.tile_height], tile_directory ) - envelopes = self.data_envelopes(image, resolution) jobs = [] with MaxQueuePool(ThreadPoolExecutor, self.max_workers) as pool: for i in range(0, len(patches), self.batch_size): + envelopes = self.data_envelopes(image, resolution) # requestRegions( # self: pixelengine.PixelEngine.View, # region: List[List[int]], From 17c47850e31bc17d77d901275f60169a50e80ee3 Mon Sep 17 00:00:00 2001 From: Melissa Linkert Date: Mon, 15 Sep 2025 17:42:21 -0500 Subject: [PATCH 17/21] Add option to configure 16 to 8 bit filtering --- isyntax2raw/__init__.py | 5 +++-- isyntax2raw/cli/isyntax2raw.py | 9 +++++++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/isyntax2raw/__init__.py b/isyntax2raw/__init__.py index e3f48ff..8649093 100644 --- a/isyntax2raw/__init__.py +++ b/isyntax2raw/__init__.py @@ -86,7 +86,7 @@ class WriteTiles(object): def __init__( self, tile_width, tile_height, resolutions, max_workers, - batch_size, fill_color, nested, input_path, output_path + batch_size, fill_color, nested, linear16to8, input_path, output_path ): self.tile_width = tile_width self.tile_height = tile_height @@ -95,6 +95,7 @@ def __init__( self.batch_size = batch_size self.fill_color = fill_color self.nested = nested + self.linear16to8 = linear16to8 self.input_path = input_path self.slide_directory = output_path @@ -264,7 +265,7 @@ def get_image_metadata_sdk_v1(self, image_no): def get_view(self, img): view = img.source_view - if img.image_type != "WSI": + if img.image_type != "WSI" or self.linear16to8 is False: return view if self.user_view is None: diff --git a/isyntax2raw/cli/isyntax2raw.py b/isyntax2raw/cli/isyntax2raw.py index 56a3152..81fb621 100644 --- a/isyntax2raw/cli/isyntax2raw.py +++ b/isyntax2raw/cli/isyntax2raw.py @@ -60,6 +60,10 @@ def cli(): "--nested/--no-nested", default=True, show_default=True, help="Whether to use '/' as the chunk path separator" ) +@click.option( + "--linear16to8", default=False, show_default=True, + help="apply built-in linear 16 to 8 bit filter" +) @click.option( "--debug", is_flag=True, help="enable debugging", @@ -68,12 +72,13 @@ def cli(): @click.argument("output_path") def write_tiles( tile_width, tile_height, resolutions, max_workers, batch_size, - fill_color, nested, debug, input_path, output_path + fill_color, nested, linear16to8, debug, input_path, output_path ): setup_logging(debug) with WriteTiles( tile_width, tile_height, resolutions, max_workers, - batch_size, fill_color, nested, input_path, output_path + batch_size, fill_color, nested, linear16to8, + input_path, output_path ) as wt: wt.write_metadata() wt.write_label_image() From bd41c24af5af4750d0112ab8015c749a8af9c1af Mon Sep 17 00:00:00 2001 From: Melissa Linkert Date: Tue, 16 Sep 2025 10:17:57 -0500 Subject: [PATCH 18/21] Remove unnecessary check on bits_per_pixel --- isyntax2raw/__init__.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/isyntax2raw/__init__.py b/isyntax2raw/__init__.py index 8649093..accbd68 100644 --- a/isyntax2raw/__init__.py +++ b/isyntax2raw/__init__.py @@ -662,8 +662,7 @@ def create_tile_directory(self, series, resolution, width, height): ) def make_planar(self, pixels, tile_width, tile_height): - if self.bits_per_pixel is not None: - pixels.dtype = self.get_data_type(self.bits_per_pixel) + pixels.dtype = self.get_data_type(self.bits_per_pixel) r = pixels[0::3] g = pixels[1::3] From efafa141fbb45acb9fe7a27db972d813f3eb27e5 Mon Sep 17 00:00:00 2001 From: Melissa Linkert Date: Tue, 16 Sep 2025 10:27:16 -0500 Subject: [PATCH 19/21] Omit compression method/ratio from metadata map if not defined --- isyntax2raw/__init__.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/isyntax2raw/__init__.py b/isyntax2raw/__init__.py index accbd68..c57b306 100644 --- a/isyntax2raw/__init__.py +++ b/isyntax2raw/__init__.py @@ -281,6 +281,9 @@ def get_image_metadata_sdk_v2(self, image_no): view = self.get_view(img) image_scale_factor = view.scale + # compression method and ratio are informational only + # these values are not needed for decompression + # and may be missing for label/macro images in particular compression_method = None try: compression_method = img.lossy_image_compression_method @@ -296,10 +299,6 @@ def get_image_metadata_sdk_v2(self, image_no): image_metadata = { "Image type": image_type, - "Lossy image compression method": - compression_method, - "Lossy image compression ratio": - compression_ratio, "Image dimension names": view.dimension_names, "Image dimension types": @@ -313,6 +312,13 @@ def get_image_metadata_sdk_v2(self, image_no): "Block size": img.block_size(), } + if compression_method is not None: + image_metadata[ + "Lossy image compression method"] = compression_method + + if compression_ratio is not None: + image_metadata["Lossy image compression ratio"] = compression_ratio + if image_type == "WSI": image_metadata["Color space transform"] = \ img.colorspace_transform From 95f75f99e56e47f222f065ff743a4789b732af83 Mon Sep 17 00:00:00 2001 From: Melissa Linkert Date: Tue, 16 Sep 2025 19:51:48 -0500 Subject: [PATCH 20/21] Remove one more unnecessary check --- isyntax2raw/__init__.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/isyntax2raw/__init__.py b/isyntax2raw/__init__.py index c57b306..9d7e1f5 100644 --- a/isyntax2raw/__init__.py +++ b/isyntax2raw/__init__.py @@ -793,9 +793,8 @@ def write_tile( ) height = int(height) pixel_buffer_size = width * height * 3 - if self.bits_per_pixel is not None: - bpp = self.bits_per_pixel / 8 - pixel_buffer_size = pixel_buffer_size * bpp + bpp = self.bits_per_pixel / 8 + pixel_buffer_size = pixel_buffer_size * bpp pixels = np.empty( int(pixel_buffer_size), From 3929942faf56c1d9486dce4b17d8e73c1c2c30d5 Mon Sep 17 00:00:00 2001 From: Melissa Linkert Date: Thu, 18 Sep 2025 09:35:34 -0500 Subject: [PATCH 21/21] Update pillow version to <8.4.0, to keep support for Python 3.6.x --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index cc44210..4ce92ed 100644 --- a/setup.py +++ b/setup.py @@ -81,7 +81,7 @@ def read(fname): setup_requires=['flake8'], install_requires=[ 'click==7.0', - 'pillow==7.1.0', + 'pillow<8.4.0', 'numpy==1.17.3', 'zarr==2.8.1', 'kajiki==0.8.2',