From 35d46b461246c6adbd2acb9e8a47111fdff98eb1 Mon Sep 17 00:00:00 2001 From: Carter Francis Date: Wed, 2 Oct 2024 09:02:23 -0500 Subject: [PATCH 1/9] First commit for writing and reading 4D DM5 files --- rsciio/dm5/_api.py | 407 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 407 insertions(+) create mode 100644 rsciio/dm5/_api.py diff --git a/rsciio/dm5/_api.py b/rsciio/dm5/_api.py new file mode 100644 index 000000000..bfcbefb60 --- /dev/null +++ b/rsciio/dm5/_api.py @@ -0,0 +1,407 @@ +from asyncore import write +from importlib.metadata import metadata + +import h5py +import dask.array as da +import numpy as np +from rsciio.hspy._api import HyperspyWriter + + +# for some reason this is slightly different in dm4??? +# Data Type 10: Integer? +# +data_types = { + np.short().dtype: [2, 2], # 2 byte integer signed + np.uint8().dtype: [6, 1], # 1 byte integer unsigned + np.float32().dtype: [2, 4], # 4 byte real (IEEE 754) + np.uint16().dtype: [10,2], # 2 byte integer unsigned + + np.int32().dtype: [23, 4], # 4 byte integer signed + np.uint32().dtype: [23, 4], # 4 byte integer signed + } + +class DM5: + """ + The internal representation of a DM5 file. + + The basic structure of a DM5 file is: + - ImageList: a list of images. This is the main data structure in a DM5 file. + - Image Behavior: a list of behaviors that can be applied to images. (e.g. Image Shift, Image Rotation) (Not implemented) + - Thumbnails: A slice of the image data that can be used to display the image as a thumbnail. (Not implemented) + - Image Source List: A list of sources for the images. (Not implemented) + + + + """ + def __init__(self, file_path, mode='r'): + self.file = h5py.File(file_path, mode=mode) + + if mode == 'r': + self.image_list = self.file["ImageList"] # list of images + else: + self.image_list = self.file.create_group("ImageList") + + self.images = [] + + def write_sources(self, signal_dimensions, navigation_dimensions): + """ + I think that DM has a similar concept of a Navigator and a signal + in hyperspy. + """ + + self.file.create_group("ImageSourceList") + self.file["ImageSourceList"].create_group("[0]") + + if signal_dimensions==2 and navigation_dimensions ==2: + # this is for 4D Data + self.file["ImageSourceList"]["[0]"].attrs.update({"ClassName":"ImageSource:4DSummed", # Need the right ClassName + "Do Sum": 1, # Sum the image? + "ImageRef":0, # Image to take the sum of... (Similar to Hyperspy) + "LayerFstEnd": 0, + "LayerFstStart": 0, + "LayerSndEnd": 0, + "LayerSndStart": 0, + "Summed Fst Dimension": 0, # Sum 1st dimension + "Summed Snd Dimension": 1, # Sum 2nd dimension + }) + self.file["ImageSourceList"]["[0]"].create_group("Id") + self.file["ImageSourceList"]["[0]"]["Id"].attrs.update({"[0]": 0}) + + elif signal_dimensions ==2 and navigation_dimensions==1: + #Insitu Imaging (Still need to check what the correct default behavior is...) + self.file["ImageSourceList"]["[0]"].attrs.update( + {"ClassName": "ImageSource:Summed", # Need the right ImageSource! + "Do Sum": 1, # Sum the image? + "ImageRef": 0, # Image to take the sum of... (Similar to Hyperspy) + "LayerEnd": 0, + "LayerStart": 0, + "Summed Dimension": 0, # Sum 1st dimension? + }) + self.file["ImageSourceList"]["[0]"].create_group("Id") + self.file["ImageSourceList"]["[0]"]["Id"].attrs.update({"[0]": 0}) + elif signal_dimensions == 1 and navigation_dimensions == 1: + + + + # Do we need to write anything here? + + def write_document_objects(self): + """ + DM uses Document Objects similar to markers/ROIs in hyperspy. + """ + self.file.create_group("DocumentObjectList") + self.file["DocumentObjectList"].create_group("[0]") + self.file["DocumentObjectList"]["[0]"].attrs.update({"AnnotationType": 20, + 'ImageSource': 0, + 'ImageDisplayType': 1, # 1 = Image + 'RangeAdjust': 1.0, + 'SparseSurvey_GridSize': 32, + 'SparseSurvey_NumberPixels': 64, + 'SparseSurvey_UseNumberPixels': 1, + 'SurveyTechnique': 2, + }) + self.file["DocumentObjectList"]["[0]"].create_group("ImageDisplayInfo") + self.file["DocumentObjectList"]["[0]"]["ImageDisplayInfo"].attrs.update({"EstimatedMin":0, + "HiLimitContrastDeltaTriggerPercentage":0}) + + def write_header_info(self): + """ + Just write the minimum "header" info to open the file in DM. + """ + self.file.attrs.update({"InImageMode": 1}) # The rest of the attributes are for defining window size... + + def write_image_behavior(self): + """ + DM uses Image Behaviors to apply transformations to images. + + Write only the minimum required to open the file in DM. + """ + self.file.create_group("Image Behavior") + self.file["Image Behavior"].attrs.update({"ViewDisplayID": 8}) + + def read_images(self): + for image_group in self.image_list.values(): + self.images.append(Image(image_group)) + + def write_image(self, data, axes_dicts=None, metadata =None, brightness=None): + """ + Write an image to the DM5 file. + """ + + previous_images = [int(k.strip("[ ]")) for k in self.image_list.keys()] + if len(previous_images) == 0: + new_image_number = '[0]' + else: + new_image_number = f"[{max(previous_images) + 1}]" + + self.image_list.create_group(new_image_number) + self.image_list[new_image_number].create_group("ImageData") + self.image_list[new_image_number].create_group("ImageTags") + self.image_list[new_image_number].create_group("UniqueID") + + image = Image(self.image_list[new_image_number]) + self.images.append(image) + + image.update_data(data) + if axes_dicts is not None: + for axis, axis_dict in axes_dicts.items(): + image.update_calibration(axis, **axis_dict) + image.update_dimension(axis, axis_dict["size"]) + else: + for axis in range(len(data.shape)): + image.update_calibration(axis) + image.update_dimension(axis) + + image.update_metadata(metadata) + + image.update_brightness(brightness) + self.write_sources() + self.write_document_objects() + self.write_image_behavior() + self.write_header_info() + + + +class Image: + """ + The internal representation of an image in a DM5 file. + + Each image has: + - ImageData: the actual image data + - Data: the actual image data + - Calibrations: the calibration data + - Dimension: the dimensions of the image + - Tags: Arbitrary tags that can be used to store metadata about the image + - UniqueID: a unique identifier for the image + """ + + def __init__(self, image_group, tags=None, unique_id=None): + self.image_data = image_group['ImageData'] + self.image_tags = image_group['ImageTags'] + self.unique_id = image_group['UniqueID'] + + def __str__(self): + return f"Image: {self.image_data['Data'].shape}" + + + def get_axis_dict(self, axis): + """ + Get the calibration data for a given axis. + + Parameters + ---------- + axis : int + The axis to get the calibration data for (Starting from 0). + + Notes + ----- + Axis is 0-indexed. + """ + try: + calibration_dict = dict(self.image_data['Calibrations']["Dimension"][f"[{axis}]"]) + axis_dict = {"name": calibration_dict.get("Label", ""), + "offset": calibration_dict.get("Origin", 0), + "scale": calibration_dict.get("Scale", 1), + "unit": calibration_dict.get("Unit", ""), + "size": self._get_dimension(axis)} + + except KeyError: # if calibration data is not present. Reads incomplete files... + axis_dict = {"name": "", + "offset": 0, + "scale": 1, + "unit": "", + "size": self._get_dimension(axis)} + return axis_dict + + def update_calibration(self, axis, label="", origin=0, scale=1, unit=""): + """ + Add calibration data to the image for a given axis. + """ + if not "Calibrations" in self.image_data: + self.image_data.create_group("Calibrations") + if not "Dimension" in self.image_data['Calibrations']: + self.image_data['Calibrations'].create_group("Dimension") + if not f"[{axis}]" in self.image_data['Calibrations']["Dimension"]: + self.image_data['Calibrations']["Dimension"].create_group(f"[{axis}]") + + self.image_data['Calibrations']["Dimension"][f"[{axis}]"].attrs.update({ + "Label": label, + "Origin": origin, + "Scale": scale, + "Unit": unit + }) + self.image_data['Calibrations'].attrs.update({"DisplayCalibratedUnits": 1}) + + + def brightness(self): + """ + Get the brightness of the image. + """ + try: + dict(self.image_data["Calibrations"]["Brightness"].attrs) + except KeyError: + return {} + + def update_brightness(self, brightness=None): + """ + Update the brightness of the image. + """ + if brightness is None: + brightness = {"Label": "b", + "Origin": 0, + "Scale": 1, + "Units": "b"} + if not "Calibrations" in self.image_data: + self.image_data.create_group("Calibrations") + if not "Brightness" in self.image_data["Calibrations"]: + self.image_data["Calibrations"].create_group("Brightness") + self.image_data["Calibrations"]["Brightness"].attrs.update(brightness) + + def update_dimension(self, axis, length=None): + """ + Update the dimension of the image for a given axis. + + This is two places in the DM5 file??? + + Under Calibrations and under Dimension. I think that only the Calibrations should be updated. + + """ + if not "Dimensions" in self.image_data: + self.image_data.create_group("Dimensions") + if length is None: + length = self._get_dimension(axis) + self.image_data['Dimensions'].attrs.update({f"[{axis}]": length}) + + def _get_dimension(self, axis): + try: + return self.image_data['Dimensions'].attrs[f"[{axis}]"] + except KeyError: + shape = self.image_data['Data'].shape + return shape[len(shape) - axis-1] + + def get_data(self, lazy=False): + """ + Get the image data. + """ + if lazy: + return da.from_array(self.image_data['Data']) + else: + return np.array(self.image_data['Data']) + + def update_data(self, data): + """ + Update the image data. + """ + HyperspyWriter.overwrite_dataset(self.image_data, + data, + "Data") + self.image_data.attrs.update({"DataType": data_types[data.dtype][0], + "PixelDepth": data_types[data.dtype][1]}) + + + def get_metadata(self): + """ + Get the metadata for the image. + """ + original_metadata = _group2dict(self.image_tags) + # translate to Hyperspy metadata format + + metadata = {} + metadata["General"] = {} + metadata["General"]["title"] = "" # DM uses the filename as the title + + + metadata[""] + if "Acquisition" in original_metadata: + metadata["Acquisition"] = original_metadata["Acquisition"] + + # The tag structure of DMFiles is arbitrary for the most part. So we can + # just copy the dict structure of the hyperspy metadata. + if "Microscope Info" in original_metadata: + metadata["Acquisition_instrument"] = {} + metadata["Acquisition_instrument"]["TEM"] = {} + metadata["Acquisition_instrument"]["TEM"]["beam_energy "] = ( + original_metadata["Microscope Info"].get("Voltage", 0)/1000) + metadata["Acquisition_instrument"]["TEM"]["acquisition_mode"] = ( + original_metadata["Microscope Info"].get("Illumination Mode", "Unknown")) + metadata["Acquisition_instrument"]["TEM"]["magnification"] = ( + original_metadata["Microscope Info"].get("Indicated Magnification", 0)) + metadata["Acquisition_instrument"]["TEM"]["camera_length"] = ( + original_metadata["Microscope Info"].get("STEM Camera Length", 0)) + metadata["Acquisition_instrument"] = original_metadata["Microscope Info"] + metadata["Acquisition_instrument"] = {} + return metadata, original_metadata + + + def update_metadata(self, metadata=None): + """ + Update the metadata for the image. + + + """ + if metadata is None: + metadata = {} + formatted_metadata = {} + + formatted_metadata["Acquisition"] = {} + + # Test + formatted_metadata["Meta Data"] = {} + formatted_metadata["Meta Data"]["Format"] = "Diffraction image" + formatted_metadata["Meta Data"]["Acquisition Mode"] = "Parallel imaging" + formatted_metadata["Meta Data"]["Experiment keywords"] = {} + formatted_metadata["Meta Data"]["Experiment keywords"]["[0]"] = "Label: Diffraction" + # Data is reversed from usual... + formatted_metadata["Meta Data"]["Data Order Swapped"] = "true" + dict2group(formatted_metadata, self.image_tags) + return + + def to_signal_dict(self): + """ + Convert the image to a Hyperspy signal dictionary. + """ + data = self.get_data() + metadata, original_metadata = self.get_metadata() + axes = [] + for axis in range(len(data.shape)): + axes.append(self.get_axis_dict(axis + 1)) + return {"data": data, "metadata": metadata, "original_metadata": original_metadata, "axes": axes} + + + + +def dict2group(dictionary, group): + for key, value in dictionary.items(): + if isinstance(value, dict): + subgroup = group.create_group(key) + dict2group(value, subgroup) + else: + group.attrs[key] = value + +def _group2dict(group, dictionary=None): + if dictionary is None: + dictionary = {} + for key, value in group.attrs.items(): + if isinstance(value, bytes): + try: + value = value.decode() + except UnicodeDecodeError: + value = "Decoding error" + if isinstance(value, (np.bytes_, str)): + if value == "_None_": + value = None + elif isinstance(value, np.bool_): + value = bool(value) + elif isinstance(value, np.ndarray) and value.dtype.char == "S": + # Convert strings to unicode + value = value.astype("U") + if value.dtype.str.endswith("U1"): + value = value.tolist() + else: + dictionary[key] = value + + if not isinstance(group, h5py.Dataset): + for key in group.keys(): + dictionary[key] = {} + _group2dict(group[key], dictionary[key]) + return dictionary \ No newline at end of file From c8a8f2a73d635ea81d7ac76290e4cf2d32dc711a Mon Sep 17 00:00:00 2001 From: Carter Francis Date: Mon, 7 Oct 2024 15:33:05 -0500 Subject: [PATCH 2/9] Add Tests for generating and saving dm5 datasets --- rsciio/dm5/_api.py | 382 ++++++++++++++++++++++++++++++--------- rsciio/tests/test_dm5.py | 73 ++++++++ 2 files changed, 369 insertions(+), 86 deletions(-) create mode 100644 rsciio/tests/test_dm5.py diff --git a/rsciio/dm5/_api.py b/rsciio/dm5/_api.py index bfcbefb60..935b00443 100644 --- a/rsciio/dm5/_api.py +++ b/rsciio/dm5/_api.py @@ -4,7 +4,11 @@ import h5py import dask.array as da import numpy as np + +from rsciio._docstrings import FILENAME_DOC, LAZY_DOC, RETURNS_DOC from rsciio.hspy._api import HyperspyWriter +from logging import getLogger +_logger = getLogger(__name__) # for some reason this is slightly different in dm4??? @@ -12,14 +16,21 @@ # data_types = { np.short().dtype: [2, 2], # 2 byte integer signed + np.float32().dtype: [2, 4], # 4 byte real (IEEE 754) + np.uint8().dtype: [6, 1], # 1 byte integer unsigned - np.float32().dtype: [2, 4], # 4 byte real (IEEE 754) + np.int8().dtype: [6, 1], # 1 byte integer signed + np.int16().dtype: [1, 2], # 2 byte integer signed + np.int32().dtype: [7, 4], # 4 byte integer signed + np.uint32().dtype: [7, 4], # 4 byte integer signed + np.float64().dtype: [12, 8], # 8 byte real (IEEE 754) np.uint16().dtype: [10,2], # 2 byte integer unsigned + np.complex64().dtype: [13, 8], # 8 byte complex + np.complex128().dtype: [13, 16], # 16 byte complex - np.int32().dtype: [23, 4], # 4 byte integer signed - np.uint32().dtype: [23, 4], # 4 byte integer signed } + class DM5: """ The internal representation of a DM5 file. @@ -28,7 +39,10 @@ class DM5: - ImageList: a list of images. This is the main data structure in a DM5 file. - Image Behavior: a list of behaviors that can be applied to images. (e.g. Image Shift, Image Rotation) (Not implemented) - Thumbnails: A slice of the image data that can be used to display the image as a thumbnail. (Not implemented) - - Image Source List: A list of sources for the images. (Not implemented) + - Image Source List: This helps to define how the data should be displayed. A couple of things to note. The + Index [0] [1] etc. is not the same as the index of the image in the ImageList. I think you can have multiple 'views' + of the same image. Instead the ImageRef attribute is used to reference the image in the ImageList. The ClassName + attribute is used to define the type of image and can be [4DSummed, Summed, Simple]. The ImageSourceList is used to @@ -52,8 +66,7 @@ def write_sources(self, signal_dimensions, navigation_dimensions): self.file.create_group("ImageSourceList") self.file["ImageSourceList"].create_group("[0]") - if signal_dimensions==2 and navigation_dimensions ==2: - # this is for 4D Data + if signal_dimensions==2 and navigation_dimensions ==2: # this is for 4D Data self.file["ImageSourceList"]["[0]"].attrs.update({"ClassName":"ImageSource:4DSummed", # Need the right ClassName "Do Sum": 1, # Sum the image? "ImageRef":0, # Image to take the sum of... (Similar to Hyperspy) @@ -63,52 +76,141 @@ def write_sources(self, signal_dimensions, navigation_dimensions): "LayerSndStart": 0, "Summed Fst Dimension": 0, # Sum 1st dimension "Summed Snd Dimension": 1, # Sum 2nd dimension - }) + }) # data is reversed from usual in DM but not numpy + self.file["ImageSourceList"]["[0]"].create_group("Id") + self.file["ImageSourceList"]["[0]"]["Id"].attrs.update({"[0]": 0}) + + # Write the Document Objects similar to ROIs in Hyperspy for Visualization + self.file.create_group("DocumentObjectList") + self.file["DocumentObjectList"].create_group("[0]") + self.file["DocumentObjectList"]["[0]"].attrs.update({"AnnotationType": 20, # Create an image + 'ImageSource': 0, # + 'ImageDisplayType': 1, # 1 = Image + 'RangeAdjust': 1.0, + 'SparseSurvey_GridSize': 32, + 'SparseSurvey_NumberPixels': 64, # fast survey? + 'SparseSurvey_UseNumberPixels': 1, + 'SurveyTechnique': 2, + }) + self.file["DocumentObjectList"]["[0]"].create_group("ImageDisplayInfo") + self.file["DocumentObjectList"]["[0]"]["ImageDisplayInfo"].attrs.update({"EstimatedMin": 0, + "HiLimitContrastDeltaTriggerPercentage": 0, + }) + + + elif signal_dimensions == 1 and navigation_dimensions == 2: # Spectrum Image (Data might need to be reversed...) + self.file["ImageSourceList"]["[0]"].attrs.update({"ClassName": "ImageSource:Summed", + "Do Sum": 1, + "ImageRef": 0, + "LayerEnd": 0, + "LayerStart": 0, + "Summed Dimension": 2, + }) # Sum last dimension self.file["ImageSourceList"]["[0]"].create_group("Id") self.file["ImageSourceList"]["[0]"]["Id"].attrs.update({"[0]": 0}) - elif signal_dimensions ==2 and navigation_dimensions==1: - #Insitu Imaging (Still need to check what the correct default behavior is...) + # Write the Document Objects similar to ROIs in Hyperspy for Visualization. + self.file.create_group("DocumentObjectList") + self.file["DocumentObjectList"].create_group("[0]") + self.file["DocumentObjectList"]["[0]"].attrs.update({"AnnotationType": 20, + 'ImageSource': 0, # Reference to the ImageSource + 'ImageDisplayType': 1, # 1 = Image? + 'IsMoveable': 1, + 'IsResizable': 1, + 'IsSelectable': 1, + 'IsTransferrable': 1, #sp? + 'IsTranslatable': 1, + 'IsVisible': 1, + 'UniqueID': 8, + }) + + + + elif signal_dimensions ==2 and navigation_dimensions==1: # Image Stack (In Situ) self.file["ImageSourceList"]["[0]"].attrs.update( {"ClassName": "ImageSource:Summed", # Need the right ImageSource! "Do Sum": 1, # Sum the image? "ImageRef": 0, # Image to take the sum of... (Similar to Hyperspy) "LayerEnd": 0, "LayerStart": 0, - "Summed Dimension": 0, # Sum 1st dimension? + "Summed Dimension": 2, # Sum last dimension? }) + self.file.attrs.update({"InImageMode": 1}) self.file["ImageSourceList"]["[0]"].create_group("Id") self.file["ImageSourceList"]["[0]"]["Id"].attrs.update({"[0]": 0}) - elif signal_dimensions == 1 and navigation_dimensions == 1: - - + self.file.create_group("DocumentObjectList") + self.file["DocumentObjectList"].create_group("[0]") + self.file["DocumentObjectList"]["[0]"].attrs.update({"AnnotationType": 20, + 'ImageSource': 0, # Reference to the ImageSource + 'ImageDisplayType': 1, # 1 = Image? + 'IsMoveable': 1, + 'IsResizable': 1, + 'IsSelectable': 1, + 'IsTransferrable': 1, #sp? + 'IsTranslatable': 1, + 'IsVisible': 1, + 'UniqueID': 8, + }) + self.file["DocumentObjectList"]["[0]"].create_group("AnnotationGroupList") + self.file["DocumentObjectList"]["[0]"]["AnnotationGroupList"].create_group("[0]") + self.file["DocumentObjectList"]["[0]"]["AnnotationGroupList"]["[0]"].attrs.update({"AnnotationType": 23, + "Name": "SICursor", + "Rectangle": (0, 0, 1, 1), + }) + self.file["DocumentObjectList"]["[0]"].create_group("ImageDisplayInfo") + self.file["DocumentObjectList"]["[0]"]["ImageDisplayInfo"].attrs.update({"EstimatedMin": 0, + "HiLimitContrastDeltaTriggerPercentage": 0}) - # Do we need to write anything here? - - def write_document_objects(self): - """ - DM uses Document Objects similar to markers/ROIs in hyperspy. - """ - self.file.create_group("DocumentObjectList") - self.file["DocumentObjectList"].create_group("[0]") - self.file["DocumentObjectList"]["[0]"].attrs.update({"AnnotationType": 20, - 'ImageSource': 0, - 'ImageDisplayType': 1, # 1 = Image - 'RangeAdjust': 1.0, - 'SparseSurvey_GridSize': 32, - 'SparseSurvey_NumberPixels': 64, - 'SparseSurvey_UseNumberPixels': 1, - 'SurveyTechnique': 2, - }) - self.file["DocumentObjectList"]["[0]"].create_group("ImageDisplayInfo") - self.file["DocumentObjectList"]["[0]"]["ImageDisplayInfo"].attrs.update({"EstimatedMin":0, - "HiLimitContrastDeltaTriggerPercentage":0}) + elif signal_dimensions == 1 and navigation_dimensions == 1: + self.file["ImageSourceList"]["[0]"].attrs.update( + {"ClassName": "ImageSource:Summed", # Need the right ImageSource! + "Do Sum": 1, # Sum the image? + "ImageRef": 0, # Image to take the sum of... (Similar to Hyperspy) + "LayerEnd": 0, + "LayerStart": 0, + "Summed Dimension": 1, # Sum last dimension? + }) + self.file["ImageSourceList"]["[0]"].create_group("Id") + self.file["ImageSourceList"]["[0]"]["Id"].attrs.update({"[0]": 0}) + self.file.create_group("DocumentObjectList") + self.file["DocumentObjectList"].create_group("[0]") + self.file["DocumentObjectList"]["[0]"].attrs.update({"AnnotationType": 20, + 'ImageSource': 0, # Reference to the ImageSource + 'ImageDisplayType': 1, # 1 = Image? + 'RangeAdjust': 1.0, + 'IsMoveable': 1, + 'IsResizable': 1, + 'IsSelectable': 1, + 'IsTransferrable': 1, # sp? + 'IsTranslatable': 1, + 'IsVisible': 1, + 'UniqueID': 8, + }) + else: + self.file["ImageSourceList"]["[0]"].attrs.update( + {"ClassName": "ImageSource:Simple", # Need the right ImageSource! + "ImageRef": 0, # Reference to self (Usually 0 is a ??thumbnail?? which we don't write) + }) + self.file.create_group("DocumentObjectList") + self.file["DocumentObjectList"].create_group("[0]") + self.file["DocumentObjectList"]["[0]"].attrs.update({"AnnotationType": 20, + 'ImageSource': 0, # Reference to the ImageSource + 'ImageDisplayType': 1, # 1 = Image? + 'RangeAdjust': 1.0, + 'IsMoveable': 1, + 'IsResizable': 1, + 'IsSelectable': 1, + 'IsTransferrable': 1, # sp? + 'IsTranslatable': 1, + 'IsVisible': 1, + 'UniqueID': 8, + }) def write_header_info(self): """ Just write the minimum "header" info to open the file in DM. """ - self.file.attrs.update({"InImageMode": 1}) # The rest of the attributes are for defining window size... + self.file.attrs.update({"InImageMode": 1}) # The rest of the attributes are for defining the window size... def write_image_behavior(self): """ @@ -121,9 +223,9 @@ def write_image_behavior(self): def read_images(self): for image_group in self.image_list.values(): - self.images.append(Image(image_group)) + self.images.append(Image(image_group, file=self.file)) - def write_image(self, data, axes_dicts=None, metadata =None, brightness=None): + def write_image(self, data, axes_dicts, metadata =None, brightness=None): """ Write an image to the DM5 file. """ @@ -139,24 +241,22 @@ def write_image(self, data, axes_dicts=None, metadata =None, brightness=None): self.image_list[new_image_number].create_group("ImageTags") self.image_list[new_image_number].create_group("UniqueID") - image = Image(self.image_list[new_image_number]) + image = Image(self.image_list[new_image_number], file=self.file) self.images.append(image) + navigation_dimensions = len([axis for axis in axes_dicts if axis["navigate"] == True]) + signal_dimensions = len([axis for axis in axes_dicts if axis["navigate"] == False]) image.update_data(data) - if axes_dicts is not None: - for axis, axis_dict in axes_dicts.items(): - image.update_calibration(axis, **axis_dict) - image.update_dimension(axis, axis_dict["size"]) - else: - for axis in range(len(data.shape)): - image.update_calibration(axis) - image.update_dimension(axis) + for axis, axis_dict in enumerate(axes_dicts): + image.update_calibration(axis, **axis_dict) + image.update_dimension(axis, axis_dict["size"]) - image.update_metadata(metadata) + image.update_metadata(metadata, signal_dimensions=signal_dimensions, + navigation_dimensions=navigation_dimensions) image.update_brightness(brightness) - self.write_sources() - self.write_document_objects() + self.write_sources(navigation_dimensions= navigation_dimensions, + signal_dimensions=signal_dimensions) self.write_image_behavior() self.write_header_info() @@ -175,14 +275,53 @@ class Image: - UniqueID: a unique identifier for the image """ - def __init__(self, image_group, tags=None, unique_id=None): + def __init__(self, image_group, tags=None, unique_id=None, file=None): self.image_data = image_group['ImageData'] self.image_tags = image_group['ImageTags'] self.unique_id = image_group['UniqueID'] + self.file = file def __str__(self): return f"Image: {self.image_data['Data'].shape}" + @property + def ndim(self): + return len(self.image_data['Data'].shape) + + def signal_dimensions(self): + """ + Get the number of signal dimensions. + """ + _, original_metadata = self.get_metadata() + meta = original_metadata.get("Meta Data", {}) + format = meta.get("Format", "Unknown") + if format == "Diffraction image": + return 2 + elif format == "Spectrum image": + return 1 + elif format == "Spectrum": + return 1 + elif format == "Image": + return 2 + else: # Now we just try to guess from the DocumentObjectList + try: + class_name =self.file["ImageSourceList"]["[0]"].attrs["ClassName"] + except KeyError: + raise KeyError("A Valid DM5 File needs to have an ImageSourceList with a " + "ClassName attribute to determine how the data should be displayed.") + if self.ndim == 2 and class_name == "ImageSource:Simple": + return 2 + elif self.ndim == 1 and class_name == "ImageSource:Simple": + return 1 + elif self.ndim == 4 and class_name == "ImageSource:4DSummed": + return 2 + else: + + raise NotImplementedError("Determining the Type for unlabeled DM5 files. Please add the" + " 'Image', 'Spectrum', 'Spectrum Image' 'Diffraction image' tag to the" + "Meta Data.Format attribute in the ImageTags group.") + def navigation_dimensions(self): + return self.ndim - self.signal_dimensions() def get_axis_dict(self, axis): """ @@ -197,38 +336,40 @@ def get_axis_dict(self, axis): ----- Axis is 0-indexed. """ - try: - calibration_dict = dict(self.image_data['Calibrations']["Dimension"][f"[{axis}]"]) - axis_dict = {"name": calibration_dict.get("Label", ""), + + axis = self.ndim - axis - 1 + navigate = axis >= self.signal_dimensions() + calibration_dict = dict(self.image_data['Calibrations']["Dimension"][f"[{axis}]"].attrs) + axis_dict = {"name": calibration_dict.get("Label", ""), "offset": calibration_dict.get("Origin", 0), "scale": calibration_dict.get("Scale", 1), - "unit": calibration_dict.get("Unit", ""), - "size": self._get_dimension(axis)} - - except KeyError: # if calibration data is not present. Reads incomplete files... - axis_dict = {"name": "", - "offset": 0, - "scale": 1, - "unit": "", - "size": self._get_dimension(axis)} + "units": calibration_dict.get("Units", ""), + "size": self._get_dimension(axis), + "navigate": navigate + } + return axis_dict - def update_calibration(self, axis, label="", origin=0, scale=1, unit=""): + def update_calibration(self, axis, name="", offset=0, scale=1, units="", **kwds): """ Add calibration data to the image for a given axis. """ + axis = self.ndim - axis - 1 if not "Calibrations" in self.image_data: self.image_data.create_group("Calibrations") if not "Dimension" in self.image_data['Calibrations']: self.image_data['Calibrations'].create_group("Dimension") if not f"[{axis}]" in self.image_data['Calibrations']["Dimension"]: self.image_data['Calibrations']["Dimension"].create_group(f"[{axis}]") - + if not isinstance(units, str): # Handle Undefined Units + units = "" + if not isinstance(name, str): + name = "" self.image_data['Calibrations']["Dimension"][f"[{axis}]"].attrs.update({ - "Label": label, - "Origin": origin, - "Scale": scale, - "Unit": unit + "Label": name, + "Origin": float(offset), + "Scale": float(scale), + "Units": units }) self.image_data['Calibrations'].attrs.update({"DisplayCalibratedUnits": 1}) @@ -266,6 +407,7 @@ def update_dimension(self, axis, length=None): Under Calibrations and under Dimension. I think that only the Calibrations should be updated. """ + axis = self.ndim - axis - 1 if not "Dimensions" in self.image_data: self.image_data.create_group("Dimensions") if length is None: @@ -277,7 +419,7 @@ def _get_dimension(self, axis): return self.image_data['Dimensions'].attrs[f"[{axis}]"] except KeyError: shape = self.image_data['Data'].shape - return shape[len(shape) - axis-1] + return shape[axis] def get_data(self, lazy=False): """ @@ -311,7 +453,6 @@ def get_metadata(self): metadata["General"]["title"] = "" # DM uses the filename as the title - metadata[""] if "Acquisition" in original_metadata: metadata["Acquisition"] = original_metadata["Acquisition"] @@ -333,27 +474,35 @@ def get_metadata(self): return metadata, original_metadata - def update_metadata(self, metadata=None): + def update_metadata(self, metadata=None, signal_dimensions=None, navigation_dimensions=None): """ Update the metadata for the image. - - """ if metadata is None: metadata = {} formatted_metadata = {} formatted_metadata["Acquisition"] = {} - - # Test formatted_metadata["Meta Data"] = {} - formatted_metadata["Meta Data"]["Format"] = "Diffraction image" - formatted_metadata["Meta Data"]["Acquisition Mode"] = "Parallel imaging" - formatted_metadata["Meta Data"]["Experiment keywords"] = {} - formatted_metadata["Meta Data"]["Experiment keywords"]["[0]"] = "Label: Diffraction" - # Data is reversed from usual... - formatted_metadata["Meta Data"]["Data Order Swapped"] = "true" + if signal_dimensions == 2 and navigation_dimensions == 2: # 4D Data + formatted_metadata["Meta Data"]["Format"] = "Diffraction image" + formatted_metadata["Meta Data"]["Acquisition Mode"] = "Parallel imaging" + formatted_metadata["Meta Data"]["Experiment keywords"] = {} + formatted_metadata["Meta Data"]["Experiment keywords"]["[0]"] = "Label: Diffraction" + elif signal_dimensions == 1 and navigation_dimensions == 2: + formatted_metadata["Meta Data"]["Format"] = "Spectrum image" + elif signal_dimensions == 1 and navigation_dimensions == 1: + formatted_metadata["Meta Data"]["Format"] = "Spectrum" + elif signal_dimensions == 2 and navigation_dimensions <= 1: + formatted_metadata["Meta Data"]["Format"] = "Image" + formatted_metadata["Meta Data"]["Signal"] = "Image" + else: + formatted_metadata["Meta Data"]["Format"] = "Unknown" + if navigation_dimensions > 0: + formatted_metadata["Meta Data"]["IsSequence"] = "true" dict2group(formatted_metadata, self.image_tags) + self.image_tags.create_group("UserTags") + dict2group(metadata, self.image_tags["UserTags"]) return def to_signal_dict(self): @@ -364,19 +513,20 @@ def to_signal_dict(self): metadata, original_metadata = self.get_metadata() axes = [] for axis in range(len(data.shape)): - axes.append(self.get_axis_dict(axis + 1)) + axes.append(self.get_axis_dict(axis)) return {"data": data, "metadata": metadata, "original_metadata": original_metadata, "axes": axes} - - def dict2group(dictionary, group): for key, value in dictionary.items(): if isinstance(value, dict): subgroup = group.create_group(key) dict2group(value, subgroup) else: - group.attrs[key] = value + try: + group.attrs[key] = value + except TypeError: + group.attrs[key] = "_Unsupported_" def _group2dict(group, dictionary=None): if dictionary is None: @@ -398,10 +548,70 @@ def _group2dict(group, dictionary=None): if value.dtype.str.endswith("U1"): value = value.tolist() else: - dictionary[key] = value + value = value + dictionary[key] = value if not isinstance(group, h5py.Dataset): for key in group.keys(): dictionary[key] = {} _group2dict(group[key], dictionary[key]) - return dictionary \ No newline at end of file + return dictionary + +def file_reader(filename, lazy=False, **kwds): + """ + Read data from hdf5-files saved with the HyperSpy hdf5-format + specification (``.hspy``). + + Parameters + ---------- + %s + %s + **kwds : dict, optional + The keyword arguments are passed to :py:class:`h5py.File`. + + %s + """ + try: + # in case blosc compression is used + # module needs to be imported to register plugin + import hdf5plugin # noqa: F401 + except ImportError: + pass + + DM5_file = DM5(filename, mode='r') + images = DM5_file.read_images() + if len(DM5_file.images) == 1: + index = 0 + else:# len(DM5_file.images) > 1: + index = 1 # I think that the first image is a thumbnail... + metadata, original_metadata = DM5_file.images[index].get_metadata() + axes = [] + data = DM5_file.images[index].get_data(lazy=lazy) + shape = data.shape + for axis in range(len(shape)): + axes.append(DM5_file.images[index].get_axis_dict(axis)) + return [{"data": data, "metadata": metadata, "original_metadata": original_metadata, "axes": axes},] + +def file_writer(filename, signal): + """ + Write data to a HDF5 file. + + """ + axes = signal["axes"] # in array orde + data = signal["data"] + metadata = signal["metadata"] + + DM5_file = DM5(filename, mode='w') + DM5_file.write_image(data, axes_dicts=axes, metadata=metadata) + DM5_file.file.close() + + + +file_reader.__doc__ %= (FILENAME_DOC, LAZY_DOC, RETURNS_DOC) + + + + + + + diff --git a/rsciio/tests/test_dm5.py b/rsciio/tests/test_dm5.py new file mode 100644 index 000000000..81d6ae567 --- /dev/null +++ b/rsciio/tests/test_dm5.py @@ -0,0 +1,73 @@ +# -*- coding: utf-8 -*- +# Copyright 2007-2023 The HyperSpy developers +# +# This file is part of RosettaSciIO. +# +# RosettaSciIO is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# RosettaSciIO is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with RosettaSciIO. If not, see . + + +# The EMD format is a hdf5 standard proposed at Lawrence Berkeley +# National Lab (see https://emdatasets.com/ for more information). +# NOT to be confused with the FEI EMD format which was developed later. + + +from pathlib import Path + +import numpy as np +import pytest + +hs = pytest.importorskip("hyperspy.api", reason="hyperspy not installed") + + + +TEST_DATA_PATH = Path(__file__).parent / "data" / "dm5" + + +class TestDM5: + + + @pytest.mark.parametrize("navigation_dimension", [0,1, 2]) + @pytest.mark.parametrize("signal_dimension", [1, 2]) + @pytest.mark.parametrize("dtype", [np.uint8, np.uint16, np.float32, np.float64, np.complex128, + np.int8, np.int16, np.int32,]) + def test_save_load_files(self, navigation_dimension, signal_dimension, dtype, tmp_path): + + fname = tmp_path / f"test_save_files_nav{navigation_dimension}_sig{signal_dimension}_{dtype().dtype}.dm5" + + dim = navigation_dimension + signal_dimension + data_shape = [10,11,12,13,14][:dim] + data = np.ones(data_shape, dtype=dtype) + if signal_dimension == 1: + signal = hs.signals.Signal1D(data) + else: + signal = hs.signals.Signal2D(data) + names = ["a", "b", "c", "d", "e"] + for i in range(dim): + ax = signal.axes_manager[i] + ax.name = names[i]+ str(ax.size) + ax.units = names[i] +" nm" + ax.scale = 0.1 + original = [signal.axes_manager[i].name for i in range(dim)] + signal.save(fname, overwrite=True) + s = hs.load(fname) + + assert s.data.shape == data.shape + assert s.data.dtype == data.dtype + assert s.axes_manager.navigation_dimension == navigation_dimension + assert s.axes_manager.signal_dimension == signal_dimension + for i in range(dim): + assert s.axes_manager[i].name == original[i] + assert "nm" in s.axes_manager[i].units + assert s.axes_manager[i].scale == 0.1 + assert s.axes_manager[i].size == int(original[i][-2:]) \ No newline at end of file From 4073becf4902440d3397525e3514fb5be309c16f Mon Sep 17 00:00:00 2001 From: Carter Francis Date: Mon, 7 Oct 2024 15:33:18 -0500 Subject: [PATCH 3/9] Add specifications.yaml file --- rsciio/dm5/specifications.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 rsciio/dm5/specifications.yaml diff --git a/rsciio/dm5/specifications.yaml b/rsciio/dm5/specifications.yaml new file mode 100644 index 000000000..992dd500a --- /dev/null +++ b/rsciio/dm5/specifications.yaml @@ -0,0 +1,8 @@ +name: DigitalMicrograph5 +name_aliases: [DM5] +description: Read and write data from .dm5 file formats from DigitalMicrograph 5. +full_support: False +file_extensions: [dm5, DM5] +default_extension: 0 +writes: True +non_uniform_axis: False From afe7b5e2a1229183c64bcb7744777601030a9e8b Mon Sep 17 00:00:00 2001 From: Carter Francis Date: Mon, 7 Oct 2024 15:35:04 -0500 Subject: [PATCH 4/9] Add __init__ --- rsciio/dm5/__init__.py | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 rsciio/dm5/__init__.py diff --git a/rsciio/dm5/__init__.py b/rsciio/dm5/__init__.py new file mode 100644 index 000000000..644ecfb0d --- /dev/null +++ b/rsciio/dm5/__init__.py @@ -0,0 +1,8 @@ +from ._api import file_reader, file_writer + +__all__ = ["file_reader", "file_writer"] + + +def __dir__(): + return sorted(__all__) + From 25abc73abc4404f871eaaabf948998cb24e41b0b Mon Sep 17 00:00:00 2001 From: Carter Francis Date: Mon, 7 Oct 2024 15:39:15 -0500 Subject: [PATCH 5/9] Clean up unused code --- rsciio/dm5/_api.py | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/rsciio/dm5/_api.py b/rsciio/dm5/_api.py index 935b00443..9c587367a 100644 --- a/rsciio/dm5/_api.py +++ b/rsciio/dm5/_api.py @@ -1,5 +1,20 @@ -from asyncore import write -from importlib.metadata import metadata +# -*- coding: utf-8 -*- +# Copyright 2007-2023 The HyperSpy developers +# +# This file is part of RosettaSciIO. +# +# RosettaSciIO is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# RosettaSciIO is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with RosettaSciIO. If not, see . import h5py import dask.array as da @@ -7,8 +22,6 @@ from rsciio._docstrings import FILENAME_DOC, LAZY_DOC, RETURNS_DOC from rsciio.hspy._api import HyperspyWriter -from logging import getLogger -_logger = getLogger(__name__) # for some reason this is slightly different in dm4??? @@ -547,8 +560,6 @@ def _group2dict(group, dictionary=None): value = value.astype("U") if value.dtype.str.endswith("U1"): value = value.tolist() - else: - value = value dictionary[key] = value if not isinstance(group, h5py.Dataset): @@ -579,7 +590,7 @@ def file_reader(filename, lazy=False, **kwds): pass DM5_file = DM5(filename, mode='r') - images = DM5_file.read_images() + DM5_file.read_images() if len(DM5_file.images) == 1: index = 0 else:# len(DM5_file.images) > 1: From e873dfcb92824f24d6e2234414d930c863be7b44 Mon Sep 17 00:00:00 2001 From: Carter Francis Date: Mon, 7 Oct 2024 15:57:33 -0500 Subject: [PATCH 6/9] Fix specifications.yaml --- rsciio/dm5/specifications.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rsciio/dm5/specifications.yaml b/rsciio/dm5/specifications.yaml index 992dd500a..1ec4756b5 100644 --- a/rsciio/dm5/specifications.yaml +++ b/rsciio/dm5/specifications.yaml @@ -1,5 +1,5 @@ -name: DigitalMicrograph5 -name_aliases: [DM5] +name: DM5 +name_aliases: [DigitalMicrograph5, Digital Micrograph 5] description: Read and write data from .dm5 file formats from DigitalMicrograph 5. full_support: False file_extensions: [dm5, DM5] From 48bb605f1592de5f5c77167a11f92e3486ca729c Mon Sep 17 00:00:00 2001 From: Carter Francis Date: Mon, 7 Oct 2024 16:03:39 -0500 Subject: [PATCH 7/9] BugFix: Fix Brightness return --- rsciio/dm5/_api.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/rsciio/dm5/_api.py b/rsciio/dm5/_api.py index 9c587367a..7f0c464e5 100644 --- a/rsciio/dm5/_api.py +++ b/rsciio/dm5/_api.py @@ -387,12 +387,13 @@ def update_calibration(self, axis, name="", offset=0, scale=1, units="", **kwds) self.image_data['Calibrations'].attrs.update({"DisplayCalibratedUnits": 1}) + def brightness(self): """ Get the brightness of the image. """ try: - dict(self.image_data["Calibrations"]["Brightness"].attrs) + return dict(self.image_data["Calibrations"]["Brightness"].attrs) except KeyError: return {} @@ -586,7 +587,7 @@ def file_reader(filename, lazy=False, **kwds): # in case blosc compression is used # module needs to be imported to register plugin import hdf5plugin # noqa: F401 - except ImportError: + except ImportError: # pragma: no cover pass DM5_file = DM5(filename, mode='r') From 527061561b8205eb8b81ca6c4da9a3a03ff8127d Mon Sep 17 00:00:00 2001 From: Carter Francis Date: Mon, 7 Oct 2024 16:24:33 -0500 Subject: [PATCH 8/9] Refactor:Fix Failing Tests --- rsciio/dm5/__init__.py | 1 - rsciio/dm5/_api.py | 481 ++++++++++++++++++++---------------- rsciio/tests/test_dm5.py | 38 ++- rsciio/tests/test_import.py | 4 +- 4 files changed, 298 insertions(+), 226 deletions(-) diff --git a/rsciio/dm5/__init__.py b/rsciio/dm5/__init__.py index 644ecfb0d..49230cbba 100644 --- a/rsciio/dm5/__init__.py +++ b/rsciio/dm5/__init__.py @@ -5,4 +5,3 @@ def __dir__(): return sorted(__all__) - diff --git a/rsciio/dm5/_api.py b/rsciio/dm5/_api.py index 7f0c464e5..64c8eed10 100644 --- a/rsciio/dm5/_api.py +++ b/rsciio/dm5/_api.py @@ -16,32 +16,29 @@ # You should have received a copy of the GNU General Public License # along with RosettaSciIO. If not, see . -import h5py import dask.array as da +import h5py import numpy as np from rsciio._docstrings import FILENAME_DOC, LAZY_DOC, RETURNS_DOC from rsciio.hspy._api import HyperspyWriter - # for some reason this is slightly different in dm4??? # Data Type 10: Integer? # data_types = { - np.short().dtype: [2, 2], # 2 byte integer signed + np.short().dtype: [2, 2], # 2 byte integer signed np.float32().dtype: [2, 4], # 4 byte real (IEEE 754) - - np.uint8().dtype: [6, 1], # 1 byte integer unsigned - np.int8().dtype: [6, 1], # 1 byte integer signed + np.uint8().dtype: [6, 1], # 1 byte integer unsigned + np.int8().dtype: [6, 1], # 1 byte integer signed np.int16().dtype: [1, 2], # 2 byte integer signed np.int32().dtype: [7, 4], # 4 byte integer signed np.uint32().dtype: [7, 4], # 4 byte integer signed - np.float64().dtype: [12, 8], # 8 byte real (IEEE 754) - np.uint16().dtype: [10,2], # 2 byte integer unsigned - np.complex64().dtype: [13, 8], # 8 byte complex - np.complex128().dtype: [13, 16], # 16 byte complex - - } + np.float64().dtype: [12, 8], # 8 byte real (IEEE 754) + np.uint16().dtype: [10, 2], # 2 byte integer unsigned + np.complex64().dtype: [13, 8], # 8 byte complex + np.complex128().dtype: [13, 16], # 16 byte complex +} class DM5: @@ -60,11 +57,12 @@ class DM5: """ - def __init__(self, file_path, mode='r'): + + def __init__(self, file_path, mode="r"): self.file = h5py.File(file_path, mode=mode) - if mode == 'r': - self.image_list = self.file["ImageList"] # list of images + if mode == "r": + self.image_list = self.file["ImageList"] # list of images else: self.image_list = self.file.create_group("ImageList") @@ -79,151 +77,192 @@ def write_sources(self, signal_dimensions, navigation_dimensions): self.file.create_group("ImageSourceList") self.file["ImageSourceList"].create_group("[0]") - if signal_dimensions==2 and navigation_dimensions ==2: # this is for 4D Data - self.file["ImageSourceList"]["[0]"].attrs.update({"ClassName":"ImageSource:4DSummed", # Need the right ClassName - "Do Sum": 1, # Sum the image? - "ImageRef":0, # Image to take the sum of... (Similar to Hyperspy) - "LayerFstEnd": 0, - "LayerFstStart": 0, - "LayerSndEnd": 0, - "LayerSndStart": 0, - "Summed Fst Dimension": 0, # Sum 1st dimension - "Summed Snd Dimension": 1, # Sum 2nd dimension - }) # data is reversed from usual in DM but not numpy + if signal_dimensions == 2 and navigation_dimensions == 2: # this is for 4D Data + self.file["ImageSourceList"]["[0]"].attrs.update( + { + "ClassName": "ImageSource:4DSummed", # Need the right ClassName + "Do Sum": 1, # Sum the image? + "ImageRef": 0, # Image to take the sum of... (Similar to Hyperspy) + "LayerFstEnd": 0, + "LayerFstStart": 0, + "LayerSndEnd": 0, + "LayerSndStart": 0, + "Summed Fst Dimension": 0, # Sum 1st dimension + "Summed Snd Dimension": 1, # Sum 2nd dimension + } + ) # data is reversed from usual in DM but not numpy self.file["ImageSourceList"]["[0]"].create_group("Id") self.file["ImageSourceList"]["[0]"]["Id"].attrs.update({"[0]": 0}) # Write the Document Objects similar to ROIs in Hyperspy for Visualization self.file.create_group("DocumentObjectList") self.file["DocumentObjectList"].create_group("[0]") - self.file["DocumentObjectList"]["[0]"].attrs.update({"AnnotationType": 20, # Create an image - 'ImageSource': 0, # - 'ImageDisplayType': 1, # 1 = Image - 'RangeAdjust': 1.0, - 'SparseSurvey_GridSize': 32, - 'SparseSurvey_NumberPixels': 64, # fast survey? - 'SparseSurvey_UseNumberPixels': 1, - 'SurveyTechnique': 2, - }) + self.file["DocumentObjectList"]["[0]"].attrs.update( + { + "AnnotationType": 20, # Create an image + "ImageSource": 0, # + "ImageDisplayType": 1, # 1 = Image + "RangeAdjust": 1.0, + "SparseSurvey_GridSize": 32, + "SparseSurvey_NumberPixels": 64, # fast survey? + "SparseSurvey_UseNumberPixels": 1, + "SurveyTechnique": 2, + } + ) self.file["DocumentObjectList"]["[0]"].create_group("ImageDisplayInfo") - self.file["DocumentObjectList"]["[0]"]["ImageDisplayInfo"].attrs.update({"EstimatedMin": 0, - "HiLimitContrastDeltaTriggerPercentage": 0, - }) - - - elif signal_dimensions == 1 and navigation_dimensions == 2: # Spectrum Image (Data might need to be reversed...) - self.file["ImageSourceList"]["[0]"].attrs.update({"ClassName": "ImageSource:Summed", - "Do Sum": 1, - "ImageRef": 0, - "LayerEnd": 0, - "LayerStart": 0, - "Summed Dimension": 2, - }) # Sum last dimension + self.file["DocumentObjectList"]["[0]"]["ImageDisplayInfo"].attrs.update( + { + "EstimatedMin": 0, + "HiLimitContrastDeltaTriggerPercentage": 0, + } + ) + + elif ( + signal_dimensions == 1 and navigation_dimensions == 2 + ): # Spectrum Image (Data might need to be reversed...) + self.file["ImageSourceList"]["[0]"].attrs.update( + { + "ClassName": "ImageSource:Summed", + "Do Sum": 1, + "ImageRef": 0, + "LayerEnd": 0, + "LayerStart": 0, + "Summed Dimension": 2, + } + ) # Sum last dimension self.file["ImageSourceList"]["[0]"].create_group("Id") self.file["ImageSourceList"]["[0]"]["Id"].attrs.update({"[0]": 0}) # Write the Document Objects similar to ROIs in Hyperspy for Visualization. self.file.create_group("DocumentObjectList") self.file["DocumentObjectList"].create_group("[0]") - self.file["DocumentObjectList"]["[0]"].attrs.update({"AnnotationType": 20, - 'ImageSource': 0, # Reference to the ImageSource - 'ImageDisplayType': 1, # 1 = Image? - 'IsMoveable': 1, - 'IsResizable': 1, - 'IsSelectable': 1, - 'IsTransferrable': 1, #sp? - 'IsTranslatable': 1, - 'IsVisible': 1, - 'UniqueID': 8, - }) - - - - elif signal_dimensions ==2 and navigation_dimensions==1: # Image Stack (In Situ) + self.file["DocumentObjectList"]["[0]"].attrs.update( + { + "AnnotationType": 20, + "ImageSource": 0, # Reference to the ImageSource + "ImageDisplayType": 1, # 1 = Image? + "IsMoveable": 1, + "IsResizable": 1, + "IsSelectable": 1, + "IsTransferrable": 1, # sp? + "IsTranslatable": 1, + "IsVisible": 1, + "UniqueID": 8, + } + ) + + elif ( + signal_dimensions == 2 and navigation_dimensions == 1 + ): # Image Stack (In Situ) self.file["ImageSourceList"]["[0]"].attrs.update( - {"ClassName": "ImageSource:Summed", # Need the right ImageSource! - "Do Sum": 1, # Sum the image? - "ImageRef": 0, # Image to take the sum of... (Similar to Hyperspy) - "LayerEnd": 0, - "LayerStart": 0, - "Summed Dimension": 2, # Sum last dimension? - }) + { + "ClassName": "ImageSource:Summed", # Need the right ImageSource! + "Do Sum": 1, # Sum the image? + "ImageRef": 0, # Image to take the sum of... (Similar to Hyperspy) + "LayerEnd": 0, + "LayerStart": 0, + "Summed Dimension": 2, # Sum last dimension? + } + ) self.file.attrs.update({"InImageMode": 1}) self.file["ImageSourceList"]["[0]"].create_group("Id") self.file["ImageSourceList"]["[0]"]["Id"].attrs.update({"[0]": 0}) self.file.create_group("DocumentObjectList") self.file["DocumentObjectList"].create_group("[0]") - self.file["DocumentObjectList"]["[0]"].attrs.update({"AnnotationType": 20, - 'ImageSource': 0, # Reference to the ImageSource - 'ImageDisplayType': 1, # 1 = Image? - 'IsMoveable': 1, - 'IsResizable': 1, - 'IsSelectable': 1, - 'IsTransferrable': 1, #sp? - 'IsTranslatable': 1, - 'IsVisible': 1, - 'UniqueID': 8, - }) + self.file["DocumentObjectList"]["[0]"].attrs.update( + { + "AnnotationType": 20, + "ImageSource": 0, # Reference to the ImageSource + "ImageDisplayType": 1, # 1 = Image? + "IsMoveable": 1, + "IsResizable": 1, + "IsSelectable": 1, + "IsTransferrable": 1, # sp? + "IsTranslatable": 1, + "IsVisible": 1, + "UniqueID": 8, + } + ) self.file["DocumentObjectList"]["[0]"].create_group("AnnotationGroupList") - self.file["DocumentObjectList"]["[0]"]["AnnotationGroupList"].create_group("[0]") - self.file["DocumentObjectList"]["[0]"]["AnnotationGroupList"]["[0]"].attrs.update({"AnnotationType": 23, - "Name": "SICursor", - "Rectangle": (0, 0, 1, 1), - }) + self.file["DocumentObjectList"]["[0]"]["AnnotationGroupList"].create_group( + "[0]" + ) + self.file["DocumentObjectList"]["[0]"]["AnnotationGroupList"][ + "[0]" + ].attrs.update( + { + "AnnotationType": 23, + "Name": "SICursor", + "Rectangle": (0, 0, 1, 1), + } + ) self.file["DocumentObjectList"]["[0]"].create_group("ImageDisplayInfo") - self.file["DocumentObjectList"]["[0]"]["ImageDisplayInfo"].attrs.update({"EstimatedMin": 0, - "HiLimitContrastDeltaTriggerPercentage": 0}) + self.file["DocumentObjectList"]["[0]"]["ImageDisplayInfo"].attrs.update( + {"EstimatedMin": 0, "HiLimitContrastDeltaTriggerPercentage": 0} + ) elif signal_dimensions == 1 and navigation_dimensions == 1: self.file["ImageSourceList"]["[0]"].attrs.update( - {"ClassName": "ImageSource:Summed", # Need the right ImageSource! - "Do Sum": 1, # Sum the image? - "ImageRef": 0, # Image to take the sum of... (Similar to Hyperspy) - "LayerEnd": 0, - "LayerStart": 0, - "Summed Dimension": 1, # Sum last dimension? - }) + { + "ClassName": "ImageSource:Summed", # Need the right ImageSource! + "Do Sum": 1, # Sum the image? + "ImageRef": 0, # Image to take the sum of... (Similar to Hyperspy) + "LayerEnd": 0, + "LayerStart": 0, + "Summed Dimension": 1, # Sum last dimension? + } + ) self.file["ImageSourceList"]["[0]"].create_group("Id") self.file["ImageSourceList"]["[0]"]["Id"].attrs.update({"[0]": 0}) self.file.create_group("DocumentObjectList") self.file["DocumentObjectList"].create_group("[0]") - self.file["DocumentObjectList"]["[0]"].attrs.update({"AnnotationType": 20, - 'ImageSource': 0, # Reference to the ImageSource - 'ImageDisplayType': 1, # 1 = Image? - 'RangeAdjust': 1.0, - 'IsMoveable': 1, - 'IsResizable': 1, - 'IsSelectable': 1, - 'IsTransferrable': 1, # sp? - 'IsTranslatable': 1, - 'IsVisible': 1, - 'UniqueID': 8, - }) + self.file["DocumentObjectList"]["[0]"].attrs.update( + { + "AnnotationType": 20, + "ImageSource": 0, # Reference to the ImageSource + "ImageDisplayType": 1, # 1 = Image? + "RangeAdjust": 1.0, + "IsMoveable": 1, + "IsResizable": 1, + "IsSelectable": 1, + "IsTransferrable": 1, # sp? + "IsTranslatable": 1, + "IsVisible": 1, + "UniqueID": 8, + } + ) else: self.file["ImageSourceList"]["[0]"].attrs.update( - {"ClassName": "ImageSource:Simple", # Need the right ImageSource! - "ImageRef": 0, # Reference to self (Usually 0 is a ??thumbnail?? which we don't write) - }) + { + "ClassName": "ImageSource:Simple", # Need the right ImageSource! + "ImageRef": 0, # Reference to self (Usually 0 is a ??thumbnail?? which we don't write) + } + ) self.file.create_group("DocumentObjectList") self.file["DocumentObjectList"].create_group("[0]") - self.file["DocumentObjectList"]["[0]"].attrs.update({"AnnotationType": 20, - 'ImageSource': 0, # Reference to the ImageSource - 'ImageDisplayType': 1, # 1 = Image? - 'RangeAdjust': 1.0, - 'IsMoveable': 1, - 'IsResizable': 1, - 'IsSelectable': 1, - 'IsTransferrable': 1, # sp? - 'IsTranslatable': 1, - 'IsVisible': 1, - 'UniqueID': 8, - }) + self.file["DocumentObjectList"]["[0]"].attrs.update( + { + "AnnotationType": 20, + "ImageSource": 0, # Reference to the ImageSource + "ImageDisplayType": 1, # 1 = Image? + "RangeAdjust": 1.0, + "IsMoveable": 1, + "IsResizable": 1, + "IsSelectable": 1, + "IsTransferrable": 1, # sp? + "IsTranslatable": 1, + "IsVisible": 1, + "UniqueID": 8, + } + ) def write_header_info(self): """ Just write the minimum "header" info to open the file in DM. """ - self.file.attrs.update({"InImageMode": 1}) # The rest of the attributes are for defining the window size... + self.file.attrs.update( + {"InImageMode": 1} + ) # The rest of the attributes are for defining the window size... def write_image_behavior(self): """ @@ -238,14 +277,14 @@ def read_images(self): for image_group in self.image_list.values(): self.images.append(Image(image_group, file=self.file)) - def write_image(self, data, axes_dicts, metadata =None, brightness=None): + def write_image(self, data, axes_dicts, metadata=None, brightness=None): """ Write an image to the DM5 file. """ previous_images = [int(k.strip("[ ]")) for k in self.image_list.keys()] if len(previous_images) == 0: - new_image_number = '[0]' + new_image_number = "[0]" else: new_image_number = f"[{max(previous_images) + 1}]" @@ -256,25 +295,29 @@ def write_image(self, data, axes_dicts, metadata =None, brightness=None): image = Image(self.image_list[new_image_number], file=self.file) self.images.append(image) - navigation_dimensions = len([axis for axis in axes_dicts if axis["navigate"] == True]) - signal_dimensions = len([axis for axis in axes_dicts if axis["navigate"] == False]) + navigation_dimensions = len([axis for axis in axes_dicts if axis["navigate"]]) + signal_dimensions = len([axis for axis in axes_dicts if not axis["navigate"]]) image.update_data(data) for axis, axis_dict in enumerate(axes_dicts): image.update_calibration(axis, **axis_dict) image.update_dimension(axis, axis_dict["size"]) - image.update_metadata(metadata, signal_dimensions=signal_dimensions, - navigation_dimensions=navigation_dimensions) + image.update_metadata( + metadata, + signal_dimensions=signal_dimensions, + navigation_dimensions=navigation_dimensions, + ) image.update_brightness(brightness) - self.write_sources(navigation_dimensions= navigation_dimensions, - signal_dimensions=signal_dimensions) + self.write_sources( + navigation_dimensions=navigation_dimensions, + signal_dimensions=signal_dimensions, + ) self.write_image_behavior() self.write_header_info() - class Image: """ The internal representation of an image in a DM5 file. @@ -289,9 +332,9 @@ class Image: """ def __init__(self, image_group, tags=None, unique_id=None, file=None): - self.image_data = image_group['ImageData'] - self.image_tags = image_group['ImageTags'] - self.unique_id = image_group['UniqueID'] + self.image_data = image_group["ImageData"] + self.image_tags = image_group["ImageTags"] + self.unique_id = image_group["UniqueID"] self.file = file def __str__(self): @@ -299,7 +342,7 @@ def __str__(self): @property def ndim(self): - return len(self.image_data['Data'].shape) + return len(self.image_data["Data"].shape) def signal_dimensions(self): """ @@ -316,12 +359,14 @@ def signal_dimensions(self): return 1 elif format == "Image": return 2 - else: # Now we just try to guess from the DocumentObjectList + else: # Now we just try to guess from the DocumentObjectList try: - class_name =self.file["ImageSourceList"]["[0]"].attrs["ClassName"] + class_name = self.file["ImageSourceList"]["[0]"].attrs["ClassName"] except KeyError: - raise KeyError("A Valid DM5 File needs to have an ImageSourceList with a " - "ClassName attribute to determine how the data should be displayed.") + raise KeyError( + "A Valid DM5 File needs to have an ImageSourceList with a " + "ClassName attribute to determine how the data should be displayed." + ) if self.ndim == 2 and class_name == "ImageSource:Simple": return 2 elif self.ndim == 1 and class_name == "ImageSource:Simple": @@ -329,10 +374,12 @@ def signal_dimensions(self): elif self.ndim == 4 and class_name == "ImageSource:4DSummed": return 2 else: + raise NotImplementedError( + "Determining the Type for unlabeled DM5 files. Please add the" + " 'Image', 'Spectrum', 'Spectrum Image' 'Diffraction image' tag to the" + "Meta Data.Format attribute in the ImageTags group." + ) - raise NotImplementedError("Determining the Type for unlabeled DM5 files. Please add the" - " 'Image', 'Spectrum', 'Spectrum Image' 'Diffraction image' tag to the" - "Meta Data.Format attribute in the ImageTags group.") def navigation_dimensions(self): return self.ndim - self.signal_dimensions() @@ -352,14 +399,17 @@ def get_axis_dict(self, axis): axis = self.ndim - axis - 1 navigate = axis >= self.signal_dimensions() - calibration_dict = dict(self.image_data['Calibrations']["Dimension"][f"[{axis}]"].attrs) - axis_dict = {"name": calibration_dict.get("Label", ""), - "offset": calibration_dict.get("Origin", 0), - "scale": calibration_dict.get("Scale", 1), - "units": calibration_dict.get("Units", ""), - "size": self._get_dimension(axis), - "navigate": navigate - } + calibration_dict = dict( + self.image_data["Calibrations"]["Dimension"][f"[{axis}]"].attrs + ) + axis_dict = { + "name": calibration_dict.get("Label", ""), + "offset": calibration_dict.get("Origin", 0), + "scale": calibration_dict.get("Scale", 1), + "units": calibration_dict.get("Units", ""), + "size": self._get_dimension(axis), + "navigate": navigate, + } return axis_dict @@ -368,25 +418,25 @@ def update_calibration(self, axis, name="", offset=0, scale=1, units="", **kwds) Add calibration data to the image for a given axis. """ axis = self.ndim - axis - 1 - if not "Calibrations" in self.image_data: + if "Calibrations" not in self.image_data: self.image_data.create_group("Calibrations") - if not "Dimension" in self.image_data['Calibrations']: - self.image_data['Calibrations'].create_group("Dimension") - if not f"[{axis}]" in self.image_data['Calibrations']["Dimension"]: - self.image_data['Calibrations']["Dimension"].create_group(f"[{axis}]") - if not isinstance(units, str): # Handle Undefined Units + if "Dimension" not in self.image_data["Calibrations"]: + self.image_data["Calibrations"].create_group("Dimension") + if f"[{axis}]" not in self.image_data["Calibrations"]["Dimension"]: + self.image_data["Calibrations"]["Dimension"].create_group(f"[{axis}]") + if not isinstance(units, str): # Handle Undefined Units units = "" if not isinstance(name, str): name = "" - self.image_data['Calibrations']["Dimension"][f"[{axis}]"].attrs.update({ - "Label": name, - "Origin": float(offset), - "Scale": float(scale), - "Units": units - }) - self.image_data['Calibrations'].attrs.update({"DisplayCalibratedUnits": 1}) - - + self.image_data["Calibrations"]["Dimension"][f"[{axis}]"].attrs.update( + { + "Label": name, + "Origin": float(offset), + "Scale": float(scale), + "Units": units, + } + ) + self.image_data["Calibrations"].attrs.update({"DisplayCalibratedUnits": 1}) def brightness(self): """ @@ -402,13 +452,10 @@ def update_brightness(self, brightness=None): Update the brightness of the image. """ if brightness is None: - brightness = {"Label": "b", - "Origin": 0, - "Scale": 1, - "Units": "b"} - if not "Calibrations" in self.image_data: + brightness = {"Label": "b", "Origin": 0, "Scale": 1, "Units": "b"} + if "Calibrations" not in self.image_data: self.image_data.create_group("Calibrations") - if not "Brightness" in self.image_data["Calibrations"]: + if "Brightness" not in self.image_data["Calibrations"]: self.image_data["Calibrations"].create_group("Brightness") self.image_data["Calibrations"]["Brightness"].attrs.update(brightness) @@ -422,17 +469,17 @@ def update_dimension(self, axis, length=None): """ axis = self.ndim - axis - 1 - if not "Dimensions" in self.image_data: + if "Dimensions" not in self.image_data: self.image_data.create_group("Dimensions") if length is None: length = self._get_dimension(axis) - self.image_data['Dimensions'].attrs.update({f"[{axis}]": length}) + self.image_data["Dimensions"].attrs.update({f"[{axis}]": length}) def _get_dimension(self, axis): try: - return self.image_data['Dimensions'].attrs[f"[{axis}]"] + return self.image_data["Dimensions"].attrs[f"[{axis}]"] except KeyError: - shape = self.image_data['Data'].shape + shape = self.image_data["Data"].shape return shape[axis] def get_data(self, lazy=False): @@ -440,20 +487,21 @@ def get_data(self, lazy=False): Get the image data. """ if lazy: - return da.from_array(self.image_data['Data']) + return da.from_array(self.image_data["Data"]) else: - return np.array(self.image_data['Data']) + return np.array(self.image_data["Data"]) def update_data(self, data): """ Update the image data. """ - HyperspyWriter.overwrite_dataset(self.image_data, - data, - "Data") - self.image_data.attrs.update({"DataType": data_types[data.dtype][0], - "PixelDepth": data_types[data.dtype][1]}) - + HyperspyWriter.overwrite_dataset(self.image_data, data, "Data") + self.image_data.attrs.update( + { + "DataType": data_types[data.dtype][0], + "PixelDepth": data_types[data.dtype][1], + } + ) def get_metadata(self): """ @@ -464,8 +512,7 @@ def get_metadata(self): metadata = {} metadata["General"] = {} - metadata["General"]["title"] = "" # DM uses the filename as the title - + metadata["General"]["title"] = "" # DM uses the filename as the title if "Acquisition" in original_metadata: metadata["Acquisition"] = original_metadata["Acquisition"] @@ -476,19 +523,24 @@ def get_metadata(self): metadata["Acquisition_instrument"] = {} metadata["Acquisition_instrument"]["TEM"] = {} metadata["Acquisition_instrument"]["TEM"]["beam_energy "] = ( - original_metadata["Microscope Info"].get("Voltage", 0)/1000) + original_metadata["Microscope Info"].get("Voltage", 0) / 1000 + ) metadata["Acquisition_instrument"]["TEM"]["acquisition_mode"] = ( - original_metadata["Microscope Info"].get("Illumination Mode", "Unknown")) + original_metadata["Microscope Info"].get("Illumination Mode", "Unknown") + ) metadata["Acquisition_instrument"]["TEM"]["magnification"] = ( - original_metadata["Microscope Info"].get("Indicated Magnification", 0)) + original_metadata["Microscope Info"].get("Indicated Magnification", 0) + ) metadata["Acquisition_instrument"]["TEM"]["camera_length"] = ( - original_metadata["Microscope Info"].get("STEM Camera Length", 0)) + original_metadata["Microscope Info"].get("STEM Camera Length", 0) + ) metadata["Acquisition_instrument"] = original_metadata["Microscope Info"] metadata["Acquisition_instrument"] = {} return metadata, original_metadata - - def update_metadata(self, metadata=None, signal_dimensions=None, navigation_dimensions=None): + def update_metadata( + self, metadata=None, signal_dimensions=None, navigation_dimensions=None + ): """ Update the metadata for the image. """ @@ -498,11 +550,13 @@ def update_metadata(self, metadata=None, signal_dimensions=None, navigation_dime formatted_metadata["Acquisition"] = {} formatted_metadata["Meta Data"] = {} - if signal_dimensions == 2 and navigation_dimensions == 2: # 4D Data + if signal_dimensions == 2 and navigation_dimensions == 2: # 4D Data formatted_metadata["Meta Data"]["Format"] = "Diffraction image" formatted_metadata["Meta Data"]["Acquisition Mode"] = "Parallel imaging" formatted_metadata["Meta Data"]["Experiment keywords"] = {} - formatted_metadata["Meta Data"]["Experiment keywords"]["[0]"] = "Label: Diffraction" + formatted_metadata["Meta Data"]["Experiment keywords"]["[0]"] = ( + "Label: Diffraction" + ) elif signal_dimensions == 1 and navigation_dimensions == 2: formatted_metadata["Meta Data"]["Format"] = "Spectrum image" elif signal_dimensions == 1 and navigation_dimensions == 1: @@ -528,7 +582,12 @@ def to_signal_dict(self): axes = [] for axis in range(len(data.shape)): axes.append(self.get_axis_dict(axis)) - return {"data": data, "metadata": metadata, "original_metadata": original_metadata, "axes": axes} + return { + "data": data, + "metadata": metadata, + "original_metadata": original_metadata, + "axes": axes, + } def dict2group(dictionary, group): @@ -542,6 +601,7 @@ def dict2group(dictionary, group): except TypeError: group.attrs[key] = "_Unsupported_" + def _group2dict(group, dictionary=None): if dictionary is None: dictionary = {} @@ -569,6 +629,7 @@ def _group2dict(group, dictionary=None): _group2dict(group[key], dictionary[key]) return dictionary + def file_reader(filename, lazy=False, **kwds): """ Read data from hdf5-files saved with the HyperSpy hdf5-format @@ -587,43 +648,43 @@ def file_reader(filename, lazy=False, **kwds): # in case blosc compression is used # module needs to be imported to register plugin import hdf5plugin # noqa: F401 - except ImportError: # pragma: no cover + except ImportError: # pragma: no cover pass - DM5_file = DM5(filename, mode='r') + DM5_file = DM5(filename, mode="r") DM5_file.read_images() if len(DM5_file.images) == 1: index = 0 - else:# len(DM5_file.images) > 1: - index = 1 # I think that the first image is a thumbnail... + else: # len(DM5_file.images) > 1: + index = 1 # I think that the first image is a thumbnail... metadata, original_metadata = DM5_file.images[index].get_metadata() axes = [] data = DM5_file.images[index].get_data(lazy=lazy) shape = data.shape for axis in range(len(shape)): axes.append(DM5_file.images[index].get_axis_dict(axis)) - return [{"data": data, "metadata": metadata, "original_metadata": original_metadata, "axes": axes},] + return [ + { + "data": data, + "metadata": metadata, + "original_metadata": original_metadata, + "axes": axes, + }, + ] + def file_writer(filename, signal): """ Write data to a HDF5 file. """ - axes = signal["axes"] # in array orde + axes = signal["axes"] # in array orde data = signal["data"] metadata = signal["metadata"] - DM5_file = DM5(filename, mode='w') + DM5_file = DM5(filename, mode="w") DM5_file.write_image(data, axes_dicts=axes, metadata=metadata) DM5_file.file.close() - file_reader.__doc__ %= (FILENAME_DOC, LAZY_DOC, RETURNS_DOC) - - - - - - - diff --git a/rsciio/tests/test_dm5.py b/rsciio/tests/test_dm5.py index 81d6ae567..7e3adc962 100644 --- a/rsciio/tests/test_dm5.py +++ b/rsciio/tests/test_dm5.py @@ -30,23 +30,35 @@ hs = pytest.importorskip("hyperspy.api", reason="hyperspy not installed") - TEST_DATA_PATH = Path(__file__).parent / "data" / "dm5" class TestDM5: - - - @pytest.mark.parametrize("navigation_dimension", [0,1, 2]) + @pytest.mark.parametrize("navigation_dimension", [0, 1, 2]) @pytest.mark.parametrize("signal_dimension", [1, 2]) - @pytest.mark.parametrize("dtype", [np.uint8, np.uint16, np.float32, np.float64, np.complex128, - np.int8, np.int16, np.int32,]) - def test_save_load_files(self, navigation_dimension, signal_dimension, dtype, tmp_path): - - fname = tmp_path / f"test_save_files_nav{navigation_dimension}_sig{signal_dimension}_{dtype().dtype}.dm5" + @pytest.mark.parametrize( + "dtype", + [ + np.uint8, + np.uint16, + np.float32, + np.float64, + np.complex128, + np.int8, + np.int16, + np.int32, + ], + ) + def test_save_load_files( + self, navigation_dimension, signal_dimension, dtype, tmp_path + ): + fname = ( + tmp_path + / f"test_save_files_nav{navigation_dimension}_sig{signal_dimension}_{dtype().dtype}.dm5" + ) dim = navigation_dimension + signal_dimension - data_shape = [10,11,12,13,14][:dim] + data_shape = [10, 11, 12, 13, 14][:dim] data = np.ones(data_shape, dtype=dtype) if signal_dimension == 1: signal = hs.signals.Signal1D(data) @@ -55,8 +67,8 @@ def test_save_load_files(self, navigation_dimension, signal_dimension, dtype, tm names = ["a", "b", "c", "d", "e"] for i in range(dim): ax = signal.axes_manager[i] - ax.name = names[i]+ str(ax.size) - ax.units = names[i] +" nm" + ax.name = names[i] + str(ax.size) + ax.units = names[i] + " nm" ax.scale = 0.1 original = [signal.axes_manager[i].name for i in range(dim)] signal.save(fname, overwrite=True) @@ -70,4 +82,4 @@ def test_save_load_files(self, navigation_dimension, signal_dimension, dtype, tm assert s.axes_manager[i].name == original[i] assert "nm" in s.axes_manager[i].units assert s.axes_manager[i].scale == 0.1 - assert s.axes_manager[i].size == int(original[i][-2:]) \ No newline at end of file + assert s.axes_manager[i].size == int(original[i][-2:]) diff --git a/rsciio/tests/test_import.py b/rsciio/tests/test_import.py index dbe849fb1..1ca02f548 100644 --- a/rsciio/tests/test_import.py +++ b/rsciio/tests/test_import.py @@ -44,7 +44,7 @@ def test_import_all(): # Remove plugins which require not installed optional dependencies h5py = importlib.util.find_spec("h5py") if h5py is None: - plugin_name_to_remove.extend(["EMD", "HSPY", "NeXus"]) + plugin_name_to_remove.extend(["EMD", "HSPY", "NeXus", "DM5"]) imageio = importlib.util.find_spec("imageio") if imageio is None: @@ -123,7 +123,7 @@ def test_dir_plugins(plugin): pytest.importorskip("pyUSID") elif plugin["name"] == "ZSPY": pytest.importorskip("zarr") - elif plugin["name"] in ["EMD", "HSPY", "NeXus"]: + elif plugin["name"] in ["EMD", "HSPY", "NeXus", "DM5"]: pytest.importorskip("h5py") plugin_module = importlib.import_module(plugin_string) From 06ea31410cac03b3319b087407691be1f4f8ada2 Mon Sep 17 00:00:00 2001 From: Carter Francis Date: Mon, 7 Oct 2024 17:57:52 -0500 Subject: [PATCH 9/9] Testing:Expand Test coverage --- rsciio/dm5/_api.py | 105 ++++++++++++++++++++++++--------------- rsciio/tests/test_dm5.py | 42 ++++++++++++++++ 2 files changed, 107 insertions(+), 40 deletions(-) diff --git a/rsciio/dm5/_api.py b/rsciio/dm5/_api.py index 64c8eed10..e6d070589 100644 --- a/rsciio/dm5/_api.py +++ b/rsciio/dm5/_api.py @@ -337,9 +337,6 @@ def __init__(self, image_group, tags=None, unique_id=None, file=None): self.unique_id = image_group["UniqueID"] self.file = file - def __str__(self): - return f"Image: {self.image_data['Data'].shape}" - @property def ndim(self): return len(self.image_data["Data"].shape) @@ -362,7 +359,7 @@ def signal_dimensions(self): else: # Now we just try to guess from the DocumentObjectList try: class_name = self.file["ImageSourceList"]["[0]"].attrs["ClassName"] - except KeyError: + except KeyError: # pragma: no cover raise KeyError( "A Valid DM5 File needs to have an ImageSourceList with a " "ClassName attribute to determine how the data should be displayed." @@ -380,9 +377,6 @@ def signal_dimensions(self): "Meta Data.Format attribute in the ImageTags group." ) - def navigation_dimensions(self): - return self.ndim - self.signal_dimensions() - def get_axis_dict(self, axis): """ Get the calibration data for a given axis. @@ -441,10 +435,15 @@ def update_calibration(self, axis, name="", offset=0, scale=1, units="", **kwds) def brightness(self): """ Get the brightness of the image. + + Note + ---- + Hyperspy has a rudimentary concept of brightness This currently doesn't do anything but should be implemented + in the future. """ - try: + try: # pragma: no cover return dict(self.image_data["Calibrations"]["Brightness"].attrs) - except KeyError: + except KeyError: # pragma: no cover return {} def update_brightness(self, brightness=None): @@ -463,9 +462,13 @@ def update_dimension(self, axis, length=None): """ Update the dimension of the image for a given axis. - This is two places in the DM5 file??? - - Under Calibrations and under Dimension. I think that only the Calibrations should be updated. + Parameters + ---------- + axis : int + The axis to update the dimension for (Starting from 0 in array order). This will be reversed to match DM's + axis order. + length : int, optional + The length of the axis. """ axis = self.ndim - axis - 1 @@ -478,13 +481,19 @@ def update_dimension(self, axis, length=None): def _get_dimension(self, axis): try: return self.image_data["Dimensions"].attrs[f"[{axis}]"] - except KeyError: + except KeyError: # pragma: no cover shape = self.image_data["Data"].shape return shape[axis] def get_data(self, lazy=False): """ Get the image data. + + Parameters + ---------- + lazy : bool, optional + Whether to return a dask array or a numpy + """ if lazy: return da.from_array(self.image_data["Data"]) @@ -494,6 +503,11 @@ def get_data(self, lazy=False): def update_data(self, data): """ Update the image data. + + Parameters + ---------- + data : np.ndarray or da.Array + The new image data. """ HyperspyWriter.overwrite_dataset(self.image_data, data, "Data") self.image_data.attrs.update( @@ -522,7 +536,7 @@ def get_metadata(self): if "Microscope Info" in original_metadata: metadata["Acquisition_instrument"] = {} metadata["Acquisition_instrument"]["TEM"] = {} - metadata["Acquisition_instrument"]["TEM"]["beam_energy "] = ( + metadata["Acquisition_instrument"]["TEM"]["beam_energy"] = ( original_metadata["Microscope Info"].get("Voltage", 0) / 1000 ) metadata["Acquisition_instrument"]["TEM"]["acquisition_mode"] = ( @@ -534,8 +548,6 @@ def get_metadata(self): metadata["Acquisition_instrument"]["TEM"]["camera_length"] = ( original_metadata["Microscope Info"].get("STEM Camera Length", 0) ) - metadata["Acquisition_instrument"] = original_metadata["Microscope Info"] - metadata["Acquisition_instrument"] = {} return metadata, original_metadata def update_metadata( @@ -543,9 +555,21 @@ def update_metadata( ): """ Update the metadata for the image. + + Parameters + ---------- + metadata : dict, optional + The metadata to update. + signal_dimensions : int, optional + The number of signal dimensions. + navigation_dimensions : int, optional + The number of navigation dimensions. """ if metadata is None: metadata = {} + if navigation_dimensions is None and signal_dimensions is None: + signal_dimensions = self.ndim + navigation_dimensions = 0 formatted_metadata = {} formatted_metadata["Acquisition"] = {} @@ -569,26 +593,35 @@ def update_metadata( if navigation_dimensions > 0: formatted_metadata["Meta Data"]["IsSequence"] = "true" dict2group(formatted_metadata, self.image_tags) + + # Update Microscope Info + if ( + "Acquisition_instrument" in metadata + and "TEM" in metadata["Acquisition_instrument"] + ): + self.image_tags.create_group("Microscope Info") + microscope_info_dict = { + "Voltage": metadata["Acquisition_instrument"]["TEM"].get( + "beam_energy", 0 + ) + * 1000, + "Illumination Mode": metadata["Acquisition_instrument"]["TEM"].get( + "acquisition_mode", "Unknown" + ), + "Indicated Magnification": metadata["Acquisition_instrument"][ + "TEM" + ].get("magnification", 0), + "STEM Camera Length": metadata["Acquisition_instrument"]["TEM"].get( + "camera_length", 0 + ), + } + + dict2group(microscope_info_dict, self.image_tags["Microscope Info"]) + self.image_tags.create_group("UserTags") dict2group(metadata, self.image_tags["UserTags"]) return - def to_signal_dict(self): - """ - Convert the image to a Hyperspy signal dictionary. - """ - data = self.get_data() - metadata, original_metadata = self.get_metadata() - axes = [] - for axis in range(len(data.shape)): - axes.append(self.get_axis_dict(axis)) - return { - "data": data, - "metadata": metadata, - "original_metadata": original_metadata, - "axes": axes, - } - def dict2group(dictionary, group): for key, value in dictionary.items(): @@ -611,16 +644,8 @@ def _group2dict(group, dictionary=None): value = value.decode() except UnicodeDecodeError: value = "Decoding error" - if isinstance(value, (np.bytes_, str)): - if value == "_None_": - value = None elif isinstance(value, np.bool_): value = bool(value) - elif isinstance(value, np.ndarray) and value.dtype.char == "S": - # Convert strings to unicode - value = value.astype("U") - if value.dtype.str.endswith("U1"): - value = value.tolist() dictionary[key] = value if not isinstance(group, h5py.Dataset): diff --git a/rsciio/tests/test_dm5.py b/rsciio/tests/test_dm5.py index 7e3adc962..dea480d5a 100644 --- a/rsciio/tests/test_dm5.py +++ b/rsciio/tests/test_dm5.py @@ -24,6 +24,7 @@ from pathlib import Path +import dask.array as da import numpy as np import pytest @@ -83,3 +84,44 @@ def test_save_load_files( assert "nm" in s.axes_manager[i].units assert s.axes_manager[i].scale == 0.1 assert s.axes_manager[i].size == int(original[i][-2:]) + + def test_save_load_undefined_axes(self, tmp_path): + fname = tmp_path / "test_save_undefined.dm5" + + data_shape = [10, 11, 12, 13] + data = np.ones(data_shape, dtype=np.float32) + signal = hs.signals.Signal2D(data) + signal.save(fname, overwrite=True) + s = hs.load(fname) + for i in range(4): + assert s.axes_manager[i].name == "" + assert s.axes_manager[i].units == "" + + def test_save_load_metadata(self, tmp_path): + fname = tmp_path / "test_save_undefined.dm5" + + data_shape = [10, 11, 12, 13] + data = np.ones(data_shape, dtype=np.float32) + signal = hs.signals.Signal2D(data) + signal.metadata.General.title = "test" + signal.metadata.add_node("Acquisition_instrument.TEM") + signal.metadata.General["test"] = "test".encode() + signal.metadata.Acquisition_instrument.TEM.beam_energy = 200 + signal.metadata.Acquisition_instrument.TEM.magnification = 100 + signal.metadata.Acquisition_instrument.TEM.camera_length = 10 + + signal.save(fname, overwrite=True) + s = hs.load(fname) + assert s.metadata.Acquisition_instrument.TEM.beam_energy == 200 + assert s.metadata.Acquisition_instrument.TEM.camera_length == 10 + assert s.metadata.Acquisition_instrument.TEM.magnification == 100 + + def test_save_load_lazy(self, tmp_path): + fname = tmp_path / "test_save_lazy.dm5" + + data_shape = [10, 11, 12, 13] + data = np.ones(data_shape, dtype=np.float32) + signal = hs.signals.Signal2D(data) + signal.save(fname, overwrite=True) + s = hs.load(fname, lazy=True) + assert isinstance(s.data, da.Array)