add support for Velox EMD files containing EELS spectrum images

rsciio/emd/_emd_velox.py

@@ -23,11 +23,11 @@
 # and use either the EMD class or the FEIEMDReader class to read the file.
 # Writing file is only supported for EMD Berkeley file.
 
-
 import importlib
 import json
 import logging
 import os
+import re
 import time
 from datetime import datetime
 
@@ -550,6 +550,10 @@ def _parse_metadata_group(self, group, group_name):
     def _read_spectrum_stream(self):
         if not self.load_SI:
             return
+
+        if "EelsSpectrumImage" in self.d_grp.keys():
+            self._read_eels_spectrum_images()
+
         self.detector_name = "EDS"
         # Try to read the number of frames from Data/SpectrumImage
         try:
@@ -707,6 +711,150 @@ def _read_stream(key):
                 }
             )
 
+    def _read_eels_spectrum_images(self):
+
+        # get binary result to read pixel size
+        k = list(self.d_grp["SpectrumImage"].keys())[0]
+        meta = self.d_grp["SpectrumImage"][k]["Metadata"]
+        meta_dict = eval(
+            bytes(meta[:].flatten())
+            .decode("utf-8", errors="ignore")
+            .strip()
+            .rstrip("\x00")
+        )
+        binary_result = meta_dict["BinaryResult"]
+        spatial_dim = len(binary_result["Offset"])
+
+        # read pixel size
+        scale_x, x_unit = self._convert_scale_units(
+            binary_result["PixelSize"]["width"], binary_result["PixelUnitX"]
+        )
+        offset_x, _ = self._convert_scale_units(
+            binary_result["Offset"]["x"], binary_result["PixelUnitX"]
+        )
+        if spatial_dim == 2:
+            scale_y, y_unit = self._convert_scale_units(
+                binary_result["PixelSize"]["height"], binary_result["PixelUnitY"]
+            )
+            offset_y, _ = self._convert_scale_units(
+                binary_result["Offset"]["y"], binary_result["PixelUnitY"]
+            )
+
+        # Zebra detector stores the different energy ranges as separate datasets. Iterate over each of their ids:
+        for bin_id in self.d_grp["EelsSpectrumImage"].keys():
+            axes = []
+            eels_dict = {}
+            data_node_path = f"EelsSpectrumImage/{bin_id}/Data"
+            data_cube = self.d_grp[data_node_path][:]
+
+            if data_cube.ndim == 3:
+                data_cube = data_cube.transpose(2, 0, 1)
+                axes.extend(
+                    [
+                        {
+                            "name": "y",
+                            "offset": offset_y,
+                            "scale": scale_y,
+                            "size": data_cube.shape[0],
+                            "units": y_unit,
+                            "index_in_array": 1,
+                            "navigate": True,
+                        },
+                        {
+                            "name": "x",
+                            "offset": offset_x,
+                            "scale": scale_x,
+                            "size": data_cube.shape[1],
+                            "units": x_unit,
+                            "index_in_array": 0,
+                            "navigate": True,
+                        },
+                    ]
+                )
+            elif data_cube.ndim == 2:
+                axes.extend(
+                    [
+                        {
+                            "name": "x",
+                            "offset": offset_x,
+                            "scale": scale_x,
+                            "size": data_cube.shape[0],
+                            "units": x_unit,
+                            "index_in_array": 0,
+                            "navigate": True,
+                        }
+                    ]
+                )
+            else:
+                raise Exception("Unexpected dataset dimensionality")
+
+            # deal with metadata
+            meta_eels = self.d_grp[f"EelsSpectrumImage/{bin_id}/Metadata"]
+            meta_eels_dict = eval(
+                bytes(meta_eels[:].flatten())
+                .decode("utf-8", errors="ignore")
+                .strip()
+                .rstrip("\x00")
+            )
+            meta_eels_dict = self._fix_eels_metadata_dict(
+                meta_eels_dict["CustomProperties"]
+            )
+
+            acquisition_md_raw = self.d_grp["EelsSpectrumImage"][f"{bin_id}"][
+                "AcquisitionMetadata"
+            ]
+            acquisition_md = eval(
+                bytes(acquisition_md_raw[:][:, 0].flatten())
+                .decode("utf-8", errors="ignore")
+                .strip()
+                .rstrip("\x00")
+            )
+            energy_offset = acquisition_md["Data"]["offset"]
+            energy_scale = acquisition_md["Data"]["dispersion"]
+            axes.append(
+                {
+                    "name": "Energy-loss",
+                    "offset": energy_offset,
+                    "scale": energy_scale,
+                    "size": data_cube.shape[-1],
+                    "units": "eV",  # assumed
+                    "navigate": False,
+                }
+            )
+
+            # tidy up dictionary
+            md = {
+                "General": {"original_filename": self.filename},
+                "title": "EELS",
+                "Signal": {"signal_type": "EELS"},
+            }
+            md["Signal"]["signal_type"] = "EELS"
+            eels_dict["data"] = data_cube
+            eels_dict["axes"] = axes
+            md["original_metadata"] = meta_eels_dict
+            eels_dict["metadata"] = md
+            self.dictionaries.append(eels_dict)
+
+    def _fix_eels_metadata_dict(self, dict):
+        new_dict = {}
+        for k, v in dict.items():
+            new_keys = re.split(r"[.\[\]]+", k)
+            if "" in new_keys:
+                new_keys.remove("")
+            new_keys = [i.replace('"', "") for i in new_keys]
+
+            if isinstance(new_keys, list):
+                if new_keys[0] not in new_dict.keys():
+                    new_dict[new_keys[0]] = {}
+
+                d = new_dict[new_keys[0]]
+                for i in new_keys[1:-1]:
+                    if i not in d.keys():
+                        d[i] = {}
+                    d = d[i]
+                d[new_keys[-1]] = v
+        return new_dict
+
     def _get_dispersion_offset(self, original_metadata):
         try:
             for detectorname, detector in original_metadata["Detectors"].items():

rsciio/tests/registry.txt

@@ -191,6 +191,7 @@
 'emd/example_object_dtype_data.emd' 31d9aa4ccd9991d5e58eb5e86c137c24669fb6f1c469e5ac9717f49fe2c9d25f
 'emd/example_signal.emd' 4a020e8f7588ca0dd97e8e8bad549c33d48bbfb4e5cb8a4a00f58b70a3a04804
 'emd/example_spectrum.emd' 14d30ab9f2b124c1208b36c99729a7c6f73e00d6babd84da1b1a4e14273c32f6
+'emd/example_velox_EELS_EDS.emd' 11e5c0f8e69582c442b3db54466cf12c3ef107ca68ec84d9daaa53894d7a5169
 'emd/fei_emd_files.zip' 101512cb1e0d66b63ac62ef4390a8bd654714a41e416c94230229e14e0d9ea62
 'emd/fei_example_complex_fft.emd' eec20bd422428dc498334143e2a721aa793e79f399bfe16c21cb8b0313ff0c07
 'emd/fei_example_dpc_titles.emd' c06422c623e7a7b18ed8864c99d34590488b98fef85423ac33e1ea10bef66b2f