Fix split.dictionary defaults for invalid inputs and null values

tests/recipes/wrangles/test_split.py

@@ -905,6 +905,164 @@ def test_split_dictionary_default(self):
             df['Col4'][0] == 'D'
         )
 
+    @pytest.mark.parametrize("output_format", ["columns", "to_lists"])
+    @pytest.mark.parametrize("invalid", [
+        "Timed Out", "", None, float("nan"), 123, False, [],
+        "{invalid JSON}", '["not a dictionary"]', "null",
+    ])
+    def test_split_dictionary_invalid_input_uses_default(self, invalid, output_format):
+        df = wrangles.recipe.run(
+            {"wrangles": [{"split.dictionary": {
+                "input": "atts",
+                "default": {"Voltage": {}},
+                "output_format": output_format,
+            }}]},
+            dataframe=pd.DataFrame({"atts": [
+                {"Voltage": {"value": 120, "uom": "VAC"}},
+                invalid,
+                '{"Voltage": {"value": 12, "uom": "VDC"}}',
+            ]}),
+        )
+        expected = [{"value": 120, "uom": "VAC"}, {}, {"value": 12, "uom": "VDC"}]
+        if output_format == "columns":
+            assert df["Voltage"].tolist() == expected
+        else:
+            assert df["Keys"].tolist() == [["Voltage"]] * 3
+            assert df["Values"].tolist() == [[value] for value in expected]
+
+    @pytest.mark.parametrize("output_format", ["columns", "to_lists"])
+    @pytest.mark.parametrize("invalid", ["Timed Out", None, "{invalid JSON}", []])
+    def test_split_dictionary_invalid_input_without_default_raises(self, invalid, output_format):
+        with pytest.raises(ValueError, match="is not a valid Dictionary"):
+            wrangles.recipe.run(
+                {"wrangles": [{"split.dictionary": {
+                    "input": "atts", "output_format": output_format,
+                }}]},
+                dataframe=pd.DataFrame({"atts": [{"Voltage": 120}, invalid]}),
+            )
+
+    @pytest.mark.parametrize("output_format", ["columns", "to_lists"])
+    def test_split_dictionary_explicit_empty_default(self, output_format):
+        df = wrangles.recipe.run(
+            {"wrangles": [{"split.dictionary": {
+                "input": "atts", "default": {}, "output_format": output_format,
+            }}]},
+            dataframe=pd.DataFrame({"atts": ["Timed Out", None]}),
+        )
+        if output_format == "columns":
+            assert df.columns.tolist() == ["atts"]
+        else:
+            assert df["Keys"].tolist() == [[], []]
+            assert df["Values"].tolist() == [[], []]
+
+    @pytest.mark.parametrize("invalid_default", ["Timed Out", 123, []])
+    def test_split_dictionary_invalid_default_raises(self, invalid_default):
+        with pytest.raises(ValueError, match="is not a valid Dictionary"):
+            wrangles.recipe.run(
+                {"wrangles": [{"split.dictionary": {
+                    "input": "atts", "default": invalid_default,
+                }}]},
+                dataframe=pd.DataFrame({"atts": [{"Voltage": 120}]}),
+            )
+
+    @pytest.mark.parametrize("output_format", ["columns", "to_lists"])
+    @pytest.mark.parametrize("default", [{}, "", [], "fallback", 0, False])
+    def test_split_dictionary_json_null_uses_default(self, default, output_format):
+        df = wrangles.recipe.run(
+            {"wrangles": [
+                {"convert.from_json": {"input": "Source", "output": "atts"}},
+                {"split.dictionary": {
+                    "input": "atts", "default": {"Voltage": default},
+                    "output_format": output_format,
+                }},
+            ]},
+            dataframe=pd.DataFrame({"Source": [
+                '{"Voltage": {"value": 120, "uom": "VAC"}}',
+                '{"Voltage": {"value": 120, "uom": "VAC"}}',
+                '{"Voltage": null}',
+                '{}',
+            ]}),
+        )
+        expected = [{"value": 120, "uom": "VAC"}] * 2 + [default, default]
+        if output_format == "columns":
+            assert df["Voltage"].tolist() == expected
+        else:
+            assert df["Keys"].tolist() == [["Voltage"]] * 4
+            assert df["Values"].tolist() == [[value] for value in expected]
+        assert df.at[2, "atts"] == {"Voltage": None}
+
+    @pytest.mark.parametrize("output_format", ["columns", "to_lists"])
+    def test_split_dictionary_defaults_preserve_existing_values(self, output_format):
+        values = {
+            "Zero": 0, "False": False, "Blank": "", "List": [], "Dictionary": {},
+            "Nested": {"child": None}, "Unconfigured": None, "Voltage": 120,
+        }
+        defaults = {key: "fallback" for key in values if key not in ("Unconfigured", "Voltage")}
+        defaults["voltage"] = {}
+        df = wrangles.recipe.run(
+            {"wrangles": [{"split.dictionary": {
+                "input": "atts", "default": defaults, "output_format": output_format,
+            }}]},
+            dataframe=pd.DataFrame({"atts": [values]}),
+        )
+        expected = {**values, "voltage": {}}
+        if output_format == "columns":
+            expected["Unconfigured"] = ""
+            assert df[list(expected)].to_dict("records") == [expected]
+        else:
+            assert dict(zip(df.at[0, "Keys"], df.at[0, "Values"])) == expected
+
+    @pytest.mark.parametrize("output_format", ["columns", "to_lists"])
+    def test_split_dictionary_defaults_after_multiple_inputs(self, output_format):
+        df = wrangles.recipe.run(
+            {"wrangles": [{"split.dictionary": {
+                "input": ["First", "Second"],
+                "default": {"Voltage": {}, "Missing": "fallback"},
+                "output_format": output_format,
+            }}]},
+            dataframe=pd.DataFrame({
+                "First": [{"Voltage": 1}, "Timed Out", {"Voltage": 1}, {"Voltage": None}],
+                "Second": ["Timed Out", {"Voltage": 2}, {"Voltage": None}, {"Voltage": 3}],
+            }),
+        )
+        expected = [1, 2, {}, 3]
+        if output_format == "columns":
+            assert df["Voltage"].tolist() == expected
+            assert df["Missing"].tolist() == ["fallback"] * 4
+        else:
+            assert df["Keys"].tolist() == [["Voltage", "Missing"]] * 4
+            assert df["Values"].tolist() == [[value, "fallback"] for value in expected]
+
+    @pytest.mark.parametrize("output_format", ["columns", "to_lists"])
+    def test_split_dictionary_mutable_defaults_are_independent(self, output_format):
+        default = {"Voltage": {"values": []}}
+        df = wrangles.recipe.run(
+            {"wrangles": [{"split.dictionary": {
+                "input": "atts", "default": default, "output_format": output_format,
+            }}]},
+            dataframe=pd.DataFrame({"atts": [{"Voltage": None}, {}, "Timed Out"]}),
+        )
+        cells = df["Voltage"].tolist() if output_format == "columns" else [
+            row[0] for row in df["Values"]
+        ]
+        cells[0]["values"].append(120)
+        assert cells[1:] == [{"values": []}, {"values": []}]
+        assert default == {"Voltage": {"values": []}}
+
+    def test_split_dictionary_defaults_with_where_and_renaming(self):
+        df = wrangles.recipe.run(
+            {"wrangles": [{"split.dictionary": {
+                "input": "atts", "output": [{"Voltage": "Supply Voltage"}],
+                "default": {"Voltage": {}}, "where": "Process > 0",
+            }}]},
+            dataframe=pd.DataFrame({
+                "atts": [{"Voltage": None}, "Timed Out", {"Voltage": None}],
+                "Supply Voltage": ["old null", "old error", "keep"],
+                "Process": [1, 1, 0],
+            }),
+        )
+        assert df["Supply Voltage"].tolist() == [{}, {}, "keep"]
+
     def test_split_dictionary_multiple(self):
         """
         Test splitting a list of dictionaries

wrangles/recipe_wrangles/split.py

@@ -3,6 +3,7 @@
 """
 # Rename List to _list to be able to use function name list without clashing
 from typing import Union as _Union, List as _list
+from copy import deepcopy as _deepcopy
 import logging as _logging
 import pandas as _pd
 from .. import format as _format
@@ -25,7 +26,8 @@ def dictionary(
     description: |-
       Split one or more dictionaries into columns.
       The dictionary keys will be returned as the new column headers.
-      If the dictionaries contain overlapping values, the last value will be returned.
+      Dictionary keys are case-sensitive. With multiple input columns,
+      later dictionaries overwrite earlier keys before defaults are applied.
     additionalProperties: false
     required:
       - input
@@ -36,9 +38,9 @@ def dictionary(
           - integer
           - array
         description: |-
-          Name or lists of the column(s) containing dictionaries to be split.
-          If providing multiple dictionaries and the dictionaries
-          contain overlapping values, the last value will be returned.
+          Name or list of columns containing dictionaries or JSON object strings to split.
+          For overlapping keys, later input columns take precedence
+          before defaults are applied.
       output:
         type:
           - string
@@ -55,8 +57,15 @@ def dictionary(
       default:
         type: object
         description: >-
-          Provide a set of default headings and values
-          if they are not found within the input
+          Provide defaults for missing top-level keys or keys with null values
+          after merging the input dictionaries. Keys use their original names
+          before output renaming. Non-null values, including 0, false, empty
+          strings, lists, and dictionaries, are preserved. Nested values are
+          not filled recursively.
+          When default is supplied (including {}), invalid or non-dictionary
+          input cells are treated as empty dictionaries, so valid values from
+          other input columns are retained. Without default, invalid input
+          raises an error.
       output_format:
         type: string
         enum:
@@ -65,11 +74,14 @@ def dictionary(
         description: |-
           How to split the dictionary.
           columns creates one output column for each dictionary key.
+          In recipes, remaining null cells are normalized to empty strings.
           to_lists creates two output columns containing lists of keys and values.
+          Remaining null values inside these lists are preserved.
     """
     if output_format not in ["columns", "to_lists"]:
         raise ValueError("output_format must be one of: columns, to_lists")
 
+    has_default = default is not None
     if default is None:
         default = {}
     _logging.debug(f": Splitting dictionaries :: input :: {input}")
@@ -88,12 +100,24 @@ def _parse_dict_or_json(val):
         raise ValueError(f'{val} is not a valid Dictionary') from None
 
 
-    # Merge each row's dictionaries so duplicate keys follow existing behavior:
-    # later input columns overwrite earlier input columns.
-    dicts = [
-        dict(_itertools.chain.from_iterable(_parse_dict_or_json(d) for d in ([default] + row.tolist())))
-        for row in df[input].values
-    ]
+    # Validate defaults separately so invalid configuration cannot use fallback.
+    default = dict(_parse_dict_or_json(default))
+    dicts = []
+    for row in df[input].values:
+        # Keep default keys first, preserving the existing output column order.
+        merged = dict.fromkeys(default)
+        for value in row:
+            try:
+                merged.update(_parse_dict_or_json(value))
+            except ValueError:
+                if not has_default:
+                    raise
+
+        # Apply defaults after merging so later input columns still take precedence.
+        for key, value in default.items():
+            if merged[key] is None:
+                merged[key] = _deepcopy(value)
+        dicts.append(merged)
 
     if output_format == "to_lists":
         if output is None: