Python: write and read chat history files as UTF-8

python/semantic_kernel/connectors/ai/onnx/services/onnx_gen_ai_completion_base.py

@@ -40,7 +40,7 @@ def __init__(self, ai_model_path: str, **kwargs) -> None:
             raise ImportError("onnxruntime-genai is not installed.")
         try:
             json_gen_ai_config = os.path.join(ai_model_path + "/genai_config.json")
-            with open(json_gen_ai_config) as file:
+            with open(json_gen_ai_config, encoding="utf-8") as file:
                 config: dict = json.load(file)
                 enable_multi_modality = "vision" in config.get("model", {})
                 model = OnnxRuntimeGenAi.Model(ai_model_path)

python/semantic_kernel/contents/chat_history.py

@@ -427,7 +427,7 @@ def store_chat_history_to_file(self, file_path: str) -> None:
             file_path: The path to the file where the serialized data will be stored.
         """
         json_str = self.serialize()
-        with open(file_path, "w") as local_file:
+        with open(file_path, "w", encoding="utf-8") as local_file:
             local_file.write(json_str)
 
     @classmethod
@@ -442,6 +442,6 @@ def load_chat_history_from_file(cls: type[_T], file_path: str) -> _T:
         Returns:
             ChatHistory: The deserialized ChatHistory instance.
         """
-        with open(file_path) as file:
+        with open(file_path, encoding="utf-8") as file:
             json_str = file.read()
         return cls.restore_chat_history(json_str)

python/tests/unit/connectors/ai/onnx/services/test_onnx_chat_completion.py

@@ -20,6 +20,20 @@ def test_onnx_chat_completion_with_valid_env_variable(gen_ai_config, model, toke
     assert not service.enable_multi_modality
 
 
+@patch("builtins.open", new_callable=mock_open, read_data=json.dumps(gen_ai_config))
+@patch("onnxruntime_genai.Model")
+@patch("onnxruntime_genai.Tokenizer")
+def test_onnx_config_is_read_as_utf8(gen_ai_config, model, tokenizer, onnx_unit_test_env):
+    """The genai_config read must name UTF-8 rather than inherit the host default."""
+    OnnxGenAIChatCompletion(template=ONNXTemplate.PHI3, env_file_path="test.env")
+
+    config_calls = [
+        call for call in open.call_args_list if call.args and str(call.args[0]).endswith("genai_config.json")
+    ]
+    assert config_calls, [call.args for call in open.call_args_list]
+    assert all(call.kwargs.get("encoding") == "utf-8" for call in config_calls), config_calls
+
+
 @patch("builtins.open", new_callable=mock_open, read_data=json.dumps(gen_ai_config_vision))
 @patch("onnxruntime_genai.Model")
 @patch("onnxruntime_genai.Tokenizer")

python/tests/unit/contents/test_chat_history.py

@@ -747,3 +747,39 @@ def test_chat_history_prompt_round_trip_keeps_entities(text: str):
 
     restored = ChatHistory.from_rendered_prompt(chat_history.to_prompt())
     assert [message.content for message in restored.messages] == [text, text]
+
+
+def test_chat_history_file_round_trip_keeps_non_ascii(tmp_path):
+    """A history with accented and CJK text has to survive the file round trip."""
+    history = ChatHistory()
+    history.add_user_message("héllo — 世界")
+
+    file_path = tmp_path / "chat_history_utf8.json"
+    history.store_chat_history_to_file(file_path)
+
+    # The stored bytes are UTF-8 no matter what the host's default encoding is.
+    assert "héllo — 世界".encode() in file_path.read_bytes()
+
+    restored = ChatHistory.load_chat_history_from_file(file_path)
+    assert restored.messages[0].content == "héllo — 世界"
+
+
+def test_chat_history_file_io_requests_utf8_explicitly(tmp_path, monkeypatch):
+    """Both file opens have to name the encoding instead of relying on the default."""
+    recorded = []
+    real_open = open
+
+    def spy_open(*args, **kwargs):
+        recorded.append(kwargs.get("encoding"))
+        return real_open(*args, **kwargs)
+
+    monkeypatch.setattr("builtins.open", spy_open)
+
+    history = ChatHistory()
+    history.add_user_message("héllo — 世界")
+    file_path = tmp_path / "chat_history_utf8.json"
+    history.store_chat_history_to_file(file_path)
+    ChatHistory.load_chat_history_from_file(file_path)
+
+    assert recorded, "the history never touched the filesystem"
+    assert all(encoding == "utf-8" for encoding in recorded), recorded