diff --git a/tools/fp8_cast_bf16.py b/tools/fp8_cast_bf16.py
index 31d6d52a3a..554c2de6ad 100644
--- a/tools/fp8_cast_bf16.py
+++ b/tools/fp8_cast_bf16.py
@@ -92,7 +92,7 @@ def get_tensor(tensor_name):
 
             if weight_name.endswith("_scale_inv"):
                 continue
-            elif weight.element_size() == 1:  # FP8 weight
+            elif weight.element_size() == 1 and weight.is_floating_point():  # FP8 weight
                 scale_inv_name = f"{weight_name}_scale_inv"
                 try:
                     # Get scale_inv from the correct file
@@ -100,8 +100,8 @@ def get_tensor(tensor_name):
                     fp8_weight_names.append(weight_name)
                     new_state_dict[weight_name] = weight_dequant(weight, scale_inv)
                 except KeyError:
-                    print(f"Warning: Missing scale_inv tensor for {weight_name}, skipping conversion")
-                    new_state_dict[weight_name] = weight
+                    print(f"Warning: Missing scale_inv tensor for {weight_name}, upcasting to {torch.get_default_dtype()}")
+                    new_state_dict[weight_name] = weight.to(torch.get_default_dtype())
             else:
                 new_state_dict[weight_name] = weight