Skip to content

[16.0][FIX] account_statement_import_ofx: handle mislabeled OFX charsets #968

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
renatonlima wants to merge 3 commits into
base: 16.0
Choose a base branch
from
Open

[16.0][FIX] account_statement_import_ofx: handle mislabeled OFX charsets #968

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
e5aaf8d
[FIX] account_statement_import_ofx: handle mislabeled OFX charsets
renatonlima Jun 19, 2026
3d465b1
[FIX] account_statement_import_online_ofx: Fixed the latest compatibl…
renatonlima Jun 19, 2026
6a90f7b
[FIX] account_statement_import_camt: account_statement_import_camt/te…
renatonlima Jun 19, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion account_statement_import_camt/tests/test_import_bank_statement.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ def _do_parse_test(self, inputfile, goldenfile, max_diff_count=None):
self.assertLessEqual(
len(diff),
max_diff_count,
f"Actual output doesn't match expected output:\n{''.join(diff)}",
f"Actual output doesn't match expected output: \n{''.join(diff)}",
)

def _get_files_diffs(self, inputfile_path, goldenfile_path) -> list:
Expand Down
47 changes: 47 additions & 0 deletions account_statement_import_ofx/tests/test_import_bank_statement.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -108,3 +108,50 @@ def test_check_journal_bank_account(self):
}
)
bank_st.import_file_button()

def _parse_ofx_memos(self, filename):
"""Return the list of transaction memos parsed from an OFX fixture.

``_check_ofx`` is fed the raw (decoded) file bytes, exactly as the
base ``_parse_file`` does in production.
"""
ofx_file_path = get_module_resource(
"account_statement_import_ofx", "tests/test_ofx_file/", filename
)
with open(ofx_file_path, "rb") as fh:
data_file = fh.read()
ofx = self.asi_model._check_ofx(data_file=data_file)
self.assertTrue(ofx, "%s should be recognized as a valid OFX file" % filename)
return [
transaction.memo
for account in ofx.accounts
for transaction in account.statement.transactions
]

def test_ofx_encoding_latin(self):
"""UTF-8 body mislabeled as CHARSET:1252 (the reported bug).

The accented "Á" produces byte 0x81, which is undefined in cp1252 and
used to crash ofxparse with a UnicodeDecodeError.
"""
memos = self._parse_ofx_memos("test_ofx_utf8_latin.ofx")
self.assertIn("TRANSFERENCIA ROMÁRIO ção JOSÉ", memos)

def test_ofx_encoding_cp1252(self):
"""Genuine cp1252 body with the Euro sign (0x80) and Western accents."""
memos = self._parse_ofx_memos("test_ofx_cp1252_euro.ofx")
self.assertIn("Loyer 850€ café Müller", memos)

def test_ofx_encoding_utf8_euro(self):
"""UTF-8 body with characters outside latin-1 (€, em dash).

These used to be silently corrupted into mojibake; they must be
preserved verbatim.
"""
memos = self._parse_ofx_memos("test_ofx_utf8_euro.ofx")
self.assertIn("Paiement 850€ — café", memos)

def test_ofx_encoding_asian(self):
"""UTF-8 body with CJK characters, far outside any 8-bit codepage."""
memos = self._parse_ofx_memos("test_ofx_utf8_asian.ofx")
self.assertIn("送金 日本 中文 결제", memos)
54 changes: 54 additions & 0 deletions account_statement_import_ofx/tests/test_ofx_file/test_ofx_cp1252_euro.ofx
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
OFXHEADER:100
DATA:OFXSGML
VERSION:102
SECURITY:NONE
ENCODING:USASCII
CHARSET:1252
COMPRESSION:NONE
OLDFILEUID:NONE
NEWFILEUID:NONE

<OFX>
<SIGNONMSGSRSV1>
<SONRS>
<STATUS>
<CODE>0
<SEVERITY>INFO
</STATUS>
<DTSERVER>20260618100000[-03:EST]
<LANGUAGE>POR
</SONRS>
</SIGNONMSGSRSV1>
<BANKMSGSRSV1>
<STMTTRNRS>
<TRNUID>1001
<STATUS>
<CODE>0
<SEVERITY>INFO
</STATUS>
<STMTRS>
<CURDEF>EUR
<BANKACCTFROM>
<BANKID>0341
<ACCTID>123456
<ACCTTYPE>CHECKING
</BANKACCTFROM>
<BANKTRANLIST>
<DTSTART>20260617100000[-03:EST]
<DTEND>20260617100000[-03:EST]
<STMTTRN>
<TRNTYPE>CREDIT
<DTPOSTED>20260617100000[-03:EST]
<TRNAMT>850.0
<FITID>20260617001
<MEMO>Loyer 850€ café Müller
</STMTTRN>
</BANKTRANLIST>
<LEDGERBAL>
<BALAMT>1000.00
<DTASOF>20260618100000[-03:EST]
</LEDGERBAL>
</STMTRS>
</STMTTRNRS>
</BANKMSGSRSV1>
</OFX>
54 changes: 54 additions & 0 deletions account_statement_import_ofx/tests/test_ofx_file/test_ofx_utf8_asian.ofx
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
OFXHEADER:100
DATA:OFXSGML
VERSION:102
SECURITY:NONE
ENCODING:USASCII
CHARSET:1252
COMPRESSION:NONE
OLDFILEUID:NONE
NEWFILEUID:NONE

<OFX>
<SIGNONMSGSRSV1>
<SONRS>
<STATUS>
<CODE>0
<SEVERITY>INFO
</STATUS>
<DTSERVER>20260618100000[-03:EST]
<LANGUAGE>POR
</SONRS>
</SIGNONMSGSRSV1>
<BANKMSGSRSV1>
<STMTTRNRS>
<TRNUID>1001
<STATUS>
<CODE>0
<SEVERITY>INFO
</STATUS>
<STMTRS>
<CURDEF>JPY
<BANKACCTFROM>
<BANKID>0341
<ACCTID>123456
<ACCTTYPE>CHECKING
</BANKACCTFROM>
<BANKTRANLIST>
<DTSTART>20260617100000[-03:EST]
<DTEND>20260617100000[-03:EST]
<STMTTRN>
<TRNTYPE>CREDIT
<DTPOSTED>20260617100000[-03:EST]
<TRNAMT>1200.0
<FITID>20260617001
<MEMO>送金 日本 中文 결제
</STMTTRN>
</BANKTRANLIST>
<LEDGERBAL>
<BALAMT>1000.00
<DTASOF>20260618100000[-03:EST]
</LEDGERBAL>
</STMTRS>
</STMTTRNRS>
</BANKMSGSRSV1>
</OFX>
54 changes: 54 additions & 0 deletions account_statement_import_ofx/tests/test_ofx_file/test_ofx_utf8_euro.ofx
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
OFXHEADER:100
DATA:OFXSGML
VERSION:102
SECURITY:NONE
ENCODING:USASCII
CHARSET:1252
COMPRESSION:NONE
OLDFILEUID:NONE
NEWFILEUID:NONE

<OFX>
<SIGNONMSGSRSV1>
<SONRS>
<STATUS>
<CODE>0
<SEVERITY>INFO
</STATUS>
<DTSERVER>20260618100000[-03:EST]
<LANGUAGE>POR
</SONRS>
</SIGNONMSGSRSV1>
<BANKMSGSRSV1>
<STMTTRNRS>
<TRNUID>1001
<STATUS>
<CODE>0
<SEVERITY>INFO
</STATUS>
<STMTRS>
<CURDEF>EUR
<BANKACCTFROM>
<BANKID>0341
<ACCTID>123456
<ACCTTYPE>CHECKING
</BANKACCTFROM>
<BANKTRANLIST>
<DTSTART>20260617100000[-03:EST]
<DTEND>20260617100000[-03:EST]
<STMTTRN>
<TRNTYPE>CREDIT
<DTPOSTED>20260617100000[-03:EST]
<TRNAMT>850.0
<FITID>20260617001
<MEMO>Paiement 850€ — café
</STMTTRN>
</BANKTRANLIST>
<LEDGERBAL>
<BALAMT>1000.00
<DTASOF>20260618100000[-03:EST]
</LEDGERBAL>
</STMTRS>
</STMTTRNRS>
</BANKMSGSRSV1>
</OFX>
61 changes: 61 additions & 0 deletions account_statement_import_ofx/tests/test_ofx_file/test_ofx_utf8_latin.ofx
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
OFXHEADER:100
DATA:OFXSGML
VERSION:102
SECURITY:NONE
ENCODING:USASCII
CHARSET:1252
COMPRESSION:NONE
OLDFILEUID:NONE
NEWFILEUID:NONE

<OFX>
<SIGNONMSGSRSV1>
<SONRS>
<STATUS>
<CODE>0
<SEVERITY>INFO
</STATUS>
<DTSERVER>20260618100000[-03:EST]
<LANGUAGE>POR
</SONRS>
</SIGNONMSGSRSV1>
<BANKMSGSRSV1>
<STMTTRNRS>
<TRNUID>1001
<STATUS>
<CODE>0
<SEVERITY>INFO
</STATUS>
<STMTRS>
<CURDEF>BRL
<BANKACCTFROM>
<BANKID>0341
<ACCTID>123456
<ACCTTYPE>CHECKING
</BANKACCTFROM>
<BANKTRANLIST>
<DTSTART>20260617100000[-03:EST]
<DTEND>20260617100000[-03:EST]
<STMTTRN>
<TRNTYPE>CREDIT
<DTPOSTED>20260617100000[-03:EST]
<TRNAMT>5610.0
<FITID>20260617001
<MEMO>PIX RECEBIDO ROMARIO
</STMTTRN>
<STMTTRN>
<TRNTYPE>CREDIT
<DTPOSTED>20260617100000[-03:EST]
<TRNAMT>4763.22
<FITID>20260617002
<MEMO>TRANSFERENCIA ROMÁRIO ção JOSÉ
</STMTTRN>
</BANKTRANLIST>
<LEDGERBAL>
<BALAMT>1000.00
<DTASOF>20260618100000[-03:EST]
</LEDGERBAL>
</STMTRS>
</STMTTRNRS>
</BANKMSGSRSV1>
</OFX>
48 changes: 46 additions & 2 deletions account_statement_import_ofx/wizard/account_statement_import.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import io
import logging
import re

from odoo import _, api, models
from odoo.exceptions import UserError
Expand All @@ -16,16 +17,59 @@
class AccountStatementImport(models.TransientModel):
_inherit = "account.statement.import"

@api.model
def _decode_ofx_file(self, data_file):
"""Decode raw OFX bytes to text, working around mislabeled charsets.

Some banks emit OFX whose SGML header declares one charset (e.g.
``CHARSET:1252``) while the body is actually encoded differently
(UTF-8 is common). ofxparse trusts that header and decodes with a
strict cp1252 codec, which raises ``UnicodeDecodeError`` on bytes
undefined in cp1252 (e.g. ``0x81``); the import then fails with a
misleading "file format not supported" message.

We decode the bytes ourselves: UTF-8 first (it is self-validating,
so it rarely matches non-UTF-8 data by accident), then the charset
declared in the header, and finally latin-1, which maps every byte
value and therefore never raises. Returns the decoded text or None.
"""
declared = None
for line in data_file[:1024].splitlines():
if line.upper().startswith(b"CHARSET:"):
value = line.split(b":", 1)[1].strip().upper()
declared = {b"1252": "cp1252", b"8859-1": "latin-1"}.get(value)
break

encodings = ["utf-8-sig"]
if declared and declared not in encodings:
encodings.append(declared)
if "latin-1" not in encodings:
encodings.append("latin-1")

for encoding in encodings:
try:
return data_file.decode(encoding)
except UnicodeDecodeError:
continue
return None

@api.model
def _check_ofx(self, data_file):
if not OfxParser:
return False
text = self._decode_ofx_file(data_file)
if text is None:
return False
# Re-encode to UTF-8 and normalize the SGML ENCODING header so that
# ofxparse decodes the body consistently, regardless of the (possibly
# wrong) charset originally declared in the file. Passing bytes (not a
# text stream) avoids ofxparse re-decoding the body via its own logic.
text = re.sub(r"(?im)^ENCODING:.*$", "ENCODING:UTF-8", text, count=1)
try:
ofx = OfxParser.parse(io.BytesIO(data_file))
return OfxParser.parse(io.BytesIO(text.encode("utf-8")))
except Exception as e:
_logger.debug(e)
return False
return ofx

@api.model
def _prepare_ofx_transaction_line(self, transaction):
Expand Down
6 changes: 5 additions & 1 deletion account_statement_import_online_ofx/README.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,7 @@
.. image:: https://odoo-community.org/readme-banner-image
:target: https://odoo-community.org/get-involved?utm_source=readme
:alt: Odoo Community Association

===========================
Online Bank Statements: OFX
===========================
Expand All @@ -13,7 +17,7 @@ Online Bank Statements: OFX
.. |badge1| image:: https://img.shields.io/badge/maturity-Beta-yellow.png
:target: https://odoo-community.org/page/development-status
:alt: Beta
.. |badge2| image:: https://img.shields.io/badge/licence-AGPL--3-blue.png
.. |badge2| image:: https://img.shields.io/badge/license-AGPL--3-blue.png
:target: http://www.gnu.org/licenses/agpl-3.0-standalone.html
:alt: License: AGPL-3
.. |badge3| image:: https://img.shields.io/badge/github-OCA%2Fbank--statement--import-lightgray.png?logo=github
Expand Down
2 changes: 1 addition & 1 deletion account_statement_import_online_ofx/__manifest__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
"depends": [
"account_statement_import_online",
],
"external_dependencies": {"python": ["ofxtools", "ofxparse"]},
"external_dependencies": {"python": ["ofxtools==0.9.5", "ofxparse"]},
"data": [
"security/ir.model.access.csv",
"views/online_bank_statement_provider.xml",
Expand Down
Loading
Loading