From 566dc9030d67da9b0d4bc88f13a8dcc3cc2d6e1f Mon Sep 17 00:00:00 2001 From: Camila Vives Date: Fri, 22 May 2026 18:48:40 +0000 Subject: [PATCH 1/2] [FIX] account_statement_import_sheet_file: Apply fixes on top of OCA 19.0 migration - Add openpyxl support for .xlsx files (OCA only kept xlrd/.xls format) - Fix integer values in column concatenation (_get_values_from_column) - Add case-insensitive column name matching in _get_column_indexes - Use str() cast in _parse_decimal to handle non-string cell values - Remove duplicate amount_column field definition (OCA bug in PR #907) - Fix typo in amount_type help text ("igned" -> "signed") - Add _clear_amount_columns onchange to clear irrelevant fields on type change - Remove data/map_data.xml from data list (keep as demo only); no impact on existing databases since the record and its XML ID remain in place - Create sample_statement_map inline in tests with .create() instead of env.ref() so tests run independently of demo data - Update tests to use BaseCommon and cover xlsx/openpyxl scenarios - Add missing Tecnativa and BCIM copyrights to parser and tests --- .../account_statement_import_sheet_mapping.py | 12 +- ...est_account_statement_import_sheet_file.py | 500 ++++++++++++++---- .../account_statement_import_sheet_parser.py | 144 +++-- 3 files changed, 501 insertions(+), 155 deletions(-) diff --git a/account_statement_import_sheet_file/models/account_statement_import_sheet_mapping.py b/account_statement_import_sheet_file/models/account_statement_import_sheet_mapping.py index 8ca187f525..ba4867bdaa 100644 --- a/account_statement_import_sheet_file/models/account_statement_import_sheet_mapping.py +++ b/account_statement_import_sheet_file/models/account_statement_import_sheet_mapping.py @@ -77,9 +77,6 @@ class AccountStatementImportSheetMapping(models.Model): "transaction from" ), ) - amount_column = fields.Char( - help="Amount of transaction in journal's currency", - ) amount_debit_column = fields.Char( string="Debit amount column", help="Debit amount of transaction in journal's currency", @@ -122,7 +119,7 @@ class AccountStatementImportSheetMapping(models.Model): required=True, default="simple_value", help=( - "Simple value: use igned amount in amount column\n" + "Simple value: use signed amount in amount column\n" "Absolute Value: use a same column for debit and credit\n" "(absolute value + indicate sign)\n" "Distinct Credit/debit Column: use a distinct column for debit and credit" @@ -238,6 +235,13 @@ def _check_columns(self): if mapping.offset_column < 0: raise ValidationError(self.env._("Offsets cannot be negative")) + @api.onchange("amount_type") + def _clear_amount_columns(self): + self.amount_column = False + self.debit_credit_column = False + self.amount_debit_column = False + self.amount_credit_column = False + def _get_float_separators(self): self.ensure_one() separators = { diff --git a/account_statement_import_sheet_file/tests/test_account_statement_import_sheet_file.py b/account_statement_import_sheet_file/tests/test_account_statement_import_sheet_file.py index 5181397133..a1935ab269 100644 --- a/account_statement_import_sheet_file/tests/test_account_statement_import_sheet_file.py +++ b/account_statement_import_sheet_file/tests/test_account_statement_import_sheet_file.py @@ -11,22 +11,32 @@ from odoo import fields from odoo.exceptions import UserError -from odoo.tests import common -from odoo.tools import float_round, mute_logger +from odoo.tools import float_round +from odoo.addons.base.tests.common import BaseCommon -class TestAccountStatementImportSheetFile(common.TransactionCase): + +class TestAccountStatementImportSheetFile(BaseCommon): @classmethod def setUpClass(cls): super().setUpClass() + cls.now = fields.Datetime.now() cls.currency_eur = cls.env.ref("base.EUR") cls.currency_usd = cls.env.ref("base.USD") cls.currency_usd.active = True + # Make sure the currency of the company is USD, as this not always happens + # To be removed in V17: https://github.com/odoo/odoo/pull/107113 + # cls.company = cls.env.company + # cls.env.cr.execute( + # "UPDATE res_company SET currency_id = %s WHERE id = %s", + # (cls.env.ref("base.USD").id, cls.company.id), + # ) # Activate EUR for unit test, by default is not active cls.currency_eur.active = True - Mapping = cls.env["account.statement.import.sheet.mapping"] - cls.sample_statement_map = Mapping.create( + cls.sample_statement_map = cls.env[ + "account.statement.import.sheet.mapping" + ].create( { "name": "Sample Statement", "footer_lines_skip_count": 0, @@ -66,29 +76,6 @@ def setUpClass(cls): cls.mock_mapping_comma_dot._get_float_separators.return_value = (",", ".") cls.mock_mapping_dot_comma = Mock() cls.mock_mapping_dot_comma._get_float_separators.return_value = (".", ",") - cls.mock_mapping_none_none = Mock() - cls.mock_mapping_none_none._get_float_separators.return_value = ("", "") - cls.journal = cls.AccountJournal.create( - { - "name": "Bank", - "type": "bank", - "code": "BANK", - "currency_id": cls.currency_usd.id, - "suspense_account_id": cls.suspense_account.id, - } - ) - cls.statement_domain = [("journal_id", "=", cls.journal.id)] - - def _get_import_wizard(self, path): - return self.AccountStatementImport.with_context( - journal_id=self.journal.id, account_statement_import_sheet_file_test=True - ).create( - { - "statement_filename": path, - "statement_file": self._data_file(path), - "sheet_mapping_id": self.sample_statement_map.id, - } - ) def _data_file(self, filename, encoding=None): mode = "rt" if encoding else "rb" @@ -99,39 +86,130 @@ def _data_file(self, filename, encoding=None): return b64encode(data) def test_import_csv_file(self): - wizard = self._get_import_wizard("fixtures/sample_statement_en.csv") - wizard.import_file_button() - statement = self.AccountBankStatement.search(self.statement_domain) + journal = self.AccountJournal.create( + { + "name": "Bank", + "type": "bank", + "code": "BANK", + "currency_id": self.currency_usd.id, + "suspense_account_id": self.suspense_account.id, + } + ) + data = self._data_file("fixtures/sample_statement_en.csv", "utf-8") + wizard = self.AccountStatementImport.with_context(journal_id=journal.id).create( + { + "statement_filename": "fixtures/sample_statement_en.csv", + "statement_file": data, + "sheet_mapping_id": self.sample_statement_map.id, + } + ) + wizard.with_context( + account_statement_import_sheet_file_test=True + ).import_file_button() + statement = self.AccountBankStatement.search([("journal_id", "=", journal.id)]) self.assertEqual(len(statement), 1) self.assertEqual(len(statement.line_ids), 2) def test_import_empty_csv_file(self): - wizard = self._get_import_wizard("fixtures/empty_statement_en.csv") + journal = self.AccountJournal.create( + { + "name": "Bank", + "type": "bank", + "code": "BANK", + "currency_id": self.currency_usd.id, + "suspense_account_id": self.suspense_account.id, + } + ) + data = self._data_file("fixtures/empty_statement_en.csv", "utf-8") + wizard = self.AccountStatementImport.with_context(journal_id=journal.id).create( + { + "statement_filename": "fixtures/empty_statement_en.csv", + "statement_file": data, + "sheet_mapping_id": self.sample_statement_map.id, + } + ) with self.assertRaises(UserError): - wizard.import_file_button() - statement = self.AccountBankStatement.search(self.statement_domain) + wizard.with_context( + account_statement_import_sheet_file_test=True + ).import_file_button() + statement = self.AccountBankStatement.search([("journal_id", "=", journal.id)]) self.assertEqual(len(statement), 0) def test_import_xlsx_file(self): - wizard = self._get_import_wizard("fixtures/sample_statement_en.xlsx") - wizard.import_file_button() - statement = self.AccountBankStatement.search(self.statement_domain) + journal = self.AccountJournal.create( + { + "name": "Bank", + "type": "bank", + "code": "BANK", + "currency_id": self.currency_usd.id, + "suspense_account_id": self.suspense_account.id, + } + ) + data = self._data_file("fixtures/sample_statement_en.xlsx") + wizard = self.AccountStatementImport.with_context(journal_id=journal.id).create( + { + "statement_filename": "fixtures/sample_statement_en.xlsx", + "statement_file": data, + "sheet_mapping_id": self.sample_statement_map.id, + } + ) + wizard.with_context( + account_statement_import_sheet_file_test=True + ).import_file_button() + statement = self.AccountBankStatement.search([("journal_id", "=", journal.id)]) self.assertEqual(len(statement), 1) self.assertEqual(len(statement.line_ids), 2) def test_import_empty_xlsx_file(self): - wizard = self._get_import_wizard("fixtures/empty_statement_en.xlsx") + journal = self.AccountJournal.create( + { + "name": "Bank", + "type": "bank", + "code": "BANK", + "currency_id": self.currency_usd.id, + "suspense_account_id": self.suspense_account.id, + } + ) + data = self._data_file("fixtures/empty_statement_en.xlsx") + wizard = self.AccountStatementImport.with_context(journal_id=journal.id).create( + { + "statement_filename": "fixtures/empty_statement_en.xlsx", + "statement_file": data, + "sheet_mapping_id": self.sample_statement_map.id, + } + ) with self.assertRaises(UserError): - wizard.import_file_button() - statement = self.AccountBankStatement.search(self.statement_domain) + wizard.with_context( + account_statement_import_sheet_file_test=True + ).import_file_button() + statement = self.AccountBankStatement.search([("journal_id", "=", journal.id)]) self.assertEqual(len(statement), 0) def test_original_currency(self): - wizard = self._get_import_wizard("fixtures/original_currency.csv") - wizard.import_file_button() - statement = self.AccountBankStatement.search(self.statement_domain) + journal = self.AccountJournal.create( + { + "name": "Bank", + "type": "bank", + "code": "BANK", + "currency_id": self.currency_usd.id, + "suspense_account_id": self.suspense_account.id, + } + ) + data = self._data_file("fixtures/original_currency.csv", "utf-8") + wizard = self.AccountStatementImport.with_context(journal_id=journal.id).create( + { + "statement_filename": "fixtures/original_currency.csv", + "statement_file": data, + "sheet_mapping_id": self.sample_statement_map.id, + } + ) + wizard.with_context( + account_statement_import_sheet_file_test=True + ).import_file_button() + statement = self.AccountBankStatement.search([("journal_id", "=", journal.id)]) self.assertEqual(len(statement), 1) self.assertEqual(len(statement.line_ids), 1) + line = statement.line_ids self.assertEqual(line.currency_id, self.currency_usd) self.assertEqual(line.amount, 1525.0) @@ -159,12 +237,30 @@ def test_original_currency_no_header(self): "bank_account_column": "6", } ) - wizard = self._get_import_wizard("fixtures/original_currency_no_header.csv") - wizard.sheet_mapping_id = no_header_statement_map.id - wizard.import_file_button() - statement = self.AccountBankStatement.search(self.statement_domain) + journal = self.AccountJournal.create( + { + "name": "Bank", + "type": "bank", + "code": "BANK", + "currency_id": self.currency_usd.id, + "suspense_account_id": self.suspense_account.id, + } + ) + data = self._data_file("fixtures/original_currency_no_header.csv", "utf-8") + wizard = self.AccountStatementImport.with_context(journal_id=journal.id).create( + { + "statement_filename": "fixtures/original_currency.csv", + "statement_file": data, + "sheet_mapping_id": no_header_statement_map.id, + } + ) + wizard.with_context( + account_statement_import_sheet_file_test=True + ).import_file_button() + statement = self.AccountBankStatement.search([("journal_id", "=", journal.id)]) self.assertEqual(len(statement), 1) self.assertEqual(len(statement.line_ids), 1) + line = statement.line_ids self.assertEqual(line.currency_id, self.currency_usd) self.assertEqual(line.foreign_currency_id, self.currency_eur) @@ -172,43 +268,99 @@ def test_original_currency_no_header(self): self.assertEqual(line.payment_ref, "Your payment INV0001") def test_original_currency_empty(self): - wizard = self._get_import_wizard("fixtures/original_currency_empty.csv") - wizard.import_file_button() - statement = self.AccountBankStatement.search(self.statement_domain) + journal = self.AccountJournal.create( + { + "name": "Bank", + "type": "bank", + "code": "BANK", + "currency_id": self.currency_usd.id, + "suspense_account_id": self.suspense_account.id, + } + ) + data = self._data_file("fixtures/original_currency_empty.csv", "utf-8") + wizard = self.AccountStatementImport.with_context(journal_id=journal.id).create( + { + "statement_filename": "fixtures/original_currency_empty.csv", + "statement_file": data, + "sheet_mapping_id": self.sample_statement_map.id, + } + ) + wizard.with_context( + account_statement_import_sheet_file_test=True + ).import_file_button() + statement = self.AccountBankStatement.search([("journal_id", "=", journal.id)]) self.assertEqual(len(statement), 1) self.assertEqual(len(statement.line_ids), 1) + line = statement.line_ids self.assertFalse(line.foreign_currency_id) self.assertEqual(line.amount_currency, 0.0) def test_multi_currency(self): - self.sample_statement_map.write( + journal = self.AccountJournal.create( + { + "name": "Bank", + "type": "bank", + "code": "BANK", + "currency_id": self.currency_usd.id, + "suspense_account_id": self.suspense_account.id, + } + ) + statement_map = self.sample_statement_map.copy( { "currency_column": "Currency", "original_currency_column": None, "original_amount_column": None, } ) - wizard = self._get_import_wizard("fixtures/multi_currency.csv") - wizard.import_file_button() - statement = self.AccountBankStatement.search(self.statement_domain) + data = self._data_file("fixtures/multi_currency.csv", "utf-8") + wizard = self.AccountStatementImport.with_context(journal_id=journal.id).create( + { + "statement_filename": "fixtures/multi_currency.csv", + "statement_file": data, + "sheet_mapping_id": statement_map.id, + } + ) + wizard.with_context( + account_statement_import_sheet_file_test=True + ).import_file_button() + statement = self.AccountBankStatement.search([("journal_id", "=", journal.id)]) self.assertEqual(len(statement), 1) self.assertEqual(len(statement.line_ids), 1) + line = statement.line_ids self.assertFalse(line.foreign_currency_id) self.assertEqual(line.amount, -33.5) def test_balance(self): - self.sample_statement_map.write( + journal = self.AccountJournal.create( + { + "name": "Bank", + "type": "bank", + "code": "BANK", + "currency_id": self.currency_usd.id, + "suspense_account_id": self.suspense_account.id, + } + ) + statement_map = self.sample_statement_map.copy( { "balance_column": "Balance", "original_currency_column": None, "original_amount_column": None, } ) - wizard = self._get_import_wizard("fixtures/balance.csv") - wizard.import_file_button() - statement = self.AccountBankStatement.search(self.statement_domain) + data = self._data_file("fixtures/balance.csv", "utf-8") + wizard = self.AccountStatementImport.with_context(journal_id=journal.id).create( + { + "statement_filename": "fixtures/balance.csv", + "statement_file": data, + "sheet_mapping_id": statement_map.id, + } + ) + wizard.with_context( + account_statement_import_sheet_file_test=True + ).import_file_button() + statement = self.AccountBankStatement.search([("journal_id", "=", journal.id)]) self.assertEqual(len(statement), 1) self.assertEqual(len(statement.line_ids), 2) self.assertEqual(statement.balance_start, 10.0) @@ -216,7 +368,16 @@ def test_balance(self): self.assertEqual(statement.balance_end, 1510.0) def test_debit_credit(self): - self.sample_statement_map.write( + journal = self.AccountJournal.create( + { + "name": "Bank", + "type": "bank", + "code": "BANK", + "currency_id": self.currency_usd.id, + "suspense_account_id": self.suspense_account.id, + } + ) + statement_map = self.sample_statement_map.copy( { "balance_column": "Balance", "original_currency_column": None, @@ -226,9 +387,18 @@ def test_debit_credit(self): "credit_value": "C", } ) - wizard = self._get_import_wizard("fixtures/debit_credit.csv") - wizard.import_file_button() - statement = self.AccountBankStatement.search(self.statement_domain) + data = self._data_file("fixtures/debit_credit.csv", "utf-8") + wizard = self.AccountStatementImport.with_context(journal_id=journal.id).create( + { + "statement_filename": "fixtures/debit_credit.csv", + "statement_file": data, + "sheet_mapping_id": statement_map.id, + } + ) + wizard.with_context( + account_statement_import_sheet_file_test=True + ).import_file_button() + statement = self.AccountBankStatement.search([("journal_id", "=", journal.id)]) self.assertEqual(len(statement), 1) self.assertEqual(len(statement.line_ids), 2) self.assertEqual(statement.balance_start, 10.0) @@ -236,7 +406,16 @@ def test_debit_credit(self): self.assertEqual(statement.balance_end, 1510.0) def test_debit_credit_amount(self): - self.sample_statement_map.write( + journal = self.AccountJournal.create( + { + "name": "Bank", + "type": "bank", + "code": "BANK", + "currency_id": self.currency_usd.id, + "suspense_account_id": self.suspense_account.id, + } + ) + statement_map = self.sample_statement_map.copy( { "amount_type": "distinct_credit_debit", "amount_debit_column": "Debit", @@ -247,9 +426,18 @@ def test_debit_credit_amount(self): "original_amount_column": None, } ) - wizard = self._get_import_wizard("fixtures/debit_credit_amount.csv") - wizard.import_file_button() - statement = self.AccountBankStatement.search(self.statement_domain) + data = self._data_file("fixtures/debit_credit_amount.csv", "utf-8") + wizard = self.AccountStatementImport.with_context(journal_id=journal.id).create( + { + "statement_filename": "fixtures/debit_credit_amount.csv", + "statement_file": data, + "sheet_mapping_id": statement_map.id, + } + ) + wizard.with_context( + account_statement_import_sheet_file_test=True + ).import_file_button() + statement = self.AccountBankStatement.search([("journal_id", "=", journal.id)]) self.assertEqual(len(statement), 1) self.assertEqual(len(statement.line_ids), 4) self.assertEqual(statement.balance_start, 10.0) @@ -257,7 +445,16 @@ def test_debit_credit_amount(self): self.assertEqual(statement.balance_end, 1510.0) def test_metadata_separated_debit_credit_csv(self): - self.sample_statement_map.write( + journal = self.AccountJournal.create( + { + "name": "Bank", + "type": "bank", + "code": "BANK", + "currency_id": self.currency_usd.id, + "suspense_account_id": self.suspense_account.id, + } + ) + statement_map = self.sample_statement_map.copy( { "footer_lines_skip_count": 1, "header_lines_skip_count": 5, @@ -274,11 +471,19 @@ def test_metadata_separated_debit_credit_csv(self): "amount_credit_column": "Credit", } ) - wizard = self._get_import_wizard( - "fixtures/meta_data_separated_credit_debit.csv" + data = self._data_file("fixtures/meta_data_separated_credit_debit.csv", "utf-8") + wizard = self.AccountStatementImport.with_context(journal_id=journal.id).create( + { + "statement_filename": "fixtures/meta_data_separated_credit_debit.csv", + "statement_file": data, + "sheet_mapping_id": statement_map.id, + } ) - wizard.import_file_button() - statement = self.AccountBankStatement.search(self.statement_domain) + wizard.with_context( + journal_id=journal.id, + account_bank_statement_import_txt_xlsx_test=True, + ).import_file_button() + statement = self.AccountBankStatement.search([("journal_id", "=", journal.id)]) self.assertEqual(len(statement), 1) self.assertEqual(len(statement.line_ids), 4) line1 = statement.line_ids.filtered(lambda x: x.payment_ref == "LABEL 1") @@ -287,7 +492,16 @@ def test_metadata_separated_debit_credit_csv(self): self.assertEqual(line4.amount, -1300) def test_metadata_separated_debit_credit_xlsx(self): - self.sample_statement_map.write( + journal = self.AccountJournal.create( + { + "name": "Bank", + "type": "bank", + "code": "BANK", + "currency_id": self.currency_usd.id, + "suspense_account_id": self.suspense_account.id, + } + ) + statement_map = self.sample_statement_map.copy( { "footer_lines_skip_count": 1, "header_lines_skip_count": 5, @@ -304,11 +518,19 @@ def test_metadata_separated_debit_credit_xlsx(self): "amount_credit_column": "Credit", } ) - wizard = self._get_import_wizard( - "fixtures/meta_data_separated_credit_debit.xlsx" + data = self._data_file("fixtures/meta_data_separated_credit_debit.xlsx") + wizard = self.AccountStatementImport.with_context(journal_id=journal.id).create( + { + "statement_filename": "fixtures/meta_data_separated_credit_debit.xlsx", + "statement_file": data, + "sheet_mapping_id": statement_map.id, + } ) - wizard.import_file_button() - statement = self.AccountBankStatement.search(self.statement_domain) + wizard.with_context( + journal_id=journal.id, + account_bank_statement_import_txt_xlsx_test=True, + ).import_file_button() + statement = self.AccountBankStatement.search([("journal_id", "=", journal.id)]) self.assertEqual(len(statement), 1) self.assertEqual(len(statement.line_ids), 4) line1 = statement.line_ids.filtered(lambda x: x.payment_ref == "LABEL 1") @@ -318,11 +540,28 @@ def test_metadata_separated_debit_credit_xlsx(self): def test_amount_inverse_sign(self): self.sample_statement_map.amount_inverse_sign = True - wizard = self._get_import_wizard( - "fixtures/sample_statement_credit_card_inverse_sign_en.csv" + journal = self.AccountJournal.create( + { + "name": "Bank", + "type": "bank", + "code": "BANK", + "currency_id": self.currency_usd.id, + "suspense_account_id": self.suspense_account.id, + } + ) + filename = "fixtures/sample_statement_credit_card_inverse_sign_en.csv" + data = self._data_file(filename, "utf-8") + wizard = self.AccountStatementImport.with_context(journal_id=journal.id).create( + { + "statement_filename": filename, + "statement_file": data, + "sheet_mapping_id": self.sample_statement_map.id, + } ) - wizard.import_file_button() - statement = self.AccountBankStatement.search(self.statement_domain) + wizard.with_context( + account_statement_import_sheet_file_test=True + ).import_file_button() + statement = self.AccountBankStatement.search([("journal_id", "=", journal.id)]) self.assertEqual(len(statement), 1) self.assertEqual(len(statement.line_ids), 2) line1 = statement.line_ids.filtered(lambda x: x.payment_ref == "LABEL 1") @@ -355,12 +594,27 @@ def test_import_xlsx_empty_values(self): } ) ) - wizard = self._get_import_wizard( - "fixtures/sample_statement_en_empty_values.xlsx" + journal = self.AccountJournal.create( + { + "name": "Bank 2", + "type": "bank", + "code": "BAN2", + "currency_id": self.currency_usd.id, + "suspense_account_id": self.suspense_account.id, + } ) - wizard.sheet_mapping_id = sample_statement_map_empty_values.id - wizard.import_file_button() - statement = self.AccountBankStatement.search(self.statement_domain) + data = self._data_file("fixtures/sample_statement_en_empty_values.xlsx") + wizard = self.AccountStatementImport.with_context(journal_id=journal.id).create( + { + "statement_filename": "fixtures/sample_statement_en_empty_values.xlsx", + "statement_file": data, + "sheet_mapping_id": sample_statement_map_empty_values.id, + } + ) + wizard.with_context( + account_statement_import_sheet_file_test=True + ).import_file_button() + statement = self.AccountBankStatement.search([("journal_id", "=", journal.id)]) self.assertEqual(len(statement), 1) self.assertEqual(len(statement.line_ids), 3) @@ -420,11 +674,6 @@ def test_parse_decimal(self): 1234567.89, self.mock_mapping_dot_comma, ), # inverted separators - ( - "123456", - 1234.56, - self.mock_mapping_none_none, - ), # no separator ] for value, expected, mock_mapping in test_cases: @@ -453,12 +702,16 @@ def test_decimal_and_float_inputs(self): 1234.56, ) - @mute_logger( - "odoo.addons.account_statement_import_sheet_file.models." - "account_statement_import" - ) def test_offsets(self): - journal = self.journal + journal = self.AccountJournal.create( + { + "name": "Bank", + "type": "bank", + "code": "BANK", + "currency_id": self.currency_usd.id, + "suspense_account_id": self.suspense_account.id, + } + ) file_name = "fixtures/sample_statement_offsets.xlsx" data = self._data_file(file_name) wizard = self.AccountStatementImport.with_context(journal_id=journal.id).create( @@ -468,49 +721,72 @@ def test_offsets(self): "sheet_mapping_id": self.sample_statement_map.id, } ) - # First try with incorrect values with self.assertRaises(UserError): wizard.with_context( - account_statement_import_txt_xlsx_test=True + account_statement_import_sheet_file_test=True ).import_file_button() - self.sample_statement_map.write( - {"offset_column": 1, "header_lines_skip_count": 3} + statement_map_offsets = self.sample_statement_map.copy( + { + "offset_column": 1, + "header_lines_skip_count": 3, + } + ) + wizard = self.AccountStatementImport.with_context(journal_id=journal.id).create( + { + "statement_filename": file_name, + "statement_file": data, + "sheet_mapping_id": statement_map_offsets.id, + } ) wizard.with_context( - account_statement_import_txt_xlsx_test=True + account_statement_import_sheet_file_test=True ).import_file_button() - statement = self.AccountBankStatement.search(self.statement_domain) + statement = self.AccountBankStatement.search([("journal_id", "=", journal.id)]) self.assertEqual(len(statement), 1) self.assertEqual(len(statement.line_ids), 2) self.assertEqual(statement.balance_start, 0.0) self.assertEqual(statement.balance_end_real, 1491.5) self.assertEqual(statement.balance_end, 1491.5) - @mute_logger( - "odoo.addons.account_statement_import_sheet_file.models." - "account_statement_import" - ) def test_skip_empty_lines(self): - journal = self.journal + journal = self.AccountJournal.create( + { + "name": "Bank", + "type": "bank", + "code": "BANK", + "currency_id": self.currency_usd.id, + "suspense_account_id": self.suspense_account.id, + } + ) file_name = "fixtures/empty_lines_statement.csv" data = self._data_file(file_name, "utf-8") - self.sample_statement_map.skip_empty_lines = False + statement_map_empty_line = self.sample_statement_map.copy( + { + "skip_empty_lines": False, + } + ) wizard = self.AccountStatementImport.with_context(journal_id=journal.id).create( { "statement_filename": file_name, "statement_file": data, - "sheet_mapping_id": self.sample_statement_map.id, + "sheet_mapping_id": statement_map_empty_line.id, } ) with self.assertRaises(UserError): wizard.with_context( - account_statement_import_txt_xlsx_test=True + account_statement_import_sheet_file_test=True ).import_file_button() - self.sample_statement_map.skip_empty_lines = True + wizard = self.AccountStatementImport.with_context(journal_id=journal.id).create( + { + "statement_filename": file_name, + "statement_file": data, + "sheet_mapping_id": self.sample_statement_map.id, + } + ) wizard.with_context( - account_statement_import_txt_xlsx_test=True + account_statement_import_sheet_file_test=True ).import_file_button() - statement = self.AccountBankStatement.search(self.statement_domain) + statement = self.AccountBankStatement.search([("journal_id", "=", journal.id)]) self.assertEqual(len(statement), 1) self.assertEqual(len(statement.line_ids), 3) self.assertEqual(statement.balance_start, 0.0) diff --git a/account_statement_import_sheet_file/wizard/account_statement_import_sheet_parser.py b/account_statement_import_sheet_file/wizard/account_statement_import_sheet_parser.py index f02d8b87ef..ebcb42b43c 100644 --- a/account_statement_import_sheet_file/wizard/account_statement_import_sheet_parser.py +++ b/account_statement_import_sheet_file/wizard/account_statement_import_sheet_parser.py @@ -1,5 +1,6 @@ # Copyright 2019 ForgeFlow, S.L. # Copyright 2020 CorporateHub (https://corporatehub.eu) +# Copyright 2025 Tecnativa - Pedro M. Baeza # Copyright 2025 Jacques-Etienne Baudoux (BCIM) # License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl). @@ -10,9 +11,11 @@ from collections.abc import Iterable from datetime import datetime from decimal import Decimal -from io import StringIO +from io import BytesIO, StringIO from os import path +import openpyxl + from odoo import api, models from odoo.exceptions import UserError @@ -48,9 +51,25 @@ def parse_header(self, csv_or_xlsx, mapping): if header_line > 0: header_line -= 1 if isinstance(csv_or_xlsx, tuple): - header = [ - str(value).strip() for value in csv_or_xlsx[1].row_values(header_line) - ] + sheet = csv_or_xlsx[1] + if isinstance(sheet, xlrd.sheet.Sheet): + header = [str(value).strip() for value in sheet.row_values(header_line)] + else: + # openpyxl sheet (iter_rows is 1-indexed) + rows = list( + sheet.iter_rows( + min_row=header_line + 1, + max_row=header_line + 1, + values_only=True, + ) + ) + if rows: + header = [ + str(value).strip() if value is not None else "" + for value in rows[0] + ] + else: + header = [] else: [next(csv_or_xlsx) for _i in range(header_line)] header = [value.strip() for value in next(csv_or_xlsx)] @@ -124,7 +143,18 @@ def _get_column_indexes(self, header, column_name, mapping): column_indexes.append(column_index) else: if column_name_or_index: - column_indexes.append(header.index(column_name_or_index)) + if column_name_or_index in header: + column_indexes.append(header.index(column_name_or_index)) + else: + # Try case-insensitive match before raising + header_lower = [h.lower() for h in header] + column_name_lower = column_name_or_index.lower() + if column_name_lower in header_lower: + column_indexes.append( + header_lower.index(column_name_lower) + ) + else: + column_indexes.append(header.index(column_name_or_index)) return column_indexes def _get_column_names(self): @@ -149,6 +179,7 @@ def _get_column_names(self): def _parse_lines(self, mapping, data_file, currency_code): columns = dict() + csv_or_xlsx = None try: workbook = xlrd.open_workbook( file_contents=data_file, @@ -156,40 +187,54 @@ def _parse_lines(self, mapping, data_file, currency_code): mapping.file_encoding if mapping.file_encoding else None ), ) - csv_or_xlsx = ( - workbook, - workbook.sheet_by_index(0), - ) - except xlrd.XLRDError: - csv_options = {} - csv_delimiter = mapping._get_column_delimiter_character() - if csv_delimiter: - csv_options["delimiter"] = csv_delimiter - if mapping.quotechar: - csv_options["quotechar"] = mapping.quotechar + sheet = workbook.sheet_by_index(0) + csv_or_xlsx = (workbook, sheet) + except Exception: + # Try openpyxl for newer .xlsx format try: - decoded_file = data_file.decode(mapping.file_encoding or "utf-8") - except UnicodeDecodeError: - # Try auto guessing the format - detected_encoding = chardet.detect(data_file).get("encoding", False) - if not detected_encoding: - raise UserError( - self.env._("No valid encoding was found for the attached file") - ) from None - decoded_file = data_file.decode(detected_encoding) - csv_or_xlsx = reader(StringIO(decoded_file), **csv_options) + workbook = openpyxl.load_workbook( + filename=BytesIO(data_file), + read_only=True, + data_only=True, + ) + sheet = workbook.active + csv_or_xlsx = (workbook, sheet) + except Exception: + # Fall back to CSV + csv_options = {} + csv_delimiter = mapping._get_column_delimiter_character() + if csv_delimiter: + csv_options["delimiter"] = csv_delimiter + if mapping.quotechar: + csv_options["quotechar"] = mapping.quotechar + try: + decoded_file = data_file.decode(mapping.file_encoding or "utf-8") + except UnicodeDecodeError: + detected_encoding = chardet.detect(data_file).get("encoding", False) + if not detected_encoding: + raise UserError( + self.env._( + "No valid encoding was found for the attached file" + ) + ) from None + decoded_file = data_file.decode(detected_encoding) + csv_or_xlsx = reader(StringIO(decoded_file), **csv_options) + header = self.parse_header(csv_or_xlsx, mapping) - # NOTE no seria necesario debit_column y credit_column ya que tenemos los - # respectivos campos related for column_name in self._get_column_names(): columns[column_name] = self._get_column_indexes( header, column_name, mapping ) - # Get the numbers of rows of the file + # Get the number of rows if isinstance(csv_or_xlsx, tuple): - numrows = csv_or_xlsx[1].nrows + sheet = csv_or_xlsx[1] + numrows = ( + sheet.nrows + if isinstance(sheet, xlrd.sheet.Sheet) + else sheet.max_row + ) else: numrows = len(str(data_file.strip()).split("\\n")) @@ -197,7 +242,15 @@ def _parse_lines(self, mapping, data_file, currency_code): footer_line = numrows - mapping.footer_lines_skip_count if isinstance(csv_or_xlsx, tuple): - rows = range(label_line, footer_line) + sheet = csv_or_xlsx[1] + if isinstance(sheet, xlrd.sheet.Sheet): + rows = range(label_line, footer_line) + else: + rows = sheet.iter_rows( + min_row=label_line + 1, + max_row=footer_line, + values_only=False, + ) else: rows = csv_or_xlsx data = csv_or_xlsx, rows, label_line, footer_line @@ -218,7 +271,10 @@ def _get_values_from_column(self, values, columns, column_name): content_l.append(values[index]) if all(isinstance(content, str) for content in content_l): return " ".join(content_l) - return content_l[0] + elif any(isinstance(content, int) for content in content_l): + # Convert to string when concatenating integer values + return " ".join(str(content) for content in content_l) + return content_l[0] if content_l else None def _parse_one_line(self, mapping, currency_code, values, columns): # noqa: C901 # Get all the raw values from the columns processed in one dict, and extract @@ -321,12 +377,20 @@ def _parse_rows(self, mapping, currency_code, data, columns): book = csv_or_xlsx[0] sheet = csv_or_xlsx[1] values = [] - for col_index in range(mapping.offset_column, sheet.row_len(row)): - cell_type = sheet.cell_type(row, col_index) - cell_value = sheet.cell_value(row, col_index) - if cell_type == xlrd.XL_CELL_DATE: - cell_value = xldate_as_datetime(cell_value, book.datemode) - values.append(cell_value) + if isinstance(sheet, xlrd.sheet.Sheet): + for col_index in range(mapping.offset_column, sheet.row_len(row)): + cell_type = sheet.cell_type(row, col_index) + cell_value = sheet.cell_value(row, col_index) + if cell_type == xlrd.XL_CELL_DATE: + cell_value = xldate_as_datetime(cell_value, book.datemode) + values.append(cell_value) + else: + # openpyxl: row is a tuple of Cell objects + for cell in row[mapping.offset_column :]: + cell_value = cell.value + if isinstance(cell_value, datetime): + cell_value = cell_value.strftime(mapping.timestamp_format) + values.append(str(cell_value) if cell_value is not None else "") else: if index >= footer_line: continue @@ -424,7 +488,9 @@ def _parse_decimal(self, value, mapping): # decimal separator, and signs value = ( re.sub( - r"[^\d\-+" + re.escape(thousands) + re.escape(decimal) + "]+", "", value + r"[^\d\-+" + re.escape(thousands) + re.escape(decimal) + "]+", + "", + str(value), ) or "0" ) From d7d6f5bcde86f8d76dcad1fda8a80e7c42c4e895 Mon Sep 17 00:00:00 2001 From: Felipe Garcia Suez Date: Tue, 26 May 2026 11:01:08 -0300 Subject: [PATCH 2/2] [IMP]account_statement_import_sheet_file: Improve string and help for header_lines_skip_count --- account_statement_import_sheet_file/i18n/es.po | 8 ++++---- .../models/account_statement_import_sheet_mapping.py | 4 ++-- .../wizard/account_statement_import_sheet_parser.py | 8 ++------ 3 files changed, 8 insertions(+), 12 deletions(-) diff --git a/account_statement_import_sheet_file/i18n/es.po b/account_statement_import_sheet_file/i18n/es.po index 380e7d3996..0d3df2c4ad 100644 --- a/account_statement_import_sheet_file/i18n/es.po +++ b/account_statement_import_sheet_file/i18n/es.po @@ -246,8 +246,8 @@ msgstr "Recuento de líneas omitidas a pie de página" #. module: account_statement_import_sheet_file #: model:ir.model.fields,field_description:account_statement_import_sheet_file.field_account_statement_import_sheet_mapping__header_lines_skip_count -msgid "Header lines skip count" -msgstr "Recuento de líneas de cabecera omitidas" +msgid "Header row number" +msgstr "Número de fila del encabezado" #. module: account_statement_import_sheet_file #: model:ir.model.fields,field_description:account_statement_import_sheet_file.field_account_statement_import_sheet_mapping__id @@ -403,8 +403,8 @@ msgstr "" #. module: account_statement_import_sheet_file #: model:ir.model.fields,help:account_statement_import_sheet_file.field_account_statement_import_sheet_mapping__header_lines_skip_count -msgid "Set the Header lines number." -msgstr "Establezca el número de líneas de Cabecera." +msgid "Row number where the column headers are located (first row is 0)." +msgstr "Número de fila donde se encuentran los encabezados de columna (la primera fila es 0)." #. module: account_statement_import_sheet_file #: model:ir.model.fields,field_description:account_statement_import_sheet_file.field_account_statement_import__sheet_mapping_id diff --git a/account_statement_import_sheet_file/models/account_statement_import_sheet_mapping.py b/account_statement_import_sheet_file/models/account_statement_import_sheet_mapping.py index ba4867bdaa..33bdc5876d 100644 --- a/account_statement_import_sheet_file/models/account_statement_import_sheet_mapping.py +++ b/account_statement_import_sheet_file/models/account_statement_import_sheet_mapping.py @@ -170,8 +170,8 @@ class AccountStatementImportSheetMapping(models.Model): default="0", ) header_lines_skip_count = fields.Integer( - string="Header lines skip count", - help="Set the Header lines number.", + string="Header row number", + help="Row number where the column headers are located (first row is 0).", default="0", ) skip_empty_lines = fields.Boolean( diff --git a/account_statement_import_sheet_file/wizard/account_statement_import_sheet_parser.py b/account_statement_import_sheet_file/wizard/account_statement_import_sheet_parser.py index ebcb42b43c..e05cfd932f 100644 --- a/account_statement_import_sheet_file/wizard/account_statement_import_sheet_parser.py +++ b/account_statement_import_sheet_file/wizard/account_statement_import_sheet_parser.py @@ -150,9 +150,7 @@ def _get_column_indexes(self, header, column_name, mapping): header_lower = [h.lower() for h in header] column_name_lower = column_name_or_index.lower() if column_name_lower in header_lower: - column_indexes.append( - header_lower.index(column_name_lower) - ) + column_indexes.append(header_lower.index(column_name_lower)) else: column_indexes.append(header.index(column_name_or_index)) return column_indexes @@ -231,9 +229,7 @@ def _parse_lines(self, mapping, data_file, currency_code): if isinstance(csv_or_xlsx, tuple): sheet = csv_or_xlsx[1] numrows = ( - sheet.nrows - if isinstance(sheet, xlrd.sheet.Sheet) - else sheet.max_row + sheet.nrows if isinstance(sheet, xlrd.sheet.Sheet) else sheet.max_row ) else: numrows = len(str(data_file.strip()).split("\\n"))