From d32515e19d8c5543b4d5b1599e75b5f24067fc8e Mon Sep 17 00:00:00 2001 From: Justus Pendleton Date: Thu, 6 Mar 2025 17:16:17 +1030 Subject: [PATCH] Don't merge deduped entries into existing_entries. Merging pollutes the existing_entries for hooks that get run later. Instead we track them separately but consider the union(existing_entries, new_entries) when running the deduplication logic, since we also want to remove duplicates from other importers in this same. --- beangulp/__init__.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/beangulp/__init__.py b/beangulp/__init__.py index 00b02165..d0566ab9 100644 --- a/beangulp/__init__.py +++ b/beangulp/__init__.py @@ -96,9 +96,12 @@ def _extract(ctx, src, output, existing, reverse, failfast, quiet): extract.sort_extracted_entries(extracted) # Deduplicate. + deduplicated_new_entries = [] for filename, entries, account, importer in extracted: - importer.deduplicate(entries, existing_entries) - existing_entries.extend(entries) + # We want to deduplicate not just against existing_entries + # but also new entries from other importers in this run + importer.deduplicate(entries, existing_entries + deduplicated_new_entries) + deduplicated_new_entries.extend(entries) # Invoke hooks. for func in ctx.hooks: