From e02c022ffd03f53862b14dba093cf4f5ddd44e1d Mon Sep 17 00:00:00 2001 From: Eric Hills <53243273+ebhills@users.noreply.github.com> Date: Sun, 26 Jul 2026 08:26:31 -0500 Subject: [PATCH 1/9] Add dedicated part_code_matches assessment capture all code match information for downstream validation logic --- tests/recipes/wrangles/test_compute.py | 78 +++++++++++++++++- wrangles/compute.py | 105 ++++++++++++++++++++++++- 2 files changed, 180 insertions(+), 3 deletions(-) diff --git a/tests/recipes/wrangles/test_compute.py b/tests/recipes/wrangles/test_compute.py index c0d5ade59..b9d9a7afd 100644 --- a/tests/recipes/wrangles/test_compute.py +++ b/tests/recipes/wrangles/test_compute.py @@ -1,4 +1,5 @@ import wrangles +import wrangles.compute as compute import pandas as pd @@ -349,4 +350,79 @@ def test_search_score(self): assert 'scored_results' in df.columns and 'Score Summary' in df.columns assert isinstance(df.iloc[0]['scored_results'], list) and isinstance(df.iloc[0]['scored_results'][0], dict) assert isinstance(df.iloc[0]['Score Summary'], list) and isinstance(df.iloc[0]['Score Summary'][0], str) - assert len(df.iloc[0]['scored_results']) == 4 and len(df.iloc[0]['Score Summary']) == 4 \ No newline at end of file + assert len(df.iloc[0]['scored_results']) == 4 and len(df.iloc[0]['Score Summary']) == 4 + assert isinstance(df.iloc[0]['scored_results'][0]['part_code_matches'], list) + + def test_part_code_matches_preserve_match_details(self): + payloads = [{ + "search_metadata": {"query_index": 1}, + "search_results": [{ + "title": "Genuine AB-123 replacement", + "snippet": "Compatible code ZX900", + "link": "https://example.com/products/AB123X", + }], + }] + + result = compute.score_search_results( + payloads=payloads, + mpns=["AB-123"], + part_codes=["AB-123", "ZX-900"], + must_match_part_code=False, + )[0] + + assert result["part_code_matches"] == [ + { + "match_type": "MPN", + "match_level": "exact", + "result_source": "title", + "input_code": "AB-123", + "matched_code": "AB-123", + }, + { + "match_type": "MPN", + "match_level": "partial", + "result_source": "url", + "input_code": "AB-123", + "matched_code": "AB123X", + }, + { + "match_type": "Codes", + "match_level": "exact", + "result_source": "title", + "input_code": "AB-123", + "matched_code": "AB-123", + }, + { + "match_type": "Codes", + "match_level": "partial", + "result_source": "url", + "input_code": "AB-123", + "matched_code": "AB123X", + }, + { + "match_type": "Codes", + "match_level": "stripped", + "result_source": "snippet", + "input_code": "ZX-900", + "matched_code": "ZX900", + }, + ] + + def test_part_code_matches_is_empty_when_no_code_matches(self): + payloads = [{ + "search_metadata": {"query_index": 1}, + "search_results": [{ + "title": "Unrelated product", + "snippet": "No matching identifiers", + "link": "https://example.com/products/unrelated", + }], + }] + + result = compute.score_search_results( + payloads=payloads, + mpns=["AB-123"], + part_codes=["AB-123"], + must_match_part_code=False, + )[0] + + assert result["part_code_matches"] == [] diff --git a/wrangles/compute.py b/wrangles/compute.py index 734178013..3ec8addcf 100644 --- a/wrangles/compute.py +++ b/wrangles/compute.py @@ -1,5 +1,6 @@ -from urllib.parse import urlsplit +import re from typing import List, Tuple, Dict +from urllib.parse import urlsplit # Import our separated helpers from . import web as _web @@ -96,6 +97,101 @@ def _evaluate_part_code_match( return best_score, best_reason, best_ratio, best_visual +def _find_part_code_matches( + candidates: list, + raw_fields: dict, + match_type: str, + min_length_for_substring: int = 4 +) -> list: + """ + Return each field-level part-code match without reducing the result to a score. + + Match levels are mutually exclusive for a candidate and field: + - exact: the original candidate appears with alphanumeric boundaries + - stripped: the candidate matches after removing non-alphanumeric characters + - partial: the normalized candidate is embedded in a larger normalized string + """ + def _find_partial_source_code(source_text: str, norm_cand: str) -> str: + for token_match in re.finditer( + r"[a-z0-9]+(?:[-._][a-z0-9]+)*", + source_text, + re.IGNORECASE + ): + token = token_match.group() + if norm_cand in _compare.normalize_alphanum(token): + return token + + normalized_chars = [] + source_indexes = [] + for index, char in enumerate(source_text): + normalized_char = _compare.normalize_alphanum(char) + if normalized_char: + normalized_chars.extend(normalized_char) + source_indexes.extend([index] * len(normalized_char)) + + match_start = "".join(normalized_chars).find(norm_cand) + if match_start < 0: + return "" + + match_end = match_start + len(norm_cand) - 1 + return source_text[source_indexes[match_start]:source_indexes[match_end] + 1] + + matches = [] + seen = set() + + for candidate in candidates or []: + candidate_text = str(candidate).strip() + norm_cand = _compare.normalize_alphanum(candidate_text) + if not norm_cand: + continue + + exact_pattern = re.compile( + rf"(?= min_length_for_substring: + squashed_text = _compare.normalize_alphanum(source_text) + if norm_cand in squashed_text: + match_level = "partial" + matched_code = _find_partial_source_code(source_text, norm_cand) + + if not match_level: + continue + + match = { + "match_type": match_type, + "match_level": match_level, + "result_source": field_name.lower(), + "input_code": candidate_text, + "matched_code": matched_code, + } + match_key = tuple(match.values()) + if match_key not in seen: + seen.add(match_key) + matches.append(match) + + return matches + + def _evaluate_match( candidates: List[str], fields: Dict[str, str], @@ -298,6 +394,10 @@ def _get_tokens(text: str) -> list: # Entity Scoring mpn_score, mpn_reason, mpn_ratio, mpn_vis = _evaluate_part_code_match(mpns, fields_tokens, squashed_fields, mpn_exact_score, mpn_partial_base, "MPN") pc_score, pc_reason, pc_ratio, pc_vis = _evaluate_part_code_match(part_codes, fields_tokens, squashed_fields, part_code_exact_score, part_code_partial_base, "Part Code") + part_code_matches = ( + _find_part_code_matches(mpns, raw_fields, "MPN") + + _find_part_code_matches(part_codes, raw_fields, "Codes") + ) sup_score, sup_reason, sup_ratio, sup_vis = _evaluate_match(suppliers, raw_fields, supplier_exact_score, supplier_partial_base, "Supplier") @@ -417,6 +517,7 @@ def _get_tokens(text: str) -> list: scored_item = { "summary": summary, + "part_code_matches": part_code_matches, "pricing": formatted_pricing, "scoring_details": scoring_details, "metadata": metadata @@ -431,4 +532,4 @@ def _get_tokens(text: str) -> list: for idx, res in enumerate(combined_results, start=1): res["summary"]["scored_result_index"] = idx - return combined_results \ No newline at end of file + return combined_results From 171b68288c5fa33ff1651d5891383025f0feb560 Mon Sep 17 00:00:00 2001 From: Eric Hills <53243273+ebhills@users.noreply.github.com> Date: Sun, 26 Jul 2026 11:35:16 -0500 Subject: [PATCH 2/9] Update compute.py --- wrangles/compute.py | 94 ++++++++++++++++++++++----------------------- 1 file changed, 47 insertions(+), 47 deletions(-) diff --git a/wrangles/compute.py b/wrangles/compute.py index 3ec8addcf..541f3a1a8 100644 --- a/wrangles/compute.py +++ b/wrangles/compute.py @@ -24,7 +24,7 @@ def _get_supplier_site_score(supplier_names: list, netloc: str) -> int: domain_parts = netloc.split('.') if len(domain_parts) < 2: return 0 - + normalized_domain = _compare.normalize_alphanum(domain_parts[-2]) normalized_suppliers = [_compare.normalize_alphanum(s) for s in supplier_names] @@ -34,13 +34,13 @@ def _get_supplier_site_score(supplier_names: list, netloc: str) -> int: for sup_norm in normalized_suppliers: if len(normalized_domain) > 4 and normalized_domain in sup_norm: return 1 if len(sup_norm) > 4 and sup_norm in normalized_domain: return 1 - + return 0 def _evaluate_part_code_match( candidates: list, - fields_tokens: dict, + fields_tokens: dict, squashed_fields: dict, # <--- NEW PARAMETER exact_score: float, partial_base: float, @@ -55,17 +55,17 @@ def _evaluate_part_code_match( best_score, best_ratio = 0.0, 0.0 best_reason = f"No {entity_name} Match" best_visual = "" - - if not candidates: + + if not candidates: return best_score, best_reason, best_ratio, best_visual - + for candidate in candidates: norm_cand = _compare.normalize_alphanum(candidate) if not norm_cand: continue - + cand_len = len(norm_cand) is_short_code = cand_len < min_length_for_substring - + # --- THE FIX: The Long-Code Bypass --- # If the code is 4+ chars, it's safe to check the squashed text for a perfect embedded match if not is_short_code and squashed_fields: @@ -79,11 +79,11 @@ def _evaluate_part_code_match( for token in tokens: if not token: continue token_len = len(token) - - # 1. Exact Token Match + + # 1. Exact Token Match if norm_cand == token: return exact_score, f"Exact Match '{candidate}' ({entity_name}) in {field_name}", 1.0, f"**{norm_cand}**" - + # 2. Variant / Substring Match (e.g., '555' inside 'LM555CN') elif not is_short_code and norm_cand in token: ratio = cand_len / token_len @@ -178,11 +178,11 @@ def _find_partial_source_code(source_text: str, norm_cand: str) -> str: continue match = { - "match_type": match_type, + "match_source": field_name.lower(), "match_level": match_level, - "result_source": field_name.lower(), - "input_code": candidate_text, + "match_type": match_type, "matched_code": matched_code, + "input_code": candidate_text, } match_key = tuple(match.values()) if match_key not in seen: @@ -194,7 +194,7 @@ def _find_partial_source_code(source_text: str, norm_cand: str) -> str: def _evaluate_match( candidates: List[str], - fields: Dict[str, str], + fields: Dict[str, str], exact_score: float, partial_base: float, entity_name: str, @@ -207,24 +207,24 @@ def _evaluate_match( best_score, best_ratio = 0.0, 0.0 best_reason = f"No {entity_name} Match" best_match_str = "" - - if not candidates or not fields: + + if not candidates or not fields: return best_score, best_reason, best_ratio, best_match_str - + for candidate in candidates: norm_cand = _compare.normalize_alphanum(candidate) if not norm_cand: continue - + for field_name, field_text in fields.items(): if not field_text: continue - + norm_field = _compare.normalize_alphanum(field_text) if not norm_field: continue - + # 1. EXACT MATCH if norm_cand == norm_field: return exact_score, f"Exact Match '{candidate}' ({entity_name}) in {field_name}", 1.0, candidate - + # 2. SUBSTRING BYPASS if norm_cand in norm_field: score = round(1.0 * partial_base, 2) @@ -233,15 +233,15 @@ def _evaluate_match( best_reason = f"Embedded Match '{candidate}' ({entity_name}) in {field_name}" best_match_str = candidate continue - + # 3. TOKENIZED FALLBACKS (Reverse Substring & Fuzzy) clean_field = field_text.replace('/', ' ').replace('-', ' ').replace('.', ' ').replace('_', ' ') tokens = clean_field.split() - + for token in tokens: norm_token = _compare.normalize_alphanum(token) if not norm_token: continue - + # A. Reverse Substring Bypass if len(norm_token) >= 5 and norm_token in norm_cand: score = round(0.95 * partial_base, 2) @@ -259,7 +259,7 @@ def _evaluate_match( best_score, best_ratio = score, ratio best_reason = f"Fuzzy Match '{candidate}' ({entity_name}) in {field_name} [{ratio:.2f}]" best_match_str = candidate - + return best_score, best_reason, best_ratio, best_match_str ### Main Score Function ### @@ -294,7 +294,7 @@ def score_search_results( raw_terms.extend(mpns) for d in descriptions: raw_terms.append(d) - raw_terms.extend([w for w in d.split() if len(w) > 2]) + raw_terms.extend([w for w in d.split() if len(w) > 2]) unique_terms = {} for t in raw_terms: @@ -314,7 +314,7 @@ def score_search_results( for item in flat_results: link = item.get("link", "") if not link: continue - + norm_link = _web.normalize_site(link) if norm_link not in deduped_map: deduped_map[norm_link] = item @@ -324,7 +324,7 @@ def score_search_results( unique_raw_results = list(deduped_map.values()) scored_flat_results = [] - + def _get_tokens(text: str) -> list: if not text: return [] clean = text.replace('/', ' ').replace('-', ' ').replace('_', ' ') @@ -341,48 +341,48 @@ def _get_tokens(text: str) -> list: "Snippet": str(item.get("snippet", "")), "URL": str(item.get("link", "")) } - + # 2. Squashed version for the complex Part Code bypass squashed_fields = {k: _compare.normalize_alphanum(v) for k, v in raw_fields.items()} # 3. Create the tokenized version for the short Part Codes & Context Matchers fields_tokens = {k: _get_tokens(v) for k, v in raw_fields.items()} - + # --- REFACTORED CONTEXT MATH --- item_matches = [] # Pool all tokens together from title, snippet, and URL all_tokens = fields_tokens["Title"] + fields_tokens["Snippet"] + fields_tokens["URL"] - + for norm_t, orig_t in unique_terms.items(): term_matched = False - + for token in all_tokens: if not token: continue - + # 1. Exact Token Match if norm_t == token: term_matched = True break - + # 2. Embedded Match (Only if the term is > 3 chars to prevent false positives) elif len(norm_t) > 3 and norm_t in token: term_matched = True break - + # 3. Token Fuzzy Match (Typos) else: ratio, _, _ = _compare.partial_ratio(norm_t, token) if ratio >= fuzzy_match_threshold: term_matched = True break - + if term_matched: item_matches.append(orig_t) context_ratio = len(item_matches) / max(1, len(unique_terms)) context_score = round(context_ratio * context_match_base, 1) - + # Build the specific reason string for context score if item_matches: matched_terms_str = ", ".join([f"'{t}'" for t in item_matches]) @@ -398,7 +398,7 @@ def _get_tokens(text: str) -> list: _find_part_code_matches(mpns, raw_fields, "MPN") + _find_part_code_matches(part_codes, raw_fields, "Codes") ) - + sup_score, sup_reason, sup_ratio, sup_vis = _evaluate_match(suppliers, raw_fields, supplier_exact_score, supplier_partial_base, "Supplier") if mpn_score >= pc_score: @@ -419,13 +419,13 @@ def _get_tokens(text: str) -> list: part_code_found_enum = "other_code_exact" elif pc_score > 0: part_code_found_enum = "other_code_partial" - + part_code_found = part_code_found_enum != "none" supplier_found = sup_ratio >= fuzzy_match_threshold position = item.get("google_rank", 1) position_weight = _get_position_weight(position) - + supplier_site_score = 0 site = item.get("link", "") try: @@ -460,10 +460,10 @@ def _get_tokens(text: str) -> list: "score": total_score, "part_match_score": best_pc_score, "part_match_reason": pc_match_reason, - "part_match_visual": best_vis, + "part_match_visual": best_vis, "brand_score": sup_score, "brand_match_reason": sup_reason, - "brand_match_visual": sup_vis, + "brand_match_visual": sup_vis, } for k, val in sorted_remainder.items(): scoring_details[k] = val @@ -477,10 +477,10 @@ def _get_tokens(text: str) -> list: reason = "no part code match" summary = { - "scored_result_index": 0, + "scored_result_index": 0, "input_row_id": item.get("input_row_id"), "source": item.get("source"), - "score": total_score, + "score": total_score, "link": item.get("link"), "part_code_found": part_code_found_enum, # Storing the enum string here momentarily "brand_found": brand_found, @@ -522,13 +522,13 @@ def _get_tokens(text: str) -> list: "scoring_details": scoring_details, "metadata": metadata } - + scored_flat_results.append(scored_item) filtered_in = sorted([s for s in scored_flat_results if not s["summary"]["filtered"]], key=lambda x: x["summary"]["score"], reverse=True) filtered_out = sorted([s for s in scored_flat_results if s["summary"]["filtered"]], key=lambda x: x["summary"]["score"], reverse=True) combined_results = filtered_in + filtered_out - + for idx, res in enumerate(combined_results, start=1): res["summary"]["scored_result_index"] = idx From c1c09be89ff10ffaa14a0fcc718077777b6d3c35 Mon Sep 17 00:00:00 2001 From: Eric Hills <53243273+ebhills@users.noreply.github.com> Date: Sun, 26 Jul 2026 15:58:18 -0500 Subject: [PATCH 3/9] Refine part code match deduping Tighten search result part-code matching by treating code punctuation as part of token boundaries and reducing duplicate matches to the strongest explanation per occurrence. This also updates compute tests to cover redundant match removal, partial matches inside larger codes, and preferring match quality over MPN provenance. --- tests/recipes/wrangles/test_compute.py | 104 +++++++++++++++++++------ wrangles/compute.py | 42 +++++++++- 2 files changed, 121 insertions(+), 25 deletions(-) diff --git a/tests/recipes/wrangles/test_compute.py b/tests/recipes/wrangles/test_compute.py index b9d9a7afd..912ffcc0a 100644 --- a/tests/recipes/wrangles/test_compute.py +++ b/tests/recipes/wrangles/test_compute.py @@ -372,42 +372,102 @@ def test_part_code_matches_preserve_match_details(self): assert result["part_code_matches"] == [ { - "match_type": "MPN", + "match_source": "title", "match_level": "exact", - "result_source": "title", - "input_code": "AB-123", - "matched_code": "AB-123", - }, - { "match_type": "MPN", - "match_level": "partial", - "result_source": "url", - "input_code": "AB-123", - "matched_code": "AB123X", - }, - { - "match_type": "Codes", - "match_level": "exact", - "result_source": "title", - "input_code": "AB-123", "matched_code": "AB-123", + "input_code": "AB-123", }, { - "match_type": "Codes", + "match_source": "url", "match_level": "partial", - "result_source": "url", - "input_code": "AB-123", + "match_type": "MPN", "matched_code": "AB123X", + "input_code": "AB-123", }, { - "match_type": "Codes", + "match_source": "snippet", "match_level": "stripped", - "result_source": "snippet", - "input_code": "ZX-900", + "match_type": "Codes", "matched_code": "ZX900", + "input_code": "ZX-900", }, ] + def test_part_code_matches_remove_redundant_less_specific_matches(self): + payloads = [{ + "search_metadata": {"query_index": 1}, + "search_results": [{ + "title": "INA NATV6-PP-A track roller", + "snippet": "", + "link": "https://example.com/product", + }], + }] + + result = compute.score_search_results( + payloads=payloads, + mpns=["NATV6-PP-A"], + part_codes=["NATV6-PP-A", "NATV6"], + must_match_part_code=False, + )[0] + + assert result["part_code_matches"] == [{ + "match_source": "title", + "match_level": "exact", + "match_type": "MPN", + "matched_code": "NATV6-PP-A", + "input_code": "NATV6-PP-A", + }] + + def test_part_code_match_inside_larger_code_is_partial(self): + payloads = [{ + "search_metadata": {"query_index": 1}, + "search_results": [{ + "title": "INA NATV6-PP-A track roller", + "snippet": "", + "link": "https://example.com/product", + }], + }] + + result = compute.score_search_results( + payloads=payloads, + part_codes=["NATV6"], + must_match_part_code=False, + )[0] + + assert result["part_code_matches"] == [{ + "match_source": "title", + "match_level": "partial", + "match_type": "Codes", + "matched_code": "NATV6-PP-A", + "input_code": "NATV6", + }] + + def test_part_code_matches_prefer_match_quality_before_provenance(self): + payloads = [{ + "search_metadata": {"query_index": 1}, + "search_results": [{ + "title": "Genuine AB123 replacement", + "snippet": "", + "link": "https://example.com/product", + }], + }] + + result = compute.score_search_results( + payloads=payloads, + mpns=["AB-123"], + part_codes=["AB123"], + must_match_part_code=False, + )[0] + + assert result["part_code_matches"] == [{ + "match_source": "title", + "match_level": "exact", + "match_type": "Codes", + "matched_code": "AB123", + "input_code": "AB123", + }] + def test_part_code_matches_is_empty_when_no_code_matches(self): payloads = [{ "search_metadata": {"query_index": 1}, diff --git a/wrangles/compute.py b/wrangles/compute.py index 541f3a1a8..f6d4941b0 100644 --- a/wrangles/compute.py +++ b/wrangles/compute.py @@ -145,12 +145,15 @@ def _find_partial_source_code(source_text: str, norm_cand: str) -> str: if not norm_cand: continue + code_characters = r"a-z0-9._/-" exact_pattern = re.compile( - rf"(? str: return matches +def _reduce_part_code_matches(matches: list) -> list: + """ + Keep the strongest explanation for each matched code occurrence. + + Evidence is ranked by match level, MPN provenance, and input-code length. + Matches against different result fields or different source codes remain + separate. + """ + match_level_rank = {"partial": 1, "stripped": 2, "exact": 3} + match_type_rank = {"Codes": 1, "MPN": 2} + best_matches = {} + key_order = [] + + for match in matches: + match_key = ( + match.get("match_source"), + _compare.normalize_alphanum(match.get("matched_code", "")), + ) + match_rank = ( + match_level_rank.get(match.get("match_level"), 0), + match_type_rank.get(match.get("match_type"), 0), + len(_compare.normalize_alphanum(match.get("input_code", ""))), + ) + + if match_key not in best_matches: + key_order.append(match_key) + best_matches[match_key] = (match_rank, match) + elif match_rank > best_matches[match_key][0]: + best_matches[match_key] = (match_rank, match) + + return [best_matches[key][1] for key in key_order] + + def _evaluate_match( candidates: List[str], fields: Dict[str, str], @@ -394,7 +430,7 @@ def _get_tokens(text: str) -> list: # Entity Scoring mpn_score, mpn_reason, mpn_ratio, mpn_vis = _evaluate_part_code_match(mpns, fields_tokens, squashed_fields, mpn_exact_score, mpn_partial_base, "MPN") pc_score, pc_reason, pc_ratio, pc_vis = _evaluate_part_code_match(part_codes, fields_tokens, squashed_fields, part_code_exact_score, part_code_partial_base, "Part Code") - part_code_matches = ( + part_code_matches = _reduce_part_code_matches( _find_part_code_matches(mpns, raw_fields, "MPN") + _find_part_code_matches(part_codes, raw_fields, "Codes") ) From 6b788875ddd2ee092dc6a3189a3513f1dbd7e6dd Mon Sep 17 00:00:00 2001 From: Eric Hills <53243273+ebhills@users.noreply.github.com> Date: Sun, 26 Jul 2026 16:32:11 -0500 Subject: [PATCH 4/9] Further refinements to reduce redundant matches --- tests/recipes/wrangles/test_compute.py | 126 ++++++++++++++++++++----- wrangles/compute.py | 78 ++++++++++----- 2 files changed, 159 insertions(+), 45 deletions(-) diff --git a/tests/recipes/wrangles/test_compute.py b/tests/recipes/wrangles/test_compute.py index 912ffcc0a..27e5863cf 100644 --- a/tests/recipes/wrangles/test_compute.py +++ b/tests/recipes/wrangles/test_compute.py @@ -372,23 +372,16 @@ def test_part_code_matches_preserve_match_details(self): assert result["part_code_matches"] == [ { - "match_source": "title", - "match_level": "exact", "match_type": "MPN", + "match_level": "exact", + "match_source": "title", "matched_code": "AB-123", "input_code": "AB-123", }, { - "match_source": "url", - "match_level": "partial", - "match_type": "MPN", - "matched_code": "AB123X", - "input_code": "AB-123", - }, - { - "match_source": "snippet", - "match_level": "stripped", "match_type": "Codes", + "match_level": "stripped", + "match_source": "snippet", "matched_code": "ZX900", "input_code": "ZX-900", }, @@ -412,9 +405,9 @@ def test_part_code_matches_remove_redundant_less_specific_matches(self): )[0] assert result["part_code_matches"] == [{ - "match_source": "title", - "match_level": "exact", "match_type": "MPN", + "match_level": "exact", + "match_source": "title", "matched_code": "NATV6-PP-A", "input_code": "NATV6-PP-A", }] @@ -436,18 +429,18 @@ def test_part_code_match_inside_larger_code_is_partial(self): )[0] assert result["part_code_matches"] == [{ - "match_source": "title", - "match_level": "partial", "match_type": "Codes", + "match_level": "partial", + "match_source": "title", "matched_code": "NATV6-PP-A", "input_code": "NATV6", }] - def test_part_code_matches_prefer_match_quality_before_provenance(self): + def test_part_code_matches_prefer_mpn_over_duplicate_code_evidence(self): payloads = [{ "search_metadata": {"query_index": 1}, "search_results": [{ - "title": "Genuine AB123 replacement", + "title": "Genuine AB123 replacement with ZX900", "snippet": "", "link": "https://example.com/product", }], @@ -455,17 +448,106 @@ def test_part_code_matches_prefer_match_quality_before_provenance(self): result = compute.score_search_results( payloads=payloads, - mpns=["AB-123"], - part_codes=["AB123"], + mpns=["AB-12"], + part_codes=["AB123", "ZX900"], + must_match_part_code=False, + )[0] + + assert result["part_code_matches"] == [ + { + "match_type": "MPN", + "match_level": "partial", + "match_source": "title", + "matched_code": "AB123", + "input_code": "AB-12", + }, + { + "match_type": "Codes", + "match_level": "exact", + "match_source": "title", + "matched_code": "ZX900", + "input_code": "ZX900", + }, + ] + + def test_part_code_matches_url_case_difference_is_exact(self): + payloads = [{ + "search_metadata": {"query_index": 1}, + "search_results": [{ + "title": "INA track roller", + "snippet": "", + "link": "https://example.com/products/natv6-pp-a", + }], + }] + + result = compute.score_search_results( + payloads=payloads, + mpns=["NATV6-PP-A"], + part_codes=["NATV6-PP-A"], must_match_part_code=False, )[0] assert result["part_code_matches"] == [{ - "match_source": "title", + "match_type": "MPN", "match_level": "exact", + "match_source": "url", + "matched_code": "natv6-pp-a", + "input_code": "NATV6-PP-A", + }] + assert list(result["part_code_matches"][0]) == [ + "match_type", + "match_level", + "match_source", + "matched_code", + "input_code", + ] + + def test_part_code_matches_keep_only_one_best_mpn(self): + payloads = [{ + "search_metadata": {"query_index": 1}, + "search_results": [{ + "title": "NATV6-PP-A track roller", + "snippet": "Replacement NATV6-PP-A", + "link": "https://example.com/products/natv6-pp-a", + }], + }] + + result = compute.score_search_results( + payloads=payloads, + mpns=["NATV6-PP-A"], + must_match_part_code=False, + )[0] + + assert result["part_code_matches"] == [{ + "match_type": "MPN", + "match_level": "exact", + "match_source": "title", + "matched_code": "NATV6-PP-A", + "input_code": "NATV6-PP-A", + }] + + def test_part_code_matches_deduplicate_codes_across_sources(self): + payloads = [{ + "search_metadata": {"query_index": 1}, + "search_results": [{ + "title": "NATV6-X-PP-A track roller", + "snippet": "Replacement NATV6-X-PP-A", + "link": "https://example.com/products/NATV6-X-PP-A", + }], + }] + + result = compute.score_search_results( + payloads=payloads, + part_codes=["NATV6"], + must_match_part_code=False, + )[0] + + assert result["part_code_matches"] == [{ "match_type": "Codes", - "matched_code": "AB123", - "input_code": "AB123", + "match_level": "partial", + "match_source": "title", + "matched_code": "NATV6-X-PP-A", + "input_code": "NATV6", }] def test_part_code_matches_is_empty_when_no_code_matches(self): diff --git a/wrangles/compute.py b/wrangles/compute.py index f6d4941b0..807793d3e 100644 --- a/wrangles/compute.py +++ b/wrangles/compute.py @@ -145,7 +145,7 @@ def _find_partial_source_code(source_text: str, norm_cand: str) -> str: if not norm_cand: continue - code_characters = r"a-z0-9._/-" + code_characters = r"a-z0-9._-" exact_pattern = re.compile( rf"(? str: continue match = { - "match_source": field_name.lower(), - "match_level": match_level, "match_type": match_type, + "match_level": match_level, + "match_source": field_name.lower(), "matched_code": matched_code, "input_code": candidate_text, } @@ -197,35 +197,67 @@ def _find_partial_source_code(source_text: str, norm_cand: str) -> str: def _reduce_part_code_matches(matches: list) -> list: """ - Keep the strongest explanation for each matched code occurrence. + Return one best MPN match followed by unique matched Code evidence. - Evidence is ranked by match level, MPN provenance, and input-code length. - Matches against different result fields or different source codes remain - separate. + Duplicate Codes that repeat an MPN input or matched value are omitted. + Repeated Code evidence across fields is reduced to the strongest and most + specific match, preferring title, then snippet, then URL when otherwise tied. """ match_level_rank = {"partial": 1, "stripped": 2, "exact": 3} - match_type_rank = {"Codes": 1, "MPN": 2} - best_matches = {} - key_order = [] + match_source_rank = {"url": 1, "snippet": 2, "title": 3} - for match in matches: - match_key = ( - match.get("match_source"), - _compare.normalize_alphanum(match.get("matched_code", "")), - ) - match_rank = ( + def _match_rank(match: dict) -> tuple: + return ( match_level_rank.get(match.get("match_level"), 0), - match_type_rank.get(match.get("match_type"), 0), len(_compare.normalize_alphanum(match.get("input_code", ""))), + match_source_rank.get(match.get("match_source"), 0), + ) + + mpn_matches = [ + match for match in matches + if match.get("match_type") == "MPN" + ] + best_mpn = max(mpn_matches, key=_match_rank, default=None) + normalized_mpn_inputs = { + _compare.normalize_alphanum(match.get("input_code", "")) + for match in mpn_matches + } + best_mpn_matched_code = ( + _compare.normalize_alphanum(best_mpn.get("matched_code", "")) + if best_mpn else "" + ) + + best_code_matches = {} + code_key_order = [] + + for match in matches: + if match.get("match_type") != "Codes": + continue + + normalized_input = _compare.normalize_alphanum( + match.get("input_code", "") ) + normalized_matched = _compare.normalize_alphanum( + match.get("matched_code", "") + ) + if normalized_input in normalized_mpn_inputs: + continue + if best_mpn_matched_code and normalized_matched == best_mpn_matched_code: + continue - if match_key not in best_matches: - key_order.append(match_key) - best_matches[match_key] = (match_rank, match) - elif match_rank > best_matches[match_key][0]: - best_matches[match_key] = (match_rank, match) + if normalized_matched not in best_code_matches: + code_key_order.append(normalized_matched) + best_code_matches[normalized_matched] = match + elif _match_rank(match) > _match_rank( + best_code_matches[normalized_matched] + ): + best_code_matches[normalized_matched] = match - return [best_matches[key][1] for key in key_order] + reduced_matches = [best_mpn] if best_mpn else [] + reduced_matches.extend( + best_code_matches[key] for key in code_key_order + ) + return reduced_matches def _evaluate_match( From b1a4ca9c7642725106dbf89a7ce4afbe1948151f Mon Sep 17 00:00:00 2001 From: Eric Hills <53243273+ebhills@users.noreply.github.com> Date: Sun, 26 Jul 2026 16:45:02 -0500 Subject: [PATCH 5/9] further redundancy work - remove codes matches contained within the MPN match --- tests/recipes/wrangles/test_compute.py | 34 ++++++++++++++++++++++++++ wrangles/compute.py | 6 +++++ 2 files changed, 40 insertions(+) diff --git a/tests/recipes/wrangles/test_compute.py b/tests/recipes/wrangles/test_compute.py index 27e5863cf..e519dc6f6 100644 --- a/tests/recipes/wrangles/test_compute.py +++ b/tests/recipes/wrangles/test_compute.py @@ -550,6 +550,40 @@ def test_part_code_matches_deduplicate_codes_across_sources(self): "input_code": "NATV6", }] + def test_part_code_matches_remove_partial_code_within_mpn_match(self): + payloads = [{ + "search_metadata": {"query_index": 1}, + "search_results": [{ + "title": "Product details", + "snippet": "Part number 085-196-225 M10", + "link": "https://example.com/product", + }], + }] + + result = compute.score_search_results( + payloads=payloads, + mpns=["085-196-225 M10"], + part_codes=["085-196-225 M10", "225 M10", "085-196-225"], + must_match_part_code=False, + )[0] + + assert result["part_code_matches"] == [ + { + "match_type": "MPN", + "match_level": "exact", + "match_source": "snippet", + "matched_code": "085-196-225 M10", + "input_code": "085-196-225 M10", + }, + { + "match_type": "Codes", + "match_level": "exact", + "match_source": "snippet", + "matched_code": "085-196-225", + "input_code": "085-196-225", + }, + ] + def test_part_code_matches_is_empty_when_no_code_matches(self): payloads = [{ "search_metadata": {"query_index": 1}, diff --git a/wrangles/compute.py b/wrangles/compute.py index 807793d3e..eb8758bdf 100644 --- a/wrangles/compute.py +++ b/wrangles/compute.py @@ -244,6 +244,12 @@ def _match_rank(match: dict) -> tuple: continue if best_mpn_matched_code and normalized_matched == best_mpn_matched_code: continue + if ( + best_mpn_matched_code + and match.get("match_level") == "partial" + and normalized_matched in best_mpn_matched_code + ): + continue if normalized_matched not in best_code_matches: code_key_order.append(normalized_matched) From a2348708fe9539023442b6db61a4fb96ff5b6aad Mon Sep 17 00:00:00 2001 From: Eric Hills <53243273+ebhills@users.noreply.github.com> Date: Sun, 26 Jul 2026 17:28:08 -0500 Subject: [PATCH 6/9] Ongoing redundancy removal --- tests/recipes/wrangles/test_compute.py | 25 ++++++++----------------- wrangles/compute.py | 3 --- 2 files changed, 8 insertions(+), 20 deletions(-) diff --git a/tests/recipes/wrangles/test_compute.py b/tests/recipes/wrangles/test_compute.py index e519dc6f6..22da719bd 100644 --- a/tests/recipes/wrangles/test_compute.py +++ b/tests/recipes/wrangles/test_compute.py @@ -550,7 +550,7 @@ def test_part_code_matches_deduplicate_codes_across_sources(self): "input_code": "NATV6", }] - def test_part_code_matches_remove_partial_code_within_mpn_match(self): + def test_part_code_matches_remove_codes_contained_within_mpn_match(self): payloads = [{ "search_metadata": {"query_index": 1}, "search_results": [{ @@ -567,22 +567,13 @@ def test_part_code_matches_remove_partial_code_within_mpn_match(self): must_match_part_code=False, )[0] - assert result["part_code_matches"] == [ - { - "match_type": "MPN", - "match_level": "exact", - "match_source": "snippet", - "matched_code": "085-196-225 M10", - "input_code": "085-196-225 M10", - }, - { - "match_type": "Codes", - "match_level": "exact", - "match_source": "snippet", - "matched_code": "085-196-225", - "input_code": "085-196-225", - }, - ] + assert result["part_code_matches"] == [{ + "match_type": "MPN", + "match_level": "exact", + "match_source": "snippet", + "matched_code": "085-196-225 M10", + "input_code": "085-196-225 M10", + }] def test_part_code_matches_is_empty_when_no_code_matches(self): payloads = [{ diff --git a/wrangles/compute.py b/wrangles/compute.py index eb8758bdf..cd2ad2d23 100644 --- a/wrangles/compute.py +++ b/wrangles/compute.py @@ -242,11 +242,8 @@ def _match_rank(match: dict) -> tuple: ) if normalized_input in normalized_mpn_inputs: continue - if best_mpn_matched_code and normalized_matched == best_mpn_matched_code: - continue if ( best_mpn_matched_code - and match.get("match_level") == "partial" and normalized_matched in best_mpn_matched_code ): continue From fe944607a29526e53090c07bc6eb3d3e11c1ccf0 Mon Sep 17 00:00:00 2001 From: Eric Hills <53243273+ebhills@users.noreply.github.com> Date: Sun, 26 Jul 2026 17:43:19 -0500 Subject: [PATCH 7/9] added part_code_match_count gives indication of how strongly the part codes show up in the content --- tests/recipes/wrangles/test_compute.py | 31 ++++++++++++++++++++++++++ wrangles/compute.py | 11 ++++++++- 2 files changed, 41 insertions(+), 1 deletion(-) diff --git a/tests/recipes/wrangles/test_compute.py b/tests/recipes/wrangles/test_compute.py index 22da719bd..7a2ddaadb 100644 --- a/tests/recipes/wrangles/test_compute.py +++ b/tests/recipes/wrangles/test_compute.py @@ -411,6 +411,7 @@ def test_part_code_matches_remove_redundant_less_specific_matches(self): "matched_code": "NATV6-PP-A", "input_code": "NATV6-PP-A", }] + assert result["part_code_match_count"] == 3 def test_part_code_match_inside_larger_code_is_partial(self): payloads = [{ @@ -575,6 +576,35 @@ def test_part_code_matches_remove_codes_contained_within_mpn_match(self): "input_code": "085-196-225 M10", }] + def test_part_code_matches_remove_code_input_contained_within_mpn(self): + payloads = [{ + "search_metadata": {"query_index": 1}, + "search_results": [{ + "title": "LBBR 14-2LS linear ball bearing", + "snippet": "", + "link": ( + "https://example.com/lbbr-14-2ls-hv6-ewellix-" + "stainless-steel-linear-ball-bearing" + ), + }], + }] + + result = compute.score_search_results( + payloads=payloads, + mpns=["LBBR14-2LS"], + part_codes=["LBBR14-2LS", "LBBR14"], + must_match_part_code=False, + )[0] + + assert result["part_code_matches"] == [{ + "match_type": "MPN", + "match_level": "stripped", + "match_source": "title", + "matched_code": "LBBR 14-2LS", + "input_code": "LBBR14-2LS", + }] + assert result["part_code_match_count"] == 6 + def test_part_code_matches_is_empty_when_no_code_matches(self): payloads = [{ "search_metadata": {"query_index": 1}, @@ -593,3 +623,4 @@ def test_part_code_matches_is_empty_when_no_code_matches(self): )[0] assert result["part_code_matches"] == [] + assert result["part_code_match_count"] == 0 diff --git a/wrangles/compute.py b/wrangles/compute.py index cd2ad2d23..b04575261 100644 --- a/wrangles/compute.py +++ b/wrangles/compute.py @@ -226,6 +226,10 @@ def _match_rank(match: dict) -> tuple: _compare.normalize_alphanum(best_mpn.get("matched_code", "")) if best_mpn else "" ) + best_mpn_input_code = ( + _compare.normalize_alphanum(best_mpn.get("input_code", "")) + if best_mpn else "" + ) best_code_matches = {} code_key_order = [] @@ -242,6 +246,8 @@ def _match_rank(match: dict) -> tuple: ) if normalized_input in normalized_mpn_inputs: continue + if best_mpn_input_code and normalized_input in best_mpn_input_code: + continue if ( best_mpn_matched_code and normalized_matched in best_mpn_matched_code @@ -465,10 +471,12 @@ def _get_tokens(text: str) -> list: # Entity Scoring mpn_score, mpn_reason, mpn_ratio, mpn_vis = _evaluate_part_code_match(mpns, fields_tokens, squashed_fields, mpn_exact_score, mpn_partial_base, "MPN") pc_score, pc_reason, pc_ratio, pc_vis = _evaluate_part_code_match(part_codes, fields_tokens, squashed_fields, part_code_exact_score, part_code_partial_base, "Part Code") - part_code_matches = _reduce_part_code_matches( + raw_part_code_matches = ( _find_part_code_matches(mpns, raw_fields, "MPN") + _find_part_code_matches(part_codes, raw_fields, "Codes") ) + part_code_match_count = len(raw_part_code_matches) + part_code_matches = _reduce_part_code_matches(raw_part_code_matches) sup_score, sup_reason, sup_ratio, sup_vis = _evaluate_match(suppliers, raw_fields, supplier_exact_score, supplier_partial_base, "Supplier") @@ -589,6 +597,7 @@ def _get_tokens(text: str) -> list: scored_item = { "summary": summary, "part_code_matches": part_code_matches, + "part_code_match_count": part_code_match_count, "pricing": formatted_pricing, "scoring_details": scoring_details, "metadata": metadata From 24b31b5db02cef6a5bcb39bc15b017256a8594ba Mon Sep 17 00:00:00 2001 From: Eric Hills <53243273+ebhills@users.noreply.github.com> Date: Sun, 26 Jul 2026 18:24:27 -0500 Subject: [PATCH 8/9] further code matching refinements --- tests/recipes/wrangles/test_compute.py | 24 ++++++++++++++++++++++++ wrangles/compute.py | 7 ++++++- 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/tests/recipes/wrangles/test_compute.py b/tests/recipes/wrangles/test_compute.py index 7a2ddaadb..6db21167b 100644 --- a/tests/recipes/wrangles/test_compute.py +++ b/tests/recipes/wrangles/test_compute.py @@ -503,6 +503,30 @@ def test_part_code_matches_url_case_difference_is_exact(self): "input_code", ] + def test_part_code_match_fallback_reclassifies_identical_token_as_exact(self): + payloads = [{ + "search_metadata": {"query_index": 1}, + "search_results": [{ + "title": "Product details", + "snippet": "Olympic Part Code: 085-196-225.", + "link": "https://example.com/product", + }], + }] + + result = compute.score_search_results( + payloads=payloads, + part_codes=["085-196-225"], + must_match_part_code=False, + )[0] + + assert result["part_code_matches"] == [{ + "match_type": "Codes", + "match_level": "exact", + "match_source": "snippet", + "matched_code": "085-196-225", + "input_code": "085-196-225", + }] + def test_part_code_matches_keep_only_one_best_mpn(self): payloads = [{ "search_metadata": {"query_index": 1}, diff --git a/wrangles/compute.py b/wrangles/compute.py index b04575261..d342dfbc5 100644 --- a/wrangles/compute.py +++ b/wrangles/compute.py @@ -174,8 +174,13 @@ def _find_partial_source_code(source_text: str, norm_cand: str) -> str: elif len(norm_cand) >= min_length_for_substring: squashed_text = _compare.normalize_alphanum(source_text) if norm_cand in squashed_text: - match_level = "partial" matched_code = _find_partial_source_code(source_text, norm_cand) + if matched_code.casefold() == candidate_text.casefold(): + match_level = "exact" + elif _compare.normalize_alphanum(matched_code) == norm_cand: + match_level = "stripped" + else: + match_level = "partial" if not match_level: continue From 3ad0680319cba09c5e16e77e3473a6c4b962afd2 Mon Sep 17 00:00:00 2001 From: Eric Hills <53243273+ebhills@users.noreply.github.com> Date: Sat, 10 Oct 2026 19:43:27 -0500 Subject: [PATCH 9/9] Harden part-code evidence boundaries and document output contract --- .gitignore | 1 + README.md | 3 + docs/search_result_scoring.md | 79 ++++++++++++++++++ tests/recipes/wrangles/test_compute.py | 107 +++++++++++++++++++++++++ wrangles/compute.py | 41 +++++++--- wrangles/recipe_wrangles/compute.py | 2 +- 6 files changed, 221 insertions(+), 12 deletions(-) create mode 100644 docs/search_result_scoring.md diff --git a/.gitignore b/.gitignore index 453c8bef6..e3ad1d919 100644 --- a/.gitignore +++ b/.gitignore @@ -131,6 +131,7 @@ docs/* !docs/search_ai_mode.md !docs/search_retrieve_link_content.md !docs/search_ai_overview_research.md +!docs/search_result_scoring.md .DS_Store # Spyder project settings diff --git a/README.md b/README.md index a0fb6ff1e..52de3d6ef 100644 --- a/README.md +++ b/README.md @@ -190,6 +190,9 @@ Recipes are written in YAML and allow a series of Wrangles to be run as an autom For targeted web content retrieval with prompts that use values from each row, see [Retrieve content from links](docs/search_retrieve_link_content.md). +For part-code match evidence and counts in scored search results, see +[Search-result part-code evidence](docs/search_result_scoring.md). + Recipes can be triggered either from python code or a terminal command. #### Run ```python diff --git a/docs/search_result_scoring.md b/docs/search_result_scoring.md new file mode 100644 index 000000000..19bed1aca --- /dev/null +++ b/docs/search_result_scoring.md @@ -0,0 +1,79 @@ +# Search-result part-code evidence + +`compute.score_search_results` scores classic search payloads and adds two +diagnostic fields to each scored-result dictionary: + +- `part_code_matches`: reduced evidence from the result's title, snippet, and URL. +- `part_code_match_count`: the number of distinct candidate/type/source matches + collected before redundancy removal. + +These fields are nested alongside `summary`, `pricing`, `scoring_details`, and +`metadata`. They do not add a third recipe output. A single output column still +receives scored dictionaries; two output columns still receive dictionaries and +formatted summaries. The numeric scores, filtering, and result ordering are +unchanged. + +## Evidence records + +The keys in each record appear in this order: + +| Key | Values or meaning | +| --- | --- | +| `match_type` | `MPN` for an MPN input, or `Codes` for a `part_codes` input. | +| `match_level` | `exact`, `stripped`, or `partial`. | +| `match_source` | `title`, `snippet`, or `url`. | +| `matched_code` | The matching text from the result, preserving its case and separators. | +| `input_code` | The input candidate, with surrounding whitespace removed. | + +Matching is case-insensitive. `exact` means the complete source code matches the +input apart from case. `stripped` means the codes match after normalization with +the existing alphanumeric helper, which removes punctuation and spaces and folds +international characters. `partial` means the normalized input is contained in +a larger source code. Partial matching requires at least four normalized input +characters to avoid incidental matches for short codes. + +For example, `NATV6-PP-A` against `natv6-pp-a` is exact, `LBBR14-2LS` against +`LBBR 14-2LS` is stripped, and `NATV6` against `NATV6-X-PP-A` is partial. A +standalone `085-196-225` followed by sentence punctuation remains exact. + +The diagnostic levels describe the source evidence more precisely than the +existing scoring labels. In particular, the existing scoring evaluator can +award its exact-match score to normalized embedded text; this does not turn a +diagnostic partial match into an exact one. + +## Redundancy removal and counts + +Each result contains zero or one MPN record, placed before all Code records even +when the MPN match is partial. The strongest MPN evidence wins: exact before +stripped before partial, then the longer normalized input, then title before +snippet before URL. + +Code records are removed when they repeat an MPN input that matched, their +normalized input is contained in the selected MPN input, or their normalized +matched text is contained in the selected MPN matched text. Remaining records +with the same normalized matched code are collapsed across sources, using the +same strength, input-length, and source preferences. + +`part_code_match_count` is recorded before these reductions. It includes the MPN +repeated in `part_codes` and evidence from multiple sources, but does not count +identical duplicate candidates or repeated occurrences within one source. It is +an evidence count, not a count of distinct products or codes. No matches produces +an empty list and a count of zero. + +For an MPN of `NATV6-PP-A`, `part_codes` of `[NATV6-PP-A, NATV6]`, and a title of +`INA NATV6-PP-A track roller` with no other matching fields, the result includes: + +```json +{ + "part_code_matches": [ + { + "match_type": "MPN", + "match_level": "exact", + "match_source": "title", + "matched_code": "NATV6-PP-A", + "input_code": "NATV6-PP-A" + } + ], + "part_code_match_count": 3 +} +``` diff --git a/tests/recipes/wrangles/test_compute.py b/tests/recipes/wrangles/test_compute.py index 6db21167b..96699df87 100644 --- a/tests/recipes/wrangles/test_compute.py +++ b/tests/recipes/wrangles/test_compute.py @@ -1,6 +1,7 @@ import wrangles import wrangles.compute as compute import pandas as pd +import pytest class TestCaseWhen: @@ -527,6 +528,29 @@ def test_part_code_match_fallback_reclassifies_identical_token_as_exact(self): "input_code": "085-196-225", }] + @pytest.mark.parametrize("input_code,source_text,expected_level", [ + ("085-196-225 M10", "085-196-225 M100", "partial"), + ("ABCD", "XYZAB CD1", "partial"), + ("ABC", "ABC.", "exact"), + ("ABC", "LMABC123", None), + ]) + def test_part_code_match_fallback_respects_complete_code_boundaries( + self, input_code, source_text, expected_level + ): + matches = compute._find_part_code_matches( + [input_code], {"Snippet": source_text}, "Codes" + ) + + if expected_level is None: + assert matches == [] + else: + assert len(matches) == 1 + assert matches[0]["match_level"] == expected_level + assert matches[0]["matched_code"] == ( + source_text.rstrip(".") if expected_level == "exact" + else source_text + ) + def test_part_code_matches_keep_only_one_best_mpn(self): payloads = [{ "search_metadata": {"query_index": 1}, @@ -551,6 +575,14 @@ def test_part_code_matches_keep_only_one_best_mpn(self): "input_code": "NATV6-PP-A", }] + def test_sentence_punctuation_does_not_hide_stronger_exact_evidence(self): + matches = compute._find_part_code_matches( + ["AB-123"], {"Snippet": "AB-123. Alternative AB123"}, "Codes" + ) + + assert matches[0]["match_level"] == "exact" + assert matches[0]["matched_code"] == "AB-123" + def test_part_code_matches_deduplicate_codes_across_sources(self): payloads = [{ "search_metadata": {"query_index": 1}, @@ -648,3 +680,78 @@ def test_part_code_matches_is_empty_when_no_code_matches(self): assert result["part_code_matches"] == [] assert result["part_code_match_count"] == 0 + + def test_part_code_match_count_ignores_duplicate_candidates_and_occurrences(self): + payloads = [{ + "search_results": [{ + "title": "AB-123 replacement for AB-123", + "link": "https://example.com/product", + }], + }] + + result = compute.score_search_results( + payloads=payloads, + mpns=["AB-123"], + part_codes=["AB-123", "AB-123"], + must_match_part_code=False, + )[0] + + assert result["part_code_match_count"] == 2 + assert len(result["part_code_matches"]) == 1 + assert result["part_code_matches"][0]["match_type"] == "MPN" + + def test_part_code_matches_are_nested_in_single_recipe_output(self): + result = wrangles.recipe.run( + """ + wrangles: + - compute.score_search_results: + input: [results, suppliers, part_codes, MPN, Description] + output: scored_results + must_match_part_code: false + """, + dataframe=self.score_data, + ) + + assert "part_code_matches" not in result.columns + assert "part_code_match_count" not in result.columns + assert isinstance(result.iloc[0]["scored_results"][0]["part_code_matches"], list) + assert isinstance(result.iloc[0]["scored_results"][0]["part_code_match_count"], int) + + @pytest.mark.parametrize("must_match_part_code", [True, False]) + def test_part_code_evidence_does_not_change_scoring_or_filtering( + self, monkeypatch, must_match_part_code + ): + payloads = [{ + "search_metadata": {"query_index": 1}, + "search_results": [ + { + "title": "Brand AB-123 bearing", + "link": "https://example.com/products/AB-123", + "google_rank": 2, + }, + { + "title": "Brand unrelated bearing", + "link": "https://example.com/products/other", + "google_rank": 1, + }, + ], + }] + kwargs = { + "suppliers": ["Brand"], + "mpns": ["AB-123"], + "part_codes": ["AB-123", "AB12"], + "must_match_part_code": must_match_part_code, + } + with_evidence = compute.score_search_results(payloads, **kwargs) + assert with_evidence[0]["part_code_match_count"] > 0 + + monkeypatch.setattr(compute, "_find_part_code_matches", lambda *args: []) + without_evidence = compute.score_search_results(payloads, **kwargs) + + def without_diagnostics(results): + return [{ + key: value for key, value in result.items() + if key not in {"part_code_matches", "part_code_match_count"} + } for result in results] + + assert without_diagnostics(with_evidence) == without_diagnostics(without_evidence) diff --git a/wrangles/compute.py b/wrangles/compute.py index d342dfbc5..dd178e087 100644 --- a/wrangles/compute.py +++ b/wrangles/compute.py @@ -107,16 +107,17 @@ def _find_part_code_matches( Return each field-level part-code match without reducing the result to a score. Match levels are mutually exclusive for a candidate and field: - - exact: the original candidate appears with alphanumeric boundaries + - exact: the original candidate matches a complete code, ignoring case - stripped: the candidate matches after removing non-alphanumeric characters - partial: the normalized candidate is embedded in a larger normalized string """ def _find_partial_source_code(source_text: str, norm_cand: str) -> str: - for token_match in re.finditer( + code_tokens = list(re.finditer( r"[a-z0-9]+(?:[-._][a-z0-9]+)*", source_text, re.IGNORECASE - ): + )) + for token_match in code_tokens: token = token_match.group() if norm_cand in _compare.normalize_alphanum(token): return token @@ -134,7 +135,16 @@ def _find_partial_source_code(source_text: str, norm_cand: str) -> str: return "" match_end = match_start + len(norm_cand) - 1 - return source_text[source_indexes[match_start]:source_indexes[match_end] + 1] + source_start = source_indexes[match_start] + source_end = source_indexes[match_end] + 1 + # Include the whole source code when the overlap crosses whitespace. + # Otherwise M10 inside M100 would look like an exact match. + for token_match in code_tokens: + if token_match.start() <= source_start < token_match.end(): + source_start = token_match.start() + if token_match.start() < source_end <= token_match.end(): + source_end = token_match.end() + return source_text[source_start:source_end] matches = [] seen = set() @@ -145,15 +155,18 @@ def _find_partial_source_code(source_text: str, norm_cand: str) -> str: if not norm_cand: continue - code_characters = r"a-z0-9._-" + # A period is internal to a code only when it connects alphanumerics. + # Sentence punctuation must not hide exact evidence behind a variant. + left_boundary = r"(? str: elif stripped_match: match_level = "stripped" matched_code = stripped_match.group() - elif len(norm_cand) >= min_length_for_substring: + else: squashed_text = _compare.normalize_alphanum(source_text) if norm_cand in squashed_text: matched_code = _find_partial_source_code(source_text, norm_cand) @@ -179,7 +192,7 @@ def _find_partial_source_code(source_text: str, norm_cand: str) -> str: match_level = "exact" elif _compare.normalize_alphanum(matched_code) == norm_cand: match_level = "stripped" - else: + elif len(norm_cand) >= min_length_for_substring: match_level = "partial" if not match_level: @@ -363,7 +376,13 @@ def score_search_results( context_match_base: float = 2.0, fuzzy_match_threshold: float = 0.8 ) -> list: - """Core function that scores a single list of search payloads.""" + """Score search payloads and attach reduced part-code matching evidence. + + Each result includes ``part_code_matches`` and ``part_code_match_count``. + The count covers distinct candidate/type/field matches before reduction, + including the MPN's duplicate in ``part_codes``. It is not an occurrence + count within a field and does not affect scores or filtering. + """ suppliers = suppliers or [] part_codes = part_codes or [] mpns = mpns or [] diff --git a/wrangles/recipe_wrangles/compute.py b/wrangles/recipe_wrangles/compute.py index 5f1c62e0b..9ad5aa07e 100644 --- a/wrangles/recipe_wrangles/compute.py +++ b/wrangles/recipe_wrangles/compute.py @@ -83,7 +83,7 @@ def score_search_results( ) -> _pd.DataFrame: """ type: object - description: Scores and filters search results based on progressive partial/exact matching. Can return dictionaries or a parallel list of formatted strings. + description: Scores and filters search results based on progressive partial/exact matching. Each result includes reduced part_code_matches evidence and part_code_match_count before reduction. Can return dictionaries or a parallel list of formatted strings. additionalProperties: false required: - input