Skip to content

Commit 671174d

Browse files
committed
Skip figure text layers on born-digital pages; justified column paragraphs stay left; APA author lines are references; Libertine TB is bold
1 parent a60a23c commit 671174d

7 files changed

Lines changed: 193 additions & 7 deletions

File tree

‎CHANGELOG.md‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,10 @@
33
## Unreleased
44

55
### Fixed
6+
- A figure's invisible text layer on a born-digital page is no longer painted in Chinese over the figure's English pixels (guide p10 overlap and tofu).
7+
- A justified column paragraph with an indented first line and short last line is no longer centered.
8+
- An APA author line split from its title (`Shen, X., … (2024).`) is kept as a reference.
9+
- Linux Libertine `TB`/`TI`/`TBI` faces are read as bold/italic, so bold run-in headings keep their weight.
610
- A display equation that moves up with its shortened paragraph no longer leaves ghost `′ ˆ` shards and its fraction bar at the old position; wide fraction bars inside a display line move with it.
711
- An `Appendix A.2.2 …` heading run into its first sentence now ends the bibliography instead of being kept as a reference entry.
812
- A gateway HTTP 400 or 504 is retried like a 5xx; the pooled gateway returns 400 for transient upstream failures.

‎crates/rpt-core/src/font/mod.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ pub use cmap::CMap;
3232
pub use predefined::Predefined;
3333
#[cfg(test)]
3434
pub use subset::box_ttf;
35+
pub(crate) use subset::libertine_weight;
3536
#[allow(unused_imports)]
3637
pub use subset::load_cjk_font;
3738
pub use subset::{

‎crates/rpt-core/src/font/subset.rs‎

Lines changed: 35 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,22 @@ pub struct FaceStyle {
3434
pub italic: bool,
3535
}
3636

37+
/// The Type 1 Linux Libertine and Biolinum set marks weight in a short
38+
/// suffix: `LinLibertineT` is regular, `TB` bold, `TI` italic, `TBI` both
39+
/// (`RKZODD+LinLibertineTB` is a bold run-in heading).
40+
pub(crate) fn libertine_weight(font_name: &str) -> (bool, bool) {
41+
let face = font_name.rsplit('+').next().unwrap_or(font_name);
42+
for family in ["LinLibertine", "LinBiolinum"] {
43+
if let Some(rest) = face.strip_prefix(family) {
44+
let rest = rest.strip_prefix(['T', 'O']).unwrap_or(rest);
45+
if rest.len() <= 3 && rest.chars().all(|ch| ch.is_ascii_uppercase()) {
46+
return (rest.contains('B'), rest.contains('I'));
47+
}
48+
}
49+
}
50+
(false, false)
51+
}
52+
3753
pub fn face_style(font_name: &str) -> FaceStyle {
3854
let upper = font_name.to_ascii_uppercase();
3955
let serif = [
@@ -59,12 +75,16 @@ pub fn face_style(font_name: &str) -> FaceStyle {
5975
// Medium (NimbusRom-Medi, the usual LaTeX \textbf face) is not `bold`
6076
// here: a running header in that face must not be enlarged as a
6177
// subsection. `medium_weight` still selects the bold CJK face.
62-
let bold = ["BOLD", "BLACK", "HEAVY", "SEMIBOLD", "DEMIBOLD", "CMBX"]
63-
.iter()
64-
.any(|needle| upper.contains(needle));
65-
let italic = ["ITAL", "OBLIQUE", "CMTI"]
66-
.iter()
67-
.any(|needle| upper.contains(needle));
78+
let (libertine_bold, libertine_italic) = libertine_weight(font_name);
79+
let serif = serif || upper.contains("LIBERTINE");
80+
let bold = libertine_bold
81+
|| ["BOLD", "BLACK", "HEAVY", "SEMIBOLD", "DEMIBOLD", "CMBX"]
82+
.iter()
83+
.any(|needle| upper.contains(needle));
84+
let italic = libertine_italic
85+
|| ["ITAL", "OBLIQUE", "CMTI"]
86+
.iter()
87+
.any(|needle| upper.contains(needle));
6888
FaceStyle {
6989
serif,
7090
bold,
@@ -1430,6 +1450,15 @@ fn i16be(data: &[u8], at: usize) -> Option<i16> {
14301450

14311451
#[cfg(test)]
14321452
mod tests {
1453+
#[test]
1454+
fn a_libertine_weight_suffix_is_read() {
1455+
assert_eq!(libertine_weight("RKZODD+LinLibertineTB"), (true, false));
1456+
assert_eq!(libertine_weight("QKFIIL+LinLibertineT"), (false, false));
1457+
assert_eq!(libertine_weight("LinLibertineTBI"), (true, true));
1458+
assert_eq!(libertine_weight("LinBiolinumTI"), (false, true));
1459+
assert!(face_style("RKZODD+LinLibertineTB").bold);
1460+
}
1461+
14331462
use super::*;
14341463

14351464
#[test]

‎crates/rpt-core/src/overlap.rs‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,21 @@ pub fn block_alignment(
8888
// Two full lines of a centered header share both edges, the same way a
8989
// justified column does. The margin keeps that column out: a body line
9090
// sits closer to the page edge than a title.
91+
// A justified paragraph in a column (indented first line, full middle
92+
// lines, short last line) shares the column midpoint on its full lines.
93+
// A short line that still starts on the column's left edge is not
94+
// centered, so the block is not either.
95+
if let Some((col_left, col_right)) = column {
96+
let full = lines.iter().any(|line| {
97+
(line.left - col_left).abs() <= 2.5 && (line.right - col_right).abs() <= 2.5
98+
});
99+
let short_flush = lines
100+
.iter()
101+
.any(|line| (line.left - col_left).abs() <= 2.5 && line.right < col_right - size * 0.5);
102+
if lines.len() >= 2 && full && short_flush {
103+
return BlockAlign::Start;
104+
}
105+
}
91106
let page_centered = lines.len() <= 4
92107
&& lines.iter().all(|line| {
93108
(mid(line) - page_mid).abs() <= tol
@@ -625,6 +640,18 @@ fn same_baseline_ink_hits(page: u32, glyphs: &[&Glyph]) -> Vec<OverlapHit> {
625640

626641
#[cfg(test)]
627642
mod tests {
643+
#[test]
644+
fn a_justified_column_paragraph_with_a_short_last_line_is_not_centered() {
645+
let line = |left: f32, right: f32| AlignLine {
646+
left,
647+
right,
648+
size: 9.0,
649+
};
650+
let lines = [line(316.6, 557.7), line(306.6, 557.7), line(306.6, 546.9)];
651+
let align = block_alignment(&lines, [0.0, 0.0, 595.0, 842.0], Some((306.6, 557.7)));
652+
assert!(matches!(align, BlockAlign::Start));
653+
}
654+
628655
use super::*;
629656
use crate::glyph::{Disposition, Glyph, GlyphSource, SourceKind};
630657
use crate::Color;

‎crates/rpt-core/src/rewrite.rs‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23530,7 +23530,11 @@ fn bold_run_in_heading(
2353023530

2353123531
fn bold_face(font: &str) -> bool {
2353223532
let face = font.rsplit('+').next().unwrap_or(font).to_ascii_lowercase();
23533-
face.contains("bold") || face.contains("-bd") || face.ends_with(".b") || face.contains("cmbx")
23533+
face.contains("bold")
23534+
|| face.contains("-bd")
23535+
|| face.ends_with(".b")
23536+
|| face.contains("cmbx")
23537+
|| crate::font::libertine_weight(font).0
2353423538
}
2353523539

2353623540
fn block_source_text(

‎crates/rpt-core/src/segment.rs‎

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1138,6 +1138,35 @@ fn ocr_rule_noise(text: &str) -> bool {
11381138
/// A page of OCR with no sentence and several garbage tokens. `TOILETTE`
11391139
/// and `PEFARTMENT` look like words; `POBUtfttCO` and `3jOV£9rOIit` do not.
11401140
/// Two of those, and no sentence, means the page is the scan image.
1141+
/// A born-digital page can embed a screenshot as a form figure that carries
1142+
/// its own invisible text layer (`Goal prompt for GuideEvolve`). The pixels
1143+
/// stay English, so painting that layer in Chinese stacked two texts on the
1144+
/// figure. A scanned page has no visible text of its own and keeps its layer.
1145+
fn detach_figure_text_layers(lines: &mut Vec<VisualLine<'_>>) -> Vec<(u32, String)> {
1146+
let mut visible: std::collections::HashMap<u32, usize> = std::collections::HashMap::new();
1147+
for line in lines.iter() {
1148+
let count = line
1149+
.glyphs
1150+
.iter()
1151+
.filter(|glyph| !glyph.invisible && !glyph.unicode.trim().is_empty())
1152+
.count();
1153+
*visible.entry(line.page).or_default() += count;
1154+
}
1155+
let mut drop = std::collections::HashSet::new();
1156+
for line in lines.iter() {
1157+
if visible.get(&line.page).copied().unwrap_or(0) < 20 {
1158+
continue;
1159+
}
1160+
for glyph in &line.glyphs {
1161+
if glyph.invisible && matches!(glyph.source.kind, crate::glyph::SourceKind::FormXObject)
1162+
{
1163+
drop.insert(glyph.id);
1164+
}
1165+
}
1166+
}
1167+
drop_glyphs_as(lines, drop, "figure")
1168+
}
1169+
11411170
fn detach_junk_ocr_pages(lines: &mut Vec<VisualLine<'_>>) -> Vec<(u32, String)> {
11421171
let mut pages: Vec<u32> = lines.iter().map(|line| line.page).collect();
11431172
pages.sort_unstable();
@@ -1847,6 +1876,7 @@ fn assemble(
18471876
kept.extend(detach_rule_noise(&mut lines));
18481877
// A scanned cover whose OCR is `3jOV£9rOIit` has no sentence. Painting a
18491878
// translation covers the photograph with gibberish. The image stays.
1879+
kept.extend(detach_figure_text_layers(&mut lines));
18501880
kept.extend(detach_junk_ocr_pages(&mut lines));
18511881
kept.extend(detach_low_quality_ocr(&mut lines, regions, pages));
18521882
attach_markers(&mut lines);
@@ -14790,6 +14820,48 @@ mod tests {
1479014820
);
1479114821
}
1479214822

14823+
#[test]
14824+
fn a_figure_text_layer_on_a_born_digital_page_is_not_translated() {
14825+
let mut glyphs = Vec::new();
14826+
let words = [
14827+
"The",
14828+
"method",
14829+
"improves",
14830+
"accuracy",
14831+
"on",
14832+
"every",
14833+
"benchmark",
14834+
"we",
14835+
"tried",
14836+
];
14837+
let mut id = 0u32;
14838+
for row in 0..3 {
14839+
for (col, word) in words.iter().enumerate() {
14840+
glyphs.push(glyph(
14841+
id,
14842+
72.0 + col as f32 * 50.0,
14843+
700.0 - row as f32 * 12.0,
14844+
word,
14845+
false,
14846+
));
14847+
id += 1;
14848+
}
14849+
}
14850+
let mut hidden = glyph(
14851+
id,
14852+
72.0,
14853+
400.0,
14854+
"Complete Autonomy: unrestricted access",
14855+
false,
14856+
);
14857+
hidden.invisible = true;
14858+
hidden.source.kind = crate::glyph::SourceKind::FormXObject;
14859+
glyphs.push(hidden);
14860+
let segments = segment_glyphs(&glyphs);
14861+
assert!(segments.iter().all(|seg| !seg.glyph_ids.contains(&id)));
14862+
assert!(segments.iter().any(|seg| seg.text.contains("benchmark")));
14863+
}
14864+
1479314865
#[test]
1479414866
fn a_junk_ocr_cover_stays_out_of_the_segments() {
1479514867
let mut glyphs = Vec::new();

‎crates/rpt-core/src/translate/mod.rs‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4878,9 +4878,48 @@ pub(crate) fn intentional_keep_kind(source: &str) -> Option<&'static str> {
48784878
if looks_like_display_math(source) {
48794879
return Some("formula");
48804880
}
4881+
if apa_author_line(source) {
4882+
return Some("references");
4883+
}
48814884
None
48824885
}
48834886

4887+
/// `Shen, X., Chen, Z., Backes, M., Shen, Y., & Zhang, Y. (2024).` is the
4888+
/// first line of an APA entry that a column break split from its title.
4889+
/// Every name is `Surname, I.` and the line ends on the year.
4890+
pub(crate) fn apa_author_line(source: &str) -> bool {
4891+
let text = source.trim().trim_end_matches('.');
4892+
let Some(open) = text.rfind('(') else {
4893+
return false;
4894+
};
4895+
if !text.ends_with(')') {
4896+
return false;
4897+
}
4898+
let year = text[open + 1..text.len() - 1].trim_end_matches(|ch: char| ch.is_ascii_lowercase());
4899+
if year.len() != 4 || !year.chars().all(|ch| ch.is_ascii_digit()) {
4900+
return false;
4901+
}
4902+
let names = text[..open].replace('&', "");
4903+
let parts: Vec<&str> = names
4904+
.split(',')
4905+
.map(str::trim)
4906+
.filter(|part| !part.is_empty())
4907+
.collect();
4908+
if parts.len() < 2 || !parts.len().is_multiple_of(2) {
4909+
return false;
4910+
}
4911+
parts.chunks(2).all(|pair| {
4912+
let surname_ok = pair[0].chars().next().is_some_and(char::is_uppercase)
4913+
&& pair[0].split_whitespace().count() <= 3;
4914+
let initials_ok = pair[1].split_whitespace().all(|word| {
4915+
word.split(['-', '.'])
4916+
.filter(|piece| !piece.is_empty())
4917+
.all(|piece| piece.chars().count() == 1 && piece.chars().all(char::is_uppercase))
4918+
});
4919+
surname_ok && initials_ok
4920+
})
4921+
}
4922+
48844923
/// `SWE-PolyBench` and `SWE-bench`. A hyphenated token whose part is an
48854924
/// acronym or has an internal capital. `Method`, `full-history`, and a
48864925
/// contents title (`PromptTemplate`) are ordinary words.
@@ -6116,6 +6155,16 @@ fn prompt_for(
61166155

61176156
#[cfg(test)]
61186157
mod tests {
6158+
#[test]
6159+
fn an_apa_author_line_split_from_its_title_is_a_reference() {
6160+
assert!(apa_author_line(
6161+
"Shen, X., Chen, Z., Backes, M., Shen, Y., & Zhang, Y. (2024)."
6162+
));
6163+
assert!(apa_author_line("Lee, J.-H. (2021a)."));
6164+
assert!(!apa_author_line("Results, as shown, improve (2024)."));
6165+
assert!(!apa_author_line("Smith, J. found the effect in 2024."));
6166+
}
6167+
61196168
use super::*;
61206169
use crate::color::Color;
61216170
use crate::extract::{Extraction, COORDINATE_SPACE};

0 commit comments

Comments
 (0)