Fix aggressive line breaks

Author: VikParuchuri

pdftext/pdf/pages.py

@@ -157,7 +157,7 @@ def line_break():
             line_break()
             continue
 
-        if span["rotation"] != line["rotation"]:
+        if span["rotation"] != line["rotation"] and abs(span["rotation"] - line["rotation"]) >= 45:
             line_break()
             continue
 

pdftext/scripts/extract_text.py

@@ -52,7 +52,7 @@ def extract_text_cli(
             workers=kwargs["workers"],
             disable_links=True
         )
-        text = json.dumps(text)
+        text = json.dumps(text, ensure_ascii=False, indent=2)
     else:
         text = plain_text_output(
             pdf_path,

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "pdftext"
-version = "0.6.2"
+version = "0.6.3"
 description = "Extract structured text from pdfs quickly"
 authors = ["Vik Paruchuri <vik.paruchuri@gmail.com>"]
 license = "Apache-2.0"

tests/conftest.py

@@ -5,6 +5,10 @@
 def pdf_path():
     return "tests/data/adversarial.pdf"
 
+@pytest.fixture(scope="session")
+def pdf_path2():
+    return "tests/data/communication.pdf"
+
 @pytest.fixture()
 def pdf_doc(pdf_path):
     doc = pdfium.PdfDocument(pdf_path)

tests/data/communication.pdf

@@ -35,3 +35,10 @@ def test_superscripts(pdf_path):
                     if span["text"] == "∞":
                         assert span["superscript"] is True
                         return True
+
+
+def test_line_joining(pdf_path2):
+    pages = [11]
+    text = plain_text_output(pdf_path2, page_range=pages).lower()
+    assert "the axis media control viewer toolbar" in text
+    assert "axismediacontrolviewertoolbar" not in text