Speed up search

src/rard/research/migrations/0056_faceted_search.py

@@ -5,6 +5,10 @@
 from rard.utils.text_processors import make_plain_text
 
 
+def noop(_apps, _schema_editor):
+    return
+
+
 def save_objects_with_plain_text_fields(apps, schema_editor):
     db_alias = schema_editor.connection.alias
     Antiquarian = apps.get_model("research", "Antiquarian")
@@ -105,5 +109,8 @@ class Migration(migrations.Migration):
             name='plain_introduction',
             field=models.TextField(default=''),
         ),
-        migrations.RunPython(save_objects_with_plain_text_fields),
+        migrations.RunPython(
+            code=save_objects_with_plain_text_fields,
+            reverse_code=noop,
+        ),
     ]

src/rard/research/migrations/0076_add_folded_text.py

@@ -0,0 +1,47 @@
+# Generated by Django 3.2 on 2026-05-08 12:07
+
+from django.db import migrations, models
+from rard.utils.text_processors import fold_latin_and_remove_punctuation
+
+def noop(_apps, _schema_editor):
+    return
+
+
+def add_folded_text_fields(apps, schema_editor):
+    db_alias = schema_editor.connection.alias
+    OriginalText = apps.get_model("research", "OriginalText")
+    for object in OriginalText.objects.using(db_alias).all():
+        if object.plain_content:
+            object.folded_content = fold_latin_and_remove_punctuation(object.plain_content)
+            object.save()
+    HistoricalOriginalText = apps.get_model(
+        'research', 'HistoricalOriginalText'
+    )
+    for object in HistoricalOriginalText.objects.using(db_alias).all():
+        if object.plain_content:
+            object.folded_content = fold_latin_and_remove_punctuation(object.plain_content)
+            object.save()
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('research', '0075_add_testimonium_tags'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='historicaloriginaltext',
+            name='folded_content',
+            field=models.TextField(default=''),
+        ),
+        migrations.AddField(
+            model_name='originaltext',
+            name='folded_content',
+            field=models.TextField(default=''),
+        ),
+        migrations.RunPython(
+            code=add_folded_text_fields,
+            reverse_code=noop,
+        ),
+    ]

src/rard/research/models/original_text.py

@@ -7,7 +7,10 @@
 from rard.research.models.mixins import HistoryModelMixin
 from rard.research.models.reference import Reference
 from rard.utils.basemodel import BaseModel, DynamicTextField
-from rard.utils.text_processors import make_plain_text
+from rard.utils.text_processors import (
+    fold_latin_and_remove_punctuation,
+    make_plain_text,
+)
 
 
 class OriginalText(HistoryModelMixin, BaseModel):
@@ -53,6 +56,9 @@ def reference_list(self):
     # Also store copy without html or punctuation for search purposes
     plain_content = models.TextField(blank=False, default="")
 
+    # Also store a copy with all folds applied
+    folded_content = models.TextField(blank=False, default="")
+
     # to be nuked eventually. not required now but hidden from view
     # to preserve previous values in case our data migration is insufficient
     apparatus_criticus = DynamicTextField(default="", blank=True)
@@ -71,6 +77,10 @@ def save(self, *args, **kwargs):
         of list items don't get merged (and other things like that)"""
         if self.content:
             self.plain_content = make_plain_text(self.content)
+            self.folded_content = fold_latin_and_remove_punctuation(self.plain_content)
+            uf = kwargs.get("update_fields")
+            if uf is not None and "content" in uf:
+                kwargs["update_fields"] = {"plain_content", "folded_content"}.union(uf)
         super(OriginalText, self).save(*args, **kwargs)
 
     def apparatus_criticus_lines(self):

src/rard/research/tests/views/test_search.py

@@ -316,11 +316,6 @@ def do_search(search_function, keywords):
         self.assertEqual(do_search(view.fragment_search, "notme"), [f2])
         self.assertEqual(do_search(view.fragment_search, "No!TMe"), [f2])
         self.assertEqual(do_search(view.fragment_search, "*Me*"), [f1, f2])
-        self.assertEqual(do_search(view.fragment_search, "may"), [f1, f2])
-        self.assertEqual(
-            do_search(view.fragment_search, "m!£$%^&()_+-=|\\{[}];@'#<,>./ay"),
-            [f1, f2],
-        )
         self.assertEqual(do_search(view.fragment_search, "mav"), [])
         self.assertEqual(do_search(view.fragment_search, 'alcott "louisa may"'), [f1])
         self.assertEqual(do_search(view.fragment_search, 'may "louisa alcott"'), [])