Fix optional dependency loading and tokenizer handling

README.md

@@ -71,6 +71,8 @@ source venv_agent_lab/bin/activate
 ```bash
 pip install -r requirements.txt
 ```
+- This install includes the dependencies used by the optional AgentRxiv web workflow.
+- TensorFlow is not required for the core Agent Laboratory workflows and is not installed by default.
 
 4. **Install pdflatex [OPTIONAL]**
 ```bash

ai_lab_repo.py

@@ -1,13 +1,13 @@
-import PyPDF2
+import importlib
 import threading
-from app import *
 from agents import *
 from copy import copy
 from pathlib import Path
 from datetime import date
 from common_imports import *
 from mlesolver import MLESolver
 import argparse, pickle, yaml
+from pypdf import PdfReader
 
 GLOBAL_AGENTRXIV = None
 DEFAULT_LLM_BACKBONE = "o3-mini"
@@ -16,6 +16,31 @@
 os.environ["TOKENIZERS_PARALLELISM"] = "false"
 
 
+def load_agentrxiv_app():
+    optional_dependencies = {
+        "flask": "flask",
+        "flask_sqlalchemy": "flask-sqlalchemy",
+        "sentence_transformers": "sentence-transformers",
+    }
+    missing_packages = []
+    for module_name, package_name in optional_dependencies.items():
+        try:
+            importlib.import_module(module_name)
+        except ModuleNotFoundError:
+            missing_packages.append(package_name)
+
+    if missing_packages:
+        missing_list = ", ".join(sorted(missing_packages))
+        raise RuntimeError(
+            "AgentRxiv requires additional web dependencies that are not installed: "
+            f"{missing_list}. Install them with `pip install -r requirements.txt`."
+        )
+
+    from app import app, run_app, update_papers_from_uploads
+
+    return app, run_app, update_papers_from_uploads
+
+
 class LaboratoryWorkflow:
     def __init__(self, research_topic, openai_api_key, max_steps=100, num_papers_lit_review=5, agent_model_backbone=f"{DEFAULT_LLM_BACKBONE}", notes=list(), human_in_loop_flag=None, compile_pdf=True, mlesolver_max_steps=3, papersolver_max_steps=5, paper_index=0, except_if_fail=False, parallelized=False, lab_dir=None, lab_index=0, agentRxiv=False, agentrxiv_papers=5):
         """
@@ -601,20 +626,23 @@ def retrieve_full_text(self, arxiv_id):
             return "Paper ID not found?"
 
     @staticmethod
-    def read_pdf_pypdf2(pdf_path):
+    def read_pdf_text(pdf_path):
         with open(pdf_path, 'rb') as pdf_file:
-            reader = PyPDF2.PdfReader(pdf_file)
+            reader = PdfReader(pdf_file)
             text = ''
             for page_num in range(len(reader.pages)):
                 page = reader.pages[page_num]
-                text += page.extract_text()
+                page_text = page.extract_text()
+                if page_text:
+                    text += page_text
         return text
 
     def search_agentrxiv(self, search_query, num_papers):
         # Use the dynamic port here as well
         url = f'http://127.0.0.1:{5000 + self.lab_index}/api/search?q={search_query}'
         return_str = str()
         try:
+            app, _, update_papers_from_uploads = load_agentrxiv_app()
             with app.app_context():
                 update_papers_from_uploads()
             response = requests.get(url)
@@ -628,7 +656,7 @@ def search_agentrxiv(self, search_query, num_papers):
                     filename = Path(f'_tmp_{self.lab_index}.pdf')
                     response = requests.get(result['pdf_url'])
                     filename.write_bytes(response.content)
-                    self.pdf_text[arxiv_id] = self.read_pdf_pypdf2(f'_tmp_{self.lab_index}.pdf')
+                    self.pdf_text[arxiv_id] = self.read_pdf_text(f'_tmp_{self.lab_index}.pdf')
                     self.summaries[arxiv_id] = query_model(
                         prompt=self.pdf_text[arxiv_id],
                         system_prompt="Please provide a 5 sentence summary of this paper.",
@@ -647,6 +675,7 @@ def search_agentrxiv(self, search_query, num_papers):
         return return_str
 
     def run_server(self, port):
+        _, run_app, _ = load_agentrxiv_app()
         run_app(port=port)
 
 
@@ -886,6 +915,3 @@ def run_lab(parallel_lab_index):
 
 
 
-
-
-

app.py

@@ -1,11 +1,11 @@
 import random, time
+from functools import lru_cache
 
 from flask import Flask, render_template, request, redirect, url_for, flash, send_from_directory, jsonify
 from werkzeug.utils import secure_filename
 import os
-from PyPDF2 import PdfReader
+from pypdf import PdfReader
 from flask_sqlalchemy import SQLAlchemy
-from sentence_transformers import SentenceTransformer
 from sklearn.metrics.pairwise import cosine_similarity
 import numpy as np
 
@@ -58,8 +58,11 @@ def update_papers_from_uploads():
     return
     #raise Exception("FAILED TO UPDATE")
 
-# Load a pre-trained sentence transformer model
-model = SentenceTransformer('all-MiniLM-L6-v2')
+@lru_cache(maxsize=1)
+def get_embedding_model():
+    from sentence_transformers import SentenceTransformer
+
+    return SentenceTransformer('all-MiniLM-L6-v2')
 
 @app.route('/update', methods=['GET'])
 def update_on_demand():
@@ -106,6 +109,7 @@ def upload():
 def search():
     query = request.args.get('q', '')
     if query:
+        model = get_embedding_model()
         papers = Paper.query.all()
         query_embedding = model.encode([query])
         paper_texts = [paper.text for paper in papers if paper.text]
@@ -123,6 +127,7 @@ def api_search():
     query = request.args.get('q', '')
     if not query:
         return jsonify({'error': 'No query provided'}), 400
+    model = get_embedding_model()
     papers = Paper.query.all()
     if not papers:
         return jsonify({'query': query, 'results': []})
@@ -168,4 +173,4 @@ def run_app(port=5000):
     app.run(debug=False, port=port)
 
 if __name__ == '__main__':
-    run_app()
+    run_app()

common_imports.py

@@ -34,13 +34,31 @@
 # Hugging Face & Transformers
 import transformers
 
+
+class MissingOptionalDependency:
+    def __init__(self, package_name, feature_description):
+        self.package_name = package_name
+        self.feature_description = feature_description
+
+    def __bool__(self):
+        return False
+
+    def __getattr__(self, name):
+        raise ImportError(
+            f"Optional dependency '{self.package_name}' is required for {self.feature_description}. "
+            f"Install it separately before using '{name}'."
+        )
+
 # Deep learning frameworks
 import torch
 import torch.nn as nn
 import torch.optim as optim
 import torch.nn.functional as F
 from torch.utils.data import DataLoader, Dataset, random_split
-import tensorflow as tf
+try:
+    import tensorflow as tf
+except ImportError:
+    tf = MissingOptionalDependency("tensorflow", "TensorFlow-backed experiments")
 #import keras
 
 # NLP Libraries