From 5ca98ef0a989f44c84c0bc47fac833355f762112 Mon Sep 17 00:00:00 2001 From: eshulman2 Date: Mon, 24 Nov 2025 11:30:56 +0200 Subject: [PATCH] re-organize project --- .pylintrc | 4 +- Makefile | 18 +-- README.md | 148 ++++++++++-------- containers/Dockerfile.google-mcp | 2 +- containers/Dockerfile.jira-mcp | 2 +- containers/Dockerfile.registry-service | 2 +- docs/ERROR_HANDLING.md | 42 ++--- docs/PROGRESSIVE_SUMMARIZATION.md | 8 +- .../error_handling_example.py | 4 +- scripts/generate_token.py | 2 +- src/clients/meeting_actions_client.py | 2 +- src/core/agents/google_agent.py | 10 +- src/core/agents/jira_agent.py | 6 +- src/core/{base => }/error_handler.py | 11 +- .../workflow_servers/action_items_server.py | 2 +- .../action_items_dispatch_orchestrator.py | 6 +- ...eting_notes_and_generation_orchestrator.py | 9 +- .../action_items_generation_workflow.py | 12 +- .../sub_workflows/agent_dispatch_workflow.py | 12 +- .../sub_workflows/meeting_notes_workflow.py | 8 +- src/infrastructure/cache/redis_cache.py | 6 +- src/infrastructure/config/read_config.py | 2 +- src/integrations/common/__init__.py | 7 + .../{general_tools => common}/date_tools.py | 0 src/integrations/general_tools/__init__.py | 2 +- src/integrations/google/__init__.py | 7 + .../auth_utils.py => google/auth.py} | 0 .../google_tools.py => google/tools.py} | 2 +- src/integrations/google_tools/__init__.py | 2 +- src/integrations/google_tools/gmail_tools.py | 2 +- src/integrations/jira/__init__.py | 7 + .../jira_formatter.py => jira/formatter.py} | 0 .../jira_tools.py => jira/tools.py} | 32 ++-- src/integrations/jira_tools/__init__.py | 2 +- src/services/mcp/__init__.py | 7 + src/{ => services}/mcp/google_tools_mcp.py | 3 +- src/{ => services}/mcp/jira_tools_mcp.py | 2 +- src/services/registry/__init__.py | 16 ++ .../registry/agent_registry.py | 2 +- .../registry/registry_client.py | 4 +- .../{ => registry}/registry_service.py | 2 +- src/shared/__init__.py | 0 src/shared/agents/__init__.py | 0 src/{core => shared}/agents/utils.py | 0 src/shared/base/__init__.py | 0 .../base/base_agent_server.py | 12 +- src/{core => shared}/base/base_server.py | 0 .../base/base_workflow_server.py | 2 +- src/shared/common/__init__.py | 0 src/{ => shared}/common/singleton_meta.py | 0 src/shared/llm/__init__.py | 0 src/shared/llm/summarization/__init__.py | 0 .../llm/summarization/progressive.py} | 2 +- .../utils => shared/llm}/token_utils.py | 0 src/shared/resilience/__init__.py | 0 .../resilience}/circuit_breaker.py | 2 +- .../base => shared/resilience}/exceptions.py | 0 src/{core/base => shared/resilience}/retry.py | 2 +- tests/conftest.py | 2 +- tests/integration/test_api_endpoints.py | 18 +-- tests/performance/test_load_testing.py | 8 +- tests/unit/agents/test_base_agent_server.py | 22 +-- tests/unit/agents/test_base_server.py | 10 +- tests/unit/config/test_config_reader.py | 8 +- tests/unit/core/test_agent_utils.py | 20 +-- tests/unit/integrations/test_auth_utils.py | 8 +- tests/unit/integrations/test_google_tools.py | 10 +- tests/unit/test_singleton_thread_safety.py | 2 +- .../utils/test_progressive_summarization.py | 120 ++++++-------- 69 files changed, 333 insertions(+), 332 deletions(-) rename {src/core/base => examples}/error_handling_example.py (98%) rename src/core/{base => }/error_handler.py (96%) create mode 100644 src/integrations/common/__init__.py rename src/integrations/{general_tools => common}/date_tools.py (100%) create mode 100644 src/integrations/google/__init__.py rename src/integrations/{google_tools/auth_utils.py => google/auth.py} (100%) rename src/integrations/{google_tools/google_tools.py => google/tools.py} (99%) create mode 100644 src/integrations/jira/__init__.py rename src/integrations/{jira_tools/jira_formatter.py => jira/formatter.py} (100%) rename src/integrations/{jira_tools/jira_tools.py => jira/tools.py} (87%) create mode 100644 src/services/mcp/__init__.py rename src/{ => services}/mcp/google_tools_mcp.py (93%) rename src/{ => services}/mcp/jira_tools_mcp.py (97%) create mode 100644 src/services/registry/__init__.py rename src/{infrastructure => services}/registry/agent_registry.py (99%) rename src/{infrastructure => services}/registry/registry_client.py (98%) rename src/services/{ => registry}/registry_service.py (99%) create mode 100644 src/shared/__init__.py create mode 100644 src/shared/agents/__init__.py rename src/{core => shared}/agents/utils.py (100%) create mode 100644 src/shared/base/__init__.py rename src/{core => shared}/base/base_agent_server.py (97%) rename src/{core => shared}/base/base_server.py (100%) rename src/{core => shared}/base/base_workflow_server.py (97%) create mode 100644 src/shared/common/__init__.py rename src/{ => shared}/common/singleton_meta.py (100%) create mode 100644 src/shared/llm/__init__.py create mode 100644 src/shared/llm/summarization/__init__.py rename src/{infrastructure/utils/progressive_summarization.py => shared/llm/summarization/progressive.py} (99%) rename src/{infrastructure/utils => shared/llm}/token_utils.py (100%) create mode 100644 src/shared/resilience/__init__.py rename src/{core/base => shared/resilience}/circuit_breaker.py (99%) rename src/{core/base => shared/resilience}/exceptions.py (100%) rename src/{core/base => shared/resilience}/retry.py (99%) diff --git a/.pylintrc b/.pylintrc index 6e732f0..f1049d4 100644 --- a/.pylintrc +++ b/.pylintrc @@ -25,7 +25,7 @@ clear-cache-post-run=no # A comma-separated list of package or module names from where C extensions may # be loaded. Extensions are loading into the active Python interpreter and may # run arbitrary code. -extension-pkg-allow-list= +extension-pkg-allow-list=google,jira # A comma-separated list of package or module names from where C extensions may # be loaded. Extensions are loading into the active Python interpreter and may @@ -64,7 +64,7 @@ ignore-patterns=^\.# # manipulated during runtime and thus existing member attributes cannot be # deduced by static analysis). It supports qualified module names, as well as # Unix pattern matching. -ignored-modules= +ignored-modules=google,google.auth,google.auth.transport,google.oauth2,jira,jira.resources # Python code to execute, usually for sys.path manipulation such as # pygtk.require(). diff --git a/Makefile b/Makefile index 824133c..4fbba74 100644 --- a/Makefile +++ b/Makefile @@ -40,11 +40,11 @@ help: # Start MCP servers start-google-mcp: @echo "Starting Google MCP server..." - UVICORN_PORT=8100 python -m src.mcp.google_tools_mcp + UVICORN_PORT=8100 python -m src.services.mcp.google_tools_mcp start-jira-mcp: @echo "Starting JIRA MCP server..." - UVICORN_PORT=8101 python -m src.mcp.jira_tools_mcp + UVICORN_PORT=8101 python -m src.services.mcp.jira_tools_mcp # Start agent servers start-google-agent: @@ -63,17 +63,17 @@ start-workflow: # Start registry service start-registry: @echo "Starting agent registry service..." - UVICORN_PORT=8003 python -m src.services.registry_service + UVICORN_PORT=8003 python -m src.services.registry.registry_service # Start all servers start-all: @echo "Starting all servers..." @echo "Starting agent registry service..." - python -m src.services.registry_service & + python -m src.services.registry.registry_service & @echo "Starting Google MCP server..." - python -m src.mcp.google_tools_mcp & + python -m src.services.mcp.google_tools_mcp & @echo "Starting JIRA MCP server..." - python -m src.mcp.jira_tools_mcp & + python -m src.services.mcp.jira_tools_mcp & @echo "Starting Jira agent..." UVICORN_PORT=8000 python -m src.core.agents.jira_agent & @echo "Starting Google agent..." @@ -85,12 +85,12 @@ start-all: # Stop all servers stop-all: @echo "Stopping all servers..." - pkill -f "python -m src.services.registry_service" || true + pkill -f "python -m src.services.registry.registry_service" || true pkill -f "python -m src.core.agents.jira_agent" || true pkill -f "python -m src.core.agents.google_agent" || true pkill -f "python -m src.core.workflow_servers.action_items_server" || true - pkill -f "python -m src.mcp.google_tools_mcp" || true - pkill -f "python -m src.mcp.jira_tools_mcp" || true + pkill -f "python -m src.services.mcp.google_tools_mcp" || true + pkill -f "python -m src.services.mcp.jira_tools_mcp" || true @echo "All servers stopped!" # Installation and Setup diff --git a/README.md b/README.md index b0e1c1c..219bd0d 100644 --- a/README.md +++ b/README.md @@ -71,16 +71,20 @@ MeetingActions follows a microservices architecture with specialized agents and - Agent discovery and dynamic dispatch via service registry - Human review and approval workflow support -### 4. **Agent Registry Service** (`src/services/registry_service.py`) +### 4. **Agent Registry Service** (`src/services/registry/`) - **Purpose**: Service discovery and health monitoring - **Port**: 8003 +- **Components**: + - `registry_service.py`: Standalone registry server + - `registry_client.py`: Client library for service discovery + - `agent_registry.py`: Registry data models and in-memory store - **Features**: - Agent registration and discovery - Health check monitoring - Load balancing support - Service metadata management -### 5. **Google MCP Server** (`src/mcp/google_tools_mcp.py`) +### 5. **Google MCP Server** (`src/services/mcp/google_tools_mcp.py`) - **Purpose**: Model Context Protocol server for Google tools - **Port**: 8100 - **Features**: @@ -88,7 +92,7 @@ MeetingActions follows a microservices architecture with specialized agents and - Standardized tool interface - Authentication management -### 6. **JIRA MCP Server** (`src/mcp/jira_tools_mcp.py`) +### 6. **JIRA MCP Server** (`src/services/mcp/jira_tools_mcp.py`) - **Purpose**: Model Context Protocol server for JIRA tools - **Port**: 8101 - **Features**: @@ -229,81 +233,91 @@ python -m src.clients.meeting_actions_client --url http://localhost:8002 src/ ├── clients/ # Client applications │ └── meeting_actions_client.py # Interactive CLI client -├── common/ # Shared utilities and patterns -│ └── singleton_meta.py # Singleton metaclass implementation -├── core/ # Business logic and agents -│ ├── agents/ # Individual agent implementations -│ │ ├── jira_agent.py # Jira integration agent -│ │ ├── google_agent.py # Google Workspace agent -│ │ └── utils.py # Agent utility functions -│ ├── base/ # Base classes and utilities -│ │ ├── base_server.py # Base server for all services -│ │ ├── base_agent_server.py # Base for agent servers -│ │ └── base_workflow_server.py # Base for workflow servers -│ ├── schemas/ # Pydantic models and schemas -│ │ ├── __init__.py # Centralized schema exports +├── core/ # Business logic and domain +│ ├── agents/ # Individual agent implementations +│ │ ├── jira_agent.py # Jira integration agent +│ │ └── google_agent.py # Google Workspace agent +│ ├── error_handler.py # Centralized error handling and resilience +│ ├── schemas/ # Pydantic models and schemas +│ │ ├── __init__.py # Centralized schema exports │ │ ├── agent_response.py # Unified AgentResponse model │ │ └── workflow_models.py # Workflow data models -│ ├── workflows/ # Event-driven workflow definitions +│ ├── workflows/ # Event-driven workflow definitions │ │ ├── meeting_notes_and_generation_orchestrator.py │ │ ├── action_items_dispatch_orchestrator.py │ │ └── sub_workflows/ # Focused sub-workflow components │ │ ├── meeting_notes_workflow.py │ │ ├── action_items_generation_workflow.py │ │ └── agent_dispatch_workflow.py -│ └── workflow_servers/ # Workflow execution servers +│ └── workflow_servers/ # Workflow execution servers │ └── action_items_server.py # Action items server -├── infrastructure/ # Platform infrastructure -│ ├── cache/ # Redis caching with singleton pattern +├── shared/ # Shared components and utilities +│ ├── agents/ # Agent utilities +│ │ └── utils.py # Agent utility functions +│ ├── base/ # Base classes for servers +│ │ ├── base_server.py # Base server for all services +│ │ ├── base_agent_server.py # Base for agent servers +│ │ └── base_workflow_server.py # Base for workflow servers +│ ├── common/ # Common patterns and utilities +│ │ └── singleton_meta.py # Singleton metaclass implementation +│ ├── llm/ # LLM-related utilities +│ │ ├── summarization/ # Summarization tools +│ │ │ └── progressive.py # Multi-pass summarization engine +│ │ └── token_utils.py # Token counting and management +│ └── resilience/ # Resilience patterns +│ ├── circuit_breaker.py # Circuit breaker implementation +│ ├── exceptions.py # Custom exception hierarchy +│ └── retry.py # Retry logic and decorators +├── infrastructure/ # Platform infrastructure +│ ├── cache/ # Redis caching with singleton pattern │ │ ├── redis_cache.py # Redis cache implementation │ │ └── document_cache.py # Document-specific caching -│ ├── config/ # Configuration management -│ │ ├── read_config.py # Configuration reader (with progressive_summarization config) -│ │ └── models.py # Configuration models -│ ├── prompts/ # System prompts (organized by category) -│ │ ├── prompts.py # Prompt loader and management -│ │ ├── action_items/ # Action items generation prompts +│ ├── config/ # Configuration management +│ │ ├── read_config.py # Configuration reader +│ │ └── models.py # Configuration models +│ ├── prompts/ # System prompts (organized by category) +│ │ ├── prompts.py # Prompt loader and management +│ │ ├── action_items/ # Action items generation prompts │ │ │ ├── generation.txt │ │ │ ├── refinement.txt │ │ │ └── review.txt -│ │ ├── agents/ # Agent-specific prompts +│ │ ├── agents/ # Agent-specific prompts │ │ │ ├── agent_query.txt │ │ │ ├── google_context.txt │ │ │ ├── jira_context.txt │ │ │ └── tool_dispatcher_prompt.txt -│ │ ├── summarization/ # Progressive summarization prompts +│ │ ├── summarization/ # Progressive summarization prompts │ │ │ ├── basic.txt │ │ │ ├── progressive_pass1.txt │ │ │ ├── progressive_pass2.txt │ │ │ └── progressive_pass3.txt -│ │ ├── meeting_notes/ # Meeting notes processing +│ │ ├── meeting_notes/ # Meeting notes processing │ │ │ └── identify_file.txt -│ │ └── legacy/ # Deprecated prompts (backward compatibility) +│ │ └── legacy/ # Deprecated prompts (backward compatibility) │ │ └── ... -│ ├── utils/ # Utility functions -│ │ ├── progressive_summarization.py # Multi-pass summarization engine -│ │ └── token_utils.py # Token counting and management -│ ├── logging/ # Structured logging +│ ├── logging/ # Structured logging │ │ └── logging_config.py # Logging configuration -│ ├── observability/ # Langfuse integration -│ │ └── observability.py # Monitoring and tracing -│ └── registry/ # Service registry -│ ├── agent_registry.py # Agent registration -│ └── registry_client.py # Registry client -├── integrations/ # External service integrations -│ ├── google_tools/ # Google Workspace APIs -│ │ ├── google_tools.py # Google API tools -│ │ └── auth_utils.py # Authentication utilities -│ ├── jira_tools/ # Jira API integration -│ │ ├── jira_tools.py # Jira API tools -│ │ └── jira_formatter.py # Jira data formatting -│ └── general_tools/ # Utility tools -│ └── date_tools.py # Date/time utilities -├── mcp/ # Model Context Protocol servers -│ ├── google_tools_mcp.py # Google tools MCP server -│ └── jira_tools_mcp.py # JIRA tools MCP server -└── services/ # Standalone services - └── registry_service.py # Agent registry service +│ └── observability/ # Langfuse integration +│ └── observability.py # Monitoring and tracing +├── integrations/ # External service integrations +│ ├── common/ # Common integration utilities +│ │ └── date_tools.py # Date/time utilities +│ ├── google/ # Google Workspace APIs +│ │ ├── auth.py # Authentication utilities +│ │ ├── tools.py # Google API tools +│ │ └── gmail_tools.py # Gmail-specific tools +│ ├── jira/ # Jira API integration +│ │ ├── formatter.py # Jira data formatting +│ │ └── tools.py # Jira API tools +│ └── general_tools/ # Legacy general tools +└── services/ # Standalone services + ├── mcp/ # Model Context Protocol servers + │ ├── google_tools_mcp.py # Google tools MCP server (Port 8100) + │ └── jira_tools_mcp.py # JIRA tools MCP server (Port 8101) + └── registry/ # Agent registry service + ├── registry_service.py # Registry server (Port 8003) + ├── registry_client.py # Registry client + └── agent_registry.py # Registry data models tests/ # Test suite ├── unit/ # Unit tests │ ├── utils/ @@ -330,20 +344,29 @@ docs/ # Documentation - **Factory Pattern**: Dynamic model and configuration creation - **Observer Pattern**: Event-driven workflow orchestration - **Repository Pattern**: Clean data access abstraction -- **Separation of Concerns**: Clear boundaries between agents, workflows, and infrastructure +- **Separation of Concerns**: Clear boundaries between core domain, shared utilities, and infrastructure +- **Resilience Patterns**: Circuit breakers, retry logic, and error handling in `src/shared/resilience/` ### Key Architectural Improvements 1. **Unified AgentResponse Schema**: Single, consistent response model for all agents and workflows -2. **Separated Base Classes**: `BaseAgentServer` for agents, `BaseWorkflowServer` for workflows +2. **Separated Base Classes**: `BaseAgentServer` for agents, `BaseWorkflowServer` for workflows in `src/shared/base/` 3. **Centralized Schemas**: All Pydantic models organized in `src/core/schemas/` 4. **Enhanced Execution Results**: Full action item tracking in dispatch results 5. **Modular Workflows**: Composable orchestrators for better maintainability -6. **Progressive Summarization**: Multi-pass reduction system with semantic chunking +6. **Shared Utilities Layer**: Common components in `src/shared/` for cross-cutting concerns + - Agent utilities in `src/shared/agents/` + - LLM tools in `src/shared/llm/` + - Resilience patterns in `src/shared/resilience/` (circuit breakers, retry, exceptions) +7. **Progressive Summarization**: Multi-pass reduction system with semantic chunking - New workflow step separation: `prepare_meeting_notes` → `generate_action_items` - Event-based communication via `NotesReadyEvent` - 26 comprehensive unit tests with full coverage -7. **Organized Prompts Structure**: Category-based prompt organization +8. **Organized Integrations**: Clean separation by service provider + - `src/integrations/google/` for Google Workspace + - `src/integrations/jira/` for Jira + - `src/integrations/common/` for shared utilities +9. **Organized Prompts Structure**: Category-based prompt organization - `action_items/`, `agents/`, `summarization/`, `meeting_notes/`, `legacy/` - Easier prompt management and maintenance - Clear separation by feature domain @@ -705,8 +728,9 @@ open http://localhost:8002/docs ### Model Context Protocol (MCP) **Extensible Tool Framework:** -- **Google Tools MCP**: Calendar, Docs, Drive, Gmail integration -- **JIRA Tools MCP**: Issue management, project coordination, ticket operations +Located in `src/services/mcp/` - standalone MCP server implementations: +- **Google Tools MCP** (Port 8100): Calendar, Docs, Drive, Gmail integration +- **JIRA Tools MCP** (Port 8101): Issue management, project coordination, ticket operations - **Custom Protocols**: Build domain-specific tool integrations - **Agent Enhancement**: Extend capabilities through external tools @@ -762,7 +786,7 @@ print(cache.get_cache_stats()) 1. **Create Agent Class**: Extend `BaseAgentServer` ```python - from src.core.base.base_agent_server import BaseAgentServer + from src.shared.base.base_agent_server import BaseAgentServer from src.core.schemas.agent_response import AgentResponse class MyAgent(BaseAgentServer): @@ -784,7 +808,7 @@ print(cache.get_cache_stats()) 1. **Define Workflow Class**: Extend `BaseWorkflowServer` ```python - from src.core.base.base_workflow_server import BaseWorkflowServer + from src.shared.base.base_workflow_server import BaseWorkflowServer class MyWorkflowServer(BaseWorkflowServer): def additional_routes(self): @@ -819,7 +843,7 @@ from src.core.schemas import ( Use the progressive summarization engine for handling long documents: ```python -from src.infrastructure.utils.progressive_summarization import ( +from src.shared.llm.summarization.progressive import ( progressive_summarize, SummarizationStrategy, ProgressiveSummaryResult, diff --git a/containers/Dockerfile.google-mcp b/containers/Dockerfile.google-mcp index 85b8326..898c3a9 100644 --- a/containers/Dockerfile.google-mcp +++ b/containers/Dockerfile.google-mcp @@ -30,4 +30,4 @@ EXPOSE 8100 ENV PYTHONPATH=/app # Default command for Google MCP server -CMD ["python", "-m", "src.mcp.google_tools_mcp"] +CMD ["python", "-m", "src.services.mcp.google_tools_mcp"] diff --git a/containers/Dockerfile.jira-mcp b/containers/Dockerfile.jira-mcp index d009ad7..42210e6 100644 --- a/containers/Dockerfile.jira-mcp +++ b/containers/Dockerfile.jira-mcp @@ -30,4 +30,4 @@ EXPOSE 8101 ENV PYTHONPATH=/app # Default command for JIRA MCP server -CMD ["python", "-m", "src.mcp.jira_tools_mcp"] +CMD ["python", "-m", "src.services.mcp.jira_tools_mcp"] diff --git a/containers/Dockerfile.registry-service b/containers/Dockerfile.registry-service index 75eb845..fa661e8 100644 --- a/containers/Dockerfile.registry-service +++ b/containers/Dockerfile.registry-service @@ -30,4 +30,4 @@ EXPOSE 8003 ENV PYTHONPATH=/app # Default command for registry service -CMD ["uvicorn", "src.services.registry_service:app", "--host", "0.0.0.0"] +CMD ["uvicorn", "src.services.registry.registry_service:app", "--host", "0.0.0.0"] diff --git a/docs/ERROR_HANDLING.md b/docs/ERROR_HANDLING.md index e5aad68..2b01686 100644 --- a/docs/ERROR_HANDLING.md +++ b/docs/ERROR_HANDLING.md @@ -8,12 +8,12 @@ This guide explains the error handling and resilience patterns implemented in Me ### 1. Custom Exception Hierarchy -Located in: `src/core/base/exceptions.py` +Located in: `src/shared/resilience/exceptions.py` All custom exceptions inherit from `MeetingActionsError` and provide structured error information. ```python -from src.core.base.exceptions import ( +from src.shared.resilience.exceptions import ( AgentError, AgentTimeoutError, WorkflowError, @@ -58,14 +58,14 @@ MeetingActionsError (base) ### 2. Retry Decorator -Located in: `src/core/base/retry.py` +Located in: `src/shared/resilience/retry.py` Automatically retries failed operations with configurable backoff strategies. #### Basic Usage ```python -from src.core.base.error_handler import with_retry, BackoffStrategy +from src.core.error_handler import with_retry, BackoffStrategy @with_retry( max_attempts=3, @@ -112,7 +112,7 @@ async def fetch_data(url: str): ```python import httpx -from src.core.base.error_handler import with_retry +from src.core.error_handler import with_retry @with_retry( max_attempts=5, @@ -143,7 +143,7 @@ async def operation(): ### 3. Circuit Breaker Pattern -Located in: `src/core/base/circuit_breaker.py` +Located in: `src/shared/resilience/circuit_breaker.py` Protects your system from cascading failures by "opening" after too many failures. @@ -174,7 +174,7 @@ Protects your system from cascading failures by "opening" after too many failure #### Basic Usage ```python -from src.core.base.error_handler import with_circuit_breaker +from src.core.error_handler import with_circuit_breaker @with_circuit_breaker( name="google_api", @@ -208,7 +208,7 @@ async def call_google_api(endpoint: str): #### Manual Circuit Control ```python -from src.core.base.error_handler import get_circuit_breaker +from src.core.error_handler import get_circuit_breaker # Get circuit breaker circuit = get_circuit_breaker("google_api") @@ -235,7 +235,7 @@ circuit.reset() #### Monitoring All Circuits ```python -from src.core.base.circuit_breaker import get_all_circuit_breakers +from src.shared.resilience.circuit_breaker import get_all_circuit_breakers circuits = get_all_circuit_breakers() @@ -246,14 +246,14 @@ for name, breaker in circuits.items(): ### 4. Error Context Manager -Located in: `src/core/base/error_handler.py` +Located in: `src/core/error_handler.py` Automatically enriches errors with contextual information. #### Usage ```python -from src.core.base.error_handler import ErrorContext +from src.core.error_handler import ErrorContext async def process_meeting(meeting_id: str, user_id: str): """Process meeting with error context.""" @@ -293,7 +293,7 @@ When an error occurs: #### Retry + Circuit Breaker ```python -from src.core.base.error_handler import ( +from src.core.error_handler import ( with_retry, with_circuit_breaker, BackoffStrategy @@ -446,7 +446,7 @@ async def complex_workflow(meeting_id: str): ```python # Add health check endpoint from fastapi import FastAPI -from src.core.base.circuit_breaker import get_all_circuit_breakers +from src.shared.resilience.circuit_breaker import get_all_circuit_breakers app = FastAPI() @@ -477,8 +477,8 @@ async def circuit_breaker_health(): ```python from fastapi import FastAPI, Request from fastapi.responses import JSONResponse -from src.core.base.error_handler import handle_error_response -from src.core.base.exceptions import MeetingActionsError +from src.core.error_handler import handle_error_response +from src.shared.resilience.exceptions import MeetingActionsError app = FastAPI() @@ -499,7 +499,7 @@ async def meeting_actions_error_handler( ```python from fastapi import HTTPException -from src.core.base.error_handler import ( +from src.core.error_handler import ( with_retry, with_circuit_breaker, ErrorContext @@ -529,8 +529,8 @@ async def _generate_with_resilience(meeting_id: str): ```python import pytest -from src.core.base.error_handler import with_retry -from src.core.base.exceptions import MaxRetriesExceededError +from src.core.error_handler import with_retry +from src.shared.resilience.exceptions import MaxRetriesExceededError @pytest.mark.asyncio async def test_retry_eventually_succeeds(): @@ -570,8 +570,8 @@ async def test_retry_max_attempts_exceeded(): ```python import pytest -from src.core.base.error_handler import with_circuit_breaker -from src.core.base.exceptions import CircuitOpenError +from src.core.error_handler import with_circuit_breaker +from src.shared.resilience.exceptions import CircuitOpenError @pytest.mark.asyncio async def test_circuit_opens_after_failures(): @@ -614,7 +614,7 @@ async def dispatch_to_agent(item: dict, agent_url: str): ### After (With Error Handling) ```python -from src.core.base.error_handler import ( +from src.core.error_handler import ( with_retry, with_circuit_breaker, ErrorContext, diff --git a/docs/PROGRESSIVE_SUMMARIZATION.md b/docs/PROGRESSIVE_SUMMARIZATION.md index be006dd..2954795 100644 --- a/docs/PROGRESSIVE_SUMMARIZATION.md +++ b/docs/PROGRESSIVE_SUMMARIZATION.md @@ -281,7 +281,7 @@ StartEvent → prepare_meeting_notes → NotesReadyEvent → generate_action_ite ### Utility Functions -Located in `src/infrastructure/utils/progressive_summarization.py`: +Located in `src/shared/llm/summarization/progressive.py`: #### Core Functions @@ -309,7 +309,7 @@ Located in `src/infrastructure/utils/progressive_summarization.py`: **`summarize_chunk()`** - Process individual chunks **`summarize_meeting_notes()`** - Simple single-pass summarization -**`truncate_text_by_tokens()`** - Fallback text truncation (from `token_utils.py`) +**`truncate_text_by_tokens()`** - Fallback text truncation (from `src/shared/llm/token_utils`) --- @@ -497,7 +497,7 @@ pytest tests/unit/utils/test_progressive_summarization.py -v pytest tests/unit/utils/test_progressive_summarization.py::TestProgressiveSummarize -v # With coverage -pytest tests/unit/utils/test_progressive_summarization.py --cov=src/infrastructure/utils/progressive_summarization +pytest tests/unit/utils/test_progressive_summarization.py --cov=src/shared/llm/summarization/progressive ``` --- @@ -556,7 +556,7 @@ config = get_config() print(config.config.progressive_summarization) # Check token count -from src.infrastructure.utils.token_utils import count_tokens +from src.shared.llm.token_utils import count_tokens tokens = count_tokens(text, llm) print(f"Tokens: {tokens}, Threshold: {threshold}") ``` diff --git a/src/core/base/error_handling_example.py b/examples/error_handling_example.py similarity index 98% rename from src/core/base/error_handling_example.py rename to examples/error_handling_example.py index d26914b..1e4d4dc 100644 --- a/src/core/base/error_handling_example.py +++ b/examples/error_handling_example.py @@ -11,8 +11,7 @@ import httpx -from src.core.base.circuit_breaker import get_all_circuit_breakers -from src.core.base.error_handler import ( +from src.core.error_handler import ( AgentResponseError, AgentTimeoutError, AgentUnavailableError, @@ -22,6 +21,7 @@ with_retry, ) from src.infrastructure.logging.logging_config import get_logger +from src.shared.resilience.circuit_breaker import get_all_circuit_breakers logger = get_logger("error_handling_example") diff --git a/scripts/generate_token.py b/scripts/generate_token.py index 04318b3..548c7fb 100755 --- a/scripts/generate_token.py +++ b/scripts/generate_token.py @@ -27,7 +27,7 @@ from google.oauth2.credentials import Credentials from google_auth_oauthlib.flow import InstalledAppFlow -from src.integrations.google_tools.auth_utils import SCOPES +from src.integrations.google.auth import SCOPES # pylint: enable=wrong-import-position diff --git a/src/clients/meeting_actions_client.py b/src/clients/meeting_actions_client.py index 6bfc171..f704893 100644 --- a/src/clients/meeting_actions_client.py +++ b/src/clients/meeting_actions_client.py @@ -15,7 +15,7 @@ from rich.console import Console from rich.table import Table -from src.core.base.retry import BackoffStrategy, with_retry +from src.shared.resilience.retry import BackoffStrategy, with_retry console = Console() diff --git a/src/core/agents/google_agent.py b/src/core/agents/google_agent.py index 21beea6..28d32cf 100644 --- a/src/core/agents/google_agent.py +++ b/src/core/agents/google_agent.py @@ -7,16 +7,14 @@ import uvicorn from llama_index.core.agent.workflow import ReActAgent -from src.core.agents.utils import safe_load_mcp_tools -from src.core.base.base_agent_server import BaseAgentServer from src.core.schemas.agent_response import AgentResponse from src.infrastructure.config import get_config, get_model from src.infrastructure.logging.logging_config import get_logger from src.infrastructure.observability.observability import set_up_langfuse -from src.infrastructure.prompts.prompts import ( - GOOGLE_AGENT_CONTEXT, -) -from src.integrations.general_tools import DateToolsSpecs +from src.infrastructure.prompts.prompts import GOOGLE_AGENT_CONTEXT +from src.integrations.common import DateToolsSpecs +from src.shared.agents.utils import safe_load_mcp_tools +from src.shared.base.base_agent_server import BaseAgentServer set_up_langfuse() logger = get_logger("agents.google") diff --git a/src/core/agents/jira_agent.py b/src/core/agents/jira_agent.py index 30d33ea..b58a70a 100644 --- a/src/core/agents/jira_agent.py +++ b/src/core/agents/jira_agent.py @@ -6,13 +6,13 @@ import uvicorn from llama_index.core.agent.workflow import ReActAgent -from src.core.agents.utils import safe_load_mcp_tools -from src.core.base.base_agent_server import BaseAgentServer from src.core.schemas.agent_response import AgentResponse from src.infrastructure.config import get_config, get_model from src.infrastructure.logging.logging_config import get_logger from src.infrastructure.prompts.prompts import JIRA_AGENT_CONTEXT -from src.integrations.general_tools import DateToolsSpecs +from src.integrations.common import DateToolsSpecs +from src.shared.agents.utils import safe_load_mcp_tools +from src.shared.base.base_agent_server import BaseAgentServer config = get_config() logger = get_logger("agents.jira") diff --git a/src/core/base/error_handler.py b/src/core/error_handler.py similarity index 96% rename from src/core/base/error_handler.py rename to src/core/error_handler.py index 9c9dfbb..c40412d 100644 --- a/src/core/base/error_handler.py +++ b/src/core/error_handler.py @@ -10,14 +10,15 @@ from fastapi import HTTPException, status -from src.core.base.circuit_breaker import ( # noqa: F401 +from src.infrastructure.logging.logging_config import get_logger +from src.shared.resilience.circuit_breaker import ( # noqa: F401 CircuitBreaker, CircuitState, get_circuit_breaker, with_circuit_breaker, ) -from src.core.base.exceptions import * # noqa: F401, F403 -from src.core.base.exceptions import ( +from src.shared.resilience.exceptions import * # noqa: F401, F403 +from src.shared.resilience.exceptions import ( AgentAuthenticationError, AgentError, AgentResponseError, @@ -43,8 +44,8 @@ ) # Re-export for convenience -from src.core.base.retry import BackoffStrategy, with_retry # noqa: F401 -from src.infrastructure.logging.logging_config import get_logger +from src.shared.resilience.retry import BackoffStrategy # noqa: F401 +from src.shared.resilience.retry import with_retry logger = get_logger("error_handler") diff --git a/src/core/workflow_servers/action_items_server.py b/src/core/workflow_servers/action_items_server.py index 5581670..bdca41a 100644 --- a/src/core/workflow_servers/action_items_server.py +++ b/src/core/workflow_servers/action_items_server.py @@ -8,7 +8,6 @@ from langfuse import get_client as get_langfuse_client from pydantic import BaseModel, PastDate -from src.core.base.base_workflow_server import BaseWorkflowServer from src.core.schemas.workflow_models import ActionItemsList from src.core.workflows.action_items_dispatch_orchestrator import ( ActionItemsDispatchOrchestrator, @@ -19,6 +18,7 @@ from src.infrastructure.config import get_config, get_model from src.infrastructure.logging.logging_config import get_logger from src.infrastructure.observability.observability import set_up_langfuse +from src.shared.base.base_workflow_server import BaseWorkflowServer set_up_langfuse() logger = get_logger("workflow_server.action_items") diff --git a/src/core/workflows/action_items_dispatch_orchestrator.py b/src/core/workflows/action_items_dispatch_orchestrator.py index c87f471..b363c59 100644 --- a/src/core/workflows/action_items_dispatch_orchestrator.py +++ b/src/core/workflows/action_items_dispatch_orchestrator.py @@ -6,11 +6,7 @@ from typing import Any -from llama_index.core.workflow import ( - StartEvent, - Workflow, - step, -) +from llama_index.core.workflow import StartEvent, Workflow, step from src.core.schemas.workflow_models import ActionItemsList, AgentExecutionResult from src.core.workflows.common_events import StopWithErrorEvent diff --git a/src/core/workflows/meeting_notes_and_generation_orchestrator.py b/src/core/workflows/meeting_notes_and_generation_orchestrator.py index 4339501..386645d 100644 --- a/src/core/workflows/meeting_notes_and_generation_orchestrator.py +++ b/src/core/workflows/meeting_notes_and_generation_orchestrator.py @@ -8,12 +8,7 @@ from typing import Any from llama_index.core.program import LLMTextCompletionProgram -from llama_index.core.workflow import ( - Event, - StartEvent, - Workflow, - step, -) +from llama_index.core.workflow import Event, StartEvent, Workflow, step from src.core.schemas.workflow_models import ActionItemsList, AgentRoutingDecision from src.core.workflows.common_events import StopWithErrorEvent @@ -23,7 +18,7 @@ from src.core.workflows.sub_workflows.meeting_notes_workflow import MeetingNotesWorkflow from src.infrastructure.logging.logging_config import get_logger from src.infrastructure.prompts.prompts import TOOL_DISPATCHER_PROMPT -from src.infrastructure.registry.registry_client import get_registry_client +from src.services.registry.registry_client import get_registry_client logger = get_logger("workflows.meeting_notes_and_generation") diff --git a/src/core/workflows/sub_workflows/action_items_generation_workflow.py b/src/core/workflows/sub_workflows/action_items_generation_workflow.py index c22a76e..c7ab390 100644 --- a/src/core/workflows/sub_workflows/action_items_generation_workflow.py +++ b/src/core/workflows/sub_workflows/action_items_generation_workflow.py @@ -8,13 +8,7 @@ from llama_index.core.memory import Memory from llama_index.core.program import LLMTextCompletionProgram -from llama_index.core.workflow import ( - Context, - Event, - StartEvent, - Workflow, - step, -) +from llama_index.core.workflow import Context, Event, StartEvent, Workflow, step from src.core.schemas.workflow_models import ActionItemsList, ReviewFeedback from src.core.workflows.common_events import StopWithErrorEvent @@ -25,13 +19,13 @@ REFINEMENT_PROMPT, REVIEWER_PROMPT, ) -from src.infrastructure.utils.progressive_summarization import ( +from src.shared.llm.summarization.progressive import ( ProgressiveSummaryResult, SummarizationStrategy, progressive_summarize, summarize_meeting_notes, ) -from src.infrastructure.utils.token_utils import ( +from src.shared.llm.token_utils import ( count_tokens, get_max_context_tokens, should_summarize_notes, diff --git a/src/core/workflows/sub_workflows/agent_dispatch_workflow.py b/src/core/workflows/sub_workflows/agent_dispatch_workflow.py index 56ac40d..d875198 100644 --- a/src/core/workflows/sub_workflows/agent_dispatch_workflow.py +++ b/src/core/workflows/sub_workflows/agent_dispatch_workflow.py @@ -7,16 +7,10 @@ from urllib.parse import urljoin import httpx -from llama_index.core.workflow import ( - Context, - Event, - StartEvent, - Workflow, - step, -) +from llama_index.core.workflow import Context, Event, StartEvent, Workflow, step from pydantic import HttpUrl -from src.core.base.error_handler import ( +from src.core.error_handler import ( AgentResponseError, AgentTimeoutError, AgentUnavailableError, @@ -33,7 +27,7 @@ from src.core.workflows.common_events import StopWithErrorEvent from src.infrastructure.logging.logging_config import get_logger from src.infrastructure.prompts.prompts import AGENT_QUERY_PROMPT -from src.infrastructure.registry.registry_client import get_registry_client +from src.services.registry.registry_client import get_registry_client logger = get_logger("workflows.agent_dispatch") diff --git a/src/core/workflows/sub_workflows/meeting_notes_workflow.py b/src/core/workflows/sub_workflows/meeting_notes_workflow.py index c104e25..dbf4e86 100644 --- a/src/core/workflows/sub_workflows/meeting_notes_workflow.py +++ b/src/core/workflows/sub_workflows/meeting_notes_workflow.py @@ -12,13 +12,7 @@ import nest_asyncio from langfuse import get_client as get_langfuse_client from llama_index.core.program import LLMTextCompletionProgram -from llama_index.core.workflow import ( - Context, - Event, - StartEvent, - Workflow, - step, -) +from llama_index.core.workflow import Context, Event, StartEvent, Workflow, step from llama_index.tools.mcp import BasicMCPClient from pydantic import BaseModel diff --git a/src/infrastructure/cache/redis_cache.py b/src/infrastructure/cache/redis_cache.py index bdbe6e3..4c11035 100644 --- a/src/infrastructure/cache/redis_cache.py +++ b/src/infrastructure/cache/redis_cache.py @@ -5,13 +5,11 @@ import redis from redis.exceptions import ConnectionError as RedisConnectionError -from redis.exceptions import ( - RedisError, -) +from redis.exceptions import RedisError -from src.common.singleton_meta import SingletonMeta from src.infrastructure.config import get_config from src.infrastructure.logging.logging_config import get_logger +from src.shared.common.singleton_meta import SingletonMeta logger = get_logger("redis_cache") diff --git a/src/infrastructure/config/read_config.py b/src/infrastructure/config/read_config.py index 14d6189..50455af 100644 --- a/src/infrastructure/config/read_config.py +++ b/src/infrastructure/config/read_config.py @@ -13,7 +13,7 @@ model_validator, ) -from src.common.singleton_meta import SingletonMeta +from src.shared.common.singleton_meta import SingletonMeta class ObservabilityConfigSchema(BaseModel): diff --git a/src/integrations/common/__init__.py b/src/integrations/common/__init__.py new file mode 100644 index 0000000..6f713bb --- /dev/null +++ b/src/integrations/common/__init__.py @@ -0,0 +1,7 @@ +""" +Common integration tools +""" + +from src.integrations.common.date_tools import DateToolsSpecs + +__all__ = ["DateToolsSpecs"] diff --git a/src/integrations/general_tools/date_tools.py b/src/integrations/common/date_tools.py similarity index 100% rename from src/integrations/general_tools/date_tools.py rename to src/integrations/common/date_tools.py diff --git a/src/integrations/general_tools/__init__.py b/src/integrations/general_tools/__init__.py index 2ff5e7e..f87767e 100644 --- a/src/integrations/general_tools/__init__.py +++ b/src/integrations/general_tools/__init__.py @@ -2,6 +2,6 @@ Init for general specs """ -from src.integrations.general_tools.date_tools import DateToolsSpecs +from src.integrations.common.date_tools import DateToolsSpecs __all__ = ["DateToolsSpecs"] diff --git a/src/integrations/google/__init__.py b/src/integrations/google/__init__.py new file mode 100644 index 0000000..2ab9a17 --- /dev/null +++ b/src/integrations/google/__init__.py @@ -0,0 +1,7 @@ +""" +Google integration tools +""" + +from src.integrations.google.tools import GoogleToolSpec + +__all__ = ["GoogleToolSpec"] diff --git a/src/integrations/google_tools/auth_utils.py b/src/integrations/google/auth.py similarity index 100% rename from src/integrations/google_tools/auth_utils.py rename to src/integrations/google/auth.py diff --git a/src/integrations/google_tools/google_tools.py b/src/integrations/google/tools.py similarity index 99% rename from src/integrations/google_tools/google_tools.py rename to src/integrations/google/tools.py index b929443..8f84d2b 100644 --- a/src/integrations/google_tools/google_tools.py +++ b/src/integrations/google/tools.py @@ -10,7 +10,7 @@ from src.infrastructure.cache import get_document_cache from src.infrastructure.logging.logging_config import get_logger -from src.integrations.google_tools.auth_utils import authenticate +from src.integrations.google.auth import authenticate logger = get_logger("google_tools.calendar") diff --git a/src/integrations/google_tools/__init__.py b/src/integrations/google_tools/__init__.py index 13066e5..9d204ae 100644 --- a/src/integrations/google_tools/__init__.py +++ b/src/integrations/google_tools/__init__.py @@ -2,7 +2,7 @@ Initialize Google tools specs """ +from src.integrations.google.tools import GoogleToolSpec from src.integrations.google_tools.gmail_tools import GmailToolSpec -from src.integrations.google_tools.google_tools import GoogleToolSpec __all__ = ["GoogleToolSpec", "GmailToolSpec"] diff --git a/src/integrations/google_tools/gmail_tools.py b/src/integrations/google_tools/gmail_tools.py index 357e726..5308cfb 100644 --- a/src/integrations/google_tools/gmail_tools.py +++ b/src/integrations/google_tools/gmail_tools.py @@ -10,7 +10,7 @@ from llama_index.core.tools.tool_spec.base import BaseToolSpec from src.infrastructure.logging.logging_config import get_logger -from src.integrations.google_tools.auth_utils import authenticate +from src.integrations.google.auth import authenticate logger = get_logger("google_tools.gmail") diff --git a/src/integrations/jira/__init__.py b/src/integrations/jira/__init__.py new file mode 100644 index 0000000..2473770 --- /dev/null +++ b/src/integrations/jira/__init__.py @@ -0,0 +1,7 @@ +""" +Jira integration tools +""" + +from src.integrations.jira.tools import JiraToolSpec + +__all__ = ["JiraToolSpec"] diff --git a/src/integrations/jira_tools/jira_formatter.py b/src/integrations/jira/formatter.py similarity index 100% rename from src/integrations/jira_tools/jira_formatter.py rename to src/integrations/jira/formatter.py diff --git a/src/integrations/jira_tools/jira_tools.py b/src/integrations/jira/tools.py similarity index 87% rename from src/integrations/jira_tools/jira_tools.py rename to src/integrations/jira/tools.py index da8d918..b0d6d0e 100644 --- a/src/integrations/jira_tools/jira_tools.py +++ b/src/integrations/jira/tools.py @@ -7,7 +7,7 @@ from llama_index.core.tools.tool_spec.base import BaseToolSpec from src.infrastructure.logging.logging_config import get_logger -from src.integrations.jira_tools.jira_formatter import JiraFormatter +from src.integrations.jira.formatter import JiraFormatter class JiraToolSpec(BaseToolSpec): @@ -33,9 +33,7 @@ def get_fields_name_to_id(self) -> Dict[str, str]: all names will be using lower()""" self.logger.debug("Getting field name to ID mapping") try: - result = { - f["name"].lower(): f["id"] for f in self.jira_client.fields() - } + result = {f["name"].lower(): f["id"] for f in self.jira_client.fields()} self.logger.info(f"Retrieved {len(result)} field mappings") return result except JIRAError as e: @@ -94,20 +92,14 @@ def search_jira_issues( f"Searching issues with query: {query}, max_results: {max_results}" ) try: - result = self.jira_client.search_issues( - query, maxResults=max_results - ) + result = self.jira_client.search_issues(query, maxResults=max_results) self.logger.info(f"Found {len(result)} issues for query: {query}") return result except JIRAError as e: - self.logger.error( - f"Failed to search issues with query '{query}': {e}" - ) + self.logger.error(f"Failed to search issues with query '{query}': {e}") raise JIRAError from e - def create_jira_issue( - self, issue_fields: Dict, issue_type: str = "task" - ) -> Issue: + def create_jira_issue(self, issue_fields: Dict, issue_type: str = "task") -> Issue: """ Create a new Jira issue using the provided field values. @@ -140,15 +132,11 @@ def create_jira_issue( for field, value in issue_fields.items(): formatter = getattr( JiraFormatter, - fields_ids_to_types.get( - fields_names_to_id.get(field), "any" - ), + fields_ids_to_types.get(fields_names_to_id.get(field), "any"), ) issue[fields_names_to_id.get(field)] = formatter(value) - issue["issuetype"] = JiraFormatter.issue_type( - issue_type.capitalize() - ) + issue["issuetype"] = JiraFormatter.issue_type(issue_type.capitalize()) new_issue = self.jira_client.create_issue(fields=issue) self.logger.info(f"Successfully created issue: {new_issue.key}") @@ -194,8 +182,7 @@ def get_jira_issue( fields_mapping = self.get_fields_id_to_name() issue_dict = { - fields_mapping[k]: v - for k, v in issue.raw.get("fields").items() + fields_mapping[k]: v for k, v in issue.raw.get("fields").items() } else: if field_filter is None: @@ -204,8 +191,7 @@ def get_jira_issue( fields_mapping = self.get_fields_name_to_id() issue_dict = { - f: issue.get_field(fields_mapping[f.lower()]) - for f in field_filter + f: issue.get_field(fields_mapping[f.lower()]) for f in field_filter } return issue_dict diff --git a/src/integrations/jira_tools/__init__.py b/src/integrations/jira_tools/__init__.py index 1499780..41bd2fe 100644 --- a/src/integrations/jira_tools/__init__.py +++ b/src/integrations/jira_tools/__init__.py @@ -1,5 +1,5 @@ """Init module for jira tools specs""" -from src.integrations.jira_tools.jira_tools import JiraToolSpec +from src.integrations.jira.tools import JiraToolSpec __all__ = ["JiraToolSpec"] diff --git a/src/services/mcp/__init__.py b/src/services/mcp/__init__.py new file mode 100644 index 0000000..036f656 --- /dev/null +++ b/src/services/mcp/__init__.py @@ -0,0 +1,7 @@ +"""MCP (Model Context Protocol) servers package. + +This package contains MCP server implementations that expose tool capabilities +via the Model Context Protocol: +- google_tools_mcp.py: Google Workspace tools MCP server (Port 8100) +- jira_tools_mcp.py: JIRA tools MCP server (Port 8101) +""" diff --git a/src/mcp/google_tools_mcp.py b/src/services/mcp/google_tools_mcp.py similarity index 93% rename from src/mcp/google_tools_mcp.py rename to src/services/mcp/google_tools_mcp.py index e024d6b..7e9da44 100644 --- a/src/mcp/google_tools_mcp.py +++ b/src/services/mcp/google_tools_mcp.py @@ -6,7 +6,8 @@ from src.infrastructure.config import get_config from src.infrastructure.logging.logging_config import get_logger -from src.integrations.google_tools import GmailToolSpec, GoogleToolSpec +from src.integrations.google import GoogleToolSpec +from src.integrations.google_tools import GmailToolSpec logger = get_logger("mcp.google_tools") diff --git a/src/mcp/jira_tools_mcp.py b/src/services/mcp/jira_tools_mcp.py similarity index 97% rename from src/mcp/jira_tools_mcp.py rename to src/services/mcp/jira_tools_mcp.py index 73d5012..6033cb9 100644 --- a/src/mcp/jira_tools_mcp.py +++ b/src/services/mcp/jira_tools_mcp.py @@ -8,7 +8,7 @@ from src.infrastructure.config import get_config from src.infrastructure.logging.logging_config import get_logger -from src.integrations.jira_tools import JiraToolSpec +from src.integrations.jira import JiraToolSpec logger = get_logger("mcp.jira_tools") diff --git a/src/services/registry/__init__.py b/src/services/registry/__init__.py new file mode 100644 index 0000000..70b671c --- /dev/null +++ b/src/services/registry/__init__.py @@ -0,0 +1,16 @@ +"""Registry service package. + +This package contains the complete registry service implementation including: +- registry_service.py: Standalone registry server +- registry_client.py: Client for service discovery +- agent_registry.py: Registry data models +""" + +from src.services.registry.agent_registry import AgentInfo +from src.services.registry.registry_client import RegistryClient, get_registry_client + +__all__ = [ + "AgentInfo", + "RegistryClient", + "get_registry_client", +] diff --git a/src/infrastructure/registry/agent_registry.py b/src/services/registry/agent_registry.py similarity index 99% rename from src/infrastructure/registry/agent_registry.py rename to src/services/registry/agent_registry.py index f121678..a8c6d4f 100644 --- a/src/infrastructure/registry/agent_registry.py +++ b/src/services/registry/agent_registry.py @@ -5,9 +5,9 @@ from pydantic import BaseModel -from src.common.singleton_meta import SingletonMeta from src.infrastructure.cache import get_cache from src.infrastructure.logging.logging_config import get_logger +from src.shared.common.singleton_meta import SingletonMeta logger = get_logger("agent_registry") diff --git a/src/infrastructure/registry/registry_client.py b/src/services/registry/registry_client.py similarity index 98% rename from src/infrastructure/registry/registry_client.py rename to src/services/registry/registry_client.py index ef72b5c..4a808ac 100644 --- a/src/infrastructure/registry/registry_client.py +++ b/src/services/registry/registry_client.py @@ -5,10 +5,10 @@ import httpx -from src.core.base.retry import BackoffStrategy, with_retry from src.infrastructure.config import get_config from src.infrastructure.logging.logging_config import get_logger -from src.infrastructure.registry.agent_registry import AgentInfo +from src.services.registry.agent_registry import AgentInfo +from src.shared.resilience.retry import BackoffStrategy, with_retry logger = get_logger("registry_client") diff --git a/src/services/registry_service.py b/src/services/registry/registry_service.py similarity index 99% rename from src/services/registry_service.py rename to src/services/registry/registry_service.py index 5b4c762..06d7790 100644 --- a/src/services/registry_service.py +++ b/src/services/registry/registry_service.py @@ -14,7 +14,7 @@ from src.infrastructure.config import get_config from src.infrastructure.logging.logging_config import get_logger -from src.infrastructure.registry.agent_registry import AgentInfo, AgentRegistry +from src.services.registry.agent_registry import AgentInfo, AgentRegistry logger = get_logger("registry_service") diff --git a/src/shared/__init__.py b/src/shared/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/shared/agents/__init__.py b/src/shared/agents/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/core/agents/utils.py b/src/shared/agents/utils.py similarity index 100% rename from src/core/agents/utils.py rename to src/shared/agents/utils.py diff --git a/src/shared/base/__init__.py b/src/shared/base/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/core/base/base_agent_server.py b/src/shared/base/base_agent_server.py similarity index 97% rename from src/core/base/base_agent_server.py rename to src/shared/base/base_agent_server.py index fabc7e6..da008b7 100644 --- a/src/core/base/base_agent_server.py +++ b/src/shared/base/base_agent_server.py @@ -20,15 +20,15 @@ from llama_index.core.workflow import Context from pydantic import BaseModel -from src.core.base.base_server import BaseServer -from src.core.base.error_handler import ErrorContext, handle_error_response -from src.core.base.exceptions import AgentError, MeetingActionsError +from src.core.error_handler import ErrorContext, handle_error_response from src.core.schemas.agent_response import AgentResponse from src.infrastructure.config import get_config from src.infrastructure.logging.logging_config import get_logger from src.infrastructure.observability.observability import set_up_langfuse -from src.infrastructure.registry.agent_registry import AgentInfo -from src.infrastructure.registry.registry_client import get_registry_client +from src.services.registry.agent_registry import AgentInfo +from src.services.registry.registry_client import get_registry_client +from src.shared.base.base_server import BaseServer +from src.shared.resilience.exceptions import AgentError, MeetingActionsError set_up_langfuse() logger = get_logger("agents.base") @@ -249,7 +249,7 @@ async def circuit_breaker_health(): current state, failure counts, and configuration. """ # pylint: disable=import-outside-toplevel - from src.core.base.circuit_breaker import get_all_circuit_breakers + from src.shared.resilience.circuit_breaker import get_all_circuit_breakers circuits = get_all_circuit_breakers() diff --git a/src/core/base/base_server.py b/src/shared/base/base_server.py similarity index 100% rename from src/core/base/base_server.py rename to src/shared/base/base_server.py diff --git a/src/core/base/base_workflow_server.py b/src/shared/base/base_workflow_server.py similarity index 97% rename from src/core/base/base_workflow_server.py rename to src/shared/base/base_workflow_server.py index 633dca7..0c85973 100644 --- a/src/core/base/base_workflow_server.py +++ b/src/shared/base/base_workflow_server.py @@ -5,8 +5,8 @@ servers that use multiple workflow orchestrators instead of a single agent service. """ -from src.core.base.base_server import BaseServer from src.infrastructure.logging.logging_config import get_logger +from src.shared.base.base_server import BaseServer logger = get_logger("workflows.base") diff --git a/src/shared/common/__init__.py b/src/shared/common/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/common/singleton_meta.py b/src/shared/common/singleton_meta.py similarity index 100% rename from src/common/singleton_meta.py rename to src/shared/common/singleton_meta.py diff --git a/src/shared/llm/__init__.py b/src/shared/llm/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/shared/llm/summarization/__init__.py b/src/shared/llm/summarization/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/infrastructure/utils/progressive_summarization.py b/src/shared/llm/summarization/progressive.py similarity index 99% rename from src/infrastructure/utils/progressive_summarization.py rename to src/shared/llm/summarization/progressive.py index 5f547c8..ef8294c 100644 --- a/src/infrastructure/utils/progressive_summarization.py +++ b/src/shared/llm/summarization/progressive.py @@ -18,7 +18,7 @@ SUMMARIZATION_PROMPT, get_progressive_pass_prompt, ) -from src.infrastructure.utils.token_utils import ( +from src.shared.llm.token_utils import ( chunk_text_by_tokens, count_tokens, get_max_context_tokens, diff --git a/src/infrastructure/utils/token_utils.py b/src/shared/llm/token_utils.py similarity index 100% rename from src/infrastructure/utils/token_utils.py rename to src/shared/llm/token_utils.py diff --git a/src/shared/resilience/__init__.py b/src/shared/resilience/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/core/base/circuit_breaker.py b/src/shared/resilience/circuit_breaker.py similarity index 99% rename from src/core/base/circuit_breaker.py rename to src/shared/resilience/circuit_breaker.py index d2204f2..c5661cd 100644 --- a/src/core/base/circuit_breaker.py +++ b/src/shared/resilience/circuit_breaker.py @@ -11,8 +11,8 @@ from enum import Enum from typing import Callable, Optional, Tuple, Type -from src.core.base.exceptions import CircuitOpenError from src.infrastructure.logging.logging_config import get_logger +from src.shared.resilience.exceptions import CircuitOpenError logger = get_logger("circuit_breaker") diff --git a/src/core/base/exceptions.py b/src/shared/resilience/exceptions.py similarity index 100% rename from src/core/base/exceptions.py rename to src/shared/resilience/exceptions.py diff --git a/src/core/base/retry.py b/src/shared/resilience/retry.py similarity index 99% rename from src/core/base/retry.py rename to src/shared/resilience/retry.py index 767d433..319fc05 100644 --- a/src/core/base/retry.py +++ b/src/shared/resilience/retry.py @@ -12,8 +12,8 @@ from enum import Enum from typing import Callable, Optional, Tuple, Type, Union -from src.core.base.exceptions import MaxRetriesExceededError, RetryableError from src.infrastructure.logging.logging_config import get_logger +from src.shared.resilience.exceptions import MaxRetriesExceededError, RetryableError logger = get_logger("retry") diff --git a/tests/conftest.py b/tests/conftest.py index 8eb9410..abf6f94 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -10,9 +10,9 @@ import pytest from fastapi.testclient import TestClient -from src.common.singleton_meta import SingletonMeta from src.infrastructure.cache import RedisDocumentCache from src.infrastructure.config.read_config import ConfigReader, ConfigSchema +from src.shared.common.singleton_meta import SingletonMeta @pytest.fixture(scope="session", autouse=True) diff --git a/tests/integration/test_api_endpoints.py b/tests/integration/test_api_endpoints.py index 0dbc159..b29e51d 100644 --- a/tests/integration/test_api_endpoints.py +++ b/tests/integration/test_api_endpoints.py @@ -7,8 +7,8 @@ import pytest from fastapi.testclient import TestClient -from src.core.base.base_agent_server import BaseAgentServer -from src.core.base.base_server import BaseServer +from src.shared.base.base_agent_server import BaseAgentServer +from src.shared.base.base_server import BaseServer class MockWorkflowServer(BaseServer): @@ -45,7 +45,7 @@ def mock_llm(): @pytest.fixture def workflow_server(mock_llm): """Create workflow server for testing.""" - with patch("src.core.base.base_server.get_logger"): + with patch("src.shared.base.base_server.get_logger"): return MockWorkflowServer( llm=mock_llm, title="Test Workflow Server", @@ -56,8 +56,8 @@ def workflow_server(mock_llm): @pytest.fixture def agent_server(mock_llm): """Create agent server for testing.""" - with patch("src.core.base.base_agent_server.set_up_langfuse"), patch( - "src.core.base.base_agent_server.get_langfuse_client" + with patch("src.shared.base.base_agent_server.set_up_langfuse"), patch( + "src.shared.base.base_agent_server.get_langfuse_client" ): return MockAgentServerForIntegration( llm=mock_llm, @@ -97,8 +97,8 @@ def test_workflow_server_endpoints(self, workflow_server): class TestAgentServerIntegration: """Integration tests for agent server.""" - @patch("src.core.base.base_agent_server.Memory") - @patch("src.core.base.base_agent_server.get_langfuse_client") + @patch("src.shared.base.base_agent_server.Memory") + @patch("src.shared.base.base_agent_server.get_langfuse_client") def test_agent_server_full_flow(self, mock_langfuse, mock_memory, agent_server): """Test complete agent server request flow.""" client = TestClient(agent_server.app) @@ -153,8 +153,8 @@ def test_agent_server_error_handling_integration(self, agent_server): """Test error handling across the full request pipeline.""" client = TestClient(agent_server.app) - with patch("src.core.base.base_agent_server.Memory"), patch( - "src.core.base.base_agent_server.get_langfuse_client" + with patch("src.shared.base.base_agent_server.Memory"), patch( + "src.shared.base.base_agent_server.get_langfuse_client" ): # Make agent raise an exception diff --git a/tests/performance/test_load_testing.py b/tests/performance/test_load_testing.py index a71907a..9c6972b 100644 --- a/tests/performance/test_load_testing.py +++ b/tests/performance/test_load_testing.py @@ -9,7 +9,7 @@ import pytest -from src.integrations.google_tools.google_tools import GoogleToolSpec +from src.integrations.google.tools import GoogleToolSpec @pytest.mark.performance @@ -19,9 +19,9 @@ class TestGoogleToolsPerformance: @pytest.fixture def mock_google_tool_spec(self): """Create mock Google tool spec for performance testing.""" - with patch("src.integrations.google_tools.google_tools.build"), patch( - "src.integrations.google_tools.google_tools.authenticate" - ), patch("src.integrations.google_tools.google_tools.get_document_cache"): + with patch("src.integrations.google.tools.build"), patch( + "src.integrations.google.tools.authenticate" + ), patch("src.integrations.google.tools.get_document_cache"): tool_spec = GoogleToolSpec() diff --git a/tests/unit/agents/test_base_agent_server.py b/tests/unit/agents/test_base_agent_server.py index c3713fa..80af36f 100644 --- a/tests/unit/agents/test_base_agent_server.py +++ b/tests/unit/agents/test_base_agent_server.py @@ -9,8 +9,8 @@ from llama_index.core.agent.workflow import ReActAgent from llama_index.core.memory import Memory -from src.core.base.base_agent_server import BaseAgentServer, ChatQuery from src.core.schemas.agent_response import AgentResponse +from src.shared.base.base_agent_server import BaseAgentServer, ChatQuery class MockAgentServer(BaseAgentServer): @@ -35,8 +35,8 @@ def mock_llm(): @pytest.fixture def agent_server(mock_llm): """Create a test agent server instance.""" - with patch("src.core.base.base_agent_server.set_up_langfuse"), patch( - "src.core.base.base_agent_server.get_langfuse_client" + with patch("src.shared.base.base_agent_server.set_up_langfuse"), patch( + "src.shared.base.base_agent_server.get_langfuse_client" ): return MockAgentServer( llm=mock_llm, title="Test Agent", description="Test agent for unit testing" @@ -83,9 +83,9 @@ def test_description_endpoint(self, test_client): assert response.status_code == 200 assert response.json() == "Test agent for unit testing" - @patch("src.core.base.base_agent_server.Context") - @patch("src.core.base.base_agent_server.Memory") - @patch("src.core.base.base_agent_server.get_langfuse_client") + @patch("src.shared.base.base_agent_server.Context") + @patch("src.shared.base.base_agent_server.Memory") + @patch("src.shared.base.base_agent_server.get_langfuse_client") def test_chat_with_agent_success( self, mock_langfuse, mock_memory, mock_context, test_client, agent_server ): @@ -129,9 +129,9 @@ def test_chat_with_agent_success( assert "ctx" in call_args[1] assert "memory" in call_args[1] - @patch("src.core.base.base_agent_server.Context") - @patch("src.core.base.base_agent_server.Memory") - @patch("src.core.base.base_agent_server.get_langfuse_client") + @patch("src.shared.base.base_agent_server.Context") + @patch("src.shared.base.base_agent_server.Memory") + @patch("src.shared.base.base_agent_server.get_langfuse_client") def test_chat_with_agent_structured_response( self, mock_langfuse, mock_memory, mock_context, test_client, agent_server ): @@ -168,8 +168,8 @@ def test_chat_with_agent_structured_response( assert data["response"] == "Structured response content" assert data["error"] is False - @patch("src.core.base.base_agent_server.Memory") - @patch("src.core.base.base_agent_server.get_langfuse_client") + @patch("src.shared.base.base_agent_server.Memory") + @patch("src.shared.base.base_agent_server.get_langfuse_client") def test_chat_with_agent_error( self, mock_langfuse, mock_memory, test_client, agent_server ): diff --git a/tests/unit/agents/test_base_server.py b/tests/unit/agents/test_base_server.py index cd9d6f0..0104ee3 100644 --- a/tests/unit/agents/test_base_server.py +++ b/tests/unit/agents/test_base_server.py @@ -6,9 +6,9 @@ import pytest -from src.core.base.base_agent_server import BaseAgentServer, ChatQuery -from src.core.base.base_server import BaseServer from src.core.schemas.agent_response import AgentResponse +from src.shared.base.base_agent_server import BaseAgentServer, ChatQuery +from src.shared.base.base_server import BaseServer class MockAgent: @@ -105,9 +105,9 @@ def test_agent_server_initialization(self, mock_llm): assert server.app.title == "Test Agent" assert isinstance(server.service, MockAgent) - @patch("src.core.base.base_agent_server.Context") - @patch("src.core.base.base_agent_server.Memory") - @patch("src.core.base.base_agent_server.get_langfuse_client") + @patch("src.shared.base.base_agent_server.Context") + @patch("src.shared.base.base_agent_server.Memory") + @patch("src.shared.base.base_agent_server.get_langfuse_client") @pytest.mark.asyncio async def test_agent_endpoint( self, mock_langfuse, mock_memory, mock_context, mock_llm, test_client_factory diff --git a/tests/unit/config/test_config_reader.py b/tests/unit/config/test_config_reader.py index f112659..bbcd116 100644 --- a/tests/unit/config/test_config_reader.py +++ b/tests/unit/config/test_config_reader.py @@ -105,7 +105,7 @@ def test_config_loading(self, test_config): def test_reset_instance(self, temp_config_file): """Test resetting singleton instance.""" - from src.common.singleton_meta import SingletonMeta + from src.shared.common.singleton_meta import SingletonMeta # Create first instance reader1 = ConfigReader() @@ -121,7 +121,7 @@ def test_reset_instance(self, temp_config_file): def test_file_not_found_error(self): """Test error when config file doesn't exist.""" - from src.common.singleton_meta import SingletonMeta + from src.shared.common.singleton_meta import SingletonMeta # Reset singleton before test SingletonMeta.reset_instance(ConfigReader) @@ -135,7 +135,7 @@ def test_file_not_found_error(self): def test_json_decode_error(self, tmp_path): """Test error when config file has invalid JSON.""" - from src.common.singleton_meta import SingletonMeta + from src.shared.common.singleton_meta import SingletonMeta # Reset singleton before test SingletonMeta.reset_instance(ConfigReader) @@ -152,7 +152,7 @@ def test_json_decode_error(self, tmp_path): def test_validation_error(self, tmp_path): """Test error when config file has invalid schema.""" - from src.common.singleton_meta import SingletonMeta + from src.shared.common.singleton_meta import SingletonMeta # Reset singleton before test SingletonMeta.reset_instance(ConfigReader) diff --git a/tests/unit/core/test_agent_utils.py b/tests/unit/core/test_agent_utils.py index 1056812..ab19c97 100644 --- a/tests/unit/core/test_agent_utils.py +++ b/tests/unit/core/test_agent_utils.py @@ -6,13 +6,13 @@ import pytest -from src.core.agents.utils import safe_load_mcp_tools +from src.shared.agents.utils import safe_load_mcp_tools class TestAgentUtils: """Test cases for agent utilities.""" - @patch("src.core.agents.utils.aget_tools_from_mcp_url") + @patch("src.shared.agents.utils.aget_tools_from_mcp_url") def test_safe_load_mcp_tools_success(self, mock_aget_tools): """Test successful loading of MCP tools.""" # Mock the async function @@ -32,7 +32,7 @@ def test_safe_load_mcp_tools_success(self, mock_aget_tools): # Should return combined tools from both servers assert len(result) == 4 # 2 tools from each server - @patch("src.core.agents.utils.aget_tools_from_mcp_url") + @patch("src.shared.agents.utils.aget_tools_from_mcp_url") def test_safe_load_mcp_tools_empty_servers(self, mock_aget_tools): """Test with empty server list.""" result = safe_load_mcp_tools([]) @@ -41,8 +41,8 @@ def test_safe_load_mcp_tools_empty_servers(self, mock_aget_tools): assert not result mock_aget_tools.assert_not_called() - @patch("src.core.agents.utils.aget_tools_from_mcp_url") - @patch("src.core.agents.utils.logger") + @patch("src.shared.agents.utils.aget_tools_from_mcp_url") + @patch("src.shared.agents.utils.logger") def test_safe_load_mcp_tools_exception_handling(self, mock_logger, mock_aget_tools): """Test exception handling when MCP server fails.""" # Mock asyncio.run to raise an exception for the first server @@ -64,8 +64,8 @@ def test_safe_load_mcp_tools_exception_handling(self, mock_logger, mock_aget_too # Should still return tools from working server assert len(result) == 1 - @patch("src.core.agents.utils.aget_tools_from_mcp_url") - @patch("src.core.agents.utils.logger") + @patch("src.shared.agents.utils.aget_tools_from_mcp_url") + @patch("src.shared.agents.utils.logger") def test_safe_load_mcp_tools_all_servers_fail(self, mock_logger, mock_aget_tools): """Test when all servers fail.""" with patch("asyncio.run") as mock_run: @@ -80,8 +80,8 @@ def test_safe_load_mcp_tools_all_servers_fail(self, mock_logger, mock_aget_tools # Should return empty list assert not result - @patch("src.core.agents.utils.aget_tools_from_mcp_url") - @patch("src.core.agents.utils.logger") + @patch("src.shared.agents.utils.aget_tools_from_mcp_url") + @patch("src.shared.agents.utils.logger") def test_safe_load_mcp_tools_logs_info(self, mock_logger, mock_aget_tools): """Test that info logging works correctly.""" with patch("asyncio.run") as mock_run: @@ -102,7 +102,7 @@ def test_safe_load_mcp_tools_with_none_servers(self): with pytest.raises((TypeError, AttributeError)): safe_load_mcp_tools(None) - @patch("src.core.agents.utils.aget_tools_from_mcp_url") + @patch("src.shared.agents.utils.aget_tools_from_mcp_url") def test_safe_load_mcp_tools_preserves_order(self, mock_aget_tools): """Test that tools from different servers are combined in order.""" with patch("asyncio.run") as mock_run: diff --git a/tests/unit/integrations/test_auth_utils.py b/tests/unit/integrations/test_auth_utils.py index 2d6d30f..fe370e1 100644 --- a/tests/unit/integrations/test_auth_utils.py +++ b/tests/unit/integrations/test_auth_utils.py @@ -4,7 +4,7 @@ import pytest -from src.integrations.google_tools.auth_utils import authenticate +from src.integrations.google.auth import authenticate class TestAuthenticate: @@ -12,7 +12,7 @@ class TestAuthenticate: @patch("os.path.exists") @patch("builtins.open", new_callable=mock_open, read_data='{"token": "test"}') - @patch("src.integrations.google_tools.auth_utils.Credentials") + @patch("src.integrations.google.auth.Credentials") def test_authenticate_with_existing_valid_token( self, mock_credentials, mock_file, mock_exists ): @@ -50,8 +50,8 @@ def test_authenticate_container_mode_missing_token(self, mock_exists): @patch("os.path.exists") @patch("builtins.open", new_callable=mock_open, read_data='{"token": "test"}') - @patch("src.integrations.google_tools.auth_utils.Credentials") - @patch("src.integrations.google_tools.auth_utils.Request") + @patch("src.integrations.google.auth.Credentials") + @patch("src.integrations.google.auth.Request") def test_authenticate_refreshes_expired_token( self, mock_request, mock_credentials, mock_file, mock_exists ): diff --git a/tests/unit/integrations/test_google_tools.py b/tests/unit/integrations/test_google_tools.py index 064ea64..bef8e04 100644 --- a/tests/unit/integrations/test_google_tools.py +++ b/tests/unit/integrations/test_google_tools.py @@ -7,7 +7,7 @@ import pytest from googleapiclient.errors import HttpError -from src.integrations.google_tools.google_tools import GoogleToolSpec +from src.integrations.google.tools import GoogleToolSpec @pytest.fixture @@ -33,11 +33,9 @@ def google_tool_spec(mock_cache, mock_services): """Create GoogleToolSpec instance with mocked dependencies.""" calendar_service, meet_service, docs_service = mock_services - with patch("src.integrations.google_tools.google_tools.build") as mock_build, patch( - "src.integrations.google_tools.google_tools.authenticate" - ), patch( - "src.integrations.google_tools.google_tools.get_document_cache" - ) as mock_get_cache: + with patch("src.integrations.google.tools.build") as mock_build, patch( + "src.integrations.google.tools.authenticate" + ), patch("src.integrations.google.tools.get_document_cache") as mock_get_cache: mock_build.side_effect = [calendar_service, meet_service, docs_service] mock_get_cache.return_value = mock_cache diff --git a/tests/unit/test_singleton_thread_safety.py b/tests/unit/test_singleton_thread_safety.py index 6a61e0a..b55d1af 100644 --- a/tests/unit/test_singleton_thread_safety.py +++ b/tests/unit/test_singleton_thread_safety.py @@ -2,7 +2,7 @@ import threading -from src.common.singleton_meta import SingletonMeta +from src.shared.common.singleton_meta import SingletonMeta class TestSingleton(metaclass=SingletonMeta): diff --git a/tests/unit/utils/test_progressive_summarization.py b/tests/unit/utils/test_progressive_summarization.py index 1fa63ce..da7cc2f 100644 --- a/tests/unit/utils/test_progressive_summarization.py +++ b/tests/unit/utils/test_progressive_summarization.py @@ -6,7 +6,7 @@ import pytest -from src.infrastructure.utils.progressive_summarization import ( +from src.shared.llm.summarization.progressive import ( ChunkSummary, PassSummaryOutput, ProgressiveSummaryResult, @@ -78,14 +78,13 @@ async def test_successful_pass(self): # Mock token counting with patch( - "src.infrastructure.utils.progressive_summarization.count_tokens" + "src.shared.llm.summarization.progressive.count_tokens" ) as mock_count: mock_count.side_effect = [5000, 2000] # input, output # Mock LLMTextCompletionProgram with patch( - "src.infrastructure.utils.progressive_summarization." - "LLMTextCompletionProgram" + "src.shared.llm.summarization.progressive.LLMTextCompletionProgram" ) as mock_program_class: mock_program = MagicMock() mock_program_class.from_defaults.return_value = mock_program @@ -120,20 +119,19 @@ async def test_pass_with_fallback(self): mock_llm = MagicMock() with patch( - "src.infrastructure.utils.progressive_summarization.count_tokens" + "src.shared.llm.summarization.progressive.count_tokens" ) as mock_count: mock_count.side_effect = [5000, 1800] # input, truncated output # Mock LLMTextCompletionProgram to raise error with patch( - "src.infrastructure.utils.progressive_summarization." - "LLMTextCompletionProgram" + "src.shared.llm.summarization.progressive.LLMTextCompletionProgram" ) as mock_program_class: mock_program_class.from_defaults.side_effect = Exception("LLM error") # Mock truncation with patch( - "src.infrastructure.utils.progressive_summarization." + "src.shared.llm.summarization.progressive." "truncate_text_by_tokens" ) as mock_truncate: mock_truncate.return_value = "Truncated text" @@ -160,7 +158,7 @@ async def test_no_summarization_needed(self): mock_llm = MagicMock() with patch( - "src.infrastructure.utils.progressive_summarization.count_tokens" + "src.shared.llm.summarization.progressive.count_tokens" ) as mock_count: mock_count.return_value = 1000 # Already below target @@ -191,14 +189,13 @@ def mock_count_tokens(*args, **kwargs): return result with patch( - "src.infrastructure.utils.progressive_summarization.count_tokens" + "src.shared.llm.summarization.progressive.count_tokens" ) as mock_count: mock_count.side_effect = mock_count_tokens # Mock get_max_context_tokens to prevent chunking with patch( - "src.infrastructure.utils.progressive_summarization." - "get_max_context_tokens" + "src.shared.llm.summarization.progressive.get_max_context_tokens" ) as mock_max_context: mock_max_context.return_value = ( 128000 # Large enough to not trigger chunking @@ -206,8 +203,7 @@ def mock_count_tokens(*args, **kwargs): # Mock perform_summary_pass with patch( - "src.infrastructure.utils.progressive_summarization." - "perform_summary_pass" + "src.shared.llm.summarization.progressive.perform_summary_pass" ) as mock_perform: async def mock_pass(text, llm, pass_number, target_tokens): @@ -254,22 +250,20 @@ def mock_count_tokens(*args, **kwargs): return result with patch( - "src.infrastructure.utils.progressive_summarization.count_tokens" + "src.shared.llm.summarization.progressive.count_tokens" ) as mock_count: mock_count.side_effect = mock_count_tokens # Mock get_max_context_tokens to prevent chunking with patch( - "src.infrastructure.utils.progressive_summarization." - "get_max_context_tokens" + "src.shared.llm.summarization.progressive.get_max_context_tokens" ) as mock_max_context: mock_max_context.return_value = ( 128000 # Large enough to not trigger chunking ) with patch( - "src.infrastructure.utils.progressive_summarization." - "perform_summary_pass" + "src.shared.llm.summarization.progressive.perform_summary_pass" ) as mock_perform: async def mock_pass(text, llm, pass_number, target_tokens): @@ -330,13 +324,12 @@ async def test_successful_chunk_summarization(self): mock_llm = MagicMock() with patch( - "src.infrastructure.utils.progressive_summarization.count_tokens" + "src.shared.llm.summarization.progressive.count_tokens" ) as mock_count: mock_count.side_effect = [5000, 3000] # input, output with patch( - "src.infrastructure.utils.progressive_summarization." - "LLMTextCompletionProgram" + "src.shared.llm.summarization.progressive.LLMTextCompletionProgram" ) as mock_program_class: mock_program = MagicMock() mock_program_class.from_defaults.return_value = mock_program @@ -368,18 +361,17 @@ async def test_chunk_summarization_with_fallback(self): mock_llm = MagicMock() with patch( - "src.infrastructure.utils.progressive_summarization.count_tokens" + "src.shared.llm.summarization.progressive.count_tokens" ) as mock_count: mock_count.side_effect = [5000, 2800] with patch( - "src.infrastructure.utils.progressive_summarization." - "LLMTextCompletionProgram" + "src.shared.llm.summarization.progressive.LLMTextCompletionProgram" ) as mock_program_class: mock_program_class.from_defaults.side_effect = Exception("LLM error") with patch( - "src.infrastructure.utils.progressive_summarization." + "src.shared.llm.summarization.progressive." "truncate_text_by_tokens" ) as mock_truncate: mock_truncate.return_value = "Truncated chunk" @@ -405,20 +397,19 @@ async def test_chunking_summarization(self): mock_llm = MagicMock() with patch( - "src.infrastructure.utils.progressive_summarization.count_tokens" + "src.shared.llm.summarization.progressive.count_tokens" ) as mock_count: # Original: 100k, combined chunks: 15k mock_count.side_effect = [100000, 15000] with patch( - "src.infrastructure.utils.progressive_summarization." - "chunk_text_by_tokens" + "src.shared.llm.summarization.progressive.chunk_text_by_tokens" ) as mock_chunk: # Simulate 3 chunks mock_chunk.return_value = ["chunk1", "chunk2", "chunk3"] with patch( - "src.infrastructure.utils.progressive_summarization.summarize_chunk" + "src.shared.llm.summarization.progressive.summarize_chunk" ) as mock_summarize_chunk: async def mock_chunk_summary(chunk, chunk_number, llm, **kwargs): @@ -453,19 +444,18 @@ async def test_chunking_within_target(self): mock_llm = MagicMock() with patch( - "src.infrastructure.utils.progressive_summarization.count_tokens" + "src.shared.llm.summarization.progressive.count_tokens" ) as mock_count: # Original: 100k, combined chunks: 8k (within 10k target) mock_count.side_effect = [100000, 8000] with patch( - "src.infrastructure.utils.progressive_summarization." - "chunk_text_by_tokens" + "src.shared.llm.summarization.progressive.chunk_text_by_tokens" ) as mock_chunk: mock_chunk.return_value = ["chunk1", "chunk2"] with patch( - "src.infrastructure.utils.progressive_summarization.summarize_chunk" + "src.shared.llm.summarization.progressive.summarize_chunk" ) as mock_summarize_chunk: async def mock_chunk_summary(chunk, chunk_number, llm, **kwargs): @@ -502,21 +492,20 @@ async def test_chunking_triggered(self): mock_llm = MagicMock() with patch( - "src.infrastructure.utils.progressive_summarization.count_tokens" + "src.shared.llm.summarization.progressive.count_tokens" ) as mock_count: # Original: 100k tokens mock_count.return_value = 100000 with patch( - "src.infrastructure.utils.progressive_summarization." - "get_max_context_tokens" + "src.shared.llm.summarization.progressive.get_max_context_tokens" ) as mock_max_context: # Context window: 128k, threshold (50%): 64k # 100k > 64k, so chunking should trigger mock_max_context.return_value = 128000 with patch( - "src.infrastructure.utils.progressive_summarization." + "src.shared.llm.summarization.progressive." "summarize_with_chunking" ) as mock_chunk_summarize: @@ -540,7 +529,7 @@ async def mock_chunking(*args, **kwargs): # Mock progressive passes (won't be called if # chunking reduces enough) with patch( - "src.infrastructure.utils.progressive_summarization." + "src.shared.llm.summarization.progressive." "perform_summary_pass" ) as mock_perform: @@ -582,13 +571,12 @@ async def test_successful_meeting_notes_summarization(self): meeting_text = "This is a long meeting with many discussions and decisions." with patch( - "src.infrastructure.utils.progressive_summarization." - "LLMTextCompletionProgram" + "src.shared.llm.summarization.progressive.LLMTextCompletionProgram" ) as mock_program_class: mock_program = MagicMock() mock_program_class.from_defaults.return_value = mock_program - from src.infrastructure.utils.progressive_summarization import ( + from src.shared.llm.summarization.progressive import ( MeetingNotesSummary, summarize_meeting_notes, ) @@ -620,18 +608,16 @@ async def test_meeting_notes_with_fallback(self): meeting_text = "Meeting notes that will fail to summarize." with patch( - "src.infrastructure.utils.progressive_summarization." - "LLMTextCompletionProgram" + "src.shared.llm.summarization.progressive.LLMTextCompletionProgram" ) as mock_program_class: mock_program_class.from_defaults.side_effect = Exception("LLM error") with patch( - "src.infrastructure.utils.progressive_summarization." - "truncate_text_by_tokens" + "src.shared.llm.summarization.progressive.truncate_text_by_tokens" ) as mock_truncate: mock_truncate.return_value = "Truncated meeting notes" - from src.infrastructure.utils.progressive_summarization import ( + from src.shared.llm.summarization.progressive import ( summarize_meeting_notes, ) @@ -654,7 +640,7 @@ async def test_empty_text_summarization(self): mock_llm = MagicMock() with patch( - "src.infrastructure.utils.progressive_summarization.count_tokens" + "src.shared.llm.summarization.progressive.count_tokens" ) as mock_count: mock_count.return_value = 0 @@ -677,7 +663,7 @@ async def test_very_small_text(self): small_text = "Just a few words." with patch( - "src.infrastructure.utils.progressive_summarization.count_tokens" + "src.shared.llm.summarization.progressive.count_tokens" ) as mock_count: mock_count.return_value = 5 # Very small @@ -707,19 +693,17 @@ def mock_count_tokens(*args, **kwargs): return result with patch( - "src.infrastructure.utils.progressive_summarization.count_tokens" + "src.shared.llm.summarization.progressive.count_tokens" ) as mock_count: mock_count.side_effect = mock_count_tokens with patch( - "src.infrastructure.utils.progressive_summarization." - "get_max_context_tokens" + "src.shared.llm.summarization.progressive.get_max_context_tokens" ) as mock_max_context: mock_max_context.return_value = 128000 with patch( - "src.infrastructure.utils.progressive_summarization." - "perform_summary_pass" + "src.shared.llm.summarization.progressive.perform_summary_pass" ) as mock_perform: async def mock_pass(*args, **kwargs): @@ -761,19 +745,17 @@ def mock_count_tokens(*args, **kwargs): return result with patch( - "src.infrastructure.utils.progressive_summarization.count_tokens" + "src.shared.llm.summarization.progressive.count_tokens" ) as mock_count: mock_count.side_effect = mock_count_tokens with patch( - "src.infrastructure.utils.progressive_summarization." - "get_max_context_tokens" + "src.shared.llm.summarization.progressive.get_max_context_tokens" ) as mock_max_context: mock_max_context.return_value = 128000 with patch( - "src.infrastructure.utils.progressive_summarization." - "perform_summary_pass" + "src.shared.llm.summarization.progressive.perform_summary_pass" ) as mock_perform: async def mock_pass(text, llm, pass_number, target_tokens): @@ -808,28 +790,25 @@ async def test_chunking_with_single_chunk(self): mock_llm = MagicMock() with patch( - "src.infrastructure.utils.progressive_summarization.count_tokens" + "src.shared.llm.summarization.progressive.count_tokens" ) as mock_count: # Original, in summarize_with_chunking, combined, after chunking # loop check, final mock_count.side_effect = [70000, 70000, 28000, 28000, 28000, 28000] with patch( - "src.infrastructure.utils.progressive_summarization." - "get_max_context_tokens" + "src.shared.llm.summarization.progressive.get_max_context_tokens" ) as mock_max_context: mock_max_context.return_value = 128000 # Threshold: 64k with patch( - "src.infrastructure.utils.progressive_summarization." - "chunk_text_by_tokens" + "src.shared.llm.summarization.progressive.chunk_text_by_tokens" ) as mock_chunk: # Single chunk mock_chunk.return_value = ["single chunk"] with patch( - "src.infrastructure.utils.progressive_summarization." - "summarize_chunk" + "src.shared.llm.summarization.progressive.summarize_chunk" ) as mock_summarize_chunk: async def mock_chunk_summary(*args, **kwargs): @@ -881,18 +860,17 @@ def mock_count_tokens(*args, **kwargs): return result with patch( - "src.infrastructure.utils.progressive_summarization.count_tokens" + "src.shared.llm.summarization.progressive.count_tokens" ) as mock_count: mock_count.side_effect = mock_count_tokens with patch( - "src.infrastructure.utils.progressive_summarization." - "get_max_context_tokens" + "src.shared.llm.summarization.progressive.get_max_context_tokens" ) as mock_max_context: mock_max_context.return_value = 128000 with patch( - "src.infrastructure.utils.progressive_summarization." + "src.shared.llm.summarization.progressive." "summarize_with_chunking" ) as mock_chunk_summarize: @@ -915,7 +893,7 @@ async def mock_chunking(*args, **kwargs): mock_chunk_summarize.side_effect = mock_chunking with patch( - "src.infrastructure.utils.progressive_summarization." + "src.shared.llm.summarization.progressive." "perform_summary_pass" ) as mock_perform: