Repository navigation
Expand file tree
/
Copy pathsettings.py
More file actions
138 lines (116 loc) · 4.67 KB
/
Copy pathsettings.py
File metadata and controls
138 lines (116 loc) · 4.67 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
"""Application settings and runtime validation."""
from __future__ import annotations
from functools import lru_cache
import logging
import os
from pathlib import Path
import shutil
from typing import Literal
from env_utils import ENV_FILE_PATH, read_env_file
from pydantic import BaseModel, ConfigDict, Field, model_validator
try:
from pydantic_settings import BaseSettings, SettingsConfigDict
except ImportError:
class BaseSettings(BaseModel):
"""Minimal fallback when pydantic-settings is unavailable."""
model_config = ConfigDict(extra="ignore")
def __init__(self, **data: object) -> None:
merged = read_env_file(ENV_FILE_PATH, lowercase_keys=True)
for field_name in self.__class__.model_fields:
env_name = field_name.upper()
if env_name in os.environ:
merged[field_name] = os.environ[env_name]
merged.update(data)
super().__init__(**merged)
def SettingsConfigDict(**kwargs: object) -> ConfigDict:
return ConfigDict(**kwargs)
class AppSettings(BaseSettings):
"""Shared typed settings for CLI, API, and worker processes."""
model_config = SettingsConfigDict(
env_file=".env.local",
env_file_encoding="utf-8",
extra="ignore",
)
app_env: Literal["development", "production", "test"] = "development"
redis_url: str = "redis://redis:6379/0"
enable_background_jobs: bool = False
artifact_root: Path = Field(default=Path("artifacts"))
max_agents_default: int = 2
frontend_origin: str | None = None
allow_vercel_preview_origins: bool = False
openai_api_key: str | None = None
gemini_api_key: str | None = None
groq_api_key: str | None = None
openai_model: str | None = None
openai_base_url: str | None = None
openrouter_site_url: str = "https://localhost"
openrouter_app_name: str = "Autonomous Data Engineer"
enable_semantic_feature_pruning: bool = False
agent_browser_bin: str = "agent-browser"
agent_browser_native: bool = True
agent_browser_cdp_url: str | None = None
browserbase_api_key: str | None = None
browserbase_project_id: str | None = None
browserbase_cdp_url: str | None = None
llm_tracing_backend: Literal["none", "langsmith", "phoenix"] = "none"
langsmith_tracing: bool = False
langsmith_api_key: str | None = None
langsmith_endpoint: str | None = None
langsmith_project: str = "web-scraper"
phoenix_collector_endpoint: str | None = None
phoenix_project_name: str = "web-scraper"
@model_validator(mode="after")
def _validate_partial_browserbase_config(self) -> "AppSettings":
if bool(self.browserbase_api_key) ^ bool(self.browserbase_project_id):
raise ValueError(
"BROWSERBASE_API_KEY and BROWSERBASE_PROJECT_ID must be provided together."
)
return self
@property
def jobs_dir(self) -> Path:
return self.artifact_root / "jobs"
@property
def has_llm_api_key(self) -> bool:
return any(
value
for value in (
self.openai_api_key,
self.gemini_api_key,
self.groq_api_key,
)
)
def validate_for_service(self) -> None:
if not self.has_llm_api_key:
logging.getLogger(__name__).warning(
"No LLM API key configured. Service will still start, but LLM-backed jobs may fail."
)
if "/" in self.agent_browser_bin:
browser_available = Path(self.agent_browser_bin).exists()
else:
browser_available = shutil.which(self.agent_browser_bin) is not None
if not browser_available:
logging.getLogger(__name__).warning(
"AGENT_BROWSER_BIN is not installed or not on PATH. Browser fallback may be unavailable."
)
@property
def cors_origins(self) -> list[str]:
origins = [
"http://localhost:5173",
"http://127.0.0.1:5173",
"http://localhost:3000",
"http://127.0.0.1:3000",
]
if self.frontend_origin and self.frontend_origin.strip():
origins.append(self.frontend_origin.strip().rstrip("/"))
return list(dict.fromkeys(origins))
@property
def cors_origin_regex(self) -> str | None:
if self.allow_vercel_preview_origins or self.app_env != "production":
return r"https://.*\.vercel\.app"
return None
@property
def running_on_vercel(self) -> bool:
return os.environ.get("VERCEL") == "1" or bool(os.environ.get("VERCEL_ENV"))
@lru_cache(maxsize=1)
def get_settings() -> AppSettings:
return AppSettings()