Skip to content

Retriever cache #129

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
mhauskn-dr wants to merge 14 commits into
base: main
Choose a base branch
from
Open

Retriever cache #129

Show file tree
Hide file tree
Changes from 11 commits
Commits
Show all changes
14 commits
Select commit Hold shift + click to select a range
1d90e4b
Retriever cache
mhauskn-dr Jun 14, 2025
931f970
Updates
mhauskn-dr Jun 14, 2025
0e9c154
Cache updates
mhauskn-dr Jun 15, 2025
549016b
Add no-retriever studies
mhauskn-dr Jun 16, 2025
ec14d93
typo
mhauskn-dr Jun 16, 2025
efd6873
Add ray cache to indexes
mhauskn-dr Jun 16, 2025
b324524
Add option to enable/disable retriever caching
mhauskn-dr Jun 16, 2025
e2b174c
Major refactor
mhauskn-dr Jun 20, 2025
1244940
param_override
mhauskn-dr Jun 23, 2025
7f6e126
Remove no-retriever studies.
mhauskn-dr Jun 23, 2025
a18885c
ruff
mhauskn-dr Jun 23, 2025
3c1bbdd
Further updates
mhauskn-dr Jun 25, 2025
3db8110
Bugfix
mhauskn-dr Jun 26, 2025
acb2946
Change top-k defaults
mhauskn-dr Jun 26, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions syftr/configuration.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -133,6 +133,9 @@ class Paths(BaseModel):
tmp_dir / "huggingface"
)
index_cache: Annotated[Path, Field(validate_default=True)] = tmp_dir / "indexcache"
retrieval_cache: Annotated[Path, Field(validate_default=True)] = (
tmp_dir / "retrieval_cache"
)
onnx_dir: Annotated[Path, Field(validate_default=True)] = tmp_dir / "onnx"
sota_dir: Annotated[Path, Field(validate_default=True)] = data_dir / "sota"
lock_dir: Annotated[Path, Field(validate_default=True)] = tmp_dir / "syftr-locks"
Expand Down
14 changes: 8 additions & 6 deletions syftr/evaluation.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -371,15 +371,17 @@ async def aretrieve_pair(
flow: RetrieverFlow,
rate_limiter: AsyncLimiter,
raise_on_exception: bool | None = EVAL__RAISE_ON_EXCEPTION,
) -> T.Tuple[T.List[NodeWithScore] | None, float, Exception | None]:
) -> T.Tuple[
T.List[NodeWithScore] | None, float, T.List[LLMCallData], Exception | None
]:
"""Get flow's retrieved documents from an Q&A pair asynchronously."""
result, run_time, exception = await exception_catcher(
nodes, duration, call_data, exception = await exception_catcher(
func=flow.aretrieve,
return_values_on_exception=(None, np.nan),
raise_on_exception=raise_on_exception,
query=qa_pair.question,
)
return result, run_time, exception
return nodes, duration, call_data, exception


async def _aeval_retriever_pair(
Expand All @@ -392,8 +394,8 @@ async def _aeval_retriever_pair(
"""Evaluate retrieval performance on a single Q&A item."""
if not qa_pair.gold_evidence:
raise ValueError("QAPair gold_evidence is empty: %s", qa_pair)
retrieval_results, run_time, retrieval_exception = await aretrieve_pair(
qa_pair, flow, rate_limiter
retrieval_results, run_time, call_data, retrieval_exception = await aretrieve_pair(
qa_pair, flow, rate_limiter, raise_on_exception
)
if retrieval_results:
retrieved_contexts: T.List[str] = [
Expand All @@ -412,7 +414,7 @@ async def _aeval_retriever_pair(
run_time=run_time,
generation_exception=retrieval_exception,
evaluation_exception=None,
llm_call_data=[],
llm_call_data=call_data,
retriever_recall=result.score,
retriever_context_length=retrieved_contexts_length,
passing=result.passing,
Expand Down
Loading