diff --git a/.gitignore b/.gitignore index 57670195..0cb2f2e9 100644 --- a/.gitignore +++ b/.gitignore @@ -209,6 +209,7 @@ studies/box* studies/*example1--small-study--drdocs_hf.yaml studies/silver* studies/test* +studies/rise* notebooks/.nfs* @@ -216,3 +217,5 @@ notebooks/.nfs* *private* /flowgen/data/nltk-data .nfs* +data.* +/datasets \ No newline at end of file diff --git a/notebooks/create_dataset.ipynb b/notebooks/create_dataset.ipynb new file mode 100644 index 00000000..46f32d3d --- /dev/null +++ b/notebooks/create_dataset.ipynb @@ -0,0 +1,364 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "0", + "metadata": {}, + "source": [ + "# QA Dataset Generation\n", + "Given a raw text, the notebook helps to generate a custom HuggingFace QA dataset based on the given information." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1", + "metadata": {}, + "outputs": [], + "source": [ + "%reload_ext autoreload\n", + "%autoreload 2\n", + "\n", + "from IPython.core import ultratb\n", + "\n", + "ultratb.VerboseTB.tb_highlight = \"bg:#3e0054\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "\n", + "if not os.getcwd().endswith(\"syftr\"):\n", + " os.chdir(os.path.dirname(os.getcwd()))\n", + " print(f\"Changed working directory to: {os.getcwd()}\")\n", + "\n", + "from syftr.configuration import cfg" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3", + "metadata": {}, + "outputs": [], + "source": [ + "DATA_FILEPATH = \"data.md\" # Path to the raw text file\n", + "CHUNK_SIZE = 400 # Size of each text chunk\n", + "LLMS = [ # adjust to LLMs you want to use for question generation\n", + " \"gpt-4o-mini\",\n", + " \"Qwen/Qwen3-32B\",\n", + " \"google/gemma-3-27b-it\",\n", + "] # We randomly select one of the provided LLMs per chunk\n", + "NUM_PARALLEL = 50 # Number of parallel processes to use for chunk processing\n", + "# -------------------------------------------------------------------------------------------\n", + " # Add instructions that are specific to your QA generation task\n", + "CUSTOM_QA_INSTRUCTIONS = None\n", + "\n", + "assert CUSTOM_QA_INSTRUCTIONS, \"Please provide custom instructions for the QA generation.\"\n", + "\n", + "# Provide a valid dataset name\n", + "DATASET_NAME = None\n", + "assert DATASET_NAME, \"Please set the DATASET_NAME variable to a valid dataset name.\"\n", + "# -------------------------------------------------------------------------------------------\n", + "\n", + "DATASET_IS_PRIVATE = True # Set to False if you want to share the dataset publicly\n", + "\n", + "HF_DATASET_NAME = f\"DataRobot-Research/{DATASET_NAME}\" # Adjust name of the dataset on Hugging Face Hub\n", + "HF_TOKEN = cfg.hf_datasets.api_key.get_secret_value() # Get Hugging Face token from configuration\n", + "\n", + "assert HF_TOKEN, \"Please set the HF_TOKEN environment variable with your Hugging Face token.\"\n", + "\n", + "print(f\"Using Hugging Face token: {HF_TOKEN[:4]}...{HF_TOKEN[-4:]}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4", + "metadata": {}, + "outputs": [], + "source": [ + "def load_text(file_path: str) -> str:\n", + " with open(file_path, \"r\", encoding=\"utf-8\") as file:\n", + " return file.read()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5", + "metadata": {}, + "outputs": [], + "source": [ + "raw_text = load_text(DATA_FILEPATH)\n", + "print(f\"Loaded {len(raw_text)} characters from {DATA_FILEPATH}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6", + "metadata": {}, + "outputs": [], + "source": [ + "def chunk_text(text: str, chunk_size: int = 1000) -> list:\n", + " return [text[i : i + chunk_size] for i in range(0, len(text), chunk_size)]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7", + "metadata": {}, + "outputs": [], + "source": [ + "chunks = chunk_text(raw_text, CHUNK_SIZE)\n", + "print(f\"Created {len(chunks)} chunks of size {CHUNK_SIZE} characters.\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8", + "metadata": {}, + "outputs": [], + "source": [ + "from tenacity import retry, stop_after_attempt, wait_fixed\n", + "from syftr.llm import get_llm\n", + "\n", + "\n", + "@retry(stop=stop_after_attempt(5), wait=wait_fixed(2))\n", + "def generate(prompt: str, llm_name: str, **kwargs):\n", + " llm = get_llm(llm_name)\n", + " assert llm is not None, f\"LLM {llm_name} not found.\"\n", + " response = llm.complete(prompt=prompt, **kwargs)\n", + " return response.text" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9", + "metadata": {}, + "outputs": [], + "source": [ + "def generate_qa_from_chunk(\n", + " chunk: str, llm_name: str, **kwargs\n", + ") -> str:\n", + " prompt = f\"\"\"Generate a question and answer based on the text below. Make sure to not use special formatting, like markdown, but formulate the question and the answer in a plan text format. Start with the question followed by the answer. The question should be clear and concise, and the answer should be informative and directly related to the question, for instance,\n", + " \n", + " Question: Who is in charge of the project SuperGold?\n", + "\n", + " Answer: The project is led by Dr. Jane Smith.\n", + "\n", + " Note that the question should always be specific, for instance, don't use generic terms like \"the text\" but always be specific about what you mean and use concrete names whereever possible. Same with images and tables: make sure you can specify which table or image your question is about or do not ask this question. The answer should be a direct response to the question, providing relevant information from the text chunk provided below.\n", + " If you cannot generate a question and answer based on the text, return an empty string.\n", + " Moreover, follow these custom instructions: \\n\\n{CUSTOM_QA_INSTRUCTIONS}\\n\\n\n", + "\n", + " Chunk: \\n\\n{chunk}\"\"\"\n", + " response = generate(prompt, llm_name, **kwargs)\n", + " return response.strip()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "10", + "metadata": {}, + "outputs": [], + "source": [ + "import hashlib\n", + "import re\n", + "import typing as T\n", + "\n", + "def parse_qa_pairs(text: str, llm_name: str | None = None, chunk: str | None = None) -> T.List[T.Dict[str, str]]: \n", + " pattern = r\"Question:\\s*(.*?)\\s*Answer:\\s*(.*)\"\n", + " matches = re.findall(pattern, text, re.DOTALL)\n", + " parsed_pairs = []\n", + " for question, answer in matches:\n", + " pair = {\n", + " \"id\": hashlib.md5(f\"{question.strip()}_{answer.strip()}\".encode()).hexdigest(),\n", + " \"question\": question.strip(),\n", + " \"answer\": answer.strip(),\n", + " }\n", + " if llm_name:\n", + " pair[\"qtype\"] = llm_name\n", + " if chunk:\n", + " pair[\"gold_evidence\"] = [chunk.strip()]\n", + " parsed_pairs.append(pair)\n", + " return parsed_pairs" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "11", + "metadata": {}, + "outputs": [], + "source": [ + "import random\n", + "from concurrent.futures import ThreadPoolExecutor, as_completed\n", + "\n", + "\n", + "def get_qa_pairs_from_chunks(chunks: T.List[str]) -> T.List[T.Dict[str, str]]:\n", + " qa_pairs = []\n", + "\n", + " def _gen(chunk: str) -> T.List[T.Dict[str, str]]:\n", + " llm_name = random.choice(LLMS)\n", + " generated_text = generate_qa_from_chunk(chunk, llm_name, max_tokens=1024, temperature=0.7)\n", + " return parse_qa_pairs(generated_text, llm_name, chunk)\n", + "\n", + " with ThreadPoolExecutor(max_workers=NUM_PARALLEL) as executor:\n", + " futures = [executor.submit(_gen, chunk) for chunk in chunks]\n", + " results = [future.result() for future in futures]\n", + " for pairs in results:\n", + " qa_pairs.extend(pairs)\n", + " return qa_pairs" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "12", + "metadata": {}, + "outputs": [], + "source": [ + "qa_pairs = get_qa_pairs_from_chunks(chunks)\n", + "print(f\"Generated {len(qa_pairs)} Q&A pairs from {len(chunks)} chunks.\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "13", + "metadata": {}, + "outputs": [], + "source": [ + "for pair in qa_pairs:\n", + " print(\"-\"* 40)\n", + " print(f\"LLM: {pair.get('qtype', 'Unknown')}\\nQuestion: {pair['question']}\\nAnswer: {pair['answer']}\\nEvidence: {pair.get('gold_evidence', 'N/A')}\")" + ] + }, + { + "cell_type": "markdown", + "id": "14", + "metadata": {}, + "source": [ + "**Adjust the parameters to make a custom split based on your needs and the amount of data generated.**" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "15", + "metadata": {}, + "outputs": [], + "source": [ + "import datasets\n", + "\n", + "\n", + "def get_context(chunks: T.List[str]) -> str:\n", + " full_context = \"\\n\".join(chunks)\n", + " return full_context\n", + "\n", + "def prepare_hf_data(\n", + " qa_pairs: T.List[T.Dict[str, str]], \n", + " chunks: T.List[str] | None = None, \n", + " all_grounding_data_for_each_partition = True,\n", + ") -> T.Tuple[datasets.DatasetDict, datasets.DatasetDict]:\n", + " if all_grounding_data_for_each_partition:\n", + " grounding_data_train = chunks\n", + " grounding_data_test = chunks\n", + " grounding_data_holdout = chunks\n", + " grounding_data_sample = chunks[:5]\n", + " elif chunks:\n", + " grounding_data_train = get_context(chunks[:100])\n", + " grounding_data_test = get_context(chunks[100:200])\n", + " grounding_data_holdout = get_context(chunks[200:])\n", + " grounding_data_sample = get_context(chunks[:5])\n", + " else:\n", + " raise ValueError(\"Either chunks or raw_text must be provided.\")\n", + " \n", + " qa_data = datasets.DatasetDict(\n", + " {\n", + " \"train\": datasets.Dataset.from_list(qa_pairs[:100]),\n", + " \"test\": datasets.Dataset.from_list(qa_pairs[100:200]),\n", + " \"holdout\": datasets.Dataset.from_list(qa_pairs[200:]),\n", + " \"sample\": datasets.Dataset.from_list(qa_pairs[:5]), # for quick testing\n", + " }\n", + " )\n", + " grounding_data = datasets.DatasetDict(\n", + " {\n", + " \"train\": datasets.Dataset.from_dict({\"text\": grounding_data_train}),\n", + " \"test\": datasets.Dataset.from_dict({\"text\": grounding_data_test}),\n", + " \"holdout\": datasets.Dataset.from_dict({\"text\": grounding_data_holdout}),\n", + " \"sample\": datasets.Dataset.from_dict({\"text\": grounding_data_sample}),\n", + " }\n", + " )\n", + " return qa_data, grounding_data" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "16", + "metadata": {}, + "outputs": [], + "source": [ + "qa_data, grounding_data = prepare_hf_data(qa_pairs, chunks=chunks)\n", + "\n", + "qa_data.push_to_hub(\n", + " repo_id=HF_DATASET_NAME, \n", + " data_dir=\"examples\",\n", + " private=DATASET_IS_PRIVATE, \n", + " token=HF_TOKEN,\n", + " config_name=\"qa\"\n", + ")\n", + "print(f\"QA data pushed to Hugging Face Hub.\")\n", + "\n", + "grounding_data.push_to_hub(\n", + " repo_id=HF_DATASET_NAME,\n", + " data_dir=\"grounding_data\",\n", + " private=DATASET_IS_PRIVATE,\n", + " token=HF_TOKEN,\n", + " config_name=\"grounding\"\n", + ")\n", + "print(f\"Grounding data pushed to Hugging Face Hub.\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "17", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "syftr", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.7" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks/create_dataset_gemini_synthetic.ipynb b/notebooks/create_dataset_gemini_synthetic.ipynb new file mode 100644 index 00000000..c01053cd --- /dev/null +++ b/notebooks/create_dataset_gemini_synthetic.ipynb @@ -0,0 +1,248 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "0", + "metadata": {}, + "source": [ + "# QA Dataset Generation\n", + "Given a raw text, the notebook helps to generate a custom HuggingFace QA dataset based on the given information." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1", + "metadata": {}, + "outputs": [], + "source": [ + "%reload_ext autoreload\n", + "%autoreload 2\n", + "\n", + "from IPython.core import ultratb\n", + "\n", + "ultratb.VerboseTB.tb_highlight = \"bg:#3e0054\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "\n", + "if not os.getcwd().endswith(\"syftr\"):\n", + " os.chdir(os.path.dirname(os.getcwd()))\n", + " print(f\"Changed working directory to: {os.getcwd()}\")\n", + "\n", + "from syftr.configuration import cfg" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3", + "metadata": {}, + "outputs": [], + "source": [ + "DATA_FILEPATH = \"/Users/debadeepta.dey/datasets/barclays/rise-insights-report-making-data-count-with-ai-DIGITAL.md\" # Path to the raw text file\n", + "QA_PAIRS_FILEPATH = \"/Users/debadeepta.dey/datasets/barclays/rise-insights-report-making-data-count-with-ai-DIGITAL-qapairs.json\" # Path to the QA pairs file\n", + "CHUNK_SIZE = 8148 # Size of each text chunk\n", + "\n", + "# Provide a valid dataset name\n", + "DATASET_NAME = \"making-data-count-with-ai-2\"\n", + "assert DATASET_NAME, \"Please set the DATASET_NAME variable to a valid dataset name.\"\n", + "# -------------------------------------------------------------------------------------------\n", + "\n", + "DATASET_IS_PRIVATE = True # Set to False if you want to share the dataset publicly\n", + "\n", + "HF_DATASET_NAME = f\"DataRobot-Research/{DATASET_NAME}\" # Adjust name of the dataset on Hugging Face Hub\n", + "HF_TOKEN = cfg.hf_datasets.api_key.get_secret_value() # Get Hugging Face token from configuration\n", + "\n", + "assert HF_TOKEN, \"Please set the HF_TOKEN environment variable with your Hugging Face token.\"\n", + "\n", + "print(f\"Using Hugging Face token: {HF_TOKEN[:4]}...{HF_TOKEN[-4:]}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4", + "metadata": {}, + "outputs": [], + "source": [ + "def load_text(file_path: str) -> str:\n", + " with open(file_path, \"r\", encoding=\"utf-8\") as file:\n", + " return file.read()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5", + "metadata": {}, + "outputs": [], + "source": [ + "raw_text = load_text(DATA_FILEPATH)\n", + "print(f\"Loaded {len(raw_text)} characters from {DATA_FILEPATH}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6", + "metadata": {}, + "outputs": [], + "source": [ + "def chunk_text(text: str, chunk_size: int = 1000) -> list:\n", + " return [text[i : i + chunk_size] for i in range(0, len(text), chunk_size)]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7", + "metadata": {}, + "outputs": [], + "source": [ + "chunks = chunk_text(raw_text, CHUNK_SIZE)\n", + "print(f\"Created {len(chunks)} chunks of size {CHUNK_SIZE} characters.\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8", + "metadata": {}, + "outputs": [], + "source": [ + "# Load QA pairs from the JSON file\n", + "import json\n", + "def load_qa_pairs(file_path: str) -> list:\n", + " with open(file_path, \"r\", encoding=\"utf-8\") as file:\n", + " return json.load(file)\n", + "\n", + "qa_pairs = load_qa_pairs(QA_PAIRS_FILEPATH)\n", + "print(qa_pairs[:3])" + ] + }, + { + "cell_type": "markdown", + "id": "9", + "metadata": {}, + "source": [ + "**Adjust the parameters to make a custom split based on your needs and the amount of data generated.**" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "10", + "metadata": {}, + "outputs": [], + "source": [ + "import datasets\n", + "import typing as T\n", + "\n", + "def get_context(chunks: T.List[str]) -> str:\n", + " full_context = \"\\n\".join(chunks)\n", + " return full_context\n", + "\n", + "def prepare_hf_data(\n", + " qa_pairs: T.List[T.Dict[str, str]], \n", + " chunks: T.List[str] | None = None, \n", + " all_grounding_data_for_each_partition = True,\n", + ") -> T.Tuple[datasets.DatasetDict, datasets.DatasetDict]:\n", + " if all_grounding_data_for_each_partition:\n", + " grounding_data_train = chunks\n", + " grounding_data_test = chunks\n", + " grounding_data_holdout = chunks\n", + " grounding_data_sample = chunks[:5]\n", + " elif chunks:\n", + " grounding_data_train = get_context(chunks[:100])\n", + " grounding_data_test = get_context(chunks[100:200])\n", + " grounding_data_holdout = get_context(chunks[200:])\n", + " grounding_data_sample = get_context(chunks[:5])\n", + " else:\n", + " raise ValueError(\"Either chunks or raw_text must be provided.\")\n", + " \n", + " qa_data = datasets.DatasetDict(\n", + " {\n", + " \"train\": datasets.Dataset.from_list(qa_pairs[:50]),\n", + " \"test\": datasets.Dataset.from_list(qa_pairs[50:180]),\n", + " \"holdout\": datasets.Dataset.from_list(qa_pairs[180:]),\n", + " \"sample\": datasets.Dataset.from_list(qa_pairs[:5]), # for quick testing\n", + " }\n", + " )\n", + " grounding_data = datasets.DatasetDict(\n", + " {\n", + " \"train\": datasets.Dataset.from_dict({\"text\": grounding_data_train}),\n", + " \"test\": datasets.Dataset.from_dict({\"text\": grounding_data_test}),\n", + " \"holdout\": datasets.Dataset.from_dict({\"text\": grounding_data_holdout}),\n", + " \"sample\": datasets.Dataset.from_dict({\"text\": grounding_data_sample}),\n", + " }\n", + " )\n", + " return qa_data, grounding_data" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "11", + "metadata": {}, + "outputs": [], + "source": [ + "qa_data, grounding_data = prepare_hf_data(qa_pairs, chunks=chunks)\n", + "\n", + "qa_data.push_to_hub(\n", + " repo_id=HF_DATASET_NAME, \n", + " data_dir=\"examples\",\n", + " private=DATASET_IS_PRIVATE, \n", + " token=HF_TOKEN,\n", + " config_name=\"qa\"\n", + ")\n", + "print(f\"QA data pushed to Hugging Face Hub.\")\n", + "\n", + "grounding_data.push_to_hub(\n", + " repo_id=HF_DATASET_NAME,\n", + " data_dir=\"grounding_data\",\n", + " private=DATASET_IS_PRIVATE,\n", + " token=HF_TOKEN,\n", + " config_name=\"grounding\"\n", + ")\n", + "print(f\"Grounding data pushed to Hugging Face Hub.\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "12", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "syftr", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.7" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks/insights.ipynb b/notebooks/insights.ipynb index 3a42208a..03f11ea9 100644 --- a/notebooks/insights.ipynb +++ b/notebooks/insights.ipynb @@ -315,19 +315,22 @@ " # \"cerebras4--rag-and-agents-local-only--phantomwikiv050_hf--depth_20_size_10000_seed_3\",\n", "\n", " # cost optimization\n", - " \"cerebras5--rag-and-agents-cerebras-only--financebench_hf\",\n", - " \"cerebras5--rag-and-agents-local-only--financebench_hf\",\n", - " \"cerebras5--rag-and-agents-cerebras-only--phantomwikiv050_hf--depth_20_size_10000_seed_3\",\n", - " \"cerebras5--rag-and-agents-local-only--phantomwikiv050_hf--depth_20_size_10000_seed_3\",\n", + " # \"cerebras5--rag-and-agents-cerebras-only--financebench_hf\",\n", + " # \"cerebras5--rag-and-agents-local-only--financebench_hf\",\n", + " # \"cerebras5--rag-and-agents-cerebras-only--phantomwikiv050_hf--depth_20_size_10000_seed_3\",\n", + " # \"cerebras5--rag-and-agents-local-only--phantomwikiv050_hf--depth_20_size_10000_seed_3\",\n", "\n", " # \".*example1--small-study--drdocs_hf\",\n", " # \"test.*\",\n", + " # \"rise4.*\",\n", + " # \"rise5.*\",\n", + " \"rise6.*\",\n", " ],\n", " exclude_regex=[\n", " # \".*example.*\",\n", " # \".*_prompt_optimization.*\",\n", - " \".*pony.*\",\n", - " \".*psychology.*\",\n", + " # \".*pony.*\",\n", + " # \".*psychology.*\",\n", " ],\n", " focus_regex=[\n", " # \".*infinitebench.*\",\n", @@ -341,54 +344,13 @@ " # \".*_prompt_optimization.*\", \n", " # \".*retriever.*\",\n", " # \"cerebras4--rag-and-agents-local-only--phantomwikiv050_hf--depth_20_size_10000_seed_3\",\n", + " \"ris6.*\",\n", " \".*\", # use first matching study\n", " ],\n", " reset_cache=True,\n", " titles={\n", - " \"rank0--rag-and-agents--financebench_hf\": \"FinanceBench (SB0)\",\n", - " \"rank1--rag-and-agents--bright_hf\": \"Bright (SB1)\",\n", - " \"rank1--rag-and-agents--crag_hf-music\": \"CRAG Music (SB1)\",\n", - " \"rank1--rag-and-agents--crag_hf-sports\": \"CRAG Sports (SB1)\",\n", - " \"rank1--rag-and-agents--drdocs_hf\": \"DrDocs (SB1)\",\n", - " \"rank1--rag-and-agents--financebench_hf\": \"FinanceBench (SB1)\",\n", - " \"rank1--rag-and-agents--hotpotqa_hf-train_hard\": \"HotpotQA Train Hard (SB1)\",\n", - " \"rank1--rag-and-agents--infinitebench_hf\": \"InfiniteBench (SB1)\",\n", - " \"rank1--rag-and-agents--multihoprag_hf\": \"MultiHopRAG (SB1)\",\n", - " \"rank1--rag-and-agents--phantomwikiv050_hf\": \"PhantomWiki v0.50 (SB1)\",\n", - " \"rank2--rag-and-agents--bright_hf\": \"Bright (SB2)\",\n", - " \"rank2--rag-and-agents--crag_hf-music\": \"CRAG Music (SB2)\",\n", - " \"rank2--rag-and-agents--crag_hf-sports\": \"CRAG Sports (SB2)\",\n", - " \"rank2--rag-and-agents--drdocs_hf\": \"DrDocs (SB2)\",\n", - " \"rank2--rag-and-agents--financebench_hf\": \"FinanceBench (SB2)\",\n", - " \"rank2--rag-and-agents--hotpotqa_hf-train_hard\": \"HotpotQA Train Hard (SB2)\",\n", - " \"rank2--rag-and-agents--infinitebench_hf\": \"InfiniteBench (SB2)\",\n", - " \"rank2--rag-and-agents--multihoprag_hf\": \"MultiHopRAG (SB2)\",\n", - " \"rank2--rag-and-agents--phantomwikiv050_hf\": \"PhantomWiki v0.50 (SB2)\",\n", - " 'rank3--rag-and-agents--bright_hf--earth_science': \"Bright Earth Science\",\n", - " 'rank3--rag-and-agents--bright_hf--economics': \"Bright Economics\",\n", - " 'rank3--rag-and-agents--bright_hf--pony': \"Bright Pony\",\n", - " 'rank3--rag-and-agents--bright_hf--psychology': \"Bright Psychology\",\n", - " 'rank3--rag-and-agents--bright_hf--robotics': \"Bright Robotics\",\n", - " 'rank3--rag-and-agents--bright_hf--stackoverflow': \"Bright Stackoverflow\",\n", - " 'rank3--rag-and-agents--bright_hf--sustainable_living': \"Bright Sustainable Living\",\n", - " \"box1--global--financebench_hf\": \"FinanceBench (Global Optimization)\",\n", - " \"box1--global--infinitebench_hf--longbook_qa_eng\": \"InfiniteBench (Global Optimization)\",\n", - " \"box1--global--phantomwikiv050_hf--depth_20_size_10000_seed_3\": \"PhontomWiki (Global Optimization)\",\n", - " \"box1--retriever--financebench_hf\": \"FinanceBench (Block Optimization)\",\n", - " \"box1--retriever--infinitebench_hf--longbook_qa_eng\": \"InfiniteBench (Block Optimization)\",\n", - " \"box1--retriever--phantomwikiv050_hf--depth_20_size_10000_seed_3\": \"PantomWiki (Block Optimization)\",\n", - " \"cerebras2--mix-with-local--financebench_hf\": \"FinanceBench Cerebras Mix with Local (RAG and Agents)\",\n", - " \"cerebras2--mix-with-local--phantomwikiv050_hf--depth_20_size_10000_seed_3\": \"PhantomWiki Cerebras Mix with Local (RAG and Agents)\",\n", - " \"cerebras3--agents-only--financebench_hf\": \"FinanceBench Cerebras Mix with Local (Agents Only)\",\n", - " \"cerebras3--agents-only--phantomwikiv050_hf--depth_20_size_10000_seed_3\": \"PhantomWiki Cerebras Mix with Local (Agents Only)\",\n", - " \"cerebras4--rag-and-agents-cerebras-only--financebench_hf\": \"FinanceBench (Cerebras RAG and Agents)\",\n", - " \"cerebras4--rag-and-agents-local-only--financebench_hf\": \"FinanceBench (Local RAG and Agents)\",\n", - " \"cerebras4--rag-and-agents-cerebras-only--phantomwikiv050_hf--depth_20_size_10000_seed_3\": \"PhantomWiki (Cerebras RAG and Agents)\",\n", - " \"cerebras4--rag-and-agents-local-only--phantomwikiv050_hf--depth_20_size_10000_seed_3\": \"PhantomWiki (Local RAG and Agents)\",\n", - " \"cerebras5--rag-and-agents-cerebras-only--financebench_hf\": \"FinanceBench (Cerebras RAG and Agents)\",\n", - " \"cerebras5--rag-and-agents-local-only--financebench_hf\": \"FinanceBench (Local RAG and Agents)\",\n", - " \"cerebras5--rag-and-agents-cerebras-only--phantomwikiv050_hf--depth_20_size_10000_seed_3\": \"PhantomWiki (Cerebras RAG and Agents)\",\n", - " \"cerebras5--rag-and-agents-local-only--phantomwikiv050_hf--depth_20_size_10000_seed_3\": \"PhantomWiki (Local RAG and Agents)\",\n", + " \"rise4--rag-and-agents--rise_insights_hf\": \"RiseInsights (GPT-4o-mini Judge)\",\n", + " \"rise5--rag-and-agents--rise_insights_hf\": \"RiseInsights\",\n", " },\n", " insights_prefix=\"\", # the name prefix for exported figures\n", ")\n", diff --git a/notebooks/silver_bullets.ipynb b/notebooks/silver_bullets.ipynb index 477f514b..75e42224 100644 --- a/notebooks/silver_bullets.ipynb +++ b/notebooks/silver_bullets.ipynb @@ -19,7 +19,7 @@ "\n", "from IPython.core import ultratb\n", "\n", - "ultratb.VerboseTB._tb_highlight = \"bg:#3e0054\"" + "ultratb.VerboseTB.tb_highlight = \"bg:#3e0054\"" ] }, { diff --git a/syftr/configuration.py b/syftr/configuration.py index 3edd177b..738c78d1 100644 --- a/syftr/configuration.py +++ b/syftr/configuration.py @@ -138,6 +138,12 @@ class Paths(BaseModel): lock_dir: Annotated[Path, Field(validate_default=True)] = tmp_dir / "syftr-locks" nltk_dir: Annotated[Path, Field(validate_default=True)] = tmp_dir / "nltk-data" sqlite_dir: Annotated[Path, Field(validate_default=True)] = Path.home() / ".syftr" + grounding_dir: Annotated[Path, Field(validate_default=True)] = ( + REPO_ROOT / "grounding_data" + ) + rise_insights_grounding_data: Annotated[Path, Field(validate_default=True)] = ( + grounding_dir / "rise_insights.md" + ) @property def templates_without_context(self) -> Path: @@ -332,6 +338,10 @@ class HFEmbeddings(BaseModel, APIKeySerializationMixin): } +class HFDatasets(BaseModel, APIKeySerializationMixin): + api_key: SecretStr = SecretStr("NOT SET") + + class AzureOAI(BaseModel, APIKeySerializationMixin): # Use cfg.azure_oai.api_key.get_secret_value() to get value api_key: SecretStr = SecretStr("NOT SET") @@ -674,6 +684,7 @@ class Settings(BaseSettings): aws: AWS = AWS() hf_embeddings: HFEmbeddings = HFEmbeddings() + hf_datasets: HFDatasets = HFDatasets() azure_inference_llama33: AzureInferenceLlama33 = AzureInferenceLlama33() azure_inference_mistral: AzureInferenceMistral = AzureInferenceMistral() azure_inference_phi4: AzureInferencePhi4 = AzureInferencePhi4() diff --git a/syftr/plotting/insights.py b/syftr/plotting/insights.py index deb48af7..604f8cd7 100644 --- a/syftr/plotting/insights.py +++ b/syftr/plotting/insights.py @@ -22,7 +22,7 @@ from syftr.configuration import cfg from syftr.helpers import is_numeric -from syftr.llm import AZURE_GPT4O_STD +from syftr.llm import AZURE_GPT4O_MINI from syftr.studies import get_response_synthesizer_llm, get_template_name SHOW_TITLE = False @@ -560,7 +560,7 @@ def generate(prompt): cache_key = ("generate", prompt) if cache_key in CACHE: return CACHE[cache_key] - response = AZURE_GPT4O_STD.complete(prompt=prompt, temperature=0) + response = AZURE_GPT4O_MINI.complete(prompt=prompt, temperature=0) CACHE.set(cache_key, response.text, expire=60 * 60 * 24 * 7) return response.text @@ -691,8 +691,9 @@ def generate_trial_description_table(df): @log_function_call def style_pareto_table(df_pareto_descriptions, is_cost): + obj2 = "Cost" if is_cost else "Latency" df_pareto_descriptions = df_pareto_descriptions[ - ["Accuracy", "Latency", "Title", "Description"] + ["Accuracy", obj2, "Title", "Description"] ].copy() if is_cost: @@ -1962,7 +1963,13 @@ def param_pareto_plot(df: pd.DataFrame, study_name, param_col, titles=None): else: df_pareto["Title"] = "" plot_pareto_plot( - df_pareto, study_name, is_cost, df_trials, ax=axes[1], show_title=SHOW_TITLE + df_pareto, + study_name, + is_cost, + df_trials, + ax=axes[1], + titles=titles, + show_title=SHOW_TITLE, ) if SHOW_TITLE: diff --git a/syftr/scripts/experiments/rise.py b/syftr/scripts/experiments/rise.py new file mode 100644 index 00000000..88cd01aa --- /dev/null +++ b/syftr/scripts/experiments/rise.py @@ -0,0 +1,321 @@ +import argparse +import asyncio +import json +import time +import typing as T + +from syftr.configuration import cfg +from syftr.experiments import iter_all_job_logs +from syftr.helpers import get_flows_from_trials +from syftr.logger import logger +from syftr.optimization import user_confirm_delete +from syftr.optuna_helper import ( + get_completed_trials, + get_pareto_flows, +) +from syftr.ray.submit import get_client, start_study +from syftr.storage import RiseInsightsHF, SyftrQADataset +from syftr.studies import ( # noqa + DEFAULT_LLMS, + LOCAL_EMBEDDING_MODELS, + LOCAL_LLMS, + Block, + CritiqueRAGAgent, + Evaluation, + FewShotRetriever, + Hyde, + LATSRagAgent, + OptimizationConfig, + QueryDecomposition, + ReactRAGAgent, + Reranker, + Retriever, + SearchSpace, + Splitter, + StudyConfig, + SubQuestionRAGAgent, + TimeoutConfig, + TopK, + TransferLearningConfig, +) +from syftr.studyconfig_helper import build_configs + +# ------------------------------------------------------- +PREFIX = "rise" # this three parameters +BENCH_NUM = 6 # are used to name +RUN_NAME = "rag-and-agents" +# ------------------------------------------------------- +NUM_TRIALS = 2000 # total number of optimization trials per submission +MAX_CONCURRENT_TRIALS = 50 +NUM_EVAL_SAMPLES = 100 +REUSE_STUDY = False # WARNING: if set to False, exsting studies will be deleted! +RECREATE_STUDY = ( + True # WARNING: do not use with simultaneous runs using the same study! +) +EVAL_MODE: T.Literal["single", "random", "consensus"] = "random" +DRY_RUN = False # a dry run will not submit jobs but create the study configs +EMBEDDING_MAX_TIME = 3600 * 8 +MINUTES_BEFORE_NEXT_SUBMISSION = 1 + +# To seed with silver bullets, you first create the input file with the silver_bullets.ipynb notebook +CUSTOM_BASELINES = None # "pareto", "all", "silver", None +# CUSTOM_BASELINES = "silver" # "pareto", "all", "silver", None +OBJ2_NAME = "llm_cost_mean" # "p80_time", "llm_cost_mean", "retriever_context_length" +# ------------------------------------------------------- +CUSTOM_BASELINES = None # "pareto", "all", "silver", None +BASELINES_BATCH_SIZE = 100 # we require batching of baselines to avoid Ray OOM issues +BASELINES_START = 0 # you can restrict the number of baselines ... +BASELINES_END = 100 # ... to start with here to avoid OOM issues +# ------------------------------------------------------- +BASELINE_STUDIES: T.List[str] = [] + +BLOCKS = [ + Block( + name="global", + num_trials=NUM_TRIALS, + components=[ + "rag_retriever", + "splitter", + "additional_context", + "few_shot_retriever", + "hyde", + "critique_rag_agent", + "lats_rag_agent", + "react_rag_agent", + "rag_mode", + "reranker", + "response_synthesizer_llm", + "sub_question_rag", + "template_name", + ], + ), +] + +BASELINES = [] +if CUSTOM_BASELINES == "pareto": + for study in BASELINE_STUDIES: + for flow in get_pareto_flows(study, 0.9): + if flow not in BASELINES: + BASELINES.append(flow) + logger.info(f"We have {len(BASELINES)} Pareto-baselines for seeding") +elif CUSTOM_BASELINES == "all": + df_trials = get_completed_trials(study=BASELINE_STUDIES) + df_trials = df_trials.sort_values(by="number") + flows = get_flows_from_trials(df_trials) + BASELINES.extend(flows) + logger.info(f"We have {len(BASELINES)} baselines for seeding") +elif CUSTOM_BASELINES == "silver": + BASELINES = json.load(open(cfg.paths.results_dir / "silver-bullets.json", "r")) + logger.info(f"We have {len(BASELINES)} silver bullet baselines for seeding") +else: + logger.info("No custom baselines provided") + +LLMS: T.List[str] = LOCAL_LLMS + +EMBEDDING_MODELS = [ + "BAAI/bge-small-en-v1.5", + "thenlper/gte-large", + "mixedbread-ai/mxbai-embed-large-v1", + "sentence-transformers/all-MiniLM-L12-v2", + "sentence-transformers/paraphrase-multilingual-mpnet-base-v2", + "BAAI/bge-base-en-v1.5", + "BAAI/bge-large-en-v1.5", + "TencentBAC/Conan-embedding-v1", + "Linq-AI-Research/Linq-Embed-Mistral", + "Snowflake/snowflake-arctic-embed-l-v2.0", + "BAAI/bge-multilingual-gemma2", +] + +SEARCH_SPACE = SearchSpace( + few_shot_enabled=[False, True], + additional_context_enabled=[False, True], + hyde_enabled=[False, True], + reranker_enabled=[False, True], + splitter=Splitter( + methods=[ + "recursive", + "sentence", + "token", + ], + chunk_min_exp=7, + chunk_max_exp=10, + chunk_overlap_frac_min=0.0, + chunk_overlap_frac_max=0.5, + chunk_overlap_frac_step=0.05, + ), + rag_modes=[ + # "no_rag", + "rag", + "lats_rag_agent", + "react_rag_agent", + "critique_rag_agent", + "sub_question_rag", + ], + template_names=[ + "default", + "concise", + "CoT", + "finance-expert", + ], + response_synthesizer_llms=LLMS, + rag_retriever=Retriever( + embedding_models=EMBEDDING_MODELS, + methods=["dense", "sparse", "hybrid"], + top_k=TopK(kmin=1, kmax=10, log=False), + query_decomposition=QueryDecomposition( + llm_names=LLMS, + num_queries_min=2, + num_queries_max=5, + num_queries_step=1, + ), + ), + react_rag_agent=ReactRAGAgent( + subquestion_engine_llms=LLMS, + subquestion_response_synthesizer_llms=LLMS, + ), + sub_question_rag=SubQuestionRAGAgent( + subquestion_engine_llms=LLMS, + subquestion_response_synthesizer_llms=LLMS, + ), + critique_rag_agent=CritiqueRAGAgent( + subquestion_engine_llms=LLMS, + subquestion_response_synthesizer_llms=LLMS, + critique_agent_llms=LLMS, + reflection_agent_llms=LLMS, + ), + lats_rag_agent=LATSRagAgent(), + reranker=Reranker(llms=LLMS), + hyde=Hyde(llms=LLMS), + few_shot_retriever=FewShotRetriever( + embedding_models=EMBEDDING_MODELS, + ), +) + +EVALUATION = Evaluation( + mode=EVAL_MODE, + llms=[ + # "gpt-4o-mini", + "deepseek-ai/DeepSeek-R1-Distill-Llama-70B", + "Qwen/Qwen3-32B", + "nvidia/Llama-3_3-Nemotron-Super-49B", + ], + raise_on_exception=False, +) + +DATASETS: T.List[SyftrQADataset] = [RiseInsightsHF()] +assert DATASETS, "No datasets found. Please check the dataset list." + + +def get_optimization_parameters(): + optimization_config = OptimizationConfig( + method="expanding", + blocks=BLOCKS, + shuffle_blocks=False, + num_trials=NUM_TRIALS, + baselines=BASELINES, + baselines_cycle_llms=True, + shuffle_baselines=False, + max_concurrent_trials=MAX_CONCURRENT_TRIALS, + num_eval_samples=NUM_EVAL_SAMPLES, + num_eval_batch=5, + rate_limiter_max_coros=60, # control the number of concurrent evals ... + rate_limiter_period=60, # ... per given time unit + max_trial_cost=40.0, + cpus_per_trial=1, + seeder_timeout=None, # None: wait until finished, 0: don't wait + # ----------------------------------------------- + num_random_trials=50, + # ----------------------------------------------- + use_individual_baselines=False, + use_agent_baselines=False, + use_variations_of_baselines=False, + # ----------------------------------------------- + use_pareto_baselines=False, # required for transfer learning + # ----------------------------------------------- + use_pareto_pruner=False, + use_cost_pruner=True, + use_runtime_pruner=True, + # ----------------------------------------------- + use_toy_baselines=False, + # ----------------------------------------------- + sampler="tpe", + objective_2_name=OBJ2_NAME, + ) + if BASELINES: + start = BASELINES_START or 0 + end = BASELINES_END or len(BASELINES) + for i in range(start, end, BASELINES_BATCH_SIZE): + optimization_config = optimization_config.model_copy() + optimization_config.baselines = BASELINES[i : i + BASELINES_BATCH_SIZE] + yield DATASETS, SEARCH_SPACE, optimization_config, EVALUATION + else: + yield DATASETS, SEARCH_SPACE, optimization_config, EVALUATION + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument( + "--remote", + help="Use remote Ray cluster", + default=False, + action="store_true", + ) + parser.add_argument( + "--number", + type=int, + default=BENCH_NUM, + help="The benchmark number used to set up configurations", + ) + args = parser.parse_args() + cfg.ray.local = False if args.remote else cfg.ray.local + + job_ids = [] + client = get_client() + for ( + datasets, + search_space, + optimization_config, + evaluation, + ) in get_optimization_parameters(): + configs, paths = build_configs( + datasets=datasets, + search_space=search_space, + optimization_config=optimization_config, + evaluation=evaluation, + bench_num=args.number, + reuse_study=REUSE_STUDY, + recreate_study=RECREATE_STUDY, + prefix=PREFIX, + run_name=RUN_NAME, + embedding_max_time=EMBEDDING_MAX_TIME, + transfer_learning=None, + ) + + if DRY_RUN: + print("Not submitting jobs because DRY_RUN is set to True") + continue + + delete_confirmed = user_confirm_delete(configs[0]) + + # launch benchmarks + assert delete_confirmed + + for i, (config, path) in enumerate(zip(configs, paths)): + job_id = start_study( + client, path, config, delete_confirmed=delete_confirmed + ) + job_ids.append(job_id) + logger.info("Started job %s", job_id) + # This might help the checkpointing bug + sleep_time = 60 * MINUTES_BEFORE_NEXT_SUBMISSION + logger.info(f"Sleeping for {sleep_time} seconds before the next submission") + time.sleep(int(sleep_time)) + + # monitor benchmarks + log_tailers = [client.tail_job_logs(job) for job in job_ids] + + asyncio.run(iter_all_job_logs(log_tailers)) + + +if __name__ == "__main__": + main() diff --git a/syftr/storage.py b/syftr/storage.py index cc7261ee..024c7bec 100644 --- a/syftr/storage.py +++ b/syftr/storage.py @@ -1230,3 +1230,69 @@ def iter_examples(self, partition="test") -> T.Iterator[QAPair]: for i in partition_range: row = qa_examples[i] yield self._row_to_qapair(row) + + +class RiseInsightsHF(SyftrQADataset): + xname: T.Literal["rise_insights_hf"] = "rise_insights_hf" # type: ignore + description: str = """This dataset is a "Rise Insights report" titled "Making data count with AI". The report explores the evolving relationship between data and Artificial Intelligence (AI) in the financial services sector. It discusses how data has become valuable, the role of data commercialization, and various AI use cases in finance, including fighting financial crime, institutional investing, and improving customer experience. The report also addresses ethical considerations, bias, and trust in AI systems. It highlights the increasing adoption of AI by fintechs and emphasizes the importance of data strategy for banks to innovate and create new revenue streams. Additionally, it features updates from Rise global sites and their initiatives to support fintech startups.""" + + def _load_grounding_dataset(self) -> datasets.DatasetDict: + with distributed_lock( + self.name, timeout_s=self.load_examples_timeout_s, host_only=True + ): + dataset = datasets.load_dataset( + "DataRobot-Research/making-data-count-with-ai-2", + name="grounding", + cache_dir=cfg.paths.huggingface_cache.as_posix(), + token=cfg.hf_datasets.api_key.get_secret_value(), + ) + assert isinstance(dataset, datasets.DatasetDict) + return dataset + + def _load_qa_dataset(self) -> datasets.DatasetDict: + with distributed_lock( + self.name, timeout_s=self.load_examples_timeout_s, host_only=True + ): + dataset = datasets.load_dataset( + "DataRobot-Research/making-data-count-with-ai-2", + name="qa", + cache_dir=cfg.paths.huggingface_cache.as_posix(), + token=cfg.hf_datasets.api_key.get_secret_value(), + ) + assert isinstance(dataset, datasets.DatasetDict) + return dataset + + @overrides + def iter_grounding_data(self, partition="notused") -> T.Iterator[Document]: + # There is no partition. The grounding dataset is the same + # across all partitions of the qa pairs. + # This setting needs to fit the way the dataset is structured. + grounding_dataset = self._load_grounding_dataset() + for row in grounding_dataset["train"]: + yield Document(text=row["text"]) + + def _row_to_qapair(self, row, id: int): + """Dataset-specific conversion of row to QAPair struct. + + Invoked by iter_examples. + + Default implementation assumes row is already in QAPair format. + """ + return QAPair( + question=row["question"], + answer=row["answer"], + id=str(id), + context={}, + supporting_facts=[], + difficulty="default", + qtype=row.get("qtype", "default"), + gold_evidence=row.get("gold_evidence", []), + ) + + @overrides + def iter_examples(self, partition="test") -> T.Iterator[QAPair]: + assert partition in self.storage_partitions + partition = self._get_storage_partition(partition) + qa_examples = self._load_qa_dataset() + for id, row in enumerate(qa_examples[partition]): + yield self._row_to_qapair(row, id=id) diff --git a/tests/functional/test_hf_datasets.py b/tests/functional/test_hf_datasets.py index 08c9b5c6..840e7a8c 100644 --- a/tests/functional/test_hf_datasets.py +++ b/tests/functional/test_hf_datasets.py @@ -10,6 +10,7 @@ PartitionMap, PhantomWikiV001HF, PhantomWikiv050, + RiseInsightsHF, SyntheticCragTask3HF, SyntheticFinanceBenchHF, SyntheticHotPotQAHF, @@ -918,3 +919,14 @@ def test_phantomwikiv001_hf(): docs = list(bright_ds.iter_grounding_data()) assert len(docs) > 0 + + +def test_riseinsights_hf(): + for partition in ["train", "test", "sample", "holdout"]: + riseinsights_ds = RiseInsightsHF(partition_map=PartitionMap(test=partition)) + + examples = list(riseinsights_ds.iter_examples()) + assert len(examples) > 0 + + docs = list(riseinsights_ds.iter_grounding_data()) + assert len(docs) > 0