diff --git a/lib/finetune/uv.lock b/lib/finetune/uv.lock index b0663c6..ff7237b 100644 --- a/lib/finetune/uv.lock +++ b/lib/finetune/uv.lock @@ -999,11 +999,12 @@ dev = [ [[package]] name = "londonaicentre-mesa-utils" -version = "1.2.0" +version = "1.3.0" source = { editable = "../utils" } dependencies = [ { name = "boto3" }, { name = "bs4" }, + { name = "httpx" }, { name = "litellm" }, { name = "londonaicentre-mesa-types" }, { name = "markdown" }, @@ -1014,6 +1015,7 @@ dependencies = [ requires-dist = [ { name = "boto3", specifier = ">=1.41.1" }, { name = "bs4", specifier = ">=0.0.2" }, + { name = "httpx", specifier = ">=0.28.1" }, { name = "litellm", specifier = ">=1.80.0" }, { name = "londonaicentre-mesa-types", editable = "../types" }, { name = "markdown", specifier = ">=3.10" }, @@ -1025,6 +1027,7 @@ dev = [ { name = "boto3-stubs", specifier = ">=1.41.4" }, { name = "mypy", specifier = ">=1.18.2" }, { name = "pytest", specifier = ">=9.0.2" }, + { name = "pytest-mock", specifier = ">=3.15.1" }, { name = "ruff", specifier = ">=0.14.6" }, { name = "types-markdown", specifier = ">=3.10.0.20251106" }, ] diff --git a/pipelines/paedawen/config.yaml b/pipelines/paedawen/config.yaml new file mode 100644 index 0000000..4e13d59 --- /dev/null +++ b/pipelines/paedawen/config.yaml @@ -0,0 +1,38 @@ +# Default fine-tuning config (training params ONLY). +# +# Each trainer translates this into native form via config.py +# Operational args (instance type, AWS config, work_dir, quantize, ...) are NOT here +# they stay per-trainer constructor inputs because they differ between HF and MLX. +# +# Cross-library translation reference: +# lora.alpha + lora.rank -> mlx scale = alpha / rank (e.g. 64 / 32 = 2.0) +# lora.target_modules -> mlx keys = ["self_attn." / "mlp.", ...] +# epochs -> mlx iters = ceil(num_samples / batch_size) * epochs +# (only set mlx.iters for override of derived value) + +training: + base_model: Qwen/Qwen3-4B-Instruct-2507 # which pre-trained model to start from + epochs: 3 # how many full passes over the training data + learning_rate: 2e-4 # how big a step the model takes each update (too high = unstable, too low = slow) + batch_size: 1 # samples processed at once; keep at 1 for long (16k-32k) samples to avoid running out of memory + max_seq_length: 16384 # longest sample (in tokens) the model will read - anything longer is cut off + lora: + rank: 32 # size of the small trainable add-on; higher = more capacity to learn but more memory + alpha: 64 # how strongly the add-on influences the model (mlx scale = alpha / rank = 2.0) + dropout: 0.05 # randomly ignores 5% of connections while training to reduce overfitting + target_modules: [q_proj, k_proj, v_proj, o_proj, gate_proj, up_proj, down_proj] # which parts of each layer get the trainable add-on + + mlx: # optional; consumed ONLY by the MLX translation + iters: null # total training steps; left null to auto-derive from epochs, set a number to override + num_layers: 16 # how many of the model's top layers to fine-tune - 16 is good balance, -1 means all layers + seed: 42 # fixed random seed so runs are reproducible + save_every: 1000 # save a checkpoint every N steps + steps_per_report: 10 # print training progress every N steps + steps_per_eval: 200 # check performance on held-out data every N steps + val_batches: 0 # 0 = no validation (TrainingDataHandler writes train.jsonl only) + grad_accumulation_steps: 4 # micro-steps accumulated before each optimizer update + grad_checkpoint: true # trade compute for memory by recomputing activations + weight_decay: 0.01 # AdamW weight decay; null uses the mlx default + lr_scheduler_type: cosine # decay the learning rate to 0 over training; null keeps it constant + warmup_ratio: 0.05 # fraction of iters spent warming up to learning_rate + lr_schedule: null # raw mlx_lm passthrough for a custom schedule; overrides lr_scheduler_type diff --git a/pipelines/paedawen/finetune.ipynb b/pipelines/paedawen/finetune.ipynb new file mode 100644 index 0000000..5b01226 --- /dev/null +++ b/pipelines/paedawen/finetune.ipynb @@ -0,0 +1,96 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "11a389f2", + "metadata": {}, + "source": [ + "### Paedawen: Model fine-tuning" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4cc82e36", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "from dotenv import load_dotenv\n", + "load_dotenv()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "950d260a", + "metadata": {}, + "outputs": [], + "source": [ + "from mesa_runtime import MesaRuntime\n", + "runtime: MesaRuntime = MesaRuntime(\n", + " os.environ[\"BASE_URL\"], os.environ[\"USERNAME\"], os.environ[\"PASSWORD\"]\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b477ce37", + "metadata": {}, + "outputs": [], + "source": [ + "runtime.list_training_data()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7567abd5", + "metadata": {}, + "outputs": [], + "source": [ + "run_id: int = runtime.dispatch(\n", + " \"20260720-223834_paedawen-batch\",\n", + " model_name=\"paedawen\",\n", + " schema=\"paedacute\",\n", + " description=\"Extracting acute triage signs from paediatric triage notes\",\n", + " version=\"0.1.0\",\n", + " config=\"config.yaml\",\n", + " config_from_file=True\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ad3a0857", + "metadata": {}, + "outputs": [], + "source": [ + "runtime.get_run(run_id=run_id)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "paedawen (3.13.14.final.0)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.13.14" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/pipelines/paedawen/pyproject.toml b/pipelines/paedawen/pyproject.toml index fc55869..ed5ce1b 100644 --- a/pipelines/paedawen/pyproject.toml +++ b/pipelines/paedawen/pyproject.toml @@ -9,8 +9,10 @@ dependencies = [ "londonaicentre-docsynth", "londonaicentre-mesa-datagen", "londonaicentre-paedacuteschema>=1.2.0", + "mesa-runtime", ] [tool.uv.sources] londonaicentre-docsynth = { git = "ssh://git@github.com/londonaicentre/MESA-Docsynth", branch = "spike/mesa-build-refactor" } londonaicentre-mesa-datagen = { path = "../../lib/datagen", editable = true } +mesa-runtime = { git = "https://github.com/londonaicentre/mesa-runtime.git", subdirectory = "lib" } diff --git a/pipelines/paedawen/uv.lock b/pipelines/paedawen/uv.lock index 80e5df5..724d785 100644 --- a/pipelines/paedawen/uv.lock +++ b/pipelines/paedawen/uv.lock @@ -1187,11 +1187,12 @@ dev = [ [[package]] name = "londonaicentre-mesa-utils" -version = "1.2.0" +version = "1.3.0" source = { editable = "../../lib/utils" } dependencies = [ { name = "boto3" }, { name = "bs4" }, + { name = "httpx" }, { name = "litellm" }, { name = "londonaicentre-mesa-types" }, { name = "markdown" }, @@ -1202,6 +1203,7 @@ dependencies = [ requires-dist = [ { name = "boto3", specifier = ">=1.41.1" }, { name = "bs4", specifier = ">=0.0.2" }, + { name = "httpx", specifier = ">=0.28.1" }, { name = "litellm", specifier = ">=1.80.0" }, { name = "londonaicentre-mesa-types", editable = "../../lib/types" }, { name = "markdown", specifier = ">=3.10" }, @@ -1213,6 +1215,7 @@ dev = [ { name = "boto3-stubs", specifier = ">=1.41.4" }, { name = "mypy", specifier = ">=1.18.2" }, { name = "pytest", specifier = ">=9.0.2" }, + { name = "pytest-mock", specifier = ">=3.15.1" }, { name = "ruff", specifier = ">=0.14.6" }, { name = "types-markdown", specifier = ">=3.10.0.20251106" }, ] @@ -1324,6 +1327,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", size = 9979, upload-time = "2022-08-14T12:40:09.779Z" }, ] +[[package]] +name = "mesa-runtime" +version = "0.1.0" +source = { git = "https://github.com/londonaicentre/mesa-runtime.git?subdirectory=lib#d9b2c9b1bd75ed493c7cb59a34e37efdec92b3a5" } +dependencies = [ + { name = "httpx" }, + { name = "pydantic" }, +] + [[package]] name = "multidict" version = "6.7.1" @@ -1452,6 +1464,7 @@ dependencies = [ { name = "londonaicentre-docsynth" }, { name = "londonaicentre-mesa-datagen" }, { name = "londonaicentre-paedacuteschema" }, + { name = "mesa-runtime" }, ] [package.metadata] @@ -1461,6 +1474,7 @@ requires-dist = [ { name = "londonaicentre-docsynth", git = "ssh://git@github.com/londonaicentre/MESA-Docsynth?branch=spike%2Fmesa-build-refactor" }, { name = "londonaicentre-mesa-datagen", editable = "../../lib/datagen" }, { name = "londonaicentre-paedacuteschema", specifier = ">=1.2.0" }, + { name = "mesa-runtime", git = "https://github.com/londonaicentre/mesa-runtime.git?subdirectory=lib" }, ] [[package]]