Massive PR to merge all the clodius changes

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.20.3
+current_version = 0.20.4
 tag = True
 commit = True
 

.gitattributes

@@ -0,0 +1,20 @@
+# Binary genomics data files tracked with Git LFS
+*.cool filter=lfs diff=lfs merge=lfs -text
+*.mv5 filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.hdf5 filter=lfs diff=lfs merge=lfs -text
+*.bam filter=lfs diff=lfs merge=lfs -text
+*.bai filter=lfs diff=lfs merge=lfs -text
+*.beddb filter=lfs diff=lfs merge=lfs -text
+*.bb filter=lfs diff=lfs merge=lfs -text
+*.bigWig filter=lfs diff=lfs merge=lfs -text
+*.hitile filter=lfs diff=lfs merge=lfs -text
+data/*.fna filter=lfs diff=lfs merge=lfs -text
+data/*.gff.gz filter=lfs diff=lfs merge=lfs -text
+data/*.vcf.gz filter=lfs diff=lfs merge=lfs -text
+data/*.bed.gz filter=lfs diff=lfs merge=lfs -text
+data/*.bed.1.gz filter=lfs diff=lfs merge=lfs -text
+data/*.gz.tbi filter=lfs diff=lfs merge=lfs -text
+data/*.multires filter=lfs diff=lfs merge=lfs -text
+data/*.gff filter=lfs diff=lfs merge=lfs -text
+data/SRR1770413.sorted.short.bam.bai filter=lfs diff=lfs merge=lfs -text

.github/workflows/ci.yml

@@ -14,7 +14,7 @@ jobs:
     - uses: actions/checkout@v3
     - uses: actions/setup-python@v4
       with:
-        python-version: '3.10'
+        python-version: '3.12'
 
     - name: Install Dependencies
       run: |
@@ -29,21 +29,12 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ['3.9', '3.10']
+        python-version: ['3.12']
 
     steps:
     - uses: actions/checkout@v3
-
-    - name: Cache Fixtures
-      id: cache-fixtures
-      uses: actions/cache@v3
       with:
-        path: data/
-        key: ${{ runner.os }}-{{ hashFiles('get_test_data.sh') }}-{{ hashFiles('.gitignore') }}
-
-    - name: Download Fixtures
-      if: steps.cache-fixtures.outputs.cache-hit != 'true'
-      run: ./get_test_data.sh
+        lfs: true
 
     - name: Set Up Python ${{ matrix.python-version }}
       uses: actions/setup-python@v4

.gitignore

@@ -1,6 +1,8 @@
 notebooks/Scratch.ipynb
 notebooks/VCF.ipynb
 
+settings.local.json
+
 *.py[cod]
 __pycache__
 *~
@@ -40,7 +42,36 @@ Thumbs.db
 old
 tmp
 checkpoint
-data/
+data/*
+!data/Dixon2012-J1-NcoI-R1-filtered.100kb.multires.cool
+!data/hic-resolutions.cool
+!data/sample_htime.json
+!data/gene_annotations.short.db
+!data/wgEncodeCaltechRnaSeqHuvecR1x75dTh1014IlnaPlusSignalRep2.bigWig
+!data/points_density.h5
+!data/corrected.geneListwithStrand.bed.multires
+!data/labels.h5
+!data/SRR1770413.sorted.short.bam
+!data/SRR1770413.sorted.short.bam.bai
+!data/SRR1770413.different_index_filename.bai
+!data/SRR1770413.mismatched_bai.bam
+!data/geneAnnotationsExonUnions.1000.bed.v3.beddb
+!data/masterlist_DHSs_733samples_WM20180608_all_mean_signal_colorsMax.bed.bb
+!data/GCA_000350705.1_Esch_coli_KTE11_V1_genomic.short.fna.fai
+!data/GCA_002918705.1_ASM291870v1_genomic.gff.gz
+!data/genomic.10k.gff
+!data/genomic.10k.gff.gz
+!data/chm13v1.chrom.sizes
+!data/hg38.chrom.sizes
+!data/test.1.vcf.gz
+!data/no_item_rgb.bed
+!data/regions.valid.bed.1.gz
+!data/regions.valid.bed
+!data/regions.valid.bed.gz
+!data/regions.valid.bed.gz.tbi
+!data/regions.spaces.bed
+!data/genomic.10k.gff.gz.tbi
+!data/GCA_000350705.1_Esch_coli_KTE11_V1_genomic.short.fna
 output/
 COMMANDS
 npm-debug.log

CHANGELOG.md

@@ -1,3 +1,11 @@
+v0.21.0
+
+- Huge set of changes to support file-pointer based tileset functions
+
+v0.20.4
+
+- Fix overflow issue in cooler files
+
 v0.20.3
 
 - Add chromsizes tileset_info function

CLAUDE.md

@@ -0,0 +1,43 @@
+# Clodius
+
+A Python library and CLI tool for aggregating large genomic datasets into tile-based formats for display at multiple resolutions (used by [HiGlass](https://higlass.io)).
+
+## Project Structure
+
+- `clodius/` — main package
+  - `cli/` — Click-based CLI commands (`aggregate.py`, `convert.py`)
+  - `tiles/` — tile generation modules per file type (bigwig, cooler, bed, etc.)
+  - `models/` — Pydantic data models
+- `test/` — pytest tests mirroring the source layout
+- `test/sample_data/` — small sample files used by tests
+
+## Development Setup
+
+```shell
+pip install -e ".[dev]"
+```
+
+## Common Commands
+
+Run all tests:
+```shell
+pytest
+```
+
+Run a specific test:
+```shell
+pytest test/cli_test.py::test_clodius_aggregate_bedgraph
+```
+
+Lint:
+```shell
+flake8 clodius
+```
+
+## Key Conventions
+
+- **Linting**: flake8 (configured via `pyproject.toml`)
+- **Tests**: pytest with coverage (`pytest --cov=clodius`)
+- **Build**: hatchling
+- **Main branch**: `develop` (use this as the base for PRs)
+- **Python packaging**: `pyproject.toml` (no `setup.py`)

README.md

@@ -55,6 +55,40 @@ install `clodius` with develop mode:
 pip install -e ".[dev]"
 ```
 
+## Test Fixtures (Git LFS)
+
+Test data files in `data/` are stored in [Git LFS](https://git-lfs.com/). They are downloaded automatically when you clone the repository with LFS enabled:
+
+```shell
+git lfs install  # once per machine
+git clone <repo>  # LFS files downloaded automatically
+# or, in an existing clone:
+git lfs pull
+```
+
+### Adding a new test fixture
+
+1. **Check if the file type is already tracked** — open [.gitattributes](.gitattributes) and look for a matching pattern (e.g. `data/*.gz`, `*.bam`). If not, add a new tracking rule:
+
+   ```shell
+   git lfs track "data/*.ext"   # adds a line to .gitattributes
+   git add .gitattributes
+   ```
+
+2. **Allow the file through `.gitignore`** — `data/*` is ignored by default. Add a negation line for your file:
+
+   ```
+   !data/your_new_file.ext
+   ```
+
+3. **Stage and commit as normal:**
+
+   ```shell
+   git add data/your_new_file.ext
+   git commit -m "Add test fixture: your_new_file.ext"
+   git push  # LFS objects are uploaded automatically
+   ```
+
 ## Testing
 
 

clodius/__init__.py

@@ -1 +1 @@
-__version__ = "0.20.3"
+__version__ = "0.32.0"