From 3e9a6201548b3e9d79e3a163730a4fe58c93401c Mon Sep 17 00:00:00 2001 From: Kamil Laurent Date: Tue, 28 Apr 2026 13:45:32 +0200 Subject: [PATCH 1/4] add hash and parth information when generating FK Tables --- src/pineko/cli/convolve.py | 1 + src/pineko/evolve.py | 11 +++++++++++ src/pineko/theory.py | 1 + 3 files changed, 13 insertions(+) diff --git a/src/pineko/cli/convolve.py b/src/pineko/cli/convolve.py index 83133f39..83f17e7f 100644 --- a/src/pineko/cli/convolve.py +++ b/src/pineko/cli/convolve.py @@ -108,6 +108,7 @@ def subcommand( assumptions=assumptions, comparison_pdfs=pdfs, min_as=min_as, + grid_path=pathlib.Path(grid_path), ) if len(operators) > 1: diff --git a/src/pineko/evolve.py b/src/pineko/evolve.py index 9ae02ba2..227c8273 100644 --- a/src/pineko/evolve.py +++ b/src/pineko/evolve.py @@ -1,6 +1,7 @@ """Tools related to evolution/eko.""" import copy +import hashlib import json import logging import os @@ -303,6 +304,7 @@ def evolve_grid( assumptions="Nf6Ind", comparison_pdfs: Optional[list[str]] = None, min_as=None, + grid_path: Optional[os.PathLike] = None, ): """Convolute grid with EKO from file paths. @@ -332,6 +334,8 @@ def evolve_grid( if given, a comparison table (with / without evolution) will be printed min_as: None or int minimum power of strong coupling + grid_path : str or os.PathLike or None + path to the grid file, used to store grid hash metadata """ order_mask = pineappl.boc.Order.create_mask(grid.orders(), max_as, max_al, True) if min_as is not None and min_as > 1: @@ -433,6 +437,13 @@ def prepare(operator, convolution_types): fktable.set_metadata("pineko_version", version.__version__) fktable.set_metadata("theory_card", json.dumps(theory_meta)) + # Store grid hash and path information + if grid_path is not None: + grid_path_obj = pathlib.Path(grid_path) + grid_hash = hashlib.md5(grid_path_obj.read_bytes()).hexdigest() + grid_files = {grid_path_obj.stem: {"hash": grid_hash, "path": str(grid_path_obj.resolve())}} + fktable.set_metadata("grid_files", json.dumps(grid_files)) + # compare before/after comparison = None if comparison_pdfs is not None: diff --git a/src/pineko/theory.py b/src/pineko/theory.py index 4a8e3e36..00a6c115 100644 --- a/src/pineko/theory.py +++ b/src/pineko/theory.py @@ -565,6 +565,7 @@ def fk(self, name, grid_path, tcard, pdfs): theory_meta=tcard, assumptions=assumptions, comparison_pdfs=pdfs, + grid_path=grid_path, ) if n_ekos > 1: From 0a184f6fb76c6882692a8a666c6f284dede83f97 Mon Sep 17 00:00:00 2001 From: Kamil Laurent Date: Tue, 26 May 2026 12:04:22 +0200 Subject: [PATCH 2/4] made info in the metadata more readable; removed absolute path from metadata, keeping path to grids from pineko --- src/pineko/evolve.py | 43 ++++++++++++++++++++++++++++++++++++------- 1 file changed, 36 insertions(+), 7 deletions(-) diff --git a/src/pineko/evolve.py b/src/pineko/evolve.py index 5c3c2db7..04a8d5ff 100644 --- a/src/pineko/evolve.py +++ b/src/pineko/evolve.py @@ -68,7 +68,7 @@ def construct_atlas(tcard): return atlas -def construct_empty_fktable(grid, theory_meta, fktable_path): +def construct_empty_fktable(grid, theory_meta, fktable_path, grid_path=None): """Construct and write a structurally valid but numerically empty FK table. "Empty" means the FK table carries a single trivial order ``(0,0,0,0,0)`` @@ -114,6 +114,21 @@ def construct_empty_fktable(grid, theory_meta, fktable_path): fktable = pineappl.fk_table.FkTable(empty_grid) fktable.set_metadata("pineko_version", version.__version__) fktable.set_metadata("theory_card", json.dumps(theory_meta)) + if grid_path is not None: + grid_path_obj = pathlib.Path(grid_path).resolve() + grid_hash = hashlib.md5(grid_path_obj.read_bytes()).hexdigest() + grid_path_parts = grid_path_obj.parts + if "pineko" in grid_path_parts: + pineko_idx = grid_path_parts.index("pineko") + display_path = str(pathlib.Path("/", *grid_path_parts[pineko_idx + 1 :])) + elif "data" in grid_path_parts: + data_idx = grid_path_parts.index("data") + display_path = str(pathlib.Path("/", *grid_path_parts[data_idx:])) + else: + display_path = grid_path_obj.name + fktable.set_metadata("grid_hash", grid_hash) + fktable.set_metadata("grid_theory", grid_path_obj.parent.name) + fktable.set_metadata("grid_path", display_path) fktable.write_lz4(str(fktable_path)) return fktable @@ -403,7 +418,9 @@ def evolve_grid( # A grid with no orders is a valid input for some workflows. In that case we # cannot build evolution kinematics, so we directly emit an empty FK table. if len(grid.orders()) == 0: - fktable = construct_empty_fktable(grid, theory_meta, fktable_path) + fktable = construct_empty_fktable( + grid, theory_meta, fktable_path, grid_path=grid_path + ) return grid, fktable, None order_mask = pineappl.boc.Order.create_mask(grid.orders(), max_as, max_al, True) @@ -427,7 +444,9 @@ def evolve_grid( # `XGrid` requires at least 2 points, otherwise it panics. In this case, we # simply return an empty FK table instead. if (len(x_grid) < 2) or (len(muf2_grid) == 0) or (len(mur2_grid) == 0): - fktable = construct_empty_fktable(grid, theory_meta, fktable_path) + fktable = construct_empty_fktable( + grid, theory_meta, fktable_path, grid_path=grid_path + ) return grid, fktable, None xif = 1.0 if operators[0].operator_card.configs.scvar_method is not None else xif @@ -514,12 +533,22 @@ def prepare(operator, convolution_types): fktable.set_metadata("pineko_version", version.__version__) fktable.set_metadata("theory_card", json.dumps(theory_meta)) - # Store grid hash and path information + # Store grid hash, theory folder, and repo-style grid path information if grid_path is not None: - grid_path_obj = pathlib.Path(grid_path) + grid_path_obj = pathlib.Path(grid_path).resolve() grid_hash = hashlib.md5(grid_path_obj.read_bytes()).hexdigest() - grid_files = {grid_path_obj.stem: {"hash": grid_hash, "path": str(grid_path_obj.resolve())}} - fktable.set_metadata("grid_files", json.dumps(grid_files)) + grid_path_parts = grid_path_obj.parts + if "pineko" in grid_path_parts: + pineko_idx = grid_path_parts.index("pineko") + display_path = str(pathlib.Path("/", *grid_path_parts[pineko_idx + 1 :])) + elif "data" in grid_path_parts: + data_idx = grid_path_parts.index("data") + display_path = str(pathlib.Path("/", *grid_path_parts[data_idx:])) + else: + display_path = grid_path_obj.name + fktable.set_metadata("grid_hash", grid_hash) + fktable.set_metadata("grid_theory", grid_path_obj.parent.name) + fktable.set_metadata("grid_path", display_path) # compare before/after comparison = None From 6f6f40227c7027c9c99898b43b581b6a5720dcd0 Mon Sep 17 00:00:00 2001 From: Kamil Laurent Date: Wed, 27 May 2026 17:27:25 +0200 Subject: [PATCH 3/4] simplified metadata --- src/pineko/evolve.py | 52 ++++++++++++++------------------------------ 1 file changed, 16 insertions(+), 36 deletions(-) diff --git a/src/pineko/evolve.py b/src/pineko/evolve.py index 04a8d5ff..e8d8029a 100644 --- a/src/pineko/evolve.py +++ b/src/pineko/evolve.py @@ -68,6 +68,20 @@ def construct_atlas(tcard): return atlas +def set_fktable_metadata(fktable, theory_meta, grid_path=None): + """Store the common FK metadata and, if available, grid provenance.""" + fktable.set_metadata("pineko_version", version.__version__) + fktable.set_metadata("theory_card", json.dumps(theory_meta)) + if grid_path is None: + return + + grid_path_obj = pathlib.Path(grid_path).resolve() + grid_hash = hashlib.md5(grid_path_obj.read_bytes()).hexdigest() + fktable.set_metadata("grid_hash", grid_hash) + fktable.set_metadata("grid_theory", grid_path_obj.parent.name) + fktable.set_metadata("grid_name", grid_path_obj.name) + + def construct_empty_fktable(grid, theory_meta, fktable_path, grid_path=None): """Construct and write a structurally valid but numerically empty FK table. @@ -112,23 +126,7 @@ def construct_empty_fktable(grid, theory_meta, fktable_path, grid_path=None): scale_funcs=grid.scales, ) fktable = pineappl.fk_table.FkTable(empty_grid) - fktable.set_metadata("pineko_version", version.__version__) - fktable.set_metadata("theory_card", json.dumps(theory_meta)) - if grid_path is not None: - grid_path_obj = pathlib.Path(grid_path).resolve() - grid_hash = hashlib.md5(grid_path_obj.read_bytes()).hexdigest() - grid_path_parts = grid_path_obj.parts - if "pineko" in grid_path_parts: - pineko_idx = grid_path_parts.index("pineko") - display_path = str(pathlib.Path("/", *grid_path_parts[pineko_idx + 1 :])) - elif "data" in grid_path_parts: - data_idx = grid_path_parts.index("data") - display_path = str(pathlib.Path("/", *grid_path_parts[data_idx:])) - else: - display_path = grid_path_obj.name - fktable.set_metadata("grid_hash", grid_hash) - fktable.set_metadata("grid_theory", grid_path_obj.parent.name) - fktable.set_metadata("grid_path", display_path) + set_fktable_metadata(fktable, theory_meta, grid_path=grid_path) fktable.write_lz4(str(fktable_path)) return fktable @@ -530,25 +528,7 @@ def prepare(operator, convolution_types): f"eko_operator_card{suffix}", json.dumps(operator.operator_card.raw) ) - fktable.set_metadata("pineko_version", version.__version__) - fktable.set_metadata("theory_card", json.dumps(theory_meta)) - - # Store grid hash, theory folder, and repo-style grid path information - if grid_path is not None: - grid_path_obj = pathlib.Path(grid_path).resolve() - grid_hash = hashlib.md5(grid_path_obj.read_bytes()).hexdigest() - grid_path_parts = grid_path_obj.parts - if "pineko" in grid_path_parts: - pineko_idx = grid_path_parts.index("pineko") - display_path = str(pathlib.Path("/", *grid_path_parts[pineko_idx + 1 :])) - elif "data" in grid_path_parts: - data_idx = grid_path_parts.index("data") - display_path = str(pathlib.Path("/", *grid_path_parts[data_idx:])) - else: - display_path = grid_path_obj.name - fktable.set_metadata("grid_hash", grid_hash) - fktable.set_metadata("grid_theory", grid_path_obj.parent.name) - fktable.set_metadata("grid_path", display_path) + set_fktable_metadata(fktable, theory_meta, grid_path=grid_path) # compare before/after comparison = None From bbf62d2ba2b5b38de06d97f13e182680560ec4f8 Mon Sep 17 00:00:00 2001 From: Kamil Laurent Date: Thu, 28 May 2026 10:41:15 +0200 Subject: [PATCH 4/4] Addressing metadata of merged FK Tables --- src/pineko/fonll.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/pineko/fonll.py b/src/pineko/fonll.py index e9fe54ea..468a9fa8 100644 --- a/src/pineko/fonll.py +++ b/src/pineko/fonll.py @@ -325,6 +325,11 @@ def produce_combined_fk( Path(configs.configs["paths"]["theory_cards"]) / f"{theoryid}.yaml" ) update_fk_theorycard(combined_fk, input_theorycard_path) + grid_hash = " ".join( + f"{idx:02d}-{fk.metadata['grid_hash']}" for idx, fk in enumerate(fk_dict.values()) + ) + combined_fk.set_metadata("grid_hash", grid_hash) + combined_fk.set_metadata("grid_theory", str(theoryid)) # save final FONLL fktable fk_folder = Path(configs.configs["paths"]["fktables"]) / str(theoryid) fk_folder.mkdir(exist_ok=True)