Ploughshare download

src/pinefarm/cli/run.py

@@ -190,18 +190,18 @@ def run_dataset(runner):
 
             # collect results in the output pineappl grid
             runner.generate_pineappl()
-
-            table.print_table(
-                table.convolute_grid(
-                    runner.grid, runner.pdf, integrated=isinstance(runner, mg5.Mg5)
-                ),
-                runner.results(),
-                runner.dest,
-            )
-
-            # TODO: annotate_version should be a post-processing step
-            # however at the moment only works in 1-grid cases
-            runner.annotate_versions()
+            if not hasattr(runner, "ps_link"):
+                table.print_table(
+                    table.convolute_grid(
+                        runner.grid, runner.pdf, integrated=isinstance(runner, mg5.Mg5)
+                    ),
+                    runner.results(),
+                    runner.dest,
+                )
+
+                # TODO: annotate_version should be a post-processing step
+                # however at the moment only works in 1-grid cases
+                runner.annotate_versions()
 
         runner.postprocess()
 

src/pinefarm/external/__init__.py

@@ -59,4 +59,9 @@ def decide_external_tool(dsname: str):
 
         return mg5.Mg5, "blue"
 
+    if (configs["paths"]["runcards"] / dsname / "ploughshare_link.txt").exists():
+        from . import plough
+
+        return plough.Plough, "purple"
+
     raise ValueError(f"pinefarm could not discover the tool to use for {dsname}")

src/pinefarm/external/interface.py

@@ -199,7 +199,7 @@ def postprocess(self):
         entries = {}
         if metadata.exists():
             for line in metadata.read_text().splitlines():
-                k, v = line.split("=")
+                k, v = line.split("=", 1)
                 entries[k] = v
 
         for ext in ["*.pineappl.lz4", "*.pineappl"]:

src/pinefarm/external/plough.py

@@ -0,0 +1,80 @@
+"""Download grids + convert them to pineappl format."""
+
+import os
+import shutil
+import tarfile
+import urllib.request
+
+import pineappl
+import requests
+
+from .. import table
+from . import interface
+
+PLOUGHSHARE_LINK_FILENAME = "ploughshare_link.txt"
+GRIDS_TMP = "grids"
+
+
+class Plough(interface.External):
+    """Interface provider."""
+
+    def __init__(self, pinecard, theorycard, *args, **kwargs):
+        super().__init__(pinecard, theorycard, *args, **kwargs)
+        self.ps_link = self.source / PLOUGHSHARE_LINK_FILENAME
+        self.link = self.ps_link.read_text()
+
+        self.filename = self.link.rsplit("/")[-1]
+        self.dir_name = self.filename.rsplit(".", 1)[0]
+        self.tarball = self.dest / self.filename
+
+    def run(self):
+        """Download and extract the .tgz file."""
+        print("Downloading from ploughshare...")
+        try:
+            self.download_to_dest()
+            if self.tarball.exists():
+                print(f"Grids successfully downloaded to {self.tarball}")
+            else:
+                raise FileNotFoundError(
+                    f"{self.tarball} not found but the download didn't seem to fail?"
+                )
+        except Exception as e:
+            raise FileNotFoundError(f"{self.tarball} could not be downloaded!") from e
+        print(f"Grids successfully downloaded to {self.tarball}")
+        print("Extracting files...")
+        self.extract_tarball()
+        print(f"Grids successfully extracted to {self.dest}")
+        self.cleanup()
+
+    def results(self):
+        """Results are collected and compared at the pineappl (script) level."""
+        pass
+
+    def collect_versions(self):
+        """No additional programs involved."""
+        return {}
+
+    def generate_pineappl(self):
+        """Grids are converted in postrun.sh."""
+        return
+
+    def download_to_dest(self):
+        """Download the file to the output folder."""
+        urllib.request.urlretrieve(self.link, self.dest / self.filename)
+
+    def extract_tarball(self):
+        """Extract the contents."""
+        with tarfile.open(self.tarball, "r:*") as tf:
+            tf.extractall(self.dest)
+        self.grids_dir = self.dest / self.dir_name / GRIDS_TMP
+        grids_list = sorted(os.listdir(self.grids_dir))
+        for i, grid in enumerate(grids_list):
+            extension = grid.split(".", 2)[2]
+            print(extension)
+            os.rename(self.grids_dir / grid, self.dest / f"grid_{i}.{extension}")
+
+    def cleanup(self):
+        """Delete unnecessary files and create tmp.pineappl.lz4 to allow postprocessing."""
+        shutil.rmtree(self.dest / self.dir_name)
+        self.tarball.unlink()
+        open(self.dest / "tmp.pineappl.lz4", "x")