Improve ref sim ci

.github/workflows/compare-post-benchmark.yaml

@@ -77,18 +77,18 @@ jobs:
       - name: Run benchmark
         run: |
           mkdir artifacts/
-          mpiexec -n 1 -np 1 pytest --refsim=${{ matrix.id }} --benchmark-only --benchmark-json artifacts/output_${{ matrix.id }}.json -s
+          UCX_TLS=self,sm,tcp mpiexec -n 1 -np 1 pytest --refsim=${{ matrix.id }} --benchmark-only --benchmark-json artifacts/output_${{ matrix.id }}.json -s
 
-      # upload the benchmark output as an artifact with name key corresponding to the current
-      # workflow run and attempt only store artifacts for 1 day
+      # upload the benchmark output as an artifact with name key corresponding to the currentworkflow run
+      # stores artifacts for 5 days in case something is blocking a failed test rerun
       - name: Upload result artifacts
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: ${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.id }}
           path: artifacts/
           if-no-files-found: error
           include-hidden-files: true
-          retention-days: 1
+          retention-days: 5
 
   collate-post-benchmark:
     # Job that loads the saved artifacts corresponding to the specific workflow run and attempt id,
@@ -134,8 +134,8 @@ jobs:
         uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
         with:
           github-token: "${{ secrets.GITHUB_TOKEN }}"
-          pattern: ${{ github.run_id }}-${{ github.run_attempt }}-*
-          merge-multiple: true
+          pattern: ${{ github.run_id }}-*
+          merge-multiple: false
           path: artifacts
 
       # prints directory info recursively, removable
@@ -148,20 +148,31 @@ jobs:
       # then takes the benchmark timing info from n-1 files and adds it to the first file.
       # With this approach, benchmark comparison output is only a single table, and we only
       # have one comment on alert in the workflow.
+      # added code to identify latest run id artifacts from each test when loading artifacts
+      # across multiple run attempts
       - uses: jannekem/run-python-script-action@9d8e2e0878d575fb6073277f38ce3f10ebf4f059 # v1.8
         with:
           script: |
+            from pathlib import Path
             import os
             import json
 
-            # make list of paths to artifact files excluding hidden files
-            filepath_arr = [os.path.join('artifacts', bench) for bench in os.listdir('artifacts') if not bench.startswith('.')]
-            print(filepath_arr)
+            # make dictionary of run artifacts keeping only the latest run attempt for each
+            # reference simulation
+            parsed_run_attempts = {}
+            for folder in Path("artifacts").iterdir():
+                _, num, ref_sim = folder.name.split("-")
+                if ref_sim not in parsed_run_attempts or parsed_run_attempts[ref_sim][0] < num:
+                    parsed_run_attempts[ref_sim] = [num, folder]
+
+            # there should only be 1 file in each directory because each directory should be
+            # a unique run and reference simulation
+            files_to_stitch = [next(val[1].iterdir()) for val in parsed_run_attempts.values()]
 
             output_jsons = []
 
-            # open each filepath in the filepath_arr, load it as a json, and append it to an empty list
-            for filepath in filepath_arr:
+            # open each filepath, load it as a json, and append it to an empty list
+            for filepath in files_to_stitch:
                 with open(filepath) as f:
                     output_jsons.append(json.load(f))
 
@@ -172,10 +183,26 @@ jobs:
             for json_out in output_jsons[1:]:
                 net_json['benchmarks'].append(json_out['benchmarks'][0])
 
+            # DEBUG (keeping for now just to confirm behavior when a reference simulation
+            # actually fails)
+            print(parsed_run_attempts)
+            print(files_to_stitch)
+            print(len(net_json['benchmarks']))
+            print(net_json)
+
             # save net_json as json with name output.json in current working directory
             with open('output.json', 'w') as f:
                 json.dump(net_json, f)
 
+      - name: Print errors
+        if: steps.script.outputs.error == 'true'
+        run: |
+          printenv "SCRIPT_STDOUT"
+          printenv "SCRIPT_STDERR"
+        env:
+          SCRIPT_STDOUT: ${{ steps.script.outputs.stdout }}
+          SCRIPT_STDERR: ${{ steps.script.outputs.stderr }}
+
       # Print github event_name and ref_name and the boolean check for whether gh-pages should be updated
       - name: Print Event, Ref, and Upload Boolean
         run: |
@@ -185,7 +212,7 @@ jobs:
 
       # Compares the data from the specified "output-file-path" and compares
       # with the latest data from the gh-pages branch. If performance regression
-      # occurs, fails the test and alerts. Will only comment in if performance
+      # occurs, alerts but does not fail. Will only comment in if performance
       # regression has occurred.
       # NOTE: it is important that this does not modify gh-pages on pull request
       # https://github.com/benchmark-action/github-action-benchmark?tab=readme-ov-file#caveats
@@ -199,9 +226,9 @@ jobs:
           output-file-path: output.json
           # Where the previous data file is stored
           # should fail consistently
-          alert-threshold: "120%"
+          alert-threshold: "150%"
           # Workflow will fail when an alert happens
-          fail-on-alert: true
+          fail-on-alert: false
           # Comment on the PR if the branch is not a fork
           comment-on-alert: true
           # Enable Job Summary for PRs

.github/workflows/testsuite.yaml

@@ -69,7 +69,7 @@ jobs:
       - name: Run Tests Linux/MacOS
         if: matrix.os != 'windows-latest'
         run: |
-          pytest -n auto --cov=pyuvsim --cov-config=.coveragerc --cov-report xml:./coverage.xml --junitxml=test-reports/xunit.xml
+          UCX_TLS=self,sm,tcp pytest -n auto --cov=pyuvsim --cov-config=.coveragerc --cov-report xml:./coverage.xml --junitxml=test-reports/xunit.xml
       - name: Run Tests Windows
         if: matrix.os == 'windows-latest'
         # Windows can't handle pytest-xdist in our experience
@@ -124,7 +124,7 @@ jobs:
 
       - name: Run Tests
         run: |
-          pytest -n auto --cov=pyuvsim --cov-config=.coveragerc --cov-report xml:./coverage.xml
+          UCX_TLS=self,sm,tcp pytest -n auto --cov=pyuvsim --cov-config=.coveragerc --cov-report xml:./coverage.xml
 
       - name: check coverage report
         run: |
@@ -178,7 +178,7 @@ jobs:
 
       - name: Run Tests
         run: |
-          pytest -n auto --cov=pyuvsim --cov-config=.coveragerc --cov-report xml:./coverage.xml --junitxml=test-reports/xunit.xml
+          UCX_TLS=self,sm,tcp pytest -n auto --cov=pyuvsim --cov-config=.coveragerc --cov-report xml:./coverage.xml --junitxml=test-reports/xunit.xml
 
       - name: check coverage report
         run: |
@@ -289,7 +289,7 @@ jobs:
 
       - name: Run Tests
         run: |
-          mpiexec -n 1 pytest tests/test_run_ref.py -k "test_download or test_run_ref_sim_cli" --cov=pyuvsim --cov-config=.coveragerc --cov-report xml:./coverage.xml --junitxml=test-reports/xunit.xml
+          UCX_TLS=self,sm,tcp mpiexec -n 1 pytest tests/test_run_ref.py -k "test_download or test_run_ref_sim_cli" --cov=pyuvsim --cov-config=.coveragerc --cov-report xml:./coverage.xml --junitxml=test-reports/xunit.xml
 
       - name: check coverage report
         run: |

src/pyuvsim/cli.py

@@ -622,16 +622,17 @@ def download_gleam_vot(url="file://gleam.vot", row_limit=None):
     """
     Download gleam.vot to pyuvsim cache specified by astropy.
 
+    Currently only used for downloading the gleam catalog with a set row limit.
+
     Runs import_file_to_cache to format the installed file to be recognizable through astropy.
-    Includes adding a fake url with which to specify the file. The fake download url acts as
-    the key to load the file in astropy -- allowing filename specification in yaml files
-    without any path details as astropy handles searching the cache with get_cached_urls
-    and download_file.
+    Includes adding a fake url with which to specify the file.
 
     Parameters
     ----------
     url: str
         A fake url for the gleam.vot file.
+    row_limit: int
+        The number of rows to download. Note that the specific rows chosen may not be consistent.
     """
     # check if url already cached
     if url in get_cached_urls("pyuvsim"):
@@ -701,14 +702,13 @@ def download_data_files(argv=None):
         - "mwa": mwa uvbeam file
         - "healpix": gsm 2016 nside 128 healpix map saved as skyh5
 
-    All files require astropy. Gleam additionally requires pyradiosky and astroquery.
+    All files require astropy. Gleam additionally requires pyradiosky and astroquery if specifying
+    the row_limit.
 
     Downloads files to astropy default cache location for pyuvsim e.g. "get_cache_dir(pyuvsim)"
 
     The intention is to have all files downloaded under the astropy paradigm, and thus findable
-    in cache throug the same paradigm with methods get_cached_urls and download_file. This allows
-    us to list only the file url corresponding to the astropy cached file in either yaml instead
-    of having to worry about relative or absolute path variables.
+    in cache through the same paradigm with methods get_cached_urls and download_file.
     """
     parser = argparse.ArgumentParser(
         description="A command-line script to download large data files "
@@ -722,7 +722,8 @@ def download_data_files(argv=None):
         "mwa": "http://ws.mwatelescope.org/static/mwa_full_embedded_element_pattern.h5",
         # gsm 2016 nside 128 healpix map
         "healpix": "https://repository.library.brown.edu/storage/bdr:eafzyycj/content/",
-        "gleam": "",
+        # hosting gleam catalog on BDR because download is slow otherwise
+        "gleam": "https://repository.library.brown.edu/storage/bdr:68m9yvr7/content/",
     }
 
     # create some strings for formatted help output
@@ -768,8 +769,8 @@ def download_data_files(argv=None):
     # for each file keyword, check that it exists in the dictionary, then try to download
     for file in args.files:
         if file in file_download_dict:
-            # call download_gleam_vot if gleam, otherwise download from url
-            if file == "gleam":
+            # call download_gleam_vot if gleam AND row_limit is specified, otherwise use url
+            if file == "gleam" and args.row_limit is not None:
                 filepath = download_gleam_vot(row_limit=args.row_limit)
             else:
                 filepath = download_file_using_astropy(file_download_dict[file])

src/pyuvsim/data/test_config/obsparam_ref_1.4_source_axis.yaml

@@ -7,7 +7,7 @@ freq:
   channel_width: 100000.0
 sources:
   filetype: "gleam"
-  catalog: "file://gleam.vot"
+  catalog: "https://repository.library.brown.edu/storage/bdr:68m9yvr7/content/"
   non_nan: "all"
   non_negative: True
 telescope:

tests/test_run_ref.py

@@ -29,9 +29,11 @@
     "files",
     [
         ("--clear", "--row_limit", "75001", "gleam", 1),
+        ("--row_limit", "75001", "gleam", 2),
         ("mwa", 1),
         ("healpix", 1),
         ("fail", 1),
+        ("gleam", 1),
         ("gleam", 2),
     ],
 )