diff --git a/.github/workflows/compare-post-benchmark.yaml b/.github/workflows/compare-post-benchmark.yaml index 378bbb09..287a1487 100644 --- a/.github/workflows/compare-post-benchmark.yaml +++ b/.github/workflows/compare-post-benchmark.yaml @@ -77,10 +77,10 @@ jobs: - name: Run benchmark run: | mkdir artifacts/ - mpiexec -n 1 -np 1 pytest --refsim=${{ matrix.id }} --benchmark-only --benchmark-json artifacts/output_${{ matrix.id }}.json -s + UCX_TLS=self,sm,tcp mpiexec -n 1 -np 1 pytest --refsim=${{ matrix.id }} --benchmark-only --benchmark-json artifacts/output_${{ matrix.id }}.json -s - # upload the benchmark output as an artifact with name key corresponding to the current - # workflow run and attempt only store artifacts for 1 day + # upload the benchmark output as an artifact with name key corresponding to the currentworkflow run + # stores artifacts for 5 days in case something is blocking a failed test rerun - name: Upload result artifacts uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: @@ -88,7 +88,7 @@ jobs: path: artifacts/ if-no-files-found: error include-hidden-files: true - retention-days: 1 + retention-days: 5 collate-post-benchmark: # Job that loads the saved artifacts corresponding to the specific workflow run and attempt id, @@ -134,8 +134,8 @@ jobs: uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: github-token: "${{ secrets.GITHUB_TOKEN }}" - pattern: ${{ github.run_id }}-${{ github.run_attempt }}-* - merge-multiple: true + pattern: ${{ github.run_id }}-* + merge-multiple: false path: artifacts # prints directory info recursively, removable @@ -148,20 +148,31 @@ jobs: # then takes the benchmark timing info from n-1 files and adds it to the first file. # With this approach, benchmark comparison output is only a single table, and we only # have one comment on alert in the workflow. + # added code to identify latest run id artifacts from each test when loading artifacts + # across multiple run attempts - uses: jannekem/run-python-script-action@9d8e2e0878d575fb6073277f38ce3f10ebf4f059 # v1.8 with: script: | + from pathlib import Path import os import json - # make list of paths to artifact files excluding hidden files - filepath_arr = [os.path.join('artifacts', bench) for bench in os.listdir('artifacts') if not bench.startswith('.')] - print(filepath_arr) + # make dictionary of run artifacts keeping only the latest run attempt for each + # reference simulation + parsed_run_attempts = {} + for folder in Path("artifacts").iterdir(): + _, num, ref_sim = folder.name.split("-") + if ref_sim not in parsed_run_attempts or parsed_run_attempts[ref_sim][0] < num: + parsed_run_attempts[ref_sim] = [num, folder] + + # there should only be 1 file in each directory because each directory should be + # a unique run and reference simulation + files_to_stitch = [next(val[1].iterdir()) for val in parsed_run_attempts.values()] output_jsons = [] - # open each filepath in the filepath_arr, load it as a json, and append it to an empty list - for filepath in filepath_arr: + # open each filepath, load it as a json, and append it to an empty list + for filepath in files_to_stitch: with open(filepath) as f: output_jsons.append(json.load(f)) @@ -172,10 +183,26 @@ jobs: for json_out in output_jsons[1:]: net_json['benchmarks'].append(json_out['benchmarks'][0]) + # DEBUG (keeping for now just to confirm behavior when a reference simulation + # actually fails) + print(parsed_run_attempts) + print(files_to_stitch) + print(len(net_json['benchmarks'])) + print(net_json) + # save net_json as json with name output.json in current working directory with open('output.json', 'w') as f: json.dump(net_json, f) + - name: Print errors + if: steps.script.outputs.error == 'true' + run: | + printenv "SCRIPT_STDOUT" + printenv "SCRIPT_STDERR" + env: + SCRIPT_STDOUT: ${{ steps.script.outputs.stdout }} + SCRIPT_STDERR: ${{ steps.script.outputs.stderr }} + # Print github event_name and ref_name and the boolean check for whether gh-pages should be updated - name: Print Event, Ref, and Upload Boolean run: | @@ -185,7 +212,7 @@ jobs: # Compares the data from the specified "output-file-path" and compares # with the latest data from the gh-pages branch. If performance regression - # occurs, fails the test and alerts. Will only comment in if performance + # occurs, alerts but does not fail. Will only comment in if performance # regression has occurred. # NOTE: it is important that this does not modify gh-pages on pull request # https://github.com/benchmark-action/github-action-benchmark?tab=readme-ov-file#caveats @@ -199,9 +226,9 @@ jobs: output-file-path: output.json # Where the previous data file is stored # should fail consistently - alert-threshold: "120%" + alert-threshold: "150%" # Workflow will fail when an alert happens - fail-on-alert: true + fail-on-alert: false # Comment on the PR if the branch is not a fork comment-on-alert: true # Enable Job Summary for PRs diff --git a/.github/workflows/testsuite.yaml b/.github/workflows/testsuite.yaml index 938acdda..cbe720c0 100644 --- a/.github/workflows/testsuite.yaml +++ b/.github/workflows/testsuite.yaml @@ -69,7 +69,7 @@ jobs: - name: Run Tests Linux/MacOS if: matrix.os != 'windows-latest' run: | - pytest -n auto --cov=pyuvsim --cov-config=.coveragerc --cov-report xml:./coverage.xml --junitxml=test-reports/xunit.xml + UCX_TLS=self,sm,tcp pytest -n auto --cov=pyuvsim --cov-config=.coveragerc --cov-report xml:./coverage.xml --junitxml=test-reports/xunit.xml - name: Run Tests Windows if: matrix.os == 'windows-latest' # Windows can't handle pytest-xdist in our experience @@ -124,7 +124,7 @@ jobs: - name: Run Tests run: | - pytest -n auto --cov=pyuvsim --cov-config=.coveragerc --cov-report xml:./coverage.xml + UCX_TLS=self,sm,tcp pytest -n auto --cov=pyuvsim --cov-config=.coveragerc --cov-report xml:./coverage.xml - name: check coverage report run: | @@ -178,7 +178,7 @@ jobs: - name: Run Tests run: | - pytest -n auto --cov=pyuvsim --cov-config=.coveragerc --cov-report xml:./coverage.xml --junitxml=test-reports/xunit.xml + UCX_TLS=self,sm,tcp pytest -n auto --cov=pyuvsim --cov-config=.coveragerc --cov-report xml:./coverage.xml --junitxml=test-reports/xunit.xml - name: check coverage report run: | @@ -289,7 +289,7 @@ jobs: - name: Run Tests run: | - mpiexec -n 1 pytest tests/test_run_ref.py -k "test_download or test_run_ref_sim_cli" --cov=pyuvsim --cov-config=.coveragerc --cov-report xml:./coverage.xml --junitxml=test-reports/xunit.xml + UCX_TLS=self,sm,tcp mpiexec -n 1 pytest tests/test_run_ref.py -k "test_download or test_run_ref_sim_cli" --cov=pyuvsim --cov-config=.coveragerc --cov-report xml:./coverage.xml --junitxml=test-reports/xunit.xml - name: check coverage report run: | diff --git a/src/pyuvsim/cli.py b/src/pyuvsim/cli.py index b52ba9c7..57ff4b70 100644 --- a/src/pyuvsim/cli.py +++ b/src/pyuvsim/cli.py @@ -622,16 +622,17 @@ def download_gleam_vot(url="file://gleam.vot", row_limit=None): """ Download gleam.vot to pyuvsim cache specified by astropy. + Currently only used for downloading the gleam catalog with a set row limit. + Runs import_file_to_cache to format the installed file to be recognizable through astropy. - Includes adding a fake url with which to specify the file. The fake download url acts as - the key to load the file in astropy -- allowing filename specification in yaml files - without any path details as astropy handles searching the cache with get_cached_urls - and download_file. + Includes adding a fake url with which to specify the file. Parameters ---------- url: str A fake url for the gleam.vot file. + row_limit: int + The number of rows to download. Note that the specific rows chosen may not be consistent. """ # check if url already cached if url in get_cached_urls("pyuvsim"): @@ -701,14 +702,13 @@ def download_data_files(argv=None): - "mwa": mwa uvbeam file - "healpix": gsm 2016 nside 128 healpix map saved as skyh5 - All files require astropy. Gleam additionally requires pyradiosky and astroquery. + All files require astropy. Gleam additionally requires pyradiosky and astroquery if specifying + the row_limit. Downloads files to astropy default cache location for pyuvsim e.g. "get_cache_dir(pyuvsim)" The intention is to have all files downloaded under the astropy paradigm, and thus findable - in cache throug the same paradigm with methods get_cached_urls and download_file. This allows - us to list only the file url corresponding to the astropy cached file in either yaml instead - of having to worry about relative or absolute path variables. + in cache through the same paradigm with methods get_cached_urls and download_file. """ parser = argparse.ArgumentParser( description="A command-line script to download large data files " @@ -722,7 +722,8 @@ def download_data_files(argv=None): "mwa": "http://ws.mwatelescope.org/static/mwa_full_embedded_element_pattern.h5", # gsm 2016 nside 128 healpix map "healpix": "https://repository.library.brown.edu/storage/bdr:eafzyycj/content/", - "gleam": "", + # hosting gleam catalog on BDR because download is slow otherwise + "gleam": "https://repository.library.brown.edu/storage/bdr:68m9yvr7/content/", } # create some strings for formatted help output @@ -768,8 +769,8 @@ def download_data_files(argv=None): # for each file keyword, check that it exists in the dictionary, then try to download for file in args.files: if file in file_download_dict: - # call download_gleam_vot if gleam, otherwise download from url - if file == "gleam": + # call download_gleam_vot if gleam AND row_limit is specified, otherwise use url + if file == "gleam" and args.row_limit is not None: filepath = download_gleam_vot(row_limit=args.row_limit) else: filepath = download_file_using_astropy(file_download_dict[file]) diff --git a/src/pyuvsim/data/test_config/obsparam_ref_1.4_source_axis.yaml b/src/pyuvsim/data/test_config/obsparam_ref_1.4_source_axis.yaml index c2f4ba67..f2ac32f6 100644 --- a/src/pyuvsim/data/test_config/obsparam_ref_1.4_source_axis.yaml +++ b/src/pyuvsim/data/test_config/obsparam_ref_1.4_source_axis.yaml @@ -7,7 +7,7 @@ freq: channel_width: 100000.0 sources: filetype: "gleam" - catalog: "file://gleam.vot" + catalog: "https://repository.library.brown.edu/storage/bdr:68m9yvr7/content/" non_nan: "all" non_negative: True telescope: diff --git a/tests/test_run_ref.py b/tests/test_run_ref.py index 086cdde3..8af14e47 100644 --- a/tests/test_run_ref.py +++ b/tests/test_run_ref.py @@ -29,9 +29,11 @@ "files", [ ("--clear", "--row_limit", "75001", "gleam", 1), + ("--row_limit", "75001", "gleam", 2), ("mwa", 1), ("healpix", 1), ("fail", 1), + ("gleam", 1), ("gleam", 2), ], )