Python 3.9

.conda/meta.yaml

@@ -14,7 +14,6 @@ requirements:
     - {{ compiler('c') }} # [unix]
   host:
     - cython >=0.25.2
-    - lpsolve55
     - numpy
     - openbabel >=3
     - pydas >=1.0.2
@@ -39,7 +38,6 @@ requirements:
     - h5py
     - jinja2
     - jupyter
-    - lpsolve55
     - markupsafe
     - matplotlib >=1.5
     - mopac

.github/workflows/CI.yml

@@ -33,8 +33,8 @@
   schedule:
     # * is a special character in YAML so you have to quote this string
     - cron: "0 8 * * *"
-  # runs on all branches on both RMG-Py and forks
-  push:
+  # allow running on RMG-Py on a pushed branch, only if triggered manually
+  workflow_dispatch:
   # runs on PRs against RMG-Py (and anywhere else, but we add this for RMG-Py)
   pull_request:
 
@@ -48,11 +48,22 @@
   # if running on RMG-Py but requiring changes on an un-merged branch of RMG-database, replace
   # main with the name of the branch
   RMG_DATABASE_BRANCH: main
-
+  # julia parallel pre-compilation leads to race conditions and hangs, so we limit it to run in serial
+  JULIA_NUM_PRECOMPILE_TASKS: 1
 
 jobs:
-  build-osx:
-    runs-on: macos-13
+  build-and-test:
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.9"]
+        os: [macos-13, macos-latest, ubuntu-latest]
+        include-rms: ["", "with RMS"]
+        exclude:
+          - os: macos-13  # GitHub's runners just aren't up to the task of installing Julia
+            include-rms: 'with RMS'
+    runs-on: ${{ matrix.os }}
+    name: Python ${{ matrix.python-version }} ${{ matrix.os }} Build and Test ${{ matrix.include-rms }}
     # skip scheduled runs from forks
     if: ${{ !( github.repository != 'ReactionMechanismGenerator/RMG-Py' && github.event_name == 'schedule' ) }}
     defaults:
@@ -62,348 +73,370 @@
       - name: Checkout RMG-Py
         uses: actions/checkout@v4
 
-      # Step to create a custom condarc.yml before setting up conda
-      - name: Create custom conda config file
-        run: |
-          RUNNER_CWD=$(pwd)
-          echo "channels:" > $RUNNER_CWD/condarc.yml
-          echo "  - conda-forge" >> $RUNNER_CWD/condarc.yml
-          echo "  - rmg" >> $RUNNER_CWD/condarc.yml
-          echo "  - cantera" >> $RUNNER_CWD/condarc.yml
-          echo "show_channel_urls: true" >> $RUNNER_CWD/condarc.yml
-
-      # configures the mamba environment manager and builds the environment
-      - name: Setup Miniforge Python 3.7
+      - name: Setup Miniforge Python ${{ matrix.python-version }}
         uses: conda-incubator/setup-miniconda@v3
         with:
           environment-file: environment.yml
           miniforge-variant: Miniforge3
           miniforge-version: latest
-          python-version: 3.7
-          condarc-file: condarc.yml
+          python-version: ${{ matrix.python-version }}
           activate-environment: rmg_env
-          use-mamba: true
+          auto-update-conda: true
+          show-channel-urls: true
+          conda-remove-defaults: "true"
 
       # list the environment for debugging purposes
-      - name: mamba info
+      - name: conda info
         run: |
-          mamba info
-          mamba list
+          conda info
+          conda list
 
       # Clone RMG-database
       - name: Clone RMG-database
         run: |
           cd ..
           git clone -b $RMG_DATABASE_BRANCH https://github.com/ReactionMechanismGenerator/RMG-database.git
 
-      # modify env variables as directed in the RMG installation instructions
-      - name: Set Environment Variables
+      # RMG build step
+      - run: make install
+
+      - name: Setup Juliaup
+        if: matrix.include-rms == 'with RMS'
+        uses: julia-actions/install-juliaup@v2
+        with:
+          channel: '1.10'
+
+      - name: Set some env vars
+        if: matrix.include-rms == 'with RMS'
         run: |
-          RUNNER_CWD=$(pwd)
-          echo "PYTHONPATH=$RUNNER_CWD/RMG-Py:$PYTHONPATH" >> $GITHUB_ENV
-          echo "$RUNNER_CWD/RMG-Py" >> $GITHUB_PATH
+          # https://juliapy.github.io/PythonCall.jl/stable/pythoncall/#If-you-already-have-Python-and-required-Python-packages-installed
+          echo "JULIA_CONDAPKG_BACKEND=Null" >> $GITHUB_ENV
+          echo "JULIA_PYTHONCALL_EXE=$CONDA_PREFIX/bin/python" >> $GITHUB_ENV
 
-      # RMG build step
-      - name: make RMG
+      # RMS installation and linking to Julia
+      - name: Install and link Julia dependencies
+        if: matrix.include-rms == 'with RMS'
+        timeout-minutes: 120 # this usually takes 20-45 minutes (or hangs for 6+ hours).
+        run: . install_rms.sh
+
+      - name: Set some other env vars
+        if: matrix.include-rms == 'with RMS'
         run: |
-          make clean
-          make
+          # ensure that juliacall in Python uses the correct julia executable and packages: https://github.com/JuliaPy/PyJuliaPkg?tab=readme-ov-file#which-julia-gets-used
+          echo "PYTHON_JULIAPKG_EXE=$(which julia)" >> $GITHUB_ENV
+          echo "PYTHON_JULIAPKG_PROJECT=$HOME/.julia/packages" >> $GITHUB_ENV
+
+      - name: Install Q2DTor
+        run: echo "" | make q2dtor
+
+      # non-regression testing
+      - name: Run Unit, Functional, and Database Tests
+        run: make test-all
 
-  build-and-test-linux:
+  regression-test:
+    needs: build-and-test
     runs-on: ubuntu-latest
+    name: Regression Test
     # skip scheduled runs from forks
     if: ${{ !( github.repository != 'ReactionMechanismGenerator/RMG-Py' && github.event_name == 'schedule' ) }}
     env: 
       # This is true only if this is a reference case for the regression testing:
       REFERENCE_JOB: ${{ github.ref == 'refs/heads/main' && github.repository == 'ReactionMechanismGenerator/RMG-Py' }}
     defaults:
       run:
         shell: bash -l {0}
     steps:
       - name: Checkout RMG-Py
         uses: actions/checkout@v4
 
-      # Step to create a custom condarc.yml before setting up conda
-      - name: Create custom condarc.yml
-        run: |
-          RUNNER_CWD=$(pwd)
-          echo "channels:" > $RUNNER_CWD/condarc.yml
-          echo "  - conda-forge" >> $RUNNER_CWD/condarc.yml
-          echo "  - rmg" >> $RUNNER_CWD/condarc.yml
-          echo "  - cantera" >> $RUNNER_CWD/condarc.yml
-          echo "show_channel_urls: true" >> $RUNNER_CWD/condarc.yml
-
-      # configures the mamba environment manager and builds the environment
-      - name: Setup Miniforge Python 3.7
+      - name: Setup Miniforge Python 3.9
         uses: conda-incubator/setup-miniconda@v3
         with:
           environment-file: environment.yml
           miniforge-variant: Miniforge3
           miniforge-version: latest
-          python-version: 3.7
-          condarc-file: condarc.yml
+          python-version: 3.9
           activate-environment: rmg_env
-          use-mamba: true
+          auto-update-conda: true
+          show-channel-urls: true
+          conda-remove-defaults: "true"
 
       # list the environment for debugging purposes
-      - name: mamba info
+      - name: conda info
         run: |
-          mamba info
-          mamba list
+          conda info
+          conda list
 
       # Clone RMG-database
       - name: Clone RMG-database
         run: |
           cd ..
           git clone -b $RMG_DATABASE_BRANCH https://github.com/ReactionMechanismGenerator/RMG-database.git
 
-      # modify env variables as directed in the RMG installation instructions
-      - name: Set Environment Variables
+      # RMG build step
+      - run: make install
+
+      - name: Make separate RMS conda env
         run: |
-          RUNNER_CWD=$(pwd)
-          echo "PYTHONPATH=$RUNNER_CWD/RMG-Py:$PYTHONPATH" >> $GITHUB_ENV
-          echo "$RUNNER_CWD/RMG-Py" >> $GITHUB_PATH
+          conda create --name rmg_env_with_rms --clone rmg_env
 
-      # RMG build step
-      - name: make RMG
+      - name: Setup Juliaup
+        uses: julia-actions/install-juliaup@v2
+        with:
+          channel: '1.10'
+
+      - name: Set some env vars
         run: |
-          make clean
-          make
+          conda activate rmg_env_with_rms
+          # https://juliapy.github.io/PythonCall.jl/stable/pythoncall/#If-you-already-have-Python-and-required-Python-packages-installed
+          echo "JULIA_CONDAPKG_BACKEND=Null" >> $GITHUB_ENV
+          echo "JULIA_PYTHONCALL_EXE=$CONDA_PREFIX/bin/python" >> $GITHUB_ENV
 
       # RMS installation and linking to Julia
       - name: Install and link Julia dependencies
         timeout-minutes: 120 # this usually takes 20-45 minutes (or hangs for 6+ hours).
         run: |
-          python -c "import julia; julia.install(); import diffeqpy; diffeqpy.install()"
-          julia -e 'using Pkg; Pkg.add(PackageSpec(name="ReactionMechanismSimulator",rev="for_rmg")); using ReactionMechanismSimulator'
+          conda activate rmg_env_with_rms
+          . install_rms.sh
+
+      - name: Set some other env vars
+        run: |
+          conda activate rmg_env_with_rms
+          # ensure that juliacall in Python uses the correct julia executable and packages: https://github.com/JuliaPy/PyJuliaPkg?tab=readme-ov-file#which-julia-gets-used
+          echo "PYTHON_JULIAPKG_EXE=$(which julia)" >> $GITHUB_ENV
+          echo "PYTHON_JULIAPKG_PROJECT=$HOME/.julia/packages" >> $GITHUB_ENV
 
       - name: Install Q2DTor
         run: echo "" | make q2dtor
 
-      # non-regression testing
-      - name: Run Unit, Functional, and Database Tests
-        # aggregate into one command so we only have to eat the collection time once
-        run: make test-all
-
       # Regression Testing - Test Execution
       - name: Regression Tests - Execution
         id: regression-execution
-        timeout-minutes: 60
         run: |
+          conda activate rmg_env_with_rms
           for regr_test in aromatics liquid_oxidation nitrogen oxidation sulfur superminimal RMS_constantVIdealGasReactor_superminimal RMS_CSTR_liquid_oxidation RMS_liquidSurface_ch4o2cat fragment RMS_constantVIdealGasReactor_fragment minimal_surface;
           do
-            if python-jl rmg.py test/regression/"$regr_test"/input.py; then
+            if python rmg.py test/regression/"$regr_test"/input.py; then
               echo "$regr_test" "Executed Successfully"
             else
               echo "$regr_test" "Failed to Execute" | tee -a $GITHUB_STEP_SUMMARY
               export FAILED=Yes
             fi
           done
           if [[ ${FAILED} ]]; then
             echo "One or more regression tests could not be executed." | tee -a $GITHUB_STEP_SUMMARY
             echo "Please download the failed results or check the above log to see why." | tee -a $GITHUB_STEP_SUMMARY
             exit 1
           fi
 
       # Upload Regression Results as Failed if above step failed
       - name: Upload Failed Results
         if: ${{ failure() && steps.regression-execution.conclusion == 'failure' }}
         uses: actions/upload-artifact@v4
         with:
           name: failed_regression_results
           path: |
             test/regression
 
       # Upload Regression Results as Stable if Scheduled or Push to Main
       - name: Upload Results as Reference
         # upload the results for scheduled CI (on main) and pushes to main
         if: ${{ env.REFERENCE_JOB == 'true' }}
         uses: actions/upload-artifact@v4
         with:
           name: stable_regression_results
           path: |
             test/regression
 
       # Upload Regression Results as Dynamic if Push to non-main Branch
       - name: Upload Results as Dynamic
         if: ${{ env.REFERENCE_JOB == 'false' }}
         uses: actions/upload-artifact@v4
         with:
           name: dynamic_regression_results
           path: |
             test/regression
 
       - name: mkdir stable_regression_results
         if: ${{ env.REFERENCE_JOB == 'false' }}
         run: mkdir stable_regression_results
 
       # Retrieve Stable Results for reference
       - name : Find ID of Reference Results
         env:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         # this will search for the last successful execution of CI on main
         run: |
           run_id=$(gh run list -R ReactionMechanismGenerator/RMG-Py --workflow="Continuous Integration" --branch main --limit 15 --json databaseId,conclusion --jq 'map(select(.conclusion == "success")) | .[0].databaseId')
           echo "CI_RUN_ID=$run_id" >> $GITHUB_ENV
 
       - name: Retrieve Stable Regression Results
         if: ${{ env.REFERENCE_JOB == 'false' }}
         uses: actions/download-artifact@v4
         with:
         # download stable regression results
           run-id: ${{ env.CI_RUN_ID }}
           repository: ReactionMechanismGenerator/RMG-Py
           github-token: ${{ secrets.GITHUB_TOKEN }}
           name: stable_regression_results
           path: stable_regression_results
 
       # Regression Testing - Actual Comparisons
       - name: Regression Tests - Compare to Baseline
         id: regression-comparison
         if: ${{ env.REFERENCE_JOB == 'false' }}
         env:
           REFERENCE: stable_regression_results
         run: |
+          # we _do not_ run in the with_rms env here
+          # conda activate rmg_env_with_rms
+
           exec 2> >(tee -a regression.stderr >&2) 1> >(tee -a regression.stdout)
+          touch checkModels.log
           mkdir -p "test/regression-diff"
           for regr_test in aromatics liquid_oxidation nitrogen oxidation sulfur superminimal RMS_constantVIdealGasReactor_superminimal RMS_CSTR_liquid_oxidation fragment RMS_constantVIdealGasReactor_fragment minimal_surface;
           do
             echo ""
             echo "### Regression test $regr_test:"
             # Memory Usage and Execution Time
             echo -n 'Reference: '
             grep "Execution time" $REFERENCE/"$regr_test"/RMG.log | tail -1
             echo -n 'Current:   '
             grep "Execution time" test/regression/"$regr_test"/RMG.log | tail -1
             echo -n 'Reference: '
             grep "Memory used:" $REFERENCE/"$regr_test"/RMG.log | tail -1
             echo -n 'Current:   '
             grep "Memory used:" test/regression/"$regr_test"/RMG.log | tail -1
 
             echo "<details>"
             # Compare the edge and core
-            if python-jl scripts/checkModels.py \
+            if python scripts/checkModels.py \
                 "$regr_test-core" \
                 $REFERENCE/"$regr_test"/chemkin/chem_annotated.inp \
                 $REFERENCE/"$regr_test"/chemkin/species_dictionary.txt \
                 test/regression/"$regr_test"/chemkin/chem_annotated.inp \
-                test/regression/"$regr_test"/chemkin/species_dictionary.txt
+                test/regression/"$regr_test"/chemkin/species_dictionary.txt &> checkModels.log
             then
               echo "<summary>$regr_test Passed Core Comparison ✅</summary>"
             else
               echo "<summary>$regr_test Failed Core Comparison ❌</summary>"
               cp "$regr_test-core.log" test/regression-diff/
               export FAILED=Yes
             fi
             echo "" # blank line so next block is interpreted as markdown
             cat "$regr_test-core.log" || (echo "Dumping the whole log failed, please download it from GitHub actions. Here are the first 100 lines:" && head -n100 "$regr_test-core.log")
             echo "</details>"
             echo "<details>"
-            if python-jl scripts/checkModels.py \
+            if python scripts/checkModels.py \
                 "$regr_test-edge" \
                 $REFERENCE/"$regr_test"/chemkin/chem_edge_annotated.inp \
                 $REFERENCE/"$regr_test"/chemkin/species_edge_dictionary.txt \
                 test/regression/"$regr_test"/chemkin/chem_edge_annotated.inp \
-                test/regression/"$regr_test"/chemkin/species_edge_dictionary.txt
+                test/regression/"$regr_test"/chemkin/species_edge_dictionary.txt &> checkModels.log
             then
               echo "<summary>$regr_test Passed Edge Comparison ✅</summary>"
             else
               echo "<summary>$regr_test Failed Edge Comparison ❌</summary>"
               cp "$regr_test-edge.log" test/regression-diff/
               export FAILED=Yes
             fi
             echo "" # blank line so next block is interpreted as markdown
             cat "$regr_test-edge.log" || (echo "Dumping the whole log failed, please download it from GitHub actions. Here are the first 100 lines:" && head -n100 "$regr_test-core.log")
             echo "</details>"
 
             # Check for Regression between Reference and Dynamic (skip superminimal)
             if [ -f test/regression/"$regr_test"/regression_input.py ];
             then
               echo "<details>"
-              if python-jl rmgpy/tools/regression.py \
+              if python rmgpy/tools/regression.py \
                 test/regression/"$regr_test"/regression_input.py \
                 $REFERENCE/"$regr_test"/chemkin \
-                test/regression/"$regr_test"/chemkin
+                test/regression/"$regr_test"/chemkin &> checkModels.log
               then
                 echo "<summary>$regr_test Passed Observable Testing ✅</summary>"
               else
                 echo "<summary>$regr_test Failed Observable Testing ❌</summary>"
                 export FAILED=Yes
               fi
               echo "</details>"
             fi
             echo ""
           done
           if [[ ${FAILED} ]]; then
             echo "⚠️ One or more regression tests failed." | tee -a $GITHUB_STEP_SUMMARY >&2
             echo "Please download the failed results and run the tests locally or check the log to see why." | tee -a $GITHUB_STEP_SUMMARY >&2
           fi
 
       - name: Prepare Results for PR Comment
         if: ${{ env.REFERENCE_JOB == 'false' }}
         env:
           PR_NUMBER: ${{ github.event.number || github.event.after || github.event_name }}
         run: |
           echo $PR_NUMBER > summary.txt
           echo "## Regression Testing Results" >> summary.txt
           cat regression.stderr >> summary.txt
           echo "<details>" >> summary.txt
           echo "<summary>Detailed regression test results.</summary>" >> summary.txt
           cat regression.stdout >> summary.txt
           echo "</details>" >> summary.txt
           echo "" >> summary.txt
+          echo "<details>" >> summary.txt
+          echo "<summary>Debugging info for `checkModels.py` (if any).</summary>" >> summary.txt
+          cat checkModels.log >> summary.txt
+          echo "</details>" >> summary.txt
+          echo "" >> summary.txt
           echo "_beep boop this comment was written by a bot_ :robot:" >> summary.txt
           cat summary.txt > $GITHUB_STEP_SUMMARY
 
       - name: Upload regression summary artifact
        # the annotate workflow uses this artifact to add a comment to the PR
         uses: actions/upload-artifact@v4
         if : ${{ github.event_name == 'pull_request' }}
         with:
           name: regression_summary
           path: summary.txt
 
       - name: Upload Comparison Results
         uses: actions/upload-artifact@v4
         with:
           name: regression_test_comparison_results
           path: |
             test/regression-diff
 
       # Install and Call codecov only if the tests were successful (permitting failures in the regression comparison tests)
       - name: Code coverage install and run
         if: success() || ( failure() && steps.regression-execution.conclusion == 'success' )
         run: |
-          mamba install -y -c conda-forge codecov
+          conda install -y -c conda-forge codecov
           codecov
 
   build-and-push-docker:
     # after testing and on pushes to main, build and push docker image
     # technically we could live without the 'needs' since _in theory_
     # nothing will ever be merged into main that fails the tests, but
     # who knows ¯\_(ツ)_/¯
     #
     # taken from https://github.com/docker/build-push-action
-    needs: build-and-test-linux
+    needs: regression-test
     runs-on: ubuntu-latest
     if: github.ref == 'refs/heads/main' && github.repository == 'ReactionMechanismGenerator/RMG-Py'
     steps:
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v2
 
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v2
 
       - name: Login to Docker Hub
         uses: docker/login-action@v2
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
 
       - name: Build and Push
         uses: docker/build-push-action@v6
         with:
           push: true
           tags: reactionmechanismgenerator/rmg:latest
           build-args: |
             RMS_Branch=for_rmg