Merge pull request #382 from lsst/tickets/OPSIM-1093

Tickets/opsim 1093

.github/workflows/build_docs.yaml

@@ -18,31 +18,44 @@ jobs:
       - uses: conda-incubator/setup-miniconda@v2
         with:
           auto-update-conda: true
-          python-version: "3.11"
-          channels: conda-forge
+          python-version: ${{ matrix.python-version }}
+          miniforge-variant: Mambaforge
+          channels: conda-forge,defaults
+          use-mamba: true
           channel-priority: strict
           show-channel-urls: true
 
-      - name: configure and install requirements and documenteer
+      - name: configure conda and install requirements
         shell: bash -l {0}
         run: |
-          conda config --set always_yes yes
-          conda install --quiet --file=requirements.txt
-          conda install --quiet pip
-          pip install "documenteer[guide]"
+          mamba install --quiet --file=requirements.txt          
+          mamba install --quiet pip
+          pip install "documenteer[pipelines]"
 
       - name: install rubin_sim
         shell: bash -l {0}
         run: |
           echo `pwd`
           ls ${{ github.workspace }}
-          python -m pip install .
+          python -m pip install . --no-deps
 
-      - name: download rubin_sim_data components needed to set up metrics for metricList
+      - name: Access rubin-sim-data cache
+        id: cache-rs
+        uses: actions/cache@v3
+        env:
+          cache-name: cached-rubin-sim-data
+        with:
+          path: ~/rubin_sim_data
+          key: ${{ env.cache-name }}
+          restore-keys: |
+            ${{ env.cache-name }}
+
+      - name: Update rubin-sim-data if needed
         shell: bash -l {0}
         run: |
-          export RUBIN_SIM_DATA_DIR=${{ github.workspace }}
-          rs_download_data --tdqm_disable -d 'maf,throughputs'
+          export RUBIN_SIM_DATA_DIR=~/rubin_sim_data
+          # Download anything that is not current
+          rs_download_data --tdqm_disable --update --dirs "maf,throughputs"
 
       - name: check conda and documenteer
         shell: bash -l {0}
@@ -52,16 +65,17 @@ jobs:
       - name: build docs
         shell: bash -l {0}
         run: |
-          export RUBIN_SIM_DATA_DIR=${{ github.workspace }}
+          export RUBIN_SIM_DATA_DIR=~/rubin_sim_data
           cd doc
           python metric_list.py
-          package-docs build
+          make html
+          #package-docs build
 
       - name: upload documentation
         uses: lsst-sqre/ltd-upload@v1
         with:
           project: "rubin-sim"
-          dir: "docs/_build/html"
+          dir: "doc/_build/html"
           username: ${{ secrets.ltd_username }}
           password: ${{ secrets.ltd_password }}
 

.github/workflows/cache.yaml

@@ -9,30 +9,31 @@ on:
 
 jobs:
   make-cache:
-    runs-on: ubuntu-latest
+    runs-on: macos-latest
     steps:
       - uses: actions/checkout@v4
-
-      - name: Install conda
-        uses: conda-incubator/setup-miniconda@v2
+      - uses: conda-incubator/setup-miniconda@v2
         with:
-          python-version: "3.11"
           auto-update-conda: true
-          channels: conda-forge,defaults
+          python-version: ${{ matrix.python-version }}
           miniforge-variant: Mambaforge
+          channels: conda-forge,defaults
           use-mamba: true
           channel-priority: strict
           show-channel-urls: true
-          activate-environment: rubin-sim
 
-      - name: Install rubin-sim
-        id: rs-install
+      - name: Configure conda and install requirements
         shell: bash -l {0}
         run: |
-          mamba install --quiet rubin_sim           
-          mamba list rubin-sim | grep -v "#" | awk '{print $2}' > ${{ github.workspace }}/rs_version
-          echo "rs-version" `cat ${{ github.workspace }}/rs_version`
-          echo "rs-version=`cat ${{ github.workspace }}/rs_version`" >> $GITHUB_OUTPUT          
+          mamba install --quiet --file=requirements.txt
+          mamba install --quiet --file=test-requirements.txt
+
+      - name: Install rubin_sim from git
+        shell: bash -l {0}
+        run: |
+          echo `pwd`
+          ls ${{ github.workspace }}
+          python -m pip install . --no-deps      
 
       - name: Access rubin-sim-data cache
         id: cache-rs
@@ -41,26 +42,16 @@ jobs:
           cache-name: cached-rubin-sim-data
         with:
           path: ~/rubin_sim_data
-          key: ${{ env.cache-name }}-v${{ steps.rs-install.outputs.rs-version }}
+          key: ${{ env.cache-name }}
           restore-keys: |
-            ${{ env.cache-name }}-v
+            ${{ env.cache-name }}
 
-      - if: ${{ steps.cache-rs.outputs.cache-hit == 'true' }}
-        name: List cache contents.
+      - name: Download data.
         shell: bash -l {0}
         run: |
           export RUBIN_SIM_DATA_DIR=~/rubin_sim_data
-          echo $RUBIN_SIM_DATA_DIR contents
-          ls $RUBIN_SIM_DATA_DIR
-          echo "__contents of versions.txt__"
-          cat $RUBIN_SIM_DATA_DIR/versions.txt
-
-      - if: ${{ steps.cache-rs.outputs.cache-hit != 'true' }}
-        name: Download data.
-        shell: bash -l {0}
-        run: |
-          export RUBIN_SIM_DATA_DIR=~/rubin_sim_data       
-          rs_download_data --tdqm_disable
+          scheduler_download_data --tdqm_disable --update
+          rs_download_data --tdqm_disable --update
 
       - name: Check data
         shell: bash -l {0}

.github/workflows/build_pypi.yml → .github/workflows/test_and_build.yaml

@@ -1,71 +1,84 @@
-name: Build and Publish PyPI
+name: Run CI
 
 on:
+  # Trigger the workflow on push (to main) or pull request
   push:
-    branches:
-      - main
     tags:
       - "*"
+    branches:
+      - "main"
+  pull_request:
+    branches:
+      - "main"
   workflow_dispatch:
 
+
 jobs:
   tests:
-    name: Check Tests (${{ matrix.python-version }}, ${{ matrix.os }})
+    name: Run Unit Tests (${{ matrix.python-version }}, ${{ matrix.os }})
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
       matrix:
-        os: ["ubuntu-latest"]
+        os: ["ubuntu-latest", "macos-latest"]
         python-version: ["3.11"]
     steps:
       - uses: actions/checkout@v4
       - uses: conda-incubator/setup-miniconda@v2
         with:
-          python-version: ${{ matrix.python-version }}
           auto-update-conda: true
-          channels: conda-forge,defaults
+          python-version: ${{ matrix.python-version }}
           miniforge-variant: Mambaforge
+          channels: conda-forge,defaults
           use-mamba: true
           channel-priority: strict
           show-channel-urls: true
 
       - name: configure conda and install requirements
         shell: bash -l {0}
         run: |
-          mamba config --set always_yes yes
           mamba install --quiet --file=requirements.txt
           mamba install --quiet --file=test-requirements.txt
 
       - name: install rubin_sim
         shell: bash -l {0}
         run: |
           echo `pwd`
-          python -m pip install .
+          ls ${{ github.workspace }}
+          python -m pip install . --no-deps
+
+      - name: Access rubin-sim-data cache
+        id: cache-rs
+        uses: actions/cache@v3
+        env:
+          cache-name: cached-rubin-sim-data
+        with:
+          path: ~/rubin_sim_data
+          key: ${{ env.cache-name }}
+          restore-keys: |
+            ${{ env.cache-name }}
 
-      - name: download rubin_sim_data components needed for unit tests
+      - name: Update rubin-sim-data if needed
         shell: bash -l {0}
         run: |
-          export RUBIN_SIM_DATA_DIR=${{ github.workspace }}/data_dir
-          rs_download_data --tdqm_disable -d 'site_models,throughputs,skybrightness,skybrightness_pre'
-          rs_download_data --tdqm_disable -d tests --force
+          export RUBIN_SIM_DATA_DIR=~/rubin_sim_data
+          # Download anything that is not current
+          scheduler_download_data --tdqm_disable --update
+          rs_download_data --tdqm_disable --update
 
       - name: conda list
         shell: bash -l {0}
         run: conda list
 
-      - name: run unit tests
+      - name: run tests
         shell: bash -l {0}
         run: |
-          export RUBIN_SIM_DATA_DIR=${{ github.workspace }}/data_dir
-          pytest -r a -v --cov=rubin_sim --cov=tests --cov-report=xml --cov-report=term --cov-branch
-
-      - name: Upload coverage to codecov
-        uses: codecov/codecov-action@v2
-        with:
-          file: coverage.xml
+          export RUBIN_SIM_DATA_DIR=~/rubin_sim_data
+          #pytest -r a -v 
+          pytest -r a -v --cov=rubin_scheduler --cov=tests --cov-report=xml --cov-report=term --cov-branch
 
   pypi:
-    name: Build and upload to PyPI
+    name: Build and Publish to PyPI
     runs-on: ubuntu-latest
     needs: [tests]
     if: startsWith(github.ref, 'refs/tags/')