wpi

.github/workflows/deploy_recipe.yaml

@@ -16,35 +16,29 @@ jobs:
     runs-on: ubuntu-latest
     steps:
 
+
+      - name: Checkout Repo
       - uses: actions/checkout@v4
+
+      - name: Setup python
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+
       - name: "Authenticate to Google Cloud"
         id: "auth"
         uses: "google-github-actions/auth@v2"
         with:
           credentials_json: "${{ secrets.LEAP_BAKERY_SERVICE_ACCOUNT }}"
 
-      - uses: actions/setup-python@v5
-        with:
-          python-version: '3.12'
 
       - name: "Install deps"
         run: |
-          python -m venv .venv
-          source .venv/bin/activate
           python -m pip install --upgrade pip
-          python -m pip install apache-beam[gcp] xarray_beam xarray kerchunk fastparquet pyarrow gcsfs google-cloud
-          python -m pip install -e .
-          echo "$VIRTUAL_ENV/bin" >> $GITHUB_PATH
-          echo "VIRTUAL_ENV=$VIRTUAL_ENV" >> $GITHUB_ENV
+          python -m pip install -e .[test]
+          python -m pip install apache-beam[gcp]
 
 
-      # - name: Get service account for beam
-      #   id: get-account
-      #   run: |
-      #     echo "SA_EMAIL=$(gcloud config get-value account)" >> $GITHUB_OUTPUT
-      #     echo $SA_EMAIL
-      #     echo $GITHUB_OUTPUT
-
 
       - name : "Deploy Beam pipeline"
         run: |

pyproject.toml

@@ -1,3 +1,25 @@
+[build-system]
+requires = ["setuptools>=45", "setuptools_scm[toml]>=6.2"]
+
+[project]
+name = "xbeam-virtualizarr"
+
+dynamic = ["version"]
+
+requires-python = ">=3.9"
+
+dependencies = [
+        'apache-beam[gcp]',
+        'xarray_beam',
+        'xarray',
+        'kerchunk',
+        'fastparquet',
+        'pyarrow',
+        'gcsfs',
+        'google-cloud']
+
+
+
 [tool.ruff]
 # Exclude a variety of commonly ignored directories.
 exclude = [

xbeam_virtualizarr/recipe.py

@@ -26,10 +26,9 @@ def run(argv=None, save_main_session=True):
 
     combined_ds = xr.open_dataset(reference_path, engine="kerchunk", chunks={})
     # subset the reference zarr
-    source_dataset = combined_ds.isel(day=slice(0, 1000))[
+    source_dataset = combined_ds.isel(day=slice(0, 32))[
         ["air_temperature"]
-    ]  # ~ 6.5 gb
-
+    ]  # ~ 3.2 gb
     source_chunks = dict(source_dataset.sizes)
     target_chunks = {"day": 16, "lat": 585, "lon": 1386}  # ~ full map 100MB chunks
     template = xbeam.make_template(source_dataset)