Sync Google Sheets data #13309
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Sync Google Sheets data | |
| on: | |
| schedule: | |
| - cron: "50 * * * *" # 50 past every hour | |
| workflow_dispatch: | |
| push: | |
| branches: | |
| - develop | |
| jobs: | |
| pull-sheets: | |
| name: Sync Google Sheets data | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Clone BDDL repo | |
| uses: actions/checkout@v3 | |
| with: | |
| path: bddl | |
| ref: develop | |
| - name: Clone ig pipeline repo | |
| uses: actions/checkout@v3 | |
| with: | |
| repository: StanfordVL/ig_pipeline | |
| path: ig_pipeline | |
| token: ${{ secrets.REPO_TOKEN }} | |
| - name: Setup python | |
| uses: actions/setup-python@v2 | |
| with: | |
| python-version: "3.8" | |
| architecture: x64 | |
| - name: Authenticate on Google Cloud | |
| uses: 'google-github-actions/auth@v1' | |
| with: | |
| credentials_json: '${{ secrets.GCP_CREDENTIALS }}' | |
| # See if we need to re-pull any ig_pipeline data from DVC. | |
| - name: Check cache for ig_pipeline data | |
| id: cache-pipeline | |
| uses: actions/cache@v3 | |
| with: | |
| key: ig_pipeline-${{ hashFiles('ig_pipeline/dvc.lock') }} | |
| path: | | |
| ig_pipeline/artifacts/pipeline/combined_room_object_list.json | |
| ig_pipeline/artifacts/pipeline/combined_room_object_list_future.json | |
| ig_pipeline/artifacts/pipeline/object_inventory.json | |
| ig_pipeline/artifacts/pipeline/object_inventory_future.json | |
| - if: ${{ steps.cache-pipeline.outputs.cache-hit != 'true' }} | |
| name: Install dvc | |
| run: pip install dvc[gs] | |
| - if: ${{ steps.cache-pipeline.outputs.cache-hit != 'true' }} | |
| name: Pull dvc data | |
| working-directory: ig_pipeline | |
| run: dvc pull combined_room_object_list combined_room_object_list_future object_inventory object_inventory_future | |
| - if: ${{ steps.cache-pipeline.outputs.cache-hit != 'true' }} | |
| name: Unprotect data | |
| working-directory: ig_pipeline | |
| run: dvc unprotect artifacts/pipeline/combined_room_object_list.json artifacts/pipeline/combined_room_object_list_future.json artifacts/pipeline/object_inventory.json artifacts/pipeline/object_inventory_future.json | |
| - name: Copy over ig_pipeline files | |
| run: cp ig_pipeline/artifacts/pipeline/{combined_room_object_list,combined_room_object_list_future,object_inventory,object_inventory_future}.json bddl/bddl/generated_data | |
| - name: Install BDDL | |
| working-directory: bddl | |
| run: pip install -e . | |
| - name: Install dev requirements | |
| working-directory: bddl | |
| run: pip install -r requirements-dev.txt | |
| - name: Refresh sheets data | |
| working-directory: bddl | |
| run: python -m bddl.data_generation.pull_sheets | |
| - name: Refresh derivative data | |
| working-directory: bddl | |
| run: python -m bddl.data_generation.generate_datafiles | |
| - uses: stefanzweifel/git-auto-commit-action@v4 | |
| with: | |
| commit_message: "Sync Google Sheets data" | |
| repository: bddl |