Merge pull request #122 from creativecommons/refine-automation

Merge refine-automation branch into gsoc2024-dev-1 branch

.github/workflows/fetch.yml

@@ -1,15 +1,10 @@
 name: Fetch Data
 
 on:
-  push:
+  # push:
   schedule:
-    - cron: '15 1 1-20 1,4,7,10 *'
-    - cron: '15 5 1-20 1,4,7,10 *'
-    - cron: '15 9 1-20 1,4,7,10 *'
-    - cron: '15 13 1-20 1,4,7,10 *'
-    - cron: '15 17 1-20 1,4,7,10 *'
-    - cron: '15 21 1-20 1,4,7,10 *'
-    - cron: '15 23 1-20 1,4,7,10 *'
+    # 1:15am onwards, days 1-20, first month of each quarter
+    - cron: '15 1,5,9,13,17,21,23 1-20 1,4,7,10 *'
   workflow_dispatch:
 
 jobs:
@@ -34,29 +29,10 @@ jobs:
           pip install --upgrade pip
           pip install pipenv
           pipenv sync --system
-          pip install -r requirements.txt
 
       - name: Set PYTHONPATH
         run: echo "PYTHONPATH=./scripts" >> $GITHUB_ENV
 
       - name: Run fetch script
         run: |
           python scripts/1-fetch/gcs_fetched.py
-
-      - name: Fetch and merge changes
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        run: |
-          python scripts/shared.py --operation fetch_and_merge --branch fetch-automation
-
-      - name: Add and commit changes
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        run: |
-          python scripts/shared.py --operation add_and_commit --message "Automated data fetch and commit"
-
-      - name: Push changes
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        run: |
-          python scripts/shared.py --operation push

.github/workflows/process.yml

@@ -3,13 +3,8 @@ name: Process Data
 on:
   # push:
   schedule:
-    - cron: '15 1 1-20 1,4,7,10 *'
-    - cron: '15 5 1-20 1,4,7,10 *'
-    - cron: '15 9 1-20 1,4,7,10 *'
-    - cron: '15 13 1-20 1,4,7,10 *'
-    - cron: '15 17 1-20 1,4,7,10 *'
-    - cron: '15 21 1-20 1,4,7,10 *'
-    - cron: '15 23 1-20 1,4,7,10 *'
+    # 1:15am onwards, days 1-20, second month of each quarter
+    - cron: '15 1,5,9,13,17,21,23 1-20 2,5,8,11 *'
   workflow_dispatch:
 
 jobs:
@@ -30,29 +25,10 @@ jobs:
           pip install --upgrade pip
           pip install pipenv
           pipenv sync --system
-          pip install -r requirements.txt
 
       - name: Set PYTHONPATH
         run: echo "PYTHONPATH=./scripts" >> $GITHUB_ENV
 
       - name: Run process script
         run: |
           python scripts/2-process/gcs_processed.py
-
-      - name: Fetch and merge changes
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        run: |
-          python scripts/shared.py --operation fetch_and_merge --branch fetch-automation
-
-      - name: Add and commit changes
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        run: |
-          python scripts/shared.py --operation add_and_commit --message "Automated data processing and commit"
-
-      - name: Push changes
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        run: |
-          python scripts/shared.py --operation push

.github/workflows/report.yml

@@ -3,15 +3,11 @@ name: Generate Report
 on:
   push:
   schedule:
-    - cron: '15 1 1-20 1,4,7,10 *'
-    - cron: '15 5 1-20 1,4,7,10 *'
-    - cron: '15 9 1-20 1,4,7,10 *'
-    - cron: '15 13 1-20 1,4,7,10 *'
-    - cron: '15 17 1-20 1,4,7,10 *'
-    - cron: '15 21 1-20 1,4,7,10 *'
-    - cron: '15 23 1-20 1,4,7,10 *'
+    # 1:15am onwards, days 1-20, third month of each quarter
+    - cron: '15 1,5,9,13,17,21,23 1-20 3,6,9,12 *'
   workflow_dispatch:
 
+
 jobs:
   generate-report:
     runs-on: ubuntu-latest
@@ -30,29 +26,10 @@ jobs:
           pip install --upgrade pip
           pip install pipenv
           pipenv sync --system
-          pip install -r requirements.txt
 
       - name: Set PYTHONPATH
         run: echo "PYTHONPATH=./scripts" >> $GITHUB_ENV
 
       - name: Run report script
         run: |
           python scripts/3-report/gcs_reports.py
-
-      - name: Fetch and merge changes
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        run: |
-          python scripts/shared.py --operation fetch_and_merge --branch fetch-automation
-
-      - name: Add and commit changes
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        run: |
-          python scripts/shared.py --operation add_and_commit --message "Automated data report and commit"
-
-      - name: Push changes
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        run: |
-          python scripts/shared.py --operation push

Pipfile

@@ -5,6 +5,7 @@ name = "pypi"
 
 [packages]
 flickrapi = "*"
+GitPython = "*"
 google-api-python-client = "*"
 internetarchive = "*"
 jupyterlab = ">=3.6.7"