From 89c5431204e5d0854e04a51c577fe13b49b4b8ce Mon Sep 17 00:00:00 2001
From: bendnorman <bdn29@cornell.edu>
Date: Wed, 16 Aug 2023 16:21:21 -0800
Subject: [PATCH] Add github action that syncs s3 distribution bucket logs with
 a gcs bucket

---
 .github/workflows/pudl-s3-logs-sync.yml | 61 +++++++++++++++++++++++++
 1 file changed, 61 insertions(+)
 create mode 100644 .github/workflows/pudl-s3-logs-sync.yml

diff --git a/.github/workflows/pudl-s3-logs-sync.yml b/.github/workflows/pudl-s3-logs-sync.yml
new file mode 100644
index 0000000000..d3e02791b2
--- /dev/null
+++ b/.github/workflows/pudl-s3-logs-sync.yml
@@ -0,0 +1,61 @@
+---
+name: pudl-s3-logs-sync
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: "0 6 * * 1-5" # Weekdays at midnight on MST
+
+env:
+  PUDL_S3_LOG_BUCKET: s3://pudl.catalyst.coop-logs
+  PUDL_GCS_LOG_BUCKET: gs://pudl-s3-logs.catalyst.coop
+
+jobs:
+  pudl-s3-logs-sync:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      id-token: write
+    strategy:
+      fail-fast: false
+    defaults:
+      run:
+        shell: bash -l {0}
+
+    steps:
+      - name: Set default gcp credentials
+        id: gcloud-auth
+        uses: "google-github-actions/auth@v1"
+        with:
+          credentials_json: "${{ secrets.PUDL_S3_LOGS_SYNC_SA_KEY }}"
+
+      - name: Set up Cloud SDK
+        uses: google-github-actions/setup-gcloud@v1
+
+      - name: Sync s3 log bucket with gs bucket
+        run: |
+          gsutil -u catalyst-cooperative-pudl -m rsync -r ${{ env.PUDL_S3_LOG_BUCKET }} ${{ env.PUDL_GCS_LOG_BUCKET }}
+
+  pudl-s3-logs-sync-notify:
+    runs-on: ubuntu-latest
+    if: ${{ always() }}
+    needs: pudl-s3-logs-sync
+    steps:
+      - name: Inform the Codemonkeys
+        uses: 8398a7/action-slack@v3
+        with:
+          status: custom
+          fields: workflow,job,commit,repo,ref,author,took
+          custom_payload: |
+            {
+              username: 'action-slack',
+              icon_emoji: ':octocat:',
+              attachments: [{
+                color: '${{ needs.pudl-s3-logs-sync.result }}' === 'success' ? 'good' : '${{ needs.pudl-s3-logs-sync.result }}' === 'failure' ? 'danger' : 'warning',
+                text: `${process.env.AS_REPO}@${process.env.AS_REF}\n ${process.env.AS_WORKFLOW} (${process.env.AS_COMMIT})\n by ${process.env.AS_AUTHOR}\n Status: ${{ needs.pudl-s3-logs-sync.result }}`,
+              }]
+            }
+        env:
+          GITHUB_TOKEN: ${{ github.token }} # required
+          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} # required
+          MATRIX_CONTEXT: ${{ toJson(matrix) }} # required