Add GH Action to drop staging db if PR is closed (#196)

* Add GH Action to drop staging db if PR is closed * update name and add print to macro --------- Co-authored-by: Noel Gomez <[email protected]>
datacoves · Feb 2, 2024 · a9dcefe · a9dcefe
1 parent aa98ca5
commit a9dcefe
Show file tree

Hide file tree

Showing 3 changed files with 72 additions and 4 deletions.
diff --git a/.github/workflows/closed_py_cleanup.yml b/.github/workflows/closed_py_cleanup.yml
@@ -0,0 +1,64 @@
+name: Drop test db if pr is closed
+
+on:  # yamllint disable-line rule:truthy
+  pull_request:
+    types:
+      - closed
+
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+# This cancels a run if another change is pushed to the same branch
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  dbt:
+    name: Pull Request dbt Tests
+    runs-on: ubuntu-latest
+
+    # Set environment variables in
+    # https://github.com//<your org>/<your repo>/settings/variables/actions
+    #
+    # Alternatively, You can define multiple ENV for different workflows.
+    # https://github.com/<org>/<repo>/settings/environments
+    # environment: PR_ENV
+    container: datacoves/ci-basic-dbt-snowflake:2
+
+    defaults:
+      run:
+        working-directory: /__w/${{ github.event.repository.name }}/${{ github.event.repository.name }}/transform
+
+    env:
+      DBT_PROFILES_DIR: /__w/${{ github.event.repository.name }}/${{ github.event.repository.name }}/automate/dbt
+      DATACOVES__DBT_HOME: /__w/${{ github.event.repository.name }}/${{ github.event.repository.name }}/transform
+      DATACOVES__YAML_DAGS_FOLDER: /__w/${{ github.event.repository.name }}/${{ github.event.repository.name }}/schedule
+
+      DATACOVES__MAIN__ACCOUNT:   ${{ vars.DATACOVES__MAIN__ACCOUNT }}
+
+      DATACOVES__MAIN__DATABASE:  ${{ vars.DATACOVES__MAIN__DATABASE }}_PR_${{ github.event.number }}
+      DATACOVES__MAIN__SCHEMA:    ${{ vars.DATACOVES__MAIN__SCHEMA }}
+
+      DATACOVES__MAIN__ROLE:      ${{ vars.DATACOVES__MAIN__ROLE }}
+      DATACOVES__MAIN__WAREHOUSE: ${{ vars.DATACOVES__MAIN__WAREHOUSE }}
+
+      DATACOVES__MAIN__USER:      ${{ vars.DATACOVES__MAIN__USER }}
+      DATACOVES__MAIN__PASSWORD:  ${{ secrets.DATACOVES__MAIN__PASSWORD }}
+
+
+    steps:
+      - name: Checkout branch
+        uses: actions/[email protected]
+        with:
+          fetch-depth: 0
+          ref: ${{ github.event.pull_request.head.sha }}
+
+      - name: Set Secure Directory
+        run: git config --global --add safe.directory /__w/${{ github.event.repository.name }}/${{ github.event.repository.name }}
+
+      - name: Install dbt packages
+        run: "dbt deps"
+
+      - name: Drop PR database
+        run: "dbt --no-write-json run-operation drop_recreate_db --args '{db_name: ${{env.DATACOVES__MAIN__DATABASE}}, recreate: False}'"  # yamllint disable-line rule:line-length
diff --git a/transform/macros/tooling/drop_recreate_db.sql b/transform/macros/tooling/drop_recreate_db.sql
@@ -1,6 +1,6 @@
 {# This macro drops and recreates in a given database #}
 {#
-    To run: 
+    To run:
     dbt run-operation drop_recreate_db --args '{db_name: dev_commercial_dw2, recreate: False}'
 #}
 
@@ -11,10 +11,12 @@
         drop database if exists {{ db_name }};
         {% if recreate %}
             create database if not exists {{ db_name }};
-            {{ log("Recreating Database: " ~ db_name, info=true) }}
+            {{ print("Recreating Database: " ~ db_name) }}
+        {% else %}
+            {{ print("Dropped Database: " ~ db_name) }}
         {% endif %}
     {% endset %}
-    
+
     {% do run_query(drop_recreate_sql) %}
 
     {% if recreate %}
@@ -27,4 +29,4 @@
         {% do run_query(apply_grants_sql) %}
     {% endif %}
 
-{%- endmacro -%}
+{%- endmacro -%}
diff --git a/transform/models/L1_inlets/loans/personal_loans.yml b/transform/models/L1_inlets/loans/personal_loans.yml
@@ -8,6 +8,8 @@ models:
         description: The state in which the borrower resides
         meta:
           masking_policy: masking_policy_pii_string
+        tests:
+          - not_null
       - name: annual_inc
         description: The borrower's annual income
         meta: