-
Notifications
You must be signed in to change notification settings - Fork 5
132 lines (103 loc) · 4.78 KB
/
merge_to_main.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
name: Perform deployment to production
on: # yamllint disable-line rule:truthy
pull_request:
types:
- closed
branches:
- main
paths:
- .github/workflows/*
- automate/*
- visualize/*
- orchestrate/*
- transform/*
- transform/**/*
# Allows you to run this workflow manually from the Actions tab
workflow_dispatch:
jobs:
build:
# Need to make sure that when the PR was closed, it was actually merged.
if: github.event.pull_request.merged == true
name: Deployment Script
runs-on: ubuntu-latest
# Set environment variables in
# https://github.com//<your org>/<your repo>/settings/variables/actions
#
# Alternatively, You can define multiple ENV for different workflows.
# https://github.com/<org>/<repo>/settings/environments
# environment: PR_ENV
container: datacoves/ci-basic-dbt-snowflake:2
defaults:
run:
working-directory: /__w/${{ github.event.repository.name }}/${{ github.event.repository.name }}/transform
env:
DBT_PROFILES_DIR: /__w/${{ github.event.repository.name }}/${{ github.event.repository.name }}/automate/dbt
DATACOVES__DBT_HOME: /__w/${{ github.event.repository.name }}/${{ github.event.repository.name }}/transform
DATACOVES__MAIN__ACCOUNT: ${{ vars.DATACOVES__MAIN__ACCOUNT }}
DATACOVES__MAIN__DATABASE: ${{ vars.DATACOVES__MAIN__DATABASE }}
DATACOVES__MAIN__SCHEMA: ${{ vars.DATACOVES__MAIN__SCHEMA }}
DATACOVES__MAIN__ROLE: ${{ vars.DATACOVES__MAIN__ROLE }}
DATACOVES__MAIN__WAREHOUSE: ${{ vars.DATACOVES__MAIN__WAREHOUSE }}
DATACOVES__MAIN__USER: ${{ vars.DATACOVES__MAIN__USER }}
DATACOVES__MAIN__PASSWORD: ${{ secrets.DATACOVES__MAIN__PASSWORD }}
# This is used by datacoves to drop the staging database for blue/green
# deployments, most likely you don't want to set this, we use it for demos
DATACOVES__DROP_DB_ON_FAIL: ${{ vars.DATACOVES__DROP_DB_ON_FAIL }}
# this is only here temporarilly until we updtae dbt-coves
DATACOVES__AIRFLOW_DAGS_YML_PATH: ''
DATACOVES__AIRFLOW_DAGS_PATH: ''
DATACOVES__AIRBYTE_HOST_NAME: ''
DATACOVES__AIRBYTE_PORT: ''
steps:
- name: Checkout branch
uses: actions/checkout@v2
with:
ref: ${{ github.event.push.head.sha }}
fetch-depth: 0
- name: Set Secure Directory
run: git config --global --add safe.directory /__w/${{ github.event.repository.name }}/${{ github.event.repository.name }}
- name: Install dbt packages
run: "dbt deps"
- name: Get prod manifest
id: prod-manifest
run: "../automate/dbt/get_artifacts.sh"
# this is needed bec our sample dynamic table is not getting refreshed
- name: Refresh dynamic tables before blue-green
run: "dbt run -s materialization config.materialized:dynamic_table --full-refresh"
- name: Run blue / green deployment
if: contains(github.event.pull_request.labels.*.name, 'full-refresh') != true
id: run-blue-green
env:
MANIFEST_FOUND: ${{ steps.prod-manifest.outputs.manifest_found }}
run: "../automate/blue_green_run.py --ci-cd-run"
- name: Run blue / green deployment with full-refresh
if: contains(github.event.pull_request.labels.*.name, 'full-refresh')
id: run-blue-green-full-refresh
env:
MANIFEST_FOUND: ${{ steps.prod-manifest.outputs.manifest_found }}
run: "../automate/blue_green_run.py --ci-cd-run --full-refresh"
- name: Drop orphaned relations in db that are no longer in dbt
run: "dbt run-operation drop_orphaned_relations --args '{\"dry_run\": false}'"
- name: Generate dbt documentation
run: "dbt docs generate"
- name: Deploy docs 🚀
uses: JamesIves/[email protected]
with:
branch: dbt-docs
folder: transform/target
- name: Deploy Streamlit App
run: "dbt run-operation deploy_streamlit_app --args \"{app_path: '../visualize/streamlit/loans-example', app_file: loans.py}\""
- name: Upload dbt artifacts
run: "dbt run-operation upload_artifacts"
- uses: fregante/setup-git-user@v2
- name: Bump dbt project and git project version
run: "../automate/dbt/bump_dbt_project.sh"
- name: Push version changes
uses: ad-m/[email protected]
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
branch: ${{ github.ref_name }}
tags: true
- name: Drop staging db on Failure
if: always() && (steps.run-blue-green.outcome == 'failure')
run: "dbt --no-write-json run-operation drop_staging_db --args 'db_name: ${{env.DATACOVES__STAGING__DATABASE}}'" # yamllint disable-line rule:line-length