Update testing structure for dbt Labs testing support - postgres (#181)

* add tox * add postgres/redshift/bq * add more wh, update var names * fix profiles key * move supported adapters * Add CI workflow * update to allow circleci to keep working * fix BQ var name * remove config from profile * move to just support postgres * fix vars * use premade workflow * add newline * use merged version * add comments about future adapters * use tag * update readme * Add lines * reowkr profile so it can be reused * add sample env files and fix comment * Update .circleci/config.yml Co-authored-by: Doug Beatty <[email protected]> * let circleCI set the schema * fix readme instructions * undo non-postgres changes --------- Co-authored-by: Doug Beatty <[email protected]>
dbt-labs · Oct 18, 2024 · e532bd4 · e532bd4
1 parent 171343d
commit e532bd4
Show file tree

Hide file tree

Showing 12 changed files with 119 additions and 51 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -10,7 +10,7 @@ jobs:
       - checkout
 
       - run:
-          run: setup_creds
+          name: setup_creds
           command: |
             echo $BIGQUERY_SERVICE_ACCOUNT_JSON > ${HOME}/bigquery-service-key.json
 
@@ -29,11 +29,12 @@ jobs:
       - run:
           name: "Run Tests - Postgres"
           environment:
-            POSTGRES_TEST_HOST: localhost
-            POSTGRES_TEST_USER: root
-            POSTGRES_TEST_PASS: ""
-            POSTGRES_TEST_PORT: 5432
-            POSTGRES_TEST_DBNAME: circle_test
+            POSTGRES_HOST: localhost
+            POSTGRES_USER: root
+            DBT_ENV_SECRET_POSTGRES_PASS: ""
+            POSTGRES_PORT: 5432
+            POSTGRES_DATABASE: circle_test
+            POSTGRES_SCHEMA: codegen_integration_tests_postgres
           command: |
             . dbt_venv/bin/activate
             cd integration_tests

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -0,0 +1,33 @@
+# **what?**
+# Run tests for dbt-codegen against supported adapters
+
+# **why?**
+# To ensure that dbt-codegen works as expected with all supported adapters
+
+# **when?**
+# On every PR, and every push to main and when manually triggered
+
+name: Package Integration Tests
+
+on:
+    push:
+        branches:
+            - main
+    pull_request:
+    workflow_dispatch:
+
+jobs:
+  run-tests:
+      uses: dbt-labs/dbt-package-testing/.github/workflows/run_tox.yml@v1
+      # this just tests with postgres so no variables need to be passed through.
+      # When it's time to add more adapters you will need to pass through inputs for
+      # the other adapters as shown in the below example for redshift
+    #   with:
+    #     # redshift
+    #     REDSHIFT_HOST: ${{ vars.REDSHIFT_HOST }}
+    #     REDSHIFT_USER: ${{ vars.REDSHIFT_USER }}
+    #     REDSHIFT_DATABASE: ${{ vars.REDSHIFT_DATABASE }}
+    #     REDSHIFT_SCHEMA: "integration_tests_redshift_${{ github.run_number }}"
+    #     REDSHIFT_PORT: ${{ vars.REDSHIFT_PORT }}
+    #   secrets:
+    #     DBT_ENV_SECRET_REDSHIFT_PASS: ${{ secrets.DBT_ENV_SECRET_REDSHIFT_PASS }}
diff --git a/Makefile b/Makefile
@@ -4,6 +4,11 @@
 test: ## Run the integration tests.
 	@./run_test.sh $(target)
 
+.PHONY: test_tox
+test: ## Run the integration tests with tox
+	@\
+	tox -e dbt_integration_$(target)
+
 .PHONY: dev
 dev: ## Installs dbt-* packages in develop mode along with development dependencies.
 	@\

diff --git a/integration_tests/.env/bigquery.env b/integration_tests/.env/bigquery.env
@@ -0,0 +1,3 @@
+BIGQUERY_PROJECT=
+BIGQUERY_SCHEMA=
+BIGQUERY_KEYFILE_JSON=
diff --git a/integration_tests/.env/postgres.env b/integration_tests/.env/postgres.env
@@ -0,0 +1,6 @@
+POSTGRES_HOST=localhost
+POSTGRES_USER=root
+DBT_ENV_SECRET_POSTGRES_PASS=password
+POSTGRES_PORT=5432
+POSTGRES_DATABASE=codegen_test
+POSTGRES_SCHEMA=codegen_integration_tests_postgres
diff --git a/integration_tests/.env/redshift.env b/integration_tests/.env/redshift.env
@@ -0,0 +1,5 @@
+REDSHIFT_TEST_HOST=
+REDSHIFT_TEST_USER=
+REDSHIFT_TEST_PASS=
+REDSHIFT_TEST_DBNAME=
+REDSHIFT_TEST_PORT=
diff --git a/integration_tests/.env/snowflake.env b/integration_tests/.env/snowflake.env
@@ -0,0 +1,6 @@
+SNOWFLAKE_TEST_ACCOUNT=
+SNOWFLAKE_TEST_USER=
+SNOWFLAKE_TEST_PASSWORD=
+SNOWFLAKE_TEST_ROLE=
+SNOWFLAKE_TEST_DATABASE=
+SNOWFLAKE_TEST_WAREHOUSE=
diff --git a/integration_tests/README.md b/integration_tests/README.md
@@ -44,35 +44,7 @@ You can set these env vars in a couple ways:
 - **Temporary**: Set these environment variables in your shell before running the tests. This is the easiest way to get started, but you'll have to set them every time you open a new terminal.
 - **Reusable**: If you anticipate developing for multiple sessions, set these environment variables in your shell profile (like `~/.bashrc` or `~/.zshrc`). This way, you won't have to set them every time you open a new terminal.
 
-The environment variables you'll need to set for each adapter are:
-
-```bash
-# Postgres — these are the defaults for the Docker container so actually have values
-export POSTGRES_TEST_HOST=localhost
-export POSTGRES_TEST_USER=root
-export POSTGRES_TEST_PASS=''
-export POSTGRES_TEST_PORT=5432
-export POSTGRES_TEST_DBNAME=circle_test
-
-# BigQuery
-export BIGQUERY_SERVICE_KEY_PATH=
-export BIGQUERY_TEST_DATABASE=
-
-# Redshift
-export REDSHIFT_TEST_HOST=
-export REDSHIFT_TEST_USER=
-export REDSHIFT_TEST_PASS=
-export REDSHIFT_TEST_DBNAME=
-export REDSHIFT_TEST_PORT=
-
-# Snowflake
-export SNOWFLAKE_TEST_ACCOUNT=
-export SNOWFLAKE_TEST_USER=
-export SNOWFLAKE_TEST_PASSWORD=
-export SNOWFLAKE_TEST_ROLE=
-export SNOWFLAKE_TEST_DATABASE=
-export SNOWFLAKE_TEST_WAREHOUSE=
-```
+The environment variables you'll need to set for each adapter can be found in [integration_tests/.env/](integration_tests/.env/).
 
 ### Setup Postgres or other database targets
 
@@ -139,7 +111,9 @@ source .venv/bin/activate
 
 ## Write or modify an integration test
 
-### Run the integration tests
+Run all the tests _before_ you start developing to make sure everything is working as expected before you start making changes. Nothing is worse than spending a ton of time troubleshooting a failing test, only to realize it was failing before you touched anything. This will also ensure that you have the correct environment variables set up and that your database is running.
+
+### Run the Circle CI integration tests
 
 To run all the integration tests on your local machine like they will get run in CI:
 
@@ -157,7 +131,13 @@ make test target=postgres
 ./run_test.sh postgres
 ```
 
-Run all the tests _before_ you start developing to make sure everything is working as expected before you start making changes. Nothing is worse than spending a ton of time troubleshooting a failing test, only to realize it was failing before you touched anything. This will also ensure that you have the correct environment variables set up and that your database is running.
+### Run the tox Supported Tests
+
+To run all the integration tests on your local machine like they will get run in the CI (using GitHub workflows with tox):
+
+```shell
+make test_tox target=postgres
+```
 
 ### Creating a new integration test
 

diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml
@@ -15,6 +15,10 @@ clean-targets:
   - "target"
   - "dbt_packages"
 
+flags:
+  send_anonymous_usage_stats: False
+  use_colors: True
+
 seeds:
   +schema: raw_data
   +quote_columns: false

diff --git a/integration_tests/profiles.yml b/integration_tests/profiles.yml
@@ -1,23 +1,20 @@
 
-# HEY! This file is used in the dbt-codegen integrations tests with CircleCI.
-# You should __NEVER__ check credentials into version control. Thanks for reading :)
-
-config:
-    send_anonymous_usage_stats: False
-    use_colors: True
+# HEY! This file is used in the dbt-codegen integrations tests with GitHub CI.
+# You should __NEVER__ check credentials into version control. That's why we use environment variables everywhere.
+# Thanks for reading :)
 
 integration_tests:
   target: postgres
   outputs:
     postgres:
-      type: postgres
-      host: "{{ env_var('POSTGRES_TEST_HOST') }}"
-      user: "{{ env_var('POSTGRES_TEST_USER') }}"
-      pass: "{{ env_var('POSTGRES_TEST_PASS') }}"
-      port: "{{ env_var('POSTGRES_TEST_PORT') | as_number }}"
-      dbname: "{{ env_var('POSTGRES_TEST_DBNAME') }}"
-      schema: codegen_integration_tests_postgres
-      threads: 1
+      type: "postgres"
+      host: "{{ env_var('POSTGRES_HOST') }}"
+      user: "{{ env_var('POSTGRES_USER') }}"
+      pass: "{{ env_var('DBT_ENV_SECRET_POSTGRES_PASS') }}"
+      port: "{{ env_var('POSTGRES_PORT') | as_number }}"
+      dbname: "{{ env_var('POSTGRES_DATABASE') }}"
+      schema: "{{ env_var('POSTGRES_SCHEMA') }}"
+      threads: 5
 
     redshift:
       type: redshift

diff --git a/supported_adapters.env b/supported_adapters.env
@@ -0,0 +1 @@
+SUPPORTED_ADAPTERS=postgres
diff --git a/tox.ini b/tox.ini
@@ -0,0 +1,27 @@
+[tox]
+skipsdist = True
+envlist = lint_all, testenv
+
+[testenv]
+passenv =
+    # postgres env vars
+    POSTGRES_HOST
+    POSTGRES_USER
+    DBT_ENV_SECRET_POSTGRES_PASS
+    POSTGRES_PORT
+    POSTGRES_DATABASE
+    POSTGRES_SCHEMA
+
+# Postgres integration tests for centralized dbt testing
+# run dbt commands directly, assumes dbt is already installed in environment
+[testenv:dbt_integration_postgres]
+changedir = integration_tests
+allowlist_externals = 
+    dbt
+skip_install = true
+commands =
+    dbt --warn-error deps --target postgres
+    dbt --warn-error run-operation create_source_table --target postgres
+    dbt --warn-error seed --target postgres --full-refresh
+    dbt --warn-error run --target postgres
+    dbt --warn-error test --target postgres