From 44284bfc467b853fd7b2abe5e3c9444b0478656b Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Tue, 11 Jun 2024 10:02:37 -0500 Subject: [PATCH 01/24] add tox --- .circleci/config.yml | 6 +++ integration_tests/profiles.yml | 19 ++++---- tox.ini | 84 ++++++++++++++++++++++++++++++++++ 3 files changed, 100 insertions(+), 9 deletions(-) create mode 100644 tox.ini diff --git a/.circleci/config.yml b/.circleci/config.yml index 2585c60..6d49002 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -34,6 +34,7 @@ jobs: POSTGRES_TEST_PASS: "" POSTGRES_TEST_PORT: 5432 POSTGRES_TEST_DBNAME: circle_test + POSTGRES_TEST_SCHEMA: codegen_integration_tests_postgres command: | . dbt_venv/bin/activate cd integration_tests @@ -45,6 +46,8 @@ jobs: - run: name: "Run Tests - Redshift" + environment: + REDSHIFT_TEST_SCHEMA: codegen_integration_tests_redshift command: | . dbt_venv/bin/activate echo `pwd` @@ -57,6 +60,8 @@ jobs: - run: name: "Run Tests - Snowflake" + environment: + SNOWFLAKE_TEST_SCHEMA: codegen_integration_tests_snowflake command: | . dbt_venv/bin/activate echo `pwd` @@ -71,6 +76,7 @@ jobs: name: "Run Tests - BigQuery" environment: BIGQUERY_SERVICE_KEY_PATH: "/home/circleci/bigquery-service-key.json" + REDSHIFT_TEST_SCHEMA: codegen_integration_tests_bigquery command: | . dbt_venv/bin/activate diff --git a/integration_tests/profiles.yml b/integration_tests/profiles.yml index ffe7a71..6829dbf 100644 --- a/integration_tests/profiles.yml +++ b/integration_tests/profiles.yml @@ -1,6 +1,7 @@ # HEY! This file is used in the dbt-codegen integrations tests with CircleCI. -# You should __NEVER__ check credentials into version control. Thanks for reading :) +# You should __NEVER__ check credentials into version control. That's why this uses env vars. +# Thanks for reading :) config: send_anonymous_usage_stats: False @@ -13,37 +14,37 @@ integration_tests: type: postgres host: "{{ env_var('POSTGRES_TEST_HOST') }}" user: "{{ env_var('POSTGRES_TEST_USER') }}" - pass: "{{ env_var('POSTGRES_TEST_PASS') }}" + pass: "{{ env_var('DBT_ENV_SECRET_POSTGRES_TEST_PASS') }}" port: "{{ env_var('POSTGRES_TEST_PORT') | as_number }}" dbname: "{{ env_var('POSTGRES_TEST_DBNAME') }}" - schema: codegen_integration_tests_postgres + schema: "{{ env_var('POSTGRES_TEST_SCHEMA') }}" threads: 1 redshift: type: redshift host: "{{ env_var('REDSHIFT_TEST_HOST') }}" user: "{{ env_var('REDSHIFT_TEST_USER') }}" - pass: "{{ env_var('REDSHIFT_TEST_PASS') }}" + pass: "{{ env_var('DBT_ENV_SECRET_REDSHIFT_TEST_PASS') }}" dbname: "{{ env_var('REDSHIFT_TEST_DBNAME') }}" port: "{{ env_var('REDSHIFT_TEST_PORT') | as_number }}" - schema: codegen_integration_tests_redshift + schema: "{{ env_var('REDSHIFT_TEST_SCHEMA') }}" threads: 1 bigquery: type: bigquery method: service-account - keyfile: "{{ env_var('BIGQUERY_SERVICE_KEY_PATH') }}" + keyfile: "{{ env_var('DBT_ENV_SECRET_BIGQUERY_SERVICE_KEY_PATH') }}" project: "{{ env_var('BIGQUERY_TEST_DATABASE') }}" - schema: codegen_integration_tests_bigquery + schema: "{{ env_var('BIGQUERY_TEST_SCHEMA') }}" threads: 1 snowflake: type: snowflake account: "{{ env_var('SNOWFLAKE_TEST_ACCOUNT') }}" user: "{{ env_var('SNOWFLAKE_TEST_USER') }}" - password: "{{ env_var('SNOWFLAKE_TEST_PASSWORD') }}" + password: "{{ env_var('DBT_ENV_SECRET_SNOWFLAKE_TEST_PASS') }}" role: "{{ env_var('SNOWFLAKE_TEST_ROLE') }}" database: "{{ env_var('SNOWFLAKE_TEST_DATABASE') }}" warehouse: "{{ env_var('SNOWFLAKE_TEST_WAREHOUSE') }}" - schema: codegen_integration_tests_snowflake + schema: "{{ env_var('SNOWFLAKE_TEST_SCHEMA') }}" threads: 1 diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..8c4fe1d --- /dev/null +++ b/tox.ini @@ -0,0 +1,84 @@ +[tox] +skipsdist = True +envlist = lint_all, testenv +# this needs to be defined on a single line. tox has problems with multiline lists +# This list determined which adapters will be run by dbt centralized testing +supported_adapters = snowflake + +[testenv] +passenv = + DBT_PROFILES_DIR + # postgres env vars + POSTGRES_TEST_HOST + POSTGRES_TEST_USER + DBT_ENV_SECRET_POSTGRES_TEST_PASS + POSTGRES_TEST_PORT + POSTGRES_TEST_DBNAME + POSTGRES_TEST_SCHEMA + # snowflake env vars + SNOWFLAKE_TEST_ACCOUNT + SNOWFLAKE_TEST_USER + DBT_ENV_SECRET_SNOWFLAKE_TEST_PASS + SNOWFLAKE_TEST_ROLE + SNOWFLAKE_TEST_DATABASE + SNOWFLAKE_TEST_WAREHOUSE + SNOWFLAKE_TEST_SCHEMA + +# This is required to ensure that the correct adapters are tested +[testenv:list_supported_adapters] +allowlist_externals = echo +commands = echo {[tox]supported_adapters} + +# Snowflake integration tests for centralized dbt testing +# run dbt commands directly, assumes dbt is already installed in environment +[testenv:dbt_integration_snowflake] +changedir = integration_tests +allowlist_externals = + dbt +skip_install = true +commands = + dbt --warn-error deps --target snowflake + dbt --warn-error run-operation create_source_table --target snowflake + dbt --warn-error seed --target snowflake --full-refresh + dbt --warn-error run --target snowflake + dbt --warn-error test --target snowflake + +# Postgres integration tests for centralized dbt testing +# run dbt commands directly, assumes dbt is already installed in environment +[testenv:dbt_integration_postgres] +changedir = integration_tests +allowlist_externals = + dbt +skip_install = true +commands = + dbt --warn-error deps --target postgres + dbt --warn-error run-operation create_source_table --target postgres + dbt --warn-error seed --target postgres --full-refresh + dbt --warn-error run --target postgres + dbt --warn-error test --target postgres + +# BigQuery integration tests for centralized dbt testing +[testenv:dbt_integration_bigquery] +changedir = integration_tests +allowlist_externals = + dbt +skip_install = true +commands = + dbt --warn-error deps --target bigquery + dbt --warn-error run-operation create_source_table --target bigquery + dbt --warn-error seed --target bigquery --full-refresh + dbt --warn-error run --target bigquery + dbt --warn-error test --target bigquery + +# redshift integration tests for centralized dbt testing +[testenv:dbt_integration_redshift] +changedir = integration_tests +allowlist_externals = + dbt +skip_install = true +commands = + dbt --warn-error deps --target redshift + dbt --warn-error run-operation create_source_table --target redshift + dbt --warn-error seed --target redshift --full-refresh + dbt --warn-error run --target redshift + dbt --warn-error test --target redshift \ No newline at end of file From b7b521c491e60de1d5fbc481325b57101c128420 Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Fri, 14 Jun 2024 08:55:54 -0500 Subject: [PATCH 02/24] add postgres/redshift/bq --- tox.ini | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/tox.ini b/tox.ini index 8c4fe1d..63e237c 100644 --- a/tox.ini +++ b/tox.ini @@ -3,17 +3,16 @@ skipsdist = True envlist = lint_all, testenv # this needs to be defined on a single line. tox has problems with multiline lists # This list determined which adapters will be run by dbt centralized testing -supported_adapters = snowflake +supported_adapters = postgres, snowflake, redshift, bigquery [testenv] passenv = - DBT_PROFILES_DIR # postgres env vars POSTGRES_TEST_HOST POSTGRES_TEST_USER DBT_ENV_SECRET_POSTGRES_TEST_PASS POSTGRES_TEST_PORT - POSTGRES_TEST_DBNAME + POSTGRES_TEST_DBNAMEs POSTGRES_TEST_SCHEMA # snowflake env vars SNOWFLAKE_TEST_ACCOUNT @@ -23,6 +22,17 @@ passenv = SNOWFLAKE_TEST_DATABASE SNOWFLAKE_TEST_WAREHOUSE SNOWFLAKE_TEST_SCHEMA + # redshift + REDSHIFT_TEST_HOST + REDSHIFT_TEST_USER + DBT_ENV_SECRET_REDSHIFT_TEST_PASS + REDSHIFT_TEST_DATABASE + REDSHIFT_TEST_SCHEMA + REDSHIFT_TEST_PORT + # bigquery + BIGQUERY_PROJECT + DBT_ENV_SECRET_BIGQUERY_KEYFILE_JSON + BIGQUERY_TEST_SCHEMA # This is required to ensure that the correct adapters are tested [testenv:list_supported_adapters] From 7e7a3e4aa5a1c97d47cc54331db15906775825bb Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Fri, 14 Jun 2024 11:24:49 -0500 Subject: [PATCH 03/24] add more wh, update var names --- integration_tests/README.md | 34 ++++++++++----------- integration_tests/dbt_project.yml | 4 +++ integration_tests/profiles.yml | 50 ++++++++++++++----------------- tox.ini | 40 ++++++++++++------------- 4 files changed, 64 insertions(+), 64 deletions(-) diff --git a/integration_tests/README.md b/integration_tests/README.md index 1337e57..d7c5108 100644 --- a/integration_tests/README.md +++ b/integration_tests/README.md @@ -48,30 +48,30 @@ The environment variables you'll need to set for each adapter are: ```bash # Postgres — these are the defaults for the Docker container so actually have values -export POSTGRES_TEST_HOST=localhost -export POSTGRES_TEST_USER=root -export POSTGRES_TEST_PASS='' -export POSTGRES_TEST_PORT=5432 -export POSTGRES_TEST_DBNAME=circle_test +export POSTGRES_HOST=localhost +export POSTGRES_USER=root +export POSTGRES_PASS='' +export POSTGRES_PORT=5432 +export POSTGRES_DATABASE=circle_test # BigQuery export BIGQUERY_SERVICE_KEY_PATH= -export BIGQUERY_TEST_DATABASE= +export BIGQUERY_PROJECT= # Redshift -export REDSHIFT_TEST_HOST= -export REDSHIFT_TEST_USER= -export REDSHIFT_TEST_PASS= -export REDSHIFT_TEST_DBNAME= -export REDSHIFT_TEST_PORT= +export REDSHIFT_HOST= +export REDSHIFT_USER= +export REDSHIFT_PASS= +export REDSHIFT_DATABASE= +export REDSHIFT_PORT= # Snowflake -export SNOWFLAKE_TEST_ACCOUNT= -export SNOWFLAKE_TEST_USER= -export SNOWFLAKE_TEST_PASSWORD= -export SNOWFLAKE_TEST_ROLE= -export SNOWFLAKE_TEST_DATABASE= -export SNOWFLAKE_TEST_WAREHOUSE= +export SNOWFLAKE_ACCOUNT= +export SNOWFLAKE_USER= +export SNOWFLAKE_PASSWORD= +export SNOWFLAKE_ROLE= +export SNOWFLAKE_DATABASE= +export SNOWFLAKE_WAREHOUSE= ``` ### Setup Postgres or other database targets diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index 6f194d0..7d0c8f2 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -15,6 +15,10 @@ clean-targets: - "target" - "dbt_packages" +flags: + send_anonymous_usage_stats: False + use_colors: True + seeds: +schema: raw_data +quote_columns: false diff --git a/integration_tests/profiles.yml b/integration_tests/profiles.yml index 6829dbf..62367a4 100644 --- a/integration_tests/profiles.yml +++ b/integration_tests/profiles.yml @@ -3,48 +3,44 @@ # You should __NEVER__ check credentials into version control. That's why this uses env vars. # Thanks for reading :) -config: - send_anonymous_usage_stats: False - use_colors: True - integration_tests: target: postgres outputs: postgres: type: postgres - host: "{{ env_var('POSTGRES_TEST_HOST') }}" - user: "{{ env_var('POSTGRES_TEST_USER') }}" - pass: "{{ env_var('DBT_ENV_SECRET_POSTGRES_TEST_PASS') }}" - port: "{{ env_var('POSTGRES_TEST_PORT') | as_number }}" - dbname: "{{ env_var('POSTGRES_TEST_DBNAME') }}" - schema: "{{ env_var('POSTGRES_TEST_SCHEMA') }}" + host: "{{ env_var('POSTGRES_HOST') }}" + user: "{{ env_var('POSTGRES_USER') }}" + pass: "{{ env_var('DBT_ENV_SECRET_POSTGRES_PASS') }}" + port: "{{ env_var('POSTGRES_PORT') | as_number }}" + dbname: "{{ env_var('POSTGRES_DATABASE') }}" + schema: "{{ env_var('POSTGRES_SCHEMA') }}" threads: 1 redshift: type: redshift - host: "{{ env_var('REDSHIFT_TEST_HOST') }}" - user: "{{ env_var('REDSHIFT_TEST_USER') }}" - pass: "{{ env_var('DBT_ENV_SECRET_REDSHIFT_TEST_PASS') }}" - dbname: "{{ env_var('REDSHIFT_TEST_DBNAME') }}" - port: "{{ env_var('REDSHIFT_TEST_PORT') | as_number }}" - schema: "{{ env_var('REDSHIFT_TEST_SCHEMA') }}" + host: "{{ env_var('REDSHIFT_HOST') }}" + user: "{{ env_var('REDSHIFT_USER') }}" + pass: "{{ env_var('DBT_ENV_SECRET_REDSHIFT_PASS') }}" + dbname: "{{ env_var('REDSHIFT_DATABASE') }}" + port: "{{ env_var('REDSHIFT_PORT') | as_number }}" + schema: "{{ env_var('REDSHIFT_SCHEMA') }}" threads: 1 bigquery: type: bigquery - method: service-account - keyfile: "{{ env_var('DBT_ENV_SECRET_BIGQUERY_SERVICE_KEY_PATH') }}" - project: "{{ env_var('BIGQUERY_TEST_DATABASE') }}" - schema: "{{ env_var('BIGQUERY_TEST_SCHEMA') }}" + method: service-account-json + keyfile: "{{ env_var('DBT_ENV_SECRET_BIGQUERY_KEYFILE_JSON') | as_native }}" + project: "{{ env_var('BIGQUERY_PROJECT') }}" + schema: "{{ env_var('BIGQUERY_SCHEMA') }}" threads: 1 snowflake: type: snowflake - account: "{{ env_var('SNOWFLAKE_TEST_ACCOUNT') }}" - user: "{{ env_var('SNOWFLAKE_TEST_USER') }}" - password: "{{ env_var('DBT_ENV_SECRET_SNOWFLAKE_TEST_PASS') }}" - role: "{{ env_var('SNOWFLAKE_TEST_ROLE') }}" - database: "{{ env_var('SNOWFLAKE_TEST_DATABASE') }}" - warehouse: "{{ env_var('SNOWFLAKE_TEST_WAREHOUSE') }}" - schema: "{{ env_var('SNOWFLAKE_TEST_SCHEMA') }}" + account: "{{ env_var('SNOWFLAKE_ACCOUNT') }}" + user: "{{ env_var('SNOWFLAKE_USER') }}" + password: "{{ env_var('DBT_ENV_SECRET_SNOWFLAKE_PASS') }}" + role: "{{ env_var('SNOWFLAKE_ROLE') }}" + database: "{{ env_var('SNOWFLAKE_DATABASE') }}" + warehouse: "{{ env_var('SNOWFLAKE_WAREHOUSE') }}" + schema: "{{ env_var('SNOWFLAKE_SCHEMA') }}" threads: 1 diff --git a/tox.ini b/tox.ini index 63e237c..209c3b5 100644 --- a/tox.ini +++ b/tox.ini @@ -8,31 +8,31 @@ supported_adapters = postgres, snowflake, redshift, bigquery [testenv] passenv = # postgres env vars - POSTGRES_TEST_HOST - POSTGRES_TEST_USER - DBT_ENV_SECRET_POSTGRES_TEST_PASS - POSTGRES_TEST_PORT - POSTGRES_TEST_DBNAMEs - POSTGRES_TEST_SCHEMA + POSTGRES_HOST + POSTGRES_USER + DBT_ENV_SECRET_POSTGRES_PASS + POSTGRES_PORT + POSTGRES_DATABASE + POSTGRES_SCHEMA # snowflake env vars - SNOWFLAKE_TEST_ACCOUNT - SNOWFLAKE_TEST_USER - DBT_ENV_SECRET_SNOWFLAKE_TEST_PASS - SNOWFLAKE_TEST_ROLE - SNOWFLAKE_TEST_DATABASE - SNOWFLAKE_TEST_WAREHOUSE - SNOWFLAKE_TEST_SCHEMA + SNOWFLAKE_ACCOUNT + SNOWFLAKE_USER + DBT_ENV_SECRET_SNOWFLAKE_PASS + SNOWFLAKE_ROLE + SNOWFLAKE_DATABASE + SNOWFLAKE_WAREHOUSE + SNOWFLAKE_SCHEMA # redshift - REDSHIFT_TEST_HOST - REDSHIFT_TEST_USER - DBT_ENV_SECRET_REDSHIFT_TEST_PASS - REDSHIFT_TEST_DATABASE - REDSHIFT_TEST_SCHEMA - REDSHIFT_TEST_PORT + REDSHIFT_HOST + REDSHIFT_USER + DBT_ENV_SECRET_REDSHIFT_PASS + REDSHIFT_DATABASE + REDSHIFT_SCHEMA + REDSHIFT_PORT # bigquery BIGQUERY_PROJECT DBT_ENV_SECRET_BIGQUERY_KEYFILE_JSON - BIGQUERY_TEST_SCHEMA + BIGQUERY_SCHEMA # This is required to ensure that the correct adapters are tested [testenv:list_supported_adapters] From c67d35c8a367188e5212581e4f856e2b44cd0914 Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Fri, 14 Jun 2024 11:35:45 -0500 Subject: [PATCH 04/24] fix profiles key --- integration_tests/profiles.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/integration_tests/profiles.yml b/integration_tests/profiles.yml index 62367a4..3f38f74 100644 --- a/integration_tests/profiles.yml +++ b/integration_tests/profiles.yml @@ -29,10 +29,10 @@ integration_tests: bigquery: type: bigquery method: service-account-json - keyfile: "{{ env_var('DBT_ENV_SECRET_BIGQUERY_KEYFILE_JSON') | as_native }}" project: "{{ env_var('BIGQUERY_PROJECT') }}" - schema: "{{ env_var('BIGQUERY_SCHEMA') }}" - threads: 1 + keyfile_json: "{{ env_var('DBT_ENV_SECRET_BIGQUERY_KEYFILE_JSON') | as_native }}" + dataset: "{{ env_var('BIGQUERY_SCHEMA') }}" + threads: 10 snowflake: type: snowflake From f049256dd6eba6fd6b0b1c07e83b097cebf7daaa Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Fri, 28 Jun 2024 12:03:04 -0500 Subject: [PATCH 05/24] move supported adapters --- supported_adapters.env | 1 + tox.ini | 8 -------- 2 files changed, 1 insertion(+), 8 deletions(-) create mode 100644 supported_adapters.env diff --git a/supported_adapters.env b/supported_adapters.env new file mode 100644 index 0000000..ed5ad27 --- /dev/null +++ b/supported_adapters.env @@ -0,0 +1 @@ +SUPPORTED_ADAPTERS=postgres,snowflake,redshift,bigquery \ No newline at end of file diff --git a/tox.ini b/tox.ini index 209c3b5..d2df2e9 100644 --- a/tox.ini +++ b/tox.ini @@ -1,9 +1,6 @@ [tox] skipsdist = True envlist = lint_all, testenv -# this needs to be defined on a single line. tox has problems with multiline lists -# This list determined which adapters will be run by dbt centralized testing -supported_adapters = postgres, snowflake, redshift, bigquery [testenv] passenv = @@ -34,11 +31,6 @@ passenv = DBT_ENV_SECRET_BIGQUERY_KEYFILE_JSON BIGQUERY_SCHEMA -# This is required to ensure that the correct adapters are tested -[testenv:list_supported_adapters] -allowlist_externals = echo -commands = echo {[tox]supported_adapters} - # Snowflake integration tests for centralized dbt testing # run dbt commands directly, assumes dbt is already installed in environment [testenv:dbt_integration_snowflake] From 7f2b6b6cf5b14b193feb4fb26d1aeabb12b9693e Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Wed, 24 Jul 2024 18:38:44 -0500 Subject: [PATCH 06/24] Add CI workflow --- .github/workflows/ci.yml | 139 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 139 insertions(+) create mode 100644 .github/workflows/ci.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..a2647fa --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,139 @@ +# **what?** +# Run tests for dbt-codegen against supported adapters + +# **why?** +# To ensure that dbt-codegen works as expected with all supported adapters + +# **when?** +# On every PR, and every push to main and when manually triggered + +name: Package Integration Tests + +on: + push: + branches: + - main + pull_request: + workflow_dispatch: + inputs: + adapter: + description: The adapter to test against. Defaults to all supported adapters when blank. + type: string + required: false + +env: + PYTHON_VERSION: "3.11" + +jobs: + determine-supported-adapters: + runs-on: ubuntu-latest + outputs: + adapters: ${{ steps.supported-adapters.outputs.adapters }} + steps: + - name: "Checkout ${{ github.event.repository }}" + uses: actions/checkout@v4 + + - name: "Set up Python ${{ env.PYTHON_VERSION }}" + uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - name: "Install tox" + run: | + python -m pip install --upgrade pip + pip install tox + + - name: "Get list of supported adapters or use input adapter only" + id: list-adapters + run: | + if [ -z "${{ inputs.adapter }}" ]; then + # github adds a pip freeze and a new line we need to strip out + source supported_adapters.env + echo $SUPPORTED_ADAPTERS + echo "test_adapters=$SUPPORTED_ADAPTERS" >> $GITHUB_OUTPUT + else + echo "test_adapters=${{ inputs.adapter }}" >> $GITHUB_OUTPUT + fi + + - name: "Format adapter list for use as the matrix" + id: supported-adapters + run: | + # Convert to JSON array and output + supported_adapters=$(echo "${{ steps.list-adapters.outputs.test_adapters }}" | jq -Rc 'split(",")') + echo $supported_adapters + echo "adapters=$supported_adapters" >> $GITHUB_OUTPUT + + - name: "[ANNOTATION] ${{ github.event.repository.name }} - Testing ${{ steps.supported-adapters.outputs.adapters }}" + run: | + title="${{ github.event.repository.name }} - adapters to test" + message="The workflow will run tests for the following adapters: ${{ steps.supported-adapters.outputs.adapters }}" + echo "::notice $title::$message" + + run-tests: + runs-on: ubuntu-latest + needs: [determine-supported-adapters] + services: + postgres: + image: postgres + env: + POSTGRES_USER: ${{ vars.POSTGRES_USER }} + POSTGRES_PASSWORD: ${{ secrets.POSTGRES_PASS }} + POSTGRES_DB: ${{ vars.POSTGRES_DATABASE }} + POSTGRES_HOST: ${{ vars.POSTGRES_HOST }} + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + - 5432:5432 + strategy: + fail-fast: false + matrix: + adapter: ${{fromJson(needs.determine-supported-adapters.outputs.adapters)}} + + steps: + - name: "Checkout ${{ github.event.repository }} " + uses: actions/checkout@v4 + + - name: "Set up Python ${{ env.PYTHON_VERSION }}" + uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - name: "Install ${{ matrix.adapter }}" + run: | + python -m pip install --upgrade pip + pip install dbt-${{ matrix.adapter }} + + - name: "Install tox" + run: | + python -m pip install --upgrade pip + pip install tox + + - name: "Run integration tests with tox on ${{ matrix.adapter }}" + run: | + tox -e dbt_integration_${{ matrix.adapter }} + env: + POSTGRES_HOST: ${{ vars.POSTGRES_HOST }} + POSTGRES_USER: ${{ vars.POSTGRES_USER }} + DBT_ENV_SECRET_POSTGRES_PASS: ${{ secrets.POSTGRES_PASS }} + POSTGRES_PORT: 5432 + POSTGRES_DATABASE: ${{ vars.POSTGRES_DATABASE }} + POSTGRES_SCHEMA: "dbt_codegen_integration_tests_postgres_${{ github.run_number }}" + SNOWFLAKE_ACCOUNT: ${{ secrets.SNOWFLAKE_ACCOUNT }} + SNOWFLAKE_USER: ${{ vars.SNOWFLAKE_USER }} + DBT_ENV_SECRET_SNOWFLAKE_PASS: ${{ secrets.SNOWFLAKE_PASS }} + SNOWFLAKE_ROLE: ${{ vars.SNOWFLAKE_ROLE }} + SNOWFLAKE_DATABASE: ${{ vars.SNOWFLAKE_DATABASE }} + SNOWFLAKE_WAREHOUSE: ${{ vars.SNOWFLAKE_WAREHOUSE }} + SNOWFLAKE_SCHEMA: "dbt_codegen_integration_tests_snowflake_${{ github.run_number }}" + REDSHIFT_HOST: ${{ vars.REDSHIFT_HOST }} + REDSHIFT_USER: ${{ vars.REDSHIFT_USER }} + DBT_ENV_SECRET_REDSHIFT_PASS: ${{ secrets.REDSHIFT_PASS }} + REDSHIFT_DATABASE: ${{ vars.REDSHIFT_DATABASE }} + REDSHIFT_SCHEMA: "dbt_codegen_integration_tests_redshift_${{ github.run_number }}" + REDSHIFT_PORT: 5439 + BIGQUERY_PROJECT: ${{ vars.BIGQUERY_PROJECT }} + BIGQUERY_KEYFILE_JSON: ${{ secrets.BIGQUERY_KEYFILE_JSON }} + BIGQUERY_SCHEMA: "dbt_codegen_integration_tests_bigquery_${{ github.run_number }}" From 8f7962563dc34bd64c3ea14f81dd03d8f5d88079 Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Wed, 24 Jul 2024 19:02:42 -0500 Subject: [PATCH 07/24] update to allow circleci to keep working --- .circleci/config.yml | 4 ++ integration_tests/ci/sample.profiles.yml | 49 ++++++++++++++++++++++++ integration_tests/profiles.yml | 24 ++++++------ 3 files changed, 66 insertions(+), 11 deletions(-) create mode 100644 integration_tests/ci/sample.profiles.yml diff --git a/.circleci/config.yml b/.circleci/config.yml index 6d49002..0b55312 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -38,6 +38,7 @@ jobs: command: | . dbt_venv/bin/activate cd integration_tests + cp ci/sample.profiles.yml profiles.yml dbt --warn-error deps --target postgres dbt --warn-error run-operation create_source_table --target postgres dbt --warn-error seed --target postgres --full-refresh @@ -52,6 +53,7 @@ jobs: . dbt_venv/bin/activate echo `pwd` cd integration_tests + cp ci/sample.profiles.yml profiles.yml dbt --warn-error deps --target redshift dbt --warn-error run-operation create_source_table --target redshift dbt --warn-error seed --target redshift --full-refresh @@ -66,6 +68,7 @@ jobs: . dbt_venv/bin/activate echo `pwd` cd integration_tests + cp ci/sample.profiles.yml profiles.yml dbt --warn-error deps --target snowflake dbt --warn-error run-operation create_source_table --target snowflake dbt --warn-error seed --target snowflake --full-refresh @@ -82,6 +85,7 @@ jobs: . dbt_venv/bin/activate echo `pwd` cd integration_tests + cp ci/sample.profiles.yml profiles.yml dbt --warn-error deps --target bigquery dbt --warn-error run-operation create_source_table --target bigquery dbt --warn-error seed --target bigquery --full-refresh diff --git a/integration_tests/ci/sample.profiles.yml b/integration_tests/ci/sample.profiles.yml new file mode 100644 index 0000000..c96e81a --- /dev/null +++ b/integration_tests/ci/sample.profiles.yml @@ -0,0 +1,49 @@ + +# HEY! This file is used in the dbt-codegen integrations tests with CircleCI. +# You should __NEVER__ check credentials into version control. Thanks for reading :) + +config: + send_anonymous_usage_stats: False + use_colors: True + +integration_tests: + target: postgres + outputs: + postgres: + type: postgres + host: "{{ env_var('POSTGRES_TEST_HOST') }}" + user: "{{ env_var('POSTGRES_TEST_USER') }}" + pass: "{{ env_var('POSTGRES_TEST_PASS') }}" + port: "{{ env_var('POSTGRES_TEST_PORT') | as_number }}" + dbname: "{{ env_var('POSTGRES_TEST_DBNAME') }}" + schema: codegen_integration_tests_postgres + threads: 1 + + redshift: + type: redshift + host: "{{ env_var('REDSHIFT_TEST_HOST') }}" + user: "{{ env_var('REDSHIFT_TEST_USER') }}" + pass: "{{ env_var('REDSHIFT_TEST_PASS') }}" + dbname: "{{ env_var('REDSHIFT_TEST_DBNAME') }}" + port: "{{ env_var('REDSHIFT_TEST_PORT') | as_number }}" + schema: codegen_integration_tests_redshift + threads: 1 + + bigquery: + type: bigquery + method: service-account + keyfile: "{{ env_var('BIGQUERY_SERVICE_KEY_PATH') }}" + project: "{{ env_var('BIGQUERY_TEST_DATABASE') }}" + schema: codegen_integration_tests_bigquery + threads: 1 + + snowflake: + type: snowflake + account: "{{ env_var('SNOWFLAKE_TEST_ACCOUNT') }}" + user: "{{ env_var('SNOWFLAKE_TEST_USER') }}" + password: "{{ env_var('SNOWFLAKE_TEST_PASSWORD') }}" + role: "{{ env_var('SNOWFLAKE_TEST_ROLE') }}" + database: "{{ env_var('SNOWFLAKE_TEST_DATABASE') }}" + warehouse: "{{ env_var('SNOWFLAKE_TEST_WAREHOUSE') }}" + schema: codegen_integration_tests_snowflake + threads: 1 \ No newline at end of file diff --git a/integration_tests/profiles.yml b/integration_tests/profiles.yml index 3f38f74..508cb1d 100644 --- a/integration_tests/profiles.yml +++ b/integration_tests/profiles.yml @@ -1,41 +1,43 @@ -# HEY! This file is used in the dbt-codegen integrations tests with CircleCI. -# You should __NEVER__ check credentials into version control. That's why this uses env vars. +# HEY! This file is used in the dbt-utils integrations tests with CircleCI. +# You should __NEVER__ check credentials into version control. That's why we use environment variables everywhere. # Thanks for reading :) integration_tests: target: postgres outputs: postgres: - type: postgres + type: "postgres" host: "{{ env_var('POSTGRES_HOST') }}" user: "{{ env_var('POSTGRES_USER') }}" pass: "{{ env_var('DBT_ENV_SECRET_POSTGRES_PASS') }}" port: "{{ env_var('POSTGRES_PORT') | as_number }}" dbname: "{{ env_var('POSTGRES_DATABASE') }}" schema: "{{ env_var('POSTGRES_SCHEMA') }}" - threads: 1 + threads: 5 redshift: - type: redshift + type: "redshift" host: "{{ env_var('REDSHIFT_HOST') }}" user: "{{ env_var('REDSHIFT_USER') }}" pass: "{{ env_var('DBT_ENV_SECRET_REDSHIFT_PASS') }}" dbname: "{{ env_var('REDSHIFT_DATABASE') }}" port: "{{ env_var('REDSHIFT_PORT') | as_number }}" schema: "{{ env_var('REDSHIFT_SCHEMA') }}" - threads: 1 + threads: 5 bigquery: - type: bigquery - method: service-account-json + type: "bigquery" + method: "service-account-json" project: "{{ env_var('BIGQUERY_PROJECT') }}" - keyfile_json: "{{ env_var('DBT_ENV_SECRET_BIGQUERY_KEYFILE_JSON') | as_native }}" dataset: "{{ env_var('BIGQUERY_SCHEMA') }}" threads: 10 + keyfile_json: + "{{ env_var('BIGQUERY_KEYFILE_JSON') | as_native}}" + job_retries: 3 snowflake: - type: snowflake + type: "snowflake" account: "{{ env_var('SNOWFLAKE_ACCOUNT') }}" user: "{{ env_var('SNOWFLAKE_USER') }}" password: "{{ env_var('DBT_ENV_SECRET_SNOWFLAKE_PASS') }}" @@ -43,4 +45,4 @@ integration_tests: database: "{{ env_var('SNOWFLAKE_DATABASE') }}" warehouse: "{{ env_var('SNOWFLAKE_WAREHOUSE') }}" schema: "{{ env_var('SNOWFLAKE_SCHEMA') }}" - threads: 1 + threads: 10 From b3ebee77bf5f55266ba9045e1bd3effcf6547d77 Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Wed, 24 Jul 2024 19:05:26 -0500 Subject: [PATCH 08/24] fix BQ var name --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index d2df2e9..952710d 100644 --- a/tox.ini +++ b/tox.ini @@ -28,7 +28,7 @@ passenv = REDSHIFT_PORT # bigquery BIGQUERY_PROJECT - DBT_ENV_SECRET_BIGQUERY_KEYFILE_JSON + BIGQUERY_KEYFILE_JSON BIGQUERY_SCHEMA # Snowflake integration tests for centralized dbt testing From 68dcd8b3b47ea2bee576b7d767c59c1d0e754113 Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Thu, 25 Jul 2024 08:59:25 -0500 Subject: [PATCH 09/24] remove config from profile --- integration_tests/ci/sample.profiles.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/integration_tests/ci/sample.profiles.yml b/integration_tests/ci/sample.profiles.yml index c96e81a..cb43468 100644 --- a/integration_tests/ci/sample.profiles.yml +++ b/integration_tests/ci/sample.profiles.yml @@ -2,9 +2,8 @@ # HEY! This file is used in the dbt-codegen integrations tests with CircleCI. # You should __NEVER__ check credentials into version control. Thanks for reading :) -config: - send_anonymous_usage_stats: False - use_colors: True +# This profile only exists for use with circleCI. +# Once CircleCi is deprecated, this profile can be removed. integration_tests: target: postgres From 0253b5d3e0f552862378ff30200ba594c72e24f5 Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Thu, 5 Sep 2024 08:20:08 -0500 Subject: [PATCH 10/24] move to just support postgres --- .circleci/config.yml | 6 ++-- .github/workflows/ci.yml | 25 +++----------- integration_tests/profiles.yml | 33 +------------------ supported_adapters.env | 2 +- tox.ini | 59 ---------------------------------- 5 files changed, 10 insertions(+), 115 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 0b55312..ad8aa2d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -10,7 +10,7 @@ jobs: - checkout - run: - run: setup_creds + name: setup_creds command: | echo $BIGQUERY_SERVICE_ACCOUNT_JSON > ${HOME}/bigquery-service-key.json @@ -79,14 +79,14 @@ jobs: name: "Run Tests - BigQuery" environment: BIGQUERY_SERVICE_KEY_PATH: "/home/circleci/bigquery-service-key.json" - REDSHIFT_TEST_SCHEMA: codegen_integration_tests_bigquery + BIGQUERY_TEST_SCHEMA: codegen_integration_tests_bigquery command: | . dbt_venv/bin/activate echo `pwd` cd integration_tests cp ci/sample.profiles.yml profiles.yml - dbt --warn-error deps --target bigquery + dbt --warn-error deps --target bigquery-circleci dbt --warn-error run-operation create_source_table --target bigquery dbt --warn-error seed --target bigquery --full-refresh dbt --warn-error run --target bigquery diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a2647fa..97ef782 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -114,26 +114,11 @@ jobs: - name: "Run integration tests with tox on ${{ matrix.adapter }}" run: | tox -e dbt_integration_${{ matrix.adapter }} + # postgres runs in the container, so nothing below is an actual secret env: - POSTGRES_HOST: ${{ vars.POSTGRES_HOST }} - POSTGRES_USER: ${{ vars.POSTGRES_USER }} - DBT_ENV_SECRET_POSTGRES_PASS: ${{ secrets.POSTGRES_PASS }} + POSTGRES_HOST: postgres_db + POSTGRES_USER: postgres + DBT_ENV_SECRET_POSTGRES_PASS: postgres POSTGRES_PORT: 5432 - POSTGRES_DATABASE: ${{ vars.POSTGRES_DATABASE }} + POSTGRES_DATABASE: postgres POSTGRES_SCHEMA: "dbt_codegen_integration_tests_postgres_${{ github.run_number }}" - SNOWFLAKE_ACCOUNT: ${{ secrets.SNOWFLAKE_ACCOUNT }} - SNOWFLAKE_USER: ${{ vars.SNOWFLAKE_USER }} - DBT_ENV_SECRET_SNOWFLAKE_PASS: ${{ secrets.SNOWFLAKE_PASS }} - SNOWFLAKE_ROLE: ${{ vars.SNOWFLAKE_ROLE }} - SNOWFLAKE_DATABASE: ${{ vars.SNOWFLAKE_DATABASE }} - SNOWFLAKE_WAREHOUSE: ${{ vars.SNOWFLAKE_WAREHOUSE }} - SNOWFLAKE_SCHEMA: "dbt_codegen_integration_tests_snowflake_${{ github.run_number }}" - REDSHIFT_HOST: ${{ vars.REDSHIFT_HOST }} - REDSHIFT_USER: ${{ vars.REDSHIFT_USER }} - DBT_ENV_SECRET_REDSHIFT_PASS: ${{ secrets.REDSHIFT_PASS }} - REDSHIFT_DATABASE: ${{ vars.REDSHIFT_DATABASE }} - REDSHIFT_SCHEMA: "dbt_codegen_integration_tests_redshift_${{ github.run_number }}" - REDSHIFT_PORT: 5439 - BIGQUERY_PROJECT: ${{ vars.BIGQUERY_PROJECT }} - BIGQUERY_KEYFILE_JSON: ${{ secrets.BIGQUERY_KEYFILE_JSON }} - BIGQUERY_SCHEMA: "dbt_codegen_integration_tests_bigquery_${{ github.run_number }}" diff --git a/integration_tests/profiles.yml b/integration_tests/profiles.yml index 508cb1d..0e4ac1c 100644 --- a/integration_tests/profiles.yml +++ b/integration_tests/profiles.yml @@ -1,5 +1,5 @@ -# HEY! This file is used in the dbt-utils integrations tests with CircleCI. +# HEY! This file is used in the dbt-utils integrations tests with GitHub CI. # You should __NEVER__ check credentials into version control. That's why we use environment variables everywhere. # Thanks for reading :) @@ -15,34 +15,3 @@ integration_tests: dbname: "{{ env_var('POSTGRES_DATABASE') }}" schema: "{{ env_var('POSTGRES_SCHEMA') }}" threads: 5 - - redshift: - type: "redshift" - host: "{{ env_var('REDSHIFT_HOST') }}" - user: "{{ env_var('REDSHIFT_USER') }}" - pass: "{{ env_var('DBT_ENV_SECRET_REDSHIFT_PASS') }}" - dbname: "{{ env_var('REDSHIFT_DATABASE') }}" - port: "{{ env_var('REDSHIFT_PORT') | as_number }}" - schema: "{{ env_var('REDSHIFT_SCHEMA') }}" - threads: 5 - - bigquery: - type: "bigquery" - method: "service-account-json" - project: "{{ env_var('BIGQUERY_PROJECT') }}" - dataset: "{{ env_var('BIGQUERY_SCHEMA') }}" - threads: 10 - keyfile_json: - "{{ env_var('BIGQUERY_KEYFILE_JSON') | as_native}}" - job_retries: 3 - - snowflake: - type: "snowflake" - account: "{{ env_var('SNOWFLAKE_ACCOUNT') }}" - user: "{{ env_var('SNOWFLAKE_USER') }}" - password: "{{ env_var('DBT_ENV_SECRET_SNOWFLAKE_PASS') }}" - role: "{{ env_var('SNOWFLAKE_ROLE') }}" - database: "{{ env_var('SNOWFLAKE_DATABASE') }}" - warehouse: "{{ env_var('SNOWFLAKE_WAREHOUSE') }}" - schema: "{{ env_var('SNOWFLAKE_SCHEMA') }}" - threads: 10 diff --git a/supported_adapters.env b/supported_adapters.env index ed5ad27..14c965c 100644 --- a/supported_adapters.env +++ b/supported_adapters.env @@ -1 +1 @@ -SUPPORTED_ADAPTERS=postgres,snowflake,redshift,bigquery \ No newline at end of file +SUPPORTED_ADAPTERS=postgres \ No newline at end of file diff --git a/tox.ini b/tox.ini index 952710d..9918e82 100644 --- a/tox.ini +++ b/tox.ini @@ -11,39 +11,6 @@ passenv = POSTGRES_PORT POSTGRES_DATABASE POSTGRES_SCHEMA - # snowflake env vars - SNOWFLAKE_ACCOUNT - SNOWFLAKE_USER - DBT_ENV_SECRET_SNOWFLAKE_PASS - SNOWFLAKE_ROLE - SNOWFLAKE_DATABASE - SNOWFLAKE_WAREHOUSE - SNOWFLAKE_SCHEMA - # redshift - REDSHIFT_HOST - REDSHIFT_USER - DBT_ENV_SECRET_REDSHIFT_PASS - REDSHIFT_DATABASE - REDSHIFT_SCHEMA - REDSHIFT_PORT - # bigquery - BIGQUERY_PROJECT - BIGQUERY_KEYFILE_JSON - BIGQUERY_SCHEMA - -# Snowflake integration tests for centralized dbt testing -# run dbt commands directly, assumes dbt is already installed in environment -[testenv:dbt_integration_snowflake] -changedir = integration_tests -allowlist_externals = - dbt -skip_install = true -commands = - dbt --warn-error deps --target snowflake - dbt --warn-error run-operation create_source_table --target snowflake - dbt --warn-error seed --target snowflake --full-refresh - dbt --warn-error run --target snowflake - dbt --warn-error test --target snowflake # Postgres integration tests for centralized dbt testing # run dbt commands directly, assumes dbt is already installed in environment @@ -58,29 +25,3 @@ commands = dbt --warn-error seed --target postgres --full-refresh dbt --warn-error run --target postgres dbt --warn-error test --target postgres - -# BigQuery integration tests for centralized dbt testing -[testenv:dbt_integration_bigquery] -changedir = integration_tests -allowlist_externals = - dbt -skip_install = true -commands = - dbt --warn-error deps --target bigquery - dbt --warn-error run-operation create_source_table --target bigquery - dbt --warn-error seed --target bigquery --full-refresh - dbt --warn-error run --target bigquery - dbt --warn-error test --target bigquery - -# redshift integration tests for centralized dbt testing -[testenv:dbt_integration_redshift] -changedir = integration_tests -allowlist_externals = - dbt -skip_install = true -commands = - dbt --warn-error deps --target redshift - dbt --warn-error run-operation create_source_table --target redshift - dbt --warn-error seed --target redshift --full-refresh - dbt --warn-error run --target redshift - dbt --warn-error test --target redshift \ No newline at end of file From 2b2d023e8f0830de5485b087d58afba5fb6286d7 Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Thu, 5 Sep 2024 08:59:40 -0500 Subject: [PATCH 11/24] fix vars --- .github/workflows/ci.yml | 26 ++++++++++++++++---------- integration_tests/README.md | 34 +++++++++++++++++----------------- 2 files changed, 33 insertions(+), 27 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 97ef782..45b16f2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -23,6 +23,12 @@ on: env: PYTHON_VERSION: "3.11" + POSTGRES_HOST: "localhost" + POSTGRES_USER: "root" + DBT_ENV_SECRET_POSTGRES_PASS: "public_password" + POSTGRES_PORT: "5432" + POSTGRES_DATABASE: "postgres_test" + POSTGRES_SCHEMA: "dbt_codegen_integration_tests_postgres_${{ github.run_number }}" jobs: determine-supported-adapters: @@ -76,10 +82,10 @@ jobs: postgres: image: postgres env: - POSTGRES_USER: ${{ vars.POSTGRES_USER }} - POSTGRES_PASSWORD: ${{ secrets.POSTGRES_PASS }} - POSTGRES_DB: ${{ vars.POSTGRES_DATABASE }} - POSTGRES_HOST: ${{ vars.POSTGRES_HOST }} + POSTGRES_USER: ${{ env.POSTGRES_USER }} + POSTGRES_PASSWORD: ${{ env.DBT_ENV_SECRET_POSTGRES_PASS }} + POSTGRES_DB: ${{ env.POSTGRES_DATABASE }} + POSTGRES_HOST: ${{ env.POSTGRES_HOST }} options: >- --health-cmd pg_isready --health-interval 10s @@ -116,9 +122,9 @@ jobs: tox -e dbt_integration_${{ matrix.adapter }} # postgres runs in the container, so nothing below is an actual secret env: - POSTGRES_HOST: postgres_db - POSTGRES_USER: postgres - DBT_ENV_SECRET_POSTGRES_PASS: postgres - POSTGRES_PORT: 5432 - POSTGRES_DATABASE: postgres - POSTGRES_SCHEMA: "dbt_codegen_integration_tests_postgres_${{ github.run_number }}" + POSTGRES_HOST: ${{ env.POSTGRES_HOST }} + POSTGRES_USER: ${{ env.POSTGRES_USER }} + DBT_ENV_SECRET_POSTGRES_PASS: ${{ env.DBT_ENV_SECRET_POSTGRES_PASS }} + POSTGRES_PORT: ${{ env.POSTGRES_PORT }} + POSTGRES_DATABASE: ${{ env.POSTGRES_DATABASE }} + POSTGRES_SCHEMA: ${{ env.POSTGRES_SCHEMA }} diff --git a/integration_tests/README.md b/integration_tests/README.md index d7c5108..1337e57 100644 --- a/integration_tests/README.md +++ b/integration_tests/README.md @@ -48,30 +48,30 @@ The environment variables you'll need to set for each adapter are: ```bash # Postgres — these are the defaults for the Docker container so actually have values -export POSTGRES_HOST=localhost -export POSTGRES_USER=root -export POSTGRES_PASS='' -export POSTGRES_PORT=5432 -export POSTGRES_DATABASE=circle_test +export POSTGRES_TEST_HOST=localhost +export POSTGRES_TEST_USER=root +export POSTGRES_TEST_PASS='' +export POSTGRES_TEST_PORT=5432 +export POSTGRES_TEST_DBNAME=circle_test # BigQuery export BIGQUERY_SERVICE_KEY_PATH= -export BIGQUERY_PROJECT= +export BIGQUERY_TEST_DATABASE= # Redshift -export REDSHIFT_HOST= -export REDSHIFT_USER= -export REDSHIFT_PASS= -export REDSHIFT_DATABASE= -export REDSHIFT_PORT= +export REDSHIFT_TEST_HOST= +export REDSHIFT_TEST_USER= +export REDSHIFT_TEST_PASS= +export REDSHIFT_TEST_DBNAME= +export REDSHIFT_TEST_PORT= # Snowflake -export SNOWFLAKE_ACCOUNT= -export SNOWFLAKE_USER= -export SNOWFLAKE_PASSWORD= -export SNOWFLAKE_ROLE= -export SNOWFLAKE_DATABASE= -export SNOWFLAKE_WAREHOUSE= +export SNOWFLAKE_TEST_ACCOUNT= +export SNOWFLAKE_TEST_USER= +export SNOWFLAKE_TEST_PASSWORD= +export SNOWFLAKE_TEST_ROLE= +export SNOWFLAKE_TEST_DATABASE= +export SNOWFLAKE_TEST_WAREHOUSE= ``` ### Setup Postgres or other database targets From b9af112d17d8a906449356383098c6823efd9fcd Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Fri, 6 Sep 2024 14:41:56 -0500 Subject: [PATCH 12/24] use premade workflow --- .github/workflows/ci.yml | 113 +-------------------------------------- 1 file changed, 2 insertions(+), 111 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 45b16f2..881c175 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,116 +15,7 @@ on: - main pull_request: workflow_dispatch: - inputs: - adapter: - description: The adapter to test against. Defaults to all supported adapters when blank. - type: string - required: false - -env: - PYTHON_VERSION: "3.11" - POSTGRES_HOST: "localhost" - POSTGRES_USER: "root" - DBT_ENV_SECRET_POSTGRES_PASS: "public_password" - POSTGRES_PORT: "5432" - POSTGRES_DATABASE: "postgres_test" - POSTGRES_SCHEMA: "dbt_codegen_integration_tests_postgres_${{ github.run_number }}" jobs: - determine-supported-adapters: - runs-on: ubuntu-latest - outputs: - adapters: ${{ steps.supported-adapters.outputs.adapters }} - steps: - - name: "Checkout ${{ github.event.repository }}" - uses: actions/checkout@v4 - - - name: "Set up Python ${{ env.PYTHON_VERSION }}" - uses: actions/setup-python@v5 - with: - python-version: ${{ env.PYTHON_VERSION }} - - - name: "Install tox" - run: | - python -m pip install --upgrade pip - pip install tox - - - name: "Get list of supported adapters or use input adapter only" - id: list-adapters - run: | - if [ -z "${{ inputs.adapter }}" ]; then - # github adds a pip freeze and a new line we need to strip out - source supported_adapters.env - echo $SUPPORTED_ADAPTERS - echo "test_adapters=$SUPPORTED_ADAPTERS" >> $GITHUB_OUTPUT - else - echo "test_adapters=${{ inputs.adapter }}" >> $GITHUB_OUTPUT - fi - - - name: "Format adapter list for use as the matrix" - id: supported-adapters - run: | - # Convert to JSON array and output - supported_adapters=$(echo "${{ steps.list-adapters.outputs.test_adapters }}" | jq -Rc 'split(",")') - echo $supported_adapters - echo "adapters=$supported_adapters" >> $GITHUB_OUTPUT - - - name: "[ANNOTATION] ${{ github.event.repository.name }} - Testing ${{ steps.supported-adapters.outputs.adapters }}" - run: | - title="${{ github.event.repository.name }} - adapters to test" - message="The workflow will run tests for the following adapters: ${{ steps.supported-adapters.outputs.adapters }}" - echo "::notice $title::$message" - - run-tests: - runs-on: ubuntu-latest - needs: [determine-supported-adapters] - services: - postgres: - image: postgres - env: - POSTGRES_USER: ${{ env.POSTGRES_USER }} - POSTGRES_PASSWORD: ${{ env.DBT_ENV_SECRET_POSTGRES_PASS }} - POSTGRES_DB: ${{ env.POSTGRES_DATABASE }} - POSTGRES_HOST: ${{ env.POSTGRES_HOST }} - options: >- - --health-cmd pg_isready - --health-interval 10s - --health-timeout 5s - --health-retries 5 - ports: - - 5432:5432 - strategy: - fail-fast: false - matrix: - adapter: ${{fromJson(needs.determine-supported-adapters.outputs.adapters)}} - - steps: - - name: "Checkout ${{ github.event.repository }} " - uses: actions/checkout@v4 - - - name: "Set up Python ${{ env.PYTHON_VERSION }}" - uses: actions/setup-python@v5 - with: - python-version: ${{ env.PYTHON_VERSION }} - - - name: "Install ${{ matrix.adapter }}" - run: | - python -m pip install --upgrade pip - pip install dbt-${{ matrix.adapter }} - - - name: "Install tox" - run: | - python -m pip install --upgrade pip - pip install tox - - - name: "Run integration tests with tox on ${{ matrix.adapter }}" - run: | - tox -e dbt_integration_${{ matrix.adapter }} - # postgres runs in the container, so nothing below is an actual secret - env: - POSTGRES_HOST: ${{ env.POSTGRES_HOST }} - POSTGRES_USER: ${{ env.POSTGRES_USER }} - DBT_ENV_SECRET_POSTGRES_PASS: ${{ env.DBT_ENV_SECRET_POSTGRES_PASS }} - POSTGRES_PORT: ${{ env.POSTGRES_PORT }} - POSTGRES_DATABASE: ${{ env.POSTGRES_DATABASE }} - POSTGRES_SCHEMA: ${{ env.POSTGRES_SCHEMA }} + run-tests: + uses: dbt-labs/dbt-package-testing/.github/workflows/run_tox.yml@er/use-action \ No newline at end of file From 91af3a0ea8ad3fe438309b06d3680a147617c6c7 Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Fri, 6 Sep 2024 14:53:26 -0500 Subject: [PATCH 13/24] add newline --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 881c175..bd8d960 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,4 +18,4 @@ on: jobs: run-tests: - uses: dbt-labs/dbt-package-testing/.github/workflows/run_tox.yml@er/use-action \ No newline at end of file + uses: dbt-labs/dbt-package-testing/.github/workflows/run_tox.yml@er/use-action From c65a371ce5bf254e6fd1ecf3e0a632aa8273d2de Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Mon, 9 Sep 2024 09:44:13 -0500 Subject: [PATCH 14/24] use merged version --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bd8d960..a29f504 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,4 +18,4 @@ on: jobs: run-tests: - uses: dbt-labs/dbt-package-testing/.github/workflows/run_tox.yml@er/use-action + uses: dbt-labs/dbt-package-testing/.github/workflows/run_tox.yml@main From a188739681378275977ab04dc1a396dce0d34b17 Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Mon, 9 Sep 2024 09:49:34 -0500 Subject: [PATCH 15/24] add comments about future adapters --- .github/workflows/ci.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a29f504..0d00222 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,3 +19,15 @@ on: jobs: run-tests: uses: dbt-labs/dbt-package-testing/.github/workflows/run_tox.yml@main + # this just tests with postgres so no variables need to be passed through. + # When it's time to add more adapters you will need to pass through inputs for + # the other adapters as shown in the below example for redshift + # with: + # # redshift + # REDSHIFT_HOST: ${{ vars.REDSHIFT_HOST }} + # REDSHIFT_USER: ${{ vars.REDSHIFT_USER }} + # REDSHIFT_DATABASE: ${{ vars.REDSHIFT_DATABASE }} + # REDSHIFT_SCHEMA: "integration_tests_redshift_${{ github.run_number }}" + # REDSHIFT_PORT: ${{ vars.REDSHIFT_PORT }} + # secrets: + # DBT_ENV_SECRET_REDSHIFT_PASS: ${{ secrets.DBT_ENV_SECRET_REDSHIFT_PASS }} From 39a45ff8442eec7b577ed52b9146e8bb8f408716 Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Thu, 26 Sep 2024 09:25:38 -0500 Subject: [PATCH 16/24] use tag --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0d00222..6a8d288 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,7 +18,7 @@ on: jobs: run-tests: - uses: dbt-labs/dbt-package-testing/.github/workflows/run_tox.yml@main + uses: dbt-labs/dbt-package-testing/.github/workflows/run_tox.yml@v1 # this just tests with postgres so no variables need to be passed through. # When it's time to add more adapters you will need to pass through inputs for # the other adapters as shown in the below example for redshift From b7b13f35bcae7b612f5e20da60178c4f2967502f Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Thu, 26 Sep 2024 10:11:04 -0500 Subject: [PATCH 17/24] update readme --- Makefile | 5 +++++ integration_tests/README.md | 16 +++++++++++++--- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 9472484..e3545fc 100644 --- a/Makefile +++ b/Makefile @@ -4,6 +4,11 @@ test: ## Run the integration tests. @./run_test.sh $(target) +.PHONY: test_tox +test: ## Run the integration tests with tox + @\ + tox -e dbt_integration_$(target) + .PHONY: dev dev: ## Installs dbt-* packages in develop mode along with development dependencies. @\ diff --git a/integration_tests/README.md b/integration_tests/README.md index 1337e57..d6f2856 100644 --- a/integration_tests/README.md +++ b/integration_tests/README.md @@ -44,7 +44,7 @@ You can set these env vars in a couple ways: - **Temporary**: Set these environment variables in your shell before running the tests. This is the easiest way to get started, but you'll have to set them every time you open a new terminal. - **Reusable**: If you anticipate developing for multiple sessions, set these environment variables in your shell profile (like `~/.bashrc` or `~/.zshrc`). This way, you won't have to set them every time you open a new terminal. -The environment variables you'll need to set for each adapter are: +The environment variables you'll need to set for each adapter when running tests with the bash script: ```bash # Postgres — these are the defaults for the Docker container so actually have values @@ -74,6 +74,8 @@ export SNOWFLAKE_TEST_DATABASE= export SNOWFLAKE_TEST_WAREHOUSE= ``` +The environment variables you'll need to set for each adapter when running tests with tox can be found in [integration_tests/.env/](integration_tests/.env/). + ### Setup Postgres or other database targets As mentioned, you'll need a target database to run the integration tests and develop against. You can use a cloud warehouse, but the easiest and free way to work is to use Postgres locally. We include a `docker-compose.yml` file that will spin up a Postgres container for you to make this easy. @@ -139,7 +141,9 @@ source .venv/bin/activate ## Write or modify an integration test -### Run the integration tests +Run all the tests _before_ you start developing to make sure everything is working as expected before you start making changes. Nothing is worse than spending a ton of time troubleshooting a failing test, only to realize it was failing before you touched anything. This will also ensure that you have the correct environment variables set up and that your database is running. + +### Run the Circle CI integration tests To run all the integration tests on your local machine like they will get run in CI: @@ -157,7 +161,13 @@ make test target=postgres ./run_test.sh postgres ``` -Run all the tests _before_ you start developing to make sure everything is working as expected before you start making changes. Nothing is worse than spending a ton of time troubleshooting a failing test, only to realize it was failing before you touched anything. This will also ensure that you have the correct environment variables set up and that your database is running. +### Run the tox Supported Tests + +To run all the integration tests on your local machine like they will get run in the CI (using GitHub workflows with tox): + +```shell +make test_tox target=postgres +``` ### Creating a new integration test From f8ea9b57e60081c788f5f7560b27b7b01387d6c8 Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Thu, 26 Sep 2024 11:13:42 -0500 Subject: [PATCH 18/24] Add lines --- integration_tests/ci/sample.profiles.yml | 2 +- supported_adapters.env | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/integration_tests/ci/sample.profiles.yml b/integration_tests/ci/sample.profiles.yml index cb43468..1f78543 100644 --- a/integration_tests/ci/sample.profiles.yml +++ b/integration_tests/ci/sample.profiles.yml @@ -45,4 +45,4 @@ integration_tests: database: "{{ env_var('SNOWFLAKE_TEST_DATABASE') }}" warehouse: "{{ env_var('SNOWFLAKE_TEST_WAREHOUSE') }}" schema: codegen_integration_tests_snowflake - threads: 1 \ No newline at end of file + threads: 1 diff --git a/supported_adapters.env b/supported_adapters.env index 14c965c..79c9a19 100644 --- a/supported_adapters.env +++ b/supported_adapters.env @@ -1 +1 @@ -SUPPORTED_ADAPTERS=postgres \ No newline at end of file +SUPPORTED_ADAPTERS=postgres From f167e3f4b1ceb57131efe69fdc70419a632f466c Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Thu, 26 Sep 2024 13:30:09 -0500 Subject: [PATCH 19/24] reowkr profile so it can be reused --- .circleci/config.yml | 22 +++++------ integration_tests/ci/sample.profiles.yml | 48 ------------------------ integration_tests/profiles.yml | 31 +++++++++++++++ 3 files changed, 40 insertions(+), 61 deletions(-) delete mode 100644 integration_tests/ci/sample.profiles.yml diff --git a/.circleci/config.yml b/.circleci/config.yml index ad8aa2d..5e112f0 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -29,16 +29,15 @@ jobs: - run: name: "Run Tests - Postgres" environment: - POSTGRES_TEST_HOST: localhost - POSTGRES_TEST_USER: root - POSTGRES_TEST_PASS: "" - POSTGRES_TEST_PORT: 5432 - POSTGRES_TEST_DBNAME: circle_test - POSTGRES_TEST_SCHEMA: codegen_integration_tests_postgres + POSTGRES_HOST: localhost + POSTGRES_USER: root + DBT_ENV_SECRET_POSTGRES_PASS: "" + POSTGRES_PORT: 5432 + POSTGRES_DATABASE: circle_test + POSTGRES_SCHEMA: codegen_integration_tests_postgres command: | . dbt_venv/bin/activate cd integration_tests - cp ci/sample.profiles.yml profiles.yml dbt --warn-error deps --target postgres dbt --warn-error run-operation create_source_table --target postgres dbt --warn-error seed --target postgres --full-refresh @@ -48,12 +47,11 @@ jobs: - run: name: "Run Tests - Redshift" environment: - REDSHIFT_TEST_SCHEMA: codegen_integration_tests_redshift + REDSHIFT_SCHEMA: codegen_integration_tests_redshift command: | . dbt_venv/bin/activate echo `pwd` cd integration_tests - cp ci/sample.profiles.yml profiles.yml dbt --warn-error deps --target redshift dbt --warn-error run-operation create_source_table --target redshift dbt --warn-error seed --target redshift --full-refresh @@ -63,12 +61,11 @@ jobs: - run: name: "Run Tests - Snowflake" environment: - SNOWFLAKE_TEST_SCHEMA: codegen_integration_tests_snowflake + SNOWFLAKE_SCHEMA: codegen_integration_tests_snowflake command: | . dbt_venv/bin/activate echo `pwd` cd integration_tests - cp ci/sample.profiles.yml profiles.yml dbt --warn-error deps --target snowflake dbt --warn-error run-operation create_source_table --target snowflake dbt --warn-error seed --target snowflake --full-refresh @@ -79,13 +76,12 @@ jobs: name: "Run Tests - BigQuery" environment: BIGQUERY_SERVICE_KEY_PATH: "/home/circleci/bigquery-service-key.json" - BIGQUERY_TEST_SCHEMA: codegen_integration_tests_bigquery + BIGQUERY_SCHEMA: codegen_integration_tests_bigquery command: | . dbt_venv/bin/activate echo `pwd` cd integration_tests - cp ci/sample.profiles.yml profiles.yml dbt --warn-error deps --target bigquery-circleci dbt --warn-error run-operation create_source_table --target bigquery dbt --warn-error seed --target bigquery --full-refresh diff --git a/integration_tests/ci/sample.profiles.yml b/integration_tests/ci/sample.profiles.yml deleted file mode 100644 index cb43468..0000000 --- a/integration_tests/ci/sample.profiles.yml +++ /dev/null @@ -1,48 +0,0 @@ - -# HEY! This file is used in the dbt-codegen integrations tests with CircleCI. -# You should __NEVER__ check credentials into version control. Thanks for reading :) - -# This profile only exists for use with circleCI. -# Once CircleCi is deprecated, this profile can be removed. - -integration_tests: - target: postgres - outputs: - postgres: - type: postgres - host: "{{ env_var('POSTGRES_TEST_HOST') }}" - user: "{{ env_var('POSTGRES_TEST_USER') }}" - pass: "{{ env_var('POSTGRES_TEST_PASS') }}" - port: "{{ env_var('POSTGRES_TEST_PORT') | as_number }}" - dbname: "{{ env_var('POSTGRES_TEST_DBNAME') }}" - schema: codegen_integration_tests_postgres - threads: 1 - - redshift: - type: redshift - host: "{{ env_var('REDSHIFT_TEST_HOST') }}" - user: "{{ env_var('REDSHIFT_TEST_USER') }}" - pass: "{{ env_var('REDSHIFT_TEST_PASS') }}" - dbname: "{{ env_var('REDSHIFT_TEST_DBNAME') }}" - port: "{{ env_var('REDSHIFT_TEST_PORT') | as_number }}" - schema: codegen_integration_tests_redshift - threads: 1 - - bigquery: - type: bigquery - method: service-account - keyfile: "{{ env_var('BIGQUERY_SERVICE_KEY_PATH') }}" - project: "{{ env_var('BIGQUERY_TEST_DATABASE') }}" - schema: codegen_integration_tests_bigquery - threads: 1 - - snowflake: - type: snowflake - account: "{{ env_var('SNOWFLAKE_TEST_ACCOUNT') }}" - user: "{{ env_var('SNOWFLAKE_TEST_USER') }}" - password: "{{ env_var('SNOWFLAKE_TEST_PASSWORD') }}" - role: "{{ env_var('SNOWFLAKE_TEST_ROLE') }}" - database: "{{ env_var('SNOWFLAKE_TEST_DATABASE') }}" - warehouse: "{{ env_var('SNOWFLAKE_TEST_WAREHOUSE') }}" - schema: codegen_integration_tests_snowflake - threads: 1 \ No newline at end of file diff --git a/integration_tests/profiles.yml b/integration_tests/profiles.yml index 0e4ac1c..cb84d4e 100644 --- a/integration_tests/profiles.yml +++ b/integration_tests/profiles.yml @@ -15,3 +15,34 @@ integration_tests: dbname: "{{ env_var('POSTGRES_DATABASE') }}" schema: "{{ env_var('POSTGRES_SCHEMA') }}" threads: 5 + + redshift: + type: "redshift" + host: "{{ env_var('REDSHIFT_HOST') }}" + user: "{{ env_var('REDSHIFT_USER') }}" + pass: "{{ env_var('DBT_ENV_SECRET_REDSHIFT_PASS') }}" + dbname: "{{ env_var('REDSHIFT_DATABASE') }}" + port: "{{ env_var('REDSHIFT_PORT') | as_number }}" + schema: "{{ env_var('REDSHIFT_SCHEMA') }}" + threads: 5 + + bigquery: + type: "bigquery" + method: "service-account-json" + project: "{{ env_var('BIGQUERY_PROJECT') }}" + dataset: "{{ env_var('BIGQUERY_SCHEMA') }}" + threads: 10 + keyfile_json: + "{{ env_var('BIGQUERY_KEYFILE_JSON') | as_native }}" + job_retries: 3 + + snowflake: + type: "snowflake" + account: "{{ env_var('SNOWFLAKE_ACCOUNT') }}" + user: "{{ env_var('SNOWFLAKE_USER') }}" + password: "{{ env_var('DBT_ENV_SECRET_SNOWFLAKE_PASS') }}" + role: "{{ env_var('SNOWFLAKE_ROLE') }}" + database: "{{ env_var('SNOWFLAKE_DATABASE') }}" + warehouse: "{{ env_var('SNOWFLAKE_WAREHOUSE') }}" + schema: "{{ env_var('SNOWFLAKE_SCHEMA') }}" + threads: 10 \ No newline at end of file From 145729a6170ef2d872fff883b5f4bccdc2e202aa Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Thu, 26 Sep 2024 14:12:48 -0500 Subject: [PATCH 20/24] add sample env files and fix comment --- integration_tests/.env/bigquery.env | 3 +++ integration_tests/.env/postgres.env | 6 ++++++ integration_tests/.env/redshift.env | 6 ++++++ integration_tests/.env/snowflake.env | 7 +++++++ integration_tests/profiles.yml | 2 +- 5 files changed, 23 insertions(+), 1 deletion(-) create mode 100644 integration_tests/.env/bigquery.env create mode 100644 integration_tests/.env/postgres.env create mode 100644 integration_tests/.env/redshift.env create mode 100644 integration_tests/.env/snowflake.env diff --git a/integration_tests/.env/bigquery.env b/integration_tests/.env/bigquery.env new file mode 100644 index 0000000..6039f6b --- /dev/null +++ b/integration_tests/.env/bigquery.env @@ -0,0 +1,3 @@ +BIGQUERY_KEYFILE_JSON= +BIGQUERY_PROJECT= +BIGQUERY_SCHEMA=codegen_integration_tests_bigquery diff --git a/integration_tests/.env/postgres.env b/integration_tests/.env/postgres.env new file mode 100644 index 0000000..4fd155c --- /dev/null +++ b/integration_tests/.env/postgres.env @@ -0,0 +1,6 @@ +POSTGRES_HOST=localhost +POSTGRES_USER=root +DBT_ENV_SECRET_POSTGRES_PASS=password +POSTGRES_PORT=5432 +POSTGRES_DATABASE=codegen_test +POSTGRES_SCHEMA=codegen_integration_tests_postgres diff --git a/integration_tests/.env/redshift.env b/integration_tests/.env/redshift.env new file mode 100644 index 0000000..c0d613d --- /dev/null +++ b/integration_tests/.env/redshift.env @@ -0,0 +1,6 @@ +REDSHIFT_HOST= +REDSHIFT_USER= +DBT_ENV_SECRET_REDSHIFT_PASS= +REDSHIFT_DATABASE= +REDSHIFT_PORT= +REDSHIFT_SCHEMA=codegen_integration_tests_redshift diff --git a/integration_tests/.env/snowflake.env b/integration_tests/.env/snowflake.env new file mode 100644 index 0000000..b0c1267 --- /dev/null +++ b/integration_tests/.env/snowflake.env @@ -0,0 +1,7 @@ +SNOWFLAKE_ACCOUNT= +SNOWFLAKE_USER= +DBT_ENV_SECRET_SNOWFLAKE_PASS= +SNOWFLAKE_ROLE= +SNOWFLAKE_DATABASE= +SNOWFLAKE_WAREHOUSE= +SNOWFLAKE_SCHEMA=codegen_integration_tests_snowflake diff --git a/integration_tests/profiles.yml b/integration_tests/profiles.yml index cb84d4e..7eeb317 100644 --- a/integration_tests/profiles.yml +++ b/integration_tests/profiles.yml @@ -1,5 +1,5 @@ -# HEY! This file is used in the dbt-utils integrations tests with GitHub CI. +# HEY! This file is used in the dbt-codegen integrations tests with GitHub CI. # You should __NEVER__ check credentials into version control. That's why we use environment variables everywhere. # Thanks for reading :) From 91d104c29ddc67733bb8da0ff3399993968bf154 Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Thu, 26 Sep 2024 14:18:12 -0500 Subject: [PATCH 21/24] Update .circleci/config.yml Co-authored-by: Doug Beatty <44704949+dbeatty10@users.noreply.github.com> --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 5e112f0..4b199da 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -82,7 +82,7 @@ jobs: . dbt_venv/bin/activate echo `pwd` cd integration_tests - dbt --warn-error deps --target bigquery-circleci + dbt --warn-error deps --target bigquery dbt --warn-error run-operation create_source_table --target bigquery dbt --warn-error seed --target bigquery --full-refresh dbt --warn-error run --target bigquery From 85f2eac62fda2da009e0f3590e656b024f156a41 Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Thu, 26 Sep 2024 14:36:34 -0500 Subject: [PATCH 22/24] let circleCI set the schema --- .circleci/config.yml | 5 ----- integration_tests/profiles.yml | 2 +- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 4b199da..6746196 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -46,8 +46,6 @@ jobs: - run: name: "Run Tests - Redshift" - environment: - REDSHIFT_SCHEMA: codegen_integration_tests_redshift command: | . dbt_venv/bin/activate echo `pwd` @@ -60,8 +58,6 @@ jobs: - run: name: "Run Tests - Snowflake" - environment: - SNOWFLAKE_SCHEMA: codegen_integration_tests_snowflake command: | . dbt_venv/bin/activate echo `pwd` @@ -76,7 +72,6 @@ jobs: name: "Run Tests - BigQuery" environment: BIGQUERY_SERVICE_KEY_PATH: "/home/circleci/bigquery-service-key.json" - BIGQUERY_SCHEMA: codegen_integration_tests_bigquery command: | . dbt_venv/bin/activate diff --git a/integration_tests/profiles.yml b/integration_tests/profiles.yml index 7eeb317..24f42af 100644 --- a/integration_tests/profiles.yml +++ b/integration_tests/profiles.yml @@ -45,4 +45,4 @@ integration_tests: database: "{{ env_var('SNOWFLAKE_DATABASE') }}" warehouse: "{{ env_var('SNOWFLAKE_WAREHOUSE') }}" schema: "{{ env_var('SNOWFLAKE_SCHEMA') }}" - threads: 10 \ No newline at end of file + threads: 10 From e85ab69eb1d0d6c5e509df3c47469a96c000ccfb Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Thu, 26 Sep 2024 14:46:34 -0500 Subject: [PATCH 23/24] fix readme instructions --- integration_tests/README.md | 32 +------------------------------- 1 file changed, 1 insertion(+), 31 deletions(-) diff --git a/integration_tests/README.md b/integration_tests/README.md index d6f2856..ed9c863 100644 --- a/integration_tests/README.md +++ b/integration_tests/README.md @@ -44,37 +44,7 @@ You can set these env vars in a couple ways: - **Temporary**: Set these environment variables in your shell before running the tests. This is the easiest way to get started, but you'll have to set them every time you open a new terminal. - **Reusable**: If you anticipate developing for multiple sessions, set these environment variables in your shell profile (like `~/.bashrc` or `~/.zshrc`). This way, you won't have to set them every time you open a new terminal. -The environment variables you'll need to set for each adapter when running tests with the bash script: - -```bash -# Postgres — these are the defaults for the Docker container so actually have values -export POSTGRES_TEST_HOST=localhost -export POSTGRES_TEST_USER=root -export POSTGRES_TEST_PASS='' -export POSTGRES_TEST_PORT=5432 -export POSTGRES_TEST_DBNAME=circle_test - -# BigQuery -export BIGQUERY_SERVICE_KEY_PATH= -export BIGQUERY_TEST_DATABASE= - -# Redshift -export REDSHIFT_TEST_HOST= -export REDSHIFT_TEST_USER= -export REDSHIFT_TEST_PASS= -export REDSHIFT_TEST_DBNAME= -export REDSHIFT_TEST_PORT= - -# Snowflake -export SNOWFLAKE_TEST_ACCOUNT= -export SNOWFLAKE_TEST_USER= -export SNOWFLAKE_TEST_PASSWORD= -export SNOWFLAKE_TEST_ROLE= -export SNOWFLAKE_TEST_DATABASE= -export SNOWFLAKE_TEST_WAREHOUSE= -``` - -The environment variables you'll need to set for each adapter when running tests with tox can be found in [integration_tests/.env/](integration_tests/.env/). +The environment variables you'll need to set for each adapter can be found in [integration_tests/.env/](integration_tests/.env/). ### Setup Postgres or other database targets From 978f67ea72a09f0d2801ec33114693b3425e93b2 Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Fri, 11 Oct 2024 07:41:28 -0500 Subject: [PATCH 24/24] undo non-postgres changes --- integration_tests/.env/bigquery.env | 4 +-- integration_tests/.env/redshift.env | 11 +++---- integration_tests/.env/snowflake.env | 13 ++++---- integration_tests/profiles.yml | 48 +++++++++++++--------------- 4 files changed, 36 insertions(+), 40 deletions(-) diff --git a/integration_tests/.env/bigquery.env b/integration_tests/.env/bigquery.env index 6039f6b..4972fca 100644 --- a/integration_tests/.env/bigquery.env +++ b/integration_tests/.env/bigquery.env @@ -1,3 +1,3 @@ -BIGQUERY_KEYFILE_JSON= BIGQUERY_PROJECT= -BIGQUERY_SCHEMA=codegen_integration_tests_bigquery +BIGQUERY_SCHEMA= +BIGQUERY_KEYFILE_JSON= diff --git a/integration_tests/.env/redshift.env b/integration_tests/.env/redshift.env index c0d613d..baf768c 100644 --- a/integration_tests/.env/redshift.env +++ b/integration_tests/.env/redshift.env @@ -1,6 +1,5 @@ -REDSHIFT_HOST= -REDSHIFT_USER= -DBT_ENV_SECRET_REDSHIFT_PASS= -REDSHIFT_DATABASE= -REDSHIFT_PORT= -REDSHIFT_SCHEMA=codegen_integration_tests_redshift +REDSHIFT_TEST_HOST= +REDSHIFT_TEST_USER= +REDSHIFT_TEST_PASS= +REDSHIFT_TEST_DBNAME= +REDSHIFT_TEST_PORT= diff --git a/integration_tests/.env/snowflake.env b/integration_tests/.env/snowflake.env index b0c1267..eb10da9 100644 --- a/integration_tests/.env/snowflake.env +++ b/integration_tests/.env/snowflake.env @@ -1,7 +1,6 @@ -SNOWFLAKE_ACCOUNT= -SNOWFLAKE_USER= -DBT_ENV_SECRET_SNOWFLAKE_PASS= -SNOWFLAKE_ROLE= -SNOWFLAKE_DATABASE= -SNOWFLAKE_WAREHOUSE= -SNOWFLAKE_SCHEMA=codegen_integration_tests_snowflake +SNOWFLAKE_TEST_ACCOUNT= +SNOWFLAKE_TEST_USER= +SNOWFLAKE_TEST_PASSWORD= +SNOWFLAKE_TEST_ROLE= +SNOWFLAKE_TEST_DATABASE= +SNOWFLAKE_TEST_WAREHOUSE= diff --git a/integration_tests/profiles.yml b/integration_tests/profiles.yml index 24f42af..0db3973 100644 --- a/integration_tests/profiles.yml +++ b/integration_tests/profiles.yml @@ -17,32 +17,30 @@ integration_tests: threads: 5 redshift: - type: "redshift" - host: "{{ env_var('REDSHIFT_HOST') }}" - user: "{{ env_var('REDSHIFT_USER') }}" - pass: "{{ env_var('DBT_ENV_SECRET_REDSHIFT_PASS') }}" - dbname: "{{ env_var('REDSHIFT_DATABASE') }}" - port: "{{ env_var('REDSHIFT_PORT') | as_number }}" - schema: "{{ env_var('REDSHIFT_SCHEMA') }}" - threads: 5 + type: redshift + host: "{{ env_var('REDSHIFT_TEST_HOST') }}" + user: "{{ env_var('REDSHIFT_TEST_USER') }}" + pass: "{{ env_var('REDSHIFT_TEST_PASS') }}" + dbname: "{{ env_var('REDSHIFT_TEST_DBNAME') }}" + port: "{{ env_var('REDSHIFT_TEST_PORT') | as_number }}" + schema: codegen_integration_tests_redshift + threads: 1 bigquery: - type: "bigquery" - method: "service-account-json" - project: "{{ env_var('BIGQUERY_PROJECT') }}" - dataset: "{{ env_var('BIGQUERY_SCHEMA') }}" - threads: 10 - keyfile_json: - "{{ env_var('BIGQUERY_KEYFILE_JSON') | as_native }}" - job_retries: 3 + type: bigquery + method: service-account + keyfile: "{{ env_var('BIGQUERY_SERVICE_KEY_PATH') }}" + project: "{{ env_var('BIGQUERY_TEST_DATABASE') }}" + schema: codegen_integration_tests_bigquery + threads: 1 snowflake: - type: "snowflake" - account: "{{ env_var('SNOWFLAKE_ACCOUNT') }}" - user: "{{ env_var('SNOWFLAKE_USER') }}" - password: "{{ env_var('DBT_ENV_SECRET_SNOWFLAKE_PASS') }}" - role: "{{ env_var('SNOWFLAKE_ROLE') }}" - database: "{{ env_var('SNOWFLAKE_DATABASE') }}" - warehouse: "{{ env_var('SNOWFLAKE_WAREHOUSE') }}" - schema: "{{ env_var('SNOWFLAKE_SCHEMA') }}" - threads: 10 + type: snowflake + account: "{{ env_var('SNOWFLAKE_TEST_ACCOUNT') }}" + user: "{{ env_var('SNOWFLAKE_TEST_USER') }}" + password: "{{ env_var('SNOWFLAKE_TEST_PASSWORD') }}" + role: "{{ env_var('SNOWFLAKE_TEST_ROLE') }}" + database: "{{ env_var('SNOWFLAKE_TEST_DATABASE') }}" + warehouse: "{{ env_var('SNOWFLAKE_TEST_WAREHOUSE') }}" + schema: codegen_integration_tests_snowflake + threads: 1