diff --git a/.circleci/config.yml b/.circleci/config.yml index b1f45ec..bfccbe7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -15,6 +15,7 @@ jobs: - run: name: "Setup dbt" command: | + sudo apt install libsasl2-dev python3 -m venv venv . venv/bin/activate pip install --upgrade pip setuptools @@ -53,4 +54,4 @@ jobs: dbt deps dbt seed --target bigquery --full-refresh dbt run --target bigquery --full-refresh - dbt test --target bigquery \ No newline at end of file + dbt test --target bigquery diff --git a/.gitignore b/.gitignore index c81afa2..6ad2ce6 100644 --- a/.gitignore +++ b/.gitignore @@ -3,4 +3,5 @@ dbt_modules/ logs/ keyfile.json .DS_Store -develop/ \ No newline at end of file +develop/ +dbt_packages/ \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..212a88a --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,11 @@ +# dbt_marketo_source v0.6.0 +🎉 dbt v1.0.0 Compatibility 🎉 +## 🚨 Breaking Changes 🚨 +- Adjusts the `require-dbt-version` to now be within the range [">=1.0.0", "<2.0.0"]. Additionally, the package has been updated for dbt v1.0.0 compatibility. If you are using a dbt version <1.0.0, you will need to upgrade in order to leverage the latest version of the package. + - For help upgrading your package, I recommend reviewing this GitHub repo's Release Notes on what changes have been implemented since your last upgrade. + - For help upgrading your dbt project to dbt v1.0.0, I recommend reviewing dbt-labs [upgrading to 1.0.0 docs](https://docs.getdbt.com/docs/guides/migration-guide/upgrading-to-1-0-0) for more details on what changes must be made. +- Upgrades the package dependency to refer to the latest `dbt_fivetran_utils`. The latest `dbt_fivetran_utils` package also has a dependency on `dbt_utils` [">=0.8.0", "<0.9.0"]. + - Please note, if you are installing a version of `dbt_utils` in your `packages.yml` that is not in the range above then you will encounter a package dependency error. + +# dbt_marketo_source v0.1.0 -> v0.5.0 +Refer to the relevant release notes on the Github repository for specific details for the previous releases. Thank you! \ No newline at end of file diff --git a/README.md b/README.md index b9e955a..1c749dc 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -[![Apache License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) ![dbt Logo and Version](https://img.shields.io/static/v1?logo=dbt&label=dbt-version&message=0.20.x&color=orange) +[![Apache License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) # Marketo Source ([docs](https://fivetran-dbt-marketo.netlify.app/#!/overview)) This package models Marketo data from [Fivetran's connector](https://fivetran.com/docs/applications/marketo). It uses data in the format described by [this ERD](https://fivetran.com/docs/applications/marketo#schema). @@ -29,7 +29,7 @@ Include in your `packages.yml` ```yaml packages: - package: fivetran/marketo_source - version: [">=0.5.0", "<0.6.0"] + version: [">=0.6.0", "<0.7.0"] ``` ## Configuration diff --git a/dbt_project.yml b/dbt_project.yml index 7c625f6..af663ee 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -1,34 +1,33 @@ - name: 'marketo_source' -version: '0.5.0' +version: '0.6.0' config-version: 2 -require-dbt-version: ">=0.20.0" +require-dbt-version: [">=1.0.0", "<2.0.0"] vars: - marketo__enable_programs: False - marketo__enable_campaigns: False + marketo__enable_programs: false + marketo__enable_campaigns: false - marketo_source: - lead: "{{ source('marketo','lead') }}" - lead_describe: "{{ source('marketo','lead_describe') }}" - campaign: "{{ source('marketo','campaign') }}" - program: "{{ source('marketo','program') }}" - email_template_history: "{{ source('marketo','email_template_history') }}" - activity_change_data_value: "{{ source('marketo','activity_change_data_value') }}" - activity_click_email: "{{ source('marketo','activity_click_email') }}" - activity_click_link: "{{ source('marketo','activity_click_link') }}" - activity_email_delivered: "{{ source('marketo','activity_email_delivered') }}" - activity_email_bounced: "{{ source('marketo','activity_email_bounced') }}" - activity_open_email: "{{ source('marketo','activity_open_email') }}" - activity_unsubscribe_email: "{{ source('marketo','activity_unsubscribe_email') }}" - activity_send_email: "{{ source('marketo','activity_send_email') }}" - activity_delete_lead: "{{ source('marketo','activity_delete_lead') }}" - activity_merge_leads: "{{ source('marketo','activity_merge_leads') }}" + marketo_source: + lead: "{{ source('marketo','lead') }}" + lead_describe: "{{ source('marketo','lead_describe') }}" + campaign: "{{ source('marketo','campaign') }}" + program: "{{ source('marketo','program') }}" + email_template_history: "{{ source('marketo','email_template_history') }}" + activity_change_data_value: "{{ source('marketo','activity_change_data_value') }}" + activity_click_email: "{{ source('marketo','activity_click_email') }}" + activity_click_link: "{{ source('marketo','activity_click_link') }}" + activity_email_delivered: "{{ source('marketo','activity_email_delivered') }}" + activity_email_bounced: "{{ source('marketo','activity_email_bounced') }}" + activity_open_email: "{{ source('marketo','activity_open_email') }}" + activity_unsubscribe_email: "{{ source('marketo','activity_unsubscribe_email') }}" + activity_send_email: "{{ source('marketo','activity_send_email') }}" + activity_delete_lead: "{{ source('marketo','activity_delete_lead') }}" + activity_merge_leads: "{{ source('marketo','activity_merge_leads') }}" -models: - marketo_source: - +materialized: table - +schema: stg_marketo - tmp: - +materialized: view \ No newline at end of file +models: + marketo_source: + +materialized: table + +schema: stg_marketo + tmp: + +materialized: view diff --git a/integration_tests/requirements.txt b/integration_tests/requirements.txt index a682a70..9edc95a 100644 --- a/integration_tests/requirements.txt +++ b/integration_tests/requirements.txt @@ -1 +1,6 @@ -dbt~=0.20.0 +dbt-snowflake==1.0.0 +dbt-bigquery==1.0.0 +dbt-redshift==1.0.0 +dbt-postgres==1.0.0 +dbt-spark==1.0.0 +dbt-spark[PyHive]==1.0.0 diff --git a/integration_tests/data/marketo_activity_change_data_value_data.csv b/integration_tests/seeds/marketo_activity_change_data_value_data.csv similarity index 100% rename from integration_tests/data/marketo_activity_change_data_value_data.csv rename to integration_tests/seeds/marketo_activity_change_data_value_data.csv diff --git a/integration_tests/data/marketo_activity_click_email_data.csv b/integration_tests/seeds/marketo_activity_click_email_data.csv similarity index 100% rename from integration_tests/data/marketo_activity_click_email_data.csv rename to integration_tests/seeds/marketo_activity_click_email_data.csv diff --git a/integration_tests/data/marketo_activity_delete_lead_data.csv b/integration_tests/seeds/marketo_activity_delete_lead_data.csv similarity index 100% rename from integration_tests/data/marketo_activity_delete_lead_data.csv rename to integration_tests/seeds/marketo_activity_delete_lead_data.csv diff --git a/integration_tests/data/marketo_activity_email_bounced_data.csv b/integration_tests/seeds/marketo_activity_email_bounced_data.csv similarity index 100% rename from integration_tests/data/marketo_activity_email_bounced_data.csv rename to integration_tests/seeds/marketo_activity_email_bounced_data.csv diff --git a/integration_tests/data/marketo_activity_email_delivered_data.csv b/integration_tests/seeds/marketo_activity_email_delivered_data.csv similarity index 100% rename from integration_tests/data/marketo_activity_email_delivered_data.csv rename to integration_tests/seeds/marketo_activity_email_delivered_data.csv diff --git a/integration_tests/data/marketo_activity_merge_leads_data.csv b/integration_tests/seeds/marketo_activity_merge_leads_data.csv similarity index 100% rename from integration_tests/data/marketo_activity_merge_leads_data.csv rename to integration_tests/seeds/marketo_activity_merge_leads_data.csv diff --git a/integration_tests/data/marketo_activity_open_email_data.csv b/integration_tests/seeds/marketo_activity_open_email_data.csv similarity index 100% rename from integration_tests/data/marketo_activity_open_email_data.csv rename to integration_tests/seeds/marketo_activity_open_email_data.csv diff --git a/integration_tests/data/marketo_activity_send_email_data.csv b/integration_tests/seeds/marketo_activity_send_email_data.csv similarity index 100% rename from integration_tests/data/marketo_activity_send_email_data.csv rename to integration_tests/seeds/marketo_activity_send_email_data.csv diff --git a/integration_tests/data/marketo_activity_unsubscribe_email_data.csv b/integration_tests/seeds/marketo_activity_unsubscribe_email_data.csv similarity index 100% rename from integration_tests/data/marketo_activity_unsubscribe_email_data.csv rename to integration_tests/seeds/marketo_activity_unsubscribe_email_data.csv diff --git a/integration_tests/data/marketo_campaign_data.csv b/integration_tests/seeds/marketo_campaign_data.csv similarity index 100% rename from integration_tests/data/marketo_campaign_data.csv rename to integration_tests/seeds/marketo_campaign_data.csv diff --git a/integration_tests/data/marketo_email_template_history_data.csv b/integration_tests/seeds/marketo_email_template_history_data.csv similarity index 100% rename from integration_tests/data/marketo_email_template_history_data.csv rename to integration_tests/seeds/marketo_email_template_history_data.csv diff --git a/integration_tests/data/marketo_lead_data.csv b/integration_tests/seeds/marketo_lead_data.csv similarity index 100% rename from integration_tests/data/marketo_lead_data.csv rename to integration_tests/seeds/marketo_lead_data.csv diff --git a/integration_tests/data/marketo_lead_describe_data.csv b/integration_tests/seeds/marketo_lead_describe_data.csv similarity index 100% rename from integration_tests/data/marketo_lead_describe_data.csv rename to integration_tests/seeds/marketo_lead_describe_data.csv diff --git a/integration_tests/data/marketo_program_data.csv b/integration_tests/seeds/marketo_program_data.csv similarity index 100% rename from integration_tests/data/marketo_program_data.csv rename to integration_tests/seeds/marketo_program_data.csv diff --git a/packages.yml b/packages.yml index 14c292e..e8d9d9e 100644 --- a/packages.yml +++ b/packages.yml @@ -1,3 +1,3 @@ packages: - package: fivetran/fivetran_utils - version: [">=0.2.0", "<0.3.0"] + version: [">=0.3.0", "<0.4.0"]