diff --git a/gitlab/dbt/serverless-ci-dbt.yml b/gitlab/dbt/serverless-ci-dbt.yml new file mode 100644 index 00000000..fc504c98 --- /dev/null +++ b/gitlab/dbt/serverless-ci-dbt.yml @@ -0,0 +1,91 @@ +variables: + DISABLE_FAST_DEPLOYS: + DAGSTER_CLOUD_URL: $DAGSTER_CLOUD_URL + DAGSTER_CLOUD_API_TOKEN: $DAGSTER_CLOUD_API_TOKEN + DAGSTER_PROJECT_NAME: "dagster_dbt_scaffold" + DBT_PROJECT_DIR: "${CI_PROJECT_DIR}" + DBT_PACKAGE_DATA_DIR: "${CI_PROJECT_DIR}/$DAGSTER_PROJECT_NAME/dbt-project" + +deploy-branch: + stage: deploy + rules: + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' + image: ghcr.io/dagster-io/dagster-cloud-action:0.1.27 + script: + # first create the branch deployment + - export PR_TIMESTAMP=$(git log -1 --format='%cd' --date=unix) + - export PR_MESSAGE=$(git log -1 --format='%s') + - export PR_EMAIL=$(git log -1 --format='%ae') + - export PR_NAME=$(git log -1 --format='%an') + - export DEPLOYMENT_NAME=$(dagster-cloud branch-deployment create-or-update + --url $DAGSTER_CLOUD_URL + --api-token $DAGSTER_CLOUD_API_TOKEN + --git-repo-name $CI_PROJECT_NAME + --branch-name $CI_COMMIT_REF_NAME + --branch-url "${CI_PROJECT_URL}/-/tree/${CI_COMMIT_REF_NAME}" + --pull-request-url "${CI_PROJECT_URL}/-/merge_requests/${CI_MERGE_REQUEST_IID}" + --pull-request-id $CI_MERGE_REQUEST_IID + --commit-hash $CI_COMMIT_SHORT_SHA + --timestamp $PR_TIMESTAMP + --commit-message "${PR_MESSAGE}" + --author-name "${PR_NAME}" + --author-email $PR_EMAIL) + # install dbt package + - pip install pip --upgrade + - cd $DBT_PROJECT_DIR/$DAGSTER_PROJECT_NAME + - pip install . MarkupSafe==2.0.1 'click>8.1.0' 'Jinja2>3.0.0' + - pip install pyOpenSSL --upgrade + - cd - + - which rsync || ( apt-get update -y && apt-get -y install rsync ) + - rsync -avz --exclude $DAGSTER_PROJECT_NAME --exclude .git $DBT_PROJECT_DIR/ $DBT_PACKAGE_DATA_DIR + - dbt deps --project-dir $DBT_PACKAGE_DATA_DIR --profiles-dir $DBT_PACKAGE_DATA_DIR + - dbt parse --project-dir $DBT_PACKAGE_DATA_DIR --profiles-dir $DBT_PACKAGE_DATA_DIR + - rm $DBT_PACKAGE_DATA_DIR/target/partial_parse.msgpack + # then deploy to that branch + - /gitlab_action/deploy.py ./dagster_cloud.yaml $DEPLOYMENT_NAME + environment: + name: branch/$CI_COMMIT_REF_NAME + on_stop: close_branch + +close_branch: + stage: deploy + image: ghcr.io/dagster-io/dagster-cloud-action:0.1.27 + when: manual + only: + - merge_requests + script: + - export PR_TIMESTAMP=$(git log -1 --format='%cd' --date=unix) + - dagster-cloud branch-deployment create-or-update + --url $DAGSTER_CLOUD_URL + --api-token $DAGSTER_CLOUD_API_TOKEN + --git-repo-name $CI_PROJECT_NAME + --branch-name $CI_COMMIT_REF_NAME + --branch-url "${CI_PROJECT_URL}/-/tree/${CI_COMMIT_REF_NAME}" + --pull-request-url "${CI_PROJECT_URL}/-/merge_requests/${CI_MERGE_REQUEST_IID}" + --pull-request-id $CI_MERGE_REQUEST_IID + --pull-request-status "CLOSED" + --commit-hash $CI_COMMIT_SHORT_SHA + --timestamp $PR_TIMESTAMP + environment: + name: branch/$CI_COMMIT_REF_NAME + action: stop + +deploy: + stage: deploy + rules: + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + image: ghcr.io/dagster-io/dagster-cloud-action:0.1.27 + script: + # install dbt package + - pip install pip --upgrade + - cd $DBT_PROJECT_DIR/$DAGSTER_PROJECT_NAME + - pip install . MarkupSafe==2.0.1 'click>8.1.0' 'Jinja2>3.0.0' + - pip install pyOpenSSL --upgrade + - cd - + - which rsync || ( apt-get update -y && apt-get -y install rsync ) + - rsync -avz --exclude $DAGSTER_PROJECT_NAME --exclude .git $DBT_PROJECT_DIR/ $DBT_PACKAGE_DATA_DIR + - dbt deps --project-dir $DBT_PACKAGE_DATA_DIR --profiles-dir $DBT_PACKAGE_DATA_DIR + - dbt parse --project-dir $DBT_PACKAGE_DATA_DIR --profiles-dir $DBT_PACKAGE_DATA_DIR + - rm $DBT_PACKAGE_DATA_DIR/target/partial_parse.msgpack + # deploy + - /gitlab_action/deploy.py ./dagster_cloud.yaml