From b38fdd8492435a226aa89ec14c765884a0860fb4 Mon Sep 17 00:00:00 2001
From: Claire Lin <claire@elementl.com>
Date: Mon, 14 Aug 2023 10:40:30 -0700
Subject: [PATCH] [GitLab Serverless CI] Build python package with dbt project
 (#148)

---
 gitlab/dbt/serverless-ci-dbt.yml | 91 ++++++++++++++++++++++++++++++++
 1 file changed, 91 insertions(+)
 create mode 100644 gitlab/dbt/serverless-ci-dbt.yml

diff --git a/gitlab/dbt/serverless-ci-dbt.yml b/gitlab/dbt/serverless-ci-dbt.yml
new file mode 100644
index 00000000..fc504c98
--- /dev/null
+++ b/gitlab/dbt/serverless-ci-dbt.yml
@@ -0,0 +1,91 @@
+variables:
+  DISABLE_FAST_DEPLOYS:
+  DAGSTER_CLOUD_URL: $DAGSTER_CLOUD_URL
+  DAGSTER_CLOUD_API_TOKEN: $DAGSTER_CLOUD_API_TOKEN
+  DAGSTER_PROJECT_NAME: "dagster_dbt_scaffold"
+  DBT_PROJECT_DIR: "${CI_PROJECT_DIR}"
+  DBT_PACKAGE_DATA_DIR: "${CI_PROJECT_DIR}/$DAGSTER_PROJECT_NAME/dbt-project"
+
+deploy-branch:
+  stage: deploy
+  rules:
+    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
+  image: ghcr.io/dagster-io/dagster-cloud-action:0.1.27
+  script:
+    # first create the branch deployment
+    - export PR_TIMESTAMP=$(git log -1 --format='%cd' --date=unix)
+    - export PR_MESSAGE=$(git log -1 --format='%s')
+    - export PR_EMAIL=$(git log -1 --format='%ae')
+    - export PR_NAME=$(git log -1 --format='%an')
+    - export DEPLOYMENT_NAME=$(dagster-cloud branch-deployment create-or-update
+      --url $DAGSTER_CLOUD_URL
+      --api-token $DAGSTER_CLOUD_API_TOKEN
+      --git-repo-name $CI_PROJECT_NAME
+      --branch-name $CI_COMMIT_REF_NAME
+      --branch-url "${CI_PROJECT_URL}/-/tree/${CI_COMMIT_REF_NAME}"
+      --pull-request-url "${CI_PROJECT_URL}/-/merge_requests/${CI_MERGE_REQUEST_IID}"
+      --pull-request-id $CI_MERGE_REQUEST_IID
+      --commit-hash $CI_COMMIT_SHORT_SHA
+      --timestamp $PR_TIMESTAMP
+      --commit-message "${PR_MESSAGE}"
+      --author-name "${PR_NAME}"
+      --author-email $PR_EMAIL)
+    # install dbt package
+    - pip install pip --upgrade
+    - cd $DBT_PROJECT_DIR/$DAGSTER_PROJECT_NAME
+    - pip install . MarkupSafe==2.0.1 'click>8.1.0' 'Jinja2>3.0.0'
+    - pip install pyOpenSSL --upgrade
+    - cd -
+    - which rsync || ( apt-get update -y && apt-get -y install rsync )
+    - rsync -avz --exclude $DAGSTER_PROJECT_NAME --exclude .git $DBT_PROJECT_DIR/ $DBT_PACKAGE_DATA_DIR
+    - dbt deps --project-dir $DBT_PACKAGE_DATA_DIR --profiles-dir $DBT_PACKAGE_DATA_DIR
+    - dbt parse --project-dir $DBT_PACKAGE_DATA_DIR --profiles-dir $DBT_PACKAGE_DATA_DIR
+    - rm $DBT_PACKAGE_DATA_DIR/target/partial_parse.msgpack
+    # then deploy to that branch
+    - /gitlab_action/deploy.py ./dagster_cloud.yaml $DEPLOYMENT_NAME
+  environment:
+    name: branch/$CI_COMMIT_REF_NAME
+    on_stop: close_branch
+
+close_branch:
+  stage: deploy
+  image: ghcr.io/dagster-io/dagster-cloud-action:0.1.27
+  when: manual
+  only:
+    - merge_requests
+  script:
+    - export PR_TIMESTAMP=$(git log -1 --format='%cd' --date=unix)
+    - dagster-cloud branch-deployment create-or-update
+      --url $DAGSTER_CLOUD_URL
+      --api-token $DAGSTER_CLOUD_API_TOKEN
+      --git-repo-name $CI_PROJECT_NAME
+      --branch-name $CI_COMMIT_REF_NAME
+      --branch-url "${CI_PROJECT_URL}/-/tree/${CI_COMMIT_REF_NAME}"
+      --pull-request-url "${CI_PROJECT_URL}/-/merge_requests/${CI_MERGE_REQUEST_IID}"
+      --pull-request-id $CI_MERGE_REQUEST_IID
+      --pull-request-status "CLOSED"
+      --commit-hash $CI_COMMIT_SHORT_SHA
+      --timestamp $PR_TIMESTAMP
+  environment:
+    name: branch/$CI_COMMIT_REF_NAME
+    action: stop
+
+deploy:
+  stage: deploy
+  rules:
+    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
+  image: ghcr.io/dagster-io/dagster-cloud-action:0.1.27
+  script:
+    # install dbt package
+    - pip install pip --upgrade
+    - cd $DBT_PROJECT_DIR/$DAGSTER_PROJECT_NAME
+    - pip install . MarkupSafe==2.0.1 'click>8.1.0' 'Jinja2>3.0.0'
+    - pip install pyOpenSSL --upgrade
+    - cd -
+    - which rsync || ( apt-get update -y && apt-get -y install rsync )
+    - rsync -avz --exclude $DAGSTER_PROJECT_NAME --exclude .git $DBT_PROJECT_DIR/ $DBT_PACKAGE_DATA_DIR
+    - dbt deps --project-dir $DBT_PACKAGE_DATA_DIR --profiles-dir $DBT_PACKAGE_DATA_DIR
+    - dbt parse --project-dir $DBT_PACKAGE_DATA_DIR --profiles-dir $DBT_PACKAGE_DATA_DIR
+    - rm $DBT_PACKAGE_DATA_DIR/target/partial_parse.msgpack
+    # deploy
+    - /gitlab_action/deploy.py ./dagster_cloud.yaml