From 13257d642ea1ba27f79b4408129309766419c9b1 Mon Sep 17 00:00:00 2001 From: gibsondan Date: Tue, 1 Oct 2024 16:37:04 -0500 Subject: [PATCH] Build dagster-cloud-action PEX for more platforms Summary: Make our dagster-cloud-cli work on github action runners that have python versions other than 3.8 installed. --- .github/workflows/serverless_launch_job.yml | 27 ------------------- .../serverless_launch_job_definitions.yml | 7 ++--- .github/workflows/tests.yml | 4 +-- CONTRIBUTING.md | 8 +++--- scripts/release.py | 18 ++++++++++--- tests/test_pex_builder.py | 18 ++++--------- 6 files changed, 28 insertions(+), 54 deletions(-) delete mode 100644 .github/workflows/serverless_launch_job.yml diff --git a/.github/workflows/serverless_launch_job.yml b/.github/workflows/serverless_launch_job.yml deleted file mode 100644 index e548da00..00000000 --- a/.github/workflows/serverless_launch_job.yml +++ /dev/null @@ -1,27 +0,0 @@ -name: Serverless Job Launch -on: - pull_request: - types: [opened, synchronize, reopened, closed] -env: - DAGSTER_CLOUD_URL: ${{ secrets.DAGSTER_CLOUD_SERVERLESS_URL }} - -jobs: - dagster_cloud_launch: - runs-on: ubuntu-latest - name: Dagster Serverless Job Launch - steps: - - name: Checkout - uses: actions/checkout@v3 - with: - ref: ${{ github.sha }} - - name: Launch a job on Dagster Cloud serverless - uses: ./actions/utils/run - with: - # differently named location (manually deployed), so that this launch integration test - # does not have race-y collisions with the deployment integration tests - location_name: from_gh_action_for_launch - repository_name: repo - job_name: simple_job - env: - DAGSTER_CLOUD_API_TOKEN: ${{ secrets.DAGSTER_CLOUD_SERVERLESS_API_TOKEN }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/serverless_launch_job_definitions.yml b/.github/workflows/serverless_launch_job_definitions.yml index 50976eb3..af7a60d8 100644 --- a/.github/workflows/serverless_launch_job_definitions.yml +++ b/.github/workflows/serverless_launch_job_definitions.yml @@ -3,7 +3,7 @@ on: pull_request: types: [opened, synchronize, reopened, closed] env: - DAGSTER_CLOUD_URL: ${{ secrets.DAGSTER_CLOUD_SERVERLESS_URL }} + DAGSTER_CLOUD_URL: "https://dagster-cloud-action-github-workflow-serverless.dagster.cloud" jobs: dagster_cloud_launch: @@ -17,10 +17,7 @@ jobs: - name: Launch a job on Dagster Cloud serverless uses: ./actions/utils/run with: - # differently named location (manually deployed), so that this launch integration test - # does not have race-y collisions with the deployment integration tests - # no repository name, since it uses Definitions - location_name: from_gh_action_for_launch_definitions + location_name: from_gh_action job_name: simple_job env: DAGSTER_CLOUD_API_TOKEN: ${{ secrets.DAGSTER_CLOUD_SERVERLESS_API_TOKEN }} diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 95941735..d28b8597 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -1,4 +1,4 @@ -name: Tests +name: Tests on: push: @@ -11,7 +11,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v4 with: - python-version: "3.8" + python-version: "3.11" - name: Install dependencies run: | python -m pip install --upgrade pip diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 58bfaa1c..8410646b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -12,12 +12,12 @@ If you have a PAT handy you can just do: % echo $YOUR_GITHUB_PAT | docker login ghcr.io -u $YOUR_GITHUB_USERNAME --password-stdin ``` -## Step 2. Determine the new point version and create a release branch, +## Step 2. Determine the new point version and create a release branch, ``` % git checkout main % git fetch origin --tags --force -% git tag +% git tag pex-v0.1 pex-v0.1.14 prha @@ -44,7 +44,7 @@ The point version is the next unused `v0.1.*` version, eg `v0.1.22` above. Creat ## Step 3. Build and deploy a new docker-cloud-action image, a new dagster.cloud.pex and update code references to docker -A script does this work. **Note**: a virtual environment using Python3.8 is required to run the script. +A script does this work. **Note**: a virtual environment using Python3.11 is required to run the script. ``` # Note no 'v' prefix @@ -106,7 +106,7 @@ git push -f origin v0.1.22 # Step 6. Promote Most users point at a dot version tag for the GitHub Action, e.g. `@v0.1` and `@pex-v0.1`. -If you are releasing a fix or non-breaking feature, you want to move this tag so existing users get access to your changes. +If you are releasing a fix or non-breaking feature, you want to move this tag so existing users get access to your changes. > Due an unfortunate bug, the ci-summary step in many user's workflow is pinned to v0.1.27, so we need v0.1.27 to follow v0.1 as well. > See https://github.com/dagster-io/dagster-cloud-hybrid-quickstart/commit/2149359dd1076f3bcd6f652dc00cb74ac9c81636 diff --git a/scripts/release.py b/scripts/release.py index edc630d7..f3ba77a5 100755 --- a/scripts/release.py +++ b/scripts/release.py @@ -104,6 +104,16 @@ def update_dagster_cloud_pex( info("Using PyPI for dagster package") dagster_pkg = "dagster" + platform_args = [] + + # each of the default versions used by ubuntu 20.04 / 22.04 / 24.04 respectively + for py_version in ["38", "310", "312"]: + platform_args.extend( + [ + f"--platform=manylinux2014_x86_64-cp-{py_version}-cp{py_version}", + ] + ) + info("Building generated/gha/dagster-cloud.pex") args = [ "pex", @@ -112,12 +122,14 @@ def update_dagster_cloud_pex( dagster_pkg, "PyGithub", "-o=dagster-cloud.pex", - "--platform=manylinux2014_x86_64-cp-38-cp38", - "--platform=macosx_12_0_x86_64-cp-38-cp38", - "--platform=macosx_12_0_arm64-cp-38-cp38", + *platform_args, + "--platform=macosx_12_0_x86_64-cp-311-cp311", + "--platform=macosx_12_0_arm64-cp-311-cp311", "--pip-version=23.0", "--resolver-version=pip-2020-resolver", "--venv=prepend", + # use a /bin/sh entrypoint that is better at choosing a python interpreter to use + "--sh-boot", "-v", ] print(f"Running {args}") diff --git a/tests/test_pex_builder.py b/tests/test_pex_builder.py index 4d3a9e0d..e7782939 100644 --- a/tests/test_pex_builder.py +++ b/tests/test_pex_builder.py @@ -2,12 +2,7 @@ import subprocess import tempfile from contextlib import contextmanager -from pathlib import Path from typing import List -from unittest import mock - -import pytest -import requests @contextmanager @@ -23,18 +18,16 @@ def run_dagster_cloud_serverless_cmd(dagster_cloud_pex_path, args: List[str]): build_output_dir, ], capture_output=True, + check=False, ) if proc.returncode: raise ValueError( - "Failed to run dagster-cloud.pex:" - + (proc.stdout + proc.stderr).decode("utf-8") + "Failed to run dagster-cloud.pex:" + (proc.stdout + proc.stderr).decode("utf-8") ) all_files = os.listdir(build_output_dir) pex_files = { - filename - for filename in all_files - if filename.endswith(".pex") and filename != ".pex" + filename for filename in all_files if filename.endswith(".pex") and filename != ".pex" } yield (build_output_dir, list(pex_files), list(set(all_files) - pex_files)) @@ -48,6 +41,7 @@ def test_pex_build_only(repo_root, dagster_cloud_pex_path): str(dagster_project1), "--api-token=fake", "--url=fake", + "--python-version=3.11", ], ) as ( build_output_dir, @@ -56,9 +50,7 @@ def test_pex_build_only(repo_root, dagster_cloud_pex_path): ): # one source-HASH.pex and one deps-HASH.pex file are expected assert 2 == len(pex_files) - pex_file_by_alias = { - filename.split("-", 1)[0]: filename for filename in pex_files - } + pex_file_by_alias = {filename.split("-", 1)[0]: filename for filename in pex_files} assert {"source", "deps"} == set(pex_file_by_alias)