Build dagster-cloud-action PEX for more platforms

Summary:
Make our dagster-cloud-cli work on github action runners that have python versions other than 3.8 installed.

.github/workflows/serverless_launch_job_definitions.yml

@@ -3,7 +3,7 @@ on:
   pull_request:
     types: [opened, synchronize, reopened, closed]
 env:
-  DAGSTER_CLOUD_URL: ${{ secrets.DAGSTER_CLOUD_SERVERLESS_URL }}
+  DAGSTER_CLOUD_URL: "https://dagster-cloud-action-github-workflow-serverless.dagster.cloud"
 
 jobs:
   dagster_cloud_launch:
@@ -17,10 +17,7 @@ jobs:
       - name: Launch a job on Dagster Cloud serverless
         uses: ./actions/utils/run
         with:
-          # differently named location (manually deployed), so that this launch integration test
-          # does not have race-y collisions with the deployment integration tests
-          # no repository name, since it uses Definitions
-          location_name: from_gh_action_for_launch_definitions
+          location_name: from_gh_action
           job_name: simple_job
         env:
           DAGSTER_CLOUD_API_TOKEN: ${{ secrets.DAGSTER_CLOUD_SERVERLESS_API_TOKEN }}

.github/workflows/tests.yml

@@ -1,4 +1,4 @@
-name: Tests 
+name: Tests
 on:
   push:
 
@@ -11,7 +11,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v4
         with:
-          python-version: "3.8"
+          python-version: "3.11"
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip

CONTRIBUTING.md

@@ -12,12 +12,12 @@ If you have a PAT handy you can just do:
 % echo $YOUR_GITHUB_PAT | docker login ghcr.io -u $YOUR_GITHUB_USERNAME --password-stdin
 ```
 
-## Step 2. Determine the new point version and create a release branch, 
+## Step 2. Determine the new point version and create a release branch,
 
 ```
 % git checkout main
 % git fetch origin --tags --force
-% git tag 
+% git tag
 pex-v0.1
 pex-v0.1.14
 prha
@@ -44,7 +44,7 @@ The point version is the next unused `v0.1.*` version, eg `v0.1.22` above. Creat
 
 ## Step 3. Build and deploy a new docker-cloud-action image, a new dagster.cloud.pex and update code references to docker
 
-A script does this work. **Note**: a virtual environment using Python3.8 is required to run the script.
+A script does this work. **Note**: a virtual environment using Python3.11 is required to run the script.
 
 ```
 # Note no 'v' prefix
@@ -106,7 +106,7 @@ git push -f origin v0.1.22
 
 # Step 6. Promote
 Most users point at a dot version tag for the GitHub Action, e.g. `@v0.1` and `@pex-v0.1`.
-If you are releasing a fix or non-breaking feature, you want to move this tag so existing users get access to your changes. 
+If you are releasing a fix or non-breaking feature, you want to move this tag so existing users get access to your changes.
 
 > Due an unfortunate bug, the ci-summary step in many user's workflow is pinned to v0.1.27, so we need v0.1.27 to follow v0.1 as well.
 > See https://github.com/dagster-io/dagster-cloud-hybrid-quickstart/commit/2149359dd1076f3bcd6f652dc00cb74ac9c81636

scripts/release.py

@@ -104,6 +104,16 @@ def update_dagster_cloud_pex(
         info("Using PyPI for dagster package")
         dagster_pkg = "dagster"
 
+    platform_args = []
+
+    # each of the default versions used by ubuntu 20.04 / 22.04 / 24.04 respectively
+    for py_version in ["38", "310", "312"]:
+        platform_args.extend(
+            [
+                f"--platform=manylinux2014_x86_64-cp-{py_version}-cp{py_version}",
+            ]
+        )
+
     info("Building generated/gha/dagster-cloud.pex")
     args = [
         "pex",
@@ -112,12 +122,14 @@ def update_dagster_cloud_pex(
         dagster_pkg,
         "PyGithub",
         "-o=dagster-cloud.pex",
-        "--platform=manylinux2014_x86_64-cp-38-cp38",
-        "--platform=macosx_12_0_x86_64-cp-38-cp38",
-        "--platform=macosx_12_0_arm64-cp-38-cp38",
+        *platform_args,
+        "--platform=macosx_12_0_x86_64-cp-311-cp311",
+        "--platform=macosx_12_0_arm64-cp-311-cp311",
         "--pip-version=23.0",
         "--resolver-version=pip-2020-resolver",
         "--venv=prepend",
+        # use a /bin/sh entrypoint that is better at choosing a python interpreter to use
+        "--sh-boot",
         "-v",
     ]
     print(f"Running {args}")

tests/test_pex_builder.py

@@ -2,12 +2,7 @@
 import subprocess
 import tempfile
 from contextlib import contextmanager
-from pathlib import Path
 from typing import List
-from unittest import mock
-
-import pytest
-import requests
 
 
 @contextmanager
@@ -23,18 +18,16 @@ def run_dagster_cloud_serverless_cmd(dagster_cloud_pex_path, args: List[str]):
                 build_output_dir,
             ],
             capture_output=True,
+            check=False,
         )
         if proc.returncode:
             raise ValueError(
-                "Failed to run dagster-cloud.pex:"
-                + (proc.stdout + proc.stderr).decode("utf-8")
+                "Failed to run dagster-cloud.pex:" + (proc.stdout + proc.stderr).decode("utf-8")
             )
 
         all_files = os.listdir(build_output_dir)
         pex_files = {
-            filename
-            for filename in all_files
-            if filename.endswith(".pex") and filename != ".pex"
+            filename for filename in all_files if filename.endswith(".pex") and filename != ".pex"
         }
         yield (build_output_dir, list(pex_files), list(set(all_files) - pex_files))
 
@@ -48,6 +41,7 @@ def test_pex_build_only(repo_root, dagster_cloud_pex_path):
             str(dagster_project1),
             "--api-token=fake",
             "--url=fake",
+            "--python-version=3.11",
         ],
     ) as (
         build_output_dir,
@@ -56,9 +50,7 @@ def test_pex_build_only(repo_root, dagster_cloud_pex_path):
     ):
         # one source-HASH.pex and one deps-HASH.pex file are expected
         assert 2 == len(pex_files)
-        pex_file_by_alias = {
-            filename.split("-", 1)[0]: filename for filename in pex_files
-        }
+        pex_file_by_alias = {filename.split("-", 1)[0]: filename for filename in pex_files}
 
         assert {"source", "deps"} == set(pex_file_by_alias)