From fd4562048e7bebea4577b98ab3a110e5784c3f2b Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Tue, 10 Sep 2024 20:47:23 -0500 Subject: [PATCH 01/10] pants: record test dependencies that could not easily be inferred --- st2tests/st2tests/fixtures/packs/BUILD | 2 ++ 1 file changed, 2 insertions(+) diff --git a/st2tests/st2tests/fixtures/packs/BUILD b/st2tests/st2tests/fixtures/packs/BUILD index 025cf82aac..71ed30b00c 100644 --- a/st2tests/st2tests/fixtures/packs/BUILD +++ b/st2tests/st2tests/fixtures/packs/BUILD @@ -8,8 +8,10 @@ pack_metadata_in_git_submodule( sources=[ "test_content_version/pack.yaml", "test_content_version/**/*.yaml", + "!test_content_version/.github/workflows/*.yaml", "test_content_version/icon.png", "test_content_version/requirements.txt", + "test_content_version/.git", # file that is git ignored, but used by the tests ], ) From 73c84ffc825776f46ce62ce6deda9ce373ba146c Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Thu, 12 Sep 2024 13:23:48 -0500 Subject: [PATCH 02/10] pants: capture .git/modules so pants can run pythonrunner tests --- BUILD | 23 +++++++++++++++++++ BUILD.environment | 23 +++++++++++++++++++ .../runners/python_runner/tests/unit/BUILD | 2 ++ pants.toml | 4 ++++ st2tests/st2tests/fixtures/packs/BUILD | 3 +-- 5 files changed, 53 insertions(+), 2 deletions(-) create mode 100644 BUILD.environment diff --git a/BUILD b/BUILD index a56fcf6b6f..25c87e41c7 100644 --- a/BUILD +++ b/BUILD @@ -93,3 +93,26 @@ file( name="logs_directory", source="logs/.gitignore", ) + +files( + name="gitmodules", + sources=[ + ".gitmodules", + "**/.git", + ], +) + +shell_command( + name="capture_git_modules", + environment="in_repo_workspace", + command="cp -r .git/modules {chroot}/.git", + tools=["cp"], + # execution_dependencies allows pants to invalidate the output + # of this command if the .gitmodules file changes (for example: + # if a submodule gets updated to a different commit). + # Theoretically, nothing else should modify .git/modules/. + execution_dependencies=[":gitmodules"], + output_dependencies=[":gitmodules"], + output_directories=[".git/modules"], + workdir="/", +) diff --git a/BUILD.environment b/BUILD.environment new file mode 100644 index 0000000000..5c1f26cdd3 --- /dev/null +++ b/BUILD.environment @@ -0,0 +1,23 @@ +# Everything listed in pants.toml [evironments-preview.names] should be defined here. +# Relevant docs: +# - https://www.pantsbuild.org/stable/docs/using-pants/environments +# - https://www.pantsbuild.org/stable/reference/targets/experimental_workspace_environment +# - https://www.pantsbuild.org/stable/reference/targets/local_environment +# - https://www.pantsbuild.org/stable/reference/targets/docker_environment + +# This file MUST NOT use any macros. + +experimental_workspace_environment( + name="in_repo_workspace", + description=( + """ + This allows shell_command and similar to in the repo, instead of in a sandbox. + Only use this environment for commands or goals that are idempotent. + Ideally, such commands do NOT change anything in the repo. + + If you need to capture output, note that output gets captured from a temporary + sandbox, not from the repo root. So, you may need to copy output files into + the sandbox with something like `cp path/to/file {chroot}/path/to/file`. + """ + ), +) diff --git a/contrib/runners/python_runner/tests/unit/BUILD b/contrib/runners/python_runner/tests/unit/BUILD index 39ad860aa4..656e54f328 100644 --- a/contrib/runners/python_runner/tests/unit/BUILD +++ b/contrib/runners/python_runner/tests/unit/BUILD @@ -9,11 +9,13 @@ python_tests( "test_output_schema.py": dict( dependencies=[ "st2tests/st2tests/resources/packs/pythonactions/actions/pascal_row.py", + "//:capture_git_modules", ], ), "test_pythonrunner.py": dict( dependencies=[ "st2tests/st2tests/resources/packs/pythonactions/actions", + "//:capture_git_modules", ], stevedore_namespaces=[ "st2common.metrics.driver", diff --git a/pants.toml b/pants.toml index 8cda497564..f532780afe 100644 --- a/pants.toml +++ b/pants.toml @@ -249,5 +249,9 @@ extra_env_vars = [ [twine] install_from_resolve = "twine" +[environments-preview.names] +# https://www.pantsbuild.org/stable/docs/using-pants/environments +in_repo_workspace = "//:in_repo_workspace" + [cli.alias] --all-changed = "--changed-since=HEAD --changed-dependents=transitive" diff --git a/st2tests/st2tests/fixtures/packs/BUILD b/st2tests/st2tests/fixtures/packs/BUILD index 71ed30b00c..a5006ef9d1 100644 --- a/st2tests/st2tests/fixtures/packs/BUILD +++ b/st2tests/st2tests/fixtures/packs/BUILD @@ -8,11 +8,10 @@ pack_metadata_in_git_submodule( sources=[ "test_content_version/pack.yaml", "test_content_version/**/*.yaml", - "!test_content_version/.github/workflows/*.yaml", "test_content_version/icon.png", "test_content_version/requirements.txt", - "test_content_version/.git", # file that is git ignored, but used by the tests ], + # NOTE: If you need the git metadata, make sure to depend on //:capture_git_modules ) st2_shell_sources_and_resources( From f07d3633781fbd79af7c54d0313a424bcb3378eb Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Thu, 12 Sep 2024 16:35:10 -0500 Subject: [PATCH 03/10] pants: workaround GHA using fetch-depth=1 for submodules I could change fetch-depth, but that would change it for both submodules and for the st2.git checkout. --- BUILD | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/BUILD b/BUILD index 25c87e41c7..26d3fe9998 100644 --- a/BUILD +++ b/BUILD @@ -102,6 +102,11 @@ files( ], ) +run_shell_command( + name="git_submodules_fetch", + command="git submodule foreach 'git fetch --all'", +) + shell_command( name="capture_git_modules", environment="in_repo_workspace", @@ -111,7 +116,10 @@ shell_command( # of this command if the .gitmodules file changes (for example: # if a submodule gets updated to a different commit). # Theoretically, nothing else should modify .git/modules/. - execution_dependencies=[":gitmodules"], + execution_dependencies=[ + ":gitmodules", + ":git_submodules_fetch", + ], output_dependencies=[":gitmodules"], output_directories=[".git/modules"], workdir="/", From bab34f99942ef9ad585690ddcf543341b07e792d Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Thu, 12 Sep 2024 17:00:12 -0500 Subject: [PATCH 04/10] pants: use GHA task to fetch submodules instead of pants --- .github/workflows/test.yaml | 7 +++++++ BUILD | 10 +--------- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 5bfb2b3815..12446f47e1 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -76,6 +76,13 @@ jobs: with: # a test uses a submodule, and pants needs access to it to calculate deps. submodules: 'true' + # sadly, the submodule will only have fetch-depth=1, which is what we want + # for st2.git, but not for the submodules. We still want actions/checkout + # to do the initial checkout, however, so that it adds auth for fetching + # in the submodule. + + - name: Fetch repository submodules + run: git submodule update --init --recursive --remote - name: 'Set up Python (${{ matrix.python-version }})' uses: actions/setup-python@v5 diff --git a/BUILD b/BUILD index 26d3fe9998..25c87e41c7 100644 --- a/BUILD +++ b/BUILD @@ -102,11 +102,6 @@ files( ], ) -run_shell_command( - name="git_submodules_fetch", - command="git submodule foreach 'git fetch --all'", -) - shell_command( name="capture_git_modules", environment="in_repo_workspace", @@ -116,10 +111,7 @@ shell_command( # of this command if the .gitmodules file changes (for example: # if a submodule gets updated to a different commit). # Theoretically, nothing else should modify .git/modules/. - execution_dependencies=[ - ":gitmodules", - ":git_submodules_fetch", - ], + execution_dependencies=[":gitmodules"], output_dependencies=[":gitmodules"], output_directories=[".git/modules"], workdir="/", From 5721cd9bc1b815b97adb75cd349b356a37f1ca4a Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Thu, 12 Sep 2024 17:41:51 -0500 Subject: [PATCH 05/10] pants: try again to get git submodules working --- .github/workflows/test.yaml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 12446f47e1..69a4d3d2c5 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -75,14 +75,17 @@ jobs: uses: actions/checkout@v4 with: # a test uses a submodule, and pants needs access to it to calculate deps. - submodules: 'true' + submodules: 'recursive' # sadly, the submodule will only have fetch-depth=1, which is what we want # for st2.git, but not for the submodules. We still want actions/checkout # to do the initial checkout, however, so that it adds auth for fetching # in the submodule. - name: Fetch repository submodules - run: git submodule update --init --recursive --remote + run: | + git submodule update --init --recursive --remote + git submodule status + git submodule foreach 'git tag' - name: 'Set up Python (${{ matrix.python-version }})' uses: actions/setup-python@v5 From 48b0cb4d0cadceb9b4ab4e23e3c6394a113f5022 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Thu, 12 Sep 2024 17:49:49 -0500 Subject: [PATCH 06/10] pants: try again to get git submodules working --- .github/workflows/test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 69a4d3d2c5..7b39b61a72 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -83,8 +83,8 @@ jobs: - name: Fetch repository submodules run: | - git submodule update --init --recursive --remote git submodule status + git submodule foreach 'git fetch --all --tags' git submodule foreach 'git tag' - name: 'Set up Python (${{ matrix.python-version }})' From 86a5d8b1eb87a88db87ae8c5140c012aeffbe18d Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Thu, 3 Oct 2024 01:12:25 -0500 Subject: [PATCH 07/10] correct comment about git submodules --- BUILD | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/BUILD b/BUILD index 25c87e41c7..8a365a257e 100644 --- a/BUILD +++ b/BUILD @@ -109,8 +109,11 @@ shell_command( tools=["cp"], # execution_dependencies allows pants to invalidate the output # of this command if the .gitmodules file changes (for example: - # if a submodule gets updated to a different commit). - # Theoretically, nothing else should modify .git/modules/. + # if a submodule gets updated to a different repo). + # Sadly this does not get invalidated if the submodule commit + # is updated. In our case, that should be rare. To work around + # If you update a submodule, + # this, kill the `pantsd` process after updating a submodule. execution_dependencies=[":gitmodules"], output_dependencies=[":gitmodules"], output_directories=[".git/modules"], From 089158d94122f9a3b5d4a25302ec832e7ea24f4f Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Fri, 4 Oct 2024 23:33:57 -0500 Subject: [PATCH 08/10] update changelog entry --- CHANGELOG.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 2bb9bc49bc..ff69e3abef 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -64,7 +64,8 @@ Added * Continue introducing `pants `_ to improve DX (Developer Experience) working on StackStorm, improve our security posture, and improve CI reliability thanks in part to pants' use of PEX lockfiles. This is not a user-facing addition. - #6118 #6141 #6133 #6120 #6181 #6183 #6200 #6237 #6229 #6240 #6241 #6244 #6251 #6253 #6254 + #6118 #6141 #6133 #6120 #6181 #6183 #6200 #6237 #6229 #6240 #6241 #6244 #6251 #6253 + #6254 #6258 Contributed by @cognifloyd * Build of ST2 EL9 packages #6153 Contributed by @amanda11 From f7461f85dc2fbb94f74c5d736f9fa371cfb8f919 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Sat, 5 Oct 2024 10:13:21 -0500 Subject: [PATCH 09/10] drop stray comment --- BUILD | 1 - 1 file changed, 1 deletion(-) diff --git a/BUILD b/BUILD index 8a365a257e..f33988a645 100644 --- a/BUILD +++ b/BUILD @@ -112,7 +112,6 @@ shell_command( # if a submodule gets updated to a different repo). # Sadly this does not get invalidated if the submodule commit # is updated. In our case, that should be rare. To work around - # If you update a submodule, # this, kill the `pantsd` process after updating a submodule. execution_dependencies=[":gitmodules"], output_dependencies=[":gitmodules"], From a25bcfd9b7b04405cd6a64f631595a0d4ec024f3 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Sat, 5 Oct 2024 10:41:53 -0500 Subject: [PATCH 10/10] typo --- BUILD.environment | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/BUILD.environment b/BUILD.environment index 5c1f26cdd3..f549e53f3e 100644 --- a/BUILD.environment +++ b/BUILD.environment @@ -11,7 +11,7 @@ experimental_workspace_environment( name="in_repo_workspace", description=( """ - This allows shell_command and similar to in the repo, instead of in a sandbox. + This allows shell_command and similar to run in the repo, instead of in a sandbox. Only use this environment for commands or goals that are idempotent. Ideally, such commands do NOT change anything in the repo.