Pants: Add BUILD metadata to handle git submodule (#6258)

.github/workflows/test.yaml

@@ -75,7 +75,17 @@ jobs:
         uses: actions/checkout@v4
         with:
           # a test uses a submodule, and pants needs access to it to calculate deps.
-          submodules: 'true'
+          submodules: 'recursive'
+          # sadly, the submodule will only have fetch-depth=1, which is what we want
+          # for st2.git, but not for the submodules. We still want actions/checkout
+          # to do the initial checkout, however, so that it adds auth for fetching
+          # in the submodule.
+
+      - name: Fetch repository submodules
+        run: |
+          git submodule status
+          git submodule foreach 'git fetch --all --tags'
+          git submodule foreach 'git tag'
 
       - name: 'Set up Python (${{ matrix.python-version }})'
         uses: actions/setup-python@v5

BUILD

@@ -93,3 +93,28 @@ file(
     name="logs_directory",
     source="logs/.gitignore",
 )
+
+files(
+    name="gitmodules",
+    sources=[
+        ".gitmodules",
+        "**/.git",
+    ],
+)
+
+shell_command(
+    name="capture_git_modules",
+    environment="in_repo_workspace",
+    command="cp -r .git/modules {chroot}/.git",
+    tools=["cp"],
+    # execution_dependencies allows pants to invalidate the output
+    # of this command if the .gitmodules file changes (for example:
+    # if a submodule gets updated to a different repo).
+    # Sadly this does not get invalidated if the submodule commit
+    # is updated. In our case, that should be rare. To work around
+    # this, kill the `pantsd` process after updating a submodule.
+    execution_dependencies=[":gitmodules"],
+    output_dependencies=[":gitmodules"],
+    output_directories=[".git/modules"],
+    workdir="/",
+)

BUILD.environment

@@ -0,0 +1,23 @@
+# Everything listed in pants.toml [evironments-preview.names] should be defined here.
+# Relevant docs:
+# - https://www.pantsbuild.org/stable/docs/using-pants/environments
+# - https://www.pantsbuild.org/stable/reference/targets/experimental_workspace_environment
+# - https://www.pantsbuild.org/stable/reference/targets/local_environment
+# - https://www.pantsbuild.org/stable/reference/targets/docker_environment
+
+# This file MUST NOT use any macros.
+
+experimental_workspace_environment(
+    name="in_repo_workspace",
+    description=(
+        """
+        This allows shell_command and similar to run in the repo, instead of in a sandbox.
+        Only use this environment for commands or goals that are idempotent.
+        Ideally, such commands do NOT change anything in the repo.
+
+        If you need to capture output, note that output gets captured from a temporary
+        sandbox, not from the repo root. So, you may need to copy output files into
+        the sandbox with something like `cp path/to/file {chroot}/path/to/file`.
+        """
+    ),
+)

CHANGELOG.rst

@@ -64,7 +64,8 @@ Added
 * Continue introducing `pants <https://www.pantsbuild.org/docs>`_ to improve DX (Developer Experience)
   working on StackStorm, improve our security posture, and improve CI reliability thanks in part
   to pants' use of PEX lockfiles. This is not a user-facing addition.
-  #6118 #6141 #6133 #6120 #6181 #6183 #6200 #6237 #6229 #6240 #6241 #6244 #6251 #6253 #6254
+  #6118 #6141 #6133 #6120 #6181 #6183 #6200 #6237 #6229 #6240 #6241 #6244 #6251 #6253
+  #6254 #6258
   Contributed by @cognifloyd
 * Build of ST2 EL9 packages #6153
   Contributed by @amanda11

contrib/runners/python_runner/tests/unit/BUILD

@@ -9,11 +9,13 @@ python_tests(
         "test_output_schema.py": dict(
             dependencies=[
                 "st2tests/st2tests/resources/packs/pythonactions/actions/pascal_row.py",
+                "//:capture_git_modules",
             ],
         ),
         "test_pythonrunner.py": dict(
             dependencies=[
                 "st2tests/st2tests/resources/packs/pythonactions/actions",
+                "//:capture_git_modules",
             ],
             stevedore_namespaces=[
                 "st2common.metrics.driver",

pants.toml

@@ -249,5 +249,9 @@ extra_env_vars = [
 [twine]
 install_from_resolve = "twine"
 
+[environments-preview.names]
+# https://www.pantsbuild.org/stable/docs/using-pants/environments
+in_repo_workspace = "//:in_repo_workspace"
+
 [cli.alias]
 --all-changed = "--changed-since=HEAD --changed-dependents=transitive"

st2tests/st2tests/fixtures/packs/BUILD

@@ -11,6 +11,7 @@ pack_metadata_in_git_submodule(
         "test_content_version/icon.png",
         "test_content_version/requirements.txt",
     ],
+    # NOTE: If you need the git metadata, make sure to depend on //:capture_git_modules
 )
 
 st2_shell_sources_and_resources(