Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

StackStorm Packs - Support for Git Submodules #5963

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions CHANGELOG.rst
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,8 @@ Added
* Expose environment variable ST2_ACTION_DEBUG to all StackStorm actions.
Contributed by @maxfactor1

* Added option to checkout git submodules when downloading/installing packs #5814
Contributed by @jk464

3.8.0 - November 18, 2022
-------------------------
Expand Down
5 changes: 5 additions & 0 deletions contrib/packs/actions/download.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -28,3 +28,8 @@
items:
type: "string"
required: false
checkout_submodules:
type: "boolean"
description: "Set to True to checkout git submodules present in the pack"
required: false
default: false
5 changes: 5 additions & 0 deletions contrib/packs/actions/install.meta.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -37,4 +37,9 @@
required: false
description: Action timeout in seconds. Action will get killed if it doesn't finish in timeout
type: integer
checkout_submodules:
type: "boolean"
description: "Set to True to checkout git submodules present in the pack"
required: false
default: false

10 changes: 9 additions & 1 deletion contrib/packs/actions/pack_mgmt/download.py
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,13 @@ def __init__(self, config=None, action_service=None):
os.environ["no_proxy"] = self.no_proxy

def run(
self, packs, abs_repo_base, verifyssl=True, force=False, dependency_list=None
self,
packs,
abs_repo_base,
verifyssl=True,
force=False,
dependency_list=None,
checkout_submodules=False,
):
result = {}
pack_url = None
Expand All @@ -86,6 +92,7 @@ def run(
proxy_config=self.proxy_config,
force_permissions=True,
logger=self.logger,
checkout_submodules=checkout_submodules,
)
pack_url, pack_ref, pack_result = pack_result
result[pack_ref] = pack_result
Expand All @@ -99,6 +106,7 @@ def run(
proxy_config=self.proxy_config,
force_permissions=True,
logger=self.logger,
checkout_submodules=checkout_submodules,
)
pack_url, pack_ref, pack_result = pack_result
result[pack_ref] = pack_result
Expand Down
2 changes: 2 additions & 0 deletions contrib/packs/actions/workflows/install.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ input:
- force
- skip_dependencies
- timeout
- checkout_submodules

vars:
- packs_list: null
Expand All @@ -30,6 +31,7 @@ tasks:
packs: <% ctx().packs %>
force: <% ctx().force %>
dependency_list: <% ctx().dependency_list %>
checkout_submodules: <% ctx().checkout_submodules %>
next:
- when: <% succeeded() %>
do: make_a_prerun
Expand Down
3 changes: 3 additions & 0 deletions st2api/st2api/controllers/v1/packs.py
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,9 @@ def post(self, pack_install_request, requester_user=None):
if pack_install_request.skip_dependencies:
parameters["skip_dependencies"] = True

if pack_install_request.checkout_submodules:
parameters["checkout_submodules"] = True

if not requester_user:
requester_user = UserDB(name=cfg.CONF.system_user.user)

Expand Down
12 changes: 12 additions & 0 deletions st2api/tests/unit/controllers/v1/test_packs.py
Original file line number Diff line number Diff line change
Expand Up @@ -210,6 +210,18 @@ def test_install_with_skip_dependencies_parameter(self, _handle_schedule_executi
self.assertEqual(resp.status_int, 202)
self.assertEqual(resp.json, {"execution_id": "123"})

@mock.patch.object(ActionExecutionsControllerMixin, "_handle_schedule_execution")
def test_install_with_checkout_submodules_parameter(
self, _handle_schedule_execution
):
_handle_schedule_execution.return_value = Response(json={"id": "123"})
payload = {"packs": ["some"], "checkout_submodules": True}

resp = self.app.post_json("/v1/packs/install", payload)

self.assertEqual(resp.status_int, 202)
self.assertEqual(resp.json, {"execution_id": "123"})

@mock.patch.object(ActionExecutionsControllerMixin, "_handle_schedule_execution")
def test_uninstall(self, _handle_schedule_execution):
_handle_schedule_execution.return_value = Response(json={"id": "123"})
Expand Down
7 changes: 7 additions & 0 deletions st2client/st2client/commands/pack.py
Original file line number Diff line number Diff line change
Expand Up @@ -266,6 +266,12 @@ def __init__(self, resource, *args, **kwargs):
default=False,
help="Skip pack dependency installation.",
)
self.parser.add_argument(
"--checkout-submodules",
action="store_true",
default=False,
help="Checkout git submodules present in the pack.",
)

def run(self, args, **kwargs):
is_structured_output = args.json or args.yaml
Expand All @@ -279,6 +285,7 @@ def run(self, args, **kwargs):
args.packs,
force=args.force,
skip_dependencies=args.skip_dependencies,
checkout_submodules=args.checkout_submodules,
**kwargs,
)

Expand Down
10 changes: 9 additions & 1 deletion st2client/st2client/models/core.py
Original file line number Diff line number Diff line change
Expand Up @@ -628,12 +628,20 @@ class AsyncRequest(Resource):

class PackResourceManager(ResourceManager):
@add_auth_token_to_kwargs_from_env
def install(self, packs, force=False, skip_dependencies=False, **kwargs):
def install(
self,
packs,
force=False,
skip_dependencies=False,
checkout_submodules=False,
**kwargs,
):
url = "/%s/install" % (self.resource.get_url_path_name())
payload = {
"packs": packs,
"force": force,
"skip_dependencies": skip_dependencies,
"checkout_submodules": checkout_submodules,
}
response = self.client.post(url, payload, **kwargs)
if response.status_code != http_client.OK:
Expand Down
5 changes: 5 additions & 0 deletions st2common/st2common/models/api/pack.py
Original file line number Diff line number Diff line change
Expand Up @@ -394,6 +394,11 @@ class PackInstallRequestAPI(BaseAPI):
"description": "Set to True to skip pack dependency installations.",
"default": False,
},
"checkout_submodules": {
"type": "boolean",
"description": "Checkout git submodules present in the pack.",
"default": False,
},
},
}

Expand Down
4 changes: 4 additions & 0 deletions st2common/st2common/openapi.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -5173,6 +5173,10 @@ definitions:
type: boolean
description: Set to True to skip pack dependency installations.
default: false
checkout_submodules:
type: boolean
description: Set to True to checkout git submodules present in the pack.
default: false
required:
- packs
PacksUninstall:
Expand Down
4 changes: 4 additions & 0 deletions st2common/st2common/openapi.yaml.j2
Original file line number Diff line number Diff line change
Expand Up @@ -5169,6 +5169,10 @@ definitions:
type: boolean
description: Set to True to skip pack dependency installations.
default: false
checkout_submodules:
type: boolean
description: Set to True to checkout git submodules present in the pack.
default: false
required:
- packs
PacksUninstall:
Expand Down
14 changes: 13 additions & 1 deletion st2common/st2common/util/pack_management.py
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,7 @@ def download_pack(
force_owner_group=True,
force_permissions=True,
logger=LOG,
checkout_submodules=False,
):
"""
Download the pack and move it to /opt/stackstorm/packs.
Expand All @@ -99,6 +100,10 @@ def download_pack(
:param force: Force the installation and ignore / delete the lock file if it already exists.
:type force: ``bool``

:param checkout_submodules: Whether to also checkout git submodules present
in the pack
:type checkout_submodules: ``bool``

:return: (pack_url, pack_ref, result)
:rtype: tuple
"""
Expand Down Expand Up @@ -163,6 +168,7 @@ def download_pack(
repo_url=pack_url,
verify_ssl=verify_ssl,
ref=pack_version,
checkout_submodules=checkout_submodules,
)

pack_metadata = get_pack_metadata(pack_dir=abs_local_path)
Expand Down Expand Up @@ -190,7 +196,9 @@ def download_pack(
return tuple(result)


def clone_repo(temp_dir, repo_url, verify_ssl=True, ref="master"):
def clone_repo(
temp_dir, repo_url, verify_ssl=True, ref="master", checkout_submodules=False
):
# Switch to non-interactive mode
os.environ["GIT_TERMINAL_PROMPT"] = "0"
os.environ["GIT_ASKPASS"] = "/bin/echo"
Expand All @@ -204,6 +212,10 @@ def clone_repo(temp_dir, repo_url, verify_ssl=True, ref="master"):
# future.
repo = Repo.clone_from(repo_url, temp_dir)

# Checkout any Git Submodules if requested.
if checkout_submodules:
repo.submodule_update(recursive=False)

is_local_repo = repo_url.startswith("file://")

try:
Expand Down