From 38a0b9337c90604186bca2fbc2fd3598de3ae658 Mon Sep 17 00:00:00 2001 From: jk464 Date: Tue, 29 Nov 2022 15:59:36 +0000 Subject: [PATCH] Added option to checkout git submodules when downloading/installing packs --- CHANGELOG.rst | 2 ++ contrib/packs/actions/download.yaml | 5 +++++ contrib/packs/actions/install.meta.yaml | 5 +++++ contrib/packs/actions/pack_mgmt/download.py | 10 +++++++++- contrib/packs/actions/workflows/install.yaml | 2 ++ st2api/st2api/controllers/v1/packs.py | 3 +++ st2api/tests/unit/controllers/v1/test_packs.py | 12 ++++++++++++ st2client/st2client/commands/pack.py | 7 +++++++ st2client/st2client/models/core.py | 10 +++++++++- st2common/st2common/models/api/pack.py | 5 +++++ st2common/st2common/openapi.yaml | 4 ++++ st2common/st2common/openapi.yaml.j2 | 4 ++++ st2common/st2common/util/pack_management.py | 14 +++++++++++++- 13 files changed, 80 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 96941c1401..94f7274828 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -28,6 +28,8 @@ Added * Expose environment variable ST2_ACTION_DEBUG to all StackStorm actions. Contributed by @maxfactor1 +* Added option to checkout git submodules when downloading/installing packs #5814 + Contributed by @jk464 3.8.0 - November 18, 2022 ------------------------- diff --git a/contrib/packs/actions/download.yaml b/contrib/packs/actions/download.yaml index 0849616b65..6683ce1b09 100644 --- a/contrib/packs/actions/download.yaml +++ b/contrib/packs/actions/download.yaml @@ -28,3 +28,8 @@ items: type: "string" required: false + checkout_submodules: + type: "boolean" + description: "Set to True to checkout git submodules present in the pack" + required: false + default: false diff --git a/contrib/packs/actions/install.meta.yaml b/contrib/packs/actions/install.meta.yaml index 191accd1c3..eecb0e5c28 100644 --- a/contrib/packs/actions/install.meta.yaml +++ b/contrib/packs/actions/install.meta.yaml @@ -37,4 +37,9 @@ required: false description: Action timeout in seconds. Action will get killed if it doesn't finish in timeout type: integer + checkout_submodules: + type: "boolean" + description: "Set to True to checkout git submodules present in the pack" + required: false + default: false diff --git a/contrib/packs/actions/pack_mgmt/download.py b/contrib/packs/actions/pack_mgmt/download.py index cc0f7cd8fb..39ebf625d7 100644 --- a/contrib/packs/actions/pack_mgmt/download.py +++ b/contrib/packs/actions/pack_mgmt/download.py @@ -71,7 +71,13 @@ def __init__(self, config=None, action_service=None): os.environ["no_proxy"] = self.no_proxy def run( - self, packs, abs_repo_base, verifyssl=True, force=False, dependency_list=None + self, + packs, + abs_repo_base, + verifyssl=True, + force=False, + dependency_list=None, + checkout_submodules=False, ): result = {} pack_url = None @@ -86,6 +92,7 @@ def run( proxy_config=self.proxy_config, force_permissions=True, logger=self.logger, + checkout_submodules=checkout_submodules, ) pack_url, pack_ref, pack_result = pack_result result[pack_ref] = pack_result @@ -99,6 +106,7 @@ def run( proxy_config=self.proxy_config, force_permissions=True, logger=self.logger, + checkout_submodules=checkout_submodules, ) pack_url, pack_ref, pack_result = pack_result result[pack_ref] = pack_result diff --git a/contrib/packs/actions/workflows/install.yaml b/contrib/packs/actions/workflows/install.yaml index a67a754146..013788517a 100644 --- a/contrib/packs/actions/workflows/install.yaml +++ b/contrib/packs/actions/workflows/install.yaml @@ -9,6 +9,7 @@ input: - force - skip_dependencies - timeout + - checkout_submodules vars: - packs_list: null @@ -30,6 +31,7 @@ tasks: packs: <% ctx().packs %> force: <% ctx().force %> dependency_list: <% ctx().dependency_list %> + checkout_submodules: <% ctx().checkout_submodules %> next: - when: <% succeeded() %> do: make_a_prerun diff --git a/st2api/st2api/controllers/v1/packs.py b/st2api/st2api/controllers/v1/packs.py index 0642c2f90e..3cb0461cc4 100644 --- a/st2api/st2api/controllers/v1/packs.py +++ b/st2api/st2api/controllers/v1/packs.py @@ -103,6 +103,9 @@ def post(self, pack_install_request, requester_user=None): if pack_install_request.skip_dependencies: parameters["skip_dependencies"] = True + if pack_install_request.checkout_submodules: + parameters["checkout_submodules"] = True + if not requester_user: requester_user = UserDB(name=cfg.CONF.system_user.user) diff --git a/st2api/tests/unit/controllers/v1/test_packs.py b/st2api/tests/unit/controllers/v1/test_packs.py index a6771c6c92..43079f72c8 100644 --- a/st2api/tests/unit/controllers/v1/test_packs.py +++ b/st2api/tests/unit/controllers/v1/test_packs.py @@ -210,6 +210,18 @@ def test_install_with_skip_dependencies_parameter(self, _handle_schedule_executi self.assertEqual(resp.status_int, 202) self.assertEqual(resp.json, {"execution_id": "123"}) + @mock.patch.object(ActionExecutionsControllerMixin, "_handle_schedule_execution") + def test_install_with_checkout_submodules_parameter( + self, _handle_schedule_execution + ): + _handle_schedule_execution.return_value = Response(json={"id": "123"}) + payload = {"packs": ["some"], "checkout_submodules": True} + + resp = self.app.post_json("/v1/packs/install", payload) + + self.assertEqual(resp.status_int, 202) + self.assertEqual(resp.json, {"execution_id": "123"}) + @mock.patch.object(ActionExecutionsControllerMixin, "_handle_schedule_execution") def test_uninstall(self, _handle_schedule_execution): _handle_schedule_execution.return_value = Response(json={"id": "123"}) diff --git a/st2client/st2client/commands/pack.py b/st2client/st2client/commands/pack.py index 4fa2d06647..3e63f5a6b5 100644 --- a/st2client/st2client/commands/pack.py +++ b/st2client/st2client/commands/pack.py @@ -266,6 +266,12 @@ def __init__(self, resource, *args, **kwargs): default=False, help="Skip pack dependency installation.", ) + self.parser.add_argument( + "--checkout-submodules", + action="store_true", + default=False, + help="Checkout git submodules present in the pack.", + ) def run(self, args, **kwargs): is_structured_output = args.json or args.yaml @@ -279,6 +285,7 @@ def run(self, args, **kwargs): args.packs, force=args.force, skip_dependencies=args.skip_dependencies, + checkout_submodules=args.checkout_submodules, **kwargs, ) diff --git a/st2client/st2client/models/core.py b/st2client/st2client/models/core.py index e66e0e7800..2f117b6bb8 100644 --- a/st2client/st2client/models/core.py +++ b/st2client/st2client/models/core.py @@ -628,12 +628,20 @@ class AsyncRequest(Resource): class PackResourceManager(ResourceManager): @add_auth_token_to_kwargs_from_env - def install(self, packs, force=False, skip_dependencies=False, **kwargs): + def install( + self, + packs, + force=False, + skip_dependencies=False, + checkout_submodules=False, + **kwargs, + ): url = "/%s/install" % (self.resource.get_url_path_name()) payload = { "packs": packs, "force": force, "skip_dependencies": skip_dependencies, + "checkout_submodules": checkout_submodules, } response = self.client.post(url, payload, **kwargs) if response.status_code != http_client.OK: diff --git a/st2common/st2common/models/api/pack.py b/st2common/st2common/models/api/pack.py index 8c31718b0e..4657b6f2e1 100644 --- a/st2common/st2common/models/api/pack.py +++ b/st2common/st2common/models/api/pack.py @@ -394,6 +394,11 @@ class PackInstallRequestAPI(BaseAPI): "description": "Set to True to skip pack dependency installations.", "default": False, }, + "checkout_submodules": { + "type": "boolean", + "description": "Checkout git submodules present in the pack.", + "default": False, + }, }, } diff --git a/st2common/st2common/openapi.yaml b/st2common/st2common/openapi.yaml index e86e42727d..0df6090621 100644 --- a/st2common/st2common/openapi.yaml +++ b/st2common/st2common/openapi.yaml @@ -5173,6 +5173,10 @@ definitions: type: boolean description: Set to True to skip pack dependency installations. default: false + checkout_submodules: + type: boolean + description: Set to True to checkout git submodules present in the pack. + default: false required: - packs PacksUninstall: diff --git a/st2common/st2common/openapi.yaml.j2 b/st2common/st2common/openapi.yaml.j2 index f053f0f3d0..48b2b2c857 100644 --- a/st2common/st2common/openapi.yaml.j2 +++ b/st2common/st2common/openapi.yaml.j2 @@ -5169,6 +5169,10 @@ definitions: type: boolean description: Set to True to skip pack dependency installations. default: false + checkout_submodules: + type: boolean + description: Set to True to checkout git submodules present in the pack. + default: false required: - packs PacksUninstall: diff --git a/st2common/st2common/util/pack_management.py b/st2common/st2common/util/pack_management.py index ae3c256184..46e91acf41 100644 --- a/st2common/st2common/util/pack_management.py +++ b/st2common/st2common/util/pack_management.py @@ -79,6 +79,7 @@ def download_pack( force_owner_group=True, force_permissions=True, logger=LOG, + checkout_submodules=False, ): """ Download the pack and move it to /opt/stackstorm/packs. @@ -99,6 +100,10 @@ def download_pack( :param force: Force the installation and ignore / delete the lock file if it already exists. :type force: ``bool`` + :param checkout_submodules: Whether to also checkout git submodules present + in the pack + :type checkout_submodules: ``bool`` + :return: (pack_url, pack_ref, result) :rtype: tuple """ @@ -163,6 +168,7 @@ def download_pack( repo_url=pack_url, verify_ssl=verify_ssl, ref=pack_version, + checkout_submodules=checkout_submodules, ) pack_metadata = get_pack_metadata(pack_dir=abs_local_path) @@ -190,7 +196,9 @@ def download_pack( return tuple(result) -def clone_repo(temp_dir, repo_url, verify_ssl=True, ref="master"): +def clone_repo( + temp_dir, repo_url, verify_ssl=True, ref="master", checkout_submodules=False +): # Switch to non-interactive mode os.environ["GIT_TERMINAL_PROMPT"] = "0" os.environ["GIT_ASKPASS"] = "/bin/echo" @@ -204,6 +212,10 @@ def clone_repo(temp_dir, repo_url, verify_ssl=True, ref="master"): # future. repo = Repo.clone_from(repo_url, temp_dir) + # Checkout any Git Submodules if requested. + if checkout_submodules: + repo.submodule_update(recursive=False) + is_local_repo = repo_url.startswith("file://") try: