diff --git a/.cspell.json b/.cspell.json index 5d20dddbc74..259d9440126 100644 --- a/.cspell.json +++ b/.cspell.json @@ -29,12 +29,15 @@ "KNXNX0kxdddddddoc", "Koning", "Ksection", + "NOTIF", "OXXXXKd", "Paren", "Prego", "Programmez", "QIDAQAB", + "REPOSITORYNAME", "RQASWB", + "SOURCEBRANCHNAME", "Ywarn", "abefhkmnptuvx", "admiralawkbar", diff --git a/megalinter/config.py b/megalinter/config.py index d891812eb20..f9eed734bd6 100644 --- a/megalinter/config.py +++ b/megalinter/config.py @@ -197,9 +197,10 @@ def get(request_id, config_var=None, default=None): val = "false" return val + def get_first_var_set(request_id, config_vars=[], default=None): for config_var in config_vars: - val = get(request_id,config_var, None) + val = get(request_id, config_var, None) if val is not None and val != "": if isinstance(val, bool): if val is True: @@ -209,6 +210,7 @@ def get_first_var_set(request_id, config_vars=[], default=None): return val return default + def set(request_id, config_var, value): global RUN_CONFIGS assert request_id in RUN_CONFIGS, "Config has not been initialized yet !" diff --git a/megalinter/reporters/ApiReporter.py b/megalinter/reporters/ApiReporter.py index 1e0f34b4c46..6e83d3fe646 100644 --- a/megalinter/reporters/ApiReporter.py +++ b/megalinter/reporters/ApiReporter.py @@ -34,12 +34,13 @@ def __init__(self, params=None): def manage_activation(self): if config.get(self.master.request_id, "API_REPORTER", "false") == "true": - if ( - config.exists(self.master.request_id, "API_REPORTER_URL") or - config.exists(self.master.request_id, "NOTIF_API_URL") - ): + if config.exists( + self.master.request_id, "API_REPORTER_URL" + ) or config.exists(self.master.request_id, "NOTIF_API_URL"): self.is_active = True - self.api_url = config.get_first_var_set(self.master.request_id, ["API_REPORTER_URL","NOTIF_API_URL"]) + self.api_url = config.get_first_var_set( + self.master.request_id, ["API_REPORTER_URL", "NOTIF_API_URL"] + ) else: logging.error("You need to define API_REPORTER_URL to use ApiReporter") @@ -52,20 +53,21 @@ def produce_report(self): # Call API self.send_to_api() # Handle Metrics API if requested - if ( - config.exists(self.master.request_id, "API_REPORTER_METRICS_URL") or - config.exists(self.master.request_id, "NOTIF_API_METRICS_URL") - ): - self.api_metrics_url = config.get_first_var_set(self.master.request_id, [ - "API_REPORTER_METRICS_URL","NOTIF_API_METRICS_URL" - ]) + if config.exists( + self.master.request_id, "API_REPORTER_METRICS_URL" + ) or config.exists(self.master.request_id, "NOTIF_API_METRICS_URL"): + self.api_metrics_url = config.get_first_var_set( + self.master.request_id, + ["API_REPORTER_METRICS_URL", "NOTIF_API_METRICS_URL"], + ) self.build_metrics_payload() self.send_to_metrics_api() - def build_payload(self): # Git info - repo_info = get_git_context_info(self.master.request_id, os.path.realpath(self.master.github_workspace)) + repo_info = get_git_context_info( + self.master.request_id, os.path.realpath(self.master.github_workspace) + ) git_identifier = f"{repo_info["repo_name"]}/{repo_info["branch_name"]}" org_identifier = self.get_org_identifier(repo_info["branch_name"]) self.payload = { @@ -92,7 +94,7 @@ def build_payload(self): } linter_payload_data = { "linterDocUrl": linter_doc_url, - "jobUrl": repo_info["job_url"] + "jobUrl": repo_info["job_url"], } # Status linter_payload_data["severity"] = ( @@ -128,17 +130,19 @@ def build_payload(self): linter_payload["data"] = linter_payload_data self.payload["linters"].append(linter_payload) - def get_org_identifier(self, branch_name: str): org_identifier = config.get( - self.master.request_id, "API_REPORTER_ORG_IDENTIFIER", None - ) + self.master.request_id, "API_REPORTER_ORG_IDENTIFIER", None + ) if org_identifier is not None: return org_identifier # Workaround for sfdx-hardis, but it's better to set ENV variable API_REPORTER_ORG_IDENTIFIER - return branch_name.replace("monitoring_","").replace( - "_","-").replace("__","--").replace("_sandbox","__sandbox") - + return ( + branch_name.replace("monitoring_", "") + .replace("_", "-") + .replace("__", "--") + .replace("_sandbox", "__sandbox") + ) def format_payload(self): if ( @@ -176,19 +180,29 @@ def send_to_api(self): "content-type": "application/json", } # Use username & password - if ( - config.exists(self.master.request_id, "API_REPORTER_BASIC_AUTH_USERNAME") or - config.exists(self.master.request_id, "NOTIF_API_BASIC_AUTH_USERNAME") - ): + if config.exists( + self.master.request_id, "API_REPORTER_BASIC_AUTH_USERNAME" + ) or config.exists(self.master.request_id, "NOTIF_API_BASIC_AUTH_USERNAME"): session.auth = ( - config.get_first_var_set(self.master.request_id, ["API_REPORTER_BASIC_AUTH_USERNAME","NOTIF_API_BASIC_AUTH_USERNAME"]), - config.get_first_var_set(self.master.request_id, ["API_REPORTER_BASIC_AUTH_PASSWORD","NOTIF_API_BASIC_AUTH_PASSWORD"]), + config.get_first_var_set( + self.master.request_id, + [ + "API_REPORTER_BASIC_AUTH_USERNAME", + "NOTIF_API_BASIC_AUTH_USERNAME", + ], + ), + config.get_first_var_set( + self.master.request_id, + [ + "API_REPORTER_BASIC_AUTH_PASSWORD", + "NOTIF_API_BASIC_AUTH_PASSWORD", + ], + ), ) # Use token - if ( - config.exists(self.master.request_id, "API_REPORTER_BEARER_TOKEN") or - config.exists(self.master.request_id, "NOTIF_API_BEARER_TOKEN") - ): + if config.exists( + self.master.request_id, "API_REPORTER_BEARER_TOKEN" + ) or config.exists(self.master.request_id, "NOTIF_API_BEARER_TOKEN"): headers["Authorization"] = ( f"Bearer {config.get_first_var_set(self.master.request_id, ["API_REPORTER_BEARER_TOKEN","NOTIF_API_BEARER_TOKEN"])}" ) @@ -200,7 +214,9 @@ def send_to_api(self): logging.info( f"[Api Reporter] Successfully posted data to {self.api_url}" ) - if config.get_first_var_set(self.master.request_id,[ "API_REPORTER_DEBUG","NOTIF_API_DEBUG"]): + if config.get_first_var_set( + self.master.request_id, ["API_REPORTER_DEBUG", "NOTIF_API_DEBUG"] + ): logging.info(json.dumps(obj=self.payloadFormatted, indent=True)) else: logging.warning( @@ -221,22 +237,23 @@ def send_to_api(self): # Build something like MetricName,source=sfdx-hardis,orgIdentifier=hardis-group metric=12.7,min=0,max=70,percent=0.63 def build_metrics_payload(self): - metric_base_tags= ( - f"source={self.payload.source}," + - f"orgIdentifier={self.payload.orgIdentifier},"+ - f"gitIdentifier={self.payload.gitIdentifier}," + metric_base_tags = ( + f"source={self.payload["source"]}," + + f"orgIdentifier={self.payload["orgIdentifier"]}," + + f"gitIdentifier={self.payload["gitIdentifier"]}," ) all_metrics_lines = [] for linter in self.master.linters: if linter.is_active is True: metric_id = linter.linter_name metric_line = ( - metric_id +","+ - metric_base_tags + - f"descriptor={linter.descriptor_id},"+ - f"linter={linter.linter_name},"+ - f"linterKey={linter.name}"+ - " " + metric_id + + "," + + metric_base_tags + + f"descriptor={linter.descriptor_id}," + + f"linter={linter.linter_name}," + + f"linterKey={linter.name}" + + " " ) metric_line += f"metric={linter.total_number_errors}" # Number of files & errors @@ -251,7 +268,6 @@ def build_metrics_payload(self): all_metrics_lines += [metric_line] self.metrics_payload = ",".join(all_metrics_lines) - def send_to_metrics_api(self): session = requests.Session() headers = { @@ -259,19 +275,31 @@ def send_to_metrics_api(self): "content-type": "application/json", } # Use username & password - if ( - config.exists(self.master.request_id, "API_REPORTER_METRICS_BASIC_AUTH_USERNAME") or - config.exists(self.master.request_id, "NOTIF_API_METRICS_BASIC_AUTH_USERNAME") + if config.exists( + self.master.request_id, "API_REPORTER_METRICS_BASIC_AUTH_USERNAME" + ) or config.exists( + self.master.request_id, "NOTIF_API_METRICS_BASIC_AUTH_USERNAME" ): session.auth = ( - config.get_first_var_set(self.master.request_id, ["API_REPORTER_METRICS_BASIC_AUTH_USERNAME","NOTIF_API_METRICS_BASIC_AUTH_USERNAME"]), - config.get_first_var_set(self.master.request_id, ["API_REPORTER_METRICS_BASIC_AUTH_PASSWORD","NOTIF_API_METRICS_BASIC_AUTH_PASSWORD"]), + config.get_first_var_set( + self.master.request_id, + [ + "API_REPORTER_METRICS_BASIC_AUTH_USERNAME", + "NOTIF_API_METRICS_BASIC_AUTH_USERNAME", + ], + ), + config.get_first_var_set( + self.master.request_id, + [ + "API_REPORTER_METRICS_BASIC_AUTH_PASSWORD", + "NOTIF_API_METRICS_BASIC_AUTH_PASSWORD", + ], + ), ) # Use token - if ( - config.exists(self.master.request_id, "API_REPORTER_METRICS_BEARER_TOKEN") or - config.exists(self.master.request_id, "NOTIF_API_METRICS_BEARER_TOKEN") - ): + if config.exists( + self.master.request_id, "API_REPORTER_METRICS_BEARER_TOKEN" + ) or config.exists(self.master.request_id, "NOTIF_API_METRICS_BEARER_TOKEN"): headers["Authorization"] = ( f"Bearer {config.get_first_var_set(self.master.request_id, ["API_REPORTER_METRICS_BEARER_TOKEN","NOTIF_API_METRICS_BEARER_TOKEN"])}" ) @@ -283,7 +311,9 @@ def send_to_metrics_api(self): logging.info( f"[Api Reporter Metrics] Successfully posted data to {self.api_url}" ) - if config.get_first_var_set(self.master.request_id,[ "API_REPORTER_DEBUG","NOTIF_API_DEBUG"]): + if config.get_first_var_set( + self.master.request_id, ["API_REPORTER_DEBUG", "NOTIF_API_DEBUG"] + ): logging.info(json.dumps(obj=self.payloadFormatted, indent=True)) else: logging.warning( @@ -300,4 +330,4 @@ def send_to_metrics_api(self): logging.warning( f"[Api Reporter Metrics] Error posting data to {self.api_url}:" f"Connection error {str(e)}" - ) \ No newline at end of file + ) diff --git a/megalinter/tests/test_megalinter/config_test.py b/megalinter/tests/test_megalinter/config_test.py index c5d0fc402b2..ba256219f05 100644 --- a/megalinter/tests/test_megalinter/config_test.py +++ b/megalinter/tests/test_megalinter/config_test.py @@ -433,28 +433,28 @@ def test_get_first_var_set(self): config.init_config( request_id, None, - { - "NOTIF_API_REPORTER_URL": "test", - "TEST_VAR": "test2" - }, + {"NOTIF_API_REPORTER_URL": "test", "TEST_VAR": "test2"}, + ) + res = config.get_first_var_set( + request_id, ["API_REPORTER_URL", "NOTIF_API_REPORTER_URL"], None ) - res = config.get_first_var_set(request_id,["API_REPORTER_URL","NOTIF_API_REPORTER_URL"], None) self.assertTrue( res == "test", "get_first_var_set is failing", ) - res2 = config.get_first_var_set(request_id,["NOPE","NOT_HERE"], "default_value") + res2 = config.get_first_var_set( + request_id, ["NOPE", "NOT_HERE"], "default_value" + ) self.assertTrue( res2 == "default_value", "get_first_var_set is failing with default value", ) - res3 = config.get_first_var_set(request_id,["TEST_VAR","TEST_VAR_2"], None) + res3 = config.get_first_var_set(request_id, ["TEST_VAR", "TEST_VAR_2"], None) self.assertTrue( res3 == "test2", "get_first_var_set is failing", ) - def replace_branch_in_input_files(self): root = ( ".automation" diff --git a/megalinter/utils.py b/megalinter/utils.py index 24f49cd2288..f46467c7b80 100644 --- a/megalinter/utils.py +++ b/megalinter/utils.py @@ -316,66 +316,66 @@ def is_git_repo(path): except git.InvalidGitRepositoryError: return False + def get_git_context_info(request_id, path): - # Repo name - repo_name = config.get_first_var_set( - request_id, - [ - "GITHUB_REPOSITORY", - "GIT_URL", - "CI_PROJECT_NAME", - "BITBUCKET_REPO_SLUG", - "BUILD_REPOSITORYNAME" - ], - None) - if repo_name is not None: - repo_name = repo_name.split("/")[-1] #Get last portion - # Branch name - branch_name = config.get_first_var_set( - request_id, - [ - "GITHUB_HEAD_REF", - "GITHUB_REF_NAME", - "GIT_BRANCH", - "CI_COMMIT_REF_NAME", - "BITBUCKET_BRANCH", - "BUILD_SOURCEBRANCHNAME" - ], - None) - if repo_name is None: - try: - repo = git.Repo( - path, - search_parent_directories=True, - ) - repo_name = repo.working_tree_dir.split("/")[-1] - except Exception as e: - repo_name = "?" - if branch_name is None: - try: - repo = git.Repo( - path, - search_parent_directories=True, - ) - repo_name_1 = repo.working_tree_dir.split("/")[-1] - branch = repo_name_1.active_branch - branch_name = branch.name - except Exception as e: - branch_name = "?" - # Job URL - job_url = config.get_first_var_set( - request_id, - [ - "GITHUB_JOB_URL", - "CI_JOB_URL" - # TODO: Handle Azure, BitBucket & Jenkins - ], - "") - return { - "repo_name": repo_name, - "branch_name": branch_name, - "job_url": job_url - } + # Repo name + repo_name = config.get_first_var_set( + request_id, + [ + "GITHUB_REPOSITORY", + "GIT_URL", + "CI_PROJECT_NAME", + "BITBUCKET_REPO_SLUG", + "BUILD_REPOSITORYNAME", + ], + None, + ) + if repo_name is not None: + repo_name = repo_name.split("/")[-1] # Get last portion + # Branch name + branch_name = config.get_first_var_set( + request_id, + [ + "GITHUB_HEAD_REF", + "GITHUB_REF_NAME", + "GIT_BRANCH", + "CI_COMMIT_REF_NAME", + "BITBUCKET_BRANCH", + "BUILD_SOURCEBRANCHNAME", + ], + None, + ) + if repo_name is None: + try: + repo = git.Repo( + path, + search_parent_directories=True, + ) + repo_name = repo.working_tree_dir.split("/")[-1] + except Exception as e: + repo_name = "?" + if branch_name is None: + try: + repo = git.Repo( + path, + search_parent_directories=True, + ) + repo_name_1 = repo.working_tree_dir.split("/")[-1] + branch = repo_name_1.active_branch + branch_name = branch.name + except Exception as e: + branch_name = "?" + # Job URL + job_url = config.get_first_var_set( + request_id, + [ + "GITHUB_JOB_URL", + "CI_JOB_URL", + # TODO: Handle Azure, BitBucket & Jenkins + ], + "", + ) + return {"repo_name": repo_name, "branch_name": branch_name, "job_url": job_url} def check_updated_file(file, repo_home, changed_files=None):