From 15527694c0174518d46fd490f11e48c003b84849 Mon Sep 17 00:00:00 2001 From: Stelios Voutsinas Date: Thu, 12 Oct 2023 19:29:11 +0300 Subject: [PATCH 1/3] Fix lint issues --- gdmp_benchmark/gdmp_benchmark.py | 199 +++++++------------------------ 1 file changed, 40 insertions(+), 159 deletions(-) diff --git a/gdmp_benchmark/gdmp_benchmark.py b/gdmp_benchmark/gdmp_benchmark.py index 227016e..740f892 100644 --- a/gdmp_benchmark/gdmp_benchmark.py +++ b/gdmp_benchmark/gdmp_benchmark.py @@ -1,9 +1,9 @@ """ Module that can be used to run benchmarks against an instance of the Gaia Data Mining Platform - """ # pylint: disable-msg=too-many-locals # pylint: disable-msg=too-many-arguments +# pylint: disable-msg=bad-continuation import sys import subprocess @@ -25,9 +25,7 @@ class InvalidConfigurationError(Exception): - """ - Exception to be raised when the configuration passed in is invalid - """ + """Exception to be raised when the configuration passed in is invalid""" __module__ = Exception.__module__ @@ -40,7 +38,6 @@ def validate_not_empty(val): """ Args: val: The value to validate - Raises: ValueError: If value is empty """ @@ -54,7 +51,6 @@ def validate_positive(val): """ Args: val: The value to validate - Raises: ValueError: If value is not positive TypeError: If value is not an int @@ -73,7 +69,6 @@ def validate(instance): Validate the parameter types of a class instance Args: instance: The instance of a class - Raises: TypeError: If type does not match Type hint """ @@ -85,9 +80,7 @@ def validate(instance): class Status(str, Enum): - """ - Status of the timing for a notebook run - """ + """Status of the timing for a notebook run""" SLOW = "SLOW" FAST = "FAST" @@ -103,11 +96,7 @@ def __repr__(self): return self.value def to_json(self): - """ - Format as json - Returns: - str - """ + """Return as json""" return self.value @@ -115,7 +104,6 @@ def to_json(self): class Timing: """ Stores the Timing info of Notebook run. - Attributes: result (Status): The result of the operation. totaltime (int): The total execution time @@ -140,7 +128,6 @@ def percent_change(self) -> str: Get percentage change (Total time - expected time) / expected_time Returns: str: Percentage change - """ if self.expected and self.totaltime > 0: return f"{((self.totaltime - self.expected) / self.expected) * 100:.2f}" @@ -169,11 +156,7 @@ def __repr__(self): ) def to_json(self): - """ - Format as json - Returns: - dict - """ + """Return as json""" return { "result": self.result, "elapsed": f"{self.totaltime:.2f}", @@ -183,11 +166,7 @@ def to_json(self): } def to_dict(self): - """ - To Dict method - Returns: - dict: The dict - """ + """Return as dictionary""" dict_view = self.__dict__ dict_view["percent_change"] = self.percent_change return dict_view @@ -225,19 +204,11 @@ class Results: name: str = "" def __post_init__(self): - """ - post_init method - Returns: - None - """ + """post_init method""" validate(self) def to_dict(self): - """ - To Dictionary - Returns: - dict: The dictionary - """ + """Return as dictionary""" return { "result": self.result, "msg": self.msg, @@ -263,25 +234,8 @@ def __str__(self): } ) - def __repr__(self): - return json.dumps( - { - "name": self.name, - "result": str(self.result), - "outputs": self.outputs, - "messages": self.messages, - "time": self.time.to_json(), - "logs": self.logs, - }, - indent=4, - ) - def to_json(self) -> str: - """ - Convert to JSON - Returns: - str - """ + """Convert to JSON""" return json.dumps( { @@ -296,12 +250,11 @@ def to_json(self) -> str: ) -class Alertable(Protocol): - def send_alert(self, message: Results, only_on_error: bool = True) -> None: - ... - - class AlertStrategies(Enum): + """ + Alerting Strategies + """ + ONLY_ON_ERROR = "ONLY_ON_ERROR" SLOW_AND_ERROR = "SLOW_AND_ERROR" ALLWAYS = "ALLWAYS" @@ -321,11 +274,16 @@ class AlertStrategies(Enum): class SlackAlerter: + """Slack Alerter Class, used to send out alerts to slack channels""" + def __init__(self, webhook): self.webhook = webhook @staticmethod def send_http(msg: str, url: str): + """ + Send an http request with msg as a 'text' parameter + """ # Send the HTTP POST request to the webhook URL response = requests.post( url, json={"text": msg}, headers={"Content-Type": "application/json"} @@ -336,12 +294,13 @@ def send_http(msg: str, url: str): logging.info("Message sent successfully to Slack!") else: logging.warning( - f"Failed to send message. Status code: {response.status_code}" + "Failed to send message. Status code: %s", response.status_code ) def send_alert( self, content: list, alert_strategy: AlertStrategies.SLOW_AND_ERROR ) -> None: + """Send an alert""" for test in content: if test.result in ALERT_STRATEGIES_MAP.get( alert_strategy, True @@ -351,6 +310,7 @@ def send_alert( @staticmethod def format_message(content: Results) -> str: + """Format a message""" return json.dumps(content, default=str) @@ -380,22 +340,11 @@ def __post_init__(self): class NotebookHandler(Protocol): - """ - Protocol for a Notebook handling Class - """ + """Protocol for a Notebook handling Class""" @staticmethod def create_notebook(config: str, filepath: str, messages: list) -> str: - """ - Create a Notebook - Args: - config: - filepath: - messages: - - Returns: - str: - """ + """Create a notebook""" # pylint: disable=W0107 pass @@ -403,43 +352,19 @@ def create_notebook(config: str, filepath: str, messages: list) -> str: def execute_notebook( config: str, notebookid: str, filepath: str, messages: list ) -> tuple: - """ - Execute a notebook - - Args: - config (str): The configuration for the user - notebookid (str): The notebook ID - filepath (str): The path for the notebook to create - messages (list): The list of messages to append to - - Returns: - output: A list of output, each element being a single cell output - msg: Result message - status: Status message - """ + """Execute a notebook""" # pylint: disable=W0107 pass @staticmethod def print_notebook(notebookid: str, config: str) -> dict: - """ - Print notebook - - Args: - notebookid: ID of the notebook - config: User configuration file - - Returns: - dict: JSON dictionary of notebook - """ + """Print a notebook""" # pylint: disable=W0107 pass @staticmethod def delete_notebook(notebookid: str, config: str) -> None: """ - Delete notebook - Args: notebookid (str): The ID of the notebook to delete config (str): The configuration for the user @@ -453,20 +378,16 @@ def delete_notebook(notebookid: str, config: str) -> None: class ZDairiNotebookHandler: """ - Implementation of the Notebook Handler Protocol - Provides methods for creating, executing, printing - and deleting notebooks + Implementation of the Notebook Handler Protocol.cProvides methods for creating, executing, + printing and deleting notebooks """ @staticmethod def delete_notebook(notebookid: str, config: str) -> None: """ - Delete notebook - Args: notebookid (str): The ID of the notebook to delete config (str): The configuration for the user - Returns: None """ @@ -482,13 +403,10 @@ def delete_notebook(notebookid: str, config: str) -> None: @staticmethod def create_notebook(config: str, filepath: str, messages: list) -> str: """ - Create a notebook - Args: config (str): The configuration for the user filepath (str): The path for the notebook to create messages (list): The list of messages to append to - Returns: notebookid: The ID for the new notebook """ @@ -526,7 +444,6 @@ def print_notebook(notebookid: str, config: str) -> dict: Args: notebookid: ID of the notebook config: User configuration file - Returns: dict: JSON dictionary of notebook """ @@ -554,7 +471,6 @@ def execute_notebook( notebookid (str): The notebook ID filepath (str): The path for the notebook to create messages (list): The list of messages to append to - Returns: output: A list of output, each element being a single cell output msg: Result message @@ -605,9 +521,7 @@ def execute_notebook( class GDMPBenchmarker: - """ - Class used to run benchmarks for the Gaia Data Mining platform - """ + """Class used to run benchmarks for the Gaia Data Mining platform""" DEFAULT_DIR = "/tmp/" DEFAULT_USER_CONFIG = "user1.yml" @@ -629,11 +543,7 @@ def __init__( @staticmethod def get_note(path: str) -> Dict[str, str]: - """ - Get the json file given a path (URL or file), and return as a json object - :type path: str - :rtype: dict - """ + """Get the json file given a path (URL or file), and return as a json object""" if path.startswith("http"): res = requests.get(path, timeout=30).text data = json.loads(res, strict=False) @@ -646,7 +556,6 @@ def generate_zdairi_user_configs(self) -> int: """ Generate the user configurations that can be used by the zdairi lib These need to be yml files with the following format: - zeppelin_url: zeppelin_auth: zeppelin_user: @@ -654,7 +563,6 @@ def generate_zdairi_user_configs(self) -> int: Returns: int: Number of users - Raises: InvalidConfigurationError: If User configuration is not a valid Json file """ @@ -690,10 +598,8 @@ def generate_zdairi_user_configs(self) -> int: def _get_user_config(self, concurrent: bool) -> str: """ - Args: concurrent (bool): Whether this is a concurrent test - Returns: config (str): The user config path """ @@ -715,9 +621,6 @@ def _write_data_to_file(data: dict, filepath: str) -> None: Args: data: Dictionary of the data filepath: The file to write to - - Returns: - None """ data["name"] = filepath with open(filepath, "w+", encoding="utf-8") as cred: @@ -734,7 +637,6 @@ def run_notebook( filepath: String with the filepath name: Name of the notebooks concurrent: Whether the notebook is part of a concurrent run - Returns: Results: The results """ @@ -795,7 +697,6 @@ def run( delay_start: Number of seconds to delay start of test delay_notebook: Number of seconds to delay each notebook delete: Whether to delete the notebooks after the test - Returns: List of Results """ @@ -805,7 +706,6 @@ def parse_notebook_config(note_config: str): Parse the notebook configuration Args: note_config: Notebook config as string - Returns: list: Notebook list """ @@ -848,10 +748,8 @@ def _run_parallel( delay_start: Delay start in seconds delay_notebook: Delay to start of notebook in seconds delete: Whether to delete the notebooks after the run - Returns: dict: The results - Raises: ValueError: If User count exceeds maximum """ @@ -891,7 +789,6 @@ def _validate_output(actual, expected, cell_number): bool: Whether the output is valid Status: Result Status str: Output message - """ out_valid = True result_status_msg = Status.PASS @@ -926,7 +823,6 @@ def _run_single( delay_start: Delay ot the start of the run in seconds delay_notebook: Delay to the start of the notebook in seconds delete: Whether to delete the notebooks after the run - Returns: dict: The results """ @@ -983,42 +879,27 @@ def _run_single( def main(args: List[str] = None): - """ - Main Function - Args: - args: Arguments - - """ + """Main method""" user_config_docs = """The user configuration file in JSON format. - { - "users": [ - { + { "users": [{ "username": "user1", "shirouser": { "name": "user1", "password": "pass1" - } - - } - ] + } + }] } """ notebook_config_docs = """The notebook configuration file in JSON format. - { - "notebooks" : [ - { - "name" : "GaiaDMPSetup", - "filepath" : "/path/GaiaDMP_validation.json", - "totaltime" : 50, - "results" : [] - } - - - ] + { "notebooks" : [{ + "name" : "GaiaDMPSetup", + "filepath" : "/path/GaiaDMP_validation.json", + "totaltime" : 50, + "results" : [] + }] } -""" - + """ parser = argparse.ArgumentParser( description="Gaia Data Mining Platform Benchmarking Tool" ) From a66aba8049fc27a2fc17909a70cc73ff48e84a0d Mon Sep 17 00:00:00 2001 From: Stelios Voutsinas Date: Thu, 12 Oct 2023 19:31:55 +0300 Subject: [PATCH 2/3] Fix lint issues 2 --- gdmp_benchmark/gdmp_benchmark.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/gdmp_benchmark/gdmp_benchmark.py b/gdmp_benchmark/gdmp_benchmark.py index 740f892..f4b2723 100644 --- a/gdmp_benchmark/gdmp_benchmark.py +++ b/gdmp_benchmark/gdmp_benchmark.py @@ -3,7 +3,6 @@ """ # pylint: disable-msg=too-many-locals # pylint: disable-msg=too-many-arguments -# pylint: disable-msg=bad-continuation import sys import subprocess @@ -286,7 +285,10 @@ def send_http(msg: str, url: str): """ # Send the HTTP POST request to the webhook URL response = requests.post( - url, json={"text": msg}, headers={"Content-Type": "application/json"} + url, + json={"text": msg}, + headers={"Content-Type": "application/json"}, + timeout=300, ) # Check the response status From 864e1e44c5e3c9773606ef533225f5e03500eb95 Mon Sep 17 00:00:00 2001 From: Stelios Voutsinas Date: Thu, 12 Oct 2023 19:33:31 +0300 Subject: [PATCH 3/3] Fix lint issues 3 --- gdmp_benchmark/gdmp_benchmark.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/gdmp_benchmark/gdmp_benchmark.py b/gdmp_benchmark/gdmp_benchmark.py index f4b2723..8725f5c 100644 --- a/gdmp_benchmark/gdmp_benchmark.py +++ b/gdmp_benchmark/gdmp_benchmark.py @@ -280,10 +280,7 @@ def __init__(self, webhook): @staticmethod def send_http(msg: str, url: str): - """ - Send an http request with msg as a 'text' parameter - """ - # Send the HTTP POST request to the webhook URL + """Send an http request with msg as a 'text' parameter""" response = requests.post( url, json={"text": msg}, @@ -291,7 +288,6 @@ def send_http(msg: str, url: str): timeout=300, ) - # Check the response status if response.status_code == 200: logging.info("Message sent successfully to Slack!") else: