From c2f06e40df74acfa089b06a428d6af1ab9c07767 Mon Sep 17 00:00:00 2001 From: Zack Cerza Date: Thu, 31 Aug 2023 10:34:41 -0700 Subject: [PATCH 1/2] orchestra: Treat EOFError as SSHException Signed-off-by: Zack Cerza --- teuthology/orchestra/connection.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/teuthology/orchestra/connection.py b/teuthology/orchestra/connection.py index ebfee2fad..b5d39d824 100644 --- a/teuthology/orchestra/connection.py +++ b/teuthology/orchestra/connection.py @@ -107,7 +107,7 @@ def connect(user_at_host, host_key=None, keep_alive=False, timeout=60, break except paramiko.AuthenticationException as e: log.error(f"Error authenticating with {host}: {str(e)}") - except paramiko.SSHException: + except (paramiko.SSHException, EOFError): msg = f"Error authenticating with {host}" if not key_filename: log.error(msg + ": No SSH private key found!") From e07fee9c6ba99d62de57040accfe940dddba780e Mon Sep 17 00:00:00 2001 From: Zack Cerza Date: Thu, 31 Aug 2023 11:10:09 -0700 Subject: [PATCH 2/2] orchestra: Move connection exception handling ... to inside the retry loop. Also, add an increment to the safe_while instance we use. Signed-off-by: Zack Cerza --- teuthology/orchestra/connection.py | 32 +++++++++++++++--------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/teuthology/orchestra/connection.py b/teuthology/orchestra/connection.py index b5d39d824..8190fa507 100644 --- a/teuthology/orchestra/connection.py +++ b/teuthology/orchestra/connection.py @@ -96,23 +96,23 @@ def connect(user_at_host, host_key=None, keep_alive=False, timeout=60, log.debug(connect_args) - try: - if not retry: - ssh.connect(**connect_args) - else: - # Retries are implemented using safe_while - with safe_while(sleep=1, action='connect to ' + host) as proceed: - while proceed(): + if not retry: + ssh.connect(**connect_args) + else: + with safe_while(sleep=1, increment=3, action='connect to ' + host) as proceed: + while proceed(): + auth_err_msg = f"Error authenticating with {host}" + try: ssh.connect(**connect_args) break - except paramiko.AuthenticationException as e: - log.error(f"Error authenticating with {host}: {str(e)}") - except (paramiko.SSHException, EOFError): - msg = f"Error authenticating with {host}" - if not key_filename: - log.error(msg + ": No SSH private key found!") - raise - else: - log.exception(msg) + except EOFError: + log.error(f"{auth_err_msg}: EOFError") + except paramiko.AuthenticationException as e: + log.error(f"{auth_err_msg}: {repr(e)}") + except paramiko.SSHException as e: + auth_err_msg = f"{auth_err_msg}: {repr(e)}" + if not key_filename: + auth_err_msg = f"{auth_err_msg} (No SSH private key found!)" + log.exception(auth_err_msg) ssh.get_transport().set_keepalive(keep_alive) return ssh