From 406d8f9e4ec403d8fbceb024a9d1a1437116d91a Mon Sep 17 00:00:00 2001 From: Dag Wieers Date: Sun, 7 Oct 2018 20:25:10 +0200 Subject: [PATCH] Implement urllib3 native retries --- winrm/protocol.py | 14 +++++++------- winrm/transport.py | 35 +++++++++++++++-------------------- 2 files changed, 22 insertions(+), 27 deletions(-) diff --git a/winrm/protocol.py b/winrm/protocol.py index 4f5a4449..7570459a 100644 --- a/winrm/protocol.py +++ b/winrm/protocol.py @@ -26,8 +26,8 @@ class Protocol(object): DEFAULT_OPERATION_TIMEOUT_SEC = 20 DEFAULT_MAX_ENV_SIZE = 153600 DEFAULT_LOCALE = 'en-US' - DEFAULT_RECONNECTION_RETRIES = 5 - DEFAULT_RECONNECTION_SLEEP = 5 + DEFAULT_RECONNECTION_RETRIES = 4 + DEFAULT_RECONNECTION_BACKOFF = 2.0 def __init__( self, endpoint, transport='plaintext', username=None, @@ -42,7 +42,7 @@ def __init__( credssp_disable_tlsv1_2=False, send_cbt=True, reconnection_retries = DEFAULT_RECONNECTION_RETRIES, - reconnection_sleep = DEFAULT_RECONNECTION_SLEEP, + reconnection_backoff = DEFAULT_RECONNECTION_BACKOFF, ): """ @param string endpoint: the WinRM webservice endpoint @@ -61,8 +61,8 @@ def __init__( @param int operation_timeout_sec: maximum allowed time in seconds for any single wsman HTTP operation (default 20). Note that operation timeouts while receiving output (the only wsman operation that should take any significant time, and where these timeouts are expected) will be silently retried indefinitely. # NOQA @param string kerberos_hostname_override: the hostname to use for the kerberos exchange (defaults to the hostname in the endpoint URL) @param bool message_encryption_enabled: Will encrypt the WinRM messages if set to True and the transport auth supports message encryption (Default True). - @param int reconnection_retries: Number of retries on Connection Refused - @param int reconnection_sleep: Number of seconds to sleep between reconnection attempts + @param int reconnection_retries: Number of retries on connection problems + @param float reconnection_backoff: Number of seconds to backoff in between reconnection attempts (first sleeps X, then sleeps 2*X, then sleeps 4*X, ...) """ try: @@ -96,7 +96,7 @@ def __init__( credssp_disable_tlsv1_2=credssp_disable_tlsv1_2, send_cbt=send_cbt, reconnection_retries=reconnection_retries, - reconnection_sleep=reconnection_sleep, + reconnection_backoff=reconnection_backoff, ) self.username = username @@ -109,7 +109,7 @@ def __init__( self.kerberos_hostname_override = kerberos_hostname_override self.credssp_disable_tlsv1_2 = credssp_disable_tlsv1_2 self.reconnection_retries = reconnection_retries - self.reconnection_sleep = reconnection_sleep + self.reconnection_backoff = reconnection_backoff def open_shell(self, i_stream='stdin', o_stream='stdout stderr', working_directory=None, env_vars=None, noprofile=False, diff --git a/winrm/transport.py b/winrm/transport.py index b67157f5..cc6195e2 100644 --- a/winrm/transport.py +++ b/winrm/transport.py @@ -1,8 +1,7 @@ from __future__ import unicode_literals -import inspect -import os import sys -import time +import os +import inspect is_py2 = sys.version[0] == '2' @@ -64,8 +63,8 @@ def __init__( message_encryption='auto', credssp_disable_tlsv1_2=False, send_cbt=True, - reconnection_retries=5, - reconnection_sleep=5, + reconnection_retries=4, + reconnection_backoff=2.0, ): self.endpoint = endpoint self.username = username @@ -83,7 +82,7 @@ def __init__( self.credssp_disable_tlsv1_2 = credssp_disable_tlsv1_2 self.send_cbt = send_cbt self.reconnection_retries = reconnection_retries - self.reconnection_sleep = reconnection_sleep + self.reconnection_backoff = reconnection_backoff if self.server_cert_validation not in [None, 'validate', 'ignore']: raise WinRMError('invalid server_cert_validation mode: %s' % self.server_cert_validation) @@ -158,6 +157,15 @@ def build_session(self): settings = session.merge_environment_settings(url=self.endpoint, proxies={}, stream=None, verify=None, cert=None) + # Retry on connection errors, with a backoff factor + retries = requests.packages.urllib3.util.retry.Retry(total=self.reconnection_retries, + connect=self.reconnection_retries, + read=self.reconnection_retries, + backoff_factor=self.reconnection_backoff, + status_forcelist=[425, 429, 503]) + session.mount('http://', requests.adapters.HTTPAdapter(max_retries=retries)) + session.mount('https://', requests.adapters.HTTPAdapter(max_retries=retries)) + # get proxy settings from env # FUTURE: allow proxy to be passed in directly to supersede this value session.proxies = settings['proxies'] @@ -266,20 +274,7 @@ def send_message(self, message): def _send_message_request(self, prepared_request, message): try: - - # Retry connection on 'Connection refused' - for attempt in range(self.reconnection_retries): - try: - response = self.session.send(prepared_request, timeout=self.read_timeout_sec) - except requests.packages.urllib3.exceptions.NewConnectionError as e: - time.sleep(self.reconnection_sleep) -# except requests.exceptions.ConnectionError as e: -# if attempt == 4 or 'connection refused' not in str(e).lower(): -# raise -# time.sleep(self.reconnection_sleep) - else: - break - + response = self.session.send(prepared_request, timeout=self.read_timeout_sec) response.raise_for_status() return response except requests.HTTPError as ex: