From 47826ce1a720f872d07b01323f170be8d1463fd3 Mon Sep 17 00:00:00 2001 From: Jon Mease Date: Fri, 31 Aug 2018 08:19:38 -0400 Subject: [PATCH 1/3] Only retry on request errors with 5XX status code Also, switch to exponential backoff to give the servers more breathing room. Tries in seconds: 0, 1, 2, 4, 8, 16, 16, 16... for up to 3 minutes total. --- plotly/api/v2/utils.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/plotly/api/v2/utils.py b/plotly/api/v2/utils.py index cbd1efaa8b1..62678143db2 100644 --- a/plotly/api/v2/utils.py +++ b/plotly/api/v2/utils.py @@ -111,8 +111,18 @@ def get_headers(): return headers -@retry(wait_random_min=100, wait_random_max=1000, wait_exponential_max=10000, - stop_max_delay=120000) +def should_retry(exception): + if isinstance(exception, exceptions.PlotlyRequestError): + if (isinstance(exception.status_code, int) and + 500 <= exception.status_code < 600): + # Retry on 5XX errors. + return True + + return False + + +@retry(wait_exponential_multiplier=1000, wait_exponential_max=16000, + stop_max_delay=180000, retry_on_exception=should_retry) def request(method, url, **kwargs): """ Central place to make any api v2 api request. From 6b411ac449a5a0bda1c3929ce229b20dca5e48da Mon Sep 17 00:00:00 2001 From: Jon Mease Date: Fri, 31 Aug 2018 08:38:30 -0400 Subject: [PATCH 2/3] Reconnect stream write/hearbeat on 502 errors --- plotly/plotly/plotly.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plotly/plotly/plotly.py b/plotly/plotly/plotly.py index b41b2430223..d425f07d7ee 100644 --- a/plotly/plotly/plotly.py +++ b/plotly/plotly/plotly.py @@ -578,7 +578,7 @@ def get_streaming_specs(self): return streaming_specs - def heartbeat(self, reconnect_on=(200, '', 408)): + def heartbeat(self, reconnect_on=(200, '', 408, 502)): """ Keep stream alive. Streams will close after ~1 min of inactivity. @@ -616,7 +616,7 @@ def open(self): self._stream = chunked_requests.Stream(**streaming_specs) def write(self, trace, layout=None, - reconnect_on=(200, '', 408)): + reconnect_on=(200, '', 408, 502)): """ Write to an open stream. From 5678c24156454bd188589a5fe9bc22cb55d2156f Mon Sep 17 00:00:00 2001 From: Jon Mease Date: Fri, 31 Aug 2018 08:42:39 -0400 Subject: [PATCH 3/3] Added retrying logic to v1 api request function --- plotly/api/v1/utils.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/plotly/api/v1/utils.py b/plotly/api/v1/utils.py index abfdf745c3e..24d827b7b79 100644 --- a/plotly/api/v1/utils.py +++ b/plotly/api/v1/utils.py @@ -2,9 +2,11 @@ import requests from requests.exceptions import RequestException +from retrying import retry from plotly import config, exceptions from plotly.api.utils import basic_auth +from plotly.api.v2.utils import should_retry def validate_response(response): @@ -59,6 +61,8 @@ def get_headers(): return headers +@retry(wait_exponential_multiplier=1000, wait_exponential_max=16000, + stop_max_delay=180000, retry_on_exception=should_retry) def request(method, url, **kwargs): """ Central place to make any v1 api request.