From 4278b0f89e82417de8734be1db280942e3ec9747 Mon Sep 17 00:00:00 2001 From: simon987 Date: Sat, 27 Feb 2021 08:58:21 -0500 Subject: [PATCH] web retry_codes fix --- hexlib/env.py | 2 +- hexlib/web.py | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/hexlib/env.py b/hexlib/env.py index 8085b87..90da20a 100644 --- a/hexlib/env.py +++ b/hexlib/env.py @@ -18,5 +18,5 @@ def get_web(): rps=os.environ.get("RPS", 1), logger=stdout_logger, cookie_file=os.environ.get("COOKIE_FILE", None), - retry_codes=set(os.environ.get("RETRY_CODES", "").split(",")) + retry_codes=set(int(x) for x in os.environ.get("RETRY_CODES", "").split(",")) ) diff --git a/hexlib/web.py b/hexlib/web.py index 3ad1175..8ac1398 100644 --- a/hexlib/web.py +++ b/hexlib/web.py @@ -109,13 +109,14 @@ def download_file(url, destination, session=None, headers=None, overwrite=False, err_cb(e) retries -= 1 + class Web: def __init__(self, proxy=None, rps=1, retries=3, logger=None, cookie_file=None, retry_codes=None, session=None): self._cookie_file = cookie_file self._proxy = proxy self._logger = logger self._current_req = None - if retry_codes is None: + if retry_codes is None or not retry_codes: retry_codes = {502, 504, 522, 524, 429} self._retry_codes = retry_codes @@ -166,5 +167,3 @@ class Web: if self._logger and r is not None: self._logger.debug(self._format_url("GET", url, kwargs, r) + " %.2fs" % (time() - time_start)) return r - -