simon987/hexlib
1
0
Fork 0
mirror of https://github.com/simon987/hexlib.git synced 2025-04-10 14:06:43 +00:00
Code Issues Projects Releases Wiki Activity

web retry_codes fix

Browse Source
This commit is contained in:
simon987 2021-02-27 08:58:21 -05:00
parent 9738819428
commit 4278b0f89e
2 changed files with 3 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
hexlib/env.py
View File

@ -18,5 +18,5 @@ def get_web():
rps=os.environ.get("RPS", 1),
logger=stdout_logger,
cookie_file=os.environ.get("COOKIE_FILE", None),
retry_codes=set(os.environ.get("RETRY_CODES", "").split(","))
retry_codes=set(int(x) for x in os.environ.get("RETRY_CODES", "").split(","))
)

5
hexlib/web.py
View File

@ -109,13 +109,14 @@ def download_file(url, destination, session=None, headers=None, overwrite=False,
err_cb(e)
retries -= 1
class Web:
def __init__(self, proxy=None, rps=1, retries=3, logger=None, cookie_file=None, retry_codes=None, session=None):
self._cookie_file = cookie_file
self._proxy = proxy
self._logger = logger
self._current_req = None
if retry_codes is None:
if retry_codes is None or not retry_codes:
retry_codes = {502, 504, 522, 524, 429}
self._retry_codes = retry_codes
@ -166,5 +167,3 @@ class Web:
if self._logger and r is not None:
self._logger.debug(self._format_url("GET", url, kwargs, r) + " %.2fs" % (time() - time_start))
return r