Compare commits

...

2 Commits

Author SHA1 Message Date
43cb6c4a7b web retry_codes fix 2021-02-27 12:07:48 -05:00
4278b0f89e web retry_codes fix 2021-02-27 08:58:21 -05:00
2 changed files with 3 additions and 4 deletions

View File

@ -18,5 +18,5 @@ def get_web():
rps=os.environ.get("RPS", 1),
logger=stdout_logger,
cookie_file=os.environ.get("COOKIE_FILE", None),
retry_codes=set(os.environ.get("RETRY_CODES", "").split(","))
retry_codes=set(int(x) if x else None for x in os.environ.get("RETRY_CODES", "").split(","))
)

View File

@ -109,13 +109,14 @@ def download_file(url, destination, session=None, headers=None, overwrite=False,
err_cb(e)
retries -= 1
class Web:
def __init__(self, proxy=None, rps=1, retries=3, logger=None, cookie_file=None, retry_codes=None, session=None):
self._cookie_file = cookie_file
self._proxy = proxy
self._logger = logger
self._current_req = None
if retry_codes is None:
if retry_codes is None or not retry_codes:
retry_codes = {502, 504, 522, 524, 429}
self._retry_codes = retry_codes
@ -166,5 +167,3 @@ class Web:
if self._logger and r is not None:
self._logger.debug(self._format_url("GET", url, kwargs, r) + " %.2fs" % (time() - time_start))
return r