simon987/hexlib
1
0
Fork 0
mirror of https://github.com/simon987/hexlib.git synced 2025-04-22 02:56:43 +00:00
Code Issues Projects Releases Wiki Activity

Compare commits

base: simon987:43cb6c4a7bad8081fc042738c922ea394ce43700
Branches Tags
simon987:master
simon987:1.89
..
compare: simon987:973881942858be0a80adb66feb4464c84e01ba89
Branches Tags
simon987:master
simon987:1.89

No commits in common. "43cb6c4a7bad8081fc042738c922ea394ce43700" and "973881942858be0a80adb66feb4464c84e01ba89" have entirely different histories.
43cb6c4a7b ... 9738819428

2 changed files with 4 additions and 3 deletions
Show Stats Expand all files Collapse all files

2
hexlib/env.py
View File

@ -18,5 +18,5 @@ def get_web():
rps=os.environ.get("RPS", 1), rps=os.environ.get("RPS", 1),
logger=stdout_logger, logger=stdout_logger,
cookie_file=os.environ.get("COOKIE_FILE", None), cookie_file=os.environ.get("COOKIE_FILE", None),
retry_codes=set(int(x) if x else None for x in os.environ.get("RETRY_CODES", "").split(",")) retry_codes=set(os.environ.get("RETRY_CODES", "").split(","))
) )

5
hexlib/web.py
View File

@ -109,14 +109,13 @@ def download_file(url, destination, session=None, headers=None, overwrite=False,
err_cb(e) err_cb(e)
retries -= 1 retries -= 1
class Web: class Web:
def __init__(self, proxy=None, rps=1, retries=3, logger=None, cookie_file=None, retry_codes=None, session=None): def __init__(self, proxy=None, rps=1, retries=3, logger=None, cookie_file=None, retry_codes=None, session=None):
self._cookie_file = cookie_file self._cookie_file = cookie_file
self._proxy = proxy self._proxy = proxy
self._logger = logger self._logger = logger
self._current_req = None self._current_req = None
if retry_codes is None or not retry_codes: if retry_codes is None:
retry_codes = {502, 504, 522, 524, 429} retry_codes = {502, 504, 522, 524, 429}
self._retry_codes = retry_codes self._retry_codes = retry_codes
@ -167,3 +166,5 @@ class Web:
if self._logger and r is not None: if self._logger and r is not None:
self._logger.debug(self._format_url("GET", url, kwargs, r) + " %.2fs" % (time() - time_start)) self._logger.debug(self._format_url("GET", url, kwargs, r) + " %.2fs" % (time() - time_start))
return r return r