simon987/hexlib
1
0
Fork 0
mirror of https://github.com/simon987/hexlib.git synced 2025-04-22 02:56:43 +00:00
Code Issues Projects Releases Wiki Activity

Compare commits

base: simon987:7ecd55a1c64656735bad00f61c5bb6746491d5e7
Branches Tags
simon987:master
simon987:1.89
..
compare: simon987:c295b5d30b952726a98af2ca8f03f8ca967c5bdb
Branches Tags
simon987:master
simon987:1.89

No commits in common. "7ecd55a1c64656735bad00f61c5bb6746491d5e7" and "c295b5d30b952726a98af2ca8f03f8ca967c5bdb" have entirely different histories.
7ecd55a1c6 ... c295b5d30b

3 changed files with 3 additions and 13 deletions
Show Stats Expand all files Collapse all files

4
hexlib/env.py
View File

@ -37,15 +37,13 @@ def redis_publish(rdb, item, item_project, item_type, item_subproject=None, item
def get_web(session=None): def get_web(session=None):
ua = UserAgent() ua = UserAgent()
retry_codes = os.environ.get("RETRY_CODES", "")
web = Web( web = Web(
session=session, session=session,
proxy=os.environ.get("PROXY", None), proxy=os.environ.get("PROXY", None),
rps=os.environ.get("RPS", 1), rps=os.environ.get("RPS", 1),
logger=stdout_logger, logger=stdout_logger,
cookie_file=os.environ.get("COOKIE_FILE", None), cookie_file=os.environ.get("COOKIE_FILE", None),
retry_codes=set(int(x) for x in retry_codes) if retry_codes else None, retry_codes=set(int(x) if x else None for x in os.environ.get("RETRY_CODES", "").split(",")),
retries=int(os.environ.get("RETRIES", 3)), retries=int(os.environ.get("RETRIES", 3)),
retry_sleep=int(os.environ.get("RETRY_SLEEP", 0)), retry_sleep=int(os.environ.get("RETRY_SLEEP", 0)),
ua=ua[os.environ.get("USER_AGENT")] if os.environ.get("USER_AGENT", None) is not None else None ua=ua[os.environ.get("USER_AGENT")] if os.environ.get("USER_AGENT", None) is not None else None

10
hexlib/web.py
View File

@ -81,14 +81,6 @@ def cookiejar_filter(cj, pattern):
return filtered_cj return filtered_cj
def cookiejar_filter_name(cj, pattern):
filtered_cj = RequestsCookieJar()
for c in cj:
if re.match(pattern, c.name):
filtered_cj.set_cookie(c)
return filtered_cj
def url_query_value(url, arg, as_list=False): def url_query_value(url, arg, as_list=False):
qs = urlparse(url).query qs = urlparse(url).query
parsed_qs = parse_qs(qs) parsed_qs = parse_qs(qs)
@ -142,7 +134,7 @@ class Web:
self._logger = logger self._logger = logger
self._current_req = None self._current_req = None
if retry_codes is None or not retry_codes: if retry_codes is None or not retry_codes:
retry_codes = {502, 504, 520, 522, 524, 429} retry_codes = {502, 504, 522, 524, 429}
self._retry_codes = retry_codes self._retry_codes = retry_codes
if session is None: if session is None:

2
setup.py
View File

@ -2,7 +2,7 @@ from setuptools import setup
setup( setup(
name="hexlib", name="hexlib",
version="1.62", version="1.59",
description="Misc utility methods", description="Misc utility methods",
author="simon987", author="simon987",
author_email="me@simon987.net", author_email="me@simon987.net",