simon987/hexlib
1
0
Fork 0
mirror of https://github.com/simon987/hexlib.git synced 2025-04-18 01:16:43 +00:00
Code Issues Projects Releases Wiki Activity

Add useragent option in Web

Browse Source
This commit is contained in:
simon987 2021-03-07 11:02:26 -05:00
parent 66d37e0be2
commit 53a03baaa4
3 changed files with 13 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
hexlib/env.py
View File

@ -1,5 +1,7 @@
import redis import redis
import os import os
from fake_useragent import UserAgent
from hexlib.log import stdout_logger from hexlib.log import stdout_logger
from hexlib.web import Web from hexlib.web import Web
@ -13,10 +15,13 @@ def get_redis():
def get_web(): def get_web():
ua = UserAgent()
return Web( return Web(
proxy=os.environ.get("PROXY", None), proxy=os.environ.get("PROXY", None),
rps=os.environ.get("RPS", 1), rps=os.environ.get("RPS", 1),
logger=stdout_logger, logger=stdout_logger,
cookie_file=os.environ.get("COOKIE_FILE", None), cookie_file=os.environ.get("COOKIE_FILE", None),
retry_codes=set(int(x) if x else None for x in os.environ.get("RETRY_CODES", "").split(",")) retry_codes=set(int(x) if x else None for x in os.environ.get("RETRY_CODES", "").split(",")),
ua=ua[os.environ.get("USER_AGENT")] if os.environ.get("USER_AGENT", None) is not None else None
) )

6
hexlib/web.py
View File

@ -111,7 +111,8 @@ def download_file(url, destination, session=None, headers=None, overwrite=False,
class Web: class Web:
def __init__(self, proxy=None, rps=1, retries=3, logger=None, cookie_file=None, retry_codes=None, session=None): def __init__(self, proxy=None, rps=1, retries=3, logger=None, cookie_file=None, retry_codes=None, session=None,
ua=None):
self._cookie_file = cookie_file self._cookie_file = cookie_file
self._proxy = proxy self._proxy = proxy
self._logger = logger self._logger = logger
@ -125,6 +126,9 @@ class Web:
self._session = session self._session = session
if ua is not None:
session.headers["User-Agent"] = ua
if self._cookie_file: if self._cookie_file:
self._session.cookies = load_cookiejar(cookie_file) self._session.cookies = load_cookiejar(cookie_file)

4
setup.py
View File

@ -2,7 +2,7 @@ from setuptools import setup
setup( setup(
name="hexlib", name="hexlib",
version="1.30", version="1.31",
description="Misc utility methods", description="Misc utility methods",
author="simon987", author="simon987",
author_email="me@simon987.net", author_email="me@simon987.net",
@ -13,6 +13,6 @@ setup(
]}, ]},
install_requires=[ install_requires=[
"ImageHash", "influxdb", "siphash", "python-dateutil", "redis", "orjson", "zstandard", "ImageHash", "influxdb", "siphash", "python-dateutil", "redis", "orjson", "zstandard",
"u-msgpack-python", "psycopg2-binary" "u-msgpack-python", "psycopg2-binary", "fake-useragent"
] ]
) )