simon987/hexlib
1
0
Fork 0
mirror of https://github.com/simon987/hexlib.git synced 2025-04-19 17:56:43 +00:00
Code Issues Projects Releases Wiki Activity

Add get_soup()

Browse Source
This commit is contained in:
simon987 2021-04-05 19:17:27 -04:00
parent 5ee1629c79
commit c79b3bfafd
2 changed files with 9 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
hexlib/web.py
View File

@ -5,6 +5,7 @@ from datetime import datetime
from base64 import b64encode, b64decode from base64 import b64encode, b64decode
from http.cookiejar import Cookie from http.cookiejar import Cookie
from time import time from time import time
from bs4 import BeautifulSoup
import requests import requests
import orjson as json import orjson as json
@ -171,3 +172,9 @@ class Web:
if self._logger and r is not None: if self._logger and r is not None:
self._logger.debug(self._format_url("GET", url, kwargs, r) + " %.2fs" % (time() - time_start)) self._logger.debug(self._format_url("GET", url, kwargs, r) + " %.2fs" % (time() - time_start))
return r return r
def get_soup(self, url, **kwargs):
r = self.get(url, **kwargs)
if not r:
return None
return BeautifulSoup(r.content, "html.parser")

4
setup.py
View File

@ -2,7 +2,7 @@ from setuptools import setup
setup( setup(
name="hexlib", name="hexlib",
version="1.33", version="1.34",
description="Misc utility methods", description="Misc utility methods",
author="simon987", author="simon987",
author_email="me@simon987.net", author_email="me@simon987.net",
@ -13,6 +13,6 @@ setup(
]}, ]},
install_requires=[ install_requires=[
"ImageHash", "influxdb", "siphash", "python-dateutil", "redis", "orjson", "zstandard", "ImageHash", "influxdb", "siphash", "python-dateutil", "redis", "orjson", "zstandard",
"u-msgpack-python", "psycopg2-binary", "fake-useragent" "u-msgpack-python", "psycopg2-binary", "fake-useragent", "bs4"
] ]
) )