simon987/vanwanet_scrape
1
0
Fork 0
mirror of https://github.com/simon987/vanwanet_scrape.git synced 2025-04-10 14:06:43 +00:00
Code Issues Projects Releases Wiki Activity

Add logging

Browse Source
This commit is contained in:
simon 2019-12-17 11:03:43 -05:00
parent ec03e5e412
commit 1c86a781e3
2 changed files with 8 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
setup.py
View File

@ -2,7 +2,7 @@ from setuptools import setup
setup(
name="vanwanet_scrape",
version="1.0",
version="1.1",
description="VanwaTech DDoS protection bypass",
author="simon987",
author_email="me@simon987.net",

10
vanwanet_scrape/scraper.py
View File

@ -15,10 +15,11 @@ SUB_PATTRN = re.compile(r'document\.cookie="(.+)";location.+$')
class Scraper:
def __init__(self, domains: list, headers=None, proxies=None):
def __init__(self, domains: list, headers=None, proxies=None, logger=None):
self._session = requests.session()
self._domains = domains
self._session.cookies = CookieJar()
self.logger = logger
if headers:
self._session.headers = headers
@ -32,10 +33,13 @@ class Scraper:
r = self._get(url, **kwargs)
if Scraper._is_challenge_page(r):
cookie = Scraper._execute_challenge(Scraper._transform_js(Scraper._get_js(r)))
cookie_str = Scraper._execute_challenge(Scraper._transform_js(Scraper._get_js(r)))
if self.logger:
self.logger.debug("Executed challenge, got %s" % (cookie_from_string(cookie_str, ""),))
for domain in self._domains:
self._session.cookies.set_cookie(cookie_from_string(cookie, domain))
self._session.cookies.set_cookie(cookie_from_string(cookie_str, domain))
return self.get(url, **kwargs)
return r