simon987/od-database
1
0
Fork 0
mirror of https://github.com/simon987/od-database.git synced 2025-04-24 12:45:51 +00:00
Code Issues Projects Releases Wiki Activity

Bugfix post-pycurl update

Browse Source
This commit is contained in:
Simon 2018-08-23 12:37:27 -04:00
parent cadaf14c1b
commit 484a0baf9d
1 changed files with 3 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
crawl_server/remote_http.py
View File

@ -179,13 +179,14 @@ class HttpDirectory(RemoteDirectory):
while retries > 0: while retries > 0:
try: try:
raw_headers = BytesIO() raw_headers = BytesIO()
curl.setopt(pycurl.URL, url) curl.setopt(pycurl.URL, url.encode("utf-8", errors="ignore"))
curl.setopt(pycurl.HEADERFUNCTION, raw_headers.write) curl.setopt(pycurl.HEADERFUNCTION, raw_headers.write)
curl.perform() curl.perform()
stripped_url = url[len(base_url) - 1:] stripped_url = url[len(base_url) - 1:]
headers = HttpDirectory._parse_dict_header(raw_headers.getvalue().decode("utf-8", errors="ignore")) headers = HttpDirectory._parse_dict_header(raw_headers.getvalue().decode("utf-8", errors="ignore"))
raw_headers.close() raw_headers.close()
curl.close()
path, name = os.path.split(stripped_url) path, name = os.path.split(stripped_url)
date = headers.get("Last-Modified", "1970-01-01") date = headers.get("Last-Modified", "1970-01-01")
@ -208,7 +209,7 @@ class HttpDirectory(RemoteDirectory):
while retries > 0: while retries > 0:
try: try:
content = BytesIO() content = BytesIO()
self.curl.setopt(pycurl.URL, url) self.curl.setopt(pycurl.URL, url.encode("utf-8", errors="ignore"))
self.curl.setopt(pycurl.WRITEDATA, content) self.curl.setopt(pycurl.WRITEDATA, content)
self.curl.perform() self.curl.perform()