simon987/od-database
1
0
Fork 0
mirror of https://github.com/simon987/od-database.git synced 2025-04-20 10:56:47 +00:00
Code Issues Projects Releases Wiki Activity

Bugfix post-pycurl update pt. 2

Browse Source
This commit is contained in:
Simon 2018-08-23 12:40:13 -04:00
parent 484a0baf9d
commit 6ffc43601b
1 changed files with 2 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
crawl_server/remote_http.py
View File

@ -162,6 +162,8 @@ class HttpDirectory(RemoteDirectory):
handles = [self._curl_handle() for _ in range(len(urls_to_request))] handles = [self._curl_handle() for _ in range(len(urls_to_request))]
files = pool.starmap(self._request_file, zip(handles, urls_to_request, repeat(self.base_url))) files = pool.starmap(self._request_file, zip(handles, urls_to_request, repeat(self.base_url)))
pool.close() pool.close()
for handle in handles:
handle.close()
for file in files: for file in files:
if file: if file:
yield file yield file
@ -186,7 +188,6 @@ class HttpDirectory(RemoteDirectory):
stripped_url = url[len(base_url) - 1:] stripped_url = url[len(base_url) - 1:]
headers = HttpDirectory._parse_dict_header(raw_headers.getvalue().decode("utf-8", errors="ignore")) headers = HttpDirectory._parse_dict_header(raw_headers.getvalue().decode("utf-8", errors="ignore"))
raw_headers.close() raw_headers.close()
curl.close()
path, name = os.path.split(stripped_url) path, name = os.path.split(stripped_url)
date = headers.get("Last-Modified", "1970-01-01") date = headers.get("Last-Modified", "1970-01-01")