simon987/od-database
1
0
Fork 0
mirror of https://github.com/simon987/od-database.git synced 2025-04-19 18:36:44 +00:00
Code Issues Projects Releases Wiki Activity

Re-added timeout that was accidentally deleted

Browse Source
This commit is contained in:
Simon 2018-06-16 22:20:15 -04:00
parent 1283cc9599
commit e6175c84c9
1 changed files with 2 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
crawl_server/remote_http.py
View File

@ -92,7 +92,7 @@ class HttpDirectory(RemoteDirectory):
retries = HttpDirectory.MAX_RETRIES retries = HttpDirectory.MAX_RETRIES
while retries > 0: while retries > 0:
try: try:
r = self.session.get(url) r = self.session.get(url, timeout=40)
return r.content, r.encoding return r.content, r.encoding
except RequestException: except RequestException:
retries -= 1 retries -= 1
@ -134,7 +134,7 @@ class HttpDirectory(RemoteDirectory):
retries = HttpDirectory.MAX_RETRIES retries = HttpDirectory.MAX_RETRIES
while retries > 0: while retries > 0:
try: try:
r = self.session.head(url, allow_redirects=False, timeout=50) r = self.session.head(url, allow_redirects=False, timeout=40)
stripped_url = url[len(self.base_url) - 1:] stripped_url = url[len(self.base_url) - 1:]