simon987/od-database
1
0
Fork 0
mirror of https://github.com/simon987/od-database.git synced 2025-04-20 02:46:45 +00:00
Code Issues Projects Releases Wiki Activity

Another fix for encoding problems

Browse Source
This commit is contained in:
Simon 2018-06-18 20:30:18 -04:00
parent 788d3749d4
commit 677bfa03ea
1 changed files with 1 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
crawl_server/remote_http.py
View File

@ -151,7 +151,7 @@ class HttpDirectory(RemoteDirectory):
try: try:
r = self.session.get(url, stream=True, timeout=40) r = self.session.get(url, stream=True, timeout=40)
for chunk in r.iter_content(chunk_size=4096): for chunk in r.iter_content(chunk_size=4096):
yield chunk.decode(r.encoding, errors="ignore") yield chunk.decode(r.encoding if r.encoding else "utf-8", errors="ignore")
r.close() r.close()
del r del r
break break