simon987/od-database
1
0
Fork 0
mirror of https://github.com/simon987/od-database.git synced 2025-04-20 02:46:45 +00:00
Code Issues Projects Releases Wiki Activity

Attempt to fix Unicode errors part two

Browse Source
This commit is contained in:
Simon 2018-06-16 15:30:44 -04:00
parent c309aa25c8
commit 86144935e3
1 changed files with 1 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
crawl_server/remote_http.py
View File

@ -113,7 +113,7 @@ class HttpDirectory(RemoteDirectory):
for link in links: for link in links:
result.append((link.text, link.get("href"))) result.append((link.text, link.get("href")))
except UnicodeDecodeError: except UnicodeDecodeError:
tree = etree.HTML(body.decode("utf-8", errors="ignore"), parser=self.parser) tree = etree.HTML(body.decode("utf-8", errors="ignore").encode("utf-8"), parser=self.parser)
links = [] links = []
try: try:
links = tree.findall(".//a/[@href]") links = tree.findall(".//a/[@href]")