simon987/od-database
1
0
Fork 0
mirror of https://github.com/simon987/od-database.git synced 2025-04-19 10:26:44 +00:00
Code Issues Projects Releases Wiki Activity

Attempt to fix unicode decode errors

Browse Source
This commit is contained in:
Simon 2018-06-16 15:20:23 -04:00
parent 9d0a0a8b42
commit c309aa25c8
1 changed files with 18 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
crawl_server/remote_http.py
View File

@ -102,6 +102,7 @@ class HttpDirectory(RemoteDirectory):
def _parse_links(self, body: bytes) -> list: def _parse_links(self, body: bytes) -> list:
result = list() result = list()
try:
tree = etree.HTML(body, parser=self.parser) tree = etree.HTML(body, parser=self.parser)
links = [] links = []
try: try:
@ -111,6 +112,16 @@ class HttpDirectory(RemoteDirectory):
for link in links: for link in links:
result.append((link.text, link.get("href"))) result.append((link.text, link.get("href")))
except UnicodeDecodeError:
tree = etree.HTML(body.decode("utf-8", errors="ignore"), parser=self.parser)
links = []
try:
links = tree.findall(".//a/[@href]")
except AttributeError:
pass
for link in links:
result.append((link.text, link.get("href")))
return result return result