simon987/od-database
1
0
Fork 0
mirror of https://github.com/simon987/od-database.git synced 2025-12-14 23:29:04 +00:00
Code Issues Projects Releases Wiki Activity

Started working on post-crawl callbacks and basic auth for crawl servers

Browse Source
This commit is contained in:
Simon
2018-06-14 15:05:56 -04:00
parent 1bd58468eb
commit 83ca579ec7
13 changed files with 142 additions and 56 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
crawl_server/remote_http.py
View File

@@ -36,7 +36,7 @@ class HttpDirectory(RemoteDirectory):
def __init__(self, url):
super().__init__(url)
self.parser = etree.HTMLParser(collect_ids=False)
self.parser = etree.HTMLParser(collect_ids=False, encoding="utf-8")
def list_dir(self, path) -> list:
results = []