simon987/od-database
1
0
Fork 0
mirror of https://github.com/simon987/od-database.git synced 2025-12-14 15:19:03 +00:00
Code Issues Projects Releases Wiki Activity

Crawler performance improvements

Browse Source
This commit is contained in:
Simon
2018-07-25 11:27:50 -04:00
parent fbbe952e4d
commit 34d1f375a8
2 changed files with 8 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
crawl_server/remote_http.py
View File

@@ -102,7 +102,7 @@ class HttpDirectory(RemoteDirectory):
def list_dir(self, path):
current_dir_name = path[path.rstrip("/").rfind("/") + 1: -1]
path_identifier = hashlib.sha1(current_dir_name.encode())
path_identifier = hashlib.md5(current_dir_name.encode())
path_url = urljoin(self.base_url, path, "")
body = self._stream_body(path_url)
anchors = self._parse_links(body)