simon987/od-database
1
0
Fork 0
mirror of https://github.com/simon987/od-database.git synced 2025-12-16 08:09:04 +00:00
Code Issues Projects Releases Wiki Activity

barebones crawl_server microservice

Browse Source
This commit is contained in:
Simon
2018-06-11 19:00:43 -04:00
parent 8421cc0885
commit d849227798
14 changed files with 264 additions and 220 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
crawler/http.py
View File

@@ -54,7 +54,7 @@ class HttpDirectory(RemoteDirectory):
if self._should_ignore(link):
continue
file_url = urljoin(path_url, link[1])
file_url = urljoin(path_url, link.url)
path, file_name = os.path.split(file_url[len(self.base_url) - 1:])
if self._isdir(link):