simon987/od-database
1
0
Fork 0
mirror of https://github.com/simon987/od-database.git synced 2025-12-16 16:19:05 +00:00
Code Issues Projects Releases Wiki Activity

barebones crawl_server microservice

Browse Source
This commit is contained in:
Simon
2018-06-11 19:00:43 -04:00
parent 8421cc0885
commit d849227798
14 changed files with 264 additions and 220 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
crawler/crawler.py
View File

@@ -71,7 +71,7 @@ class RemoteDirectoryCrawler:
try:
directory = RemoteDirectoryFactory.get_directory(self.url)
root_listing = directory.list_dir("/dl2/") # todo get path
root_listing = directory.list_dir("/")
directory.close()
except TimeoutError:
return

2
crawler/http.py
View File

@@ -54,7 +54,7 @@ class HttpDirectory(RemoteDirectory):
if self._should_ignore(link):
continue
file_url = urljoin(path_url, link[1])
file_url = urljoin(path_url, link.url)
path, file_name = os.path.split(file_url[len(self.base_url) - 1:])
if self._isdir(link):