simon987/od-database
1
0
Fork 0
mirror of https://github.com/simon987/od-database.git synced 2025-12-14 07:09:03 +00:00
Code Issues Projects Releases Wiki Activity

More bugfixes for looping directories, some work on task dispatching

Browse Source
This commit is contained in:
Simon
2018-06-21 20:50:26 -04:00
parent 14d384e366
commit 7a4432e4d0
3 changed files with 14 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
crawl_server/crawler.py
View File

@@ -87,8 +87,8 @@ class RemoteDirectoryCrawler:
try:
try:
directory = RemoteDirectoryFactory.get_directory(self.url)
path, root_listing = directory.list_dir("")
self.crawled_paths.append(path)
path_id, root_listing = directory.list_dir(urlparse(self.url).path)
self.crawled_paths.append(path_id)
directory.close()
except TimeoutError:
return CrawlResult(0, "timeout")