simon987/od-database
1
0
Fork 0
mirror of https://github.com/simon987/od-database.git synced 2025-12-13 23:09:01 +00:00
Code Issues Projects Releases Wiki Activity

Bug fixes for FTP crawler

Browse Source
This commit is contained in:
Simon
2018-06-13 15:54:45 -04:00
parent 9bde8cb629
commit 1bd58468eb
5 changed files with 45 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
crawl_server/task_manager.py
View File

@@ -53,7 +53,7 @@ class TaskManager:
print("Starting task " + task.url)
crawler = RemoteDirectoryCrawler(task.url, 100)
crawler = RemoteDirectoryCrawler(task.url, 30)
crawl_result = crawler.crawl_directory("./crawled/" + str(task.website_id) + ".json")
result.file_count = crawl_result.file_count