simon987/od-database
1
0
Fork 0
mirror of https://github.com/simon987/od-database.git synced 2025-12-13 23:09:01 +00:00
Code Issues Projects Releases Wiki Activity

Bug fixes with html parsing

Browse Source
This commit is contained in:
Simon
2018-06-14 20:02:06 -04:00
parent f3c7b551d2
commit 81fde6cc30
5 changed files with 13 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
crawl_server/task_manager.py
View File

@@ -54,7 +54,7 @@ class TaskManager:
print("Starting task " + task.url)
crawler = RemoteDirectoryCrawler(task.url, 100)
crawler = RemoteDirectoryCrawler(task.url, 10)
crawl_result = crawler.crawl_directory("./crawled/" + str(task.website_id) + ".json")
result.file_count = crawl_result.file_count