simon987/od-database
1
0
Fork 0
mirror of https://github.com/simon987/od-database.git synced 2025-12-13 23:09:01 +00:00
Code Issues Projects Releases Wiki Activity

Updated readme and UI fixes

Browse Source
This commit is contained in:
Simon
2018-06-22 13:22:58 -04:00
parent 9d3fc2d71b
commit e824b2bf3c
8 changed files with 51 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
crawl_server/task_manager.py
View File

@@ -1,3 +1,4 @@
import config
from crawl_server.database import TaskManagerDatabase, Task, TaskResult
from multiprocessing import Manager, Pool
from apscheduler.schedulers.background import BackgroundScheduler
@@ -59,7 +60,7 @@ class TaskManager:
print("Starting task " + task.url)
crawler = RemoteDirectoryCrawler(task.url, 20)
crawler = RemoteDirectoryCrawler(task.url, config.CRAWL_SERVER_THREADS)
crawl_result = crawler.crawl_directory("./crawled/" + str(task.website_id) + ".json")
del crawler