simon987/od-database
1
0
Fork 0
mirror of https://github.com/simon987/od-database.git synced 2025-12-13 23:09:01 +00:00
Code Issues Projects Releases Wiki Activity

Completed tasks are now fetched by the TaskDispatcher

Browse Source
This commit is contained in:
Simon
2018-06-12 14:16:05 -04:00
parent d61fd75890
commit 6c912ea8c5
2 changed files with 15 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
task.py
View File

@@ -26,7 +26,9 @@ class CrawlServer:
def get_completed_tasks(self) -> list:
r = requests.get(self.url + "/task/completed")
return []
return [
TaskResult(r["status_code"], r["file_count"], r["start_time"], r["end_time"], r["website_id"])
for r in json.loads(r.text)]
def get_queued_tasks(self) -> list:
@@ -62,7 +64,13 @@ class TaskDispatcher:
]
def check_completed_tasks(self):
return self._get_available_crawl_server().get_completed_tasks()
completed_tasks = []
for server in self.crawl_servers:
completed_tasks.extend(server.get_completed_tasks())
if completed_tasks:
print(str(len(completed_tasks)) + " completed tasks. Will index immediately")
def dispatch_task(self, task: Task):
self._get_available_crawl_server().queue_task(task)