simon987/od-database
1
0
Fork 0
mirror of https://github.com/simon987/od-database.git synced 2025-12-13 23:09:01 +00:00
Code Issues Projects Releases Wiki Activity

Added button to queue empty websites

Browse Source
This commit is contained in:
Simon
2018-06-24 19:33:15 -04:00
parent f6ee338c0f
commit 059d9fd366
4 changed files with 40 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
task.py
View File

@@ -174,18 +174,17 @@ class TaskDispatcher:
for server in self.db.get_crawl_servers():
for task in server.fetch_completed_tasks():
print("Completed task")
# All files are overwritten
self.search.delete_docs(task.website_id)
file_list = server.fetch_website_files(task.website_id)
if file_list:
if task.file_count:
# All files are overwritten
self.search.delete_docs(task.website_id)
file_list = server.fetch_website_files(task.website_id)
self.search.import_json(file_list, task.website_id)
# File list is safe to delete once indexed
server.free_website_files(task.website_id)
# Update last_modified date for website
self.db.update_website_date_if_exists(task.website_id)
# File list is safe to delete once indexed
server.free_website_files(task.website_id)
def dispatch_task(self, task: Task):
self._get_available_crawl_server().queue_task(task)
@@ -207,7 +206,7 @@ class TaskDispatcher:
return server_with_most_free_slots
def get_queued_tasks(self) -> list:
def get_queued_tasks(self):
queued_tasks_by_server = self._get_queued_tasks_by_server()
for queued_tasks in queued_tasks_by_server.values():