diff --git a/app.py b/app.py index 785d8d8..7a97016 100644 --- a/app.py +++ b/app.py @@ -602,7 +602,7 @@ def api_complete_task(): filename = "./tmp/" + str(task_result.website_id) + ".json" if not os.path.exists(filename): filename = None - taskManager.complete_task(filename) + taskManager.complete_task(filename, task, task_result, name) if filename and os.path.exists(filename): os.remove(filename) diff --git a/tasks.py b/tasks.py index 258277e..dd94835 100644 --- a/tasks.py +++ b/tasks.py @@ -65,9 +65,8 @@ class TaskManager: self.search = ElasticSearchEngine("od-database") self.db = database.Database("db.sqlite3") - def complete_task(self, file_list): + def complete_task(self, file_list, task, task_result, crawler_name): - file_list, task, task_result, crawler_name = self.to_index_queue.pop() self.search.delete_docs(task_result.website_id) if file_list: @@ -87,6 +86,7 @@ class TaskManager: self.db.log_result(task_result) + def queue_task(self, task: Task): self.db.put_task(task) print("Queued task and made it available to crawlers: " + str(task.website_id))