simon987/od-database
1
0
Fork 0
mirror of https://github.com/simon987/od-database.git synced 2025-04-19 18:36:44 +00:00
Code Issues Projects Releases Wiki Activity

revert task queuing pt. 3 ._.

Browse Source
This commit is contained in:
Simon 2018-11-18 11:13:51 -05:00
parent 801d056da8
commit 98f43f817a
2 changed files with 3 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
app.py
View File

@ -602,7 +602,7 @@ def api_complete_task():
filename = "./tmp/" + str(task_result.website_id) + ".json" filename = "./tmp/" + str(task_result.website_id) + ".json"
if not os.path.exists(filename): if not os.path.exists(filename):
filename = None filename = None
taskManager.complete_task(filename) taskManager.complete_task(filename, task, task_result, name)
if filename and os.path.exists(filename): if filename and os.path.exists(filename):
os.remove(filename) os.remove(filename)

4
tasks.py
View File

@ -65,9 +65,8 @@ class TaskManager:
self.search = ElasticSearchEngine("od-database") self.search = ElasticSearchEngine("od-database")
self.db = database.Database("db.sqlite3") self.db = database.Database("db.sqlite3")
def complete_task(self, file_list): def complete_task(self, file_list, task, task_result, crawler_name):
file_list, task, task_result, crawler_name = self.to_index_queue.pop()
self.search.delete_docs(task_result.website_id) self.search.delete_docs(task_result.website_id)
if file_list: if file_list:
@ -87,6 +86,7 @@ class TaskManager:
self.db.log_result(task_result) self.db.log_result(task_result)
def queue_task(self, task: Task): def queue_task(self, task: Task):
self.db.put_task(task) self.db.put_task(task)
print("Queued task and made it available to crawlers: " + str(task.website_id)) print("Queued task and made it available to crawlers: " + str(task.website_id))