From 59b1d249ba871bd5ad645559d74a91d1c04f5dcc Mon Sep 17 00:00:00 2001 From: simon987 Date: Sat, 2 Feb 2019 11:45:46 -0500 Subject: [PATCH] hotfix attempt 3 --- api.py | 5 +---- database.py | 12 ++++++++---- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/api.py b/api.py index 242e5c8..6113c8e 100644 --- a/api.py +++ b/api.py @@ -31,10 +31,7 @@ def setup_api(app): oddb.logger.info("No queued tasks, creating a new one") try: - website_id = oddb.db.get_oldest_website_id() - website = oddb.db.get_website_by_id(website_id) - task = Task(website_id, website.url) - oddb.db.put_task(task, name) + task = oddb.db.make_task_for_oldest() except: oddb.logger.error("Couldn't create new task") abort(404) diff --git a/database.py b/database.py index 7f20997..4cbd329 100644 --- a/database.py +++ b/database.py @@ -153,13 +153,17 @@ class Database: website_id = cursor.fetchone() return website_id[0] if website_id else None - def get_oldest_website_id(self): + def make_task_for_oldest(self, assigned_crawler): with sqlite3.connect(self.db_path) as conn: cursor = conn.cursor() - cursor.execute("SELECT Website.id FROM Website WHERE Website.id not in (SELECT website_id FROM Queue) " - "ORDER BY last_modified ASC LIMIT 1") - return cursor.fetchone()[0] + cursor.execute("INSERT INTO QUEUE (website_id, url, assigned_crawler) SELECT Website.id, Website.url, ? FROM Website WHERE Website.id not in (SELECT website_id FROM Queue) " + "ORDER BY last_modified ASC LIMIT 1", (assigned_crawler, )) + + cursor.execute("SELECT id, website_id, url, priority, callback_type, callback_args FROM Queue " + "WHERE id=LAST_INSERT_ID()") + task = cursor.fetchone() + return Task(task[1], task[2], task[3], task[4], task[5]) def delete_website(self, website_id):