From 4c51598441c8b5b823cebde0a47ad2d4e1f74417 Mon Sep 17 00:00:00 2001 From: Simon Date: Fri, 16 Nov 2018 22:59:26 -0500 Subject: [PATCH] Get queued tasks temporarily returns only non-ftp websites --- app.py | 4 ++-- database.py | 10 ++++++---- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/app.py b/app.py index 1278af5..41cad72 100644 --- a/app.py +++ b/app.py @@ -550,7 +550,7 @@ def api_get_task(): name = db.check_api_token(token) if name: - task = db.pop_task(name) + task = db.pop_task(name, False) if task: print("Assigning task " + str(task.website_id) + " to " + name) @@ -562,7 +562,7 @@ def api_get_task(): task = Task(website_id, website.url) db.put_task(task) - task = db.pop_task(name) + task = db.pop_task(name, False) return Response(str(task), mimetype="application/json") else: diff --git a/database.py b/database.py index a5c0b05..a12d428 100644 --- a/database.py +++ b/database.py @@ -381,14 +381,16 @@ class Database: return [Task(t[0], t[1], t[2], t[3], t[4]) for t in db_tasks] - def pop_task(self, name) -> Task: + def pop_task(self, name, ftp: bool) -> Task: with sqlite3.connect(self.db_path) as conn: cursor = conn.cursor() - cursor.execute("SELECT id, website_id, url, priority, callback_type, callback_args " - "FROM Queue WHERE assigned_crawler is NULL " - "ORDER BY priority DESC, Queue.id ASC LIMIT 1") + cursor.execute("SELECT id, website_id, url, priority, callback_type, callback_args " + + "FROM Queue WHERE assigned_crawler is NULL " + + ("AND url LIKE 'ftp%'" if ftp else "") + + "ORDER BY priority DESC, Queue.id" + + "ASC LIMIT 1") task = cursor.fetchone() if task: