simon987/od-database
1
0
Fork 0
mirror of https://github.com/simon987/od-database.git synced 2025-04-20 10:56:47 +00:00
Code Issues Projects Releases Wiki Activity

Get queued tasks temporarily returns only non-ftp websites

Browse Source
This commit is contained in:
Simon 2018-11-16 22:59:26 -05:00
parent e3f3a9cf7f
commit 4c51598441
2 changed files with 8 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
app.py
View File

@ -550,7 +550,7 @@ def api_get_task():
name = db.check_api_token(token) name = db.check_api_token(token)
if name: if name:
task = db.pop_task(name) task = db.pop_task(name, False)
if task: if task:
print("Assigning task " + str(task.website_id) + " to " + name) print("Assigning task " + str(task.website_id) + " to " + name)
@ -562,7 +562,7 @@ def api_get_task():
task = Task(website_id, website.url) task = Task(website_id, website.url)
db.put_task(task) db.put_task(task)
task = db.pop_task(name) task = db.pop_task(name, False)
return Response(str(task), mimetype="application/json") return Response(str(task), mimetype="application/json")
else: else:

10
database.py
View File

@ -381,14 +381,16 @@ class Database:
return [Task(t[0], t[1], t[2], t[3], t[4]) for t in db_tasks] return [Task(t[0], t[1], t[2], t[3], t[4]) for t in db_tasks]
def pop_task(self, name) -> Task: def pop_task(self, name, ftp: bool) -> Task:
with sqlite3.connect(self.db_path) as conn: with sqlite3.connect(self.db_path) as conn:
cursor = conn.cursor() cursor = conn.cursor()
cursor.execute("SELECT id, website_id, url, priority, callback_type, callback_args " cursor.execute("SELECT id, website_id, url, priority, callback_type, callback_args " +
"FROM Queue WHERE assigned_crawler is NULL " "FROM Queue WHERE assigned_crawler is NULL " +
"ORDER BY priority DESC, Queue.id ASC LIMIT 1") ("AND url LIKE 'ftp%'" if ftp else "") +
"ORDER BY priority DESC, Queue.id" +
"ASC LIMIT 1")
task = cursor.fetchone() task = cursor.fetchone()
if task: if task: