simon987/od-database
1
0
Fork 0
mirror of https://github.com/simon987/od-database.git synced 2025-12-13 23:09:01 +00:00
Code Issues Projects Releases Wiki Activity

Get queued tasks temporarily returns only non-ftp websites

Browse Source
This commit is contained in:
Simon
2018-11-16 22:59:26 -05:00
parent e3f3a9cf7f
commit 4c51598441
2 changed files with 8 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
database.py
View File

@@ -381,14 +381,16 @@ class Database:
return [Task(t[0], t[1], t[2], t[3], t[4]) for t in db_tasks]
def pop_task(self, name) -> Task:
def pop_task(self, name, ftp: bool) -> Task:
with sqlite3.connect(self.db_path) as conn:
cursor = conn.cursor()
cursor.execute("SELECT id, website_id, url, priority, callback_type, callback_args "
"FROM Queue WHERE assigned_crawler is NULL "
"ORDER BY priority DESC, Queue.id ASC LIMIT 1")
cursor.execute("SELECT id, website_id, url, priority, callback_type, callback_args " +
"FROM Queue WHERE assigned_crawler is NULL " +
("AND url LIKE 'ftp%'" if ftp else "") +
"ORDER BY priority DESC, Queue.id" +
"ASC LIMIT 1")
task = cursor.fetchone()
if task: