simon987/od-database
1
0
Fork 0
mirror of https://github.com/simon987/od-database.git synced 2025-04-10 14:06:45 +00:00
Code Issues Projects Releases Wiki Activity

hotfix attempt 3

Browse Source
This commit is contained in:
simon987 2019-02-02 11:45:46 -05:00
parent 0ff6ea1682
commit 59b1d249ba
2 changed files with 9 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
api.py
View File

@ -31,10 +31,7 @@ def setup_api(app):
oddb.logger.info("No queued tasks, creating a new one")
try:
website_id = oddb.db.get_oldest_website_id()
website = oddb.db.get_website_by_id(website_id)
task = Task(website_id, website.url)
oddb.db.put_task(task, name)
task = oddb.db.make_task_for_oldest()
except:
oddb.logger.error("Couldn't create new task")
abort(404)

12
database.py
View File

@ -153,13 +153,17 @@ class Database:
website_id = cursor.fetchone()
return website_id[0] if website_id else None
def get_oldest_website_id(self):
def make_task_for_oldest(self, assigned_crawler):
with sqlite3.connect(self.db_path) as conn:
cursor = conn.cursor()
cursor.execute("SELECT Website.id FROM Website WHERE Website.id not in (SELECT website_id FROM Queue) "
"ORDER BY last_modified ASC LIMIT 1")
return cursor.fetchone()[0]
cursor.execute("INSERT INTO QUEUE (website_id, url, assigned_crawler) SELECT Website.id, Website.url, ? FROM Website WHERE Website.id not in (SELECT website_id FROM Queue) "
"ORDER BY last_modified ASC LIMIT 1", (assigned_crawler, ))
cursor.execute("SELECT id, website_id, url, priority, callback_type, callback_args FROM Queue "
"WHERE id=LAST_INSERT_ID()")
task = cursor.fetchone()
return Task(task[1], task[2], task[3], task[4], task[5])
def delete_website(self, website_id):