mirror of
https://github.com/simon987/od-database.git
synced 2025-12-13 23:09:01 +00:00
Use task_tracker for task tracking
This commit is contained in:
40
database.py
40
database.py
@@ -1,4 +1,3 @@
|
||||
import json
|
||||
import os
|
||||
import sqlite3
|
||||
import uuid
|
||||
@@ -30,32 +29,6 @@ class ApiClient:
|
||||
self.name = name
|
||||
|
||||
|
||||
class Task:
|
||||
|
||||
def __init__(self, website_id: int, url: str, priority: int = 1,
|
||||
callback_type: str = None, callback_args: str = None):
|
||||
self.website_id = website_id
|
||||
self.url = url
|
||||
self.priority = priority
|
||||
self.callback_type = callback_type
|
||||
self.callback_args = json.loads(callback_args) if callback_args else {}
|
||||
|
||||
def to_json(self):
|
||||
return {
|
||||
"website_id": self.website_id,
|
||||
"url": self.url,
|
||||
"priority": self.priority,
|
||||
"callback_type": self.callback_type,
|
||||
"callback_args": json.dumps(self.callback_args)
|
||||
}
|
||||
|
||||
def __str__(self):
|
||||
return json.dumps(self.to_json())
|
||||
|
||||
def __repr__(self):
|
||||
return self.__str__()
|
||||
|
||||
|
||||
class Database:
|
||||
|
||||
def __init__(self, db_path):
|
||||
@@ -152,19 +125,6 @@ class Database:
|
||||
website_id = cursor.fetchone()
|
||||
return website_id[0] if website_id else None
|
||||
|
||||
def make_task_for_oldest(self, assigned_crawler):
|
||||
|
||||
# TODO: task_tracker
|
||||
with sqlite3.connect(self.db_path) as conn:
|
||||
cursor = conn.cursor()
|
||||
cursor.execute("INSERT INTO QUEUE (website_id, url, assigned_crawler) SELECT Website.id, Website.url, ? FROM Website WHERE Website.id not in (SELECT website_id FROM Queue) "
|
||||
"ORDER BY last_modified ASC LIMIT 1", (assigned_crawler, ))
|
||||
|
||||
cursor.execute("SELECT id, website_id, url, priority, callback_type, callback_args FROM Queue "
|
||||
"WHERE id=LAST_INSERT_ROWID()")
|
||||
task = cursor.fetchone()
|
||||
return Task(task[1], task[2], task[3], task[4], task[5])
|
||||
|
||||
def delete_website(self, website_id):
|
||||
|
||||
with sqlite3.connect(self.db_path) as conn:
|
||||
|
||||
Reference in New Issue
Block a user