simon987/od-database
1
0
Fork 0
mirror of https://github.com/simon987/od-database.git synced 2025-04-10 14:06:45 +00:00
Code Issues Projects Releases Wiki Activity

Only queue http tasks (temp)

Browse Source
This commit is contained in:
simon987 2019-04-06 09:07:17 -04:00
parent 310f343423
commit 06ae89f4d2
2 changed files with 5 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
database.py
View File

@ -309,14 +309,15 @@ class Database:
conn.commit()
def get_oldest_updated_websites(self, size: int):
def get_oldest_updated_websites(self, size: int, prefix: str):
with psycopg2.connect(self.db_conn_str) as conn:
cursor = conn.cursor()
cursor.execute("SELECT id, url, last_modified FROM website "
"WHERE url LIKE %s "
"ORDER BY last_modified ASC LIMIT %s",
(size,))
(prefix + "%", size, ))
return [Website(url=r[1],
website_id=r[0],
last_modified=r[2],

4
tasks.py
View File

@ -133,7 +133,7 @@ class TaskManager:
def _generate_crawling_tasks(self):
# TODO: Insert more in-depth re-crawl logic here
websites_to_crawl = self.db.get_oldest_updated_websites(config.RECRAWL_POOL_SIZE)
websites_to_crawl = self.db.get_oldest_updated_websites(config.RECRAWL_POOL_SIZE, prefix="http")
def recrawl(website: Website):
crawl_task = Task(website.id, website.url,
@ -141,7 +141,7 @@ class TaskManager:
)
self.queue_task(crawl_task)
pool = ThreadPool(processes=10)
pool = ThreadPool(processes=3)
pool.map(func=recrawl, iterable=websites_to_crawl)
pool.close()