simon987/od-database
1
0
Fork 0
mirror of https://github.com/simon987/od-database.git synced 2025-12-14 07:09:03 +00:00
Code Issues Projects Releases Wiki Activity

Only queue http tasks (temp)

Browse Source
This commit is contained in:
simon987
2019-04-06 09:07:17 -04:00
parent 310f343423
commit 06ae89f4d2
2 changed files with 5 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
tasks.py
View File

@@ -133,7 +133,7 @@ class TaskManager:
def _generate_crawling_tasks(self):
# TODO: Insert more in-depth re-crawl logic here
websites_to_crawl = self.db.get_oldest_updated_websites(config.RECRAWL_POOL_SIZE)
websites_to_crawl = self.db.get_oldest_updated_websites(config.RECRAWL_POOL_SIZE, prefix="http")
def recrawl(website: Website):
crawl_task = Task(website.id, website.url,
@@ -141,7 +141,7 @@ class TaskManager:
)
self.queue_task(crawl_task)
pool = ThreadPool(processes=10)
pool = ThreadPool(processes=3)
pool.map(func=recrawl, iterable=websites_to_crawl)
pool.close()