From 0c3d0b38e611c091ff90204d7f90c199dbb7fd8a Mon Sep 17 00:00:00 2001 From: simon987 Date: Sat, 6 Apr 2019 09:21:02 -0400 Subject: [PATCH] Don't use multiprocessing for recrawl task --- tasks.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tasks.py b/tasks.py index 012235a..51ce6ca 100644 --- a/tasks.py +++ b/tasks.py @@ -2,7 +2,6 @@ import json import logging import os import time -from multiprocessing.pool import ThreadPool from threading import Thread from uuid import uuid4 @@ -141,9 +140,8 @@ class TaskManager: ) self.queue_task(crawl_task) - pool = ThreadPool(processes=3) - pool.map(func=recrawl, iterable=websites_to_crawl) - pool.close() + for w in websites_to_crawl: + recrawl(w) def queue_task(self, task: Task): max_assign_time = 24 * 7 * 3600