From d43cf3b0ce72b0c722e099e6ee1315542c4745eb Mon Sep 17 00:00:00 2001 From: Simon Date: Fri, 20 Jul 2018 14:11:17 -0400 Subject: [PATCH] Empty queue timeout increased to avoid that all workers die before the website is dropped --- crawl_server/crawler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crawl_server/crawler.py b/crawl_server/crawler.py index 6854c7f..b103167 100644 --- a/crawl_server/crawler.py +++ b/crawl_server/crawler.py @@ -140,7 +140,7 @@ class RemoteDirectoryCrawler: while directory: try: - path = in_q.get(timeout=150) + path = in_q.get(timeout=2000) except Empty: logger.debug("in_q is Empty") directory.close()