From 88bf634cb64344085925bffbfadc78ed657efaff Mon Sep 17 00:00:00 2001 From: Simon Fortier Date: Fri, 5 Apr 2019 09:30:20 -0400 Subject: [PATCH] Update config.yml --- config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config.yml b/config.yml index a420c80..5e60845 100644 --- a/config.yml +++ b/config.yml @@ -47,7 +47,7 @@ output: # Crawler settings crawl: # Number of sites that can be processed at once - tasks: 100 + tasks: 25 # Number of connections per site # Please be careful with this setting! @@ -81,4 +81,4 @@ crawl: # in memory. # A negative value will cause all jobs # to be stored in memory. (Don't do this) - job_buffer: 5000 + job_buffer: -1