simon987/od-database-crawler
1
0
Fork 0
mirror of https://github.com/terorie/od-database-crawler.git synced 2025-04-19 18:36:43 +00:00
Code Issues Projects Releases Wiki Activity

Update config.yml

Browse Source
This commit is contained in:
Richard Patel 2018-11-17 04:19:04 +01:00
parent d596882b40
commit 7fdffff58f
No known key found for this signature in database
GPG Key ID: C268B2BBDA2ABECB
1 changed files with 9 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
config.yml
View File

@ -16,13 +16,18 @@ output:
# Crawler settings # Crawler settings
crawl: crawl:
# Number of sites that can be # Number of sites that can be processed at once
# processed at once tasks: 100
tasks: 300
# Number of connections per site # Number of connections per site
connections: 20 # Please be careful with this setting!
# The crawler fires fast and more than
# ten connections can overwhelm a server.
connections: 10
# How often to retry getting data # How often to retry getting data
# from the site before giving up # from the site before giving up
retries: 5 retries: 5
# Time before discarding a network request # Time before discarding a network request
timeout: 10s timeout: 10s