simon987/od-database-crawler
1
0
Fork 0
mirror of https://github.com/terorie/od-database-crawler.git synced 2025-12-16 08:19:06 +00:00
Code Issues Projects Releases Wiki Activity

Add TCP timeout option

Browse Source
This commit is contained in:
Richard Patel
2018-11-20 03:29:10 +01:00
parent b846498030
commit 86ec78cae1
3 changed files with 29 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
config.yml
View File

@@ -52,9 +52,17 @@ crawl:
# from the site before giving up
retries: 5
# Time before discarding a failed connection attempt
dial_timeout: 10s
# Time before discarding a network request
timeout: 10s
timeout: 30s
# Crawler User-Agent
# If empty, no User-Agent header is sent.
user-agent: "Mozilla/5.0 (X11; od-database-crawler) Gecko/20100101 Firefox/52.0"
# Job buffer size (per task)
# Higher values cause less disk writes
# but require more memory.
job_buffer: 5000