simon987/od-database-crawler
1
0
Fork 0
mirror of https://github.com/terorie/od-database-crawler.git synced 2025-12-16 08:19:06 +00:00
Code Issues Projects Releases Wiki Activity

Add log file support

Browse Source
This commit is contained in:
Richard Patel
2018-11-18 14:46:52 +01:00
parent dc4e4212a0
commit f3620262fc
4 changed files with 51 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
config.yml
View File

@@ -23,13 +23,20 @@ server:
output:
# Crawl statistics
crawl_stats: 1s
# CPU/RAM/Job queue stats
resource_stats: 10s
# More output? (Every listed dir)
verbose: false
# Print HTTP errors (Super spammy)
http: false
# Log file
# If empty, no log file is created.
log: crawler.log
# Crawler settings
crawl:
# Number of sites that can be processed at once
@@ -49,4 +56,5 @@ crawl:
timeout: 10s
# Crawler User-Agent
# If empty, no User-Agent header is sent.
user-agent: "Mozilla/5.0 (X11; od-database-crawler) Gecko/20100101 Firefox/52.0"