mirror of
https://github.com/terorie/od-database-crawler.git
synced 2025-04-10 14:06:42 +00:00
61 lines
1.3 KiB
YAML
61 lines
1.3 KiB
YAML
# OD-Database server settings
|
|
server:
|
|
# Connection URL
|
|
url: http://od-db.mine.terorie.com/api
|
|
|
|
# Server auth token
|
|
token:
|
|
|
|
# Request timeout
|
|
timeout: 60s
|
|
|
|
# Recheck interval
|
|
# The crawler periodically asks the server
|
|
# for new jobs. Sets the minimum wait time
|
|
# between /task/get requests to the server.
|
|
recheck: 1s
|
|
|
|
# Upload chunk size
|
|
# If the value is too high, the upload fails.
|
|
upload_chunk: 1 MB
|
|
|
|
# Log output settings
|
|
output:
|
|
# Crawl statistics
|
|
crawl_stats: 1s
|
|
|
|
# CPU/RAM/Job queue stats
|
|
resource_stats: 10s
|
|
|
|
# More output? (Every listed dir)
|
|
verbose: false
|
|
|
|
# Print HTTP errors (Super spammy)
|
|
http: false
|
|
|
|
# Log file
|
|
# If empty, no log file is created.
|
|
log: crawler.log
|
|
|
|
# Crawler settings
|
|
crawl:
|
|
# Number of sites that can be processed at once
|
|
tasks: 100
|
|
|
|
# Number of connections per site
|
|
# Please be careful with this setting!
|
|
# The crawler fires fast and more than
|
|
# ten connections can overwhelm a server.
|
|
connections: 10
|
|
|
|
# How often to retry getting data
|
|
# from the site before giving up
|
|
retries: 5
|
|
|
|
# Time before discarding a network request
|
|
timeout: 10s
|
|
|
|
# Crawler User-Agent
|
|
# If empty, no User-Agent header is sent.
|
|
user-agent: "Mozilla/5.0 (X11; od-database-crawler) Gecko/20100101 Firefox/52.0"
|