mirror of
https://github.com/terorie/od-database-crawler.git
synced 2025-04-18 01:46:43 +00:00
39 lines
860 B
YAML
39 lines
860 B
YAML
# OD-Database server settings
|
|
server:
|
|
# Connection URL
|
|
url: http://od-db.mine.terorie.com/api
|
|
# Server auth token
|
|
token:
|
|
# Upload chunk size
|
|
# If the value is too high, the upload fails.
|
|
upload_chunk: 1 MB
|
|
|
|
# Log output settings
|
|
output:
|
|
# Crawl statistics
|
|
crawl_stats: 1s
|
|
# CPU/RAM/Job queue stats
|
|
resource_stats: 10s
|
|
# More output? (Every listed dir)
|
|
verbose: false
|
|
# Print HTTP errors (Super spammy)
|
|
http: false
|
|
|
|
# Crawler settings
|
|
crawl:
|
|
# Number of sites that can be processed at once
|
|
tasks: 100
|
|
|
|
# Number of connections per site
|
|
# Please be careful with this setting!
|
|
# The crawler fires fast and more than
|
|
# ten connections can overwhelm a server.
|
|
connections: 10
|
|
|
|
# How often to retry getting data
|
|
# from the site before giving up
|
|
retries: 5
|
|
|
|
# Time before discarding a network request
|
|
timeout: 10s
|