simon987/od-database-crawler
1
0
Fork 0
mirror of https://github.com/terorie/od-database-crawler.git synced 2025-04-19 18:36:43 +00:00
Code Issues Projects Releases Wiki Activity

Better config

Browse Source
This commit is contained in:
Richard Patel 2018-10-28 14:19:09 +01:00
parent ddfdce9d0f
commit c196b6f20d
No known key found for this signature in database
GPG Key ID: C268B2BBDA2ABECB
2 changed files with 51 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
config.go
View File

@ -11,22 +11,25 @@ var config struct {
Token string Token string
Retries int Retries int
Workers int Workers int
Tasks int
StatsInterval time.Duration StatsInterval time.Duration
Verbose bool Verbose bool
} }
const ( const (
ConfServerUrl = "server_url" ConfServerUrl = "server.url"
ConfToken = "token" ConfToken = "server.token"
ConfRetries = "retries" ConfTasks = "crawl.tasks"
ConfWorkers = "workers" ConfRetries = "crawl.retries"
ConfStatsInterval = "stats_interval" ConfWorkers = "crawl.connections"
ConfVerbose = "verbose" ConfStatsInterval = "output.stats_interval"
ConfVerbose = "output.verbose"
) )
func prepareConfig() { func prepareConfig() {
viper.SetDefault(ConfRetries, 3) viper.SetDefault(ConfRetries, 5)
viper.SetDefault(ConfWorkers, 50) viper.SetDefault(ConfWorkers, 2)
viper.SetDefault(ConfTasks, 3)
viper.SetDefault(ConfStatsInterval, 3 * time.Second) viper.SetDefault(ConfStatsInterval, 3 * time.Second)
viper.SetDefault(ConfVerbose, false) viper.SetDefault(ConfVerbose, false)
} }
@ -41,12 +44,12 @@ func readConfig() {
config.ServerUrl = viper.GetString(ConfServerUrl) config.ServerUrl = viper.GetString(ConfServerUrl)
if config.ServerUrl == "" { if config.ServerUrl == "" {
logrus.Fatal("config: server_url not set!") configMissing(ConfServerUrl)
} }
config.Token = viper.GetString(ConfToken) config.Token = viper.GetString(ConfToken)
if config.Token == "" { if config.Token == "" {
logrus.Fatal("config: token not set!") configMissing(ConfToken)
} }
config.Retries = viper.GetInt(ConfRetries) config.Retries = viper.GetInt(ConfRetries)
@ -56,7 +59,12 @@ func readConfig() {
config.Workers = viper.GetInt(ConfWorkers) config.Workers = viper.GetInt(ConfWorkers)
if config.Workers <= 0 { if config.Workers <= 0 {
logrus.Fatal("config: illegal value %d for workers!", config.Workers) configOOB(ConfWorkers, config.Workers)
}
config.Tasks = viper.GetInt(ConfTasks)
if config.Tasks <= 0 {
configOOB(ConfTasks, config.Tasks)
} }
config.StatsInterval = viper.GetDuration(ConfStatsInterval) config.StatsInterval = viper.GetDuration(ConfStatsInterval)
@ -66,3 +74,11 @@ func readConfig() {
logrus.SetLevel(logrus.DebugLevel) logrus.SetLevel(logrus.DebugLevel)
} }
} }
func configMissing(key string) {
logrus.Fatalf("config: %s not set!", key)
}
func configOOB(key string, v int) {
logrus.Fatal("config: illegal value %d for %key!", v, key)
}

30
config.yml
View File

@ -1,6 +1,24 @@
server_url: localhost:6969 # OD-Database server settings
token: abc server:
stats_interval: 1s # Connection URL
verbose: true url: localhost:6969
retries: 5 # Server auth token
workers: 80 token:
# Log output settings
output:
# Statistics printing interval
stats_interval: 1s
# More output? (Every listed dir)
verbose: false
# Crawler settings
crawl:
# Number of sites that can be
# processed at once
tasks: 3
# Number of connections per site
connections: 2
# How often to retry getting data
# from the site before giving up
retries: 5