simon987/od-database-crawler
1
0
Fork 0
mirror of https://github.com/terorie/od-database-crawler.git synced 2025-12-13 15:19:03 +00:00
Code Issues Projects Releases Wiki Activity

Timeout and results saving

Browse Source
This commit is contained in:
Richard Patel
2018-11-15 20:14:31 +01:00
parent a268c6dbcf
commit ffde1a9e5d
7 changed files with 86 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
config.go
View File

@@ -13,6 +13,7 @@ var config struct {
Token string
Retries int
Workers int
Timeout time.Duration
Tasks int32
CrawlStats time.Duration
AllocStats time.Duration
@@ -25,6 +26,7 @@ const (
ConfTasks = "crawl.tasks"
ConfRetries = "crawl.retries"
ConfWorkers = "crawl.connections"
ConfTimeout = "crawl.timeout"
ConfCrawlStats = "output.crawl_stats"
ConfAllocStats = "output.resource_stats"
ConfVerbose = "output.verbose"
@@ -34,6 +36,7 @@ func prepareConfig() {
viper.SetDefault(ConfRetries, 5)
viper.SetDefault(ConfWorkers, 2)
viper.SetDefault(ConfTasks, 3)
viper.SetDefault(ConfTimeout, 10 * time.Second)
viper.SetDefault(ConfCrawlStats, 3 * time.Second)
viper.SetDefault(ConfAllocStats, 0)
viper.SetDefault(ConfVerbose, false)
@@ -73,6 +76,8 @@ func readConfig() {
configOOB(ConfTasks, int(config.Tasks))
}
config.Timeout = viper.GetDuration(ConfTimeout)
config.CrawlStats = viper.GetDuration(ConfCrawlStats)
config.AllocStats = viper.GetDuration(ConfAllocStats)