simon987/od-database-crawler
1
0
Fork 0
mirror of https://github.com/terorie/od-database-crawler.git synced 2025-12-13 15:19:03 +00:00
Code Issues Projects Releases Wiki Activity

Add job buffer size parameter

Browse Source
This commit is contained in:
Richard Patel
2018-11-20 03:42:32 +01:00
parent 86ec78cae1
commit 4dbe2aef2b
3 changed files with 29 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
config.go
View File

@@ -25,6 +25,7 @@ var config struct {
AllocStats time.Duration
Verbose bool
PrintHTTP bool
JobBufferSize int
}
const (
@@ -39,6 +40,7 @@ const (
ConfUserAgent = "crawl.user-agent"
ConfDialTimeout = "crawl.dial_timeout"
ConfTimeout = "crawl.timeout"
ConfJobBufferSize = "crawl.job_buffer"
ConfCrawlStats = "output.crawl_stats"
ConfAllocStats = "output.resource_stats"
ConfVerbose = "output.verbose"
@@ -53,6 +55,7 @@ func prepareConfig() {
viper.SetDefault(ConfUserAgent, "")
viper.SetDefault(ConfDialTimeout, 10 * time.Second)
viper.SetDefault(ConfTimeout, 30 * time.Second)
viper.SetDefault(ConfJobBufferSize, 5000)
viper.SetDefault(ConfCrawlStats, 3 * time.Second)
viper.SetDefault(ConfAllocStats, 0)
viper.SetDefault(ConfVerbose, false)
@@ -112,6 +115,8 @@ func readConfig() {
setTimeout(viper.GetDuration(ConfTimeout))
config.JobBufferSize = viper.GetInt(ConfJobBufferSize)
config.CrawlStats = viper.GetDuration(ConfCrawlStats)
config.AllocStats = viper.GetDuration(ConfAllocStats)