diff --git a/config.go b/config.go index a9f5771..6576126 100644 --- a/config.go +++ b/config.go @@ -21,8 +21,6 @@ var config struct { Workers int UserAgent string Tasks int32 - CrawlStats time.Duration - AllocStats time.Duration Verbose bool PrintHTTP bool JobBufferSize int @@ -125,10 +123,6 @@ func readConfig() { config.JobBufferSize = viper.GetInt(ConfJobBufferSize) - config.CrawlStats = viper.GetDuration(ConfCrawlStats) - - config.AllocStats = viper.GetDuration(ConfAllocStats) - config.Verbose = viper.GetBool(ConfVerbose) if config.Verbose { logrus.SetLevel(logrus.DebugLevel) diff --git a/stats.go b/stats.go index 80c149e..5e7b688 100644 --- a/stats.go +++ b/stats.go @@ -3,6 +3,7 @@ package main import ( "context" "github.com/sirupsen/logrus" + "github.com/spf13/viper" "math" "runtime" "sync/atomic" @@ -19,11 +20,14 @@ func Stats(c context.Context) { var crawlTicker <-chan time.Time var allocTicker <-chan time.Time - if config.CrawlStats != 0 { - crawlTicker = time.NewTicker(config.CrawlStats).C + crawlInterval := viper.GetDuration(ConfCrawlStats) + allocInterval := viper.GetDuration(ConfAllocStats) + + if crawlInterval != 0 { + crawlTicker = time.Tick(crawlInterval) } - if config.AllocStats != 0 { - allocTicker = time.NewTicker(config.AllocStats).C + if allocInterval != 0 { + allocTicker = time.Tick(allocInterval) } for { @@ -32,7 +36,7 @@ func Stats(c context.Context) { startedNow := atomic.LoadUint64(&totalStarted) perSecond := float64(startedNow - startedLast) / - config.CrawlStats.Seconds() + crawlInterval.Seconds() // Round to .5 perSecond *= 2 diff --git a/worker.go b/worker.go index 2d6e39a..1ec632a 100644 --- a/worker.go +++ b/worker.go @@ -42,7 +42,7 @@ func (w *WorkerContext) Worker(results chan<- File) { } func (w *WorkerContext) step(results chan<- File, job Job) { - defer w.finishJob(&job) + defer w.finishJob() var f File @@ -175,7 +175,7 @@ func (w *WorkerContext) queueJob(job Job) { } } -func (w *WorkerContext) finishJob(job *Job) { +func (w *WorkerContext) finishJob() { w.OD.Wait.Done() }