simon987/od-database-crawler
1
0
Fork 0
mirror of https://github.com/terorie/od-database-crawler.git synced 2025-04-10 14:06:42 +00:00
Code Issues Projects Releases Wiki Activity

Minor cleanup

Browse Source
This commit is contained in:
Richard Patel 2018-12-18 15:31:33 +01:00
parent 4b8275c7bf
commit b244cdae80
No known key found for this signature in database
GPG Key ID: C268B2BBDA2ABECB
3 changed files with 11 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
config.go
View File

@ -21,8 +21,6 @@ var config struct {
Workers int Workers int
UserAgent string UserAgent string
Tasks int32 Tasks int32
CrawlStats time.Duration
AllocStats time.Duration
Verbose bool Verbose bool
PrintHTTP bool PrintHTTP bool
JobBufferSize int JobBufferSize int
@ -125,10 +123,6 @@ func readConfig() {
config.JobBufferSize = viper.GetInt(ConfJobBufferSize) config.JobBufferSize = viper.GetInt(ConfJobBufferSize)
config.CrawlStats = viper.GetDuration(ConfCrawlStats)
config.AllocStats = viper.GetDuration(ConfAllocStats)
config.Verbose = viper.GetBool(ConfVerbose) config.Verbose = viper.GetBool(ConfVerbose)
if config.Verbose { if config.Verbose {
logrus.SetLevel(logrus.DebugLevel) logrus.SetLevel(logrus.DebugLevel)

14
stats.go
View File

@ -3,6 +3,7 @@ package main
import ( import (
"context" "context"
"github.com/sirupsen/logrus" "github.com/sirupsen/logrus"
"github.com/spf13/viper"
"math" "math"
"runtime" "runtime"
"sync/atomic" "sync/atomic"
@ -19,11 +20,14 @@ func Stats(c context.Context) {
var crawlTicker <-chan time.Time var crawlTicker <-chan time.Time
var allocTicker <-chan time.Time var allocTicker <-chan time.Time
if config.CrawlStats != 0 { crawlInterval := viper.GetDuration(ConfCrawlStats)
crawlTicker = time.NewTicker(config.CrawlStats).C allocInterval := viper.GetDuration(ConfAllocStats)
if crawlInterval != 0 {
crawlTicker = time.Tick(crawlInterval)
} }
if config.AllocStats != 0 { if allocInterval != 0 {
allocTicker = time.NewTicker(config.AllocStats).C allocTicker = time.Tick(allocInterval)
} }
for { for {
@ -32,7 +36,7 @@ func Stats(c context.Context) {
startedNow := atomic.LoadUint64(&totalStarted) startedNow := atomic.LoadUint64(&totalStarted)
perSecond := float64(startedNow - startedLast) / perSecond := float64(startedNow - startedLast) /
config.CrawlStats.Seconds() crawlInterval.Seconds()
// Round to .5 // Round to .5
perSecond *= 2 perSecond *= 2

4
worker.go
View File

@ -42,7 +42,7 @@ func (w *WorkerContext) Worker(results chan<- File) {
} }
func (w *WorkerContext) step(results chan<- File, job Job) { func (w *WorkerContext) step(results chan<- File, job Job) {
defer w.finishJob(&job) defer w.finishJob()
var f File var f File
@ -175,7 +175,7 @@ func (w *WorkerContext) queueJob(job Job) {
} }
} }
func (w *WorkerContext) finishJob(job *Job) { func (w *WorkerContext) finishJob() {
w.OD.Wait.Done() w.OD.Wait.Done()
} }