mirror of
https://github.com/terorie/od-database-crawler.git
synced 2025-04-16 08:56:44 +00:00
Add stats interval parameter
This commit is contained in:
parent
79f540bf29
commit
a507110787
@ -3,6 +3,7 @@ package main
|
||||
import (
|
||||
"github.com/sirupsen/logrus"
|
||||
"github.com/spf13/viper"
|
||||
"time"
|
||||
)
|
||||
|
||||
var config struct {
|
||||
@ -10,6 +11,7 @@ var config struct {
|
||||
Token string
|
||||
Retries int
|
||||
Workers int
|
||||
StatsInterval time.Duration
|
||||
}
|
||||
|
||||
const (
|
||||
@ -17,11 +19,13 @@ const (
|
||||
ConfToken = "token"
|
||||
ConfRetries = "retries"
|
||||
ConfWorkers = "workers"
|
||||
ConfStatsInterval = "stats_interval"
|
||||
)
|
||||
|
||||
func prepareConfig() {
|
||||
viper.SetDefault(ConfRetries, 3)
|
||||
viper.SetDefault(ConfWorkers, 50)
|
||||
viper.SetDefault(ConfStatsInterval, 3 * time.Second)
|
||||
}
|
||||
|
||||
func readConfig() {
|
||||
@ -51,4 +55,6 @@ func readConfig() {
|
||||
if config.Workers <= 0 {
|
||||
logrus.Fatal("config: illegal value %d for workers!", config.Workers)
|
||||
}
|
||||
|
||||
config.StatsInterval = viper.GetDuration(ConfStatsInterval)
|
||||
}
|
||||
|
@ -1,2 +1,3 @@
|
||||
server_url: localhost:6969
|
||||
token: abc
|
||||
token: abc
|
||||
stats: 5s
|
25
scheduler.go
25
scheduler.go
@ -2,10 +2,7 @@ package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"github.com/sirupsen/logrus"
|
||||
"net/url"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
)
|
||||
|
||||
type Job struct {
|
||||
@ -49,28 +46,6 @@ func (r *RemoteDir) Watch() {
|
||||
r.Wait.Wait()
|
||||
}
|
||||
|
||||
func Stats(c context.Context) {
|
||||
var startedLast uint64 = 0
|
||||
ticker := time.NewTicker(time.Second).C
|
||||
for {
|
||||
select {
|
||||
case <-ticker:
|
||||
startedNow := atomic.LoadUint64(&totalStarted)
|
||||
logrus.WithFields(logrus.Fields{
|
||||
"per_second": startedNow - startedLast,
|
||||
"done": atomic.LoadUint64(&totalDone),
|
||||
"retries": atomic.LoadUint64(&totalRetries),
|
||||
"aborted": atomic.LoadUint64(&totalAborted),
|
||||
}).Info("Stats")
|
||||
|
||||
startedLast = startedNow
|
||||
|
||||
case <-c.Done():
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func makeJobBuffer() (chan<- Job, <-chan Job) {
|
||||
in := make(chan Job)
|
||||
out := make(chan Job)
|
||||
|
45
stats.go
Normal file
45
stats.go
Normal file
@ -0,0 +1,45 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"github.com/sirupsen/logrus"
|
||||
"math"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
)
|
||||
|
||||
var totalStarted uint64
|
||||
var totalDone uint64
|
||||
var totalRetries uint64
|
||||
var totalAborted uint64
|
||||
|
||||
func Stats(c context.Context) {
|
||||
var startedLast uint64 = 0
|
||||
ticker := time.NewTicker(config.StatsInterval).C
|
||||
for {
|
||||
select {
|
||||
case <-ticker:
|
||||
startedNow := atomic.LoadUint64(&totalStarted)
|
||||
|
||||
perSecond := float64(startedNow - startedLast) /
|
||||
config.StatsInterval.Seconds()
|
||||
|
||||
// Round to .5
|
||||
perSecond *= 2
|
||||
perSecond = math.Round(perSecond)
|
||||
perSecond /= 2
|
||||
|
||||
logrus.WithFields(logrus.Fields{
|
||||
"per_second": perSecond,
|
||||
"done": atomic.LoadUint64(&totalDone),
|
||||
"retries": atomic.LoadUint64(&totalRetries),
|
||||
"aborted": atomic.LoadUint64(&totalAborted),
|
||||
}).Info("Stats")
|
||||
|
||||
startedLast = startedNow
|
||||
|
||||
case <-c.Done():
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
Loading…
x
Reference in New Issue
Block a user