mirror of
https://github.com/terorie/od-database-crawler.git
synced 2025-12-13 15:19:03 +00:00
Add stats interval parameter
This commit is contained in:
25
scheduler.go
25
scheduler.go
@@ -2,10 +2,7 @@ package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"github.com/sirupsen/logrus"
|
||||
"net/url"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
)
|
||||
|
||||
type Job struct {
|
||||
@@ -49,28 +46,6 @@ func (r *RemoteDir) Watch() {
|
||||
r.Wait.Wait()
|
||||
}
|
||||
|
||||
func Stats(c context.Context) {
|
||||
var startedLast uint64 = 0
|
||||
ticker := time.NewTicker(time.Second).C
|
||||
for {
|
||||
select {
|
||||
case <-ticker:
|
||||
startedNow := atomic.LoadUint64(&totalStarted)
|
||||
logrus.WithFields(logrus.Fields{
|
||||
"per_second": startedNow - startedLast,
|
||||
"done": atomic.LoadUint64(&totalDone),
|
||||
"retries": atomic.LoadUint64(&totalRetries),
|
||||
"aborted": atomic.LoadUint64(&totalAborted),
|
||||
}).Info("Stats")
|
||||
|
||||
startedLast = startedNow
|
||||
|
||||
case <-c.Done():
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func makeJobBuffer() (chan<- Job, <-chan Job) {
|
||||
in := make(chan Job)
|
||||
out := make(chan Job)
|
||||
|
||||
Reference in New Issue
Block a user