simon987/od-database-crawler
1
0
Fork 0
mirror of https://github.com/terorie/od-database-crawler.git synced 2025-04-10 05:56:42 +00:00
Code Issues Projects Releases Wiki Activity

Fix queue_count in log

Browse Source
This commit is contained in:
Richard Patel 2018-11-20 02:49:03 +01:00
parent 85d2aac9d4
commit 4f3140a39f
No known key found for this signature in database
GPG Key ID: C268B2BBDA2ABECB
3 changed files with 7 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
queue.go
View File

@ -3,6 +3,7 @@ package main
import (
"github.com/beeker1121/goque"
"sync"
"sync/atomic"
)
const (
@ -11,7 +12,6 @@ const (
type BufferedQueue struct {
q *goque.Queue
inBuf []Job
outBuf []Job
m sync.Mutex
}
@ -24,6 +24,7 @@ func OpenQueue(dataDir string) (bq *BufferedQueue, err error) {
}
func (q *BufferedQueue) Enqueue(job *Job) error {
atomic.AddInt64(&totalQueued, 1)
if q.directEnqueue(job) {
return nil
}
@ -36,6 +37,7 @@ func (q *BufferedQueue) Enqueue(job *Job) error {
func (q *BufferedQueue) Dequeue() (job Job, err error) {
if q.directDequeue(&job) {
atomic.AddInt64(&totalQueued, -1)
return job, nil
}
@ -43,6 +45,8 @@ func (q *BufferedQueue) Dequeue() (job Job, err error) {
item, err = q.q.Dequeue()
if err != nil { return }
atomic.AddInt64(&totalQueued, -1)
var gob JobGob
err = item.ToObject(&gob)
if err != nil { return }

2
scheduler.go
View File

@ -16,7 +16,7 @@ import (
var activeTasksLock sync.Mutex
var activeTasks = make(map[uint64]bool)
var numActiveTasks int32
var totalBuffered int64
var totalQueued int64
func Schedule(c context.Context, remotes <-chan *OD) {
go Stats(c)

2
stats.go
View File

@ -57,7 +57,7 @@ func Stats(c context.Context) {
runtime.ReadMemStats(&mem)
logrus.WithFields(logrus.Fields{
"queue_count": atomic.LoadInt64(&totalBuffered),
"queue_count": atomic.LoadInt64(&totalQueued),
"heap": FormatByteCount(mem.Alloc),
"objects": mem.HeapObjects,
"num_gc": mem.NumGC,