simon987/od-database-crawler
1
0
Fork 0
mirror of https://github.com/terorie/od-database-crawler.git synced 2025-04-19 02:16:43 +00:00
Code Issues Projects Releases Wiki Activity

Fix queue_count in log

Browse Source
This commit is contained in:
Richard Patel 2018-11-20 02:49:03 +01:00
parent 85d2aac9d4
commit 4f3140a39f
No known key found for this signature in database
GPG Key ID: C268B2BBDA2ABECB
3 changed files with 7 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
queue.go
View File

@ -3,6 +3,7 @@ package main
import ( import (
"github.com/beeker1121/goque" "github.com/beeker1121/goque"
"sync" "sync"
"sync/atomic"
) )
const ( const (
@ -11,7 +12,6 @@ const (
type BufferedQueue struct { type BufferedQueue struct {
q *goque.Queue q *goque.Queue
inBuf []Job
outBuf []Job outBuf []Job
m sync.Mutex m sync.Mutex
} }
@ -24,6 +24,7 @@ func OpenQueue(dataDir string) (bq *BufferedQueue, err error) {
} }
func (q *BufferedQueue) Enqueue(job *Job) error { func (q *BufferedQueue) Enqueue(job *Job) error {
atomic.AddInt64(&totalQueued, 1)
if q.directEnqueue(job) { if q.directEnqueue(job) {
return nil return nil
} }
@ -36,6 +37,7 @@ func (q *BufferedQueue) Enqueue(job *Job) error {
func (q *BufferedQueue) Dequeue() (job Job, err error) { func (q *BufferedQueue) Dequeue() (job Job, err error) {
if q.directDequeue(&job) { if q.directDequeue(&job) {
atomic.AddInt64(&totalQueued, -1)
return job, nil return job, nil
} }
@ -43,6 +45,8 @@ func (q *BufferedQueue) Dequeue() (job Job, err error) {
item, err = q.q.Dequeue() item, err = q.q.Dequeue()
if err != nil { return } if err != nil { return }
atomic.AddInt64(&totalQueued, -1)
var gob JobGob var gob JobGob
err = item.ToObject(&gob) err = item.ToObject(&gob)
if err != nil { return } if err != nil { return }

2
scheduler.go
View File

@ -16,7 +16,7 @@ import (
var activeTasksLock sync.Mutex var activeTasksLock sync.Mutex
var activeTasks = make(map[uint64]bool) var activeTasks = make(map[uint64]bool)
var numActiveTasks int32 var numActiveTasks int32
var totalBuffered int64 var totalQueued int64
func Schedule(c context.Context, remotes <-chan *OD) { func Schedule(c context.Context, remotes <-chan *OD) {
go Stats(c) go Stats(c)

2
stats.go
View File

@ -57,7 +57,7 @@ func Stats(c context.Context) {
runtime.ReadMemStats(&mem) runtime.ReadMemStats(&mem)
logrus.WithFields(logrus.Fields{ logrus.WithFields(logrus.Fields{
"queue_count": atomic.LoadInt64(&totalBuffered), "queue_count": atomic.LoadInt64(&totalQueued),
"heap": FormatByteCount(mem.Alloc), "heap": FormatByteCount(mem.Alloc),
"objects": mem.HeapObjects, "objects": mem.HeapObjects,
"num_gc": mem.NumGC, "num_gc": mem.NumGC,