simon987/od-database-crawler
1
0
Fork 0
mirror of https://github.com/terorie/od-database-crawler.git synced 2025-04-18 18:06:45 +00:00
Code Issues Projects Releases Wiki Activity

Remove too many crawler block

Browse Source 
More logging
This commit is contained in:
Richard Patel 2018-10-28 18:17:04 +01:00
parent bfd7302be8
commit fa37d45378
No known key found for this signature in database
GPG Key ID: C268B2BBDA2ABECB
1 changed files with 6 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
scheduler.go
View File

@ -2,8 +2,8 @@ package main
import (
"context"
"github.com/sirupsen/logrus"
"sync/atomic"
"time"
)
var activeTasks int32
@ -17,14 +17,8 @@ func Schedule(c context.Context, remotes <-chan *OD) {
return
case remote := <-remotes:
for atomic.LoadInt32(&activeTasks) > config.Tasks {
select {
case <-time.After(time.Second):
break
case <-c.Done():
return
}
}
logrus.WithField("url", remote.BaseUri.String()).
Info("Starting crawler")
// Spawn workers
remote.WCtx.in, remote.WCtx.out = makeJobBuffer(c)
@ -53,6 +47,9 @@ func (r *OD) Watch() {
r.Wait.Wait()
close(r.WCtx.in)
atomic.AddInt32(&activeTasks, -1)
logrus.WithField("url", r.BaseUri.String()).
Info("Crawler finished")
}
func makeJobBuffer(c context.Context) (chan<- Job, <-chan Job) {