simon987/od-database-crawler
1
0
Fork 0
mirror of https://github.com/terorie/od-database-crawler.git synced 2025-04-19 10:26:43 +00:00
Code Issues Projects Releases Wiki Activity

Reduce WaitQueue usage

Browse Source
This commit is contained in:
Richard Patel 2018-11-12 00:38:22 +01:00
parent 4c071171eb
commit a268c6dbcf
No known key found for this signature in database
GPG Key ID: C268B2BBDA2ABECB
2 changed files with 2 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
scheduler.go
View File

@ -35,7 +35,6 @@ func Schedule(c context.Context, remotes <-chan *OD) {
UriStr: remote.BaseUri.String(), UriStr: remote.BaseUri.String(),
Fails: 0, Fails: 0,
}) })
globalWait.Done()
// Upload result when ready // Upload result when ready
go remote.Watch() go remote.Watch()
@ -51,6 +50,8 @@ func (r *OD) Watch() {
logrus.WithField("url", r.BaseUri.String()). logrus.WithField("url", r.BaseUri.String()).
Info("Crawler finished") Info("Crawler finished")
globalWait.Done()
} }
func makeJobBuffer(c context.Context) (chan<- Job, <-chan Job) { func makeJobBuffer(c context.Context) (chan<- Job, <-chan Job) {

2
worker.go
View File

@ -134,7 +134,6 @@ func DoJob(job *Job, f *File) (newJobs []Job, err error) {
func (w WorkerContext) queueJob(job Job) { func (w WorkerContext) queueJob(job Job) {
job.OD.Wait.Add(1) job.OD.Wait.Add(1)
globalWait.Add(1)
if w.numRateLimits > 0 { if w.numRateLimits > 0 {
if time.Since(w.lastRateLimit) > 5 * time.Second { if time.Since(w.lastRateLimit) > 5 * time.Second {
@ -151,5 +150,4 @@ func (w WorkerContext) queueJob(job Job) {
func (w WorkerContext) finishJob(job *Job) { func (w WorkerContext) finishJob(job *Job) {
job.OD.Wait.Done() job.OD.Wait.Done()
globalWait.Done()
} }