diff --git a/scheduler.go b/scheduler.go index c6b183d..eb0b526 100644 --- a/scheduler.go +++ b/scheduler.go @@ -35,7 +35,6 @@ func Schedule(c context.Context, remotes <-chan *OD) { UriStr: remote.BaseUri.String(), Fails: 0, }) - globalWait.Done() // Upload result when ready go remote.Watch() @@ -51,6 +50,8 @@ func (r *OD) Watch() { logrus.WithField("url", r.BaseUri.String()). Info("Crawler finished") + + globalWait.Done() } func makeJobBuffer(c context.Context) (chan<- Job, <-chan Job) { diff --git a/worker.go b/worker.go index 687f5f8..cafc393 100644 --- a/worker.go +++ b/worker.go @@ -134,7 +134,6 @@ func DoJob(job *Job, f *File) (newJobs []Job, err error) { func (w WorkerContext) queueJob(job Job) { job.OD.Wait.Add(1) - globalWait.Add(1) if w.numRateLimits > 0 { if time.Since(w.lastRateLimit) > 5 * time.Second { @@ -151,5 +150,4 @@ func (w WorkerContext) queueJob(job Job) { func (w WorkerContext) finishJob(job *Job) { job.OD.Wait.Done() - globalWait.Done() }