diff --git a/scheduler.go b/scheduler.go index a87bb27..9febcd5 100644 --- a/scheduler.go +++ b/scheduler.go @@ -22,17 +22,16 @@ func Schedule(c context.Context, remotes <-chan *OD) { go Stats(c) for remote := range remotes { + // Create HTTP client + remote.WCtx.OD = remote + remote.WCtx.Prepare() + logrus.WithField("url", remote.BaseUri.String()). Info("Starting crawler") - // Create HTTP client - remote.WCtx.Prepare() - // Collect results results := make(chan File) - remote.WCtx.OD = remote - // Get queue path queuePath := path.Join("queue", fmt.Sprintf("%d", remote.Task.WebsiteId))