simon987/od-database-crawler
1
0
Fork 0
mirror of https://github.com/terorie/od-database-crawler.git synced 2025-04-20 02:46:43 +00:00
Code Issues Projects Releases Wiki Activity

Fix segfault

Browse Source
This commit is contained in:
Richard Patel 2019-02-09 16:50:45 +01:00
parent d69cd4400e
commit c72f4ba475
No known key found for this signature in database
GPG Key ID: C268B2BBDA2ABECB
1 changed files with 4 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
scheduler.go
View File

@ -22,17 +22,16 @@ func Schedule(c context.Context, remotes <-chan *OD) {
go Stats(c) go Stats(c)
for remote := range remotes { for remote := range remotes {
// Create HTTP client
remote.WCtx.OD = remote
remote.WCtx.Prepare()
logrus.WithField("url", remote.BaseUri.String()). logrus.WithField("url", remote.BaseUri.String()).
Info("Starting crawler") Info("Starting crawler")
// Create HTTP client
remote.WCtx.Prepare()
// Collect results // Collect results
results := make(chan File) results := make(chan File)
remote.WCtx.OD = remote
// Get queue path // Get queue path
queuePath := path.Join("queue", fmt.Sprintf("%d", remote.Task.WebsiteId)) queuePath := path.Join("queue", fmt.Sprintf("%d", remote.Task.WebsiteId))