simon987/od-database-crawler
1
0
Fork 0
mirror of https://github.com/terorie/od-database-crawler.git synced 2025-12-13 15:19:03 +00:00
Code Issues Projects Releases Wiki Activity

Use fasthttp.PipelineClient

Browse Source
This commit is contained in:
Richard Patel
2019-02-09 16:46:36 +01:00
parent b18b70f798
commit d69cd4400e
5 changed files with 46 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
scheduler.go
View File

@@ -25,6 +25,9 @@ func Schedule(c context.Context, remotes <-chan *OD) {
logrus.WithField("url", remote.BaseUri.String()).
Info("Starting crawler")
// Create HTTP client
remote.WCtx.Prepare()
// Collect results
results := make(chan File)