simon987/od-database-crawler
1
0
Fork 0
mirror of https://github.com/terorie/od-database-crawler.git synced 2025-12-13 15:19:03 +00:00
Code Issues Projects Releases Wiki Activity

Use fasthttp.PipelineClient

Browse Source
This commit is contained in:
Richard Patel
2019-02-09 16:46:36 +01:00
parent b18b70f798
commit d69cd4400e
5 changed files with 46 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
worker.go
View File

@@ -3,6 +3,7 @@ package main
import (
"github.com/beeker1121/goque"
"github.com/sirupsen/logrus"
"github.com/valyala/fasthttp"
"math"
"sort"
"strings"
@@ -18,6 +19,11 @@ type WorkerContext struct {
Queue *BufferedQueue
lastRateLimit time.Time
numRateLimits int
client *fasthttp.PipelineClient
}
func (w *WorkerContext) Prepare() {
w.client = newHTTPClient(&w.OD.BaseUri)
}
func (w *WorkerContext) Worker(results chan<- File) {
@@ -91,7 +97,7 @@ func (w *WorkerContext) DoJob(job *Job, f *File) (newJobs []Job, err error) {
if len(job.Uri.Path) == 0 { return }
if job.Uri.Path[len(job.Uri.Path)-1] == '/' {
// Load directory
links, err := GetDir(job, f)
links, err := w.GetDir(job, f)
if err != nil {
if !isErrSilent(err) {
logrus.WithError(err).
@@ -141,7 +147,7 @@ func (w *WorkerContext) DoJob(job *Job, f *File) (newJobs []Job, err error) {
}
} else {
// Load file
err := GetFile(job.Uri, f)
err := w.GetFile(job.Uri, f)
if err != nil {
if !isErrSilent(err) {
logrus.WithError(err).