simon987/od-database-crawler
1
0
Fork 0
mirror of https://github.com/terorie/od-database-crawler.git synced 2025-12-13 15:19:03 +00:00
Code Issues Projects Releases Wiki Activity

Timeout and results saving

Browse Source
This commit is contained in:
Richard Patel
2018-11-15 20:14:31 +01:00
parent a268c6dbcf
commit ffde1a9e5d
7 changed files with 86 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
crawl.go
View File

@@ -3,7 +3,6 @@ package main
import (
"bytes"
"fmt"
"github.com/sirupsen/logrus"
"github.com/terorie/oddb-go/ds/redblackhash"
"github.com/terorie/oddb-go/fasturl"
"github.com/valyala/fasthttp"
@@ -28,13 +27,10 @@ func GetDir(j *Job, f *File) (links []fasturl.URL, err error) {
res := fasthttp.AcquireResponse()
defer fasthttp.ReleaseResponse(res)
err = client.Do(req, res)
err = client.DoTimeout(req, res, config.Timeout)
fasthttp.ReleaseRequest(req)
if err != nil {
logrus.Error(err)
return
}
if err != nil { return }
err = checkStatusCode(res.StatusCode())
if err != nil { return }
@@ -129,7 +125,7 @@ func GetFile(u fasturl.URL, f *File) (err error) {
res.SkipBody = true
defer fasthttp.ReleaseResponse(res)
err = client.Do(req, res)
err = client.DoTimeout(req, res, config.Timeout)
fasthttp.ReleaseRequest(req)
if err != nil { return }