simon987/od-database-crawler
1
0
Fork 0
mirror of https://github.com/terorie/od-database-crawler.git synced 2025-12-13 15:19:03 +00:00
Code Issues Projects Releases Wiki Activity

Handle HTTP statuses

Browse Source
This commit is contained in:
Richard Patel
2018-10-28 03:22:25 +01:00
parent 1c33346f45
commit 4ea5f8a410
2 changed files with 25 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
worker.go
View File

@@ -31,11 +31,6 @@ func (w WorkerContext) step(job Job) {
if err != nil {
job.Fails++
logrus.WithFields(logrus.Fields{
"error": err.Error(),
"url": job.UriStr,
}).Warningf("Crawl error: %s", err)
if job.Fails > config.Retries {
atomic.AddUint64(&totalAborted, 1)
logrus.WithField("url", job.UriStr).