Don't retry on 401/403

2025-12-13 15:19:03 +00:00 · 2018-10-28 03:47:29 +01:00
parent faad19f121
commit ab5874129f
2 changed files with 27 additions and 20 deletions
--- a/worker.go
+++ b/worker.go
@@ -35,6 +35,11 @@ func (w WorkerContext) step(job Job) {
 	if err != nil {
 		job.Fails++

+		if err == ErrForbidden {
+			// Don't attempt crawling again
+			return
+		}
+
 		if job.Fails > config.Retries {
 			atomic.AddUint64(&totalAborted, 1)
 			logrus.WithField("url", job.UriStr).