From 46c0e0bd3231267b5af4dcb52fd946bfcd0e3d12 Mon Sep 17 00:00:00 2001
From: Richard Patel <terorie@alphakevin.club>
Date: Sun, 3 Feb 2019 03:35:09 +0100
Subject: [PATCH] Smarter HTTP error handling

---
 errors.go | 28 ++++++++++++++++++++++++++++
 worker.go | 15 ++++++---------
 2 files changed, 34 insertions(+), 9 deletions(-)

diff --git a/errors.go b/errors.go
index 9ef0157..e4eacb5 100644
--- a/errors.go
+++ b/errors.go
@@ -3,6 +3,8 @@ package main
 import (
 	"errors"
 	"fmt"
+	"github.com/valyala/fasthttp"
+	"net"
 )
 
 var ErrRateLimit = errors.New("too many requests")
@@ -15,3 +17,29 @@ type HttpError struct {
 func (e HttpError) Error() string {
 	return fmt.Sprintf("http status %d", e.code)
 }
+
+func shouldRetry(err error) bool {
+	// HTTP errors
+	if httpErr, ok := err.(*HttpError); ok {
+		switch httpErr.code {
+		case fasthttp.StatusTooManyRequests:
+			return true
+		default:
+			// Don't retry HTTP error codes
+			return false
+		}
+	}
+
+	if dnsError, ok := err.(*net.DNSError); ok {
+		// Don't retry permanent DNS errors
+		return dnsError.IsTemporary
+	}
+
+	if netErr, ok := err.(*net.OpError); ok {
+		// Don't retry permanent network errors
+		return netErr.Temporary()
+	}
+
+	// Retry by default
+	return true
+}
diff --git a/worker.go b/worker.go
index 1ec632a..118e28f 100644
--- a/worker.go
+++ b/worker.go
@@ -3,7 +3,6 @@ package main
 import (
 	"github.com/beeker1121/goque"
 	"github.com/sirupsen/logrus"
-	"github.com/valyala/fasthttp"
 	"math"
 	"sort"
 	"strings"
@@ -55,14 +54,12 @@ func (w *WorkerContext) step(results chan<- File, job Job) {
 	if err != nil {
 		job.Fails++
 
-		if httpErr, ok := err.(*HttpError); ok {
-			switch httpErr.code {
-			case fasthttp.StatusTooManyRequests:
-				err = ErrRateLimit
-			default:
-				// Don't retry HTTP error codes
-				return
-			}
+		if !shouldRetry(err) {
+			atomic.AddUint64(&totalAborted, 1)
+			logrus.WithField("url", job.UriStr).
+				WithError(err).
+				Error("Giving up after failure")
+			return
 		}
 
 		if job.Fails > config.Retries {