simon987/od-database-crawler
1
0
Fork 0
mirror of https://github.com/terorie/od-database-crawler.git synced 2025-04-18 01:46:43 +00:00
Code Issues Projects Releases Wiki Activity

Ignore all HTTP errors

Browse Source
This commit is contained in:
Richard Patel 2018-11-18 14:25:06 +01:00
parent a71157b4d8
commit 6e6a4edd27
No known key found for this signature in database
GPG Key ID: C268B2BBDA2ABECB
1 changed files with 3 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
worker.go
View File

@ -42,15 +42,11 @@ func (w WorkerContext) step(results chan<- File, job Job) {
if httpErr, ok := err.(*HttpError); ok { if httpErr, ok := err.(*HttpError); ok {
switch httpErr.code { switch httpErr.code {
case
fasthttp.StatusMovedPermanently,
fasthttp.StatusFound,
fasthttp.StatusUnauthorized,
fasthttp.StatusForbidden,
fasthttp.StatusNotFound:
return
case fasthttp.StatusTooManyRequests: case fasthttp.StatusTooManyRequests:
err = ErrRateLimit err = ErrRateLimit
default:
// Don't retry HTTP error codes
return
} }
} }