simon987/od-database-crawler
1
0
Fork 0
mirror of https://github.com/terorie/od-database-crawler.git synced 2025-04-04 06:52:59 +00:00
Code Issues Projects Releases Wiki Activity

Ignore FTP URLs

Browse Source
This commit is contained in:
Richard Patel 2018-11-17 04:50:59 +01:00
parent 7fdffff58f
commit f41198b00c
No known key found for this signature in database
GPG Key ID: C268B2BBDA2ABECB
2 changed files with 8 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
fasturl/url.go
View File

@ -33,6 +33,8 @@ var Schemes = [SchemeCount]string {
"https",
}
var ErrUnknownScheme = errors.New("unknown protocol scheme")
// Error reports an error and the operation and URL that caused it.
type Error struct {
Op string
@ -353,7 +355,7 @@ func getscheme(rawurl string) (scheme Scheme, path string, err error) {
case "https":
scheme = SchemeHTTPS
default:
return SchemeInvalid, "", errors.New("unknown protocol scheme")
return SchemeInvalid, "", ErrUnknownScheme
}
path = rawurl[i+1:]

5
main.go
View File

@ -62,6 +62,11 @@ func cmdBase(clic *cli.Context) error {
case <-ticker.C:
t, err := FetchTask()
if err != nil {
if err == fasturl.ErrUnknownScheme {
// Not an error
err = nil
continue
}
logrus.WithError(err).
Error("Failed getting new task")
time.Sleep(30 * time.Second)