simon987/od-database-crawler
1
0
Fork 0
mirror of https://github.com/terorie/od-database-crawler.git synced 2025-04-10 05:56:42 +00:00
Code Issues Projects Releases Wiki Activity

Ignore FTP URLs

Browse Source
This commit is contained in:
Richard Patel 2018-11-17 04:50:59 +01:00
parent 7fdffff58f
commit f41198b00c
No known key found for this signature in database
GPG Key ID: C268B2BBDA2ABECB
2 changed files with 8 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
fasturl/url.go
View File

@ -33,6 +33,8 @@ var Schemes = [SchemeCount]string {
"https", "https",
} }
var ErrUnknownScheme = errors.New("unknown protocol scheme")
// Error reports an error and the operation and URL that caused it. // Error reports an error and the operation and URL that caused it.
type Error struct { type Error struct {
Op string Op string
@ -353,7 +355,7 @@ func getscheme(rawurl string) (scheme Scheme, path string, err error) {
case "https": case "https":
scheme = SchemeHTTPS scheme = SchemeHTTPS
default: default:
return SchemeInvalid, "", errors.New("unknown protocol scheme") return SchemeInvalid, "", ErrUnknownScheme
} }
path = rawurl[i+1:] path = rawurl[i+1:]

5
main.go
View File

@ -62,6 +62,11 @@ func cmdBase(clic *cli.Context) error {
case <-ticker.C: case <-ticker.C:
t, err := FetchTask() t, err := FetchTask()
if err != nil { if err != nil {
if err == fasturl.ErrUnknownScheme {
// Not an error
err = nil
continue
}
logrus.WithError(err). logrus.WithError(err).
Error("Failed getting new task") Error("Failed getting new task")
time.Sleep(30 * time.Second) time.Sleep(30 * time.Second)