Bump to v1.0.2

crawl.go

@@ -50,6 +50,8 @@ func GetDir(j *Job, f *File) (links []fasturl.URL, err error) {
 
 	var linkHref string
 	for {
+		err = nil
+
 		tokenType := doc.Next()
 		if tokenType == html.ErrorToken {
 			break
@@ -80,16 +82,16 @@ func GetDir(j *Job, f *File) (links []fasturl.URL, err error) {
 				linkHref = ""
 
 				if strings.LastIndexByte(href, '?') != -1 {
-					goto nextToken
+					continue
 				}
 
 				switch href {
 				case "", " ", ".", "..", "/":
-					goto nextToken
+					continue
 				}
 
 				if strings.Contains(href, "../") {
-					goto nextToken
+					continue
 				}
 
 				var link fasturl.URL
@@ -108,8 +110,6 @@ func GetDir(j *Job, f *File) (links []fasturl.URL, err error) {
 				links = append(links, link)
 			}
 		}
-
-	nextToken:
 	}
 
 	return

main.go

@@ -14,7 +14,7 @@ import (
 var app = cli.App {
 	Name:         "od-database-crawler",
 	Usage:        "OD-Database Go crawler",
-	Version:      "1.0.1",
+	Version:      "1.0.2",
 	BashComplete: cli.DefaultAppComplete,
 	Writer:       os.Stdout,
 	Action:       cmdBase,

server.go

@@ -11,6 +11,7 @@ import (
 	"net/url"
 	"os"
 	"strconv"
+	"time"
 )
 
 var serverClient = http.Client {
@@ -101,25 +102,38 @@ func uploadChunks(websiteId uint64, f *os.File) error {
 
 		multi.Close()
 
+		for retried := false; true; retried = true {
+			err = nil
+			if retried {
+				// Error occurred, retry upload
+				time.Sleep(5 * time.Second)
+			}
+
 			req, err := http.NewRequest(
 				http.MethodPost,
 				config.ServerUrl + "/task/upload",
 				&b)
 			req.Header.Set("content-type", multi.FormDataContentType())
-		if err != nil { return err }
+			if err != nil { continue }
 
 			res, err := serverClient.Do(req)
-		if err != nil { return err }
+			if err != nil { continue }
 			res.Body.Close()
 
 			if res.StatusCode != http.StatusOK {
-			return fmt.Errorf("failed to upload list part %d: %s",
+				logrus.WithField("status", res.Status).
-				iter, res.Status)
+					WithField("part", iter).
+					Errorf("Upload failed")
+				continue
+			}
+
+			// Upload successful
+			break
 		}
 
 		logrus.WithField("id", websiteId).
 			WithField("part", iter).
-			Infof("Uploading files chunk")
+			Infof("Uploaded files chunk")
 	}
 	return nil
 }