Getting tasks

main.go

@@ -9,7 +9,7 @@ import (
 	"net/http"
 	_ "net/http/pprof"
 	"os"
-	"strings"
+	"sync/atomic"
 	"time"
 )
 
@@ -20,14 +20,7 @@ var app = cli.App {
 	BashComplete: cli.DefaultAppComplete,
 	Writer:       os.Stdout,
 	Compiled:     buildDate,
-	Commands:     []cli.Command{
-		{
-			Name:      "crawl",
-			Usage:     "Crawl a list of URLs",
-			ArgsUsage: "[site, site, ...]",
-			Action:    cmdCrawler,
-		},
-	},
+	Action:       cmdBase,
 }
 
 func init() {
@@ -41,50 +34,56 @@ func main() {
 	app.Run(os.Args)
 }
 
-func cmdCrawler(clic *cli.Context) error {
+func cmdBase(clic *cli.Context) error {
 	readConfig()
 
-	if clic.NArg() == 0 {
-		cli.ShowCommandHelpAndExit(clic, "crawl", 1)
-	}
-
-	args := clic.Args()
-	remotes := make([]*OD, len(args))
-	for i, arg := range args {
-		// https://github.com/golang/go/issues/19779
-		if !strings.Contains(arg, "://") {
-			arg = "http://" + arg
-		}
-		var u fasturl.URL
-		err := u.Parse(arg)
-		if !strings.HasSuffix(u.Path, "/") {
-			u.Path += "/"
-		}
-		if err != nil { return err }
-		remotes[i] = &OD {
-			Task: &Task{
-				WebsiteId: 0,
-				Url: u.String(),
-			},
-			BaseUri: u,
-		}
-	}
-
-	c := context.Background()
+	// TODO Graceful shutdown
+	appCtx := context.Background()
+	forceCtx := context.Background()
 
 	inRemotes := make(chan *OD)
-	go Schedule(c, inRemotes)
+	go Schedule(forceCtx, inRemotes)
 
-	for _, remote := range remotes {
-		globalWait.Add(1)
-		inRemotes <- remote
+	ticker := time.NewTicker(3 * time.Second)
+	defer ticker.Stop()
+	for {
+		select {
+		case <-appCtx.Done():
+			return nil
+		case <-ticker.C:
+			t, err := FetchTask()
+			if err != nil {
+				logrus.WithError(err).
+					Error("Failed getting new task")
+				time.Sleep(30 * time.Second)
+				continue
+			}
+			if t == nil {
+				// No new task
+				if atomic.LoadInt32(&activeTasks) == 0 {
+					logrus.Info("Waiting …")
+				}
+				continue
+			}
+
+			var baseUri fasturl.URL
+			err = baseUri.Parse(t.Url)
+			if err != nil {
+				logrus.WithError(err).
+					Error("Failed getting new task")
+				time.Sleep(30 * time.Second)
+				continue
+			}
+			inRemotes <- &OD {
+				Task: t,
+				BaseUri: baseUri,
+			}
+		}
 	}
 
 	// Wait for all jobs to finish
 	globalWait.Wait()
 
-	logrus.Info("All dirs processed!")
-
 	return nil
 }