Add urfave/cli app

config.go

@@ -1,8 +1,10 @@
 package main
 
 import (
+	"fmt"
 	"github.com/sirupsen/logrus"
 	"github.com/spf13/viper"
+	"os"
 	"time"
 )
 
@@ -39,18 +41,19 @@ func readConfig() {
 	viper.SetConfigName("config")
 	err := viper.ReadInConfig()
 	if err != nil {
-		logrus.Fatal(err)
+		fmt.Fprintln(os.Stderr, err)
+		os.Exit(1)
 	}
 
 	config.ServerUrl = viper.GetString(ConfServerUrl)
-	if config.ServerUrl == "" {
+	//if config.ServerUrl == "" {
-		configMissing(ConfServerUrl)
+	//	configMissing(ConfServerUrl)
-	}
+	//}
 
 	config.Token = viper.GetString(ConfToken)
-	if config.Token == "" {
+	//if config.Token == "" {
-		configMissing(ConfToken)
+	//	configMissing(ConfToken)
-	}
+	//}
 
 	config.Retries = viper.GetInt(ConfRetries)
 	if config.Retries < 0 {
@@ -76,9 +79,11 @@ func readConfig() {
 }
 
 func configMissing(key string) {
-	logrus.Fatalf("config: %s not set!", key)
+	fmt.Fprintf(os.Stderr, "config: %s not set!\n", key)
+	os.Exit(1)
 }
 
 func configOOB(key string, v int) {
-	logrus.Fatal("config: illegal value %d for %key!", v, key)
+	fmt.Fprintf(os.Stderr, "config: illegal value %d for %key!\n", v, key)
+	os.Exit(1)
 }

crawl.go

@@ -19,10 +19,6 @@ var client fasthttp.Client
 var ErrRateLimit = errors.New("too many requests")
 var ErrForbidden = errors.New("access denied")
 
-func NewRemoteDir(u url.URL) *OD {
-	return &OD{ BaseUri: u }
-}
-
 func GetDir(j *Job, f *File) (links []url.URL, err error) {
 	f.IsDir = true
 	f.Name = path.Base(j.Uri.Path)

main.go

@@ -2,27 +2,77 @@ package main
 
 import (
 	"context"
+	"github.com/urfave/cli"
 	"net/url"
+	"os"
+	"strings"
+	"time"
 )
 
+var app = cli.App {
+	Name:         "oddb-go",
+	Usage:        "OD-Database Go crawler",
+	Version:      "0.1",
+	BashComplete: cli.DefaultAppComplete,
+	Writer:       os.Stdout,
+	Compiled:     buildDate,
+	Commands:     []cli.Command{
+		{
+			Name:      "crawl",
+			Usage:     "Crawl a list of URLs",
+			ArgsUsage: "[site, site, ...]",
+			Action:    cmdCrawler,
+		},
+	},
+}
+
 func init() {
 	prepareConfig()
 }
 
 func main() {
+	app.Run(os.Args)
+}
+
+func cmdCrawler(clic *cli.Context) error {
 	readConfig()
 
+	if clic.NArg() == 0 {
+		cli.ShowCommandHelpAndExit(clic, "crawl", 1)
+	}
+
+	args := clic.Args()
+	remotes := make([]*OD, len(args))
+	for i, arg := range args {
+		// https://github.com/golang/go/issues/19779
+		if !strings.Contains(arg, "://") {
+			arg = "http://" + arg
+		}
+		u, err := url.Parse(arg)
+		if !strings.HasSuffix(u.Path, "/") {
+			u.Path += "/"
+		}
+		if err != nil { return err }
+		remotes[i] = &OD{ BaseUri: *u }
+	}
+
 	c := context.Background()
 
-	remotes := make(chan *OD)
+	inRemotes := make(chan *OD)
-	go Schedule(c, remotes)
+	go Schedule(c, inRemotes)
-
-	u, _ := url.Parse("http://mine.terorie.com:420/")
-	remote := NewRemoteDir(*u)
 
+	for _, remote := range remotes {
 		globalWait.Add(1)
-	remotes <- remote
+		inRemotes <- remote
+	}
 
 	// Wait for all jobs to finish
 	globalWait.Wait()
+
+	return nil
 }
+
+var buildDate = time.Date(
+	2018, 10, 28,
+	17, 10, 0, 0,
+	time.UTC)