From c196b6f20d78f618a9c0e8e951070bb4937164b5 Mon Sep 17 00:00:00 2001
From: Richard Patel <terorie@alphakevin.club>
Date: Sun, 28 Oct 2018 14:19:09 +0100
Subject: [PATCH] Better config

---
 config.go  | 38 +++++++++++++++++++++++++++-----------
 config.yml | 30 ++++++++++++++++++++++++------
 2 files changed, 51 insertions(+), 17 deletions(-)

diff --git a/config.go b/config.go
index 4205c7e..ad63034 100644
--- a/config.go
+++ b/config.go
@@ -11,22 +11,25 @@ var config struct {
 	Token     string
 	Retries   int
 	Workers   int
+	Tasks     int
 	StatsInterval time.Duration
 	Verbose   bool
 }
 
 const (
-	ConfServerUrl = "server_url"
-	ConfToken     = "token"
-	ConfRetries   = "retries"
-	ConfWorkers   = "workers"
-	ConfStatsInterval = "stats_interval"
-	ConfVerbose   = "verbose"
+	ConfServerUrl = "server.url"
+	ConfToken     = "server.token"
+	ConfTasks     = "crawl.tasks"
+	ConfRetries   = "crawl.retries"
+	ConfWorkers   = "crawl.connections"
+	ConfStatsInterval = "output.stats_interval"
+	ConfVerbose   = "output.verbose"
 )
 
 func prepareConfig() {
-	viper.SetDefault(ConfRetries, 3)
-	viper.SetDefault(ConfWorkers, 50)
+	viper.SetDefault(ConfRetries, 5)
+	viper.SetDefault(ConfWorkers, 2)
+	viper.SetDefault(ConfTasks, 3)
 	viper.SetDefault(ConfStatsInterval, 3 * time.Second)
 	viper.SetDefault(ConfVerbose, false)
 }
@@ -41,12 +44,12 @@ func readConfig() {
 
 	config.ServerUrl = viper.GetString(ConfServerUrl)
 	if config.ServerUrl == "" {
-		logrus.Fatal("config: server_url not set!")
+		configMissing(ConfServerUrl)
 	}
 
 	config.Token = viper.GetString(ConfToken)
 	if config.Token == "" {
-		logrus.Fatal("config: token not set!")
+		configMissing(ConfToken)
 	}
 
 	config.Retries = viper.GetInt(ConfRetries)
@@ -56,7 +59,12 @@ func readConfig() {
 
 	config.Workers = viper.GetInt(ConfWorkers)
 	if config.Workers <= 0 {
-		logrus.Fatal("config: illegal value %d for workers!", config.Workers)
+		configOOB(ConfWorkers, config.Workers)
+	}
+
+	config.Tasks = viper.GetInt(ConfTasks)
+	if config.Tasks <= 0 {
+		configOOB(ConfTasks, config.Tasks)
 	}
 
 	config.StatsInterval = viper.GetDuration(ConfStatsInterval)
@@ -66,3 +74,11 @@ func readConfig() {
 		logrus.SetLevel(logrus.DebugLevel)
 	}
 }
+
+func configMissing(key string) {
+	logrus.Fatalf("config: %s not set!", key)
+}
+
+func configOOB(key string, v int) {
+	logrus.Fatal("config: illegal value %d for %key!", v, key)
+}
diff --git a/config.yml b/config.yml
index fbfa1c5..efb0b4b 100644
--- a/config.yml
+++ b/config.yml
@@ -1,6 +1,24 @@
-server_url: localhost:6969
-token: abc
-stats_interval: 1s
-verbose: true
-retries: 5
-workers: 80
\ No newline at end of file
+# OD-Database server settings
+server:
+  # Connection URL
+  url: localhost:6969
+  # Server auth token
+  token:
+
+# Log output settings
+output:
+  # Statistics printing interval
+  stats_interval: 1s
+  # More output? (Every listed dir)
+  verbose: false
+
+# Crawler settings
+crawl:
+  # Number of sites that can be
+  # processed at once
+  tasks: 3
+  # Number of connections per site
+  connections: 2
+  # How often to retry getting data
+  # from the site before giving up
+  retries: 5