Add resource stats logging

stats.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"github.com/sirupsen/logrus"
 	"math"
+	"runtime"
 	"sync/atomic"
 	"time"
 )
@@ -15,14 +16,23 @@ var totalAborted uint64
 
 func Stats(c context.Context) {
 	var startedLast uint64 = 0
-	ticker := time.NewTicker(config.StatsInterval).C
+	var crawlTicker <-chan time.Time
+	var allocTicker <-chan time.Time
+
+	if config.CrawlStats != 0 {
+		crawlTicker = time.NewTicker(config.CrawlStats).C
+	}
+	if config.AllocStats != 0 {
+		allocTicker = time.NewTicker(config.AllocStats).C
+	}
+
 	for {
 		select {
-		case <-ticker:
+		case <-crawlTicker:
 			startedNow := atomic.LoadUint64(&totalStarted)
 
 			perSecond := float64(startedNow - startedLast) /
-				config.StatsInterval.Seconds()
+				config.CrawlStats.Seconds()
 
 			// Round to .5
 			perSecond *= 2
@@ -34,10 +44,21 @@ func Stats(c context.Context) {
 				"done":    atomic.LoadUint64(&totalDone),
 				"retries": atomic.LoadUint64(&totalRetries),
 				"aborted": atomic.LoadUint64(&totalAborted),
-			}).Info("Stats")
+			}).Info("Crawl Stats")
 
 			startedLast = startedNow
 
+		case <-allocTicker:
+			var mem runtime.MemStats
+			runtime.ReadMemStats(&mem)
+
+			logrus.WithFields(logrus.Fields{
+				"queue_count": totalBuffered,
+				"heap": FormatByteCount(mem.Alloc),
+				"objects": mem.HeapObjects,
+				"num_gc": mem.NumGC,
+			}).Info("Resource Stats")
+
 		case <-c.Done():
 			return
 		}