od-database-crawler/stats.go

package main

import (
	"context"
	"github.com/sirupsen/logrus"
	"math"
	"runtime"
	"sync/atomic"
	"time"
)

var totalStarted uint64
var totalDone uint64
var totalRetries uint64
var totalAborted uint64

func Stats(c context.Context) {
	var startedLast uint64 = 0
	var crawlTicker <-chan time.Time
	var allocTicker <-chan time.Time

	if config.CrawlStats != 0 {
		crawlTicker = time.NewTicker(config.CrawlStats).C
	}
	if config.AllocStats != 0 {
		allocTicker = time.NewTicker(config.AllocStats).C
	}

	for {
		select {
		case <-crawlTicker:
			startedNow := atomic.LoadUint64(&totalStarted)

			perSecond := float64(startedNow - startedLast) /
				config.CrawlStats.Seconds()

			// Round to .5
			perSecond *= 2
			perSecond = math.Round(perSecond)
			perSecond /= 2

			logrus.WithFields(logrus.Fields{
				"per_second": perSecond,
				"done":    atomic.LoadUint64(&totalDone),
				"retries": atomic.LoadUint64(&totalRetries),
				"aborted": atomic.LoadUint64(&totalAborted),
			}).Info("Crawl Stats")

			startedLast = startedNow

		case <-allocTicker:
			var mem runtime.MemStats
			runtime.ReadMemStats(&mem)

			logrus.WithFields(logrus.Fields{
				"queue_count": totalBuffered,
				"heap": FormatByteCount(mem.Alloc),
				"objects": mem.HeapObjects,
				"num_gc": mem.NumGC,
			}).Info("Resource Stats")

		case <-c.Done():
			return
		}
	}
}