Add hot config reload

README.md

@@ -19,13 +19,22 @@ and error handling. Built for automated web scraping.
 ### Usage
 
 ```bash
-wget https://simon987.net/data/architeuthis/9_architeuthis.tar.gz
+wget https://simon987.net/data/architeuthis/11_architeuthis.tar.gz
-tar -xzf 9_architeuthis.tar.gz
+tar -xzf 11_architeuthis.tar.gz
 
 vim config.json # Configure settings here
 ./architeuthis
 ```
 
+### Hot config reload
+
+```bash
+# Note: this will reset current rate limiters, if there are many active
+# connections, this might cause a small request spike and go over
+# the rate limits.
+./reload.sh
+```
+
 ### Sample configuration
 
 ```json

config.go

@@ -3,6 +3,7 @@ package main
 import (
 	"encoding/json"
 	"fmt"
+	"github.com/sirupsen/logrus"
 	"golang.org/x/time/rate"
 	"io/ioutil"
 	"os"
@@ -98,6 +99,32 @@ func applyConfig(proxy *Proxy) {
 	}
 }
 
+func (b *Balancer) reloadConfig() {
+
+	b.proxyMutex.Lock()
+	loadConfig()
+
+	if b.proxies != nil {
+		b.proxies = b.proxies[:0]
+	}
+
+	for _, proxyConf := range config.Proxies {
+		proxy, err := NewProxy(proxyConf.Name, proxyConf.Url)
+		handleErr(err)
+		b.proxies = append(b.proxies, proxy)
+
+		applyConfig(proxy)
+
+		logrus.WithFields(logrus.Fields{
+			"name": proxy.Name,
+			"url":  proxy.Url,
+		}).Info("Proxy")
+	}
+	b.proxyMutex.Unlock()
+
+	logrus.Info("Reloaded config")
+}
+
 func handleErr(err error) {
 	if err != nil {
 		panic(err)

gc.go

@@ -24,11 +24,14 @@ func (b *Balancer) setupGarbageCollector() {
 func (b *Balancer) cleanAllExpiredLimits() {
 	before := 0
 	after := 0
+
+	b.proxyMutex.RLock()
 	for _, p := range b.proxies {
 		before += len(p.Limiters)
 		cleanExpiredLimits(p)
 		after += len(p.Limiters)
 	}
+	b.proxyMutex.RUnlock()
 
 	logrus.WithFields(logrus.Fields{
 		"removed": before - after,

jenkins/Jenkinsfile

@@ -26,7 +26,7 @@ pipeline {
                 sh 'cp *.go "/go/src/github.com/simon987/Architeuthis"'
                 sh 'cd /go/src/github.com/simon987/Architeuthis && go get ./...'
                 sh 'cd /go/src/github.com/simon987/Architeuthis && go build -a -installsuffix cgo -o "${WORKSPACE}/architeuthis" .'
-                sh 'tar -czf ${BUILD_NUMBER}_architeuthis.tar.gz config.json architeuthis'
+                sh 'tar -czf ${BUILD_NUMBER}_architeuthis.tar.gz config.json architeuthis reload.sh'
                 sshPut remote: remote, from: env.BUILD_NUMBER + '_architeuthis.tar.gz', into: 'architeuthis/webroot/'
             }
         }

main.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"fmt"
 	"github.com/elazarl/goproxy"
 	"github.com/pkg/errors"
 	"github.com/ryanuber/go-glob"
@@ -11,12 +12,14 @@ import (
 	"net/url"
 	"sort"
 	"strings"
+	"sync"
 	"time"
 )
 
 type Balancer struct {
-	server  *goproxy.ProxyHttpServer
+	server     *goproxy.ProxyHttpServer
-	proxies []*Proxy
+	proxies    []*Proxy
+	proxyMutex *sync.RWMutex
 }
 
 type ExpiringLimiter struct {
@@ -113,6 +116,7 @@ func New() *Balancer {
 
 	balancer := new(Balancer)
 
+	balancer.proxyMutex = &sync.RWMutex{}
 	balancer.server = goproxy.NewProxyHttpServer()
 
 	balancer.server.OnRequest().HandleConnect(goproxy.AlwaysMitm)
@@ -120,6 +124,7 @@ func New() *Balancer {
 	balancer.server.OnRequest().DoFunc(
 		func(r *http.Request, ctx *goproxy.ProxyCtx) (*http.Request, *http.Response) {
 
+			balancer.proxyMutex.RLock()
 			p := balancer.chooseProxy()
 
 			logrus.WithFields(logrus.Fields{
@@ -129,6 +134,7 @@ func New() *Balancer {
 			}).Trace("Routing request")
 
 			resp, err := p.processRequest(r)
+			balancer.proxyMutex.RUnlock()
 
 			if err != nil {
 				logrus.WithError(err).Trace("Could not complete request")
@@ -137,6 +143,17 @@ func New() *Balancer {
 
 			return nil, resp
 		})
+
+	balancer.server.NonproxyHandler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+
+		if r.URL.Path == "/reload" {
+			balancer.reloadConfig()
+			_, _ = fmt.Fprint(w, "Reloaded\n")
+		} else {
+			w.Header().Set("Content-Type", "application/json")
+			_, _ = fmt.Fprint(w, "{\"name\":\"Architeuthis\",\"version\":1.0}")
+		}
+	})
 	return balancer
 }
 
@@ -275,21 +292,8 @@ func NewProxy(name, stringUrl string) (*Proxy, error) {
 func main() {
 	logrus.SetLevel(logrus.TraceLevel)
 
-	loadConfig()
 	balancer := New()
-
+	balancer.reloadConfig()
-	for _, proxyConf := range config.Proxies {
-		proxy, err := NewProxy(proxyConf.Name, proxyConf.Url)
-		handleErr(err)
-		balancer.proxies = append(balancer.proxies, proxy)
-
-		applyConfig(proxy)
-
-		logrus.WithFields(logrus.Fields{
-			"name": proxy.Name,
-			"url":  proxy.Url,
-		}).Info("Proxy")
-	}
 
 	balancer.setupGarbageCollector()
 	balancer.Run()

reload.sh

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+
+curl -X GET localhost:5050/reload