diff --git a/config.json b/config.json index 4d62d5e..c671284 100644 --- a/config.json +++ b/config.json @@ -8,28 +8,52 @@ { "name": "p0", "url": "" - }, - { - "name": "p1", - "url": "" } ], "hosts": [ { "host": "*", - "every": "500s", + "every": "500ms", "burst": 25, "headers": { "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Cache-Control": "max-age=0", "Connection": "keep-alive", - "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:67.0) Gecko/20100101 Firefox/67.0", - "X-Test": "default" + "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:67.0) Gecko/20100101 Firefox/67.0" } }, { "host": "*.reddit.com", - "every": "2s" + "every": "2s", + "burst": 1 + }, + { + "host": ".twitter.com", + "every": "24s", + "burst": 20 + }, + { + "host": ".pbs.twimg.com", + "every": "125ms" + }, + { + "host": "*.cdninstagram", + "every": "250ms" + }, + { + "host": ".www.instagram.com", + "every": "30s", + "burst": 3 + }, + { + "host": ".deviantart.com", + "every": "2s", + "burst": 3 + }, + { + "host": ".s3.amazonaws.com", + "every": "10s", + "burst": 3 } ] } \ No newline at end of file diff --git a/main.go b/main.go index 2d483d2..3bda8c7 100644 --- a/main.go +++ b/main.go @@ -6,6 +6,7 @@ import ( "github.com/ryanuber/go-glob" "github.com/sirupsen/logrus" "golang.org/x/time/rate" + "math/rand" "net/http" "net/url" "sort" @@ -47,14 +48,15 @@ func (a ByConnectionCount) Less(i, j int) bool { func (p *Proxy) getLimiter(host string) *rate.Limiter { - expLimit, ok := p.Limiters[host] - if !ok { - newExpiringLimiter := p.makeNewLimiter(host) - return newExpiringLimiter.Limiter + for hostGlob, limiter := range p.Limiters { + if glob.Glob(hostGlob, host) { + limiter.LastRead = time.Now() + return limiter.Limiter + } } - expLimit.LastRead = time.Now() - return expLimit.Limiter + newExpiringLimiter := p.makeNewLimiter(host) + return newExpiringLimiter.Limiter } func (p *Proxy) makeNewLimiter(host string) *ExpiringLimiter { @@ -85,8 +87,26 @@ func simplifyHost(host string) string { func (b *Balancer) chooseProxy() *Proxy { + if len(b.proxies) == 0 { + return b.proxies[0] + } + sort.Sort(ByConnectionCount(b.proxies)) - return b.proxies[0] + + p0 := b.proxies[0] + proxiesWithSameConnCount := make([]*Proxy, 0) + for _, p := range b.proxies { + if p.Connections != p0.Connections { + break + } + proxiesWithSameConnCount = append(proxiesWithSameConnCount, p) + } + + if len(proxiesWithSameConnCount) > 1 { + return proxiesWithSameConnCount[rand.Intn(len(proxiesWithSameConnCount))] + } else { + return p0 + } } func New() *Balancer { @@ -105,7 +125,7 @@ func New() *Balancer { logrus.WithFields(logrus.Fields{ "proxy": p.Name, "connexions": p.Connections, - "host": r.Host, + "url": r.URL, }).Trace("Routing request") resp, err := p.processRequest(r)