package main import ( "github.com/elazarl/goproxy" redisPackage "github.com/go-redis/redis/v7" "github.com/go-redis/redis_rate/v8" influx "github.com/influxdata/influxdb1-client/v2" "math" "net/http" "net/url" "time" ) type Architeuthis struct { server *goproxy.ProxyHttpServer redis *redisPackage.Client influxdb influx.Client points chan *influx.Point } // Request/Response type RequestCtx struct { Request *http.Request Retries int LastFailedProxy string p *Proxy LastErrorWasProxyError bool RequestTime time.Time options RequestOptions configs []*HostConfig } type ResponseCtx struct { Response *http.Response ResponseTime float64 Error error ShouldRetry bool } type RequestOptions struct { DoCloudflareBypass bool } // Proxy type Proxy struct { Name string Url *url.URL HttpClient *http.Client GoodRequestCount int64 incrGood int64 BadRequestCount int64 incrBad int64 TotalRequestTime float64 incrReqTime float64 Connections int64 KillOnError bool } func (p *Proxy) AvgLatency() float64 { return p.TotalRequestTime / float64(p.GoodRequestCount+p.BadRequestCount) } func (p *Proxy) Score() float64 { if p.GoodRequestCount+p.BadRequestCount == 0 { return 1000 } var errorMod float64 var latencyMod float64 if p.BadRequestCount == 0 { errorMod = 1 } else { errorMod = math.Min(float64(p.GoodRequestCount)/float64(p.BadRequestCount), 1) } avgLatency := p.AvgLatency() switch { case avgLatency < 3: latencyMod = 1 case avgLatency < 4: latencyMod = 0.8 case avgLatency < 5: latencyMod = 0.7 case avgLatency < 9: latencyMod = 0.6 case avgLatency < 10: latencyMod = 0.5 case avgLatency < 15: latencyMod = 0.3 case avgLatency < 20: latencyMod = 0.1 default: latencyMod = 0 } return 600*errorMod + 400*latencyMod - 200*(math.Max(float64(p.Connections-1), 0)) } func (p *Proxy) getStats() proxyStat { return proxyStat{ Name: p.Name, Url: p.Url.String(), GoodRequestCount: p.GoodRequestCount, BadRequestCount: p.BadRequestCount, AvgLatency: p.AvgLatency(), Connections: p.Connections, Score: int64(p.Score()), } } type proxyStat struct { Name string Url string GoodRequestCount int64 BadRequestCount int64 AvgLatency float64 Connections int64 Score int64 } type statsData struct { TotalGood int TotalBad int Connections int AvgLatency float64 AvgScore float64 Proxies []proxyStat } type CheckMethod string const ( CheckIp CheckMethod = "check_ip" HttpOk CheckMethod = "http_ok" ) type ProxyJudge struct { url *url.URL method CheckMethod } type RedisLimiter struct { Key string Limiter *redis_rate.Limiter Limit *redis_rate.Limit } // Config type HostConfig struct { Host string `json:"host"` EveryStr string `json:"every"` Burst int `json:"burst"` Headers map[string]string `json:"headers"` RawRules []*RawHostRule `json:"rules"` IsGlob bool Every time.Duration Rules []*HostRule } type RawHostRule struct { Condition string `json:"condition"` Action string `json:"action"` Arg string `json:"arg"` } type HostRuleAction int const ( DontRetry HostRuleAction = 0 ForceRetry HostRuleAction = 1 ShouldRetry HostRuleAction = 2 ) type HostRule struct { Matches func(r *ResponseCtx) bool Action HostRuleAction Arg float64 } type ProxyConfig struct { Name string `json:"name"` Url string `json:"url"` } var config struct { Addr string `json:"addr"` TimeoutStr string `json:"timeout"` WaitStr string `json:"wait"` Multiplier float64 `json:"multiplier"` Retries int `json:"retries"` MaxErrorRatio float64 `json:"max_error"` Hosts []*HostConfig `json:"hosts"` Proxies []ProxyConfig `json:"proxies"` RedisUrl string `json:"redis_url"` Wait int64 Timeout time.Duration DefaultConfig *HostConfig Routing bool InfluxUrl string `json:"influx_url"` InfluxUser string `json:"influx_user"` InfluxPass string `json:"influx_pass"` }