diff --git a/crawl.go b/crawl.go index 5ee17f0..de5bea3 100644 --- a/crawl.go +++ b/crawl.go @@ -20,7 +20,7 @@ var tlsConfig = tls.Config { InsecureSkipVerify: true, } -func newHTTPClient(url *fasturl.URL) *fasthttp.PipelineClient { +func newHTTPClient(url fasturl.URL) *fasthttp.PipelineClient { var isTLS bool switch url.Scheme { case fasturl.SchemeHTTP: @@ -29,6 +29,8 @@ func newHTTPClient(url *fasturl.URL) *fasthttp.PipelineClient { isTLS = true } + url.AddDefaultPort() + return &fasthttp.PipelineClient { MaxConns: viper.GetInt(ConfWorkers), Addr: url.Host, diff --git a/fasturl/url.go b/fasturl/url.go index 104bd2b..0641b6e 100644 --- a/fasturl/url.go +++ b/fasturl/url.go @@ -562,6 +562,20 @@ func validOptionalPort(port string) bool { return true } +// TODO Check if RFC-compliant (99% sure not) +func (u *URL) AddDefaultPort() { + if strings.ContainsRune(u.Host, ':') { + return + } + + switch u.Scheme { + case SchemeHTTP: + u.Host += ":80" + case SchemeHTTPS: + u.Host += ":443" + } +} + // String reassembles the URL into a valid URL string. // The general form of the result is one of: // diff --git a/worker.go b/worker.go index 2bedf21..1796a3a 100644 --- a/worker.go +++ b/worker.go @@ -23,7 +23,7 @@ type WorkerContext struct { } func (w *WorkerContext) Prepare() { - w.client = newHTTPClient(&w.OD.BaseUri) + w.client = newHTTPClient(w.OD.BaseUri) } func (w *WorkerContext) Worker(results chan<- File) {