Fix missing port

crawl.go

@@ -20,7 +20,7 @@ var tlsConfig = tls.Config {
 	InsecureSkipVerify: true,
 }
 
-func newHTTPClient(url *fasturl.URL) *fasthttp.PipelineClient {
+func newHTTPClient(url fasturl.URL) *fasthttp.PipelineClient {
 	var isTLS bool
 	switch url.Scheme {
 	case fasturl.SchemeHTTP:
@@ -29,6 +29,8 @@ func newHTTPClient(url *fasturl.URL) *fasthttp.PipelineClient {
 		isTLS = true
 	}
 
+	url.AddDefaultPort()
+
 	return &fasthttp.PipelineClient {
 		MaxConns:     viper.GetInt(ConfWorkers),
 		Addr:         url.Host,

fasturl/url.go

@@ -562,6 +562,20 @@ func validOptionalPort(port string) bool {
 	return true
 }
 
+// TODO Check if RFC-compliant (99% sure not)
+func (u *URL) AddDefaultPort() {
+	if strings.ContainsRune(u.Host, ':') {
+		return
+	}
+
+	switch u.Scheme {
+	case SchemeHTTP:
+		u.Host += ":80"
+	case SchemeHTTPS:
+		u.Host += ":443"
+	}
+}
+
 // String reassembles the URL into a valid URL string.
 // The general form of the result is one of:
 //

worker.go

@@ -23,7 +23,7 @@ type WorkerContext struct {
 }
 
 func (w *WorkerContext) Prepare() {
-	w.client = newHTTPClient(&w.OD.BaseUri)
+	w.client = newHTTPClient(w.OD.BaseUri)
 }
 
 func (w *WorkerContext) Worker(results chan<- File) {