simon987/od-database-crawler
1
0
Fork 0
mirror of https://github.com/terorie/od-database-crawler.git synced 2025-04-10 05:56:42 +00:00
Code Issues Projects Releases Wiki Activity

Fix missing port

Browse Source
This commit is contained in:
Richard Patel 2019-02-09 16:58:25 +01:00
parent c72f4ba475
commit 9bc3455ee0
No known key found for this signature in database
GPG Key ID: C268B2BBDA2ABECB
3 changed files with 18 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
crawl.go
View File

@ -20,7 +20,7 @@ var tlsConfig = tls.Config {
InsecureSkipVerify: true,
}
func newHTTPClient(url *fasturl.URL) *fasthttp.PipelineClient {
func newHTTPClient(url fasturl.URL) *fasthttp.PipelineClient {
var isTLS bool
switch url.Scheme {
case fasturl.SchemeHTTP:
@ -29,6 +29,8 @@ func newHTTPClient(url *fasturl.URL) *fasthttp.PipelineClient {
isTLS = true
}
url.AddDefaultPort()
return &fasthttp.PipelineClient {
MaxConns: viper.GetInt(ConfWorkers),
Addr: url.Host,

14
fasturl/url.go
View File

@ -562,6 +562,20 @@ func validOptionalPort(port string) bool {
return true
}
// TODO Check if RFC-compliant (99% sure not)
func (u *URL) AddDefaultPort() {
if strings.ContainsRune(u.Host, ':') {
return
}
switch u.Scheme {
case SchemeHTTP:
u.Host += ":80"
case SchemeHTTPS:
u.Host += ":443"
}
}
// String reassembles the URL into a valid URL string.
// The general form of the result is one of:
//

2
worker.go
View File

@ -23,7 +23,7 @@ type WorkerContext struct {
}
func (w *WorkerContext) Prepare() {
w.client = newHTTPClient(&w.OD.BaseUri)
w.client = newHTTPClient(w.OD.BaseUri)
}
func (w *WorkerContext) Worker(results chan<- File) {