simon987/od-database-crawler
1
0
Fork 0
mirror of https://github.com/terorie/od-database-crawler.git synced 2025-12-13 23:29:02 +00:00
Code Issues Projects Releases Wiki Activity

Ignore HTTPS errors

Browse Source
This commit is contained in:
Richard Patel
2018-11-18 00:37:30 +01:00
parent 4464f34779
commit 6793086c22
4 changed files with 12 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
crawl.go
View File

@@ -2,6 +2,7 @@ package main
import (
"bytes"
"crypto/tls"
"github.com/terorie/od-database-crawler/ds/redblackhash"
"github.com/terorie/od-database-crawler/fasturl"
"github.com/valyala/fasthttp"
@@ -13,7 +14,11 @@ import (
"time"
)
var client fasthttp.Client
var client = fasthttp.Client {
TLSConfig: &tls.Config{
InsecureSkipVerify: true,
},
}
func GetDir(j *Job, f *File) (links []fasturl.URL, err error) {
f.IsDir = true