simon987/od-database-crawler
1
0
Fork 0
mirror of https://github.com/terorie/od-database-crawler.git synced 2025-04-19 10:26:43 +00:00
Code Issues Projects Releases Wiki Activity

Ignore HTTPS errors

Browse Source
This commit is contained in:
Richard Patel 2018-11-18 00:37:30 +01:00
parent 4464f34779
commit 6793086c22
No known key found for this signature in database
GPG Key ID: C268B2BBDA2ABECB
4 changed files with 12 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
config.go
View File

@ -14,7 +14,7 @@ var config struct {
Token string Token string
ServerTimeout time.Duration ServerTimeout time.Duration
Recheck time.Duration Recheck time.Duration
ChunkSize uint ChunkSize int64
Retries int Retries int
Workers int Workers int
Timeout time.Duration Timeout time.Duration
@ -78,7 +78,7 @@ func readConfig() {
config.Recheck = viper.GetDuration(ConfRecheck) config.Recheck = viper.GetDuration(ConfRecheck)
config.ChunkSize = viper.GetSizeInBytes(ConfChunkSize) config.ChunkSize = int64(viper.GetSizeInBytes(ConfChunkSize))
if config.ChunkSize < 100 { if config.ChunkSize < 100 {
configOOB(ConfChunkSize, config.ChunkSize) configOOB(ConfChunkSize, config.ChunkSize)
} }

7
crawl.go
View File

@ -2,6 +2,7 @@ package main
import ( import (
"bytes" "bytes"
"crypto/tls"
"github.com/terorie/od-database-crawler/ds/redblackhash" "github.com/terorie/od-database-crawler/ds/redblackhash"
"github.com/terorie/od-database-crawler/fasturl" "github.com/terorie/od-database-crawler/fasturl"
"github.com/valyala/fasthttp" "github.com/valyala/fasthttp"
@ -13,7 +14,11 @@ import (
"time" "time"
) )
var client fasthttp.Client var client = fasthttp.Client {
TLSConfig: &tls.Config{
InsecureSkipVerify: true,
},
}
func GetDir(j *Job, f *File) (links []fasturl.URL, err error) { func GetDir(j *Job, f *File) (links []fasturl.URL, err error) {
f.IsDir = true f.IsDir = true

4
main.go
View File

@ -148,6 +148,6 @@ func cmdCrawler(clic *cli.Context) error {
} }
var buildDate = time.Date( var buildDate = time.Date(
2018, 11, 15, 2018, 11, 17,
23, 24, 0, 0, 23, 32, 0, 0,
time.UTC) time.UTC)

8
server.go
View File

@ -13,10 +13,6 @@ import (
"strconv" "strconv"
) )
const (
fileListChunkSize int64 = 5000000 // 5 mb
)
var serverClient = http.Client { var serverClient = http.Client {
Timeout: config.ServerTimeout, Timeout: config.ServerTimeout,
} }
@ -90,11 +86,11 @@ func uploadChunks(websiteId uint64, f *os.File) error {
// Copy chunk to file_list // Copy chunk to file_list
formFile, err := multi.CreateFormFile("file_list", "file_list") formFile, err := multi.CreateFormFile("file_list", "file_list")
var n int64 var n int64
n, err = io.CopyN(formFile, f, fileListChunkSize) n, err = io.CopyN(formFile, f, config.ChunkSize)
if err != io.EOF && err != nil { if err != io.EOF && err != nil {
return err return err
} }
if n < fileListChunkSize { if n < config.ChunkSize {
err = nil err = nil
// Break at end of iteration // Break at end of iteration
eof = true eof = true