Ignore HTTPS errors

config.go

@@ -14,7 +14,7 @@ var config struct {
 	Token      string
 	ServerTimeout time.Duration
 	Recheck    time.Duration
-	ChunkSize  uint
+	ChunkSize  int64
 	Retries    int
 	Workers    int
 	Timeout    time.Duration
@@ -78,7 +78,7 @@ func readConfig() {
 
 	config.Recheck = viper.GetDuration(ConfRecheck)
 
-	config.ChunkSize = viper.GetSizeInBytes(ConfChunkSize)
+	config.ChunkSize = int64(viper.GetSizeInBytes(ConfChunkSize))
 	if config.ChunkSize < 100 {
 		configOOB(ConfChunkSize, config.ChunkSize)
 	}

crawl.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"bytes"
+	"crypto/tls"
 	"github.com/terorie/od-database-crawler/ds/redblackhash"
 	"github.com/terorie/od-database-crawler/fasturl"
 	"github.com/valyala/fasthttp"
@@ -13,7 +14,11 @@ import (
 	"time"
 )
 
-var client fasthttp.Client
+var client = fasthttp.Client {
+	TLSConfig: &tls.Config{
+		InsecureSkipVerify: true,
+	},
+}
 
 func GetDir(j *Job, f *File) (links []fasturl.URL, err error) {
 	f.IsDir = true

main.go

@@ -148,6 +148,6 @@ func cmdCrawler(clic *cli.Context) error {
 }
 
 var buildDate = time.Date(
-	2018, 11, 15,
-	23, 24, 0, 0,
+	2018, 11, 17,
+	23, 32, 0, 0,
 	time.UTC)

server.go

@@ -13,10 +13,6 @@ import (
 	"strconv"
 )
 
-const (
-	fileListChunkSize int64 = 5000000 // 5 mb
-)
-
 var serverClient = http.Client {
 	Timeout: config.ServerTimeout,
 }
@@ -90,11 +86,11 @@ func uploadChunks(websiteId uint64, f *os.File) error {
 		// Copy chunk to file_list
 		formFile, err := multi.CreateFormFile("file_list", "file_list")
 		var n int64
-		n, err = io.CopyN(formFile, f, fileListChunkSize)
+		n, err = io.CopyN(formFile, f, config.ChunkSize)
 		if err != io.EOF && err != nil {
 			return err
 		}
-		if n < fileListChunkSize {
+		if n < config.ChunkSize {
 			err = nil
 			// Break at end of iteration
 			eof = true