From 6793086c22b9d24b8bdf7d8d780861147413cf39 Mon Sep 17 00:00:00 2001
From: Richard Patel <terorie@alphakevin.club>
Date: Sun, 18 Nov 2018 00:37:30 +0100
Subject: [PATCH] Ignore HTTPS errors

---
 config.go | 4 ++--
 crawl.go  | 7 ++++++-
 main.go   | 4 ++--
 server.go | 8 ++------
 4 files changed, 12 insertions(+), 11 deletions(-)

diff --git a/config.go b/config.go
index af84b34..379586f 100644
--- a/config.go
+++ b/config.go
@@ -14,7 +14,7 @@ var config struct {
 	Token      string
 	ServerTimeout time.Duration
 	Recheck    time.Duration
-	ChunkSize  uint
+	ChunkSize  int64
 	Retries    int
 	Workers    int
 	Timeout    time.Duration
@@ -78,7 +78,7 @@ func readConfig() {
 
 	config.Recheck = viper.GetDuration(ConfRecheck)
 
-	config.ChunkSize = viper.GetSizeInBytes(ConfChunkSize)
+	config.ChunkSize = int64(viper.GetSizeInBytes(ConfChunkSize))
 	if config.ChunkSize < 100 {
 		configOOB(ConfChunkSize, config.ChunkSize)
 	}
diff --git a/crawl.go b/crawl.go
index 80dc58e..8e2dad7 100644
--- a/crawl.go
+++ b/crawl.go
@@ -2,6 +2,7 @@ package main
 
 import (
 	"bytes"
+	"crypto/tls"
 	"github.com/terorie/od-database-crawler/ds/redblackhash"
 	"github.com/terorie/od-database-crawler/fasturl"
 	"github.com/valyala/fasthttp"
@@ -13,7 +14,11 @@ import (
 	"time"
 )
 
-var client fasthttp.Client
+var client = fasthttp.Client {
+	TLSConfig: &tls.Config{
+		InsecureSkipVerify: true,
+	},
+}
 
 func GetDir(j *Job, f *File) (links []fasturl.URL, err error) {
 	f.IsDir = true
diff --git a/main.go b/main.go
index 1ca415d..4dc86bb 100644
--- a/main.go
+++ b/main.go
@@ -148,6 +148,6 @@ func cmdCrawler(clic *cli.Context) error {
 }
 
 var buildDate = time.Date(
-	2018, 11, 15,
-	23, 24, 0, 0,
+	2018, 11, 17,
+	23, 32, 0, 0,
 	time.UTC)
diff --git a/server.go b/server.go
index d469c75..382a960 100644
--- a/server.go
+++ b/server.go
@@ -13,10 +13,6 @@ import (
 	"strconv"
 )
 
-const (
-	fileListChunkSize int64 = 5000000 // 5 mb
-)
-
 var serverClient = http.Client {
 	Timeout: config.ServerTimeout,
 }
@@ -90,11 +86,11 @@ func uploadChunks(websiteId uint64, f *os.File) error {
 		// Copy chunk to file_list
 		formFile, err := multi.CreateFormFile("file_list", "file_list")
 		var n int64
-		n, err = io.CopyN(formFile, f, fileListChunkSize)
+		n, err = io.CopyN(formFile, f, config.ChunkSize)
 		if err != io.EOF && err != nil {
 			return err
 		}
-		if n < fileListChunkSize {
+		if n < config.ChunkSize {
 			err = nil
 			// Break at end of iteration
 			eof = true