Fix missing port

Dockerfile

@@ -1,15 +0,0 @@
-FROM golang:alpine as builder
-ADD . /go/src/github.com/terorie/od-database-crawler
-RUN apk add git \
- && go get -d -v github.com/terorie/od-database-crawler \
- && CGO_ENABLED=0 go install -a \
-    -installsuffix cgo \
-    -ldflags="-s -w" \
-    github.com/terorie/od-database-crawler
-
-FROM scratch
-COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
-COPY --from=builder /go/bin/od-database-crawler /bin/
-WORKDIR /oddb
-VOLUME [ "/oddb" ]
-CMD ["/bin/od-database-crawler", "server"]

README.md

@@ -1,57 +1,24 @@
-# OD-Database Crawler 🕷
+# od-database Go crawler 🚀
 [![Build Status](https://travis-ci.org/terorie/od-database-crawler.svg?branch=master)](https://travis-ci.org/terorie/od-database-crawler)
-[![](https://tokei.rs/b1/github/terorie/od-database-crawler)](https://github.com/terorie/od-database-crawler)
-[![CodeFactor](https://www.codefactor.io/repository/github/terorie/od-database-crawler/badge/master)](https://www.codefactor.io/repository/github/terorie/od-database-crawler/overview/master)
+> by terorie 2018 :P
 
  * Crawler for [__OD-Database__](https://github.com/simon987/od-database)
- * In production at https://od-db.the-eye.eu/
- * Over 880 TB actively crawled
  * Crawls HTTP open directories (standard Web Server Listings)
  * Gets name, path, size and modification time of all files
- * Lightweight and fast
+ * Lightweight and fast: __over 9000 requests per second__ on a standard laptop
 
 https://od-db.the-eye.eu/
 
-## Usage
-
-### Deploys
+#### Usage
 
  1. With Config File (if `config.yml` found in working dir)
     - Download [default config](https://github.com/terorie/od-database-crawler/blob/master/config.yml)
     - Set `server.url` and `server.token`
     - Start with `./od-database-crawler server --config <file>`
-
+ 
  2. With Flags or env
     - Override config file if it exists
     - `--help` for list of flags
     - Every flag is available as an environment variable:
       `--server.crawl_stats` ➡️ `OD_SERVER_CRAWL_STATS`
     - Start with `./od-database-crawler server <flags>`
-
- 3. With Docker
-    ```bash
-    docker run \
-        -e OD_SERVER_URL=xxx \
-        -e OD_SERVER_TOKEN=xxx \
-        terorie/od-database-crawler
-    ```
-
-### Flag reference
-
-Here are the most important config flags. For more fine control, take a look at `/config.yml`.
-
-| Flag/Environment                                        | Description                                                  | Example                             |
-| ------------------------------------------------------- | ------------------------------------------------------------ | ----------------------------------- |
-| `server.url`<br />`OD_SERVER_URL`                       | OD-DB Server URL                                             | `https://od-db.mine.the-eye.eu/api` |
-| `server.token`<br />`OD_SERVER_TOKEN`                   | OD-DB Server Access Token                                    | _Ask Hexa **TM**_                   |
-| `server.recheck`<br />`OD_SERVER_RECHECK`               | Job Fetching Interval                                        | `3s`                                |
-| `output.crawl_stats`<br />`OD_OUTPUT_CRAWL_STATS`       | Crawl Stats Logging Interval (0 = disabled)                  | `500ms`                             |
-| `output.resource_stats`<br />`OD_OUTPUT_RESORUCE_STATS` | Resource Stats Logging Interval (0 = disabled)               | `8s`                                |
-| `output.log`<br />`OD_OUTPUT_LOG`                       | Log File (none = disabled)                                   | `crawler.log`                       |
-| `crawl.tasks`<br />`OD_CRAWL_TASKS`                     | Max number of sites to crawl concurrently                    | `500`                               |
-| `crawl.connections`<br />`OD_CRAWL_CONNECTIONS`         | HTTP connections per site                                    | `1`                                 |
-| `crawl.retries`<br />`OD_CRAWL_RETRIES`                 | How often to retry after a temporary failure (e.g. `HTTP 429` or timeouts) | `5`                                 |
-| `crawl.dial_timeout`<br />`OD_CRAWL_DIAL_TIMEOUT`       | TCP Connect timeout                                          | `5s`                                |
-| `crawl.timeout`<br />`OD_CRAWL_TIMEOUT`                 | HTTP request timeout                                         | `20s`                               |
-| `crawl.user-agent`<br />`OD_CRAWL_USER_AGENT`           | HTTP Crawler User-Agent                                      | `googlebot/1.2.3`                   |
-| `crawl.job_buffer`<br />`OD_CRAWL_JOB_BUFFER`           | Number of URLs to keep in memory/cache, per job. The rest is offloaded to disk. Decrease this value if the crawler uses too much RAM. (0 = Disable Cache, -1 = Only use Cache) | `5000`                              |

config.go

@@ -4,7 +4,6 @@ import (
 	"bufio"
 	"fmt"
 	"github.com/sirupsen/logrus"
-	"github.com/spf13/pflag"
 	"github.com/spf13/viper"
 	"io"
 	"os"
@@ -27,8 +26,6 @@ var config struct {
 	JobBufferSize int
 }
 
-var onlineMode bool
-
 const (
 	ConfServerUrl  = "server.url"
 	ConfToken      = "server.token"
@@ -57,56 +54,8 @@ const (
 func prepareConfig() {
 	pf := rootCmd.PersistentFlags()
 
-	pf.SortFlags = false
-	pf.StringVar(&configFile, "config", "", "Config file")
-	configFile = os.Getenv("OD_CONFIG")
-
-	pf.String(ConfServerUrl, "http://od-db.the-eye.eu/api", "OD-DB server URL")
-
-	pf.String(ConfToken, "", "OD-DB access token (env OD_SERVER_TOKEN)")
-
-	pf.Duration(ConfServerTimeout, 60 * time.Second, "OD-DB request timeout")
-
-	pf.Duration(ConfRecheck, 1 * time.Second, "OD-DB: Poll interval for new jobs")
-
-	pf.Duration(ConfCooldown, 30 * time.Second, "OD-DB: Time to wait after a server-side error")
-
-	pf.String(ConfChunkSize, "1 MB", "OD-DB: Result upload chunk size")
-
-	pf.Uint(ConfUploadRetries, 10, "OD-DB: Max upload retries")
-
-	pf.Duration(ConfUploadRetryInterval, 30 * time.Second, "OD-DB: Time to wait between upload retries")
-
-	pf.Uint(ConfTasks, 100, "Crawler: Max concurrent tasks")
-
-	pf.Uint(ConfWorkers, 4, "Crawler: Connections per server")
-
-	pf.Uint(ConfRetries, 5, "Crawler: Request retries")
-
-	pf.Duration(ConfDialTimeout, 10 * time.Second, "Crawler: Handshake timeout")
-
-	pf.Duration(ConfTimeout, 30 * time.Second, "Crawler: Request timeout")
-
-	pf.String(ConfUserAgent, "Mozilla/5.0 (X11; od-database-crawler) Gecko/20100101 Firefox/52.0", "Crawler: User-Agent")
-
-	pf.Uint(ConfJobBufferSize, 5000, "Crawler: Task queue cache size")
-
-	pf.Duration(ConfCrawlStats, time.Second, "Log: Crawl stats interval")
-
-	pf.Duration(ConfAllocStats, 10 * time.Second, "Log: Resource stats interval")
-
-	pf.Bool(ConfVerbose, false, "Log: Print every listed dir")
-
-	pf.Bool(ConfPrintHTTP, false, "Log: Print HTTP client errors")
-
-	pf.String(ConfLogFile, "crawler.log", "Log file")
-
-	// Bind all flags to Viper
-	pf.VisitAll(func(flag *pflag.Flag) {
-		s := flag.Name
-		s = strings.TrimLeft(s, "-")
-
-		if err := viper.BindPFlag(s, flag); err != nil {
+	bind := func(s string) {
+		if err := viper.BindPFlag(s, pf.Lookup(s)); err != nil {
 			panic(err)
 		}
 		var envKey string
@@ -116,7 +65,71 @@ func prepareConfig() {
 		if err := viper.BindEnv(s, envKey); err != nil {
 			panic(err)
 		}
-	})
+	}
+
+	pf.SortFlags = false
+	pf.StringVar(&configFile, "config", "", "Config file")
+	configFile = os.Getenv("OD_CONFIG")
+
+	pf.String(ConfServerUrl, "http://od-db.the-eye.eu/api", "OD-DB server URL")
+	bind(ConfServerUrl)
+
+	pf.String(ConfToken, "", "OD-DB access token (env OD_SERVER_TOKEN)")
+	bind(ConfToken)
+
+	pf.Duration(ConfServerTimeout, 60 * time.Second, "OD-DB request timeout")
+	bind(ConfServerTimeout)
+
+	pf.Duration(ConfRecheck, 1 * time.Second, "OD-DB: Poll interval for new jobs")
+	bind(ConfRecheck)
+
+	pf.Duration(ConfCooldown, 30 * time.Second, "OD-DB: Time to wait after a server-side error")
+	bind(ConfCooldown)
+
+	pf.String(ConfChunkSize, "1 MB", "OD-DB: Result upload chunk size")
+	bind(ConfChunkSize)
+
+	pf.Uint(ConfUploadRetries, 10, "OD-DB: Max upload retries")
+	bind(ConfUploadRetries)
+
+	pf.Duration(ConfUploadRetryInterval, 30 * time.Second, "OD-DB: Time to wait between upload retries")
+	bind(ConfUploadRetryInterval)
+
+	pf.Uint(ConfTasks, 100, "Crawler: Max concurrent tasks")
+	bind(ConfTasks)
+
+	pf.Uint(ConfWorkers, 4, "Crawler: Connections per server")
+	bind(ConfWorkers)
+
+	pf.Uint(ConfRetries, 5, "Crawler: Request retries")
+	bind(ConfRetries)
+
+	pf.Duration(ConfDialTimeout, 10 * time.Second, "Crawler: Handshake timeout")
+	bind(ConfDialTimeout)
+
+	pf.Duration(ConfTimeout, 30 * time.Second, "Crawler: Request timeout")
+	bind(ConfTimeout)
+
+	pf.String(ConfUserAgent, "Mozilla/5.0 (X11; od-database-crawler) Gecko/20100101 Firefox/52.0", "Crawler: User-Agent")
+	bind(ConfUserAgent)
+
+	pf.Uint(ConfJobBufferSize, 5000, "Crawler: Task queue cache size")
+	bind(ConfJobBufferSize)
+
+	pf.Duration(ConfCrawlStats, time.Second, "Log: Crawl stats interval")
+	bind(ConfCrawlStats)
+
+	pf.Duration(ConfAllocStats, 10 * time.Second, "Log: Resource stats interval")
+	bind(ConfAllocStats)
+
+	pf.Bool(ConfVerbose, false, "Log: Print every listed dir")
+	bind(ConfVerbose)
+
+	pf.Bool(ConfPrintHTTP, false, "Log: Print HTTP client errors")
+	bind(ConfPrintHTTP)
+
+	pf.String(ConfLogFile, "crawler.log", "Log file")
+	bind(ConfLogFile)
 }
 
 func readConfig() {
@@ -144,17 +157,15 @@ func readConfig() {
 		}
 	}
 
-	if onlineMode {
-		config.ServerUrl = viper.GetString(ConfServerUrl)
-		if config.ServerUrl == "" {
-			configMissing(ConfServerUrl)
-		}
-		config.ServerUrl = strings.TrimRight(config.ServerUrl, "/")
+	config.ServerUrl = viper.GetString(ConfServerUrl)
+	if config.ServerUrl == "" {
+		configMissing(ConfServerUrl)
+	}
+	config.ServerUrl = strings.TrimRight(config.ServerUrl, "/")
 
-		config.Token = viper.GetString(ConfToken)
-		if config.Token == "" {
-			configMissing(ConfToken)
-		}
+	config.Token = viper.GetString(ConfToken)
+	if config.Token == "" {
+		configMissing(ConfToken)
 	}
 
 	config.ServerTimeout = viper.GetDuration(ConfServerTimeout)
@@ -183,10 +194,6 @@ func readConfig() {
 
 	config.UserAgent = viper.GetString(ConfUserAgent)
 
-	setDialTimeout(viper.GetDuration(ConfDialTimeout))
-
-	setTimeout(viper.GetDuration(ConfTimeout))
-
 	config.JobBufferSize = viper.GetInt(ConfJobBufferSize)
 
 	config.Verbose = viper.GetBool(ConfVerbose)

config.yml

@@ -47,13 +47,13 @@ output:
 # Crawler settings
 crawl:
   # Number of sites that can be processed at once
-  tasks: 25
+  tasks: 100
 
   # Number of connections per site
   # Please be careful with this setting!
   # The crawler fires fast and more than
   # ten connections can overwhelm a server.
-  connections: 1
+  connections: 4
 
   # How often to retry getting data
   # from the site before giving up
@@ -81,4 +81,4 @@ crawl:
   # in memory.
   # A negative value will cause all jobs
   # to be stored in memory. (Don't do this)
-  job_buffer: -1
+  job_buffer: 5000

crawl.go

@@ -3,6 +3,7 @@ package main
 import (
 	"bytes"
 	"crypto/tls"
+	"github.com/spf13/viper"
 	"github.com/terorie/od-database-crawler/ds/redblackhash"
 	"github.com/terorie/od-database-crawler/fasturl"
 	"github.com/valyala/fasthttp"
@@ -15,24 +16,35 @@ import (
 	"time"
 )
 
-var client = fasthttp.Client {
-	TLSConfig: &tls.Config{
-		InsecureSkipVerify: true,
-	},
+var tlsConfig = tls.Config {
+	InsecureSkipVerify: true,
 }
 
-func setDialTimeout(d time.Duration) {
-	client.Dial = func(addr string) (net.Conn, error) {
-		return fasthttp.DialTimeout(addr, d)
+func newHTTPClient(url fasturl.URL) *fasthttp.PipelineClient {
+	var isTLS bool
+	switch url.Scheme {
+	case fasturl.SchemeHTTP:
+		isTLS = false
+	case fasturl.SchemeHTTPS:
+		isTLS = true
+	}
+
+	url.AddDefaultPort()
+
+	return &fasthttp.PipelineClient {
+		MaxConns:     viper.GetInt(ConfWorkers),
+		Addr:         url.Host,
+		IsTLS:        isTLS,
+		TLSConfig:    &tlsConfig,
+		ReadTimeout:  viper.GetDuration(ConfTimeout),
+		WriteTimeout: viper.GetDuration(ConfTimeout) / 2,
+		Dial: func(addr string) (conn net.Conn, e error) {
+			return fasthttp.DialTimeout(addr, viper.GetDuration(ConfDialTimeout))
+		},
 	}
 }
 
-func setTimeout(d time.Duration) {
-	client.ReadTimeout = d
-	client.WriteTimeout = d / 2
-}
-
-func GetDir(j *Job, f *File) (links []fasturl.URL, err error) {
+func (w *WorkerContext) GetDir(j *Job, f *File) (links []fasturl.URL, err error) {
 	f.IsDir = true
 	f.Name = path.Base(j.Uri.Path)
 
@@ -45,7 +57,7 @@ func GetDir(j *Job, f *File) (links []fasturl.URL, err error) {
 	res := fasthttp.AcquireResponse()
 	defer fasthttp.ReleaseResponse(res)
 
-	err = client.Do(req, res)
+	err = w.client.Do(req, res)
 	fasthttp.ReleaseRequest(req)
 
 	if err != nil {
@@ -123,6 +135,14 @@ func ParseDir(body []byte, baseUri *fasturl.URL) (links []fasturl.URL, err error
 					continue
 				}
 
+				if strings.HasSuffix(link.Path, ".php") {
+					continue
+				}
+
+				if strings.Contains(link.Path, "/cgi-bin/") {
+					continue
+				}
+
 				links = append(links, link)
 			}
 		}
@@ -131,7 +151,7 @@ func ParseDir(body []byte, baseUri *fasturl.URL) (links []fasturl.URL, err error
 	return
 }
 
-func GetFile(u fasturl.URL, f *File) (err error) {
+func (w *WorkerContext) GetFile(u fasturl.URL, f *File) (err error) {
 	f.IsDir = false
 	u.Path = path.Clean(u.Path)
 	f.Name = path.Base(u.Path)
@@ -148,7 +168,7 @@ func GetFile(u fasturl.URL, f *File) (err error) {
 	res.SkipBody = true
 	defer fasthttp.ReleaseResponse(res)
 
-	err = client.Do(req, res)
+	err = w.client.Do(req, res)
 	fasthttp.ReleaseRequest(req)
 
 	if err != nil {

fasturl/url.go

@@ -562,6 +562,20 @@ func validOptionalPort(port string) bool {
 	return true
 }
 
+// TODO Check if RFC-compliant (99% sure not)
+func (u *URL) AddDefaultPort() {
+	if strings.ContainsRune(u.Host, ':') {
+		return
+	}
+
+	switch u.Scheme {
+	case SchemeHTTP:
+		u.Host += ":80"
+	case SchemeHTTPS:
+		u.Host += ":443"
+	}
+}
+
 // String reassembles the URL into a valid URL string.
 // The general form of the result is one of:
 //

go.mod

@@ -1,13 +1,14 @@
-module github.com/terorie/od-database-crawler
+module github.com/syndtr/od-database-crawler
 
 require (
 	github.com/beeker1121/goque v2.0.1+incompatible
 	github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db // indirect
-	github.com/sirupsen/logrus v1.4.0
+	github.com/sirupsen/logrus v1.3.0
 	github.com/spf13/cobra v0.0.3
-	github.com/spf13/viper v1.3.2
+	github.com/spf13/viper v1.3.1
 	github.com/syndtr/goleveldb v0.0.0-20181128100959-b001fa50d6b2 // indirect
-	github.com/valyala/fasthttp v1.2.0
+	github.com/terorie/od-database-crawler v1.1.1
+	github.com/valyala/fasthttp v1.1.0
 	golang.org/x/crypto v0.0.0-20190131182504-b8fe1690c613
 	golang.org/x/net v0.0.0-20180911220305-26e67e76b6c3
 )

go.sum

@@ -25,8 +25,6 @@ github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/9
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/sirupsen/logrus v1.3.0 h1:hI/7Q+DtNZ2kINb6qt/lS+IyXnHQe9e90POfeewL/ME=
 github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
-github.com/sirupsen/logrus v1.4.0 h1:yKenngtzGh+cUSSh6GWbxW2abRqhYUSR/t/6+2QqNvE=
-github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/spf13/afero v1.1.2 h1:m8/z1t7/fwjysjQRYbP0RD+bUIF/8tJwPdEZsI83ACI=
 github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
 github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8=
@@ -39,19 +37,17 @@ github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg=
 github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
 github.com/spf13/viper v1.3.1 h1:5+8j8FTpnFV4nEImW/ofkzEt8VoOiLXxdYIDsB73T38=
 github.com/spf13/viper v1.3.1/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s=
-github.com/spf13/viper v1.3.2 h1:VUFqw5KcqRf7i70GOzW7N+Q7+gxVBkSSqiXB12+JQ4M=
-github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/syndtr/goleveldb v0.0.0-20181128100959-b001fa50d6b2 h1:GnOzE5fEFN3b2zDhJJABEofdb51uMRNb8eqIVtdducs=
 github.com/syndtr/goleveldb v0.0.0-20181128100959-b001fa50d6b2/go.mod h1:Z4AUp2Km+PwemOoO/VB5AOx9XSsIItzFjoJlOSiYmn0=
+github.com/terorie/od-database-crawler v1.1.1 h1:Ca+ZqbZX3rVWBR8SDRzvroyxjBtUs75MQXZ9YG0gqGo=
+github.com/terorie/od-database-crawler v1.1.1/go.mod h1:vVJ7pLkudrlUNp9qu24JCzQ8N6mFsrOmX1tPXr155DQ=
 github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
 github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
 github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
 github.com/valyala/fasthttp v1.1.0 h1:3BohG7mqwj4lq7PTX//7gLbUlzNvZSPmuHFnloXT0lw=
 github.com/valyala/fasthttp v1.1.0/go.mod h1:4vX61m6KN+xDduDNwXrhIAVZaZaZiQ1luJk8LWSxF3s=
-github.com/valyala/fasthttp v1.2.0 h1:dzZJf2IuMiclVjdw0kkT+f9u4YdrapbNyGAN47E/qnk=
-github.com/valyala/fasthttp v1.2.0/go.mod h1:4vX61m6KN+xDduDNwXrhIAVZaZaZiQ1luJk8LWSxF3s=
 github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio=
 github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=

main.go

@@ -8,7 +8,6 @@ import (
 	"github.com/spf13/viper"
 	"github.com/terorie/od-database-crawler/fasturl"
 	"os"
-	"os/signal"
 	"strings"
 	"sync/atomic"
 	"time"
@@ -18,7 +17,7 @@ var configFile string
 
 var rootCmd = cobra.Command {
 	Use: "od-database-crawler",
-	Version: "1.2.2",
+	Version: "1.2.1",
 	Short: "OD-Database Go crawler",
 	Long: helpText,
 	PersistentPreRunE: preRun,
@@ -36,7 +35,7 @@ var serverCmd = cobra.Command {
 }
 
 var crawlCmd = cobra.Command {
-	Use: "crawl",
+	Use: "crawl <url>",
 	Short: "Crawl an URL",
 	Long: "Crawl the URL specified.\n" +
 		"Results will not be uploaded to the database,\n" +
@@ -62,6 +61,8 @@ func preRun(cmd *cobra.Command, args []string) error {
 	if err := os.MkdirAll("queue", 0755);
 		err != nil { panic(err) }
 
+	readConfig()
+
 	return nil
 }
 
@@ -74,31 +75,25 @@ func main() {
 }
 
 func cmdBase(_ *cobra.Command, _ []string) {
-	onlineMode = true
-	readConfig()
-
-	appCtx, soft := context.WithCancel(context.Background())
-	forceCtx, hard := context.WithCancel(context.Background())
-	go hardShutdown(forceCtx)
-	go listenCtrlC(soft, hard)
+	// TODO Graceful shutdown
+	appCtx := context.Background()
+	forceCtx := context.Background()
 
 	inRemotes := make(chan *OD)
-	go Schedule(appCtx, inRemotes)
+	go Schedule(forceCtx, inRemotes)
 
 	ticker := time.NewTicker(config.Recheck)
 	defer ticker.Stop()
 	for {
 		select {
 		case <-appCtx.Done():
-			goto shutdown
+			return
 		case <-ticker.C:
 			t, err := FetchTask()
 			if err != nil {
 				logrus.WithError(err).
 					Error("Failed to get new task")
-				if !sleep(viper.GetDuration(ConfCooldown), appCtx) {
-					goto shutdown
-				}
+				time.Sleep(viper.GetDuration(ConfCooldown))
 				continue
 			}
 			if t == nil {
@@ -114,7 +109,13 @@ func cmdBase(_ *cobra.Command, _ []string) {
 			if urlErr, ok := err.(*fasturl.Error); ok && urlErr.Err == fasturl.ErrUnknownScheme {
 				// Not an error
 				err = nil
-				// TODO FTP crawler
+
+				// Give back task
+				//err2 := CancelTask(t.WebsiteId)
+				//if err2 != nil {
+				//	logrus.Error(err2)
+				//}
+
 				continue
 			} else if err != nil {
 				logrus.WithError(err).
@@ -125,15 +126,9 @@ func cmdBase(_ *cobra.Command, _ []string) {
 			ScheduleTask(inRemotes, t, &baseUri)
 		}
 	}
-
-shutdown:
-	globalWait.Wait()
 }
 
 func cmdCrawler(_ *cobra.Command, args []string) error {
-	onlineMode = false
-	readConfig()
-
 	arg := args[0]
 	// https://github.com/golang/go/issues/19779
 	if !strings.Contains(arg, "://") {
@@ -166,30 +161,3 @@ func cmdCrawler(_ *cobra.Command, args []string) error {
 
 	return nil
 }
-
-func listenCtrlC(soft, hard context.CancelFunc) {
-	c := make(chan os.Signal)
-	signal.Notify(c, os.Interrupt)
-
-	<-c
-	logrus.Info(">>> Shutting down crawler... <<<")
-	soft()
-
-	<-c
-	logrus.Warning(">>> Force shutdown! <<<")
-	hard()
-}
-
-func hardShutdown(c context.Context) {
-	<-c.Done()
-	os.Exit(1)
-}
-
-func sleep(d time.Duration, c context.Context) bool {
-	select {
-	case <-time.After(d):
-		return true
-	case <-c.Done():
-		return false
-	}
-}

scheduler.go

@@ -22,14 +22,16 @@ func Schedule(c context.Context, remotes <-chan *OD) {
 	go Stats(c)
 
 	for remote := range remotes {
+		// Create HTTP client
+		remote.WCtx.OD = remote
+		remote.WCtx.Prepare()
+
 		logrus.WithField("url", remote.BaseUri.String()).
 			Info("Starting crawler")
 
 		// Collect results
 		results := make(chan File)
 
-		remote.WCtx.OD = remote
-
 		// Get queue path
 		queuePath := path.Join("queue", fmt.Sprintf("%d", remote.Task.WebsiteId))
 

server.go

@@ -40,9 +40,7 @@ func FetchTask() (t *Task, err error) {
 
 	t = new(Task)
 	err = json.NewDecoder(res.Body).Decode(t)
-	if _, ok := err.(*json.SyntaxError); ok {
-		return nil, fmt.Errorf("/task/get returned invalid JSON")
-	} else if err != nil { return }
+	if err != nil { return }
 
 	return
 }

worker.go

@@ -3,6 +3,7 @@ package main
 import (
 	"github.com/beeker1121/goque"
 	"github.com/sirupsen/logrus"
+	"github.com/valyala/fasthttp"
 	"math"
 	"sort"
 	"strings"
@@ -18,6 +19,11 @@ type WorkerContext struct {
 	Queue *BufferedQueue
 	lastRateLimit time.Time
 	numRateLimits int
+	client *fasthttp.PipelineClient
+}
+
+func (w *WorkerContext) Prepare() {
+	w.client = newHTTPClient(w.OD.BaseUri)
 }
 
 func (w *WorkerContext) Worker(results chan<- File) {
@@ -91,7 +97,7 @@ func (w *WorkerContext) DoJob(job *Job, f *File) (newJobs []Job, err error) {
 	if len(job.Uri.Path) == 0 { return }
 	if job.Uri.Path[len(job.Uri.Path)-1] == '/' {
 		// Load directory
-		links, err := GetDir(job, f)
+		links, err := w.GetDir(job, f)
 		if err != nil {
 			if !isErrSilent(err) {
 				logrus.WithError(err).
@@ -141,7 +147,7 @@ func (w *WorkerContext) DoJob(job *Job, f *File) (newJobs []Job, err error) {
 		}
 	} else {
 		// Load file
-		err := GetFile(job.Uri, f)
+		err := w.GetFile(job.Uri, f)
 		if err != nil {
 			if !isErrSilent(err) {
 				logrus.WithError(err).