Micro optimization pt. 2

.gitignore

@@ -1,6 +1,3 @@
 /.idea/
 .DS_Store
-/od-database-crawler
-*.log
-/queue/
-/crawled/
+/oddb-go

README.md

@@ -1,9 +1,2 @@
-# od-database Go crawler 🚀
+# oddb Go crawler
 > by terorie 2018 :P
-
- * Crawler for [__OD-Database__](https://github.com/simon987/od-database)
- * Crawls HTTP open directories (standard Web Server Listings)
- * Gets name, path, size and modification time of all files
- * Lightweight and fast: __over 9000 requests per second__ on a standard laptop
-
-https://od-db.the-eye.eu/

config.go

@@ -1,76 +1,42 @@
 package main
 
 import (
-	"bufio"
 	"fmt"
 	"github.com/sirupsen/logrus"
 	"github.com/spf13/viper"
-	"io"
 	"os"
-	"strings"
 	"time"
 )
 
 var config struct {
 	ServerUrl  string
 	Token      string
-	ServerTimeout time.Duration
-	Recheck    time.Duration
-	ChunkSize  int64
 	Retries    int
 	Workers    int
-	UserAgent  string
 	Tasks      int32
 	CrawlStats time.Duration
 	AllocStats time.Duration
 	Verbose    bool
-	PrintHTTP  bool
-	JobBufferSize int
 }
 
 const (
 	ConfServerUrl  = "server.url"
 	ConfToken      = "server.token"
-	ConfServerTimeout = "server.timeout"
-	ConfRecheck    = "server.recheck"
-	ConfCooldown   = "server.cooldown"
-	ConfChunkSize  = "server.upload_chunk"
-	ConfUploadRetries = "server.upload_retries"
-	ConfUploadRetryInterval = "server.upload_retry_interval"
-
 	ConfTasks      = "crawl.tasks"
 	ConfRetries    = "crawl.retries"
 	ConfWorkers    = "crawl.connections"
-	ConfUserAgent  = "crawl.user-agent"
-	ConfDialTimeout = "crawl.dial_timeout"
-	ConfTimeout    = "crawl.timeout"
-	ConfJobBufferSize = "crawl.job_buffer"
-
 	ConfCrawlStats = "output.crawl_stats"
 	ConfAllocStats = "output.resource_stats"
 	ConfVerbose    = "output.verbose"
-	ConfPrintHTTP  = "output.http"
-	ConfLogFile    = "output.log"
 )
 
 func prepareConfig() {
 	viper.SetDefault(ConfRetries, 5)
 	viper.SetDefault(ConfWorkers, 2)
 	viper.SetDefault(ConfTasks, 3)
-	viper.SetDefault(ConfUserAgent, "")
-	viper.SetDefault(ConfDialTimeout, 10 * time.Second)
-	viper.SetDefault(ConfTimeout, 60 * time.Second)
-	viper.SetDefault(ConfJobBufferSize, 5000)
 	viper.SetDefault(ConfCrawlStats, 3 * time.Second)
 	viper.SetDefault(ConfAllocStats, 0)
 	viper.SetDefault(ConfVerbose, false)
-	viper.SetDefault(ConfPrintHTTP, false)
-	viper.SetDefault(ConfLogFile, "")
-	viper.SetDefault(ConfRecheck, 3 * time.Second)
-	viper.SetDefault(ConfCooldown, 30 * time.Second)
-	viper.SetDefault(ConfChunkSize, "1 MB")
-	viper.SetDefault(ConfUploadRetries, 10)
-	viper.SetDefault(ConfUploadRetryInterval, 30 * time.Second)
 }
 
 func readConfig() {
@@ -83,24 +49,14 @@ func readConfig() {
 	}
 
 	config.ServerUrl = viper.GetString(ConfServerUrl)
-	if config.ServerUrl == "" {
-		configMissing(ConfServerUrl)
-	}
-	config.ServerUrl = strings.TrimRight(config.ServerUrl, "/")
+	//if config.ServerUrl == "" {
+	//	configMissing(ConfServerUrl)
+	//}
 
 	config.Token = viper.GetString(ConfToken)
-	if config.Token == "" {
-		configMissing(ConfToken)
-	}
-
-	config.ServerTimeout = viper.GetDuration(ConfServerTimeout)
-
-	config.Recheck = viper.GetDuration(ConfRecheck)
-
-	config.ChunkSize = int64(viper.GetSizeInBytes(ConfChunkSize))
-	if config.ChunkSize < 100 {
-		configOOB(ConfChunkSize, config.ChunkSize)
-	}
+	//if config.Token == "" {
+	//	configMissing(ConfToken)
+	//}
 
 	config.Retries = viper.GetInt(ConfRetries)
 	if config.Retries < 0 {
@@ -117,14 +73,6 @@ func readConfig() {
 		configOOB(ConfTasks, int(config.Tasks))
 	}
 
-	config.UserAgent = viper.GetString(ConfUserAgent)
-
-	setDialTimeout(viper.GetDuration(ConfDialTimeout))
-
-	setTimeout(viper.GetDuration(ConfTimeout))
-
-	config.JobBufferSize = viper.GetInt(ConfJobBufferSize)
-
 	config.CrawlStats = viper.GetDuration(ConfCrawlStats)
 
 	config.AllocStats = viper.GetDuration(ConfAllocStats)
@@ -133,19 +81,6 @@ func readConfig() {
 	if config.Verbose {
 		logrus.SetLevel(logrus.DebugLevel)
 	}
-
-	if filePath := viper.GetString(ConfLogFile); filePath != "" {
-		f, err := os.OpenFile(filePath, os.O_CREATE | os.O_WRONLY | os.O_APPEND, 0644)
-		bufWriter := bufio.NewWriter(f)
-		if err != nil { panic(err) }
-		exitHooks.Add(func() {
-			bufWriter.Flush()
-			f.Close()
-		})
-		logrus.SetOutput(io.MultiWriter(os.Stdout, bufWriter))
-	}
-
-	config.PrintHTTP = viper.GetBool(ConfPrintHTTP)
 }
 
 func configMissing(key string) {
@@ -153,7 +88,7 @@ func configMissing(key string) {
 	os.Exit(1)
 }
 
-func configOOB(key string, v interface{}) {
-	fmt.Fprintf(os.Stderr, "config: illegal value %v for key %s!\n", v, key)
+func configOOB(key string, v int) {
+	fmt.Fprintf(os.Stderr, "config: illegal value %d for %key!\n", v, key)
 	os.Exit(1)
 }

config.yml

@@ -1,84 +1,26 @@
 # OD-Database server settings
 server:
   # Connection URL
-  url: http://od-db.mine.terorie.com/api
-
+  url: localhost:6969
   # Server auth token
   token:
 
-  # Request timeout
-  timeout: 60s
-
-  # Recheck interval
-  # The crawler periodically asks the server
-  # for new jobs. Sets the minimum wait time
-  # between /task/get requests to the server.
-  recheck: 1s
-
-  # Time to wait after receiving an error
-  # from the server. Doesn't apply to uploads.
-  cooldown: 30s
-
-  # Upload chunk size
-  # If the value is too high, the upload fails.
-  upload_chunk: 1 MB
-
-  upload_retries: 10
-  upload_retry_interval: 30s
-
 # Log output settings
 output:
   # Crawl statistics
   crawl_stats: 1s
-
   # CPU/RAM/Job queue stats
-  resource_stats: 10s
-
+  resource_stats: 1s
   # More output? (Every listed dir)
   verbose: false
 
-  # Print HTTP errors (Super spammy)
-  http: false
-
-  # Log file
-  # If empty, no log file is created.
-  log: crawler.log
-
 # Crawler settings
 crawl:
-  # Number of sites that can be processed at once
-  tasks: 100
-
+  # Number of sites that can be
+  # processed at once
+  tasks: 3
   # Number of connections per site
-  # Please be careful with this setting!
-  # The crawler fires fast and more than
-  # ten connections can overwhelm a server.
-  connections: 4
-
+  connections: 2
   # How often to retry getting data
   # from the site before giving up
   retries: 5
-
-  # Time before discarding a failed connection attempt
-  dial_timeout: 10s
-
-  # Time before discarding a network request
-  timeout: 30s
-
-  # Crawler User-Agent
-  # If empty, no User-Agent header is sent.
-  user-agent: "Mozilla/5.0 (X11; od-database-crawler) Gecko/20100101 Firefox/52.0"
-
-  # Job buffer size (per task)
-  # Higher values cause less disk writes
-  # but require more memory.
-  #
-  # The job queue contains all URLs
-  # that should be crawled next.
-  # As it grows very large over time,
-  # it's kept mainly on disk.
-  # This sets how many jobs are kept
-  # in memory.
-  # A negative value will cause all jobs
-  # to be stored in memory. (Don't do this)
-  job_buffer: 5000

crawl.go

@@ -2,45 +2,28 @@ package main
 
 import (
 	"bytes"
-	"crypto/tls"
-	"github.com/terorie/od-database-crawler/ds/redblackhash"
-	"github.com/terorie/od-database-crawler/fasturl"
+	"encoding/base64"
+	"fmt"
+	"github.com/sirupsen/logrus"
 	"github.com/valyala/fasthttp"
 	"golang.org/x/crypto/blake2b"
 	"golang.org/x/net/html"
-	"net"
+	"golang.org/x/net/html/atom"
+	"net/url"
 	"path"
 	"strconv"
 	"strings"
 	"time"
 )
 
-var client = fasthttp.Client {
-	TLSConfig: &tls.Config{
-		InsecureSkipVerify: true,
-	},
-}
+var client fasthttp.Client
 
-func setDialTimeout(d time.Duration) {
-	client.Dial = func(addr string) (net.Conn, error) {
-		return fasthttp.DialTimeout(addr, d)
-	}
-}
-
-func setTimeout(d time.Duration) {
-	client.ReadTimeout = d
-	client.WriteTimeout = d / 2
-}
-
-func GetDir(j *Job, f *File) (links []fasturl.URL, err error) {
+func GetDir(j *Job, f *File) (links []url.URL, err error) {
 	f.IsDir = true
 	f.Name = path.Base(j.Uri.Path)
 
 	req := fasthttp.AcquireRequest()
-	if config.UserAgent != "" {
-		req.Header.SetUserAgent(config.UserAgent)
-	}
-	req.SetRequestURI(j.UriStr)
+	req.SetRequestURI(j.Uri.String())
 
 	res := fasthttp.AcquireResponse()
 	defer fasthttp.ReleaseResponse(res)
@@ -49,68 +32,72 @@ func GetDir(j *Job, f *File) (links []fasturl.URL, err error) {
 	fasthttp.ReleaseRequest(req)
 
 	if err != nil {
+		logrus.Error(err)
 		return
 	}
 
 	err = checkStatusCode(res.StatusCode())
-	if err != nil {
-		return
-	}
+	if err != nil { return }
 
 	body := res.Body()
 	doc := html.NewTokenizer(bytes.NewReader(body))
 
 	var linkHref string
+	var linkTexts []string
 	for {
-		err = nil
-
 		tokenType := doc.Next()
+		token := doc.Token()
 		if tokenType == html.ErrorToken {
 			break
 		}
 
 		switch tokenType {
 		case html.StartTagToken:
-			name, hasAttr := doc.TagName()
-			if len(name) == 1 && name[0] == 'a' {
-				for hasAttr {
-					var ks, vs []byte
-					ks, vs, hasAttr = doc.TagAttr()
-					if bytes.Equal(ks, []byte("href")) {
-						// TODO Check escape
-						linkHref = string(vs)
+			if token.DataAtom == atom.A {
+				for _, attr := range token.Attr {
+					if attr.Key == "href" {
+						linkHref = attr.Val
 						break
 					}
 				}
 			}
 
+		case html.TextToken:
+			if linkHref != "" {
+				linkTexts = append(linkTexts, token.Data)
+			}
+
 		case html.EndTagToken:
-			name, _ := doc.TagName()
-			if len(name) == 1 && name[0] == 'a' {
+			if linkHref != "" && token.DataAtom == atom.A {
 				// Copy params
 				href := linkHref
+				linkText := strings.Join(linkTexts, " ")
 
 				// Reset params
 				linkHref = ""
+				linkTexts = nil
 
-				if strings.LastIndexByte(href, '?') != -1 {
-					continue
+				// TODO Optimized decision tree
+				for _, entry := range urlBlackList {
+					if href == entry {
+						goto nextToken
+					}
+				}
+				for _, entry := range urlPartBlackList {
+					if strings.Contains(href, entry) {
+						goto nextToken
+					}
+				}
+				for _, entry := range fileNameBlackList {
+					if strings.Contains(linkText, entry) {
+						goto nextToken
+					}
 				}
 
-				switch href {
-				case "", " ", ".", "..", "/":
-					continue
-				}
+				subref, err := url.Parse(href)
+				if err != nil { continue }
 
-				if strings.Contains(href, "../") {
-					continue
-				}
-
-				var link fasturl.URL
-				err = j.Uri.ParseRel(&link, href)
-				if err != nil {
-					continue
-				}
+				link := *j.Uri.ResolveReference(subref)
 
 				if link.Scheme != j.Uri.Scheme ||
 					link.Host != j.Uri.Host ||
@@ -122,22 +109,21 @@ func GetDir(j *Job, f *File) (links []fasturl.URL, err error) {
 				links = append(links, link)
 			}
 		}
+
+		nextToken:
 	}
 
 	return
 }
 
-func GetFile(u fasturl.URL, f *File) (err error) {
+func GetFile(u url.URL, f *File) (err error) {
 	f.IsDir = false
 	u.Path = path.Clean(u.Path)
 	f.Name = path.Base(u.Path)
-	f.Path = strings.Trim(path.Dir(u.Path), "/")
+	f.Path = strings.Trim(u.Path, "/")
 
 	req := fasthttp.AcquireRequest()
 	req.Header.SetMethod("HEAD")
-	if config.UserAgent != "" {
-		req.Header.SetUserAgent(config.UserAgent)
-	}
 	req.SetRequestURI(u.String())
 
 	res := fasthttp.AcquireResponse()
@@ -147,22 +133,19 @@ func GetFile(u fasturl.URL, f *File) (err error) {
 	err = client.Do(req, res)
 	fasthttp.ReleaseRequest(req)
 
-	if err != nil {
-		return
-	}
+	if err != nil { return }
 
 	err = checkStatusCode(res.StatusCode())
-	if err != nil {
-		return
-	}
+	if err != nil { return }
 
-	f.applyContentLength(string(res.Header.Peek("content-length")))
-	f.applyLastModified(string(res.Header.Peek("last-modified")))
+	// TODO Inefficient af
+	header := res.Header.Header()
+	f.ParseHeader(header)
 
 	return nil
 }
 
-func (f *File) HashDir(links []fasturl.URL) (o redblackhash.Key) {
+func (f *File) HashDir(links []url.URL) string {
 	h, _ := blake2b.New256(nil)
 	h.Write([]byte(f.Name))
 	for _, link := range links {
@@ -170,46 +153,67 @@ func (f *File) HashDir(links []fasturl.URL) (o redblackhash.Key) {
 		h.Write([]byte(fileName))
 	}
 	sum := h.Sum(nil)
-	copy(o[:redblackhash.KeySize], sum)
-	return
+	b64sum := base64.StdEncoding.EncodeToString(sum)
+	return b64sum
 }
 
-func (f *File) applyContentLength(v string) {
-	if v == "" {
-		return
+func (f *File) ParseHeader(h []byte) {
+	var k1, k2 int
+	var v1, v2 int
+
+	// Simple finite state machine
+	state := 0
+	for i, b := range h {
+		switch state {
+		case 0:
+			if b == byte(':') {
+				state = 1
+				k2 = i
+			}
+
+		case 1:
+			state = 2
+
+		case 2:
+			state = 3
+			v1 = i
+
+		case 3:
+			if b == byte('\r') {
+				state = 4
+			}
+
+		case 4:
+			state = 0
+			v2 = i - 1
+
+			key := string(h[k1:k2])
+			val := string(h[v1:v2])
+			k1 = i
+
+			f.applyHeader(key, val)
+		}
 	}
-	size, err := strconv.ParseInt(v, 10, 64)
-	if err != nil {
-		return
-	}
-	if size < 0 {
-		return
-	}
-	f.Size = size
+
 }
 
-// TODO Cleanup
-func (f *File) applyLastModified(v string) {
-	if v == "" {
-		return
-	}
-	var t time.Time
-	var err error
-	t, err = time.Parse(time.RFC1123, v)
-	if err == nil {
-		f.MTime = t.Unix()
-		return
-	}
-	t, err = time.Parse(time.RFC850, v)
-	if err == nil {
-		f.MTime = t.Unix()
-		return
-	}
-	// TODO Parse asctime
-	t, err = time.Parse("2006-01-02", v[:10])
-	if err == nil {
-		f.MTime = t.Unix()
-		return
+func (f *File) applyHeader(k, v string) {
+	switch k {
+	case "content-length":
+		size, err := strconv.ParseInt(v, 10, 64)
+		if err != nil { break }
+		if size < 0 { break }
+		f.Size = size
+
+	case "last-modified":
+		var err error
+		f.MTime, err = time.Parse(time.RFC1123, v)
+		if err == nil { break }
+		f.MTime, err = time.Parse(time.RFC850, v)
+		if err == nil { break }
+		// TODO Parse asctime
+		f.MTime, err = time.Parse("2006-01-02", v[:10])
+		if err == nil { break }
 	}
 }
 
@@ -217,7 +221,53 @@ func checkStatusCode(status int) error {
 	switch status {
 	case fasthttp.StatusOK:
 		return nil
+
+	case fasthttp.StatusTooManyRequests:
+		return ErrRateLimit
+
+	case fasthttp.StatusForbidden,
+		fasthttp.StatusUnauthorized:
+		return ErrForbidden
+
 	default:
-		return &HttpError{status}
+		return fmt.Errorf("got HTTP status %d", status)
 	}
 }
+
+var urlBlackList = [...]string {
+	"",
+	" ",
+	".",
+	"..",
+	"/",
+}
+
+var urlPartBlackList = [...]string {
+	"?C=N&O=D",
+	"?C=M&O=A",
+	"?C=S&O=A",
+	"?C=D&O=A",
+	"?C=N;O=D",
+	"?C=M;O=A",
+	"?C=M&O=D",
+	"?C=S;O=A",
+	"?C=S&O=D",
+	"?C=D;O=A",
+	"?MA",
+	"?SA",
+	"?DA",
+	"?ND",
+	"?C=N&O=A",
+	"?C=N&O=A",
+	"?M=A",
+	"?N=D",
+	"?S=A",
+	"?D=A",
+}
+
+var fileNameBlackList = [...]string {
+	"Parent Directory",
+	" Parent Directory",
+	"../",
+}
+

ds/redblackhash/redblack.go

@@ -1,489 +0,0 @@
-// Copyright (c) 2015, Emir Pasic. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// Modifications by terorie
-
-// Package redblacktree implements a red-black tree.
-//
-// Used by TreeSet and TreeMap.
-//
-// Structure is not thread safe.
-//
-// References: http://en.wikipedia.org/wiki/Red%E2%80%93black_tree
-package redblackhash
-
-import (
-	"bytes"
-	"fmt"
-	"sync"
-)
-
-const (
-	black, red color = true, false
-	KeySize = 64
-)
-
-type color bool
-type Key [KeySize]byte
-
-// Tree holds elements of the red-black tree
-type Tree struct {
-	sync.Mutex
-	Root       *Node
-	size       int
-}
-
-// Node is a single element within the tree
-type Node struct {
-	Key    Key
-	color  color
-	Left   *Node
-	Right  *Node
-	Parent *Node
-}
-
-func (k *Key) Compare(o *Key) int {
-	return bytes.Compare(k[:], o[:])
-}
-
-// Put inserts node into the tree.
-// Key should adhere to the comparator's type assertion, otherwise method panics.
-func (tree *Tree) Put(key *Key) {
-	var insertedNode *Node
-	if tree.Root == nil {
-		// Assert key is of comparator's type for initial tree
-		tree.Root = &Node{Key: *key, color: red}
-		insertedNode = tree.Root
-	} else {
-		node := tree.Root
-		loop := true
-		for loop {
-			compare := key.Compare(&node.Key)
-			switch {
-			case compare == 0:
-				node.Key = *key
-				return
-			case compare < 0:
-				if node.Left == nil {
-					node.Left = &Node{Key: *key, color: red}
-					insertedNode = node.Left
-					loop = false
-				} else {
-					node = node.Left
-				}
-			case compare > 0:
-				if node.Right == nil {
-					node.Right = &Node{Key: *key, color: red}
-					insertedNode = node.Right
-					loop = false
-				} else {
-					node = node.Right
-				}
-			}
-		}
-		insertedNode.Parent = node
-	}
-	tree.insertCase1(insertedNode)
-	tree.size++
-}
-
-// Get searches the node in the tree by key and returns its value or nil if key is not found in tree.
-// Second return parameter is true if key was found, otherwise false.
-// Key should adhere to the comparator's type assertion, otherwise method panics.
-func (tree *Tree) Get(key *Key) (found bool) {
-	node := tree.lookup(key)
-	return node != nil
-}
-
-// Remove remove the node from the tree by key.
-// Key should adhere to the comparator's type assertion, otherwise method panics.
-func (tree *Tree) Remove(key *Key) {
-	var child *Node
-	node := tree.lookup(key)
-	if node == nil {
-		return
-	}
-	if node.Left != nil && node.Right != nil {
-		pred := node.Left.maximumNode()
-		node.Key = pred.Key
-		node = pred
-	}
-	if node.Left == nil || node.Right == nil {
-		if node.Right == nil {
-			child = node.Left
-		} else {
-			child = node.Right
-		}
-		if node.color == black {
-			node.color = nodeColor(child)
-			tree.deleteCase1(node)
-		}
-		tree.replaceNode(node, child)
-		if node.Parent == nil && child != nil {
-			child.color = black
-		}
-	}
-	tree.size--
-}
-
-// Empty returns true if tree does not contain any nodes
-func (tree *Tree) Empty() bool {
-	return tree.size == 0
-}
-
-// Size returns number of nodes in the tree.
-func (tree *Tree) Size() int {
-	return tree.size
-}
-
-// Left returns the left-most (min) node or nil if tree is empty.
-func (tree *Tree) Left() *Node {
-	var parent *Node
-	current := tree.Root
-	for current != nil {
-		parent = current
-		current = current.Left
-	}
-	return parent
-}
-
-// Right returns the right-most (max) node or nil if tree is empty.
-func (tree *Tree) Right() *Node {
-	var parent *Node
-	current := tree.Root
-	for current != nil {
-		parent = current
-		current = current.Right
-	}
-	return parent
-}
-
-// Floor Finds floor node of the input key, return the floor node or nil if no floor is found.
-// Second return parameter is true if floor was found, otherwise false.
-//
-// Floor node is defined as the largest node that is smaller than or equal to the given node.
-// A floor node may not be found, either because the tree is empty, or because
-// all nodes in the tree are larger than the given node.
-//
-// Key should adhere to the comparator's type assertion, otherwise method panics.
-func (tree *Tree) Floor(key *Key) (floor *Node, found bool) {
-	found = false
-	node := tree.Root
-	for node != nil {
-		compare := key.Compare(&node.Key)
-		switch {
-		case compare == 0:
-			return node, true
-		case compare < 0:
-			node = node.Left
-		case compare > 0:
-			floor, found = node, true
-			node = node.Right
-		}
-	}
-	if found {
-		return floor, true
-	}
-	return nil, false
-}
-
-// Ceiling finds ceiling node of the input key, return the ceiling node or nil if no ceiling is found.
-// Second return parameter is true if ceiling was found, otherwise false.
-//
-// Ceiling node is defined as the smallest node that is larger than or equal to the given node.
-// A ceiling node may not be found, either because the tree is empty, or because
-// all nodes in the tree are smaller than the given node.
-//
-// Key should adhere to the comparator's type assertion, otherwise method panics.
-func (tree *Tree) Ceiling(key *Key) (ceiling *Node, found bool) {
-	found = false
-	node := tree.Root
-	for node != nil {
-		compare := key.Compare(&node.Key)
-		switch {
-		case compare == 0:
-			return node, true
-		case compare < 0:
-			ceiling, found = node, true
-			node = node.Left
-		case compare > 0:
-			node = node.Right
-		}
-	}
-	if found {
-		return ceiling, true
-	}
-	return nil, false
-}
-
-// Clear removes all nodes from the tree.
-func (tree *Tree) Clear() {
-	tree.Root = nil
-	tree.size = 0
-}
-
-// String returns a string representation of container
-func (tree *Tree) String() string {
-	str := "RedBlackTree\n"
-	if !tree.Empty() {
-		output(tree.Root, "", true, &str)
-	}
-	return str
-}
-
-func (node *Node) String() string {
-	return fmt.Sprintf("%v", node.Key)
-}
-
-func output(node *Node, prefix string, isTail bool, str *string) {
-	if node.Right != nil {
-		newPrefix := prefix
-		if isTail {
-			newPrefix += "│   "
-		} else {
-			newPrefix += "    "
-		}
-		output(node.Right, newPrefix, false, str)
-	}
-	*str += prefix
-	if isTail {
-		*str += "└── "
-	} else {
-		*str += "┌── "
-	}
-	*str += node.String() + "\n"
-	if node.Left != nil {
-		newPrefix := prefix
-		if isTail {
-			newPrefix += "    "
-		} else {
-			newPrefix += "│   "
-		}
-		output(node.Left, newPrefix, true, str)
-	}
-}
-
-func (tree *Tree) lookup(key *Key) *Node {
-	node := tree.Root
-	for node != nil {
-		compare := key.Compare(&node.Key)
-		switch {
-		case compare == 0:
-			return node
-		case compare < 0:
-			node = node.Left
-		case compare > 0:
-			node = node.Right
-		}
-	}
-	return nil
-}
-
-func (node *Node) grandparent() *Node {
-	if node != nil && node.Parent != nil {
-		return node.Parent.Parent
-	}
-	return nil
-}
-
-func (node *Node) uncle() *Node {
-	if node == nil || node.Parent == nil || node.Parent.Parent == nil {
-		return nil
-	}
-	return node.Parent.sibling()
-}
-
-func (node *Node) sibling() *Node {
-	if node == nil || node.Parent == nil {
-		return nil
-	}
-	if node == node.Parent.Left {
-		return node.Parent.Right
-	}
-	return node.Parent.Left
-}
-
-func (tree *Tree) rotateLeft(node *Node) {
-	right := node.Right
-	tree.replaceNode(node, right)
-	node.Right = right.Left
-	if right.Left != nil {
-		right.Left.Parent = node
-	}
-	right.Left = node
-	node.Parent = right
-}
-
-func (tree *Tree) rotateRight(node *Node) {
-	left := node.Left
-	tree.replaceNode(node, left)
-	node.Left = left.Right
-	if left.Right != nil {
-		left.Right.Parent = node
-	}
-	left.Right = node
-	node.Parent = left
-}
-
-func (tree *Tree) replaceNode(old *Node, new *Node) {
-	if old.Parent == nil {
-		tree.Root = new
-	} else {
-		if old == old.Parent.Left {
-			old.Parent.Left = new
-		} else {
-			old.Parent.Right = new
-		}
-	}
-	if new != nil {
-		new.Parent = old.Parent
-	}
-}
-
-func (tree *Tree) insertCase1(node *Node) {
-	if node.Parent == nil {
-		node.color = black
-	} else {
-		tree.insertCase2(node)
-	}
-}
-
-func (tree *Tree) insertCase2(node *Node) {
-	if nodeColor(node.Parent) == black {
-		return
-	}
-	tree.insertCase3(node)
-}
-
-func (tree *Tree) insertCase3(node *Node) {
-	uncle := node.uncle()
-	if nodeColor(uncle) == red {
-		node.Parent.color = black
-		uncle.color = black
-		node.grandparent().color = red
-		tree.insertCase1(node.grandparent())
-	} else {
-		tree.insertCase4(node)
-	}
-}
-
-func (tree *Tree) insertCase4(node *Node) {
-	grandparent := node.grandparent()
-	if node == node.Parent.Right && node.Parent == grandparent.Left {
-		tree.rotateLeft(node.Parent)
-		node = node.Left
-	} else if node == node.Parent.Left && node.Parent == grandparent.Right {
-		tree.rotateRight(node.Parent)
-		node = node.Right
-	}
-	tree.insertCase5(node)
-}
-
-func (tree *Tree) insertCase5(node *Node) {
-	node.Parent.color = black
-	grandparent := node.grandparent()
-	grandparent.color = red
-	if node == node.Parent.Left && node.Parent == grandparent.Left {
-		tree.rotateRight(grandparent)
-	} else if node == node.Parent.Right && node.Parent == grandparent.Right {
-		tree.rotateLeft(grandparent)
-	}
-}
-
-func (node *Node) maximumNode() *Node {
-	if node == nil {
-		return nil
-	}
-	for node.Right != nil {
-		node = node.Right
-	}
-	return node
-}
-
-func (tree *Tree) deleteCase1(node *Node) {
-	if node.Parent == nil {
-		return
-	}
-	tree.deleteCase2(node)
-}
-
-func (tree *Tree) deleteCase2(node *Node) {
-	sibling := node.sibling()
-	if nodeColor(sibling) == red {
-		node.Parent.color = red
-		sibling.color = black
-		if node == node.Parent.Left {
-			tree.rotateLeft(node.Parent)
-		} else {
-			tree.rotateRight(node.Parent)
-		}
-	}
-	tree.deleteCase3(node)
-}
-
-func (tree *Tree) deleteCase3(node *Node) {
-	sibling := node.sibling()
-	if nodeColor(node.Parent) == black &&
-		nodeColor(sibling) == black &&
-		nodeColor(sibling.Left) == black &&
-		nodeColor(sibling.Right) == black {
-		sibling.color = red
-		tree.deleteCase1(node.Parent)
-	} else {
-		tree.deleteCase4(node)
-	}
-}
-
-func (tree *Tree) deleteCase4(node *Node) {
-	sibling := node.sibling()
-	if nodeColor(node.Parent) == red &&
-		nodeColor(sibling) == black &&
-		nodeColor(sibling.Left) == black &&
-		nodeColor(sibling.Right) == black {
-		sibling.color = red
-		node.Parent.color = black
-	} else {
-		tree.deleteCase5(node)
-	}
-}
-
-func (tree *Tree) deleteCase5(node *Node) {
-	sibling := node.sibling()
-	if node == node.Parent.Left &&
-		nodeColor(sibling) == black &&
-		nodeColor(sibling.Left) == red &&
-		nodeColor(sibling.Right) == black {
-		sibling.color = red
-		sibling.Left.color = black
-		tree.rotateRight(sibling)
-	} else if node == node.Parent.Right &&
-		nodeColor(sibling) == black &&
-		nodeColor(sibling.Right) == red &&
-		nodeColor(sibling.Left) == black {
-		sibling.color = red
-		sibling.Right.color = black
-		tree.rotateLeft(sibling)
-	}
-	tree.deleteCase6(node)
-}
-
-func (tree *Tree) deleteCase6(node *Node) {
-	sibling := node.sibling()
-	sibling.color = nodeColor(node.Parent)
-	node.Parent.color = black
-	if node == node.Parent.Left && nodeColor(sibling.Right) == red {
-		sibling.Right.color = black
-		tree.rotateLeft(node.Parent)
-	} else if nodeColor(sibling.Left) == red {
-		sibling.Left.color = black
-		tree.rotateRight(node.Parent)
-	}
-}
-
-func nodeColor(node *Node) color {
-	if node == nil {
-		return black
-	}
-	return node.color
-}

errors.go

@@ -1,17 +1,8 @@
 package main
 
-import (
-	"errors"
-	"fmt"
-)
+import "errors"
 
 var ErrRateLimit = errors.New("too many requests")
+var ErrForbidden = errors.New("access denied")
 var ErrKnown     = errors.New("already crawled")
 
-type HttpError struct {
-	code int
-}
-
-func (e HttpError) Error() string {
-	return fmt.Sprintf("http status %d", e.code)
-}

fasturl/url.go

@@ -1,869 +0,0 @@
-// Copyright 2009 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// Package fasturl parses URLs and implements query escaping.
-package fasturl
-
-// Modifications by terorie
-
-// See RFC 3986. This package generally follows RFC 3986, except where
-// it deviates for compatibility reasons. When sending changes, first
-// search old issues for history on decisions. Unit tests should also
-// contain references to issue numbers with details.
-
-import (
-	"errors"
-	"fmt"
-	"strconv"
-	"strings"
-)
-
-type Scheme int
-const (
-	SchemeInvalid = iota
-	SchemeHTTP
-	SchemeHTTPS
-	SchemeCount
-)
-
-var Schemes = [SchemeCount]string {
-	"",
-	"http",
-	"https",
-}
-
-var ErrUnknownScheme = errors.New("unknown protocol scheme")
-
-// Error reports an error and the operation and URL that caused it.
-type Error struct {
-	Op  string
-	URL string
-	Err error
-}
-
-func (e *Error) Error() string { return e.Op + " " + e.URL + ": " + e.Err.Error() }
-
-type timeout interface {
-	Timeout() bool
-}
-
-func (e *Error) Timeout() bool {
-	t, ok := e.Err.(timeout)
-	return ok && t.Timeout()
-}
-
-type temporary interface {
-	Temporary() bool
-}
-
-func (e *Error) Temporary() bool {
-	t, ok := e.Err.(temporary)
-	return ok && t.Temporary()
-}
-
-func ishex(c byte) bool {
-	switch {
-	case '0' <= c && c <= '9':
-		return true
-	case 'a' <= c && c <= 'f':
-		return true
-	case 'A' <= c && c <= 'F':
-		return true
-	}
-	return false
-}
-
-func unhex(c byte) byte {
-	switch {
-	case '0' <= c && c <= '9':
-		return c - '0'
-	case 'a' <= c && c <= 'f':
-		return c - 'a' + 10
-	case 'A' <= c && c <= 'F':
-		return c - 'A' + 10
-	}
-	return 0
-}
-
-type encoding int
-
-const (
-	encodePath encoding = 1 + iota
-	encodePathSegment
-	encodeHost
-	encodeZone
-	encodeUserPassword
-	encodeQueryComponent
-	encodeFragment
-)
-
-type EscapeError string
-
-func (e EscapeError) Error() string {
-	return "invalid URL escape " + strconv.Quote(string(e))
-}
-
-type InvalidHostError string
-
-func (e InvalidHostError) Error() string {
-	return "invalid character " + strconv.Quote(string(e)) + " in host name"
-}
-
-// Return true if the specified character should be escaped when
-// appearing in a URL string, according to RFC 3986.
-//
-// Please be informed that for now shouldEscape does not check all
-// reserved characters correctly. See golang.org/issue/5684.
-func shouldEscape(c byte, mode encoding) bool {
-	// §2.3 Unreserved characters (alphanum)
-	if 'A' <= c && c <= 'Z' || 'a' <= c && c <= 'z' || '0' <= c && c <= '9' {
-		return false
-	}
-
-	if mode == encodeHost || mode == encodeZone {
-		// §3.2.2 Host allows
-		//	sub-delims = "!" / "$" / "&" / "'" / "(" / ")" / "*" / "+" / "," / ";" / "="
-		// as part of reg-name.
-		// We add : because we include :port as part of host.
-		// We add [ ] because we include [ipv6]:port as part of host.
-		// We add < > because they're the only characters left that
-		// we could possibly allow, and Parse will reject them if we
-		// escape them (because hosts can't use %-encoding for
-		// ASCII bytes).
-		switch c {
-		case '!', '$', '&', '\'', '(', ')', '*', '+', ',', ';', '=', ':', '[', ']', '<', '>', '"':
-			return false
-		}
-	}
-
-	switch c {
-	case '-', '_', '.', '~': // §2.3 Unreserved characters (mark)
-		return false
-
-	case '$', '&', '+', ',', '/', ':', ';', '=', '?', '@': // §2.2 Reserved characters (reserved)
-		// Different sections of the URL allow a few of
-		// the reserved characters to appear unescaped.
-		switch mode {
-		case encodePath: // §3.3
-			// The RFC allows : @ & = + $ but saves / ; , for assigning
-			// meaning to individual path segments. This package
-			// only manipulates the path as a whole, so we allow those
-			// last three as well. That leaves only ? to escape.
-			return c == '?'
-
-		case encodePathSegment: // §3.3
-			// The RFC allows : @ & = + $ but saves / ; , for assigning
-			// meaning to individual path segments.
-			return c == '/' || c == ';' || c == ',' || c == '?'
-
-		case encodeUserPassword: // §3.2.1
-			// The RFC allows ';', ':', '&', '=', '+', '$', and ',' in
-			// userinfo, so we must escape only '@', '/', and '?'.
-			// The parsing of userinfo treats ':' as special so we must escape
-			// that too.
-			return c == '@' || c == '/' || c == '?' || c == ':'
-
-		case encodeQueryComponent: // §3.4
-			// The RFC reserves (so we must escape) everything.
-			return true
-
-		case encodeFragment: // §4.1
-			// The RFC text is silent but the grammar allows
-			// everything, so escape nothing.
-			return false
-		}
-	}
-
-	if mode == encodeFragment {
-		// RFC 3986 §2.2 allows not escaping sub-delims. A subset of sub-delims are
-		// included in reserved from RFC 2396 §2.2. The remaining sub-delims do not
-		// need to be escaped. To minimize potential breakage, we apply two restrictions:
-		// (1) we always escape sub-delims outside of the fragment, and (2) we always
-		// escape single quote to avoid breaking callers that had previously assumed that
-		// single quotes would be escaped. See issue #19917.
-		switch c {
-		case '!', '(', ')', '*':
-			return false
-		}
-	}
-
-	// Everything else must be escaped.
-	return true
-}
-
-// unescape unescapes a string; the mode specifies
-// which section of the URL string is being unescaped.
-func unescape(s string, mode encoding) (string, error) {
-	// Count %, check that they're well-formed.
-	n := 0
-	hasPlus := false
-	for i := 0; i < len(s); {
-		switch s[i] {
-		case '%':
-			n++
-			if i+2 >= len(s) || !ishex(s[i+1]) || !ishex(s[i+2]) {
-				s = s[i:]
-				if len(s) > 3 {
-					s = s[:3]
-				}
-				return "", EscapeError(s)
-			}
-			// Per https://tools.ietf.org/html/rfc3986#page-21
-			// in the host component %-encoding can only be used
-			// for non-ASCII bytes.
-			// But https://tools.ietf.org/html/rfc6874#section-2
-			// introduces %25 being allowed to escape a percent sign
-			// in IPv6 scoped-address literals. Yay.
-			if mode == encodeHost && unhex(s[i+1]) < 8 && s[i:i+3] != "%25" {
-				return "", EscapeError(s[i : i+3])
-			}
-			if mode == encodeZone {
-				// RFC 6874 says basically "anything goes" for zone identifiers
-				// and that even non-ASCII can be redundantly escaped,
-				// but it seems prudent to restrict %-escaped bytes here to those
-				// that are valid host name bytes in their unescaped form.
-				// That is, you can use escaping in the zone identifier but not
-				// to introduce bytes you couldn't just write directly.
-				// But Windows puts spaces here! Yay.
-				v := unhex(s[i+1])<<4 | unhex(s[i+2])
-				if s[i:i+3] != "%25" && v != ' ' && shouldEscape(v, encodeHost) {
-					return "", EscapeError(s[i : i+3])
-				}
-			}
-			i += 3
-		case '+':
-			hasPlus = mode == encodeQueryComponent
-			i++
-		default:
-			if (mode == encodeHost || mode == encodeZone) && s[i] < 0x80 && shouldEscape(s[i], mode) {
-				return "", InvalidHostError(s[i : i+1])
-			}
-			i++
-		}
-	}
-
-	if n == 0 && !hasPlus {
-		return s, nil
-	}
-
-	t := make([]byte, len(s)-2*n)
-	j := 0
-	for i := 0; i < len(s); {
-		switch s[i] {
-		case '%':
-			t[j] = unhex(s[i+1])<<4 | unhex(s[i+2])
-			j++
-			i += 3
-		case '+':
-			if mode == encodeQueryComponent {
-				t[j] = ' '
-			} else {
-				t[j] = '+'
-			}
-			j++
-			i++
-		default:
-			t[j] = s[i]
-			j++
-			i++
-		}
-	}
-	return string(t), nil
-}
-
-func escape(s string, mode encoding) string {
-	spaceCount, hexCount := 0, 0
-	for i := 0; i < len(s); i++ {
-		c := s[i]
-		if shouldEscape(c, mode) {
-			if c == ' ' && mode == encodeQueryComponent {
-				spaceCount++
-			} else {
-				hexCount++
-			}
-		}
-	}
-
-	if spaceCount == 0 && hexCount == 0 {
-		return s
-	}
-
-	t := make([]byte, len(s)+2*hexCount)
-	j := 0
-	for i := 0; i < len(s); i++ {
-		switch c := s[i]; {
-		case c == ' ' && mode == encodeQueryComponent:
-			t[j] = '+'
-			j++
-		case shouldEscape(c, mode):
-			t[j] = '%'
-			t[j+1] = "0123456789ABCDEF"[c>>4]
-			t[j+2] = "0123456789ABCDEF"[c&15]
-			j += 3
-		default:
-			t[j] = s[i]
-			j++
-		}
-	}
-	return string(t)
-}
-
-// A URL represents a parsed URL (technically, a URI reference).
-//
-// The general form represented is:
-//
-//	[scheme:][//[userinfo@]host][/]path[?query][#fragment]
-//
-// URLs that do not start with a slash after the scheme are interpreted as:
-//
-//	scheme:opaque[?query][#fragment]
-//
-// Note that the Path field is stored in decoded form: /%47%6f%2f becomes /Go/.
-// A consequence is that it is impossible to tell which slashes in the Path were
-// slashes in the raw URL and which were %2f. This distinction is rarely important,
-// but when it is, code must not use Path directly.
-// The Parse function sets both Path and RawPath in the URL it returns,
-// and URL's String method uses RawPath if it is a valid encoding of Path,
-// by calling the EscapedPath method.
-type URL struct {
-	Scheme     Scheme
-	Host       string    // host or host:port
-	Path       string    // path (relative paths may omit leading slash)
-}
-
-// Maybe rawurl is of the form scheme:path.
-// (Scheme must be [a-zA-Z][a-zA-Z0-9+-.]*)
-// If so, return scheme, path; else return "", rawurl.
-func getscheme(rawurl string) (scheme Scheme, path string, err error) {
-	for i := 0; i < len(rawurl); i++ {
-		c := rawurl[i]
-		switch {
-		case 'a' <= c && c <= 'z' || 'A' <= c && c <= 'Z':
-			// do nothing
-		case '0' <= c && c <= '9' || c == '+' || c == '-' || c == '.':
-			if i == 0 {
-				return SchemeInvalid, rawurl, nil
-			}
-		case c == ':':
-			if i == 0 {
-				return SchemeInvalid, "", errors.New("missing protocol scheme")
-			}
-			switch rawurl[:i] {
-			case "http":
-				scheme = SchemeHTTP
-			case "https":
-				scheme = SchemeHTTPS
-			default:
-				return SchemeInvalid, "", ErrUnknownScheme
-			}
-
-			path = rawurl[i+1:]
-			return
-		default:
-			// we have encountered an invalid character,
-			// so there is no valid scheme
-			return SchemeInvalid, rawurl, nil
-		}
-	}
-	return SchemeInvalid, rawurl, nil
-}
-
-// Maybe s is of the form t c u.
-// If so, return t, c u (or t, u if cutc == true).
-// If not, return s, "".
-func split(s string, c string, cutc bool) (string, string) {
-	i := strings.Index(s, c)
-	if i < 0 {
-		return s, ""
-	}
-	if cutc {
-		return s[:i], s[i+len(c):]
-	}
-	return s[:i], s[i:]
-}
-
-// Parse parses rawurl into a URL structure.
-//
-// The rawurl may be relative (a path, without a host) or absolute
-// (starting with a scheme). Trying to parse a hostname and path
-// without a scheme is invalid but may not necessarily return an
-// error, due to parsing ambiguities.
-func (u *URL) Parse(rawurl string) error {
-	// Cut off #frag
-	s, frag := split(rawurl, "#", true)
-	err := u.parse(s, false)
-	if err != nil {
-		return &Error{"parse", s, err}
-	}
-	if frag == "" {
-		return nil
-	}
-	return nil
-}
-
-// ParseRequestURI parses rawurl into a URL structure. It assumes that
-// rawurl was received in an HTTP request, so the rawurl is interpreted
-// only as an absolute URI or an absolute path.
-// The string rawurl is assumed not to have a #fragment suffix.
-// (Web browsers strip #fragment before sending the URL to a web server.)
-func (u *URL) ParseRequestURI(rawurl string) error {
-	err := u.parse(rawurl, true)
-	if err != nil {
-		return &Error{"parse", rawurl, err}
-	}
-	return nil
-}
-
-// parse parses a URL from a string in one of two contexts. If
-// viaRequest is true, the URL is assumed to have arrived via an HTTP request,
-// in which case only absolute URLs or path-absolute relative URLs are allowed.
-// If viaRequest is false, all forms of relative URLs are allowed.
-func (u *URL) parse(rawurl string, viaRequest bool) error {
-	var rest string
-	var err error
-
-	if rawurl == "" && viaRequest {
-		return errors.New("empty url")
-	}
-
-	if rawurl == "*" {
-		u.Path = "*"
-		return nil
-	}
-
-	// Split off possible leading "http:", "mailto:", etc.
-	// Cannot contain escaped characters.
-	if u.Scheme, rest, err = getscheme(rawurl); err != nil {
-		return err
-	}
-
-	if strings.HasSuffix(rest, "?") && strings.Count(rest, "?") == 1 {
-		rest = rest[:len(rest)-1]
-	} else {
-		rest, _ = split(rest, "?", true)
-	}
-
-	if !strings.HasPrefix(rest, "/") {
-		if u.Scheme != SchemeInvalid {
-			// We consider rootless paths per RFC 3986 as opaque.
-			return nil
-		}
-		if viaRequest {
-			return errors.New("invalid URI for request")
-		}
-
-		// Avoid confusion with malformed schemes, like cache_object:foo/bar.
-		// See golang.org/issue/16822.
-		//
-		// RFC 3986, §3.3:
-		// In addition, a URI reference (Section 4.1) may be a relative-path reference,
-		// in which case the first path segment cannot contain a colon (":") character.
-		colon := strings.Index(rest, ":")
-		slash := strings.Index(rest, "/")
-		if colon >= 0 && (slash < 0 || colon < slash) {
-			// First path segment has colon. Not allowed in relative URL.
-			return errors.New("first path segment in URL cannot contain colon")
-		}
-	}
-
-	if (u.Scheme != SchemeInvalid || !viaRequest && !strings.HasPrefix(rest, "///")) && strings.HasPrefix(rest, "//") {
-		var authority string
-		authority, rest = split(rest[2:], "/", false)
-		u.Host, err = parseAuthority(authority)
-		if err != nil {
-			return err
-		}
-	}
-	u.Path = rest
-	return nil
-}
-
-func parseAuthority(authority string) (host string, err error) {
-	i := strings.LastIndex(authority, "@")
-	if i < 0 {
-		host, err = parseHost(authority)
-	} else {
-		host, err = parseHost(authority[i+1:])
-	}
-	if err != nil {
-		return "", err
-	}
-	if i < 0 {
-		return host, nil
-	}
-	userinfo := authority[:i]
-	if !validUserinfo(userinfo) {
-		return "", errors.New("fasturl: invalid userinfo")
-	}
-	return host, nil
-}
-
-// parseHost parses host as an authority without user
-// information. That is, as host[:port].
-func parseHost(host string) (string, error) {
-	if strings.HasPrefix(host, "[") {
-		// Parse an IP-Literal in RFC 3986 and RFC 6874.
-		// E.g., "[fe80::1]", "[fe80::1%25en0]", "[fe80::1]:80".
-		i := strings.LastIndex(host, "]")
-		if i < 0 {
-			return "", errors.New("missing ']' in host")
-		}
-		colonPort := host[i+1:]
-		if !validOptionalPort(colonPort) {
-			return "", fmt.Errorf("invalid port %q after host", colonPort)
-		}
-
-		// RFC 6874 defines that %25 (%-encoded percent) introduces
-		// the zone identifier, and the zone identifier can use basically
-		// any %-encoding it likes. That's different from the host, which
-		// can only %-encode non-ASCII bytes.
-		// We do impose some restrictions on the zone, to avoid stupidity
-		// like newlines.
-		zone := strings.Index(host[:i], "%25")
-		if zone >= 0 {
-			host1, err := unescape(host[:zone], encodeHost)
-			if err != nil {
-				return "", err
-			}
-			host2, err := unescape(host[zone:i], encodeZone)
-			if err != nil {
-				return "", err
-			}
-			host3, err := unescape(host[i:], encodeHost)
-			if err != nil {
-				return "", err
-			}
-			return host1 + host2 + host3, nil
-		}
-	}
-
-	var err error
-	if host, err = unescape(host, encodeHost); err != nil {
-		return "", err
-	}
-	return host, nil
-}
-
-// validOptionalPort reports whether port is either an empty string
-// or matches /^:\d*$/
-func validOptionalPort(port string) bool {
-	if port == "" {
-		return true
-	}
-	if port[0] != ':' {
-		return false
-	}
-	for _, b := range port[1:] {
-		if b < '0' || b > '9' {
-			return false
-		}
-	}
-	return true
-}
-
-// String reassembles the URL into a valid URL string.
-// The general form of the result is one of:
-//
-//	scheme:opaque?query#fragment
-//	scheme://userinfo@host/path?query#fragment
-//
-// If u.Opaque is non-empty, String uses the first form;
-// otherwise it uses the second form.
-// To obtain the path, String uses u.EscapedPath().
-//
-// In the second form, the following rules apply:
-//	- if u.Scheme is empty, scheme: is omitted.
-//	- if u.User is nil, userinfo@ is omitted.
-//	- if u.Host is empty, host/ is omitted.
-//	- if u.Scheme and u.Host are empty and u.User is nil,
-//	   the entire scheme://userinfo@host/ is omitted.
-//	- if u.Host is non-empty and u.Path begins with a /,
-//	   the form host/path does not add its own /.
-//	- if u.RawQuery is empty, ?query is omitted.
-//	- if u.Fragment is empty, #fragment is omitted.
-func (u *URL) String() string {
-	var buf strings.Builder
-	if u.Scheme != SchemeInvalid {
-		buf.WriteString(Schemes[u.Scheme])
-		buf.WriteByte(':')
-	}
-	if u.Scheme != SchemeInvalid || u.Host != "" {
-		if u.Host != "" || u.Path != "" {
-			buf.WriteString("//")
-		}
-		if h := u.Host; h != "" {
-			buf.WriteString(escape(h, encodeHost))
-		}
-	}
-	path := u.Path
-	if path != "" && path[0] != '/' && u.Host != "" {
-		buf.WriteByte('/')
-	}
-	if buf.Len() == 0 {
-		// RFC 3986 §4.2
-		// A path segment that contains a colon character (e.g., "this:that")
-		// cannot be used as the first segment of a relative-path reference, as
-		// it would be mistaken for a scheme name. Such a segment must be
-		// preceded by a dot-segment (e.g., "./this:that") to make a relative-
-		// path reference.
-		if i := strings.IndexByte(path, ':'); i > -1 && strings.IndexByte(path[:i], '/') == -1 {
-			buf.WriteString("./")
-		}
-	}
-	buf.WriteString(path)
-	return buf.String()
-}
-
-func isRunesDot(r []rune) bool {
-	return len(r) == 1 && r[0] == '.'
-}
-
-func isRunesDoubleDot(r []rune) bool {
-	return len(r) == 2 && r[0] == '.' && r[1] == '.'
-}
-
-// resolvePath applies special path segments from refs and applies
-// them to base, per RFC 3986.
-func resolvePath(base, ref string) string {
-	var full string
-	if ref == "" {
-		full = base
-	} else if ref[0] != '/' {
-		i := strings.LastIndex(base, "/")
-		full = base[:i+1] + ref
-	} else {
-		full = ref
-	}
-	if full == "" {
-		return ""
-	} else if full == "/" {
-		return "/"
-	}
-
-	dst := make([]rune, len(full))
-	dst = dst[0:0]
-
-	start := 0
-	rs := []rune(full)
-	if len(rs) != 0 && rs[0] == '/' {
-		rs = rs[1:]
-	}
-	var stack []int
-	stack = append(stack, 0)
-	for i, c := range rs {
-		if i == len(rs) - 1 {
-			closingSlash := false
-			part := rs[start:]
-			if len(part) == 0 {
-				dst = append(dst, '/')
-			} else if part[len(part)-1] == '/' {
-				part = part[:len(part)-1]
-				closingSlash = true
-			}
-			switch {
-			case isRunesDot(part):
-				dst = append(dst, '/')
-			case isRunesDoubleDot(part):
-				// Cut to the last slash
-				start = i+1
-				dst = dst[:stack[len(stack)-1]]
-				if len(stack) != 1 {
-					stack = stack[:len(stack)-1]
-				}
-				dst = append(dst, '/')
-			default:
-				dst = append(dst, '/')
-				dst = append(dst, part...)
-			}
-			if closingSlash && len(dst) != 0 && dst[len(dst)-1] != '/' {
-				dst = append(dst, '/')
-			}
-		} else if c == '/' {
-			part := rs[start:i]
-			switch {
-			case isRunesDot(part):
-				start = i+1
-			case isRunesDoubleDot(part):
-				// Cut to the last slash
-				start = i+1
-				dst = dst[:stack[len(stack)-1]]
-				if len(stack) != 1 {
-					stack = stack[:len(stack)-1]
-				}
-			default:
-				start = i+1
-				stack = append(stack, len(dst))
-				dst = append(dst, '/')
-				dst = append(dst, part...)
-			}
-		}
-	}
-	return string(dst)
-
-	/*var dst []string
-	src := strings.Split(full, "/")
-	for _, elem := range src {
-		switch elem {
-		case ".":
-			// drop
-		case "..":
-			if len(dst) > 0 {
-				dst = dst[:len(dst)-1]
-			}
-		default:
-			dst = append(dst, elem)
-		}
-	}
-	if last := src[len(src)-1]; last == "." || last == ".." {
-		// Add final slash to the joined path.
-		dst = append(dst, "")
-	}
-	return "/" + strings.TrimPrefix(strings.Join(dst, "/"), "/")*/
-}
-
-// IsAbs reports whether the URL is absolute.
-// Absolute means that it has a non-empty scheme.
-func (u *URL) IsAbs() bool {
-	return u.Scheme != SchemeInvalid
-}
-
-// ParseRel parses a URL in the context of the receiver. The provided URL
-// may be relative or absolute. Parse returns nil, err on parse
-// failure, otherwise its return value is the same as ResolveReference.
-func (u *URL) ParseRel(out *URL, ref string) error {
-	var refurl URL
-
-	err := refurl.Parse(ref)
-	if err != nil {
-		return err
-	}
-
-	u.ResolveReference(out, &refurl)
-	return nil
-}
-
-// ResolveReference resolves a URI reference to an absolute URI from
-// an absolute base URI u, per RFC 3986 Section 5.2. The URI reference
-// may be relative or absolute. ResolveReference always returns a new
-// URL instance, even if the returned URL is identical to either the
-// base or reference. If ref is an absolute URL, then ResolveReference
-// ignores base and returns a copy of ref.
-func (u *URL) ResolveReference(url *URL, ref *URL) {
-	*url = *ref
-	if ref.Scheme == SchemeInvalid {
-		url.Scheme = u.Scheme
-	}
-	if ref.Scheme != SchemeInvalid || ref.Host != "" {
-		// The "absoluteURI" or "net_path" cases.
-		// We can ignore the error from setPath since we know we provided a
-		// validly-escaped path.
-		url.Path = resolvePath(ref.Path, "")
-		return
-	}
-	// The "abs_path" or "rel_path" cases.
-	url.Host = u.Host
-	url.Path = resolvePath(u.Path, ref.Path)
-	return
-}
-
-// Marshaling interface implementations.
-// Would like to implement MarshalText/UnmarshalText but that will change the JSON representation of URLs.
-
-func (u *URL) MarshalBinary() (text []byte, err error) {
-	return []byte(u.String()), nil
-}
-
-func (u *URL) UnmarshalBinary(text []byte) error {
-	var u1 URL
-	err := u1.Parse(string(text))
-	if err != nil {
-		return err
-	}
-	*u = u1
-	return nil
-}
-
-// validUserinfo reports whether s is a valid userinfo string per RFC 3986
-// Section 3.2.1:
-//     userinfo    = *( unreserved / pct-encoded / sub-delims / ":" )
-//     unreserved  = ALPHA / DIGIT / "-" / "." / "_" / "~"
-//     sub-delims  = "!" / "$" / "&" / "'" / "(" / ")"
-//                   / "*" / "+" / "," / ";" / "="
-//
-// It doesn't validate pct-encoded. The caller does that via func unescape.
-func validUserinfo(s string) bool {
-	for _, r := range s {
-		if 'A' <= r && r <= 'Z' {
-			continue
-		}
-		if 'a' <= r && r <= 'z' {
-			continue
-		}
-		if '0' <= r && r <= '9' {
-			continue
-		}
-		switch r {
-		case '-', '.', '_', ':', '~', '!', '$', '&', '\'',
-			'(', ')', '*', '+', ',', ';', '=', '%', '@':
-			continue
-		default:
-			return false
-		}
-	}
-	return true
-}
-
-func PathUnescape(s string) string {
-	newStr, err := pathUnescape(s)
-	if err != nil {
-		return s
-	} else {
-		return newStr
-	}
-}
-
-func pathUnescape(s string) (string, error) {
-	// Count %, check that they're well-formed.
-	n := 0
-	for i := 0; i < len(s); {
-		switch s[i] {
-		case '%':
-			n++
-			if i+2 >= len(s) || !ishex(s[i+1]) || !ishex(s[i+2]) {
-				s = s[i:]
-				if len(s) > 3 {
-					s = s[:3]
-				}
-				return "", EscapeError(s)
-			}
-			i += 3
-		default:
-			i++
-		}
-	}
-
-	if n == 0 {
-		return s, nil
-	}
-
-	t := make([]byte, len(s)-2*n)
-	j := 0
-	for i := 0; i < len(s); {
-		switch s[i] {
-		case '%':
-			t[j] = unhex(s[i+1])<<4 | unhex(s[i+2])
-			j++
-			i += 3
-		case '+':
-			t[j] = '+'
-			j++
-			i++
-		default:
-			t[j] = s[i]
-			j++
-			i++
-		}
-	}
-	return string(t), nil
-}

fasturl/url_test.go

@@ -1,897 +0,0 @@
-// Copyright 2009 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package fasturl
-
-import (
-	"bytes"
-	encodingPkg "encoding"
-	"encoding/gob"
-	"encoding/json"
-	"fmt"
-	"io"
-	"net"
-	"reflect"
-	"testing"
-)
-
-type URLTest struct {
-	in        string
-	out       *URL   // expected parse; RawPath="" means same as Path
-	roundtrip string // expected result of reserializing the URL; empty means same as "in".
-}
-
-var urltests = []URLTest{
-	// no path
-	{
-		"http://www.google.com",
-		&URL{
-			Scheme: SchemeHTTP,
-			Host:   "www.google.com",
-		},
-		"",
-	},
-	// path
-	{
-		"http://www.google.com/",
-		&URL{
-			Scheme: SchemeHTTP,
-			Host:   "www.google.com",
-			Path:   "/",
-		},
-		"",
-	},
-	// %20 outside query
-	{
-		"http://www.google.com/a%20b",
-		&URL{
-			Scheme:   SchemeHTTP,
-			Host:     "www.google.com",
-			Path:     "/a%20b",
-		},
-		"",
-	},
-	// leading // without scheme should create an authority
-	{
-		"//foo",
-		&URL{
-			Host: "foo",
-		},
-		"",
-	},
-	// Three leading slashes isn't an authority, but doesn't return an error.
-	// (We can't return an error, as this code is also used via
-	// ServeHTTP -> ReadRequest -> Parse, which is arguably a
-	// different URL parsing context, but currently shares the
-	// same codepath)
-	{
-		"///threeslashes",
-		&URL{
-			Path: "///threeslashes",
-		},
-		"",
-	},
-	// unescaped @ in username should not confuse host
-	{
-		"http://j@ne:password@google.com",
-		&URL{
-			Scheme: SchemeHTTP,
-			Host:   "google.com",
-		},
-		"http://google.com",
-	},
-	// unescaped @ in password should not confuse host
-	{
-		"http://jane:p@ssword@google.com",
-		&URL{
-			Scheme: SchemeHTTP,
-			Host:   "google.com",
-		},
-		"http://google.com",
-	},
-	// Relative path
-	{
-		"a/b/c",
-		&URL{
-			Path: "a/b/c",
-		},
-		"a/b/c",
-	},
-	// host subcomponent; IPv4 address in RFC 3986
-	{
-		"http://192.168.0.1/",
-		&URL{
-			Scheme: SchemeHTTP,
-			Host:   "192.168.0.1",
-			Path:   "/",
-		},
-		"",
-	},
-	// host and port subcomponents; IPv4 address in RFC 3986
-	{
-		"http://192.168.0.1:8080/",
-		&URL{
-			Scheme: SchemeHTTP,
-			Host:   "192.168.0.1:8080",
-			Path:   "/",
-		},
-		"",
-	},
-	// host subcomponent; IPv6 address in RFC 3986
-	{
-		"http://[fe80::1]/",
-		&URL{
-			Scheme: SchemeHTTP,
-			Host:   "[fe80::1]",
-			Path:   "/",
-		},
-		"",
-	},
-	// host and port subcomponents; IPv6 address in RFC 3986
-	{
-		"http://[fe80::1]:8080/",
-		&URL{
-			Scheme: SchemeHTTP,
-			Host:   "[fe80::1]:8080",
-			Path:   "/",
-		},
-		"",
-	},
-	// host subcomponent; IPv6 address with zone identifier in RFC 6874
-	{
-		"http://[fe80::1%25en0]/", // alphanum zone identifier
-		&URL{
-			Scheme: SchemeHTTP,
-			Host:   "[fe80::1%en0]",
-			Path:   "/",
-		},
-		"",
-	},
-	// host and port subcomponents; IPv6 address with zone identifier in RFC 6874
-	{
-		"http://[fe80::1%25en0]:8080/", // alphanum zone identifier
-		&URL{
-			Scheme: SchemeHTTP,
-			Host:   "[fe80::1%en0]:8080",
-			Path:   "/",
-		},
-		"",
-	},
-	// host subcomponent; IPv6 address with zone identifier in RFC 6874
-	{
-		"http://[fe80::1%25%65%6e%301-._~]/", // percent-encoded+unreserved zone identifier
-		&URL{
-			Scheme: SchemeHTTP,
-			Host:   "[fe80::1%en01-._~]",
-			Path:   "/",
-		},
-		"http://[fe80::1%25en01-._~]/",
-	},
-	// host and port subcomponents; IPv6 address with zone identifier in RFC 6874
-	{
-		"http://[fe80::1%25%65%6e%301-._~]:8080/", // percent-encoded+unreserved zone identifier
-		&URL{
-			Scheme: SchemeHTTP,
-			Host:   "[fe80::1%en01-._~]:8080",
-			Path:   "/",
-		},
-		"http://[fe80::1%25en01-._~]:8080/",
-	},
-	// golang.org/issue/12200 (colon with empty port)
-	{
-		"http://192.168.0.2:8080/foo",
-		&URL{
-			Scheme: SchemeHTTP,
-			Host:   "192.168.0.2:8080",
-			Path:   "/foo",
-		},
-		"",
-	},
-	{
-		"http://192.168.0.2:/foo",
-		&URL{
-			Scheme: SchemeHTTP,
-			Host:   "192.168.0.2:",
-			Path:   "/foo",
-		},
-		"",
-	},
-	{
-		// Malformed IPv6 but still accepted.
-		"http://2b01:e34:ef40:7730:8e70:5aff:fefe:edac:8080/foo",
-		&URL{
-			Scheme: SchemeHTTP,
-			Host:   "2b01:e34:ef40:7730:8e70:5aff:fefe:edac:8080",
-			Path:   "/foo",
-		},
-		"",
-	},
-	{
-		// Malformed IPv6 but still accepted.
-		"http://2b01:e34:ef40:7730:8e70:5aff:fefe:edac:/foo",
-		&URL{
-			Scheme: SchemeHTTP,
-			Host:   "2b01:e34:ef40:7730:8e70:5aff:fefe:edac:",
-			Path:   "/foo",
-		},
-		"",
-	},
-	{
-		"http://[2b01:e34:ef40:7730:8e70:5aff:fefe:edac]:8080/foo",
-		&URL{
-			Scheme: SchemeHTTP,
-			Host:   "[2b01:e34:ef40:7730:8e70:5aff:fefe:edac]:8080",
-			Path:   "/foo",
-		},
-		"",
-	},
-	{
-		"http://[2b01:e34:ef40:7730:8e70:5aff:fefe:edac]:/foo",
-		&URL{
-			Scheme: SchemeHTTP,
-			Host:   "[2b01:e34:ef40:7730:8e70:5aff:fefe:edac]:",
-			Path:   "/foo",
-		},
-		"",
-	},
-	// golang.org/issue/7991 and golang.org/issue/12719 (non-ascii %-encoded in host)
-	{
-		"http://hello.世界.com/foo",
-		&URL{
-			Scheme: SchemeHTTP,
-			Host:   "hello.世界.com",
-			Path:   "/foo",
-		},
-		"http://hello.%E4%B8%96%E7%95%8C.com/foo",
-	},
-	{
-		"http://hello.%e4%b8%96%e7%95%8c.com/foo",
-		&URL{
-			Scheme: SchemeHTTP,
-			Host:   "hello.世界.com",
-			Path:   "/foo",
-		},
-		"http://hello.%E4%B8%96%E7%95%8C.com/foo",
-	},
-	{
-		"http://hello.%E4%B8%96%E7%95%8C.com/foo",
-		&URL{
-			Scheme: SchemeHTTP,
-			Host:   "hello.世界.com",
-			Path:   "/foo",
-		},
-		"",
-	},
-	// golang.org/issue/10433 (path beginning with //)
-	{
-		"http://example.com//foo",
-		&URL{
-			Scheme: SchemeHTTP,
-			Host:   "example.com",
-			Path:   "//foo",
-		},
-		"",
-	},
-	// test that we can reparse the host names we accept.
-	{
-		"http://authority<\"hi\">/foo",
-		&URL{
-			Scheme: SchemeHTTP,
-			Host:   "authority<\"hi\">",
-			Path:   "/foo",
-		},
-		"",
-	},
-}
-
-// more useful string for debugging than fmt's struct printer
-func ufmt(u *URL) string {
-	return fmt.Sprintf("scheme=%q, host=%q, path=%q",
-		Schemes[u.Scheme], u.Host, u.Path)
-}
-
-func BenchmarkString(b *testing.B) {
-	b.StopTimer()
-	b.ReportAllocs()
-	for _, tt := range urltests {
-		var u URL
-		err := u.Parse(tt.in)
-		if err != nil {
-			b.Errorf("Parse(%q) returned error %s", tt.in, err)
-			continue
-		}
-		if tt.roundtrip == "" {
-			continue
-		}
-		b.StartTimer()
-		var g string
-		for i := 0; i < b.N; i++ {
-			g = u.String()
-		}
-		b.StopTimer()
-		if w := tt.roundtrip; b.N > 0 && g != w {
-			b.Errorf("Parse(%q).String() == %q, want %q", tt.in, g, w)
-		}
-	}
-}
-
-func TestParse(t *testing.T) {
-	for _, tt := range urltests {
-		var u URL
-		err := u.Parse(tt.in)
-		if err != nil {
-			t.Errorf("Parse(%q) returned error %v", tt.in, err)
-			continue
-		}
-		if !reflect.DeepEqual(&u, tt.out) {
-			t.Errorf("Parse(%q):\n\tgot  %v\n\twant %v\n", tt.in, ufmt(&u), ufmt(tt.out))
-		}
-	}
-}
-
-const pathThatLooksSchemeRelative = "//not.a.user@not.a.host/just/a/path"
-
-var parseRequestURLTests = []struct {
-	url           string
-	expectedValid bool
-}{
-	{"http://foo.com", true},
-	{"http://foo.com/", true},
-	{"http://foo.com/path", true},
-	{"/", true},
-	{pathThatLooksSchemeRelative, true},
-	{"//not.a.user@%66%6f%6f.com/just/a/path/also", true},
-	{"*", true},
-	{"http://192.168.0.1/", true},
-	{"http://192.168.0.1:8080/", true},
-	{"http://[fe80::1]/", true},
-	{"http://[fe80::1]:8080/", true},
-
-	// Tests exercising RFC 6874 compliance:
-	{"http://[fe80::1%25en0]/", true},                 // with alphanum zone identifier
-	{"http://[fe80::1%25en0]:8080/", true},            // with alphanum zone identifier
-	{"http://[fe80::1%25%65%6e%301-._~]/", true},      // with percent-encoded+unreserved zone identifier
-	{"http://[fe80::1%25%65%6e%301-._~]:8080/", true}, // with percent-encoded+unreserved zone identifier
-
-	{"foo.html", false},
-	{"../dir/", false},
-	{"http://192.168.0.%31/", false},
-	{"http://192.168.0.%31:8080/", false},
-	{"http://[fe80::%31]/", false},
-	{"http://[fe80::%31]:8080/", false},
-	{"http://[fe80::%31%25en0]/", false},
-	{"http://[fe80::%31%25en0]:8080/", false},
-
-	// These two cases are valid as textual representations as
-	// described in RFC 4007, but are not valid as address
-	// literals with IPv6 zone identifiers in URIs as described in
-	// RFC 6874.
-	{"http://[fe80::1%en0]/", false},
-	{"http://[fe80::1%en0]:8080/", false},
-}
-
-func TestParseRequestURI(t *testing.T) {
-	for _, test := range parseRequestURLTests {
-		var u URL
-		err := u.ParseRequestURI(test.url)
-		if test.expectedValid && err != nil {
-			t.Errorf("ParseRequestURI(%q) gave err %v; want no error", test.url, err)
-		} else if !test.expectedValid && err == nil {
-			t.Errorf("ParseRequestURI(%q) gave nil error; want some error", test.url)
-		}
-	}
-
-	var url URL
-	err := url.ParseRequestURI(pathThatLooksSchemeRelative)
-	if err != nil {
-		t.Fatalf("Unexpected error %v", err)
-	}
-	if url.Path != pathThatLooksSchemeRelative {
-		t.Errorf("ParseRequestURI path:\ngot  %q\nwant %q", url.Path, pathThatLooksSchemeRelative)
-	}
-}
-
-var stringURLTests = []struct {
-	url  URL
-	want string
-}{
-	// No leading slash on path should prepend slash on String() call
-	{
-		url: URL{
-			Scheme: SchemeHTTP,
-			Host:   "www.google.com",
-			Path:   "search",
-		},
-		want: "http://www.google.com/search",
-	},
-	// Relative path with first element containing ":" should be prepended with "./", golang.org/issue/17184
-	{
-		url: URL{
-			Path: "this:that",
-		},
-		want: "./this:that",
-	},
-	// Relative path with second element containing ":" should not be prepended with "./"
-	{
-		url: URL{
-			Path: "here/this:that",
-		},
-		want: "here/this:that",
-	},
-	// Non-relative path with first element containing ":" should not be prepended with "./"
-	{
-		url: URL{
-			Scheme: SchemeHTTP,
-			Host:   "www.google.com",
-			Path:   "this:that",
-		},
-		want: "http://www.google.com/this:that",
-	},
-}
-
-func TestURLString(t *testing.T) {
-	for _, tt := range urltests {
-		var u URL
-		err := u.Parse(tt.in)
-		if err != nil {
-			t.Errorf("Parse(%q) returned error %s", tt.in, err)
-			continue
-		}
-		expected := tt.in
-		if tt.roundtrip != "" {
-			expected = tt.roundtrip
-		}
-		s := u.String()
-		if s != expected {
-			t.Errorf("Parse(%q).String() == %q (expected %q)", tt.in, s, expected)
-		}
-	}
-
-	for _, tt := range stringURLTests {
-		if got := tt.url.String(); got != tt.want {
-			t.Errorf("%+v.String() = %q; want %q", tt.url, got, tt.want)
-		}
-	}
-}
-
-var resolvePathTests = []struct {
-	base, ref, expected string
-}{
-	{"a/b", ".", "/a/"},
-	{"a/b", "c", "/a/c"},
-	{"a/b", "..", "/"},
-	{"a/", "..", "/"},
-	{"a/", "../..", "/"},
-	{"a/b/c", "..", "/a/"},
-	{"a/b/c", "../d", "/a/d"},
-	{"a/b/c", ".././d", "/a/d"},
-	{"a/b", "./..", "/"},
-	{"a/./b", ".", "/a/"},
-	{"a/../", ".", "/"},
-	{"a/.././b", "c", "/c"},
-}
-
-func TestResolvePath(t *testing.T) {
-	for _, test := range resolvePathTests {
-		got := resolvePath(test.base, test.ref)
-		if got != test.expected {
-			t.Errorf("For %q + %q got %q; expected %q", test.base, test.ref, got, test.expected)
-		}
-	}
-}
-
-var resolveReferenceTests = []struct {
-	base, rel, expected string
-}{
-	// Absolute URL references
-	{"http://foo.com?a=b", "https://bar.com/", "https://bar.com/"},
-	{"http://foo.com/", "https://bar.com/?a=b", "https://bar.com/"},
-	{"http://foo.com/", "https://bar.com/?", "https://bar.com/"},
-
-	// Path-absolute references
-	{"http://foo.com/bar", "/baz", "http://foo.com/baz"},
-	{"http://foo.com/bar?a=b#f", "/baz", "http://foo.com/baz"},
-	{"http://foo.com/bar?a=b", "/baz?", "http://foo.com/baz"},
-	{"http://foo.com/bar?a=b", "/baz?c=d", "http://foo.com/baz"},
-
-	// Multiple slashes
-	{"http://foo.com/bar", "http://foo.com//baz", "http://foo.com//baz"},
-	{"http://foo.com/bar", "http://foo.com///baz/quux", "http://foo.com///baz/quux"},
-
-	// Scheme-relative
-	{"https://foo.com/bar?a=b", "//bar.com/quux", "https://bar.com/quux"},
-
-	// Path-relative references:
-
-	// ... current directory
-	{"http://foo.com", ".", "http://foo.com/"},
-	{"http://foo.com/bar", ".", "http://foo.com/"},
-	{"http://foo.com/bar/", ".", "http://foo.com/bar/"},
-
-	// ... going down
-	{"http://foo.com", "bar", "http://foo.com/bar"},
-	{"http://foo.com/", "bar", "http://foo.com/bar"},
-	{"http://foo.com/bar/baz", "quux", "http://foo.com/bar/quux"},
-
-	// ... going up
-	{"http://foo.com/bar/baz", "../quux", "http://foo.com/quux"},
-	{"http://foo.com/bar/baz", "../../../../../quux", "http://foo.com/quux"},
-	{"http://foo.com/bar", "..", "http://foo.com/"},
-	{"http://foo.com/bar/baz", "./..", "http://foo.com/"},
-	// ".." in the middle (issue 3560)
-	{"http://foo.com/bar/baz", "quux/dotdot/../tail", "http://foo.com/bar/quux/tail"},
-	{"http://foo.com/bar/baz", "quux/./dotdot/../tail", "http://foo.com/bar/quux/tail"},
-	{"http://foo.com/bar/baz", "quux/./dotdot/.././tail", "http://foo.com/bar/quux/tail"},
-	{"http://foo.com/bar/baz", "quux/./dotdot/./../tail", "http://foo.com/bar/quux/tail"},
-	{"http://foo.com/bar/baz", "quux/./dotdot/dotdot/././../../tail", "http://foo.com/bar/quux/tail"},
-	{"http://foo.com/bar/baz", "quux/./dotdot/dotdot/./.././../tail", "http://foo.com/bar/quux/tail"},
-	{"http://foo.com/bar/baz", "quux/./dotdot/dotdot/dotdot/./../../.././././tail", "http://foo.com/bar/quux/tail"},
-	{"http://foo.com/bar/baz", "quux/./dotdot/../dotdot/../dot/./tail/..", "http://foo.com/bar/quux/dot/"},
-
-	// Remove any dot-segments prior to forming the target URI.
-	// http://tools.ietf.org/html/rfc3986#section-5.2.4
-	{"http://foo.com/dot/./dotdot/../foo/bar", "../baz", "http://foo.com/dot/baz"},
-
-	// Triple dot isn't special
-	{"http://foo.com/bar", "...", "http://foo.com/..."},
-
-	// Fragment
-	{"http://foo.com/bar", ".#frag", "http://foo.com/"},
-	{"http://example.org/", "#!$&%27()*+,;=", "http://example.org/"},
-
-	// Paths with escaping (issue 16947).
-	{"http://foo.com/foo%2fbar/", "../baz", "http://foo.com/baz"},
-	{"http://foo.com/1/2%2f/3%2f4/5", "../../a/b/c", "http://foo.com/1/a/b/c"},
-	{"http://foo.com/1/2/3", "./a%2f../../b/..%2fc", "http://foo.com/1/2/b/..%2fc"},
-	{"http://foo.com/1/2%2f/3%2f4/5", "./a%2f../b/../c", "http://foo.com/1/2%2f/3%2f4/a%2f../c"},
-	{"http://foo.com/foo%20bar/", "../baz", "http://foo.com/baz"},
-	{"http://foo.com/foo", "../bar%2fbaz", "http://foo.com/bar%2fbaz"},
-	{"http://foo.com/foo%2dbar/", "./baz-quux", "http://foo.com/foo%2dbar/baz-quux"},
-
-	// RFC 3986: Normal Examples
-	// http://tools.ietf.org/html/rfc3986#section-5.4.1
-	{"http://a/b/c/d;p?q", "g", "http://a/b/c/g"},
-	{"http://a/b/c/d;p?q", "./g", "http://a/b/c/g"},
-	{"http://a/b/c/d;p?q", "g/", "http://a/b/c/g/"},
-	{"http://a/b/c/d;p?q", "/g", "http://a/g"},
-	{"http://a/b/c/d;p?q", "//g", "http://g"},
-	{"http://a/b/c/d;p?q", "?y", "http://a/b/c/d;p"},
-	{"http://a/b/c/d;p?q", "g?y", "http://a/b/c/g"},
-	{"http://a/b/c/d;p?q", "#s", "http://a/b/c/d;p"},
-	{"http://a/b/c/d;p?q", "g#s", "http://a/b/c/g"},
-	{"http://a/b/c/d;p?q", "g?y#s", "http://a/b/c/g"},
-	{"http://a/b/c/d;p?q", ";x", "http://a/b/c/;x"},
-	{"http://a/b/c/d;p?q", "g;x", "http://a/b/c/g;x"},
-	{"http://a/b/c/d;p?q", "g;x?y#s", "http://a/b/c/g;x"},
-	{"http://a/b/c/d;p?q", "", "http://a/b/c/d;p"},
-	{"http://a/b/c/d;p?q", ".", "http://a/b/c/"},
-	{"http://a/b/c/d;p?q", "./", "http://a/b/c/"},
-	{"http://a/b/c/d;p?q", "..", "http://a/b/"},
-	{"http://a/b/c/d;p?q", "../", "http://a/b/"},
-	{"http://a/b/c/d;p?q", "../g", "http://a/b/g"},
-	{"http://a/b/c/d;p?q", "../..", "http://a/"},
-	{"http://a/b/c/d;p?q", "../../", "http://a/"},
-	{"http://a/b/c/d;p?q", "../../g", "http://a/g"},
-
-	// RFC 3986: Abnormal Examples
-	// http://tools.ietf.org/html/rfc3986#section-5.4.2
-	{"http://a/b/c/d;p?q", "../../../g", "http://a/g"},
-	{"http://a/b/c/d;p?q", "../../../../g", "http://a/g"},
-	{"http://a/b/c/d;p?q", "/./g", "http://a/g"},
-	{"http://a/b/c/d;p?q", "/../g", "http://a/g"},
-	{"http://a/b/c/d;p?q", "g.", "http://a/b/c/g."},
-	{"http://a/b/c/d;p?q", ".g", "http://a/b/c/.g"},
-	{"http://a/b/c/d;p?q", "g..", "http://a/b/c/g.."},
-	{"http://a/b/c/d;p?q", "..g", "http://a/b/c/..g"},
-	{"http://a/b/c/d;p?q", "./../g", "http://a/b/g"},
-	{"http://a/b/c/d;p?q", "./g/.", "http://a/b/c/g/"},
-	{"http://a/b/c/d;p?q", "g/./h", "http://a/b/c/g/h"},
-	{"http://a/b/c/d;p?q", "g/../h", "http://a/b/c/h"},
-	{"http://a/b/c/d;p?q", "g;x=1/./y", "http://a/b/c/g;x=1/y"},
-	{"http://a/b/c/d;p?q", "g;x=1/../y", "http://a/b/c/y"},
-	{"http://a/b/c/d;p?q", "g?y/./x", "http://a/b/c/g"},
-	{"http://a/b/c/d;p?q", "g?y/../x", "http://a/b/c/g"},
-	{"http://a/b/c/d;p?q", "g#s/./x", "http://a/b/c/g"},
-	{"http://a/b/c/d;p?q", "g#s/../x", "http://a/b/c/g"},
-
-	// Extras.
-	{"https://a/b/c/d;p?q", "//g?q", "https://g"},
-	{"https://a/b/c/d;p?q", "//g#s", "https://g"},
-	{"https://a/b/c/d;p?q", "//g/d/e/f?y#s", "https://g/d/e/f"},
-	{"https://a/b/c/d;p#s", "?y", "https://a/b/c/d;p"},
-	{"https://a/b/c/d;p?q#s", "?y", "https://a/b/c/d;p"},
-}
-
-func TestResolveReference(t *testing.T) {
-	mustParse := func(url string) *URL {
-		u := new(URL)
-		err := u.Parse(url)
-		if err != nil {
-			t.Fatalf("Parse(%q) got err %v", url, err)
-		}
-		return u
-	}
-	for _, test := range resolveReferenceTests {
-		base := mustParse(test.base)
-		rel := mustParse(test.rel)
-		var url URL
-		base.ResolveReference(&url, rel)
-		if got := url.String(); got != test.expected {
-			t.Errorf("URL(%q).ResolveReference(%q)\ngot  %q\nwant %q", test.base, test.rel, got, test.expected)
-		}
-	}
-}
-
-type RequestURITest struct {
-	url *URL
-	out string
-}
-
-var requritests = []RequestURITest{
-	{
-		&URL{
-			Scheme: SchemeHTTP,
-			Host:   "example.com",
-			Path:   "",
-		},
-		"/",
-	},
-	{
-		&URL{
-			Scheme: SchemeHTTP,
-			Host:   "example.com",
-			Path:   "/a b",
-		},
-		"/a%20b",
-	},
-	{
-		&URL{
-			Scheme: SchemeHTTP,
-			Host:   "example.com",
-			Path:   "//foo",
-		},
-		"//foo",
-	},
-}
-
-func TestParseErrors(t *testing.T) {
-	tests := []struct {
-		in      string
-		wantErr bool
-	}{
-		{"http://[::1]", false},
-		{"http://[::1]:80", false},
-		{"http://[::1]:namedport", true}, // rfc3986 3.2.3
-		{"http://[::1]/", false},
-		{"http://[::1]a", true},
-		{"http://[::1]%23", true},
-		{"http://[::1%25en0]", false},     // valid zone id
-		{"http://[::1]:", false},          // colon, but no port OK
-		{"http://[::1]:%38%30", true},     // not allowed: % encoding only for non-ASCII
-		{"http://[::1%25%41]", false},     // RFC 6874 allows over-escaping in zone
-		{"http://[%10::1]", true},         // no %xx escapes in IP address
-		{"http://[::1]/%48", false},       // %xx in path is fine
-		{"http://%41:8080/", true},        // not allowed: % encoding only for non-ASCII
-
-		{"http://[]%20%48%54%54%50%2f%31%2e%31%0a%4d%79%48%65%61%64%65%72%3a%20%31%32%33%0a%0a/", true}, // golang.org/issue/11208
-		{"http://a b.com/", true},    // no space in host name please
-	}
-	for _, tt := range tests {
-		var u URL
-		err := u.Parse(tt.in)
-		if tt.wantErr {
-			if err == nil {
-				t.Errorf("Parse(%q) = %#v; want an error", tt.in, u)
-			}
-			continue
-		}
-		if err != nil {
-			t.Logf("Parse(%q) = %v; want no error", tt.in, err)
-		}
-	}
-}
-
-type shouldEscapeTest struct {
-	in     byte
-	mode   encoding
-	escape bool
-}
-
-var shouldEscapeTests = []shouldEscapeTest{
-	// Unreserved characters (§2.3)
-	{'a', encodePath, false},
-	{'a', encodeUserPassword, false},
-	{'a', encodeQueryComponent, false},
-	{'a', encodeFragment, false},
-	{'a', encodeHost, false},
-	{'z', encodePath, false},
-	{'A', encodePath, false},
-	{'Z', encodePath, false},
-	{'0', encodePath, false},
-	{'9', encodePath, false},
-	{'-', encodePath, false},
-	{'-', encodeUserPassword, false},
-	{'-', encodeQueryComponent, false},
-	{'-', encodeFragment, false},
-	{'.', encodePath, false},
-	{'_', encodePath, false},
-	{'~', encodePath, false},
-
-	// User information (§3.2.1)
-	{':', encodeUserPassword, true},
-	{'/', encodeUserPassword, true},
-	{'?', encodeUserPassword, true},
-	{'@', encodeUserPassword, true},
-	{'$', encodeUserPassword, false},
-	{'&', encodeUserPassword, false},
-	{'+', encodeUserPassword, false},
-	{',', encodeUserPassword, false},
-	{';', encodeUserPassword, false},
-	{'=', encodeUserPassword, false},
-
-	// Host (IP address, IPv6 address, registered name, port suffix; §3.2.2)
-	{'!', encodeHost, false},
-	{'$', encodeHost, false},
-	{'&', encodeHost, false},
-	{'\'', encodeHost, false},
-	{'(', encodeHost, false},
-	{')', encodeHost, false},
-	{'*', encodeHost, false},
-	{'+', encodeHost, false},
-	{',', encodeHost, false},
-	{';', encodeHost, false},
-	{'=', encodeHost, false},
-	{':', encodeHost, false},
-	{'[', encodeHost, false},
-	{']', encodeHost, false},
-	{'0', encodeHost, false},
-	{'9', encodeHost, false},
-	{'A', encodeHost, false},
-	{'z', encodeHost, false},
-	{'_', encodeHost, false},
-	{'-', encodeHost, false},
-	{'.', encodeHost, false},
-}
-
-func TestShouldEscape(t *testing.T) {
-	for _, tt := range shouldEscapeTests {
-		if shouldEscape(tt.in, tt.mode) != tt.escape {
-			t.Errorf("shouldEscape(%q, %v) returned %v; expected %v", tt.in, tt.mode, !tt.escape, tt.escape)
-		}
-	}
-}
-
-type timeoutError struct {
-	timeout bool
-}
-
-func (e *timeoutError) Error() string { return "timeout error" }
-func (e *timeoutError) Timeout() bool { return e.timeout }
-
-type temporaryError struct {
-	temporary bool
-}
-
-func (e *temporaryError) Error() string   { return "temporary error" }
-func (e *temporaryError) Temporary() bool { return e.temporary }
-
-type timeoutTemporaryError struct {
-	timeoutError
-	temporaryError
-}
-
-func (e *timeoutTemporaryError) Error() string { return "timeout/temporary error" }
-
-var netErrorTests = []struct {
-	err       error
-	timeout   bool
-	temporary bool
-}{{
-	err:       &Error{"Get", "http://google.com/", &timeoutError{timeout: true}},
-	timeout:   true,
-	temporary: false,
-}, {
-	err:       &Error{"Get", "http://google.com/", &timeoutError{timeout: false}},
-	timeout:   false,
-	temporary: false,
-}, {
-	err:       &Error{"Get", "http://google.com/", &temporaryError{temporary: true}},
-	timeout:   false,
-	temporary: true,
-}, {
-	err:       &Error{"Get", "http://google.com/", &temporaryError{temporary: false}},
-	timeout:   false,
-	temporary: false,
-}, {
-	err:       &Error{"Get", "http://google.com/", &timeoutTemporaryError{timeoutError{timeout: true}, temporaryError{temporary: true}}},
-	timeout:   true,
-	temporary: true,
-}, {
-	err:       &Error{"Get", "http://google.com/", &timeoutTemporaryError{timeoutError{timeout: false}, temporaryError{temporary: true}}},
-	timeout:   false,
-	temporary: true,
-}, {
-	err:       &Error{"Get", "http://google.com/", &timeoutTemporaryError{timeoutError{timeout: true}, temporaryError{temporary: false}}},
-	timeout:   true,
-	temporary: false,
-}, {
-	err:       &Error{"Get", "http://google.com/", &timeoutTemporaryError{timeoutError{timeout: false}, temporaryError{temporary: false}}},
-	timeout:   false,
-	temporary: false,
-}, {
-	err:       &Error{"Get", "http://google.com/", io.EOF},
-	timeout:   false,
-	temporary: false,
-}}
-
-// Test that url.Error implements net.Error and that it forwards
-func TestURLErrorImplementsNetError(t *testing.T) {
-	for i, tt := range netErrorTests {
-		err, ok := tt.err.(net.Error)
-		if !ok {
-			t.Errorf("%d: %T does not implement net.Error", i+1, tt.err)
-			continue
-		}
-		if err.Timeout() != tt.timeout {
-			t.Errorf("%d: err.Timeout(): got %v, want %v", i+1, err.Timeout(), tt.timeout)
-			continue
-		}
-		if err.Temporary() != tt.temporary {
-			t.Errorf("%d: err.Temporary(): got %v, want %v", i+1, err.Temporary(), tt.temporary)
-		}
-	}
-}
-
-var _ encodingPkg.BinaryMarshaler = (*URL)(nil)
-var _ encodingPkg.BinaryUnmarshaler = (*URL)(nil)
-
-func TestJSON(t *testing.T) {
-	var u URL
-	err := u.Parse("https://www.google.com/x?y=z")
-	if err != nil {
-		t.Fatal(err)
-	}
-	js, err := json.Marshal(&u)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	// If only we could implement TextMarshaler/TextUnmarshaler,
-	// this would work:
-	//
-	// if string(js) != strconv.Quote(u.String()) {
-	// 	t.Errorf("json encoding: %s\nwant: %s\n", js, strconv.Quote(u.String()))
-	// }
-
-	u1 := new(URL)
-	err = json.Unmarshal(js, u1)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if u1.String() != u.String() {
-		t.Errorf("json decoded to: %s\nwant: %s\n", u1, &u)
-	}
-}
-
-func TestGob(t *testing.T) {
-	var u URL
-	err := u.Parse("https://www.google.com/x?y=z")
-	if err != nil {
-		t.Fatal(err)
-	}
-	var w bytes.Buffer
-	err = gob.NewEncoder(&w).Encode(&u)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	u1 := new(URL)
-	err = gob.NewDecoder(&w).Decode(u1)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if u1.String() != u.String() {
-		t.Errorf("json decoded to: %s\nwant: %s\n", u1, &u)
-	}
-}

main.go

@@ -3,158 +3,85 @@ package main
 import (
 	"context"
 	"github.com/sirupsen/logrus"
-	"github.com/spf13/viper"
-	"github.com/terorie/od-database-crawler/fasturl"
 	"github.com/urfave/cli"
+	"log"
+	"net/http"
+	_ "net/http/pprof"
+	"net/url"
 	"os"
 	"strings"
-	"sync/atomic"
 	"time"
 )
 
-var configFile string
-
 var app = cli.App {
-	Name:         "od-database-crawler",
+	Name:         "oddb-go",
 	Usage:        "OD-Database Go crawler",
-	Version:      "1.1.1",
+	Version:      "0.1",
 	BashComplete: cli.DefaultAppComplete,
 	Writer:       os.Stdout,
-	Action:       cmdBase,
-	Commands:     []cli.Command {
+	Compiled:     buildDate,
+	Commands:     []cli.Command{
 		{
 			Name:      "crawl",
 			Usage:     "Crawl a list of URLs",
-			ArgsUsage: "<site>",
+			ArgsUsage: "[site, site, ...]",
 			Action:    cmdCrawler,
 		},
 	},
-	Flags: []cli.Flag {
-		cli.StringFlag {
-			Name: "config",
-			EnvVar: "CONFIG",
-			Destination: &configFile,
-		},
-	},
-	Before: func(i *cli.Context) error {
-		if configFile != "" {
-			viper.SetConfigFile(configFile)
-		}
-		return nil
-	},
-	After: func(i *cli.Context) error {
-		exitHooks.Execute()
-		return nil
-	},
 }
 
-var exitHooks Hooks
-
 func init() {
 	prepareConfig()
 }
 
 func main() {
-	if err := os.MkdirAll("crawled", 0755);
-		err != nil { panic(err) }
-
-	if err := os.MkdirAll("queue", 0755);
-		err != nil { panic(err) }
-
-	readConfig()
+	go func() {
+		log.Println(http.ListenAndServe("localhost:42069", nil))
+	}()
 	app.Run(os.Args)
 }
 
-func cmdBase(_ *cli.Context) error {
-	// TODO Graceful shutdown
-	appCtx := context.Background()
-	forceCtx := context.Background()
-
-	inRemotes := make(chan *OD)
-	go Schedule(forceCtx, inRemotes)
-
-	ticker := time.NewTicker(config.Recheck)
-	defer ticker.Stop()
-	for {
-		select {
-		case <-appCtx.Done():
-			return nil
-		case <-ticker.C:
-			t, err := FetchTask()
-			if err != nil {
-				logrus.WithError(err).
-					Error("Failed to get new task")
-				time.Sleep(viper.GetDuration(ConfCooldown))
-				continue
-			}
-			if t == nil {
-				// No new task
-				if atomic.LoadInt32(&numActiveTasks) == 0 {
-					logrus.Info("Waiting …")
-				}
-				continue
-			}
-
-			var baseUri fasturl.URL
-			err = baseUri.Parse(t.Url)
-			if urlErr, ok := err.(*fasturl.Error); ok && urlErr.Err == fasturl.ErrUnknownScheme {
-				// Not an error
-				err = nil
-
-				// Give back task
-				//err2 := CancelTask(t.WebsiteId)
-				//if err2 != nil {
-				//	logrus.Error(err2)
-				//}
-
-				continue
-			} else if err != nil {
-				logrus.WithError(err).
-					Error("Failed to get new task")
-				time.Sleep(viper.GetDuration(ConfCooldown))
-				continue
-			}
-			ScheduleTask(inRemotes, t, &baseUri)
-		}
-	}
-
-	return nil
-}
-
 func cmdCrawler(clic *cli.Context) error {
-	if clic.NArg() != 1 {
+	readConfig()
+
+	if clic.NArg() == 0 {
 		cli.ShowCommandHelpAndExit(clic, "crawl", 1)
 	}
 
-	arg := clic.Args()[0]
-	// https://github.com/golang/go/issues/19779
-	if !strings.Contains(arg, "://") {
-		arg = "http://" + arg
+	args := clic.Args()
+	remotes := make([]*OD, len(args))
+	for i, arg := range args {
+		// https://github.com/golang/go/issues/19779
+		if !strings.Contains(arg, "://") {
+			arg = "http://" + arg
+		}
+		u, err := url.Parse(arg)
+		if !strings.HasSuffix(u.Path, "/") {
+			u.Path += "/"
+		}
+		if err != nil { return err }
+		remotes[i] = &OD{ BaseUri: *u }
 	}
-	var u fasturl.URL
-	err := u.Parse(arg)
-	if !strings.HasSuffix(u.Path, "/") {
-		u.Path += "/"
-	}
-	if err != nil { return err }
 
-	// TODO Graceful shutdown
-	forceCtx := context.Background()
+	c := context.Background()
 
 	inRemotes := make(chan *OD)
-	go Schedule(forceCtx, inRemotes)
+	go Schedule(c, inRemotes)
 
-	ticker := time.NewTicker(3 * time.Second)
-	defer ticker.Stop()
-
-	task := Task {
-		WebsiteId: 0,
-		Url: u.String(),
+	for _, remote := range remotes {
+		globalWait.Add(1)
+		inRemotes <- remote
 	}
-	ScheduleTask(inRemotes, &task, &u)
 
 	// Wait for all jobs to finish
 	globalWait.Wait()
 
+	logrus.Info("All dirs processed!")
+
 	return nil
 }
+
+var buildDate = time.Date(
+	2018, 10, 28,
+	17, 10, 0, 0,
+	time.UTC)

model.go

@@ -1,63 +1,32 @@
 package main
 
 import (
-	"github.com/terorie/od-database-crawler/ds/redblackhash"
-	"github.com/terorie/od-database-crawler/fasturl"
+	"net/url"
 	"sync"
 	"time"
 )
 
-type Task struct {
-	WebsiteId uint64 `json:"website_id"`
-	Url       string `json:"url"`
-}
-
-type TaskResult struct {
-	StatusCode    string    `json:"status_code"`
-	FileCount     uint64    `json:"file_count"`
-	ErrorCount    uint64    `json:"-"`
-	StartTime     time.Time `json:"-"`
-	StartTimeUnix int64     `json:"start_time"`
-	EndTimeUnix   int64     `json:"end_time"`
-	WebsiteId     uint64    `json:"website_id"`
-}
-
 type Job struct {
-	Uri       fasturl.URL
+	OD        *OD
+	Uri       url.URL
 	UriStr    string
 	Fails     int
 	LastError error
 }
 
 type OD struct {
-	Task    Task
-	Result  TaskResult
 	Wait    sync.WaitGroup
-	BaseUri fasturl.URL
+	BaseUri url.URL
+	lock    sync.Mutex
+	Files   []File
 	WCtx    WorkerContext
-	Scanned redblackhash.Tree
+	Scanned sync.Map
 }
 
 type File struct {
-	Name  string `json:"name"`
-	Size  int64  `json:"size"`
-	MTime int64  `json:"mtime"`
-	Path  string `json:"path"`
-	IsDir bool   `json:"-"`
-}
-
-func (o *OD) LoadOrStoreKey(k *redblackhash.Key) (exists bool) {
-	o.Scanned.Lock()
-	defer o.Scanned.Unlock()
-
-	exists = o.Scanned.Get(k)
-	if exists { return true }
-
-	o.Scanned.Put(k)
-	return false
-}
-
-type errorString string
-func (e errorString) Error() string {
-	return string(e)
+	Name  string    `json:"name"`
+	Size  int64     `json:"size"`
+	MTime time.Time `json:"mtime"`
+	Path  string    `json:"path"`
+	IsDir bool      `json:"-"`
 }

queue.go

@@ -1,129 +0,0 @@
-package main
-
-import (
-	"github.com/beeker1121/goque"
-	"os"
-	"sync"
-	"sync/atomic"
-)
-
-type BufferedQueue struct {
-	dataDir string
-	q       *goque.Queue
-	buf     []Job
-	m       sync.Mutex
-}
-
-func OpenQueue(dataDir string) (bq *BufferedQueue, err error) {
-	bq = new(BufferedQueue)
-	if config.JobBufferSize < 0 {
-		return
-	}
-	bq.dataDir = dataDir
-	bq.q, err = goque.OpenQueue(dataDir)
-	if err != nil { return nil, err }
-	return
-}
-
-func (q *BufferedQueue) Enqueue(job *Job) error {
-	atomic.AddInt64(&totalQueued, 1)
-	if q.directEnqueue(job) {
-		return nil
-	}
-
-	var gob JobGob
-	gob.ToGob(job)
-	_, err := q.q.EnqueueObject(gob)
-	return err
-}
-
-func (q *BufferedQueue) Dequeue() (job Job, err error) {
-	if q.directDequeue(&job) {
-		atomic.AddInt64(&totalQueued, -1)
-		return job, nil
-	}
-
-	if config.JobBufferSize < 0 {
-		err = goque.ErrEmpty
-		return
-	}
-
-	var item *goque.Item
-	item, err = q.q.Dequeue()
-	if err != nil { return }
-
-	atomic.AddInt64(&totalQueued, -1)
-
-	var gob JobGob
-	err = item.ToObject(&gob)
-	if err != nil { return }
-	gob.FromGob(&job)
-
-	return
-}
-
-func (q *BufferedQueue) directEnqueue(job *Job) bool {
-	q.m.Lock()
-	defer q.m.Unlock()
-
-	bs := config.JobBufferSize
-	if len(q.buf) < bs || bs < 0 {
-		q.buf = append(q.buf, *job)
-		return true
-	} else {
-		return false
-	}
-}
-
-func (q *BufferedQueue) directDequeue(job *Job) bool {
-	q.m.Lock()
-	defer q.m.Unlock()
-
-	if len(q.buf) > 0 {
-		*job = q.buf[0]
-		q.buf = q.buf[1:]
-		return true
-	} else {
-		return false
-	}
-}
-
-// Always returns nil (But implements io.Closer)
-func (q *BufferedQueue) Close() error {
-	if config.JobBufferSize < 0 {
-		return nil
-	}
-
-	// Close ignoring errors
-	q.q.Close()
-
-	// Delete files
-	if err := os.RemoveAll(q.dataDir);
-		err != nil { panic(err) }
-
-	return nil
-}
-
-type JobGob struct {
-	Uri string
-	Fails int
-	LastError string
-}
-
-func (g *JobGob) ToGob(j *Job) {
-	g.Uri = j.UriStr
-	g.Fails = j.Fails
-	if j.LastError != nil {
-		g.LastError = j.LastError.Error()
-	}
-}
-
-func (g *JobGob) FromGob(j *Job) {
-	if err := j.Uri.Parse(g.Uri);
-		err != nil { panic(err) }
-	j.UriStr = g.Uri
-	j.Fails = g.Fails
-	if g.LastError != "" {
-		j.LastError = errorString(g.LastError)
-	}
-}

release.sh

@@ -1,25 +0,0 @@
-#!/usr/bin/env bash
-
-appname="od-database-crawler"
-tag=$1
-[ -z "$tag" ] && echo "Usage: build <version>" && exit 1
-
-name=${appname}-${tag}-windows.exe
-GOOS="windows" GOARCH="amd64" go build -ldflags="-s -w" -o $name
-gzip -f $name
-echo $name
-
-name=${appname}-${tag}-linux
-GOOS="linux" GOARCH="amd64" go build -ldflags="-s -w" -o $name
-gzip -f $name
-echo $name
-
-name=${appname}-${tag}-mac
-GOOS="darwin" GOARCH="amd64" go build -ldflags="-s -w" -o $name
-gzip -f $name
-echo $name
-
-name=${appname}-${tag}-freebsd
-GOOS="freebsd" GOARCH="amd64" go build -ldflags="-s -w" -o $name
-gzip -f $name
-echo $name

scheduler.go

@@ -2,215 +2,101 @@ package main
 
 import (
 	"context"
-	"encoding/json"
-	"fmt"
 	"github.com/sirupsen/logrus"
-	"github.com/terorie/od-database-crawler/fasturl"
-	"os"
-	"path"
-	"sync"
 	"sync/atomic"
-	"time"
 )
 
-var activeTasksLock sync.Mutex
-var activeTasks = make(map[uint64]bool)
-var numActiveTasks int32
-var totalQueued int64
+var activeTasks int32
+var totalBuffered int64
 
 func Schedule(c context.Context, remotes <-chan *OD) {
 	go Stats(c)
 
-	for remote := range remotes {
-		logrus.WithField("url", remote.BaseUri.String()).
-			Info("Starting crawler")
+	for {
+		select {
+		case <-c.Done():
+			return
 
-		// Collect results
-		results := make(chan File)
+		case remote := <-remotes:
+			logrus.WithField("url", remote.BaseUri.String()).
+				Info("Starting crawler")
 
-		remote.WCtx.OD = remote
+			// Spawn workers
+			remote.WCtx.in, remote.WCtx.out = makeJobBuffer(c)
+			for i := 0; i < config.Workers; i++ {
+				go remote.WCtx.Worker()
+			}
 
-		// Get queue path
-		queuePath := path.Join("queue", fmt.Sprintf("%d", remote.Task.WebsiteId))
+			// Enqueue initial job
+			atomic.AddInt32(&activeTasks, 1)
+			remote.WCtx.queueJob(Job{
+				OD:     remote,
+				Uri:    remote.BaseUri,
+				UriStr: remote.BaseUri.String(),
+				Fails:  0,
+			})
+			globalWait.Done()
 
-		// Delete existing queue
-		if err := os.RemoveAll(queuePath);
-			err != nil { panic(err) }
-
-		// Start new queue
-		var err error
-		remote.WCtx.Queue, err = OpenQueue(queuePath)
-		if err != nil { panic(err) }
-
-		// Spawn workers
-		for i := 0; i < config.Workers; i++ {
-			go remote.WCtx.Worker(results)
+			// Upload result when ready
+			go remote.Watch()
 		}
+	}
+}
 
-		// Enqueue initial job
-		atomic.AddInt32(&numActiveTasks, 1)
-		remote.WCtx.queueJob(Job{
-			Uri:    remote.BaseUri,
-			UriStr: remote.BaseUri.String(),
-			Fails:  0,
-		})
+func (r *OD) Watch() {
+	// Wait for all jobs on remote to finish
+	r.Wait.Wait()
+	close(r.WCtx.in)
+	atomic.AddInt32(&activeTasks, -1)
 
-		// Upload result when ready
-		go remote.Watch(results)
+	logrus.WithField("url", r.BaseUri.String()).
+		Info("Crawler finished")
+}
 
-		// Sleep if max number of tasks are active
-		for atomic.LoadInt32(&numActiveTasks) > config.Tasks {
+func makeJobBuffer(c context.Context) (chan<- Job, <-chan Job) {
+	in  := make(chan Job)
+	out := make(chan Job)
+	go bufferJobs(c, in, out)
+	return in, out
+}
+
+func bufferJobs(c context.Context, in chan Job, out chan Job) {
+	defer close(out)
+	var inQueue []Job
+	outCh := func() chan Job {
+		if len(inQueue) == 0 {
+			return nil
+		}
+		return out
+	}
+	for len(inQueue) > 0 || in != nil {
+		if len(inQueue) == 0 {
 			select {
+			case v, ok := <-in:
+				if !ok {
+					in = nil
+				} else {
+					atomic.AddInt64(&totalBuffered, 1)
+					inQueue = append(inQueue, v)
+				}
+			case <-c.Done():
+				return
+			}
+		} else {
+			select {
+			case v, ok := <-in:
+				if !ok {
+					in = nil
+				} else {
+					atomic.AddInt64(&totalBuffered, 1)
+					inQueue = append(inQueue, v)
+				}
+			case outCh() <- inQueue[0]:
+				atomic.AddInt64(&totalBuffered, -1)
+				inQueue = inQueue[1:]
 			case <-c.Done():
 				return
-			case <-time.After(time.Second):
-				continue
 			}
 		}
 	}
 }
-
-func ScheduleTask(remotes chan<- *OD, t *Task, u *fasturl.URL) {
-	if !t.register() {
-		return
-	}
-
-	globalWait.Add(1)
-	now := time.Now()
-	od := &OD {
-		Task: *t,
-		BaseUri: *u,
-		Result: TaskResult {
-			WebsiteId: t.WebsiteId,
-			StartTime: now,
-			StartTimeUnix: now.Unix(),
-		},
-	}
-	remotes <- od
-}
-
-func (t *Task) register() bool {
-	activeTasksLock.Lock()
-	defer activeTasksLock.Unlock()
-
-	if _, known := activeTasks[t.WebsiteId]; known {
-		return false
-	} else {
-		activeTasks[t.WebsiteId] = true
-		return true
-	}
-}
-
-func (t *Task) unregister() {
-	activeTasksLock.Lock()
-	delete(activeTasks, t.WebsiteId)
-	activeTasksLock.Unlock()
-}
-
-func (o *OD) Watch(results chan File) {
-	// Mark job as completely done
-	defer globalWait.Done()
-	defer o.Task.unregister()
-
-	filePath := path.Join("crawled", fmt.Sprintf("%d.json", o.Task.WebsiteId))
-
-	// Open crawl results file
-	f, err := os.OpenFile(
-		filePath,
-		os.O_CREATE | os.O_RDWR | os.O_TRUNC,
-		0644,
-	)
-	if err != nil {
-		logrus.WithError(err).
-			Error("Failed saving crawl results")
-		return
-	}
-	defer f.Close()
-	defer os.Remove(filePath)
-
-	// Listen for exit code of Collect()
-	collectErrC := make(chan error)
-
-	// Block until all results are written
-	// (closes results channel)
-	o.handleCollect(results, f, collectErrC)
-
-	// Exit code of Collect()
-	err = <-collectErrC
-	close(collectErrC)
-	if err != nil {
-		logrus.WithError(err).
-			Error("Failed saving crawl results")
-		return
-	}
-
-	// Upload results
-	err = PushResult(&o.Result, f)
-	if err != nil {
-		logrus.WithError(err).
-			Error("Failed uploading crawl results")
-		return
-	}
-}
-
-func (o *OD) handleCollect(results chan File, f *os.File, collectErrC chan error) {
-	// Begin collecting results
-	go o.Task.Collect(results, f, collectErrC)
-	defer close(results)
-
-	// Wait for all jobs on remote to finish
-	o.Wait.Wait()
-
-	// Close queue
-	if err := o.WCtx.Queue.Close(); err != nil {
-		panic(err)
-	}
-	atomic.AddInt32(&numActiveTasks, -1)
-
-	// Log finish
-
-	logrus.WithFields(logrus.Fields{
-		"id":  o.Task.WebsiteId,
-		"url": o.BaseUri.String(),
-		"duration": time.Since(o.Result.StartTime),
-	}).Info("Crawler finished")
-
-	// Set status code
-	now := time.Now()
-	o.Result.EndTimeUnix = now.Unix()
-	fileCount := atomic.LoadUint64(&o.Result.FileCount)
-	if fileCount == 0 {
-		errorCount := atomic.LoadUint64(&o.Result.ErrorCount)
-		if errorCount == 0 {
-			o.Result.StatusCode = "empty"
-		} else {
-			o.Result.StatusCode = "directory listing failed"
-		}
-	} else {
-		o.Result.StatusCode = "success"
-	}
-}
-
-func (t *Task) Collect(results chan File, f *os.File, errC chan<- error) {
-	err := t.collect(results, f)
-	if err != nil {
-		logrus.WithError(err).
-			Error("Failed saving crawl results")
-	}
-	errC <- err
-}
-
-func (t *Task) collect(results chan File, f *os.File) error {
-	for result := range results {
-		result.Path = fasturl.PathUnescape(result.Path)
-		result.Name = fasturl.PathUnescape(result.Name)
-		resJson, err := json.Marshal(result)
-		if err != nil { panic(err) }
-		_, err = f.Write(resJson)
-		if err != nil { return err }
-		_, err = f.Write([]byte{'\n'})
-		if err != nil { return err }
-	}
-
-	return nil
-}

server.go

@@ -5,34 +5,39 @@ import (
 	"encoding/json"
 	"fmt"
 	"github.com/sirupsen/logrus"
-	"github.com/spf13/viper"
 	"io"
 	"mime/multipart"
 	"net/http"
 	"net/url"
 	"os"
+	"path/filepath"
 	"strconv"
-	"time"
+	"strings"
 )
 
-var serverClient = http.Client {
-	Timeout: config.ServerTimeout,
-}
+const (
+	fileListChunkSize int64 = 5000000 // 5 mb
+)
+
+var serverClient = http.DefaultClient
 
 func FetchTask() (t *Task, err error) {
-	res, err := serverClient.PostForm(
+	escToken, _ := json.Marshal(config.Token)
+	payload := `{"token":` + string(escToken) + `}`
+
+	req, err := http.NewRequest(
+		http.MethodPost,
 		config.ServerUrl + "/task/get",
-		url.Values{ "token": {config.Token} })
+		strings.NewReader(payload))
+	if err != nil { return }
+
+	res, err := serverClient.Do(req)
 	if err != nil { return }
 	defer res.Body.Close()
 
-	switch res.StatusCode {
-	case 200:
-		break
-	case 404, 500:
-		return nil, nil
-	default:
-		return nil, fmt.Errorf("http %s", res.Status)
+	if res.StatusCode != 200 {
+		err = fmt.Errorf("http %s", res.Status)
+		return
 	}
 
 	t = new(Task)
@@ -42,17 +47,21 @@ func FetchTask() (t *Task, err error) {
 	return
 }
 
-func PushResult(result *TaskResult, f *os.File) (err error) {
-	if result.WebsiteId == 0 {
-		// Not a real result, don't push
-		return nil
-	}
+func PushResult(result *TaskResult) (err error) {
+	filePath := filepath.Join(
+		".", "crawled",
+		fmt.Sprintf("%d.json", result.WebsiteId))
 
-	// Rewind to the beginning of the file
-	_, err = f.Seek(0, 0)
-	if err != nil {
+	defer os.Remove(filePath)
+
+	f, err := os.Open(filePath)
+	if os.IsNotExist(err) {
+		err = fmt.Errorf("cannot upload result: %s does not exist", filePath)
+		return
+	} else if err != nil {
 		return
 	}
+	defer f.Close()
 
 	err = uploadChunks(result.WebsiteId, f)
 	if err != nil {
@@ -64,109 +73,104 @@ func PushResult(result *TaskResult, f *os.File) (err error) {
 		return
 	}
 
-	// Upload result ignoring errors
-	uploadResult(result)
+	err = uploadResult(result)
+	if err != nil {
+		logrus.Errorf("Failed to upload result: %s", err)
+		err2 := CancelTask(result.WebsiteId)
+		if err2 != nil {
+			logrus.Error(err2)
+		}
+		return
+	}
 
 	return
 }
 
-func uploadChunks(websiteId uint64, f *os.File) error {
-	eof := false
-	for iter := 1; !eof; iter++ {
+func uploadChunks(websiteId uint64, f *os.File) (err error) {
+	for iter := 1; iter > 0; iter++ {
 		// TODO Stream with io.Pipe?
 		var b bytes.Buffer
 
 		multi := multipart.NewWriter(&b)
 
 		// Set upload fields
-		var err error
 		err = multi.WriteField("token", config.Token)
-		if err != nil { return err }
+		if err != nil { return }
 		err = multi.WriteField("website_id", fmt.Sprintf("%d", websiteId))
-		if err != nil { return err }
+		if err != nil { return }
 
 		// Copy chunk to file_list
 		formFile, err := multi.CreateFormFile("file_list", "file_list")
-		var n int64
-		n, err = io.CopyN(formFile, f, config.ChunkSize)
-		if err != io.EOF && err != nil {
-			return err
-		}
-		if n == 0 {
-			// Don't upload, no content
-			return nil
-		} else if n < config.ChunkSize {
+		_, err = io.CopyN(formFile, f, fileListChunkSize)
+		if err == io.EOF {
+			break
+		} else if err == io.ErrUnexpectedEOF {
 			err = nil
 			// Break at end of iteration
-			eof = true
+			iter = -420
 		}
 
-		multi.Close()
+		req, err := http.NewRequest(
+			http.MethodPost,
+			config.ServerUrl + "/task/upload",
+			&b)
+		if err != nil { return err }
 
-		for retries := 0; retries < viper.GetInt(ConfUploadRetries); retries++ {
-			if retries > 0 {
-				// Error occurred, retry upload
-				time.Sleep(viper.GetDuration(ConfUploadRetryInterval))
-			}
+		res, err := serverClient.Do(req)
+		if err != nil { return err }
+		res.Body.Close()
 
-			req, err := http.NewRequest(
-				http.MethodPost,
-				config.ServerUrl + "/task/upload",
-				&b)
-			req.Header.Set("content-type", multi.FormDataContentType())
-			if err != nil { continue }
-
-			res, err := serverClient.Do(req)
-			if err != nil { continue }
-			res.Body.Close()
-
-			if res.StatusCode != http.StatusOK {
-				logrus.WithField("status", res.Status).
-					WithField("part", iter).
-					Errorf("Upload failed")
-				continue
-			}
-
-			// Upload successful
-			break
+		if res.StatusCode != http.StatusOK {
+			return fmt.Errorf("failed to upload list part %d: %s",
+				iter, res.Status)
 		}
 
-		logrus.WithField("id", websiteId).
-			WithField("part", iter).
-			Infof("Uploaded files chunk")
+		logrus.Infof("Uploading file list part %d: %s",
+			iter, res.Status)
 	}
-	return nil
+	return
 }
 
 func uploadResult(result *TaskResult) (err error) {
 	resultEnc, err := json.Marshal(result)
 	if err != nil { panic(err) }
 
-	res, err := serverClient.PostForm(
+	payload := url.Values {
+		"token": {config.Token},
+		"result": {string(resultEnc)},
+	}.Encode()
+
+	req, err := http.NewRequest(
+		http.MethodPost,
 		config.ServerUrl + "/task/complete",
-		url.Values {
-			"token": {config.Token},
-			"result": {string(resultEnc)},
-		},
-	)
+		strings.NewReader(payload))
 	if err != nil { return }
-	res.Body.Close()
 
-	if res.StatusCode != http.StatusOK {
-		return HttpError{res.StatusCode}
-	}
-
-	return
-}
-
-func CancelTask(websiteId uint64) (err error) {
-	res, err := serverClient.PostForm(
-		config.ServerUrl + "/task/cancel",
-		url.Values{
-			"token": {config.Token},
-			"website_id": {strconv.FormatUint(websiteId, 10)},
-		},
-	)
+	res, err := serverClient.Do(req)
+	if err != nil { return }
+	res.Body.Close()
+
+	if res.StatusCode != http.StatusOK {
+		return fmt.Errorf("failed to cancel task: %s", res.Status)
+	}
+
+	return
+}
+
+func CancelTask(websiteId uint64) (err error) {
+	form := url.Values{
+		"token": {config.Token},
+		"website_id": {strconv.FormatUint(websiteId, 10)},
+	}
+	encForm := form.Encode()
+
+	req, err := http.NewRequest(
+		http.MethodPost,
+		config.ServerUrl + "/task/cancel",
+		strings.NewReader(encForm))
+	if err != nil { return }
+
+	res, err := serverClient.Do(req)
 	if err != nil { return }
 	res.Body.Close()
 

stats.go

@@ -39,10 +39,6 @@ func Stats(c context.Context) {
 			perSecond = math.Round(perSecond)
 			perSecond /= 2
 
-			if perSecond <= 0 {
-				continue
-			}
-
 			logrus.WithFields(logrus.Fields{
 				"per_second": perSecond,
 				"done":    atomic.LoadUint64(&totalDone),
@@ -57,7 +53,7 @@ func Stats(c context.Context) {
 			runtime.ReadMemStats(&mem)
 
 			logrus.WithFields(logrus.Fields{
-				"queue_count": atomic.LoadInt64(&totalQueued),
+				"queue_count": totalBuffered,
 				"heap": FormatByteCount(mem.Alloc),
 				"objects": mem.HeapObjects,
 				"num_gc": mem.NumGC,

tasks.go

@@ -0,0 +1,16 @@
+package main
+
+import "time"
+
+type Task struct {
+	WebsiteId int `json:"website_id"`
+	Url string `json:"url"`
+}
+
+type TaskResult struct {
+	StatusCode int `json:"status_code"`
+	FileCount uint64 `json:"file_count"`
+	StartTime time.Time `json:"start_time"`
+	EndTime time.Time `json:"end_time"`
+	WebsiteId uint64 `json:"website_id"`
+}

util.go

@@ -1,9 +1,6 @@
 package main
 
-import (
-	"fmt"
-	"sync"
-)
+import "fmt"
 
 // https://programming.guide/go/formatting-byte-size-to-human-readable-format.html
 func FormatByteCount(b uint64) string {
@@ -19,20 +16,3 @@ func FormatByteCount(b uint64) string {
 		return fmt.Sprintf("%.1f %ciB", float64(b)/float64(div), "KMGTPE"[exp])
 	}
 }
-
-type Hooks struct {
-	m sync.Mutex
-	l []func()
-}
-
-func (h *Hooks) Add(hook func()) {
-	h.m.Lock()
-	h.l = append(h.l, hook)
-	h.m.Unlock()
-}
-
-func (h *Hooks) Execute() {
-	for _, hook := range h.l {
-		hook()
-	}
-}

worker.go

@@ -1,12 +1,8 @@
 package main
 
 import (
-	"github.com/beeker1121/goque"
 	"github.com/sirupsen/logrus"
-	"github.com/valyala/fasthttp"
 	"math"
-	"sort"
-	"strings"
 	"sync"
 	"sync/atomic"
 	"time"
@@ -15,38 +11,24 @@ import (
 var globalWait sync.WaitGroup
 
 type WorkerContext struct {
-	OD *OD
-	Queue *BufferedQueue
+	in            chan<- Job
+	out           <-chan Job
 	lastRateLimit time.Time
 	numRateLimits int
 }
 
-func (w *WorkerContext) Worker(results chan<- File) {
-	for {
-		job, err := w.Queue.Dequeue()
-		switch err {
-		case goque.ErrEmpty:
-			time.Sleep(500 * time.Millisecond)
-			continue
-
-		case goque.ErrDBClosed:
-			return
-
-		case nil:
-			w.step(results, job)
-
-		default:
-			panic(err)
-		}
+func (w WorkerContext) Worker() {
+	for job := range w.out {
+		w.step(job)
 	}
 }
 
-func (w *WorkerContext) step(results chan<- File, job Job) {
+func (w WorkerContext) step(job Job) {
 	defer w.finishJob(&job)
 
 	var f File
 
-	newJobs, err := w.DoJob(&job, &f)
+	newJobs, err := DoJob(&job, &f)
 	atomic.AddUint64(&totalStarted, 1)
 	if err == ErrKnown {
 		return
@@ -55,14 +37,9 @@ func (w *WorkerContext) step(results chan<- File, job Job) {
 	if err != nil {
 		job.Fails++
 
-		if httpErr, ok := err.(*HttpError); ok {
-			switch httpErr.code {
-			case fasthttp.StatusTooManyRequests:
-				err = ErrRateLimit
-			default:
-				// Don't retry HTTP error codes
-				return
-			}
+		if err == ErrForbidden {
+			// Don't attempt crawling again
+			return
 		}
 
 		if job.Fails > config.Retries {
@@ -85,22 +62,17 @@ func (w *WorkerContext) step(results chan<- File, job Job) {
 		w.queueJob(job)
 	}
 
-	if !f.IsDir {
-		results <- f
-	}
+	job.OD.Files = append(job.OD.Files, f)
 }
 
-func (w *WorkerContext) DoJob(job *Job, f *File) (newJobs []Job, err error) {
-	if len(job.Uri.Path) == 0 { return }
-	if job.Uri.Path[len(job.Uri.Path)-1] == '/' {
+func DoJob(job *Job, f *File) (newJobs []Job, err error) {
+	if len(job.Uri.Path) != 0 && job.Uri.Path[len(job.Uri.Path)-1] == '/' {
 		// Load directory
 		links, err := GetDir(job, f)
 		if err != nil {
-			if !isErrSilent(err) {
-				logrus.WithError(err).
-					WithField("url", job.UriStr).
-					Error("Failed to crawl dir")
-			}
+			logrus.WithError(err).
+				WithField("url", job.Uri.String()).
+				Error("Failed getting dir")
 			return nil, err
 		}
 
@@ -108,82 +80,58 @@ func (w *WorkerContext) DoJob(job *Job, f *File) (newJobs []Job, err error) {
 		hash := f.HashDir(links)
 
 		// Skip symlinked dirs
-		if w.OD.LoadOrStoreKey(&hash) {
+		if _, old := job.OD.Scanned.LoadOrStore(hash, true); old {
 			return nil, ErrKnown
 		}
 
-		// Sort by path
-		sort.Slice(links, func(i, j int) bool {
-			return strings.Compare(links[i].Path, links[j].Path) < 0
-		})
-
-		var newJobCount int
-		var lastLink string
 		for _, link := range links {
-			uriStr := link.String()
-
-			// Ignore dupes
-			if uriStr == lastLink {
+			// Skip already queued links
+			if _, old := job.OD.Scanned.LoadOrStore(link, true); old {
 				continue
 			}
-			lastLink = uriStr
-
+			job.OD.Wait.Add(1)
 			newJobs = append(newJobs, Job{
+				OD:     job.OD,
 				Uri:    link,
-				UriStr: uriStr,
+				UriStr: link.String(),
 				Fails:  0,
 			})
-
-			newJobCount++
-		}
-		if config.Verbose {
-			logrus.WithFields(logrus.Fields{
-				"url":   job.UriStr,
-				"files": newJobCount,
-			}).Debug("Listed")
 		}
+		logrus.WithFields(logrus.Fields{
+			"url":   job.UriStr,
+			"files": len(links),
+		}).Debug("Listed")
 	} else {
 		// Load file
 		err := GetFile(job.Uri, f)
 		if err != nil {
-			if !isErrSilent(err) {
-				logrus.WithError(err).
-					WithField("url", job.UriStr).
-					Error("Failed to crawl file")
-			}
+			logrus.WithError(err).
+				WithField("url", job.Uri.String()).
+				Error("Failed getting file")
 			return nil, err
 		}
-		atomic.AddUint64(&w.OD.Result.FileCount, 1)
 	}
 	return
 }
 
-func (w *WorkerContext) queueJob(job Job) {
-	w.OD.Wait.Add(1)
+func (w WorkerContext) queueJob(job Job) {
+	job.OD.Wait.Add(1)
+	globalWait.Add(1)
 
 	if w.numRateLimits > 0 {
-		if time.Since(w.lastRateLimit) > 5 * time.Second {
+		if time.Since(w.lastRateLimit) > 5*time.Second {
 			w.numRateLimits = 0
 		} else {
-			time.Sleep(time.Duration(math.Sqrt(float64(50 * w.numRateLimits))) *
+			time.Sleep(time.Duration(math.Sqrt(float64(50*w.numRateLimits))) *
 				100 * time.Millisecond)
+			w.in <- job
 		}
-	}
-
-	if err := w.Queue.Enqueue(&job); err != nil {
-		panic(err)
+	} else {
+		w.in <- job
 	}
 }
 
-func (w *WorkerContext) finishJob(job *Job) {
-	w.OD.Wait.Done()
-}
-
-func isErrSilent(err error) bool {
-	if !config.PrintHTTP {
-		if _, ok := err.(*HttpError); ok {
-			return true
-		}
-	}
-	return false
+func (w WorkerContext) finishJob(job *Job) {
+	job.OD.Wait.Done()
+	globalWait.Done()
 }