From 8f6f8fd17fbea88c0a83aeedf67a636f32c4aeac Mon Sep 17 00:00:00 2001 From: Richard Patel Date: Fri, 16 Nov 2018 04:10:45 +0100 Subject: [PATCH] fasthttp uri --- crawl.go | 32 +- fasturl/url.go | 813 --------------------------------------- fasturl/url_test.go | 897 -------------------------------------------- main.go | 12 +- model.go | 6 +- worker.go | 9 +- 6 files changed, 30 insertions(+), 1739 deletions(-) delete mode 100644 fasturl/url.go delete mode 100644 fasturl/url_test.go diff --git a/crawl.go b/crawl.go index 304beac..5415c1a 100644 --- a/crawl.go +++ b/crawl.go @@ -4,7 +4,6 @@ import ( "bytes" "fmt" "github.com/terorie/oddb-go/ds/redblackhash" - "github.com/terorie/oddb-go/fasturl" "github.com/valyala/fasthttp" "golang.org/x/crypto/blake2b" "golang.org/x/net/html" @@ -16,9 +15,9 @@ import ( var client fasthttp.Client -func GetDir(j *Job, f *File) (links []fasturl.URL, err error) { +func GetDir(j *Job, f *File) (links []fasthttp.URI, err error) { f.IsDir = true - f.Name = path.Base(j.Uri.Path) + f.Name = path.Base(string(j.Uri.Path())) req := fasthttp.AcquireRequest() req.SetRequestURI(j.UriStr) @@ -81,14 +80,15 @@ func GetDir(j *Job, f *File) (links []fasturl.URL, err error) { goto nextToken } - var link fasturl.URL - err = j.Uri.ParseRel(&link, href) + var link fasthttp.URI + j.Uri.CopyTo(&link) + link.Update(href) if err != nil { continue } - if link.Scheme != j.Uri.Scheme || - link.Host != j.Uri.Host || - link.Path == j.Uri.Path || - !strings.HasPrefix(link.Path, j.Uri.Path) { + if !bytes.Equal(link.Scheme(), j.Uri.Scheme()) || + !bytes.Equal(link.Host(), j.Uri.Host()) || + bytes.Equal(link.Path(), j.Uri.Path()) || + !bytes.HasPrefix(link.Path(), j.Uri.Path()) { continue } @@ -102,11 +102,12 @@ func GetDir(j *Job, f *File) (links []fasturl.URL, err error) { return } -func GetFile(u fasturl.URL, f *File) (err error) { +func GetFile(u fasthttp.URI, f *File) (err error) { f.IsDir = false - u.Path = path.Clean(u.Path) - f.Name = path.Base(u.Path) - f.Path = strings.Trim(u.Path, "/") + cleanPath := path.Clean(string(u.Path())) + u.SetPath(cleanPath) + f.Name = path.Base(cleanPath) + f.Path = strings.Trim(cleanPath, "/") req := fasthttp.AcquireRequest() req.Header.SetMethod("HEAD") @@ -130,12 +131,11 @@ func GetFile(u fasturl.URL, f *File) (err error) { return nil } -func (f *File) HashDir(links []fasturl.URL) (o redblackhash.Key) { +func (f *File) HashDir(links []fasthttp.URI) (o redblackhash.Key) { h, _ := blake2b.New256(nil) h.Write([]byte(f.Name)) for _, link := range links { - fileName := path.Base(link.Path) - h.Write([]byte(fileName)) + h.Write(link.Path()) } sum := h.Sum(nil) copy(o[:redblackhash.KeySize], sum) diff --git a/fasturl/url.go b/fasturl/url.go deleted file mode 100644 index f03612b..0000000 --- a/fasturl/url.go +++ /dev/null @@ -1,813 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Package fasturl parses URLs and implements query escaping. -package fasturl - -// Modifications by terorie - -// See RFC 3986. This package generally follows RFC 3986, except where -// it deviates for compatibility reasons. When sending changes, first -// search old issues for history on decisions. Unit tests should also -// contain references to issue numbers with details. - -import ( - "errors" - "fmt" - "strconv" - "strings" -) - -type Scheme int -const ( - SchemeInvalid = iota - SchemeHTTP - SchemeHTTPS - SchemeCount -) - -var Schemes = [SchemeCount]string { - "", - "http", - "https", -} - -// Error reports an error and the operation and URL that caused it. -type Error struct { - Op string - URL string - Err error -} - -func (e *Error) Error() string { return e.Op + " " + e.URL + ": " + e.Err.Error() } - -type timeout interface { - Timeout() bool -} - -func (e *Error) Timeout() bool { - t, ok := e.Err.(timeout) - return ok && t.Timeout() -} - -type temporary interface { - Temporary() bool -} - -func (e *Error) Temporary() bool { - t, ok := e.Err.(temporary) - return ok && t.Temporary() -} - -func ishex(c byte) bool { - switch { - case '0' <= c && c <= '9': - return true - case 'a' <= c && c <= 'f': - return true - case 'A' <= c && c <= 'F': - return true - } - return false -} - -func unhex(c byte) byte { - switch { - case '0' <= c && c <= '9': - return c - '0' - case 'a' <= c && c <= 'f': - return c - 'a' + 10 - case 'A' <= c && c <= 'F': - return c - 'A' + 10 - } - return 0 -} - -type encoding int - -const ( - encodePath encoding = 1 + iota - encodePathSegment - encodeHost - encodeZone - encodeUserPassword - encodeQueryComponent - encodeFragment -) - -type EscapeError string - -func (e EscapeError) Error() string { - return "invalid URL escape " + strconv.Quote(string(e)) -} - -type InvalidHostError string - -func (e InvalidHostError) Error() string { - return "invalid character " + strconv.Quote(string(e)) + " in host name" -} - -// Return true if the specified character should be escaped when -// appearing in a URL string, according to RFC 3986. -// -// Please be informed that for now shouldEscape does not check all -// reserved characters correctly. See golang.org/issue/5684. -func shouldEscape(c byte, mode encoding) bool { - // §2.3 Unreserved characters (alphanum) - if 'A' <= c && c <= 'Z' || 'a' <= c && c <= 'z' || '0' <= c && c <= '9' { - return false - } - - if mode == encodeHost || mode == encodeZone { - // §3.2.2 Host allows - // sub-delims = "!" / "$" / "&" / "'" / "(" / ")" / "*" / "+" / "," / ";" / "=" - // as part of reg-name. - // We add : because we include :port as part of host. - // We add [ ] because we include [ipv6]:port as part of host. - // We add < > because they're the only characters left that - // we could possibly allow, and Parse will reject them if we - // escape them (because hosts can't use %-encoding for - // ASCII bytes). - switch c { - case '!', '$', '&', '\'', '(', ')', '*', '+', ',', ';', '=', ':', '[', ']', '<', '>', '"': - return false - } - } - - switch c { - case '-', '_', '.', '~': // §2.3 Unreserved characters (mark) - return false - - case '$', '&', '+', ',', '/', ':', ';', '=', '?', '@': // §2.2 Reserved characters (reserved) - // Different sections of the URL allow a few of - // the reserved characters to appear unescaped. - switch mode { - case encodePath: // §3.3 - // The RFC allows : @ & = + $ but saves / ; , for assigning - // meaning to individual path segments. This package - // only manipulates the path as a whole, so we allow those - // last three as well. That leaves only ? to escape. - return c == '?' - - case encodePathSegment: // §3.3 - // The RFC allows : @ & = + $ but saves / ; , for assigning - // meaning to individual path segments. - return c == '/' || c == ';' || c == ',' || c == '?' - - case encodeUserPassword: // §3.2.1 - // The RFC allows ';', ':', '&', '=', '+', '$', and ',' in - // userinfo, so we must escape only '@', '/', and '?'. - // The parsing of userinfo treats ':' as special so we must escape - // that too. - return c == '@' || c == '/' || c == '?' || c == ':' - - case encodeQueryComponent: // §3.4 - // The RFC reserves (so we must escape) everything. - return true - - case encodeFragment: // §4.1 - // The RFC text is silent but the grammar allows - // everything, so escape nothing. - return false - } - } - - if mode == encodeFragment { - // RFC 3986 §2.2 allows not escaping sub-delims. A subset of sub-delims are - // included in reserved from RFC 2396 §2.2. The remaining sub-delims do not - // need to be escaped. To minimize potential breakage, we apply two restrictions: - // (1) we always escape sub-delims outside of the fragment, and (2) we always - // escape single quote to avoid breaking callers that had previously assumed that - // single quotes would be escaped. See issue #19917. - switch c { - case '!', '(', ')', '*': - return false - } - } - - // Everything else must be escaped. - return true -} - -// unescape unescapes a string; the mode specifies -// which section of the URL string is being unescaped. -func unescape(s string, mode encoding) (string, error) { - // Count %, check that they're well-formed. - n := 0 - hasPlus := false - for i := 0; i < len(s); { - switch s[i] { - case '%': - n++ - if i+2 >= len(s) || !ishex(s[i+1]) || !ishex(s[i+2]) { - s = s[i:] - if len(s) > 3 { - s = s[:3] - } - return "", EscapeError(s) - } - // Per https://tools.ietf.org/html/rfc3986#page-21 - // in the host component %-encoding can only be used - // for non-ASCII bytes. - // But https://tools.ietf.org/html/rfc6874#section-2 - // introduces %25 being allowed to escape a percent sign - // in IPv6 scoped-address literals. Yay. - if mode == encodeHost && unhex(s[i+1]) < 8 && s[i:i+3] != "%25" { - return "", EscapeError(s[i : i+3]) - } - if mode == encodeZone { - // RFC 6874 says basically "anything goes" for zone identifiers - // and that even non-ASCII can be redundantly escaped, - // but it seems prudent to restrict %-escaped bytes here to those - // that are valid host name bytes in their unescaped form. - // That is, you can use escaping in the zone identifier but not - // to introduce bytes you couldn't just write directly. - // But Windows puts spaces here! Yay. - v := unhex(s[i+1])<<4 | unhex(s[i+2]) - if s[i:i+3] != "%25" && v != ' ' && shouldEscape(v, encodeHost) { - return "", EscapeError(s[i : i+3]) - } - } - i += 3 - case '+': - hasPlus = mode == encodeQueryComponent - i++ - default: - if (mode == encodeHost || mode == encodeZone) && s[i] < 0x80 && shouldEscape(s[i], mode) { - return "", InvalidHostError(s[i : i+1]) - } - i++ - } - } - - if n == 0 && !hasPlus { - return s, nil - } - - t := make([]byte, len(s)-2*n) - j := 0 - for i := 0; i < len(s); { - switch s[i] { - case '%': - t[j] = unhex(s[i+1])<<4 | unhex(s[i+2]) - j++ - i += 3 - case '+': - if mode == encodeQueryComponent { - t[j] = ' ' - } else { - t[j] = '+' - } - j++ - i++ - default: - t[j] = s[i] - j++ - i++ - } - } - return string(t), nil -} - -func escape(s string, mode encoding) string { - spaceCount, hexCount := 0, 0 - for i := 0; i < len(s); i++ { - c := s[i] - if shouldEscape(c, mode) { - if c == ' ' && mode == encodeQueryComponent { - spaceCount++ - } else { - hexCount++ - } - } - } - - if spaceCount == 0 && hexCount == 0 { - return s - } - - t := make([]byte, len(s)+2*hexCount) - j := 0 - for i := 0; i < len(s); i++ { - switch c := s[i]; { - case c == ' ' && mode == encodeQueryComponent: - t[j] = '+' - j++ - case shouldEscape(c, mode): - t[j] = '%' - t[j+1] = "0123456789ABCDEF"[c>>4] - t[j+2] = "0123456789ABCDEF"[c&15] - j += 3 - default: - t[j] = s[i] - j++ - } - } - return string(t) -} - -// A URL represents a parsed URL (technically, a URI reference). -// -// The general form represented is: -// -// [scheme:][//[userinfo@]host][/]path[?query][#fragment] -// -// URLs that do not start with a slash after the scheme are interpreted as: -// -// scheme:opaque[?query][#fragment] -// -// Note that the Path field is stored in decoded form: /%47%6f%2f becomes /Go/. -// A consequence is that it is impossible to tell which slashes in the Path were -// slashes in the raw URL and which were %2f. This distinction is rarely important, -// but when it is, code must not use Path directly. -// The Parse function sets both Path and RawPath in the URL it returns, -// and URL's String method uses RawPath if it is a valid encoding of Path, -// by calling the EscapedPath method. -type URL struct { - Scheme Scheme - Host string // host or host:port - Path string // path (relative paths may omit leading slash) -} - -// Maybe rawurl is of the form scheme:path. -// (Scheme must be [a-zA-Z][a-zA-Z0-9+-.]*) -// If so, return scheme, path; else return "", rawurl. -func getscheme(rawurl string) (scheme Scheme, path string, err error) { - for i := 0; i < len(rawurl); i++ { - c := rawurl[i] - switch { - case 'a' <= c && c <= 'z' || 'A' <= c && c <= 'Z': - // do nothing - case '0' <= c && c <= '9' || c == '+' || c == '-' || c == '.': - if i == 0 { - return SchemeInvalid, rawurl, nil - } - case c == ':': - if i == 0 { - return SchemeInvalid, "", errors.New("missing protocol scheme") - } - switch rawurl[:i] { - case "http": - scheme = SchemeHTTP - case "https": - scheme = SchemeHTTPS - default: - return SchemeInvalid, "", errors.New("unknown protocol scheme") - } - - path = rawurl[i+1:] - return - default: - // we have encountered an invalid character, - // so there is no valid scheme - return SchemeInvalid, rawurl, nil - } - } - return SchemeInvalid, rawurl, nil -} - -// Maybe s is of the form t c u. -// If so, return t, c u (or t, u if cutc == true). -// If not, return s, "". -func split(s string, c string, cutc bool) (string, string) { - i := strings.Index(s, c) - if i < 0 { - return s, "" - } - if cutc { - return s[:i], s[i+len(c):] - } - return s[:i], s[i:] -} - -// Parse parses rawurl into a URL structure. -// -// The rawurl may be relative (a path, without a host) or absolute -// (starting with a scheme). Trying to parse a hostname and path -// without a scheme is invalid but may not necessarily return an -// error, due to parsing ambiguities. -func (u *URL) Parse(rawurl string) error { - // Cut off #frag - s, frag := split(rawurl, "#", true) - err := u.parse(s, false) - if err != nil { - return &Error{"parse", s, err} - } - if frag == "" { - return nil - } - return nil -} - -// ParseRequestURI parses rawurl into a URL structure. It assumes that -// rawurl was received in an HTTP request, so the rawurl is interpreted -// only as an absolute URI or an absolute path. -// The string rawurl is assumed not to have a #fragment suffix. -// (Web browsers strip #fragment before sending the URL to a web server.) -func (u *URL) ParseRequestURI(rawurl string) error { - err := u.parse(rawurl, true) - if err != nil { - return &Error{"parse", rawurl, err} - } - return nil -} - -// parse parses a URL from a string in one of two contexts. If -// viaRequest is true, the URL is assumed to have arrived via an HTTP request, -// in which case only absolute URLs or path-absolute relative URLs are allowed. -// If viaRequest is false, all forms of relative URLs are allowed. -func (u *URL) parse(rawurl string, viaRequest bool) error { - var rest string - var err error - - if rawurl == "" && viaRequest { - return errors.New("empty url") - } - - if rawurl == "*" { - u.Path = "*" - return nil - } - - // Split off possible leading "http:", "mailto:", etc. - // Cannot contain escaped characters. - if u.Scheme, rest, err = getscheme(rawurl); err != nil { - return err - } - - if strings.HasSuffix(rest, "?") && strings.Count(rest, "?") == 1 { - rest = rest[:len(rest)-1] - } else { - rest, _ = split(rest, "?", true) - } - - if !strings.HasPrefix(rest, "/") { - if u.Scheme != SchemeInvalid { - // We consider rootless paths per RFC 3986 as opaque. - return nil - } - if viaRequest { - return errors.New("invalid URI for request") - } - - // Avoid confusion with malformed schemes, like cache_object:foo/bar. - // See golang.org/issue/16822. - // - // RFC 3986, §3.3: - // In addition, a URI reference (Section 4.1) may be a relative-path reference, - // in which case the first path segment cannot contain a colon (":") character. - colon := strings.Index(rest, ":") - slash := strings.Index(rest, "/") - if colon >= 0 && (slash < 0 || colon < slash) { - // First path segment has colon. Not allowed in relative URL. - return errors.New("first path segment in URL cannot contain colon") - } - } - - if (u.Scheme != SchemeInvalid || !viaRequest && !strings.HasPrefix(rest, "///")) && strings.HasPrefix(rest, "//") { - var authority string - authority, rest = split(rest[2:], "/", false) - u.Host, err = parseAuthority(authority) - if err != nil { - return err - } - } - u.Path = rest - return nil -} - -func parseAuthority(authority string) (host string, err error) { - i := strings.LastIndex(authority, "@") - if i < 0 { - host, err = parseHost(authority) - } else { - host, err = parseHost(authority[i+1:]) - } - if err != nil { - return "", err - } - if i < 0 { - return host, nil - } - userinfo := authority[:i] - if !validUserinfo(userinfo) { - return "", errors.New("fasturl: invalid userinfo") - } - return host, nil -} - -// parseHost parses host as an authority without user -// information. That is, as host[:port]. -func parseHost(host string) (string, error) { - if strings.HasPrefix(host, "[") { - // Parse an IP-Literal in RFC 3986 and RFC 6874. - // E.g., "[fe80::1]", "[fe80::1%25en0]", "[fe80::1]:80". - i := strings.LastIndex(host, "]") - if i < 0 { - return "", errors.New("missing ']' in host") - } - colonPort := host[i+1:] - if !validOptionalPort(colonPort) { - return "", fmt.Errorf("invalid port %q after host", colonPort) - } - - // RFC 6874 defines that %25 (%-encoded percent) introduces - // the zone identifier, and the zone identifier can use basically - // any %-encoding it likes. That's different from the host, which - // can only %-encode non-ASCII bytes. - // We do impose some restrictions on the zone, to avoid stupidity - // like newlines. - zone := strings.Index(host[:i], "%25") - if zone >= 0 { - host1, err := unescape(host[:zone], encodeHost) - if err != nil { - return "", err - } - host2, err := unescape(host[zone:i], encodeZone) - if err != nil { - return "", err - } - host3, err := unescape(host[i:], encodeHost) - if err != nil { - return "", err - } - return host1 + host2 + host3, nil - } - } - - var err error - if host, err = unescape(host, encodeHost); err != nil { - return "", err - } - return host, nil -} - -// validOptionalPort reports whether port is either an empty string -// or matches /^:\d*$/ -func validOptionalPort(port string) bool { - if port == "" { - return true - } - if port[0] != ':' { - return false - } - for _, b := range port[1:] { - if b < '0' || b > '9' { - return false - } - } - return true -} - -// String reassembles the URL into a valid URL string. -// The general form of the result is one of: -// -// scheme:opaque?query#fragment -// scheme://userinfo@host/path?query#fragment -// -// If u.Opaque is non-empty, String uses the first form; -// otherwise it uses the second form. -// To obtain the path, String uses u.EscapedPath(). -// -// In the second form, the following rules apply: -// - if u.Scheme is empty, scheme: is omitted. -// - if u.User is nil, userinfo@ is omitted. -// - if u.Host is empty, host/ is omitted. -// - if u.Scheme and u.Host are empty and u.User is nil, -// the entire scheme://userinfo@host/ is omitted. -// - if u.Host is non-empty and u.Path begins with a /, -// the form host/path does not add its own /. -// - if u.RawQuery is empty, ?query is omitted. -// - if u.Fragment is empty, #fragment is omitted. -func (u *URL) String() string { - var buf strings.Builder - if u.Scheme != SchemeInvalid { - buf.WriteString(Schemes[u.Scheme]) - buf.WriteByte(':') - } - if u.Scheme != SchemeInvalid || u.Host != "" { - if u.Host != "" || u.Path != "" { - buf.WriteString("//") - } - if h := u.Host; h != "" { - buf.WriteString(escape(h, encodeHost)) - } - } - path := u.Path - if path != "" && path[0] != '/' && u.Host != "" { - buf.WriteByte('/') - } - if buf.Len() == 0 { - // RFC 3986 §4.2 - // A path segment that contains a colon character (e.g., "this:that") - // cannot be used as the first segment of a relative-path reference, as - // it would be mistaken for a scheme name. Such a segment must be - // preceded by a dot-segment (e.g., "./this:that") to make a relative- - // path reference. - if i := strings.IndexByte(path, ':'); i > -1 && strings.IndexByte(path[:i], '/') == -1 { - buf.WriteString("./") - } - } - buf.WriteString(path) - return buf.String() -} - -func isRunesDot(r []rune) bool { - return len(r) == 1 && r[0] == '.' -} - -func isRunesDoubleDot(r []rune) bool { - return len(r) == 2 && r[0] == '.' && r[1] == '.' -} - -// resolvePath applies special path segments from refs and applies -// them to base, per RFC 3986. -func resolvePath(base, ref string) string { - var full string - if ref == "" { - full = base - } else if ref[0] != '/' { - i := strings.LastIndex(base, "/") - full = base[:i+1] + ref - } else { - full = ref - } - if full == "" { - return "" - } else if full == "/" { - return "/" - } - - dst := make([]rune, len(full)) - dst = dst[0:0] - - start := 0 - rs := []rune(full) - if len(rs) != 0 && rs[0] == '/' { - rs = rs[1:] - } - var stack []int - stack = append(stack, 0) - for i, c := range rs { - if i == len(rs) - 1 { - closingSlash := false - part := rs[start:] - if len(part) == 0 { - dst = append(dst, '/') - } else if part[len(part)-1] == '/' { - part = part[:len(part)-1] - closingSlash = true - } - switch { - case isRunesDot(part): - dst = append(dst, '/') - case isRunesDoubleDot(part): - // Cut to the last slash - start = i+1 - dst = dst[:stack[len(stack)-1]] - if len(stack) != 1 { - stack = stack[:len(stack)-1] - } - dst = append(dst, '/') - default: - dst = append(dst, '/') - dst = append(dst, part...) - } - if closingSlash && len(dst) != 0 && dst[len(dst)-1] != '/' { - dst = append(dst, '/') - } - } else if c == '/' { - part := rs[start:i] - switch { - case isRunesDot(part): - start = i+1 - case isRunesDoubleDot(part): - // Cut to the last slash - start = i+1 - dst = dst[:stack[len(stack)-1]] - if len(stack) != 1 { - stack = stack[:len(stack)-1] - } - default: - start = i+1 - stack = append(stack, len(dst)) - dst = append(dst, '/') - dst = append(dst, part...) - } - } - } - return string(dst) - - /*var dst []string - src := strings.Split(full, "/") - for _, elem := range src { - switch elem { - case ".": - // drop - case "..": - if len(dst) > 0 { - dst = dst[:len(dst)-1] - } - default: - dst = append(dst, elem) - } - } - if last := src[len(src)-1]; last == "." || last == ".." { - // Add final slash to the joined path. - dst = append(dst, "") - } - return "/" + strings.TrimPrefix(strings.Join(dst, "/"), "/")*/ -} - -// IsAbs reports whether the URL is absolute. -// Absolute means that it has a non-empty scheme. -func (u *URL) IsAbs() bool { - return u.Scheme != SchemeInvalid -} - -// ParseRel parses a URL in the context of the receiver. The provided URL -// may be relative or absolute. Parse returns nil, err on parse -// failure, otherwise its return value is the same as ResolveReference. -func (u *URL) ParseRel(out *URL, ref string) error { - var refurl URL - - err := refurl.Parse(ref) - if err != nil { - return err - } - - u.ResolveReference(out, &refurl) - return nil -} - -// ResolveReference resolves a URI reference to an absolute URI from -// an absolute base URI u, per RFC 3986 Section 5.2. The URI reference -// may be relative or absolute. ResolveReference always returns a new -// URL instance, even if the returned URL is identical to either the -// base or reference. If ref is an absolute URL, then ResolveReference -// ignores base and returns a copy of ref. -func (u *URL) ResolveReference(url *URL, ref *URL) { - *url = *ref - if ref.Scheme == SchemeInvalid { - url.Scheme = u.Scheme - } - if ref.Scheme != SchemeInvalid || ref.Host != "" { - // The "absoluteURI" or "net_path" cases. - // We can ignore the error from setPath since we know we provided a - // validly-escaped path. - url.Path = resolvePath(ref.Path, "") - return - } - // The "abs_path" or "rel_path" cases. - url.Host = u.Host - url.Path = resolvePath(u.Path, ref.Path) - return -} - -// Marshaling interface implementations. -// Would like to implement MarshalText/UnmarshalText but that will change the JSON representation of URLs. - -func (u *URL) MarshalBinary() (text []byte, err error) { - return []byte(u.String()), nil -} - -func (u *URL) UnmarshalBinary(text []byte) error { - var u1 URL - err := u1.Parse(string(text)) - if err != nil { - return err - } - *u = u1 - return nil -} - -// validUserinfo reports whether s is a valid userinfo string per RFC 3986 -// Section 3.2.1: -// userinfo = *( unreserved / pct-encoded / sub-delims / ":" ) -// unreserved = ALPHA / DIGIT / "-" / "." / "_" / "~" -// sub-delims = "!" / "$" / "&" / "'" / "(" / ")" -// / "*" / "+" / "," / ";" / "=" -// -// It doesn't validate pct-encoded. The caller does that via func unescape. -func validUserinfo(s string) bool { - for _, r := range s { - if 'A' <= r && r <= 'Z' { - continue - } - if 'a' <= r && r <= 'z' { - continue - } - if '0' <= r && r <= '9' { - continue - } - switch r { - case '-', '.', '_', ':', '~', '!', '$', '&', '\'', - '(', ')', '*', '+', ',', ';', '=', '%', '@': - continue - default: - return false - } - } - return true -} diff --git a/fasturl/url_test.go b/fasturl/url_test.go deleted file mode 100644 index b89e9aa..0000000 --- a/fasturl/url_test.go +++ /dev/null @@ -1,897 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package fasturl - -import ( - "bytes" - encodingPkg "encoding" - "encoding/gob" - "encoding/json" - "fmt" - "io" - "net" - "reflect" - "testing" -) - -type URLTest struct { - in string - out *URL // expected parse; RawPath="" means same as Path - roundtrip string // expected result of reserializing the URL; empty means same as "in". -} - -var urltests = []URLTest{ - // no path - { - "http://www.google.com", - &URL{ - Scheme: SchemeHTTP, - Host: "www.google.com", - }, - "", - }, - // path - { - "http://www.google.com/", - &URL{ - Scheme: SchemeHTTP, - Host: "www.google.com", - Path: "/", - }, - "", - }, - // %20 outside query - { - "http://www.google.com/a%20b", - &URL{ - Scheme: SchemeHTTP, - Host: "www.google.com", - Path: "/a%20b", - }, - "", - }, - // leading // without scheme should create an authority - { - "//foo", - &URL{ - Host: "foo", - }, - "", - }, - // Three leading slashes isn't an authority, but doesn't return an error. - // (We can't return an error, as this code is also used via - // ServeHTTP -> ReadRequest -> Parse, which is arguably a - // different URL parsing context, but currently shares the - // same codepath) - { - "///threeslashes", - &URL{ - Path: "///threeslashes", - }, - "", - }, - // unescaped @ in username should not confuse host - { - "http://j@ne:password@google.com", - &URL{ - Scheme: SchemeHTTP, - Host: "google.com", - }, - "http://google.com", - }, - // unescaped @ in password should not confuse host - { - "http://jane:p@ssword@google.com", - &URL{ - Scheme: SchemeHTTP, - Host: "google.com", - }, - "http://google.com", - }, - // Relative path - { - "a/b/c", - &URL{ - Path: "a/b/c", - }, - "a/b/c", - }, - // host subcomponent; IPv4 address in RFC 3986 - { - "http://192.168.0.1/", - &URL{ - Scheme: SchemeHTTP, - Host: "192.168.0.1", - Path: "/", - }, - "", - }, - // host and port subcomponents; IPv4 address in RFC 3986 - { - "http://192.168.0.1:8080/", - &URL{ - Scheme: SchemeHTTP, - Host: "192.168.0.1:8080", - Path: "/", - }, - "", - }, - // host subcomponent; IPv6 address in RFC 3986 - { - "http://[fe80::1]/", - &URL{ - Scheme: SchemeHTTP, - Host: "[fe80::1]", - Path: "/", - }, - "", - }, - // host and port subcomponents; IPv6 address in RFC 3986 - { - "http://[fe80::1]:8080/", - &URL{ - Scheme: SchemeHTTP, - Host: "[fe80::1]:8080", - Path: "/", - }, - "", - }, - // host subcomponent; IPv6 address with zone identifier in RFC 6874 - { - "http://[fe80::1%25en0]/", // alphanum zone identifier - &URL{ - Scheme: SchemeHTTP, - Host: "[fe80::1%en0]", - Path: "/", - }, - "", - }, - // host and port subcomponents; IPv6 address with zone identifier in RFC 6874 - { - "http://[fe80::1%25en0]:8080/", // alphanum zone identifier - &URL{ - Scheme: SchemeHTTP, - Host: "[fe80::1%en0]:8080", - Path: "/", - }, - "", - }, - // host subcomponent; IPv6 address with zone identifier in RFC 6874 - { - "http://[fe80::1%25%65%6e%301-._~]/", // percent-encoded+unreserved zone identifier - &URL{ - Scheme: SchemeHTTP, - Host: "[fe80::1%en01-._~]", - Path: "/", - }, - "http://[fe80::1%25en01-._~]/", - }, - // host and port subcomponents; IPv6 address with zone identifier in RFC 6874 - { - "http://[fe80::1%25%65%6e%301-._~]:8080/", // percent-encoded+unreserved zone identifier - &URL{ - Scheme: SchemeHTTP, - Host: "[fe80::1%en01-._~]:8080", - Path: "/", - }, - "http://[fe80::1%25en01-._~]:8080/", - }, - // golang.org/issue/12200 (colon with empty port) - { - "http://192.168.0.2:8080/foo", - &URL{ - Scheme: SchemeHTTP, - Host: "192.168.0.2:8080", - Path: "/foo", - }, - "", - }, - { - "http://192.168.0.2:/foo", - &URL{ - Scheme: SchemeHTTP, - Host: "192.168.0.2:", - Path: "/foo", - }, - "", - }, - { - // Malformed IPv6 but still accepted. - "http://2b01:e34:ef40:7730:8e70:5aff:fefe:edac:8080/foo", - &URL{ - Scheme: SchemeHTTP, - Host: "2b01:e34:ef40:7730:8e70:5aff:fefe:edac:8080", - Path: "/foo", - }, - "", - }, - { - // Malformed IPv6 but still accepted. - "http://2b01:e34:ef40:7730:8e70:5aff:fefe:edac:/foo", - &URL{ - Scheme: SchemeHTTP, - Host: "2b01:e34:ef40:7730:8e70:5aff:fefe:edac:", - Path: "/foo", - }, - "", - }, - { - "http://[2b01:e34:ef40:7730:8e70:5aff:fefe:edac]:8080/foo", - &URL{ - Scheme: SchemeHTTP, - Host: "[2b01:e34:ef40:7730:8e70:5aff:fefe:edac]:8080", - Path: "/foo", - }, - "", - }, - { - "http://[2b01:e34:ef40:7730:8e70:5aff:fefe:edac]:/foo", - &URL{ - Scheme: SchemeHTTP, - Host: "[2b01:e34:ef40:7730:8e70:5aff:fefe:edac]:", - Path: "/foo", - }, - "", - }, - // golang.org/issue/7991 and golang.org/issue/12719 (non-ascii %-encoded in host) - { - "http://hello.世界.com/foo", - &URL{ - Scheme: SchemeHTTP, - Host: "hello.世界.com", - Path: "/foo", - }, - "http://hello.%E4%B8%96%E7%95%8C.com/foo", - }, - { - "http://hello.%e4%b8%96%e7%95%8c.com/foo", - &URL{ - Scheme: SchemeHTTP, - Host: "hello.世界.com", - Path: "/foo", - }, - "http://hello.%E4%B8%96%E7%95%8C.com/foo", - }, - { - "http://hello.%E4%B8%96%E7%95%8C.com/foo", - &URL{ - Scheme: SchemeHTTP, - Host: "hello.世界.com", - Path: "/foo", - }, - "", - }, - // golang.org/issue/10433 (path beginning with //) - { - "http://example.com//foo", - &URL{ - Scheme: SchemeHTTP, - Host: "example.com", - Path: "//foo", - }, - "", - }, - // test that we can reparse the host names we accept. - { - "http://authority<\"hi\">/foo", - &URL{ - Scheme: SchemeHTTP, - Host: "authority<\"hi\">", - Path: "/foo", - }, - "", - }, -} - -// more useful string for debugging than fmt's struct printer -func ufmt(u *URL) string { - return fmt.Sprintf("scheme=%q, host=%q, path=%q", - Schemes[u.Scheme], u.Host, u.Path) -} - -func BenchmarkString(b *testing.B) { - b.StopTimer() - b.ReportAllocs() - for _, tt := range urltests { - var u URL - err := u.Parse(tt.in) - if err != nil { - b.Errorf("Parse(%q) returned error %s", tt.in, err) - continue - } - if tt.roundtrip == "" { - continue - } - b.StartTimer() - var g string - for i := 0; i < b.N; i++ { - g = u.String() - } - b.StopTimer() - if w := tt.roundtrip; b.N > 0 && g != w { - b.Errorf("Parse(%q).String() == %q, want %q", tt.in, g, w) - } - } -} - -func TestParse(t *testing.T) { - for _, tt := range urltests { - var u URL - err := u.Parse(tt.in) - if err != nil { - t.Errorf("Parse(%q) returned error %v", tt.in, err) - continue - } - if !reflect.DeepEqual(&u, tt.out) { - t.Errorf("Parse(%q):\n\tgot %v\n\twant %v\n", tt.in, ufmt(&u), ufmt(tt.out)) - } - } -} - -const pathThatLooksSchemeRelative = "//not.a.user@not.a.host/just/a/path" - -var parseRequestURLTests = []struct { - url string - expectedValid bool -}{ - {"http://foo.com", true}, - {"http://foo.com/", true}, - {"http://foo.com/path", true}, - {"/", true}, - {pathThatLooksSchemeRelative, true}, - {"//not.a.user@%66%6f%6f.com/just/a/path/also", true}, - {"*", true}, - {"http://192.168.0.1/", true}, - {"http://192.168.0.1:8080/", true}, - {"http://[fe80::1]/", true}, - {"http://[fe80::1]:8080/", true}, - - // Tests exercising RFC 6874 compliance: - {"http://[fe80::1%25en0]/", true}, // with alphanum zone identifier - {"http://[fe80::1%25en0]:8080/", true}, // with alphanum zone identifier - {"http://[fe80::1%25%65%6e%301-._~]/", true}, // with percent-encoded+unreserved zone identifier - {"http://[fe80::1%25%65%6e%301-._~]:8080/", true}, // with percent-encoded+unreserved zone identifier - - {"foo.html", false}, - {"../dir/", false}, - {"http://192.168.0.%31/", false}, - {"http://192.168.0.%31:8080/", false}, - {"http://[fe80::%31]/", false}, - {"http://[fe80::%31]:8080/", false}, - {"http://[fe80::%31%25en0]/", false}, - {"http://[fe80::%31%25en0]:8080/", false}, - - // These two cases are valid as textual representations as - // described in RFC 4007, but are not valid as address - // literals with IPv6 zone identifiers in URIs as described in - // RFC 6874. - {"http://[fe80::1%en0]/", false}, - {"http://[fe80::1%en0]:8080/", false}, -} - -func TestParseRequestURI(t *testing.T) { - for _, test := range parseRequestURLTests { - var u URL - err := u.ParseRequestURI(test.url) - if test.expectedValid && err != nil { - t.Errorf("ParseRequestURI(%q) gave err %v; want no error", test.url, err) - } else if !test.expectedValid && err == nil { - t.Errorf("ParseRequestURI(%q) gave nil error; want some error", test.url) - } - } - - var url URL - err := url.ParseRequestURI(pathThatLooksSchemeRelative) - if err != nil { - t.Fatalf("Unexpected error %v", err) - } - if url.Path != pathThatLooksSchemeRelative { - t.Errorf("ParseRequestURI path:\ngot %q\nwant %q", url.Path, pathThatLooksSchemeRelative) - } -} - -var stringURLTests = []struct { - url URL - want string -}{ - // No leading slash on path should prepend slash on String() call - { - url: URL{ - Scheme: SchemeHTTP, - Host: "www.google.com", - Path: "search", - }, - want: "http://www.google.com/search", - }, - // Relative path with first element containing ":" should be prepended with "./", golang.org/issue/17184 - { - url: URL{ - Path: "this:that", - }, - want: "./this:that", - }, - // Relative path with second element containing ":" should not be prepended with "./" - { - url: URL{ - Path: "here/this:that", - }, - want: "here/this:that", - }, - // Non-relative path with first element containing ":" should not be prepended with "./" - { - url: URL{ - Scheme: SchemeHTTP, - Host: "www.google.com", - Path: "this:that", - }, - want: "http://www.google.com/this:that", - }, -} - -func TestURLString(t *testing.T) { - for _, tt := range urltests { - var u URL - err := u.Parse(tt.in) - if err != nil { - t.Errorf("Parse(%q) returned error %s", tt.in, err) - continue - } - expected := tt.in - if tt.roundtrip != "" { - expected = tt.roundtrip - } - s := u.String() - if s != expected { - t.Errorf("Parse(%q).String() == %q (expected %q)", tt.in, s, expected) - } - } - - for _, tt := range stringURLTests { - if got := tt.url.String(); got != tt.want { - t.Errorf("%+v.String() = %q; want %q", tt.url, got, tt.want) - } - } -} - -var resolvePathTests = []struct { - base, ref, expected string -}{ - {"a/b", ".", "/a/"}, - {"a/b", "c", "/a/c"}, - {"a/b", "..", "/"}, - {"a/", "..", "/"}, - {"a/", "../..", "/"}, - {"a/b/c", "..", "/a/"}, - {"a/b/c", "../d", "/a/d"}, - {"a/b/c", ".././d", "/a/d"}, - {"a/b", "./..", "/"}, - {"a/./b", ".", "/a/"}, - {"a/../", ".", "/"}, - {"a/.././b", "c", "/c"}, -} - -func TestResolvePath(t *testing.T) { - for _, test := range resolvePathTests { - got := resolvePath(test.base, test.ref) - if got != test.expected { - t.Errorf("For %q + %q got %q; expected %q", test.base, test.ref, got, test.expected) - } - } -} - -var resolveReferenceTests = []struct { - base, rel, expected string -}{ - // Absolute URL references - {"http://foo.com?a=b", "https://bar.com/", "https://bar.com/"}, - {"http://foo.com/", "https://bar.com/?a=b", "https://bar.com/"}, - {"http://foo.com/", "https://bar.com/?", "https://bar.com/"}, - - // Path-absolute references - {"http://foo.com/bar", "/baz", "http://foo.com/baz"}, - {"http://foo.com/bar?a=b#f", "/baz", "http://foo.com/baz"}, - {"http://foo.com/bar?a=b", "/baz?", "http://foo.com/baz"}, - {"http://foo.com/bar?a=b", "/baz?c=d", "http://foo.com/baz"}, - - // Multiple slashes - {"http://foo.com/bar", "http://foo.com//baz", "http://foo.com//baz"}, - {"http://foo.com/bar", "http://foo.com///baz/quux", "http://foo.com///baz/quux"}, - - // Scheme-relative - {"https://foo.com/bar?a=b", "//bar.com/quux", "https://bar.com/quux"}, - - // Path-relative references: - - // ... current directory - {"http://foo.com", ".", "http://foo.com/"}, - {"http://foo.com/bar", ".", "http://foo.com/"}, - {"http://foo.com/bar/", ".", "http://foo.com/bar/"}, - - // ... going down - {"http://foo.com", "bar", "http://foo.com/bar"}, - {"http://foo.com/", "bar", "http://foo.com/bar"}, - {"http://foo.com/bar/baz", "quux", "http://foo.com/bar/quux"}, - - // ... going up - {"http://foo.com/bar/baz", "../quux", "http://foo.com/quux"}, - {"http://foo.com/bar/baz", "../../../../../quux", "http://foo.com/quux"}, - {"http://foo.com/bar", "..", "http://foo.com/"}, - {"http://foo.com/bar/baz", "./..", "http://foo.com/"}, - // ".." in the middle (issue 3560) - {"http://foo.com/bar/baz", "quux/dotdot/../tail", "http://foo.com/bar/quux/tail"}, - {"http://foo.com/bar/baz", "quux/./dotdot/../tail", "http://foo.com/bar/quux/tail"}, - {"http://foo.com/bar/baz", "quux/./dotdot/.././tail", "http://foo.com/bar/quux/tail"}, - {"http://foo.com/bar/baz", "quux/./dotdot/./../tail", "http://foo.com/bar/quux/tail"}, - {"http://foo.com/bar/baz", "quux/./dotdot/dotdot/././../../tail", "http://foo.com/bar/quux/tail"}, - {"http://foo.com/bar/baz", "quux/./dotdot/dotdot/./.././../tail", "http://foo.com/bar/quux/tail"}, - {"http://foo.com/bar/baz", "quux/./dotdot/dotdot/dotdot/./../../.././././tail", "http://foo.com/bar/quux/tail"}, - {"http://foo.com/bar/baz", "quux/./dotdot/../dotdot/../dot/./tail/..", "http://foo.com/bar/quux/dot/"}, - - // Remove any dot-segments prior to forming the target URI. - // http://tools.ietf.org/html/rfc3986#section-5.2.4 - {"http://foo.com/dot/./dotdot/../foo/bar", "../baz", "http://foo.com/dot/baz"}, - - // Triple dot isn't special - {"http://foo.com/bar", "...", "http://foo.com/..."}, - - // Fragment - {"http://foo.com/bar", ".#frag", "http://foo.com/"}, - {"http://example.org/", "#!$&%27()*+,;=", "http://example.org/"}, - - // Paths with escaping (issue 16947). - {"http://foo.com/foo%2fbar/", "../baz", "http://foo.com/baz"}, - {"http://foo.com/1/2%2f/3%2f4/5", "../../a/b/c", "http://foo.com/1/a/b/c"}, - {"http://foo.com/1/2/3", "./a%2f../../b/..%2fc", "http://foo.com/1/2/b/..%2fc"}, - {"http://foo.com/1/2%2f/3%2f4/5", "./a%2f../b/../c", "http://foo.com/1/2%2f/3%2f4/a%2f../c"}, - {"http://foo.com/foo%20bar/", "../baz", "http://foo.com/baz"}, - {"http://foo.com/foo", "../bar%2fbaz", "http://foo.com/bar%2fbaz"}, - {"http://foo.com/foo%2dbar/", "./baz-quux", "http://foo.com/foo%2dbar/baz-quux"}, - - // RFC 3986: Normal Examples - // http://tools.ietf.org/html/rfc3986#section-5.4.1 - {"http://a/b/c/d;p?q", "g", "http://a/b/c/g"}, - {"http://a/b/c/d;p?q", "./g", "http://a/b/c/g"}, - {"http://a/b/c/d;p?q", "g/", "http://a/b/c/g/"}, - {"http://a/b/c/d;p?q", "/g", "http://a/g"}, - {"http://a/b/c/d;p?q", "//g", "http://g"}, - {"http://a/b/c/d;p?q", "?y", "http://a/b/c/d;p"}, - {"http://a/b/c/d;p?q", "g?y", "http://a/b/c/g"}, - {"http://a/b/c/d;p?q", "#s", "http://a/b/c/d;p"}, - {"http://a/b/c/d;p?q", "g#s", "http://a/b/c/g"}, - {"http://a/b/c/d;p?q", "g?y#s", "http://a/b/c/g"}, - {"http://a/b/c/d;p?q", ";x", "http://a/b/c/;x"}, - {"http://a/b/c/d;p?q", "g;x", "http://a/b/c/g;x"}, - {"http://a/b/c/d;p?q", "g;x?y#s", "http://a/b/c/g;x"}, - {"http://a/b/c/d;p?q", "", "http://a/b/c/d;p"}, - {"http://a/b/c/d;p?q", ".", "http://a/b/c/"}, - {"http://a/b/c/d;p?q", "./", "http://a/b/c/"}, - {"http://a/b/c/d;p?q", "..", "http://a/b/"}, - {"http://a/b/c/d;p?q", "../", "http://a/b/"}, - {"http://a/b/c/d;p?q", "../g", "http://a/b/g"}, - {"http://a/b/c/d;p?q", "../..", "http://a/"}, - {"http://a/b/c/d;p?q", "../../", "http://a/"}, - {"http://a/b/c/d;p?q", "../../g", "http://a/g"}, - - // RFC 3986: Abnormal Examples - // http://tools.ietf.org/html/rfc3986#section-5.4.2 - {"http://a/b/c/d;p?q", "../../../g", "http://a/g"}, - {"http://a/b/c/d;p?q", "../../../../g", "http://a/g"}, - {"http://a/b/c/d;p?q", "/./g", "http://a/g"}, - {"http://a/b/c/d;p?q", "/../g", "http://a/g"}, - {"http://a/b/c/d;p?q", "g.", "http://a/b/c/g."}, - {"http://a/b/c/d;p?q", ".g", "http://a/b/c/.g"}, - {"http://a/b/c/d;p?q", "g..", "http://a/b/c/g.."}, - {"http://a/b/c/d;p?q", "..g", "http://a/b/c/..g"}, - {"http://a/b/c/d;p?q", "./../g", "http://a/b/g"}, - {"http://a/b/c/d;p?q", "./g/.", "http://a/b/c/g/"}, - {"http://a/b/c/d;p?q", "g/./h", "http://a/b/c/g/h"}, - {"http://a/b/c/d;p?q", "g/../h", "http://a/b/c/h"}, - {"http://a/b/c/d;p?q", "g;x=1/./y", "http://a/b/c/g;x=1/y"}, - {"http://a/b/c/d;p?q", "g;x=1/../y", "http://a/b/c/y"}, - {"http://a/b/c/d;p?q", "g?y/./x", "http://a/b/c/g"}, - {"http://a/b/c/d;p?q", "g?y/../x", "http://a/b/c/g"}, - {"http://a/b/c/d;p?q", "g#s/./x", "http://a/b/c/g"}, - {"http://a/b/c/d;p?q", "g#s/../x", "http://a/b/c/g"}, - - // Extras. - {"https://a/b/c/d;p?q", "//g?q", "https://g"}, - {"https://a/b/c/d;p?q", "//g#s", "https://g"}, - {"https://a/b/c/d;p?q", "//g/d/e/f?y#s", "https://g/d/e/f"}, - {"https://a/b/c/d;p#s", "?y", "https://a/b/c/d;p"}, - {"https://a/b/c/d;p?q#s", "?y", "https://a/b/c/d;p"}, -} - -func TestResolveReference(t *testing.T) { - mustParse := func(url string) *URL { - u := new(URL) - err := u.Parse(url) - if err != nil { - t.Fatalf("Parse(%q) got err %v", url, err) - } - return u - } - for _, test := range resolveReferenceTests { - base := mustParse(test.base) - rel := mustParse(test.rel) - var url URL - base.ResolveReference(&url, rel) - if got := url.String(); got != test.expected { - t.Errorf("URL(%q).ResolveReference(%q)\ngot %q\nwant %q", test.base, test.rel, got, test.expected) - } - } -} - -type RequestURITest struct { - url *URL - out string -} - -var requritests = []RequestURITest{ - { - &URL{ - Scheme: SchemeHTTP, - Host: "example.com", - Path: "", - }, - "/", - }, - { - &URL{ - Scheme: SchemeHTTP, - Host: "example.com", - Path: "/a b", - }, - "/a%20b", - }, - { - &URL{ - Scheme: SchemeHTTP, - Host: "example.com", - Path: "//foo", - }, - "//foo", - }, -} - -func TestParseErrors(t *testing.T) { - tests := []struct { - in string - wantErr bool - }{ - {"http://[::1]", false}, - {"http://[::1]:80", false}, - {"http://[::1]:namedport", true}, // rfc3986 3.2.3 - {"http://[::1]/", false}, - {"http://[::1]a", true}, - {"http://[::1]%23", true}, - {"http://[::1%25en0]", false}, // valid zone id - {"http://[::1]:", false}, // colon, but no port OK - {"http://[::1]:%38%30", true}, // not allowed: % encoding only for non-ASCII - {"http://[::1%25%41]", false}, // RFC 6874 allows over-escaping in zone - {"http://[%10::1]", true}, // no %xx escapes in IP address - {"http://[::1]/%48", false}, // %xx in path is fine - {"http://%41:8080/", true}, // not allowed: % encoding only for non-ASCII - - {"http://[]%20%48%54%54%50%2f%31%2e%31%0a%4d%79%48%65%61%64%65%72%3a%20%31%32%33%0a%0a/", true}, // golang.org/issue/11208 - {"http://a b.com/", true}, // no space in host name please - } - for _, tt := range tests { - var u URL - err := u.Parse(tt.in) - if tt.wantErr { - if err == nil { - t.Errorf("Parse(%q) = %#v; want an error", tt.in, u) - } - continue - } - if err != nil { - t.Logf("Parse(%q) = %v; want no error", tt.in, err) - } - } -} - -type shouldEscapeTest struct { - in byte - mode encoding - escape bool -} - -var shouldEscapeTests = []shouldEscapeTest{ - // Unreserved characters (§2.3) - {'a', encodePath, false}, - {'a', encodeUserPassword, false}, - {'a', encodeQueryComponent, false}, - {'a', encodeFragment, false}, - {'a', encodeHost, false}, - {'z', encodePath, false}, - {'A', encodePath, false}, - {'Z', encodePath, false}, - {'0', encodePath, false}, - {'9', encodePath, false}, - {'-', encodePath, false}, - {'-', encodeUserPassword, false}, - {'-', encodeQueryComponent, false}, - {'-', encodeFragment, false}, - {'.', encodePath, false}, - {'_', encodePath, false}, - {'~', encodePath, false}, - - // User information (§3.2.1) - {':', encodeUserPassword, true}, - {'/', encodeUserPassword, true}, - {'?', encodeUserPassword, true}, - {'@', encodeUserPassword, true}, - {'$', encodeUserPassword, false}, - {'&', encodeUserPassword, false}, - {'+', encodeUserPassword, false}, - {',', encodeUserPassword, false}, - {';', encodeUserPassword, false}, - {'=', encodeUserPassword, false}, - - // Host (IP address, IPv6 address, registered name, port suffix; §3.2.2) - {'!', encodeHost, false}, - {'$', encodeHost, false}, - {'&', encodeHost, false}, - {'\'', encodeHost, false}, - {'(', encodeHost, false}, - {')', encodeHost, false}, - {'*', encodeHost, false}, - {'+', encodeHost, false}, - {',', encodeHost, false}, - {';', encodeHost, false}, - {'=', encodeHost, false}, - {':', encodeHost, false}, - {'[', encodeHost, false}, - {']', encodeHost, false}, - {'0', encodeHost, false}, - {'9', encodeHost, false}, - {'A', encodeHost, false}, - {'z', encodeHost, false}, - {'_', encodeHost, false}, - {'-', encodeHost, false}, - {'.', encodeHost, false}, -} - -func TestShouldEscape(t *testing.T) { - for _, tt := range shouldEscapeTests { - if shouldEscape(tt.in, tt.mode) != tt.escape { - t.Errorf("shouldEscape(%q, %v) returned %v; expected %v", tt.in, tt.mode, !tt.escape, tt.escape) - } - } -} - -type timeoutError struct { - timeout bool -} - -func (e *timeoutError) Error() string { return "timeout error" } -func (e *timeoutError) Timeout() bool { return e.timeout } - -type temporaryError struct { - temporary bool -} - -func (e *temporaryError) Error() string { return "temporary error" } -func (e *temporaryError) Temporary() bool { return e.temporary } - -type timeoutTemporaryError struct { - timeoutError - temporaryError -} - -func (e *timeoutTemporaryError) Error() string { return "timeout/temporary error" } - -var netErrorTests = []struct { - err error - timeout bool - temporary bool -}{{ - err: &Error{"Get", "http://google.com/", &timeoutError{timeout: true}}, - timeout: true, - temporary: false, -}, { - err: &Error{"Get", "http://google.com/", &timeoutError{timeout: false}}, - timeout: false, - temporary: false, -}, { - err: &Error{"Get", "http://google.com/", &temporaryError{temporary: true}}, - timeout: false, - temporary: true, -}, { - err: &Error{"Get", "http://google.com/", &temporaryError{temporary: false}}, - timeout: false, - temporary: false, -}, { - err: &Error{"Get", "http://google.com/", &timeoutTemporaryError{timeoutError{timeout: true}, temporaryError{temporary: true}}}, - timeout: true, - temporary: true, -}, { - err: &Error{"Get", "http://google.com/", &timeoutTemporaryError{timeoutError{timeout: false}, temporaryError{temporary: true}}}, - timeout: false, - temporary: true, -}, { - err: &Error{"Get", "http://google.com/", &timeoutTemporaryError{timeoutError{timeout: true}, temporaryError{temporary: false}}}, - timeout: true, - temporary: false, -}, { - err: &Error{"Get", "http://google.com/", &timeoutTemporaryError{timeoutError{timeout: false}, temporaryError{temporary: false}}}, - timeout: false, - temporary: false, -}, { - err: &Error{"Get", "http://google.com/", io.EOF}, - timeout: false, - temporary: false, -}} - -// Test that url.Error implements net.Error and that it forwards -func TestURLErrorImplementsNetError(t *testing.T) { - for i, tt := range netErrorTests { - err, ok := tt.err.(net.Error) - if !ok { - t.Errorf("%d: %T does not implement net.Error", i+1, tt.err) - continue - } - if err.Timeout() != tt.timeout { - t.Errorf("%d: err.Timeout(): got %v, want %v", i+1, err.Timeout(), tt.timeout) - continue - } - if err.Temporary() != tt.temporary { - t.Errorf("%d: err.Temporary(): got %v, want %v", i+1, err.Temporary(), tt.temporary) - } - } -} - -var _ encodingPkg.BinaryMarshaler = (*URL)(nil) -var _ encodingPkg.BinaryUnmarshaler = (*URL)(nil) - -func TestJSON(t *testing.T) { - var u URL - err := u.Parse("https://www.google.com/x?y=z") - if err != nil { - t.Fatal(err) - } - js, err := json.Marshal(&u) - if err != nil { - t.Fatal(err) - } - - // If only we could implement TextMarshaler/TextUnmarshaler, - // this would work: - // - // if string(js) != strconv.Quote(u.String()) { - // t.Errorf("json encoding: %s\nwant: %s\n", js, strconv.Quote(u.String())) - // } - - u1 := new(URL) - err = json.Unmarshal(js, u1) - if err != nil { - t.Fatal(err) - } - if u1.String() != u.String() { - t.Errorf("json decoded to: %s\nwant: %s\n", u1, &u) - } -} - -func TestGob(t *testing.T) { - var u URL - err := u.Parse("https://www.google.com/x?y=z") - if err != nil { - t.Fatal(err) - } - var w bytes.Buffer - err = gob.NewEncoder(&w).Encode(&u) - if err != nil { - t.Fatal(err) - } - - u1 := new(URL) - err = gob.NewDecoder(&w).Decode(u1) - if err != nil { - t.Fatal(err) - } - if u1.String() != u.String() { - t.Errorf("json decoded to: %s\nwant: %s\n", u1, &u) - } -} diff --git a/main.go b/main.go index 260b27f..448c71e 100644 --- a/main.go +++ b/main.go @@ -3,8 +3,8 @@ package main import ( "context" "github.com/sirupsen/logrus" - "github.com/terorie/oddb-go/fasturl" "github.com/urfave/cli" + "github.com/valyala/fasthttp" "log" "net/http" _ "net/http/pprof" @@ -55,12 +55,12 @@ func cmdCrawler(clic *cli.Context) error { if !strings.Contains(arg, "://") { arg = "http://" + arg } - var u fasturl.URL - err := u.Parse(arg) - if !strings.HasSuffix(u.Path, "/") { - u.Path += "/" + var u fasthttp.URI + u.Parse(nil, []byte(arg)) + uPath := string(u.Path()) + if !strings.HasSuffix(uPath, "/") { + u.SetPath(uPath + "/") } - if err != nil { return err } remotes[i] = &OD { Task: &Task{ WebsiteId: 0, diff --git a/model.go b/model.go index 1c6ab4b..8365466 100644 --- a/model.go +++ b/model.go @@ -2,14 +2,14 @@ package main import ( "github.com/terorie/oddb-go/ds/redblackhash" - "github.com/terorie/oddb-go/fasturl" + "github.com/valyala/fasthttp" "sync" "time" ) type Job struct { OD *OD - Uri fasturl.URL + Uri fasthttp.URI UriStr string Fails int LastError error @@ -18,7 +18,7 @@ type Job struct { type OD struct { Task *Task Wait sync.WaitGroup - BaseUri fasturl.URL + BaseUri fasthttp.URI WCtx WorkerContext Scanned redblackhash.Tree diff --git a/worker.go b/worker.go index 813f49b..59e0db4 100644 --- a/worker.go +++ b/worker.go @@ -1,10 +1,10 @@ package main import ( + "bytes" "github.com/sirupsen/logrus" "math" "sort" - "strings" "sync" "sync/atomic" "time" @@ -70,8 +70,9 @@ func (w WorkerContext) step(results chan<- File, job Job) { } func DoJob(job *Job, f *File) (newJobs []Job, err error) { - if len(job.Uri.Path) == 0 { return } - if job.Uri.Path[len(job.Uri.Path)-1] == '/' { + uriPath := job.Uri.Path() + if len(uriPath) == 0 { return } + if uriPath[len(uriPath)-1] == '/' { // Load directory links, err := GetDir(job, f) if err != nil { @@ -91,7 +92,7 @@ func DoJob(job *Job, f *File) (newJobs []Job, err error) { // Sort by path sort.Slice(links, func(i, j int) bool { - return strings.Compare(links[i].Path, links[j].Path) < 0 + return bytes.Compare(links[i].Path(), links[j].Path()) < 0 }) var newJobCount int