mirror of
https://github.com/terorie/od-database-crawler.git
synced 2025-04-10 05:56:42 +00:00
fasthttp uri
This commit is contained in:
Richard Patel
parent
3c39f0d621
commit
8f6f8fd17f
32
crawl.go
32
crawl.go
@ -4,7 +4,6 @@ import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"github.com/terorie/oddb-go/ds/redblackhash"
|
||||
"github.com/terorie/oddb-go/fasturl"
|
||||
"github.com/valyala/fasthttp"
|
||||
"golang.org/x/crypto/blake2b"
|
||||
"golang.org/x/net/html"
|
||||
@ -16,9 +15,9 @@ import (
|
||||
|
||||
var client fasthttp.Client
|
||||
|
||||
func GetDir(j *Job, f *File) (links []fasturl.URL, err error) {
|
||||
func GetDir(j *Job, f *File) (links []fasthttp.URI, err error) {
|
||||
f.IsDir = true
|
||||
f.Name = path.Base(j.Uri.Path)
|
||||
f.Name = path.Base(string(j.Uri.Path()))
|
||||
|
||||
req := fasthttp.AcquireRequest()
|
||||
req.SetRequestURI(j.UriStr)
|
||||
@ -81,14 +80,15 @@ func GetDir(j *Job, f *File) (links []fasturl.URL, err error) {
|
||||
goto nextToken
|
||||
}
|
||||
|
||||
var link fasturl.URL
|
||||
err = j.Uri.ParseRel(&link, href)
|
||||
var link fasthttp.URI
|
||||
j.Uri.CopyTo(&link)
|
||||
link.Update(href)
|
||||
if err != nil { continue }
|
||||
|
||||
if link.Scheme != j.Uri.Scheme ||
|
||||
link.Host != j.Uri.Host ||
|
||||
link.Path == j.Uri.Path ||
|
||||
!strings.HasPrefix(link.Path, j.Uri.Path) {
|
||||
if !bytes.Equal(link.Scheme(), j.Uri.Scheme()) ||
|
||||
!bytes.Equal(link.Host(), j.Uri.Host()) ||
|
||||
bytes.Equal(link.Path(), j.Uri.Path()) ||
|
||||
!bytes.HasPrefix(link.Path(), j.Uri.Path()) {
|
||||
continue
|
||||
}
|
||||
|
||||
@ -102,11 +102,12 @@ func GetDir(j *Job, f *File) (links []fasturl.URL, err error) {
|
||||
return
|
||||
}
|
||||
|
||||
func GetFile(u fasturl.URL, f *File) (err error) {
|
||||
func GetFile(u fasthttp.URI, f *File) (err error) {
|
||||
f.IsDir = false
|
||||
u.Path = path.Clean(u.Path)
|
||||
f.Name = path.Base(u.Path)
|
||||
f.Path = strings.Trim(u.Path, "/")
|
||||
cleanPath := path.Clean(string(u.Path()))
|
||||
u.SetPath(cleanPath)
|
||||
f.Name = path.Base(cleanPath)
|
||||
f.Path = strings.Trim(cleanPath, "/")
|
||||
|
||||
req := fasthttp.AcquireRequest()
|
||||
req.Header.SetMethod("HEAD")
|
||||
@ -130,12 +131,11 @@ func GetFile(u fasturl.URL, f *File) (err error) {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *File) HashDir(links []fasturl.URL) (o redblackhash.Key) {
|
||||
func (f *File) HashDir(links []fasthttp.URI) (o redblackhash.Key) {
|
||||
h, _ := blake2b.New256(nil)
|
||||
h.Write([]byte(f.Name))
|
||||
for _, link := range links {
|
||||
fileName := path.Base(link.Path)
|
||||
h.Write([]byte(fileName))
|
||||
h.Write(link.Path())
|
||||
}
|
||||
sum := h.Sum(nil)
|
||||
copy(o[:redblackhash.KeySize], sum)
|
||||
|
||||
813
fasturl/url.go
813
fasturl/url.go
@ -1,813 +0,0 @@
|
||||
// Copyright 2009 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
// Package fasturl parses URLs and implements query escaping.
|
||||
package fasturl
|
||||
|
||||
// Modifications by terorie
|
||||
|
||||
// See RFC 3986. This package generally follows RFC 3986, except where
|
||||
// it deviates for compatibility reasons. When sending changes, first
|
||||
// search old issues for history on decisions. Unit tests should also
|
||||
// contain references to issue numbers with details.
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type Scheme int
|
||||
const (
|
||||
SchemeInvalid = iota
|
||||
SchemeHTTP
|
||||
SchemeHTTPS
|
||||
SchemeCount
|
||||
)
|
||||
|
||||
var Schemes = [SchemeCount]string {
|
||||
"",
|
||||
"http",
|
||||
"https",
|
||||
}
|
||||
|
||||
// Error reports an error and the operation and URL that caused it.
|
||||
type Error struct {
|
||||
Op string
|
||||
URL string
|
||||
Err error
|
||||
}
|
||||
|
||||
func (e *Error) Error() string { return e.Op + " " + e.URL + ": " + e.Err.Error() }
|
||||
|
||||
type timeout interface {
|
||||
Timeout() bool
|
||||
}
|
||||
|
||||
func (e *Error) Timeout() bool {
|
||||
t, ok := e.Err.(timeout)
|
||||
return ok && t.Timeout()
|
||||
}
|
||||
|
||||
type temporary interface {
|
||||
Temporary() bool
|
||||
}
|
||||
|
||||
func (e *Error) Temporary() bool {
|
||||
t, ok := e.Err.(temporary)
|
||||
return ok && t.Temporary()
|
||||
}
|
||||
|
||||
func ishex(c byte) bool {
|
||||
switch {
|
||||
case '0' <= c && c <= '9':
|
||||
return true
|
||||
case 'a' <= c && c <= 'f':
|
||||
return true
|
||||
case 'A' <= c && c <= 'F':
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func unhex(c byte) byte {
|
||||
switch {
|
||||
case '0' <= c && c <= '9':
|
||||
return c - '0'
|
||||
case 'a' <= c && c <= 'f':
|
||||
return c - 'a' + 10
|
||||
case 'A' <= c && c <= 'F':
|
||||
return c - 'A' + 10
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
type encoding int
|
||||
|
||||
const (
|
||||
encodePath encoding = 1 + iota
|
||||
encodePathSegment
|
||||
encodeHost
|
||||
encodeZone
|
||||
encodeUserPassword
|
||||
encodeQueryComponent
|
||||
encodeFragment
|
||||
)
|
||||
|
||||
type EscapeError string
|
||||
|
||||
func (e EscapeError) Error() string {
|
||||
return "invalid URL escape " + strconv.Quote(string(e))
|
||||
}
|
||||
|
||||
type InvalidHostError string
|
||||
|
||||
func (e InvalidHostError) Error() string {
|
||||
return "invalid character " + strconv.Quote(string(e)) + " in host name"
|
||||
}
|
||||
|
||||
// Return true if the specified character should be escaped when
|
||||
// appearing in a URL string, according to RFC 3986.
|
||||
//
|
||||
// Please be informed that for now shouldEscape does not check all
|
||||
// reserved characters correctly. See golang.org/issue/5684.
|
||||
func shouldEscape(c byte, mode encoding) bool {
|
||||
// §2.3 Unreserved characters (alphanum)
|
||||
if 'A' <= c && c <= 'Z' || 'a' <= c && c <= 'z' || '0' <= c && c <= '9' {
|
||||
return false
|
||||
}
|
||||
|
||||
if mode == encodeHost || mode == encodeZone {
|
||||
// §3.2.2 Host allows
|
||||
// sub-delims = "!" / "$" / "&" / "'" / "(" / ")" / "*" / "+" / "," / ";" / "="
|
||||
// as part of reg-name.
|
||||
// We add : because we include :port as part of host.
|
||||
// We add [ ] because we include [ipv6]:port as part of host.
|
||||
// We add < > because they're the only characters left that
|
||||
// we could possibly allow, and Parse will reject them if we
|
||||
// escape them (because hosts can't use %-encoding for
|
||||
// ASCII bytes).
|
||||
switch c {
|
||||
case '!', '$', '&', '\'', '(', ')', '*', '+', ',', ';', '=', ':', '[', ']', '<', '>', '"':
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
switch c {
|
||||
case '-', '_', '.', '~': // §2.3 Unreserved characters (mark)
|
||||
return false
|
||||
|
||||
case '$', '&', '+', ',', '/', ':', ';', '=', '?', '@': // §2.2 Reserved characters (reserved)
|
||||
// Different sections of the URL allow a few of
|
||||
// the reserved characters to appear unescaped.
|
||||
switch mode {
|
||||
case encodePath: // §3.3
|
||||
// The RFC allows : @ & = + $ but saves / ; , for assigning
|
||||
// meaning to individual path segments. This package
|
||||
// only manipulates the path as a whole, so we allow those
|
||||
// last three as well. That leaves only ? to escape.
|
||||
return c == '?'
|
||||
|
||||
case encodePathSegment: // §3.3
|
||||
// The RFC allows : @ & = + $ but saves / ; , for assigning
|
||||
// meaning to individual path segments.
|
||||
return c == '/' || c == ';' || c == ',' || c == '?'
|
||||
|
||||
case encodeUserPassword: // §3.2.1
|
||||
// The RFC allows ';', ':', '&', '=', '+', '$', and ',' in
|
||||
// userinfo, so we must escape only '@', '/', and '?'.
|
||||
// The parsing of userinfo treats ':' as special so we must escape
|
||||
// that too.
|
||||
return c == '@' || c == '/' || c == '?' || c == ':'
|
||||
|
||||
case encodeQueryComponent: // §3.4
|
||||
// The RFC reserves (so we must escape) everything.
|
||||
return true
|
||||
|
||||
case encodeFragment: // §4.1
|
||||
// The RFC text is silent but the grammar allows
|
||||
// everything, so escape nothing.
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
if mode == encodeFragment {
|
||||
// RFC 3986 §2.2 allows not escaping sub-delims. A subset of sub-delims are
|
||||
// included in reserved from RFC 2396 §2.2. The remaining sub-delims do not
|
||||
// need to be escaped. To minimize potential breakage, we apply two restrictions:
|
||||
// (1) we always escape sub-delims outside of the fragment, and (2) we always
|
||||
// escape single quote to avoid breaking callers that had previously assumed that
|
||||
// single quotes would be escaped. See issue #19917.
|
||||
switch c {
|
||||
case '!', '(', ')', '*':
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// Everything else must be escaped.
|
||||
return true
|
||||
}
|
||||
|
||||
// unescape unescapes a string; the mode specifies
|
||||
// which section of the URL string is being unescaped.
|
||||
func unescape(s string, mode encoding) (string, error) {
|
||||
// Count %, check that they're well-formed.
|
||||
n := 0
|
||||
hasPlus := false
|
||||
for i := 0; i < len(s); {
|
||||
switch s[i] {
|
||||
case '%':
|
||||
n++
|
||||
if i+2 >= len(s) || !ishex(s[i+1]) || !ishex(s[i+2]) {
|
||||
s = s[i:]
|
||||
if len(s) > 3 {
|
||||
s = s[:3]
|
||||
}
|
||||
return "", EscapeError(s)
|
||||
}
|
||||
// Per https://tools.ietf.org/html/rfc3986#page-21
|
||||
// in the host component %-encoding can only be used
|
||||
// for non-ASCII bytes.
|
||||
// But https://tools.ietf.org/html/rfc6874#section-2
|
||||
// introduces %25 being allowed to escape a percent sign
|
||||
// in IPv6 scoped-address literals. Yay.
|
||||
if mode == encodeHost && unhex(s[i+1]) < 8 && s[i:i+3] != "%25" {
|
||||
return "", EscapeError(s[i : i+3])
|
||||
}
|
||||
if mode == encodeZone {
|
||||
// RFC 6874 says basically "anything goes" for zone identifiers
|
||||
// and that even non-ASCII can be redundantly escaped,
|
||||
// but it seems prudent to restrict %-escaped bytes here to those
|
||||
// that are valid host name bytes in their unescaped form.
|
||||
// That is, you can use escaping in the zone identifier but not
|
||||
// to introduce bytes you couldn't just write directly.
|
||||
// But Windows puts spaces here! Yay.
|
||||
v := unhex(s[i+1])<<4 | unhex(s[i+2])
|
||||
if s[i:i+3] != "%25" && v != ' ' && shouldEscape(v, encodeHost) {
|
||||
return "", EscapeError(s[i : i+3])
|
||||
}
|
||||
}
|
||||
i += 3
|
||||
case '+':
|
||||
hasPlus = mode == encodeQueryComponent
|
||||
i++
|
||||
default:
|
||||
if (mode == encodeHost || mode == encodeZone) && s[i] < 0x80 && shouldEscape(s[i], mode) {
|
||||
return "", InvalidHostError(s[i : i+1])
|
||||
}
|
||||
i++
|
||||
}
|
||||
}
|
||||
|
||||
if n == 0 && !hasPlus {
|
||||
return s, nil
|
||||
}
|
||||
|
||||
t := make([]byte, len(s)-2*n)
|
||||
j := 0
|
||||
for i := 0; i < len(s); {
|
||||
switch s[i] {
|
||||
case '%':
|
||||
t[j] = unhex(s[i+1])<<4 | unhex(s[i+2])
|
||||
j++
|
||||
i += 3
|
||||
case '+':
|
||||
if mode == encodeQueryComponent {
|
||||
t[j] = ' '
|
||||
} else {
|
||||
t[j] = '+'
|
||||
}
|
||||
j++
|
||||
i++
|
||||
default:
|
||||
t[j] = s[i]
|
||||
j++
|
||||
i++
|
||||
}
|
||||
}
|
||||
return string(t), nil
|
||||
}
|
||||
|
||||
func escape(s string, mode encoding) string {
|
||||
spaceCount, hexCount := 0, 0
|
||||
for i := 0; i < len(s); i++ {
|
||||
c := s[i]
|
||||
if shouldEscape(c, mode) {
|
||||
if c == ' ' && mode == encodeQueryComponent {
|
||||
spaceCount++
|
||||
} else {
|
||||
hexCount++
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if spaceCount == 0 && hexCount == 0 {
|
||||
return s
|
||||
}
|
||||
|
||||
t := make([]byte, len(s)+2*hexCount)
|
||||
j := 0
|
||||
for i := 0; i < len(s); i++ {
|
||||
switch c := s[i]; {
|
||||
case c == ' ' && mode == encodeQueryComponent:
|
||||
t[j] = '+'
|
||||
j++
|
||||
case shouldEscape(c, mode):
|
||||
t[j] = '%'
|
||||
t[j+1] = "0123456789ABCDEF"[c>>4]
|
||||
t[j+2] = "0123456789ABCDEF"[c&15]
|
||||
j += 3
|
||||
default:
|
||||
t[j] = s[i]
|
||||
j++
|
||||
}
|
||||
}
|
||||
return string(t)
|
||||
}
|
||||
|
||||
// A URL represents a parsed URL (technically, a URI reference).
|
||||
//
|
||||
// The general form represented is:
|
||||
//
|
||||
// [scheme:][//[userinfo@]host][/]path[?query][#fragment]
|
||||
//
|
||||
// URLs that do not start with a slash after the scheme are interpreted as:
|
||||
//
|
||||
// scheme:opaque[?query][#fragment]
|
||||
//
|
||||
// Note that the Path field is stored in decoded form: /%47%6f%2f becomes /Go/.
|
||||
// A consequence is that it is impossible to tell which slashes in the Path were
|
||||
// slashes in the raw URL and which were %2f. This distinction is rarely important,
|
||||
// but when it is, code must not use Path directly.
|
||||
// The Parse function sets both Path and RawPath in the URL it returns,
|
||||
// and URL's String method uses RawPath if it is a valid encoding of Path,
|
||||
// by calling the EscapedPath method.
|
||||
type URL struct {
|
||||
Scheme Scheme
|
||||
Host string // host or host:port
|
||||
Path string // path (relative paths may omit leading slash)
|
||||
}
|
||||
|
||||
// Maybe rawurl is of the form scheme:path.
|
||||
// (Scheme must be [a-zA-Z][a-zA-Z0-9+-.]*)
|
||||
// If so, return scheme, path; else return "", rawurl.
|
||||
func getscheme(rawurl string) (scheme Scheme, path string, err error) {
|
||||
for i := 0; i < len(rawurl); i++ {
|
||||
c := rawurl[i]
|
||||
switch {
|
||||
case 'a' <= c && c <= 'z' || 'A' <= c && c <= 'Z':
|
||||
// do nothing
|
||||
case '0' <= c && c <= '9' || c == '+' || c == '-' || c == '.':
|
||||
if i == 0 {
|
||||
return SchemeInvalid, rawurl, nil
|
||||
}
|
||||
case c == ':':
|
||||
if i == 0 {
|
||||
return SchemeInvalid, "", errors.New("missing protocol scheme")
|
||||
}
|
||||
switch rawurl[:i] {
|
||||
case "http":
|
||||
scheme = SchemeHTTP
|
||||
case "https":
|
||||
scheme = SchemeHTTPS
|
||||
default:
|
||||
return SchemeInvalid, "", errors.New("unknown protocol scheme")
|
||||
}
|
||||
|
||||
path = rawurl[i+1:]
|
||||
return
|
||||
default:
|
||||
// we have encountered an invalid character,
|
||||
// so there is no valid scheme
|
||||
return SchemeInvalid, rawurl, nil
|
||||
}
|
||||
}
|
||||
return SchemeInvalid, rawurl, nil
|
||||
}
|
||||
|
||||
// Maybe s is of the form t c u.
|
||||
// If so, return t, c u (or t, u if cutc == true).
|
||||
// If not, return s, "".
|
||||
func split(s string, c string, cutc bool) (string, string) {
|
||||
i := strings.Index(s, c)
|
||||
if i < 0 {
|
||||
return s, ""
|
||||
}
|
||||
if cutc {
|
||||
return s[:i], s[i+len(c):]
|
||||
}
|
||||
return s[:i], s[i:]
|
||||
}
|
||||
|
||||
// Parse parses rawurl into a URL structure.
|
||||
//
|
||||
// The rawurl may be relative (a path, without a host) or absolute
|
||||
// (starting with a scheme). Trying to parse a hostname and path
|
||||
// without a scheme is invalid but may not necessarily return an
|
||||
// error, due to parsing ambiguities.
|
||||
func (u *URL) Parse(rawurl string) error {
|
||||
// Cut off #frag
|
||||
s, frag := split(rawurl, "#", true)
|
||||
err := u.parse(s, false)
|
||||
if err != nil {
|
||||
return &Error{"parse", s, err}
|
||||
}
|
||||
if frag == "" {
|
||||
return nil
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ParseRequestURI parses rawurl into a URL structure. It assumes that
|
||||
// rawurl was received in an HTTP request, so the rawurl is interpreted
|
||||
// only as an absolute URI or an absolute path.
|
||||
// The string rawurl is assumed not to have a #fragment suffix.
|
||||
// (Web browsers strip #fragment before sending the URL to a web server.)
|
||||
func (u *URL) ParseRequestURI(rawurl string) error {
|
||||
err := u.parse(rawurl, true)
|
||||
if err != nil {
|
||||
return &Error{"parse", rawurl, err}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// parse parses a URL from a string in one of two contexts. If
|
||||
// viaRequest is true, the URL is assumed to have arrived via an HTTP request,
|
||||
// in which case only absolute URLs or path-absolute relative URLs are allowed.
|
||||
// If viaRequest is false, all forms of relative URLs are allowed.
|
||||
func (u *URL) parse(rawurl string, viaRequest bool) error {
|
||||
var rest string
|
||||
var err error
|
||||
|
||||
if rawurl == "" && viaRequest {
|
||||
return errors.New("empty url")
|
||||
}
|
||||
|
||||
if rawurl == "*" {
|
||||
u.Path = "*"
|
||||
return nil
|
||||
}
|
||||
|
||||
// Split off possible leading "http:", "mailto:", etc.
|
||||
// Cannot contain escaped characters.
|
||||
if u.Scheme, rest, err = getscheme(rawurl); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if strings.HasSuffix(rest, "?") && strings.Count(rest, "?") == 1 {
|
||||
rest = rest[:len(rest)-1]
|
||||
} else {
|
||||
rest, _ = split(rest, "?", true)
|
||||
}
|
||||
|
||||
if !strings.HasPrefix(rest, "/") {
|
||||
if u.Scheme != SchemeInvalid {
|
||||
// We consider rootless paths per RFC 3986 as opaque.
|
||||
return nil
|
||||
}
|
||||
if viaRequest {
|
||||
return errors.New("invalid URI for request")
|
||||
}
|
||||
|
||||
// Avoid confusion with malformed schemes, like cache_object:foo/bar.
|
||||
// See golang.org/issue/16822.
|
||||
//
|
||||
// RFC 3986, §3.3:
|
||||
// In addition, a URI reference (Section 4.1) may be a relative-path reference,
|
||||
// in which case the first path segment cannot contain a colon (":") character.
|
||||
colon := strings.Index(rest, ":")
|
||||
slash := strings.Index(rest, "/")
|
||||
if colon >= 0 && (slash < 0 || colon < slash) {
|
||||
// First path segment has colon. Not allowed in relative URL.
|
||||
return errors.New("first path segment in URL cannot contain colon")
|
||||
}
|
||||
}
|
||||
|
||||
if (u.Scheme != SchemeInvalid || !viaRequest && !strings.HasPrefix(rest, "///")) && strings.HasPrefix(rest, "//") {
|
||||
var authority string
|
||||
authority, rest = split(rest[2:], "/", false)
|
||||
u.Host, err = parseAuthority(authority)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
u.Path = rest
|
||||
return nil
|
||||
}
|
||||
|
||||
func parseAuthority(authority string) (host string, err error) {
|
||||
i := strings.LastIndex(authority, "@")
|
||||
if i < 0 {
|
||||
host, err = parseHost(authority)
|
||||
} else {
|
||||
host, err = parseHost(authority[i+1:])
|
||||
}
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if i < 0 {
|
||||
return host, nil
|
||||
}
|
||||
userinfo := authority[:i]
|
||||
if !validUserinfo(userinfo) {
|
||||
return "", errors.New("fasturl: invalid userinfo")
|
||||
}
|
||||
return host, nil
|
||||
}
|
||||
|
||||
// parseHost parses host as an authority without user
|
||||
// information. That is, as host[:port].
|
||||
func parseHost(host string) (string, error) {
|
||||
if strings.HasPrefix(host, "[") {
|
||||
// Parse an IP-Literal in RFC 3986 and RFC 6874.
|
||||
// E.g., "[fe80::1]", "[fe80::1%25en0]", "[fe80::1]:80".
|
||||
i := strings.LastIndex(host, "]")
|
||||
if i < 0 {
|
||||
return "", errors.New("missing ']' in host")
|
||||
}
|
||||
colonPort := host[i+1:]
|
||||
if !validOptionalPort(colonPort) {
|
||||
return "", fmt.Errorf("invalid port %q after host", colonPort)
|
||||
}
|
||||
|
||||
// RFC 6874 defines that %25 (%-encoded percent) introduces
|
||||
// the zone identifier, and the zone identifier can use basically
|
||||
// any %-encoding it likes. That's different from the host, which
|
||||
// can only %-encode non-ASCII bytes.
|
||||
// We do impose some restrictions on the zone, to avoid stupidity
|
||||
// like newlines.
|
||||
zone := strings.Index(host[:i], "%25")
|
||||
if zone >= 0 {
|
||||
host1, err := unescape(host[:zone], encodeHost)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
host2, err := unescape(host[zone:i], encodeZone)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
host3, err := unescape(host[i:], encodeHost)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return host1 + host2 + host3, nil
|
||||
}
|
||||
}
|
||||
|
||||
var err error
|
||||
if host, err = unescape(host, encodeHost); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return host, nil
|
||||
}
|
||||
|
||||
// validOptionalPort reports whether port is either an empty string
|
||||
// or matches /^:\d*$/
|
||||
func validOptionalPort(port string) bool {
|
||||
if port == "" {
|
||||
return true
|
||||
}
|
||||
if port[0] != ':' {
|
||||
return false
|
||||
}
|
||||
for _, b := range port[1:] {
|
||||
if b < '0' || b > '9' {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// String reassembles the URL into a valid URL string.
|
||||
// The general form of the result is one of:
|
||||
//
|
||||
// scheme:opaque?query#fragment
|
||||
// scheme://userinfo@host/path?query#fragment
|
||||
//
|
||||
// If u.Opaque is non-empty, String uses the first form;
|
||||
// otherwise it uses the second form.
|
||||
// To obtain the path, String uses u.EscapedPath().
|
||||
//
|
||||
// In the second form, the following rules apply:
|
||||
// - if u.Scheme is empty, scheme: is omitted.
|
||||
// - if u.User is nil, userinfo@ is omitted.
|
||||
// - if u.Host is empty, host/ is omitted.
|
||||
// - if u.Scheme and u.Host are empty and u.User is nil,
|
||||
// the entire scheme://userinfo@host/ is omitted.
|
||||
// - if u.Host is non-empty and u.Path begins with a /,
|
||||
// the form host/path does not add its own /.
|
||||
// - if u.RawQuery is empty, ?query is omitted.
|
||||
// - if u.Fragment is empty, #fragment is omitted.
|
||||
func (u *URL) String() string {
|
||||
var buf strings.Builder
|
||||
if u.Scheme != SchemeInvalid {
|
||||
buf.WriteString(Schemes[u.Scheme])
|
||||
buf.WriteByte(':')
|
||||
}
|
||||
if u.Scheme != SchemeInvalid || u.Host != "" {
|
||||
if u.Host != "" || u.Path != "" {
|
||||
buf.WriteString("//")
|
||||
}
|
||||
if h := u.Host; h != "" {
|
||||
buf.WriteString(escape(h, encodeHost))
|
||||
}
|
||||
}
|
||||
path := u.Path
|
||||
if path != "" && path[0] != '/' && u.Host != "" {
|
||||
buf.WriteByte('/')
|
||||
}
|
||||
if buf.Len() == 0 {
|
||||
// RFC 3986 §4.2
|
||||
// A path segment that contains a colon character (e.g., "this:that")
|
||||
// cannot be used as the first segment of a relative-path reference, as
|
||||
// it would be mistaken for a scheme name. Such a segment must be
|
||||
// preceded by a dot-segment (e.g., "./this:that") to make a relative-
|
||||
// path reference.
|
||||
if i := strings.IndexByte(path, ':'); i > -1 && strings.IndexByte(path[:i], '/') == -1 {
|
||||
buf.WriteString("./")
|
||||
}
|
||||
}
|
||||
buf.WriteString(path)
|
||||
return buf.String()
|
||||
}
|
||||
|
||||
func isRunesDot(r []rune) bool {
|
||||
return len(r) == 1 && r[0] == '.'
|
||||
}
|
||||
|
||||
func isRunesDoubleDot(r []rune) bool {
|
||||
return len(r) == 2 && r[0] == '.' && r[1] == '.'
|
||||
}
|
||||
|
||||
// resolvePath applies special path segments from refs and applies
|
||||
// them to base, per RFC 3986.
|
||||
func resolvePath(base, ref string) string {
|
||||
var full string
|
||||
if ref == "" {
|
||||
full = base
|
||||
} else if ref[0] != '/' {
|
||||
i := strings.LastIndex(base, "/")
|
||||
full = base[:i+1] + ref
|
||||
} else {
|
||||
full = ref
|
||||
}
|
||||
if full == "" {
|
||||
return ""
|
||||
} else if full == "/" {
|
||||
return "/"
|
||||
}
|
||||
|
||||
dst := make([]rune, len(full))
|
||||
dst = dst[0:0]
|
||||
|
||||
start := 0
|
||||
rs := []rune(full)
|
||||
if len(rs) != 0 && rs[0] == '/' {
|
||||
rs = rs[1:]
|
||||
}
|
||||
var stack []int
|
||||
stack = append(stack, 0)
|
||||
for i, c := range rs {
|
||||
if i == len(rs) - 1 {
|
||||
closingSlash := false
|
||||
part := rs[start:]
|
||||
if len(part) == 0 {
|
||||
dst = append(dst, '/')
|
||||
} else if part[len(part)-1] == '/' {
|
||||
part = part[:len(part)-1]
|
||||
closingSlash = true
|
||||
}
|
||||
switch {
|
||||
case isRunesDot(part):
|
||||
dst = append(dst, '/')
|
||||
case isRunesDoubleDot(part):
|
||||
// Cut to the last slash
|
||||
start = i+1
|
||||
dst = dst[:stack[len(stack)-1]]
|
||||
if len(stack) != 1 {
|
||||
stack = stack[:len(stack)-1]
|
||||
}
|
||||
dst = append(dst, '/')
|
||||
default:
|
||||
dst = append(dst, '/')
|
||||
dst = append(dst, part...)
|
||||
}
|
||||
if closingSlash && len(dst) != 0 && dst[len(dst)-1] != '/' {
|
||||
dst = append(dst, '/')
|
||||
}
|
||||
} else if c == '/' {
|
||||
part := rs[start:i]
|
||||
switch {
|
||||
case isRunesDot(part):
|
||||
start = i+1
|
||||
case isRunesDoubleDot(part):
|
||||
// Cut to the last slash
|
||||
start = i+1
|
||||
dst = dst[:stack[len(stack)-1]]
|
||||
if len(stack) != 1 {
|
||||
stack = stack[:len(stack)-1]
|
||||
}
|
||||
default:
|
||||
start = i+1
|
||||
stack = append(stack, len(dst))
|
||||
dst = append(dst, '/')
|
||||
dst = append(dst, part...)
|
||||
}
|
||||
}
|
||||
}
|
||||
return string(dst)
|
||||
|
||||
/*var dst []string
|
||||
src := strings.Split(full, "/")
|
||||
for _, elem := range src {
|
||||
switch elem {
|
||||
case ".":
|
||||
// drop
|
||||
case "..":
|
||||
if len(dst) > 0 {
|
||||
dst = dst[:len(dst)-1]
|
||||
}
|
||||
default:
|
||||
dst = append(dst, elem)
|
||||
}
|
||||
}
|
||||
if last := src[len(src)-1]; last == "." || last == ".." {
|
||||
// Add final slash to the joined path.
|
||||
dst = append(dst, "")
|
||||
}
|
||||
return "/" + strings.TrimPrefix(strings.Join(dst, "/"), "/")*/
|
||||
}
|
||||
|
||||
// IsAbs reports whether the URL is absolute.
|
||||
// Absolute means that it has a non-empty scheme.
|
||||
func (u *URL) IsAbs() bool {
|
||||
return u.Scheme != SchemeInvalid
|
||||
}
|
||||
|
||||
// ParseRel parses a URL in the context of the receiver. The provided URL
|
||||
// may be relative or absolute. Parse returns nil, err on parse
|
||||
// failure, otherwise its return value is the same as ResolveReference.
|
||||
func (u *URL) ParseRel(out *URL, ref string) error {
|
||||
var refurl URL
|
||||
|
||||
err := refurl.Parse(ref)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
u.ResolveReference(out, &refurl)
|
||||
return nil
|
||||
}
|
||||
|
||||
// ResolveReference resolves a URI reference to an absolute URI from
|
||||
// an absolute base URI u, per RFC 3986 Section 5.2. The URI reference
|
||||
// may be relative or absolute. ResolveReference always returns a new
|
||||
// URL instance, even if the returned URL is identical to either the
|
||||
// base or reference. If ref is an absolute URL, then ResolveReference
|
||||
// ignores base and returns a copy of ref.
|
||||
func (u *URL) ResolveReference(url *URL, ref *URL) {
|
||||
*url = *ref
|
||||
if ref.Scheme == SchemeInvalid {
|
||||
url.Scheme = u.Scheme
|
||||
}
|
||||
if ref.Scheme != SchemeInvalid || ref.Host != "" {
|
||||
// The "absoluteURI" or "net_path" cases.
|
||||
// We can ignore the error from setPath since we know we provided a
|
||||
// validly-escaped path.
|
||||
url.Path = resolvePath(ref.Path, "")
|
||||
return
|
||||
}
|
||||
// The "abs_path" or "rel_path" cases.
|
||||
url.Host = u.Host
|
||||
url.Path = resolvePath(u.Path, ref.Path)
|
||||
return
|
||||
}
|
||||
|
||||
// Marshaling interface implementations.
|
||||
// Would like to implement MarshalText/UnmarshalText but that will change the JSON representation of URLs.
|
||||
|
||||
func (u *URL) MarshalBinary() (text []byte, err error) {
|
||||
return []byte(u.String()), nil
|
||||
}
|
||||
|
||||
func (u *URL) UnmarshalBinary(text []byte) error {
|
||||
var u1 URL
|
||||
err := u1.Parse(string(text))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
*u = u1
|
||||
return nil
|
||||
}
|
||||
|
||||
// validUserinfo reports whether s is a valid userinfo string per RFC 3986
|
||||
// Section 3.2.1:
|
||||
// userinfo = *( unreserved / pct-encoded / sub-delims / ":" )
|
||||
// unreserved = ALPHA / DIGIT / "-" / "." / "_" / "~"
|
||||
// sub-delims = "!" / "$" / "&" / "'" / "(" / ")"
|
||||
// / "*" / "+" / "," / ";" / "="
|
||||
//
|
||||
// It doesn't validate pct-encoded. The caller does that via func unescape.
|
||||
func validUserinfo(s string) bool {
|
||||
for _, r := range s {
|
||||
if 'A' <= r && r <= 'Z' {
|
||||
continue
|
||||
}
|
||||
if 'a' <= r && r <= 'z' {
|
||||
continue
|
||||
}
|
||||
if '0' <= r && r <= '9' {
|
||||
continue
|
||||
}
|
||||
switch r {
|
||||
case '-', '.', '_', ':', '~', '!', '$', '&', '\'',
|
||||
'(', ')', '*', '+', ',', ';', '=', '%', '@':
|
||||
continue
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
@ -1,897 +0,0 @@
|
||||
// Copyright 2009 The Go Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package fasturl
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
encodingPkg "encoding"
|
||||
"encoding/gob"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"reflect"
|
||||
"testing"
|
||||
)
|
||||
|
||||
type URLTest struct {
|
||||
in string
|
||||
out *URL // expected parse; RawPath="" means same as Path
|
||||
roundtrip string // expected result of reserializing the URL; empty means same as "in".
|
||||
}
|
||||
|
||||
var urltests = []URLTest{
|
||||
// no path
|
||||
{
|
||||
"http://www.google.com",
|
||||
&URL{
|
||||
Scheme: SchemeHTTP,
|
||||
Host: "www.google.com",
|
||||
},
|
||||
"",
|
||||
},
|
||||
// path
|
||||
{
|
||||
"http://www.google.com/",
|
||||
&URL{
|
||||
Scheme: SchemeHTTP,
|
||||
Host: "www.google.com",
|
||||
Path: "/",
|
||||
},
|
||||
"",
|
||||
},
|
||||
// %20 outside query
|
||||
{
|
||||
"http://www.google.com/a%20b",
|
||||
&URL{
|
||||
Scheme: SchemeHTTP,
|
||||
Host: "www.google.com",
|
||||
Path: "/a%20b",
|
||||
},
|
||||
"",
|
||||
},
|
||||
// leading // without scheme should create an authority
|
||||
{
|
||||
"//foo",
|
||||
&URL{
|
||||
Host: "foo",
|
||||
},
|
||||
"",
|
||||
},
|
||||
// Three leading slashes isn't an authority, but doesn't return an error.
|
||||
// (We can't return an error, as this code is also used via
|
||||
// ServeHTTP -> ReadRequest -> Parse, which is arguably a
|
||||
// different URL parsing context, but currently shares the
|
||||
// same codepath)
|
||||
{
|
||||
"///threeslashes",
|
||||
&URL{
|
||||
Path: "///threeslashes",
|
||||
},
|
||||
"",
|
||||
},
|
||||
// unescaped @ in username should not confuse host
|
||||
{
|
||||
"http://j@ne:password@google.com",
|
||||
&URL{
|
||||
Scheme: SchemeHTTP,
|
||||
Host: "google.com",
|
||||
},
|
||||
"http://google.com",
|
||||
},
|
||||
// unescaped @ in password should not confuse host
|
||||
{
|
||||
"http://jane:p@ssword@google.com",
|
||||
&URL{
|
||||
Scheme: SchemeHTTP,
|
||||
Host: "google.com",
|
||||
},
|
||||
"http://google.com",
|
||||
},
|
||||
// Relative path
|
||||
{
|
||||
"a/b/c",
|
||||
&URL{
|
||||
Path: "a/b/c",
|
||||
},
|
||||
"a/b/c",
|
||||
},
|
||||
// host subcomponent; IPv4 address in RFC 3986
|
||||
{
|
||||
"http://192.168.0.1/",
|
||||
&URL{
|
||||
Scheme: SchemeHTTP,
|
||||
Host: "192.168.0.1",
|
||||
Path: "/",
|
||||
},
|
||||
"",
|
||||
},
|
||||
// host and port subcomponents; IPv4 address in RFC 3986
|
||||
{
|
||||
"http://192.168.0.1:8080/",
|
||||
&URL{
|
||||
Scheme: SchemeHTTP,
|
||||
Host: "192.168.0.1:8080",
|
||||
Path: "/",
|
||||
},
|
||||
"",
|
||||
},
|
||||
// host subcomponent; IPv6 address in RFC 3986
|
||||
{
|
||||
"http://[fe80::1]/",
|
||||
&URL{
|
||||
Scheme: SchemeHTTP,
|
||||
Host: "[fe80::1]",
|
||||
Path: "/",
|
||||
},
|
||||
"",
|
||||
},
|
||||
// host and port subcomponents; IPv6 address in RFC 3986
|
||||
{
|
||||
"http://[fe80::1]:8080/",
|
||||
&URL{
|
||||
Scheme: SchemeHTTP,
|
||||
Host: "[fe80::1]:8080",
|
||||
Path: "/",
|
||||
},
|
||||
"",
|
||||
},
|
||||
// host subcomponent; IPv6 address with zone identifier in RFC 6874
|
||||
{
|
||||
"http://[fe80::1%25en0]/", // alphanum zone identifier
|
||||
&URL{
|
||||
Scheme: SchemeHTTP,
|
||||
Host: "[fe80::1%en0]",
|
||||
Path: "/",
|
||||
},
|
||||
"",
|
||||
},
|
||||
// host and port subcomponents; IPv6 address with zone identifier in RFC 6874
|
||||
{
|
||||
"http://[fe80::1%25en0]:8080/", // alphanum zone identifier
|
||||
&URL{
|
||||
Scheme: SchemeHTTP,
|
||||
Host: "[fe80::1%en0]:8080",
|
||||
Path: "/",
|
||||
},
|
||||
"",
|
||||
},
|
||||
// host subcomponent; IPv6 address with zone identifier in RFC 6874
|
||||
{
|
||||
"http://[fe80::1%25%65%6e%301-._~]/", // percent-encoded+unreserved zone identifier
|
||||
&URL{
|
||||
Scheme: SchemeHTTP,
|
||||
Host: "[fe80::1%en01-._~]",
|
||||
Path: "/",
|
||||
},
|
||||
"http://[fe80::1%25en01-._~]/",
|
||||
},
|
||||
// host and port subcomponents; IPv6 address with zone identifier in RFC 6874
|
||||
{
|
||||
"http://[fe80::1%25%65%6e%301-._~]:8080/", // percent-encoded+unreserved zone identifier
|
||||
&URL{
|
||||
Scheme: SchemeHTTP,
|
||||
Host: "[fe80::1%en01-._~]:8080",
|
||||
Path: "/",
|
||||
},
|
||||
"http://[fe80::1%25en01-._~]:8080/",
|
||||
},
|
||||
// golang.org/issue/12200 (colon with empty port)
|
||||
{
|
||||
"http://192.168.0.2:8080/foo",
|
||||
&URL{
|
||||
Scheme: SchemeHTTP,
|
||||
Host: "192.168.0.2:8080",
|
||||
Path: "/foo",
|
||||
},
|
||||
"",
|
||||
},
|
||||
{
|
||||
"http://192.168.0.2:/foo",
|
||||
&URL{
|
||||
Scheme: SchemeHTTP,
|
||||
Host: "192.168.0.2:",
|
||||
Path: "/foo",
|
||||
},
|
||||
"",
|
||||
},
|
||||
{
|
||||
// Malformed IPv6 but still accepted.
|
||||
"http://2b01:e34:ef40:7730:8e70:5aff:fefe:edac:8080/foo",
|
||||
&URL{
|
||||
Scheme: SchemeHTTP,
|
||||
Host: "2b01:e34:ef40:7730:8e70:5aff:fefe:edac:8080",
|
||||
Path: "/foo",
|
||||
},
|
||||
"",
|
||||
},
|
||||
{
|
||||
// Malformed IPv6 but still accepted.
|
||||
"http://2b01:e34:ef40:7730:8e70:5aff:fefe:edac:/foo",
|
||||
&URL{
|
||||
Scheme: SchemeHTTP,
|
||||
Host: "2b01:e34:ef40:7730:8e70:5aff:fefe:edac:",
|
||||
Path: "/foo",
|
||||
},
|
||||
"",
|
||||
},
|
||||
{
|
||||
"http://[2b01:e34:ef40:7730:8e70:5aff:fefe:edac]:8080/foo",
|
||||
&URL{
|
||||
Scheme: SchemeHTTP,
|
||||
Host: "[2b01:e34:ef40:7730:8e70:5aff:fefe:edac]:8080",
|
||||
Path: "/foo",
|
||||
},
|
||||
"",
|
||||
},
|
||||
{
|
||||
"http://[2b01:e34:ef40:7730:8e70:5aff:fefe:edac]:/foo",
|
||||
&URL{
|
||||
Scheme: SchemeHTTP,
|
||||
Host: "[2b01:e34:ef40:7730:8e70:5aff:fefe:edac]:",
|
||||
Path: "/foo",
|
||||
},
|
||||
"",
|
||||
},
|
||||
// golang.org/issue/7991 and golang.org/issue/12719 (non-ascii %-encoded in host)
|
||||
{
|
||||
"http://hello.世界.com/foo",
|
||||
&URL{
|
||||
Scheme: SchemeHTTP,
|
||||
Host: "hello.世界.com",
|
||||
Path: "/foo",
|
||||
},
|
||||
"http://hello.%E4%B8%96%E7%95%8C.com/foo",
|
||||
},
|
||||
{
|
||||
"http://hello.%e4%b8%96%e7%95%8c.com/foo",
|
||||
&URL{
|
||||
Scheme: SchemeHTTP,
|
||||
Host: "hello.世界.com",
|
||||
Path: "/foo",
|
||||
},
|
||||
"http://hello.%E4%B8%96%E7%95%8C.com/foo",
|
||||
},
|
||||
{
|
||||
"http://hello.%E4%B8%96%E7%95%8C.com/foo",
|
||||
&URL{
|
||||
Scheme: SchemeHTTP,
|
||||
Host: "hello.世界.com",
|
||||
Path: "/foo",
|
||||
},
|
||||
"",
|
||||
},
|
||||
// golang.org/issue/10433 (path beginning with //)
|
||||
{
|
||||
"http://example.com//foo",
|
||||
&URL{
|
||||
Scheme: SchemeHTTP,
|
||||
Host: "example.com",
|
||||
Path: "//foo",
|
||||
},
|
||||
"",
|
||||
},
|
||||
// test that we can reparse the host names we accept.
|
||||
{
|
||||
"http://authority<\"hi\">/foo",
|
||||
&URL{
|
||||
Scheme: SchemeHTTP,
|
||||
Host: "authority<\"hi\">",
|
||||
Path: "/foo",
|
||||
},
|
||||
"",
|
||||
},
|
||||
}
|
||||
|
||||
// more useful string for debugging than fmt's struct printer
|
||||
func ufmt(u *URL) string {
|
||||
return fmt.Sprintf("scheme=%q, host=%q, path=%q",
|
||||
Schemes[u.Scheme], u.Host, u.Path)
|
||||
}
|
||||
|
||||
func BenchmarkString(b *testing.B) {
|
||||
b.StopTimer()
|
||||
b.ReportAllocs()
|
||||
for _, tt := range urltests {
|
||||
var u URL
|
||||
err := u.Parse(tt.in)
|
||||
if err != nil {
|
||||
b.Errorf("Parse(%q) returned error %s", tt.in, err)
|
||||
continue
|
||||
}
|
||||
if tt.roundtrip == "" {
|
||||
continue
|
||||
}
|
||||
b.StartTimer()
|
||||
var g string
|
||||
for i := 0; i < b.N; i++ {
|
||||
g = u.String()
|
||||
}
|
||||
b.StopTimer()
|
||||
if w := tt.roundtrip; b.N > 0 && g != w {
|
||||
b.Errorf("Parse(%q).String() == %q, want %q", tt.in, g, w)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestParse(t *testing.T) {
|
||||
for _, tt := range urltests {
|
||||
var u URL
|
||||
err := u.Parse(tt.in)
|
||||
if err != nil {
|
||||
t.Errorf("Parse(%q) returned error %v", tt.in, err)
|
||||
continue
|
||||
}
|
||||
if !reflect.DeepEqual(&u, tt.out) {
|
||||
t.Errorf("Parse(%q):\n\tgot %v\n\twant %v\n", tt.in, ufmt(&u), ufmt(tt.out))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const pathThatLooksSchemeRelative = "//not.a.user@not.a.host/just/a/path"
|
||||
|
||||
var parseRequestURLTests = []struct {
|
||||
url string
|
||||
expectedValid bool
|
||||
}{
|
||||
{"http://foo.com", true},
|
||||
{"http://foo.com/", true},
|
||||
{"http://foo.com/path", true},
|
||||
{"/", true},
|
||||
{pathThatLooksSchemeRelative, true},
|
||||
{"//not.a.user@%66%6f%6f.com/just/a/path/also", true},
|
||||
{"*", true},
|
||||
{"http://192.168.0.1/", true},
|
||||
{"http://192.168.0.1:8080/", true},
|
||||
{"http://[fe80::1]/", true},
|
||||
{"http://[fe80::1]:8080/", true},
|
||||
|
||||
// Tests exercising RFC 6874 compliance:
|
||||
{"http://[fe80::1%25en0]/", true}, // with alphanum zone identifier
|
||||
{"http://[fe80::1%25en0]:8080/", true}, // with alphanum zone identifier
|
||||
{"http://[fe80::1%25%65%6e%301-._~]/", true}, // with percent-encoded+unreserved zone identifier
|
||||
{"http://[fe80::1%25%65%6e%301-._~]:8080/", true}, // with percent-encoded+unreserved zone identifier
|
||||
|
||||
{"foo.html", false},
|
||||
{"../dir/", false},
|
||||
{"http://192.168.0.%31/", false},
|
||||
{"http://192.168.0.%31:8080/", false},
|
||||
{"http://[fe80::%31]/", false},
|
||||
{"http://[fe80::%31]:8080/", false},
|
||||
{"http://[fe80::%31%25en0]/", false},
|
||||
{"http://[fe80::%31%25en0]:8080/", false},
|
||||
|
||||
// These two cases are valid as textual representations as
|
||||
// described in RFC 4007, but are not valid as address
|
||||
// literals with IPv6 zone identifiers in URIs as described in
|
||||
// RFC 6874.
|
||||
{"http://[fe80::1%en0]/", false},
|
||||
{"http://[fe80::1%en0]:8080/", false},
|
||||
}
|
||||
|
||||
func TestParseRequestURI(t *testing.T) {
|
||||
for _, test := range parseRequestURLTests {
|
||||
var u URL
|
||||
err := u.ParseRequestURI(test.url)
|
||||
if test.expectedValid && err != nil {
|
||||
t.Errorf("ParseRequestURI(%q) gave err %v; want no error", test.url, err)
|
||||
} else if !test.expectedValid && err == nil {
|
||||
t.Errorf("ParseRequestURI(%q) gave nil error; want some error", test.url)
|
||||
}
|
||||
}
|
||||
|
||||
var url URL
|
||||
err := url.ParseRequestURI(pathThatLooksSchemeRelative)
|
||||
if err != nil {
|
||||
t.Fatalf("Unexpected error %v", err)
|
||||
}
|
||||
if url.Path != pathThatLooksSchemeRelative {
|
||||
t.Errorf("ParseRequestURI path:\ngot %q\nwant %q", url.Path, pathThatLooksSchemeRelative)
|
||||
}
|
||||
}
|
||||
|
||||
var stringURLTests = []struct {
|
||||
url URL
|
||||
want string
|
||||
}{
|
||||
// No leading slash on path should prepend slash on String() call
|
||||
{
|
||||
url: URL{
|
||||
Scheme: SchemeHTTP,
|
||||
Host: "www.google.com",
|
||||
Path: "search",
|
||||
},
|
||||
want: "http://www.google.com/search",
|
||||
},
|
||||
// Relative path with first element containing ":" should be prepended with "./", golang.org/issue/17184
|
||||
{
|
||||
url: URL{
|
||||
Path: "this:that",
|
||||
},
|
||||
want: "./this:that",
|
||||
},
|
||||
// Relative path with second element containing ":" should not be prepended with "./"
|
||||
{
|
||||
url: URL{
|
||||
Path: "here/this:that",
|
||||
},
|
||||
want: "here/this:that",
|
||||
},
|
||||
// Non-relative path with first element containing ":" should not be prepended with "./"
|
||||
{
|
||||
url: URL{
|
||||
Scheme: SchemeHTTP,
|
||||
Host: "www.google.com",
|
||||
Path: "this:that",
|
||||
},
|
||||
want: "http://www.google.com/this:that",
|
||||
},
|
||||
}
|
||||
|
||||
func TestURLString(t *testing.T) {
|
||||
for _, tt := range urltests {
|
||||
var u URL
|
||||
err := u.Parse(tt.in)
|
||||
if err != nil {
|
||||
t.Errorf("Parse(%q) returned error %s", tt.in, err)
|
||||
continue
|
||||
}
|
||||
expected := tt.in
|
||||
if tt.roundtrip != "" {
|
||||
expected = tt.roundtrip
|
||||
}
|
||||
s := u.String()
|
||||
if s != expected {
|
||||
t.Errorf("Parse(%q).String() == %q (expected %q)", tt.in, s, expected)
|
||||
}
|
||||
}
|
||||
|
||||
for _, tt := range stringURLTests {
|
||||
if got := tt.url.String(); got != tt.want {
|
||||
t.Errorf("%+v.String() = %q; want %q", tt.url, got, tt.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var resolvePathTests = []struct {
|
||||
base, ref, expected string
|
||||
}{
|
||||
{"a/b", ".", "/a/"},
|
||||
{"a/b", "c", "/a/c"},
|
||||
{"a/b", "..", "/"},
|
||||
{"a/", "..", "/"},
|
||||
{"a/", "../..", "/"},
|
||||
{"a/b/c", "..", "/a/"},
|
||||
{"a/b/c", "../d", "/a/d"},
|
||||
{"a/b/c", ".././d", "/a/d"},
|
||||
{"a/b", "./..", "/"},
|
||||
{"a/./b", ".", "/a/"},
|
||||
{"a/../", ".", "/"},
|
||||
{"a/.././b", "c", "/c"},
|
||||
}
|
||||
|
||||
func TestResolvePath(t *testing.T) {
|
||||
for _, test := range resolvePathTests {
|
||||
got := resolvePath(test.base, test.ref)
|
||||
if got != test.expected {
|
||||
t.Errorf("For %q + %q got %q; expected %q", test.base, test.ref, got, test.expected)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var resolveReferenceTests = []struct {
|
||||
base, rel, expected string
|
||||
}{
|
||||
// Absolute URL references
|
||||
{"http://foo.com?a=b", "https://bar.com/", "https://bar.com/"},
|
||||
{"http://foo.com/", "https://bar.com/?a=b", "https://bar.com/"},
|
||||
{"http://foo.com/", "https://bar.com/?", "https://bar.com/"},
|
||||
|
||||
// Path-absolute references
|
||||
{"http://foo.com/bar", "/baz", "http://foo.com/baz"},
|
||||
{"http://foo.com/bar?a=b#f", "/baz", "http://foo.com/baz"},
|
||||
{"http://foo.com/bar?a=b", "/baz?", "http://foo.com/baz"},
|
||||
{"http://foo.com/bar?a=b", "/baz?c=d", "http://foo.com/baz"},
|
||||
|
||||
// Multiple slashes
|
||||
{"http://foo.com/bar", "http://foo.com//baz", "http://foo.com//baz"},
|
||||
{"http://foo.com/bar", "http://foo.com///baz/quux", "http://foo.com///baz/quux"},
|
||||
|
||||
// Scheme-relative
|
||||
{"https://foo.com/bar?a=b", "//bar.com/quux", "https://bar.com/quux"},
|
||||
|
||||
// Path-relative references:
|
||||
|
||||
// ... current directory
|
||||
{"http://foo.com", ".", "http://foo.com/"},
|
||||
{"http://foo.com/bar", ".", "http://foo.com/"},
|
||||
{"http://foo.com/bar/", ".", "http://foo.com/bar/"},
|
||||
|
||||
// ... going down
|
||||
{"http://foo.com", "bar", "http://foo.com/bar"},
|
||||
{"http://foo.com/", "bar", "http://foo.com/bar"},
|
||||
{"http://foo.com/bar/baz", "quux", "http://foo.com/bar/quux"},
|
||||
|
||||
// ... going up
|
||||
{"http://foo.com/bar/baz", "../quux", "http://foo.com/quux"},
|
||||
{"http://foo.com/bar/baz", "../../../../../quux", "http://foo.com/quux"},
|
||||
{"http://foo.com/bar", "..", "http://foo.com/"},
|
||||
{"http://foo.com/bar/baz", "./..", "http://foo.com/"},
|
||||
// ".." in the middle (issue 3560)
|
||||
{"http://foo.com/bar/baz", "quux/dotdot/../tail", "http://foo.com/bar/quux/tail"},
|
||||
{"http://foo.com/bar/baz", "quux/./dotdot/../tail", "http://foo.com/bar/quux/tail"},
|
||||
{"http://foo.com/bar/baz", "quux/./dotdot/.././tail", "http://foo.com/bar/quux/tail"},
|
||||
{"http://foo.com/bar/baz", "quux/./dotdot/./../tail", "http://foo.com/bar/quux/tail"},
|
||||
{"http://foo.com/bar/baz", "quux/./dotdot/dotdot/././../../tail", "http://foo.com/bar/quux/tail"},
|
||||
{"http://foo.com/bar/baz", "quux/./dotdot/dotdot/./.././../tail", "http://foo.com/bar/quux/tail"},
|
||||
{"http://foo.com/bar/baz", "quux/./dotdot/dotdot/dotdot/./../../.././././tail", "http://foo.com/bar/quux/tail"},
|
||||
{"http://foo.com/bar/baz", "quux/./dotdot/../dotdot/../dot/./tail/..", "http://foo.com/bar/quux/dot/"},
|
||||
|
||||
// Remove any dot-segments prior to forming the target URI.
|
||||
// http://tools.ietf.org/html/rfc3986#section-5.2.4
|
||||
{"http://foo.com/dot/./dotdot/../foo/bar", "../baz", "http://foo.com/dot/baz"},
|
||||
|
||||
// Triple dot isn't special
|
||||
{"http://foo.com/bar", "...", "http://foo.com/..."},
|
||||
|
||||
// Fragment
|
||||
{"http://foo.com/bar", ".#frag", "http://foo.com/"},
|
||||
{"http://example.org/", "#!$&%27()*+,;=", "http://example.org/"},
|
||||
|
||||
// Paths with escaping (issue 16947).
|
||||
{"http://foo.com/foo%2fbar/", "../baz", "http://foo.com/baz"},
|
||||
{"http://foo.com/1/2%2f/3%2f4/5", "../../a/b/c", "http://foo.com/1/a/b/c"},
|
||||
{"http://foo.com/1/2/3", "./a%2f../../b/..%2fc", "http://foo.com/1/2/b/..%2fc"},
|
||||
{"http://foo.com/1/2%2f/3%2f4/5", "./a%2f../b/../c", "http://foo.com/1/2%2f/3%2f4/a%2f../c"},
|
||||
{"http://foo.com/foo%20bar/", "../baz", "http://foo.com/baz"},
|
||||
{"http://foo.com/foo", "../bar%2fbaz", "http://foo.com/bar%2fbaz"},
|
||||
{"http://foo.com/foo%2dbar/", "./baz-quux", "http://foo.com/foo%2dbar/baz-quux"},
|
||||
|
||||
// RFC 3986: Normal Examples
|
||||
// http://tools.ietf.org/html/rfc3986#section-5.4.1
|
||||
{"http://a/b/c/d;p?q", "g", "http://a/b/c/g"},
|
||||
{"http://a/b/c/d;p?q", "./g", "http://a/b/c/g"},
|
||||
{"http://a/b/c/d;p?q", "g/", "http://a/b/c/g/"},
|
||||
{"http://a/b/c/d;p?q", "/g", "http://a/g"},
|
||||
{"http://a/b/c/d;p?q", "//g", "http://g"},
|
||||
{"http://a/b/c/d;p?q", "?y", "http://a/b/c/d;p"},
|
||||
{"http://a/b/c/d;p?q", "g?y", "http://a/b/c/g"},
|
||||
{"http://a/b/c/d;p?q", "#s", "http://a/b/c/d;p"},
|
||||
{"http://a/b/c/d;p?q", "g#s", "http://a/b/c/g"},
|
||||
{"http://a/b/c/d;p?q", "g?y#s", "http://a/b/c/g"},
|
||||
{"http://a/b/c/d;p?q", ";x", "http://a/b/c/;x"},
|
||||
{"http://a/b/c/d;p?q", "g;x", "http://a/b/c/g;x"},
|
||||
{"http://a/b/c/d;p?q", "g;x?y#s", "http://a/b/c/g;x"},
|
||||
{"http://a/b/c/d;p?q", "", "http://a/b/c/d;p"},
|
||||
{"http://a/b/c/d;p?q", ".", "http://a/b/c/"},
|
||||
{"http://a/b/c/d;p?q", "./", "http://a/b/c/"},
|
||||
{"http://a/b/c/d;p?q", "..", "http://a/b/"},
|
||||
{"http://a/b/c/d;p?q", "../", "http://a/b/"},
|
||||
{"http://a/b/c/d;p?q", "../g", "http://a/b/g"},
|
||||
{"http://a/b/c/d;p?q", "../..", "http://a/"},
|
||||
{"http://a/b/c/d;p?q", "../../", "http://a/"},
|
||||
{"http://a/b/c/d;p?q", "../../g", "http://a/g"},
|
||||
|
||||
// RFC 3986: Abnormal Examples
|
||||
// http://tools.ietf.org/html/rfc3986#section-5.4.2
|
||||
{"http://a/b/c/d;p?q", "../../../g", "http://a/g"},
|
||||
{"http://a/b/c/d;p?q", "../../../../g", "http://a/g"},
|
||||
{"http://a/b/c/d;p?q", "/./g", "http://a/g"},
|
||||
{"http://a/b/c/d;p?q", "/../g", "http://a/g"},
|
||||
{"http://a/b/c/d;p?q", "g.", "http://a/b/c/g."},
|
||||
{"http://a/b/c/d;p?q", ".g", "http://a/b/c/.g"},
|
||||
{"http://a/b/c/d;p?q", "g..", "http://a/b/c/g.."},
|
||||
{"http://a/b/c/d;p?q", "..g", "http://a/b/c/..g"},
|
||||
{"http://a/b/c/d;p?q", "./../g", "http://a/b/g"},
|
||||
{"http://a/b/c/d;p?q", "./g/.", "http://a/b/c/g/"},
|
||||
{"http://a/b/c/d;p?q", "g/./h", "http://a/b/c/g/h"},
|
||||
{"http://a/b/c/d;p?q", "g/../h", "http://a/b/c/h"},
|
||||
{"http://a/b/c/d;p?q", "g;x=1/./y", "http://a/b/c/g;x=1/y"},
|
||||
{"http://a/b/c/d;p?q", "g;x=1/../y", "http://a/b/c/y"},
|
||||
{"http://a/b/c/d;p?q", "g?y/./x", "http://a/b/c/g"},
|
||||
{"http://a/b/c/d;p?q", "g?y/../x", "http://a/b/c/g"},
|
||||
{"http://a/b/c/d;p?q", "g#s/./x", "http://a/b/c/g"},
|
||||
{"http://a/b/c/d;p?q", "g#s/../x", "http://a/b/c/g"},
|
||||
|
||||
// Extras.
|
||||
{"https://a/b/c/d;p?q", "//g?q", "https://g"},
|
||||
{"https://a/b/c/d;p?q", "//g#s", "https://g"},
|
||||
{"https://a/b/c/d;p?q", "//g/d/e/f?y#s", "https://g/d/e/f"},
|
||||
{"https://a/b/c/d;p#s", "?y", "https://a/b/c/d;p"},
|
||||
{"https://a/b/c/d;p?q#s", "?y", "https://a/b/c/d;p"},
|
||||
}
|
||||
|
||||
func TestResolveReference(t *testing.T) {
|
||||
mustParse := func(url string) *URL {
|
||||
u := new(URL)
|
||||
err := u.Parse(url)
|
||||
if err != nil {
|
||||
t.Fatalf("Parse(%q) got err %v", url, err)
|
||||
}
|
||||
return u
|
||||
}
|
||||
for _, test := range resolveReferenceTests {
|
||||
base := mustParse(test.base)
|
||||
rel := mustParse(test.rel)
|
||||
var url URL
|
||||
base.ResolveReference(&url, rel)
|
||||
if got := url.String(); got != test.expected {
|
||||
t.Errorf("URL(%q).ResolveReference(%q)\ngot %q\nwant %q", test.base, test.rel, got, test.expected)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type RequestURITest struct {
|
||||
url *URL
|
||||
out string
|
||||
}
|
||||
|
||||
var requritests = []RequestURITest{
|
||||
{
|
||||
&URL{
|
||||
Scheme: SchemeHTTP,
|
||||
Host: "example.com",
|
||||
Path: "",
|
||||
},
|
||||
"/",
|
||||
},
|
||||
{
|
||||
&URL{
|
||||
Scheme: SchemeHTTP,
|
||||
Host: "example.com",
|
||||
Path: "/a b",
|
||||
},
|
||||
"/a%20b",
|
||||
},
|
||||
{
|
||||
&URL{
|
||||
Scheme: SchemeHTTP,
|
||||
Host: "example.com",
|
||||
Path: "//foo",
|
||||
},
|
||||
"//foo",
|
||||
},
|
||||
}
|
||||
|
||||
func TestParseErrors(t *testing.T) {
|
||||
tests := []struct {
|
||||
in string
|
||||
wantErr bool
|
||||
}{
|
||||
{"http://[::1]", false},
|
||||
{"http://[::1]:80", false},
|
||||
{"http://[::1]:namedport", true}, // rfc3986 3.2.3
|
||||
{"http://[::1]/", false},
|
||||
{"http://[::1]a", true},
|
||||
{"http://[::1]%23", true},
|
||||
{"http://[::1%25en0]", false}, // valid zone id
|
||||
{"http://[::1]:", false}, // colon, but no port OK
|
||||
{"http://[::1]:%38%30", true}, // not allowed: % encoding only for non-ASCII
|
||||
{"http://[::1%25%41]", false}, // RFC 6874 allows over-escaping in zone
|
||||
{"http://[%10::1]", true}, // no %xx escapes in IP address
|
||||
{"http://[::1]/%48", false}, // %xx in path is fine
|
||||
{"http://%41:8080/", true}, // not allowed: % encoding only for non-ASCII
|
||||
|
||||
{"http://[]%20%48%54%54%50%2f%31%2e%31%0a%4d%79%48%65%61%64%65%72%3a%20%31%32%33%0a%0a/", true}, // golang.org/issue/11208
|
||||
{"http://a b.com/", true}, // no space in host name please
|
||||
}
|
||||
for _, tt := range tests {
|
||||
var u URL
|
||||
err := u.Parse(tt.in)
|
||||
if tt.wantErr {
|
||||
if err == nil {
|
||||
t.Errorf("Parse(%q) = %#v; want an error", tt.in, u)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
t.Logf("Parse(%q) = %v; want no error", tt.in, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type shouldEscapeTest struct {
|
||||
in byte
|
||||
mode encoding
|
||||
escape bool
|
||||
}
|
||||
|
||||
var shouldEscapeTests = []shouldEscapeTest{
|
||||
// Unreserved characters (§2.3)
|
||||
{'a', encodePath, false},
|
||||
{'a', encodeUserPassword, false},
|
||||
{'a', encodeQueryComponent, false},
|
||||
{'a', encodeFragment, false},
|
||||
{'a', encodeHost, false},
|
||||
{'z', encodePath, false},
|
||||
{'A', encodePath, false},
|
||||
{'Z', encodePath, false},
|
||||
{'0', encodePath, false},
|
||||
{'9', encodePath, false},
|
||||
{'-', encodePath, false},
|
||||
{'-', encodeUserPassword, false},
|
||||
{'-', encodeQueryComponent, false},
|
||||
{'-', encodeFragment, false},
|
||||
{'.', encodePath, false},
|
||||
{'_', encodePath, false},
|
||||
{'~', encodePath, false},
|
||||
|
||||
// User information (§3.2.1)
|
||||
{':', encodeUserPassword, true},
|
||||
{'/', encodeUserPassword, true},
|
||||
{'?', encodeUserPassword, true},
|
||||
{'@', encodeUserPassword, true},
|
||||
{'$', encodeUserPassword, false},
|
||||
{'&', encodeUserPassword, false},
|
||||
{'+', encodeUserPassword, false},
|
||||
{',', encodeUserPassword, false},
|
||||
{';', encodeUserPassword, false},
|
||||
{'=', encodeUserPassword, false},
|
||||
|
||||
// Host (IP address, IPv6 address, registered name, port suffix; §3.2.2)
|
||||
{'!', encodeHost, false},
|
||||
{'$', encodeHost, false},
|
||||
{'&', encodeHost, false},
|
||||
{'\'', encodeHost, false},
|
||||
{'(', encodeHost, false},
|
||||
{')', encodeHost, false},
|
||||
{'*', encodeHost, false},
|
||||
{'+', encodeHost, false},
|
||||
{',', encodeHost, false},
|
||||
{';', encodeHost, false},
|
||||
{'=', encodeHost, false},
|
||||
{':', encodeHost, false},
|
||||
{'[', encodeHost, false},
|
||||
{']', encodeHost, false},
|
||||
{'0', encodeHost, false},
|
||||
{'9', encodeHost, false},
|
||||
{'A', encodeHost, false},
|
||||
{'z', encodeHost, false},
|
||||
{'_', encodeHost, false},
|
||||
{'-', encodeHost, false},
|
||||
{'.', encodeHost, false},
|
||||
}
|
||||
|
||||
func TestShouldEscape(t *testing.T) {
|
||||
for _, tt := range shouldEscapeTests {
|
||||
if shouldEscape(tt.in, tt.mode) != tt.escape {
|
||||
t.Errorf("shouldEscape(%q, %v) returned %v; expected %v", tt.in, tt.mode, !tt.escape, tt.escape)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type timeoutError struct {
|
||||
timeout bool
|
||||
}
|
||||
|
||||
func (e *timeoutError) Error() string { return "timeout error" }
|
||||
func (e *timeoutError) Timeout() bool { return e.timeout }
|
||||
|
||||
type temporaryError struct {
|
||||
temporary bool
|
||||
}
|
||||
|
||||
func (e *temporaryError) Error() string { return "temporary error" }
|
||||
func (e *temporaryError) Temporary() bool { return e.temporary }
|
||||
|
||||
type timeoutTemporaryError struct {
|
||||
timeoutError
|
||||
temporaryError
|
||||
}
|
||||
|
||||
func (e *timeoutTemporaryError) Error() string { return "timeout/temporary error" }
|
||||
|
||||
var netErrorTests = []struct {
|
||||
err error
|
||||
timeout bool
|
||||
temporary bool
|
||||
}{{
|
||||
err: &Error{"Get", "http://google.com/", &timeoutError{timeout: true}},
|
||||
timeout: true,
|
||||
temporary: false,
|
||||
}, {
|
||||
err: &Error{"Get", "http://google.com/", &timeoutError{timeout: false}},
|
||||
timeout: false,
|
||||
temporary: false,
|
||||
}, {
|
||||
err: &Error{"Get", "http://google.com/", &temporaryError{temporary: true}},
|
||||
timeout: false,
|
||||
temporary: true,
|
||||
}, {
|
||||
err: &Error{"Get", "http://google.com/", &temporaryError{temporary: false}},
|
||||
timeout: false,
|
||||
temporary: false,
|
||||
}, {
|
||||
err: &Error{"Get", "http://google.com/", &timeoutTemporaryError{timeoutError{timeout: true}, temporaryError{temporary: true}}},
|
||||
timeout: true,
|
||||
temporary: true,
|
||||
}, {
|
||||
err: &Error{"Get", "http://google.com/", &timeoutTemporaryError{timeoutError{timeout: false}, temporaryError{temporary: true}}},
|
||||
timeout: false,
|
||||
temporary: true,
|
||||
}, {
|
||||
err: &Error{"Get", "http://google.com/", &timeoutTemporaryError{timeoutError{timeout: true}, temporaryError{temporary: false}}},
|
||||
timeout: true,
|
||||
temporary: false,
|
||||
}, {
|
||||
err: &Error{"Get", "http://google.com/", &timeoutTemporaryError{timeoutError{timeout: false}, temporaryError{temporary: false}}},
|
||||
timeout: false,
|
||||
temporary: false,
|
||||
}, {
|
||||
err: &Error{"Get", "http://google.com/", io.EOF},
|
||||
timeout: false,
|
||||
temporary: false,
|
||||
}}
|
||||
|
||||
// Test that url.Error implements net.Error and that it forwards
|
||||
func TestURLErrorImplementsNetError(t *testing.T) {
|
||||
for i, tt := range netErrorTests {
|
||||
err, ok := tt.err.(net.Error)
|
||||
if !ok {
|
||||
t.Errorf("%d: %T does not implement net.Error", i+1, tt.err)
|
||||
continue
|
||||
}
|
||||
if err.Timeout() != tt.timeout {
|
||||
t.Errorf("%d: err.Timeout(): got %v, want %v", i+1, err.Timeout(), tt.timeout)
|
||||
continue
|
||||
}
|
||||
if err.Temporary() != tt.temporary {
|
||||
t.Errorf("%d: err.Temporary(): got %v, want %v", i+1, err.Temporary(), tt.temporary)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var _ encodingPkg.BinaryMarshaler = (*URL)(nil)
|
||||
var _ encodingPkg.BinaryUnmarshaler = (*URL)(nil)
|
||||
|
||||
func TestJSON(t *testing.T) {
|
||||
var u URL
|
||||
err := u.Parse("https://www.google.com/x?y=z")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
js, err := json.Marshal(&u)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// If only we could implement TextMarshaler/TextUnmarshaler,
|
||||
// this would work:
|
||||
//
|
||||
// if string(js) != strconv.Quote(u.String()) {
|
||||
// t.Errorf("json encoding: %s\nwant: %s\n", js, strconv.Quote(u.String()))
|
||||
// }
|
||||
|
||||
u1 := new(URL)
|
||||
err = json.Unmarshal(js, u1)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if u1.String() != u.String() {
|
||||
t.Errorf("json decoded to: %s\nwant: %s\n", u1, &u)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGob(t *testing.T) {
|
||||
var u URL
|
||||
err := u.Parse("https://www.google.com/x?y=z")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var w bytes.Buffer
|
||||
err = gob.NewEncoder(&w).Encode(&u)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
u1 := new(URL)
|
||||
err = gob.NewDecoder(&w).Decode(u1)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if u1.String() != u.String() {
|
||||
t.Errorf("json decoded to: %s\nwant: %s\n", u1, &u)
|
||||
}
|
||||
}
|
||||
12
main.go
12
main.go
@ -3,8 +3,8 @@ package main
|
||||
import (
|
||||
"context"
|
||||
"github.com/sirupsen/logrus"
|
||||
"github.com/terorie/oddb-go/fasturl"
|
||||
"github.com/urfave/cli"
|
||||
"github.com/valyala/fasthttp"
|
||||
"log"
|
||||
"net/http"
|
||||
_ "net/http/pprof"
|
||||
@ -55,12 +55,12 @@ func cmdCrawler(clic *cli.Context) error {
|
||||
if !strings.Contains(arg, "://") {
|
||||
arg = "http://" + arg
|
||||
}
|
||||
var u fasturl.URL
|
||||
err := u.Parse(arg)
|
||||
if !strings.HasSuffix(u.Path, "/") {
|
||||
u.Path += "/"
|
||||
var u fasthttp.URI
|
||||
u.Parse(nil, []byte(arg))
|
||||
uPath := string(u.Path())
|
||||
if !strings.HasSuffix(uPath, "/") {
|
||||
u.SetPath(uPath + "/")
|
||||
}
|
||||
if err != nil { return err }
|
||||
remotes[i] = &OD {
|
||||
Task: &Task{
|
||||
WebsiteId: 0,
|
||||
|
||||
6
model.go
6
model.go
@ -2,14 +2,14 @@ package main
|
||||
|
||||
import (
|
||||
"github.com/terorie/oddb-go/ds/redblackhash"
|
||||
"github.com/terorie/oddb-go/fasturl"
|
||||
"github.com/valyala/fasthttp"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
type Job struct {
|
||||
OD *OD
|
||||
Uri fasturl.URL
|
||||
Uri fasthttp.URI
|
||||
UriStr string
|
||||
Fails int
|
||||
LastError error
|
||||
@ -18,7 +18,7 @@ type Job struct {
|
||||
type OD struct {
|
||||
Task *Task
|
||||
Wait sync.WaitGroup
|
||||
BaseUri fasturl.URL
|
||||
BaseUri fasthttp.URI
|
||||
WCtx WorkerContext
|
||||
Scanned redblackhash.Tree
|
||||
|
||||
|
||||
@ -1,10 +1,10 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"github.com/sirupsen/logrus"
|
||||
"math"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
@ -70,8 +70,9 @@ func (w WorkerContext) step(results chan<- File, job Job) {
|
||||
}
|
||||
|
||||
func DoJob(job *Job, f *File) (newJobs []Job, err error) {
|
||||
if len(job.Uri.Path) == 0 { return }
|
||||
if job.Uri.Path[len(job.Uri.Path)-1] == '/' {
|
||||
uriPath := job.Uri.Path()
|
||||
if len(uriPath) == 0 { return }
|
||||
if uriPath[len(uriPath)-1] == '/' {
|
||||
// Load directory
|
||||
links, err := GetDir(job, f)
|
||||
if err != nil {
|
||||
@ -91,7 +92,7 @@ func DoJob(job *Job, f *File) (newJobs []Job, err error) {
|
||||
|
||||
// Sort by path
|
||||
sort.Slice(links, func(i, j int) bool {
|
||||
return strings.Compare(links[i].Path, links[j].Path) < 0
|
||||
return bytes.Compare(links[i].Path(), links[j].Path()) < 0
|
||||
})
|
||||
|
||||
var newJobCount int
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user