// Copyright 2009 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. // Package fasturl parses URLs and implements query escaping. package fasturl // Modifications by terorie // See RFC 3986. This package generally follows RFC 3986, except where // it deviates for compatibility reasons. When sending changes, first // search old issues for history on decisions. Unit tests should also // contain references to issue numbers with details. import ( "errors" "fmt" "strconv" "strings" ) type Scheme int const ( SchemeInvalid = iota SchemeHTTP SchemeHTTPS SchemeCount ) var Schemes = [SchemeCount]string { "", "http", "https", } var ErrUnknownScheme = errors.New("unknown protocol scheme") // Error reports an error and the operation and URL that caused it. type Error struct { Op string URL string Err error } func (e *Error) Error() string { return e.Op + " " + e.URL + ": " + e.Err.Error() } type timeout interface { Timeout() bool } func (e *Error) Timeout() bool { t, ok := e.Err.(timeout) return ok && t.Timeout() } type temporary interface { Temporary() bool } func (e *Error) Temporary() bool { t, ok := e.Err.(temporary) return ok && t.Temporary() } func ishex(c byte) bool { switch { case '0' <= c && c <= '9': return true case 'a' <= c && c <= 'f': return true case 'A' <= c && c <= 'F': return true } return false } func unhex(c byte) byte { switch { case '0' <= c && c <= '9': return c - '0' case 'a' <= c && c <= 'f': return c - 'a' + 10 case 'A' <= c && c <= 'F': return c - 'A' + 10 } return 0 } type encoding int const ( encodePath encoding = 1 + iota encodePathSegment encodeHost encodeZone encodeUserPassword encodeQueryComponent encodeFragment ) type EscapeError string func (e EscapeError) Error() string { return "invalid URL escape " + strconv.Quote(string(e)) } type InvalidHostError string func (e InvalidHostError) Error() string { return "invalid character " + strconv.Quote(string(e)) + " in host name" } // Return true if the specified character should be escaped when // appearing in a URL string, according to RFC 3986. // // Please be informed that for now shouldEscape does not check all // reserved characters correctly. See golang.org/issue/5684. func shouldEscape(c byte, mode encoding) bool { // §2.3 Unreserved characters (alphanum) if 'A' <= c && c <= 'Z' || 'a' <= c && c <= 'z' || '0' <= c && c <= '9' { return false } if mode == encodeHost || mode == encodeZone { // §3.2.2 Host allows // sub-delims = "!" / "$" / "&" / "'" / "(" / ")" / "*" / "+" / "," / ";" / "=" // as part of reg-name. // We add : because we include :port as part of host. // We add [ ] because we include [ipv6]:port as part of host. // We add < > because they're the only characters left that // we could possibly allow, and Parse will reject them if we // escape them (because hosts can't use %-encoding for // ASCII bytes). switch c { case '!', '$', '&', '\'', '(', ')', '*', '+', ',', ';', '=', ':', '[', ']', '<', '>', '"': return false } } switch c { case '-', '_', '.', '~': // §2.3 Unreserved characters (mark) return false case '$', '&', '+', ',', '/', ':', ';', '=', '?', '@': // §2.2 Reserved characters (reserved) // Different sections of the URL allow a few of // the reserved characters to appear unescaped. switch mode { case encodePath: // §3.3 // The RFC allows : @ & = + $ but saves / ; , for assigning // meaning to individual path segments. This package // only manipulates the path as a whole, so we allow those // last three as well. That leaves only ? to escape. return c == '?' case encodePathSegment: // §3.3 // The RFC allows : @ & = + $ but saves / ; , for assigning // meaning to individual path segments. return c == '/' || c == ';' || c == ',' || c == '?' case encodeUserPassword: // §3.2.1 // The RFC allows ';', ':', '&', '=', '+', '$', and ',' in // userinfo, so we must escape only '@', '/', and '?'. // The parsing of userinfo treats ':' as special so we must escape // that too. return c == '@' || c == '/' || c == '?' || c == ':' case encodeQueryComponent: // §3.4 // The RFC reserves (so we must escape) everything. return true case encodeFragment: // §4.1 // The RFC text is silent but the grammar allows // everything, so escape nothing. return false } } if mode == encodeFragment { // RFC 3986 §2.2 allows not escaping sub-delims. A subset of sub-delims are // included in reserved from RFC 2396 §2.2. The remaining sub-delims do not // need to be escaped. To minimize potential breakage, we apply two restrictions: // (1) we always escape sub-delims outside of the fragment, and (2) we always // escape single quote to avoid breaking callers that had previously assumed that // single quotes would be escaped. See issue #19917. switch c { case '!', '(', ')', '*': return false } } // Everything else must be escaped. return true } // unescape unescapes a string; the mode specifies // which section of the URL string is being unescaped. func unescape(s string, mode encoding) (string, error) { // Count %, check that they're well-formed. n := 0 hasPlus := false for i := 0; i < len(s); { switch s[i] { case '%': n++ if i+2 >= len(s) || !ishex(s[i+1]) || !ishex(s[i+2]) { s = s[i:] if len(s) > 3 { s = s[:3] } return "", EscapeError(s) } // Per https://tools.ietf.org/html/rfc3986#page-21 // in the host component %-encoding can only be used // for non-ASCII bytes. // But https://tools.ietf.org/html/rfc6874#section-2 // introduces %25 being allowed to escape a percent sign // in IPv6 scoped-address literals. Yay. if mode == encodeHost && unhex(s[i+1]) < 8 && s[i:i+3] != "%25" { return "", EscapeError(s[i : i+3]) } if mode == encodeZone { // RFC 6874 says basically "anything goes" for zone identifiers // and that even non-ASCII can be redundantly escaped, // but it seems prudent to restrict %-escaped bytes here to those // that are valid host name bytes in their unescaped form. // That is, you can use escaping in the zone identifier but not // to introduce bytes you couldn't just write directly. // But Windows puts spaces here! Yay. v := unhex(s[i+1])<<4 | unhex(s[i+2]) if s[i:i+3] != "%25" && v != ' ' && shouldEscape(v, encodeHost) { return "", EscapeError(s[i : i+3]) } } i += 3 case '+': hasPlus = mode == encodeQueryComponent i++ default: if (mode == encodeHost || mode == encodeZone) && s[i] < 0x80 && shouldEscape(s[i], mode) { return "", InvalidHostError(s[i : i+1]) } i++ } } if n == 0 && !hasPlus { return s, nil } t := make([]byte, len(s)-2*n) j := 0 for i := 0; i < len(s); { switch s[i] { case '%': t[j] = unhex(s[i+1])<<4 | unhex(s[i+2]) j++ i += 3 case '+': if mode == encodeQueryComponent { t[j] = ' ' } else { t[j] = '+' } j++ i++ default: t[j] = s[i] j++ i++ } } return string(t), nil } func escape(s string, mode encoding) string { spaceCount, hexCount := 0, 0 for i := 0; i < len(s); i++ { c := s[i] if shouldEscape(c, mode) { if c == ' ' && mode == encodeQueryComponent { spaceCount++ } else { hexCount++ } } } if spaceCount == 0 && hexCount == 0 { return s } t := make([]byte, len(s)+2*hexCount) j := 0 for i := 0; i < len(s); i++ { switch c := s[i]; { case c == ' ' && mode == encodeQueryComponent: t[j] = '+' j++ case shouldEscape(c, mode): t[j] = '%' t[j+1] = "0123456789ABCDEF"[c>>4] t[j+2] = "0123456789ABCDEF"[c&15] j += 3 default: t[j] = s[i] j++ } } return string(t) } // A URL represents a parsed URL (technically, a URI reference). // // The general form represented is: // // [scheme:][//[userinfo@]host][/]path[?query][#fragment] // // URLs that do not start with a slash after the scheme are interpreted as: // // scheme:opaque[?query][#fragment] // // Note that the Path field is stored in decoded form: /%47%6f%2f becomes /Go/. // A consequence is that it is impossible to tell which slashes in the Path were // slashes in the raw URL and which were %2f. This distinction is rarely important, // but when it is, code must not use Path directly. // The Parse function sets both Path and RawPath in the URL it returns, // and URL's String method uses RawPath if it is a valid encoding of Path, // by calling the EscapedPath method. type URL struct { Scheme Scheme Host string // host or host:port Path string // path (relative paths may omit leading slash) } // Maybe rawurl is of the form scheme:path. // (Scheme must be [a-zA-Z][a-zA-Z0-9+-.]*) // If so, return scheme, path; else return "", rawurl. func getscheme(rawurl string) (scheme Scheme, path string, err error) { for i := 0; i < len(rawurl); i++ { c := rawurl[i] switch { case 'a' <= c && c <= 'z' || 'A' <= c && c <= 'Z': // do nothing case '0' <= c && c <= '9' || c == '+' || c == '-' || c == '.': if i == 0 { return SchemeInvalid, rawurl, nil } case c == ':': if i == 0 { return SchemeInvalid, "", errors.New("missing protocol scheme") } switch rawurl[:i] { case "http": scheme = SchemeHTTP case "https": scheme = SchemeHTTPS default: return SchemeInvalid, "", ErrUnknownScheme } path = rawurl[i+1:] return default: // we have encountered an invalid character, // so there is no valid scheme return SchemeInvalid, rawurl, nil } } return SchemeInvalid, rawurl, nil } // Maybe s is of the form t c u. // If so, return t, c u (or t, u if cutc == true). // If not, return s, "". func split(s string, c string, cutc bool) (string, string) { i := strings.Index(s, c) if i < 0 { return s, "" } if cutc { return s[:i], s[i+len(c):] } return s[:i], s[i:] } // Parse parses rawurl into a URL structure. // // The rawurl may be relative (a path, without a host) or absolute // (starting with a scheme). Trying to parse a hostname and path // without a scheme is invalid but may not necessarily return an // error, due to parsing ambiguities. func (u *URL) Parse(rawurl string) error { // Cut off #frag s, frag := split(rawurl, "#", true) err := u.parse(s, false) if err != nil { return &Error{"parse", s, err} } if frag == "" { return nil } return nil } // ParseRequestURI parses rawurl into a URL structure. It assumes that // rawurl was received in an HTTP request, so the rawurl is interpreted // only as an absolute URI or an absolute path. // The string rawurl is assumed not to have a #fragment suffix. // (Web browsers strip #fragment before sending the URL to a web server.) func (u *URL) ParseRequestURI(rawurl string) error { err := u.parse(rawurl, true) if err != nil { return &Error{"parse", rawurl, err} } return nil } // parse parses a URL from a string in one of two contexts. If // viaRequest is true, the URL is assumed to have arrived via an HTTP request, // in which case only absolute URLs or path-absolute relative URLs are allowed. // If viaRequest is false, all forms of relative URLs are allowed. func (u *URL) parse(rawurl string, viaRequest bool) error { var rest string var err error if rawurl == "" && viaRequest { return errors.New("empty url") } if rawurl == "*" { u.Path = "*" return nil } // Split off possible leading "http:", "mailto:", etc. // Cannot contain escaped characters. if u.Scheme, rest, err = getscheme(rawurl); err != nil { return err } if strings.HasSuffix(rest, "?") && strings.Count(rest, "?") == 1 { rest = rest[:len(rest)-1] } else { rest, _ = split(rest, "?", true) } if !strings.HasPrefix(rest, "/") { if u.Scheme != SchemeInvalid { // We consider rootless paths per RFC 3986 as opaque. return nil } if viaRequest { return errors.New("invalid URI for request") } // Avoid confusion with malformed schemes, like cache_object:foo/bar. // See golang.org/issue/16822. // // RFC 3986, §3.3: // In addition, a URI reference (Section 4.1) may be a relative-path reference, // in which case the first path segment cannot contain a colon (":") character. colon := strings.Index(rest, ":") slash := strings.Index(rest, "/") if colon >= 0 && (slash < 0 || colon < slash) { // First path segment has colon. Not allowed in relative URL. return errors.New("first path segment in URL cannot contain colon") } } if (u.Scheme != SchemeInvalid || !viaRequest && !strings.HasPrefix(rest, "///")) && strings.HasPrefix(rest, "//") { var authority string authority, rest = split(rest[2:], "/", false) u.Host, err = parseAuthority(authority) if err != nil { return err } } u.Path = rest return nil } func parseAuthority(authority string) (host string, err error) { i := strings.LastIndex(authority, "@") if i < 0 { host, err = parseHost(authority) } else { host, err = parseHost(authority[i+1:]) } if err != nil { return "", err } if i < 0 { return host, nil } userinfo := authority[:i] if !validUserinfo(userinfo) { return "", errors.New("fasturl: invalid userinfo") } return host, nil } // parseHost parses host as an authority without user // information. That is, as host[:port]. func parseHost(host string) (string, error) { if strings.HasPrefix(host, "[") { // Parse an IP-Literal in RFC 3986 and RFC 6874. // E.g., "[fe80::1]", "[fe80::1%25en0]", "[fe80::1]:80". i := strings.LastIndex(host, "]") if i < 0 { return "", errors.New("missing ']' in host") } colonPort := host[i+1:] if !validOptionalPort(colonPort) { return "", fmt.Errorf("invalid port %q after host", colonPort) } // RFC 6874 defines that %25 (%-encoded percent) introduces // the zone identifier, and the zone identifier can use basically // any %-encoding it likes. That's different from the host, which // can only %-encode non-ASCII bytes. // We do impose some restrictions on the zone, to avoid stupidity // like newlines. zone := strings.Index(host[:i], "%25") if zone >= 0 { host1, err := unescape(host[:zone], encodeHost) if err != nil { return "", err } host2, err := unescape(host[zone:i], encodeZone) if err != nil { return "", err } host3, err := unescape(host[i:], encodeHost) if err != nil { return "", err } return host1 + host2 + host3, nil } } var err error if host, err = unescape(host, encodeHost); err != nil { return "", err } return host, nil } // validOptionalPort reports whether port is either an empty string // or matches /^:\d*$/ func validOptionalPort(port string) bool { if port == "" { return true } if port[0] != ':' { return false } for _, b := range port[1:] { if b < '0' || b > '9' { return false } } return true } // String reassembles the URL into a valid URL string. // The general form of the result is one of: // // scheme:opaque?query#fragment // scheme://userinfo@host/path?query#fragment // // If u.Opaque is non-empty, String uses the first form; // otherwise it uses the second form. // To obtain the path, String uses u.EscapedPath(). // // In the second form, the following rules apply: // - if u.Scheme is empty, scheme: is omitted. // - if u.User is nil, userinfo@ is omitted. // - if u.Host is empty, host/ is omitted. // - if u.Scheme and u.Host are empty and u.User is nil, // the entire scheme://userinfo@host/ is omitted. // - if u.Host is non-empty and u.Path begins with a /, // the form host/path does not add its own /. // - if u.RawQuery is empty, ?query is omitted. // - if u.Fragment is empty, #fragment is omitted. func (u *URL) String() string { var buf strings.Builder if u.Scheme != SchemeInvalid { buf.WriteString(Schemes[u.Scheme]) buf.WriteByte(':') } if u.Scheme != SchemeInvalid || u.Host != "" { if u.Host != "" || u.Path != "" { buf.WriteString("//") } if h := u.Host; h != "" { buf.WriteString(escape(h, encodeHost)) } } path := u.Path if path != "" && path[0] != '/' && u.Host != "" { buf.WriteByte('/') } if buf.Len() == 0 { // RFC 3986 §4.2 // A path segment that contains a colon character (e.g., "this:that") // cannot be used as the first segment of a relative-path reference, as // it would be mistaken for a scheme name. Such a segment must be // preceded by a dot-segment (e.g., "./this:that") to make a relative- // path reference. if i := strings.IndexByte(path, ':'); i > -1 && strings.IndexByte(path[:i], '/') == -1 { buf.WriteString("./") } } buf.WriteString(path) return buf.String() } func isRunesDot(r []rune) bool { return len(r) == 1 && r[0] == '.' } func isRunesDoubleDot(r []rune) bool { return len(r) == 2 && r[0] == '.' && r[1] == '.' } // resolvePath applies special path segments from refs and applies // them to base, per RFC 3986. func resolvePath(base, ref string) string { var full string if ref == "" { full = base } else if ref[0] != '/' { i := strings.LastIndex(base, "/") full = base[:i+1] + ref } else { full = ref } if full == "" { return "" } else if full == "/" { return "/" } dst := make([]rune, len(full)) dst = dst[0:0] start := 0 rs := []rune(full) if len(rs) != 0 && rs[0] == '/' { rs = rs[1:] } var stack []int stack = append(stack, 0) for i, c := range rs { if i == len(rs) - 1 { closingSlash := false part := rs[start:] if len(part) == 0 { dst = append(dst, '/') } else if part[len(part)-1] == '/' { part = part[:len(part)-1] closingSlash = true } switch { case isRunesDot(part): dst = append(dst, '/') case isRunesDoubleDot(part): // Cut to the last slash start = i+1 dst = dst[:stack[len(stack)-1]] if len(stack) != 1 { stack = stack[:len(stack)-1] } dst = append(dst, '/') default: dst = append(dst, '/') dst = append(dst, part...) } if closingSlash && len(dst) != 0 && dst[len(dst)-1] != '/' { dst = append(dst, '/') } } else if c == '/' { part := rs[start:i] switch { case isRunesDot(part): start = i+1 case isRunesDoubleDot(part): // Cut to the last slash start = i+1 dst = dst[:stack[len(stack)-1]] if len(stack) != 1 { stack = stack[:len(stack)-1] } default: start = i+1 stack = append(stack, len(dst)) dst = append(dst, '/') dst = append(dst, part...) } } } return string(dst) /*var dst []string src := strings.Split(full, "/") for _, elem := range src { switch elem { case ".": // drop case "..": if len(dst) > 0 { dst = dst[:len(dst)-1] } default: dst = append(dst, elem) } } if last := src[len(src)-1]; last == "." || last == ".." { // Add final slash to the joined path. dst = append(dst, "") } return "/" + strings.TrimPrefix(strings.Join(dst, "/"), "/")*/ } // IsAbs reports whether the URL is absolute. // Absolute means that it has a non-empty scheme. func (u *URL) IsAbs() bool { return u.Scheme != SchemeInvalid } // ParseRel parses a URL in the context of the receiver. The provided URL // may be relative or absolute. Parse returns nil, err on parse // failure, otherwise its return value is the same as ResolveReference. func (u *URL) ParseRel(out *URL, ref string) error { var refurl URL err := refurl.Parse(ref) if err != nil { return err } u.ResolveReference(out, &refurl) return nil } // ResolveReference resolves a URI reference to an absolute URI from // an absolute base URI u, per RFC 3986 Section 5.2. The URI reference // may be relative or absolute. ResolveReference always returns a new // URL instance, even if the returned URL is identical to either the // base or reference. If ref is an absolute URL, then ResolveReference // ignores base and returns a copy of ref. func (u *URL) ResolveReference(url *URL, ref *URL) { *url = *ref if ref.Scheme == SchemeInvalid { url.Scheme = u.Scheme } if ref.Scheme != SchemeInvalid || ref.Host != "" { // The "absoluteURI" or "net_path" cases. // We can ignore the error from setPath since we know we provided a // validly-escaped path. url.Path = resolvePath(ref.Path, "") return } // The "abs_path" or "rel_path" cases. url.Host = u.Host url.Path = resolvePath(u.Path, ref.Path) return } // Marshaling interface implementations. // Would like to implement MarshalText/UnmarshalText but that will change the JSON representation of URLs. func (u *URL) MarshalBinary() (text []byte, err error) { return []byte(u.String()), nil } func (u *URL) UnmarshalBinary(text []byte) error { var u1 URL err := u1.Parse(string(text)) if err != nil { return err } *u = u1 return nil } // validUserinfo reports whether s is a valid userinfo string per RFC 3986 // Section 3.2.1: // userinfo = *( unreserved / pct-encoded / sub-delims / ":" ) // unreserved = ALPHA / DIGIT / "-" / "." / "_" / "~" // sub-delims = "!" / "$" / "&" / "'" / "(" / ")" // / "*" / "+" / "," / ";" / "=" // // It doesn't validate pct-encoded. The caller does that via func unescape. func validUserinfo(s string) bool { for _, r := range s { if 'A' <= r && r <= 'Z' { continue } if 'a' <= r && r <= 'z' { continue } if '0' <= r && r <= '9' { continue } switch r { case '-', '.', '_', ':', '~', '!', '$', '&', '\'', '(', ')', '*', '+', ',', ';', '=', '%', '@': continue default: return false } } return true } func PathUnescape(s string) string { newStr, err := pathUnescape(s) if err != nil { return s } else { return newStr } } func pathUnescape(s string) (string, error) { // Count %, check that they're well-formed. n := 0 for i := 0; i < len(s); { switch s[i] { case '%': n++ if i+2 >= len(s) || !ishex(s[i+1]) || !ishex(s[i+2]) { s = s[i:] if len(s) > 3 { s = s[:3] } return "", EscapeError(s) } i += 3 default: i++ } } if n == 0 { return s, nil } t := make([]byte, len(s)-2*n) j := 0 for i := 0; i < len(s); { switch s[i] { case '%': t[j] = unhex(s[i+1])<<4 | unhex(s[i+2]) j++ i += 3 case '+': t[j] = '+' j++ i++ default: t[j] = s[i] j++ i++ } } return string(t), nil }