package nhttp import ( "bufio" "bytes" "errors" "fmt" "golang.org/x/net/http/httpguts" "io" _http "net/http" "net/textproto" "strconv" "sync" ) type transferReader struct { // Input Header _http.Header StatusCode int RequestMethod string ProtoMajor int ProtoMinor int // Output Body io.ReadCloser ContentLength int64 Chunked bool Close bool Trailer _http.Header } // bodyLocked is an io.Reader reading from a *body when its mutex is // already held. type bodyLocked struct { b *body } func (bl bodyLocked) Read(p []byte) (n int, err error) { if bl.b.closed { return 0, errors.New("nhttp: invalid Read on closed Body") } return bl.b.readLocked(p) } // body turns a Reader into a ReadCloser. // Close ensures that the body has been fully read // and then reads the trailer if necessary. type body struct { src io.Reader hdr any // non-nil (Response or Request) value means read trailer r *bufio.Reader // underlying wire-format reader for the trailer closing bool // is the connection to be closed after reading body? doEarlyClose bool // whether Close should stop early mu sync.Mutex // guards following, and calls to Read and Close sawEOF bool closed bool earlyClose bool // Close called and we didn't read to the end of src onHitEOF func() // if non-nil, func to call when EOF is Read } func (b *body) Read(p []byte) (n int, err error) { b.mu.Lock() defer b.mu.Unlock() if b.closed { return 0, errors.New("nhttp: invalid Read on closed Body") } return b.readLocked(p) } // Must hold b.mu. func (b *body) readLocked(p []byte) (n int, err error) { if b.sawEOF { return 0, io.EOF } n, err = b.src.Read(p) if err == io.EOF { b.sawEOF = true // Chunked case. Read the trailer. if b.hdr != nil { if e := b.readTrailer(); e != nil { err = e // Something went wrong in the trailer, we must not allow any // further reads of any kind to succeed from body, nor any // subsequent requests on the server connection. See // golang.org/issue/12027 b.sawEOF = false b.closed = true } b.hdr = nil } else { // If the server declared the Content-Length, our body is a LimitedReader // and we need to check whether this EOF arrived early. if lr, ok := b.src.(*io.LimitedReader); ok && lr.N > 0 { err = io.ErrUnexpectedEOF } } } // If we can return an EOF here along with the read data, do // so. This is optional per the io.Reader contract, but doing // so helps the HTTP transport code recycle its connection // earlier (since it will see this EOF itself), even if the // client doesn't do future reads or Close. if err == nil && n > 0 { if lr, ok := b.src.(*io.LimitedReader); ok && lr.N == 0 { err = io.EOF b.sawEOF = true } } if b.sawEOF && b.onHitEOF != nil { b.onHitEOF() } return n, err } func (b *body) Close() error { b.mu.Lock() defer b.mu.Unlock() if b.closed { return nil } var err error switch { case b.sawEOF: // Already saw EOF, so no need going to look for it. case b.hdr == nil && b.closing: // no trailer and closing the connection next. // no point in reading to EOF. case b.doEarlyClose: // Read up to maxPostHandlerReadBytes bytes of the body, looking // for EOF (and trailers), so we can re-use this connection. if lr, ok := b.src.(*io.LimitedReader); ok && lr.N > 256<<10 { // There was a declared Content-Length, and we have more bytes remaining // than our maxPostHandlerReadBytes tolerance. So, give up. b.earlyClose = true } else { var n int64 // Consume the body, or, which will also lead to us reading // the trailer headers after the body, if present. n, err = io.CopyN(io.Discard, bodyLocked{b}, 256<<10) if err == io.EOF { err = nil } if n == 256<<10 { b.earlyClose = true } } default: // Fully consume the body, which will also lead to us reading // the trailer headers after the body, if present. _, err = io.Copy(io.Discard, bodyLocked{b}) } b.closed = true return err } func (b *body) didEarlyClose() bool { b.mu.Lock() defer b.mu.Unlock() return b.earlyClose } // bodyRemains reports whether future Read calls might // yield data. func (b *body) bodyRemains() bool { b.mu.Lock() defer b.mu.Unlock() return !b.sawEOF } func (b *body) registerOnHitEOF(fn func()) { b.mu.Lock() defer b.mu.Unlock() b.onHitEOF = fn } // Determine whether to hang up after sending a request and body, or // receiving a response and body // 'header' is the request headers func shouldClose(major, minor int, header _http.Header, removeCloseHeader bool) bool { if major < 1 { return true } conv := header["Connection"] hasClose := httpguts.HeaderValuesContainsToken(conv, "close") if major == 1 && minor == 0 { return hasClose || !httpguts.HeaderValuesContainsToken(conv, "keep-alive") } if hasClose && removeCloseHeader { header.Del("Connection") } return hasClose } // msg is *Request or *Response. func readTransfer(msg any, r *bufio.Reader) (err error) { t := &transferReader{RequestMethod: "GET"} // Unify input isResponse := false switch rr := msg.(type) { case *_http.Response: t.Header = rr.Header t.StatusCode = rr.StatusCode t.ProtoMajor = rr.ProtoMajor t.ProtoMinor = rr.ProtoMinor t.Close = shouldClose(t.ProtoMajor, t.ProtoMinor, t.Header, true) isResponse = true if rr.Request != nil { t.RequestMethod = rr.Request.Method } case *_http.Request: t.Header = rr.Header t.RequestMethod = rr.Method t.ProtoMajor = rr.ProtoMajor t.ProtoMinor = rr.ProtoMinor // Transfer semantics for Requests are exactly like those for // Responses with status code 200, responding to a GET method t.StatusCode = 200 t.Close = rr.Close default: panic("unexpected type") } // Default to HTTP/1.1 if t.ProtoMajor == 0 && t.ProtoMinor == 0 { t.ProtoMajor, t.ProtoMinor = 1, 1 } // Transfer-Encoding: chunked, and overriding Content-Length. if err := t.parseTransferEncoding(); err != nil { return err } realLength, err := fixLength(isResponse, t.StatusCode, t.RequestMethod, t.Header, t.Chunked) if err != nil { return err } if isResponse && t.RequestMethod == "HEAD" { if n, err := parseContentLength(t.Header.Get("Content-Length")); err != nil { return err } else { t.ContentLength = n } } else { t.ContentLength = realLength } // Trailer t.Trailer, err = fixTrailer(t.Header, t.Chunked) if err != nil { return err } // If there is no Content-Length or chunked Transfer-Encoding on a *Response // and the status is not 1xx, 204 or 304, then the body is unbounded. // See RFC 7230, section 3.3. switch msg.(type) { case *_http.Response: if realLength == -1 && !t.Chunked && bodyAllowedForStatus(t.StatusCode) { // Unbounded body. t.Close = true } } // Prepare body reader. ContentLength < 0 means chunked encoding // or close connection when finished, since multipart is not supported yet switch { case t.Chunked: if t.RequestMethod == "HEAD" || !bodyAllowedForStatus(t.StatusCode) { t.Body = NoBody } else { t.Body = &body{src: NewChunkedReader(r), hdr: msg, r: r, closing: t.Close} } case realLength == 0: t.Body = NoBody case realLength > 0: t.Body = &body{src: io.LimitReader(r, realLength), closing: t.Close} default: // realLength < 0, i.e. "Content-Length" not mentioned in header if t.Close { // Close semantics (i.e. HTTP/1.0) t.Body = &body{src: r, closing: t.Close} } else { // Persistent connection (i.e. HTTP/1.1) t.Body = NoBody } } // Unify output switch rr := msg.(type) { case *_http.Request: rr.Body = t.Body rr.ContentLength = t.ContentLength if t.Chunked { rr.TransferEncoding = []string{"chunked"} } rr.Close = t.Close rr.Trailer = t.Trailer case *_http.Response: rr.Body = t.Body rr.ContentLength = t.ContentLength if t.Chunked { rr.TransferEncoding = []string{"chunked"} } rr.Close = t.Close rr.Trailer = t.Trailer } return nil } // parseTransferEncoding sets t.Chunked based on the Transfer-Encoding header. func (t *transferReader) parseTransferEncoding() error { raw, present := t.Header["Transfer-Encoding"] if !present { return nil } delete(t.Header, "Transfer-Encoding") // Issue 12785; ignore Transfer-Encoding on HTTP/1.0 requests. if !t.protoAtLeast(1, 1) { return nil } // Like nginx, we only support a single Transfer-Encoding header field, and // only if set to "chunked". This is one of the most security sensitive // surfaces in HTTP/1.1 due to the risk of request smuggling, so we keep it // strict and simple. if len(raw) != 1 { return errors.New(fmt.Sprintf("too many transfer encodings: %q", raw)) } if !EqualFold(raw[0], "chunked") { return errors.New(fmt.Sprintf("unsupported transfer encoding: %q", raw[0])) } // RFC 7230 3.3.2 says "A sender MUST NOT send a Content-Length header field // in any message that contains a Transfer-Encoding header field." // // but also: "If a message is received with both a Transfer-Encoding and a // Content-Length header field, the Transfer-Encoding overrides the // Content-Length. Such a message might indicate an attempt to perform // request smuggling (Section 9.5) or response splitting (Section 9.4) and // ought to be handled as an error. A sender MUST remove the received // Content-Length field prior to forwarding such a message downstream." // // Reportedly, these appear in the wild. delete(t.Header, "Content-Length") t.Chunked = true return nil } func (t *transferReader) protoAtLeast(m, n int) bool { return t.ProtoMajor > m || (t.ProtoMajor == m && t.ProtoMinor >= n) } // Determine the expected body length, using RFC 7230 Section 3.3. This // function is not a method, because ultimately it should be shared by // ReadResponse and ReadRequest. func fixLength(isResponse bool, status int, requestMethod string, header _http.Header, chunked bool) (int64, error) { isRequest := !isResponse contentLens := header["Content-Length"] // Hardening against HTTP request smuggling if len(contentLens) > 1 { // Per RFC 7230 Section 3.3.2, prevent multiple // Content-Length headers if they differ in value. // If there are dups of the value, remove the dups. // See Issue 16490. first := textproto.TrimString(contentLens[0]) for _, ct := range contentLens[1:] { if first != textproto.TrimString(ct) { return 0, fmt.Errorf("nhttp: message cannot contain multiple Content-Length headers; got %q", contentLens) } } // deduplicate Content-Length header.Del("Content-Length") header.Add("Content-Length", first) contentLens = header["Content-Length"] } // Logic based on response type or status if requestMethod == "HEAD" { // For HTTP requests, as part of hardening against request // smuggling (RFC 7230), don't allow a Content-Length header for // methods which don't permit bodies. As an exception, allow // exactly one Content-Length header if its value is "0". if isRequest && len(contentLens) > 0 && !(len(contentLens) == 1 && contentLens[0] == "0") { return 0, fmt.Errorf("nhttp: method cannot contain a Content-Length; got %q", contentLens) } return 0, nil } if status/100 == 1 { return 0, nil } switch status { case 204, 304: return 0, nil } // Logic based on Transfer-Encoding if chunked { return -1, nil } // Logic based on Content-Length var cl string if len(contentLens) == 1 { cl = textproto.TrimString(contentLens[0]) } if cl != "" { n, err := parseContentLength(cl) if err != nil { return -1, err } return n, nil } header.Del("Content-Length") if isRequest { // RFC 7230 neither explicitly permits nor forbids an // entity-body on a GET request so we permit one if // declared, but we default to 0 here (not -1 below) // if there's no mention of a body. // Likewise, all other request methods are assumed to have // no body if neither Transfer-Encoding chunked nor a // Content-Length are set. return 0, nil } // Body-EOF logic based on other methods (like closing, or chunked coding) return -1, nil } func parseContentLength(cl string) (int64, error) { cl = textproto.TrimString(cl) if cl == "" { return -1, nil } n, err := strconv.ParseUint(cl, 10, 63) if err != nil { return 0, errors.New("bad Content-Length") } return int64(n), nil } // Parse the trailer header func fixTrailer(header _http.Header, chunked bool) (_http.Header, error) { vv, ok := header["Trailer"] if !ok { return nil, nil } if !chunked { // Trailer and no chunking: // this is an invalid use case for trailer header. // Nevertheless, no error will be returned and we // let users decide if this is a valid HTTP message. // The Trailer header will be kept in Response.Header // but not populate Response.Trailer. // See issue #27197. return nil, nil } header.Del("Trailer") trailer := make(_http.Header) var err error for _, v := range vv { foreachHeaderElement(v, func(key string) { key = CanonicalHeaderKey(key) switch key { case "Transfer-Encoding", "Trailer", "Content-Length": if err == nil { err = errors.New("bad trailer key") return } } trailer[key] = nil }) } if err != nil { return nil, err } if len(trailer) == 0 { return nil, nil } return trailer, nil } func bodyAllowedForStatus(status int) bool { switch { case status >= 100 && status <= 199: return false case status == 204: return false case status == 304: return false } return true } var errTrailerEOF = errors.New("nhttp: unexpected EOF reading trailer") func (b *body) readTrailer() error { // The common case, since nobody uses trailers. buf, err := b.r.Peek(2) if bytes.Equal(buf, singleCRLF) { b.r.Discard(2) return nil } if len(buf) < 2 { return errTrailerEOF } if err != nil { return err } // Make sure there's a header terminator coming up, to prevent // a DoS with an unbounded size Trailer. It's not easy to // slip in a LimitReader here, as textproto.NewReader requires // a concrete *bufio.Reader. Also, we can't get all the way // back up to our conn's LimitedReader that *might* be backing // this bufio.Reader. Instead, a hack: we iteratively Peek up // to the bufio.Reader's max size, looking for a double CRLF. // This limits the trailer to the underlying buffer size, typically 4kB. if !seeUpcomingDoubleCRLF(b.r) { return errors.New("nhttp: suspiciously long trailer after chunked body") } hdr, err := textproto.NewReader(b.r).ReadMIMEHeader() if err != nil { if err == io.EOF { return errTrailerEOF } return err } switch rr := b.hdr.(type) { case *_http.Request: mergeSetHeader(&rr.Trailer, _http.Header(hdr)) case *_http.Response: mergeSetHeader(&rr.Trailer, _http.Header(hdr)) } return nil } func mergeSetHeader(dst *_http.Header, src _http.Header) { if *dst == nil { *dst = src return } for k, vv := range src { (*dst)[k] = vv } } var ( suppressedHeaders304 = []string{"Content-Type", "Content-Length", "Transfer-Encoding"} suppressedHeadersNoBody = []string{"Content-Length", "Transfer-Encoding"} excludedHeadersNoBody = map[string]bool{"Content-Length": true, "Transfer-Encoding": true} ) func suppressedHeaders(status int) []string { switch { case status == 304: // RFC 7232 section 4.1 return suppressedHeaders304 case !bodyAllowedForStatus(status): return suppressedHeadersNoBody } return nil }