diff --git a/http_client.go b/http_client.go index f9ab412..8c7948b 100644 --- a/http_client.go +++ b/http_client.go @@ -52,7 +52,7 @@ func NewHTTPClient(baseURL string, config *HTTPClientConfig) *HTTPClient { client.baseURL = u.String() client.host = u.Host client.scheme = u.Scheme - client.respBuf = make([]byte, 4096*10) + client.respBuf = make([]byte, 512*1024) // 500kb client.config = config return client @@ -137,6 +137,16 @@ func (c *HTTPClient) Send(data []byte) (response []byte, err error) { c.conn.SetReadDeadline(timeout) n, err := c.conn.Read(c.respBuf) + // If response large then our buffer, we need to read all reponse buffer + // Otherwise it will corrupt response of next request + // Parsing response body is non trivial thing, especially with keep-alive + // Simples case is to to close connection if response too large + // + // See https://github.com/buger/gor/issues/184 + if n == len(c.respBuf) { + c.Disconnect() + } + if err != nil { Debug("[HTTPClient] Response read error", err, c.conn) return diff --git a/http_client_test.go b/http_client_test.go index feb0326..93051b6 100644 --- a/http_client_test.go +++ b/http_client_test.go @@ -8,6 +8,8 @@ import ( "net/http/httputil" "sync" "testing" + "crypto/rand" + "bytes" _ "time" ) @@ -81,6 +83,36 @@ func TestHTTPClientSend(t *testing.T) { wg.Wait() } +// https://github.com/buger/gor/issues/184 +func TestHTTPClientResponseBuffer(t *testing.T) { + wg := new(sync.WaitGroup) + + payload := []byte("GET / HTTP/1.1\r\n\r\n") + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + + size := 1 * 1024 * 1024 // 1 MB + rb := make([]byte, size) + rand.Read(rb) + + w.Write(rb) + + wg.Done() + })) + + client := NewHTTPClient(server.URL, &HTTPClientConfig{Debug: true}) + + wg.Add(2) + client.Send(payload) + resp, _ := client.Send(payload) + + if !bytes.Equal(resp[0:8], []byte("HTTP/1.1")) { + t.Error("Response buffer contains data from previous request", string(resp[0:5])) + } + + wg.Wait() +} + func TestHTTPClientHTTPSSend(t *testing.T) { wg := new(sync.WaitGroup)