Fix large responses

http_client.go

@@ -52,7 +52,7 @@ func NewHTTPClient(baseURL string, config *HTTPClientConfig) *HTTPClient {
 	client.baseURL = u.String()
 	client.host = u.Host
 	client.scheme = u.Scheme
-	client.respBuf = make([]byte, 4096*10)
+	client.respBuf = make([]byte, 512*1024) // 500kb
 	client.config = config
 
 	return client
@@ -137,6 +137,16 @@ func (c *HTTPClient) Send(data []byte) (response []byte, err error) {
 	c.conn.SetReadDeadline(timeout)
 	n, err := c.conn.Read(c.respBuf)
 
+	// If response large then our buffer, we need to read all reponse buffer
+	// Otherwise it will corrupt response of next request
+	// Parsing response body is non trivial thing, especially with keep-alive
+	// Simples case is to to close connection if response too large
+	//
+	// See https://github.com/buger/gor/issues/184
+	if n == len(c.respBuf) {
+		c.Disconnect()
+	}
+
 	if err != nil {
 		Debug("[HTTPClient] Response read error", err, c.conn)
 		return

http_client_test.go

@@ -8,6 +8,8 @@ import (
 	"net/http/httputil"
 	"sync"
 	"testing"
+	"crypto/rand"
+	"bytes"
 	_ "time"
 )
 
@@ -81,6 +83,36 @@ func TestHTTPClientSend(t *testing.T) {
 	wg.Wait()
 }
 
+// https://github.com/buger/gor/issues/184
+func TestHTTPClientResponseBuffer(t *testing.T) {
+	wg := new(sync.WaitGroup)
+
+	payload := []byte("GET / HTTP/1.1\r\n\r\n")
+
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+
+		size := 1 * 1024 * 1024 // 1 MB
+		rb := make([]byte, size)
+		rand.Read(rb)
+
+		w.Write(rb)
+
+		wg.Done()
+	}))
+
+	client := NewHTTPClient(server.URL, &HTTPClientConfig{Debug: true})
+
+	wg.Add(2)
+	client.Send(payload)
+	resp, _ := client.Send(payload)
+
+	if !bytes.Equal(resp[0:8], []byte("HTTP/1.1")) {
+		t.Error("Response buffer contains data from previous request", string(resp[0:5]))
+	}
+
+	wg.Wait()
+}
+
 func TestHTTPClientHTTPSSend(t *testing.T) {
 	wg := new(sync.WaitGroup)