Merge remote-tracking branch 'upstream/master'

http_client.go

@@ -9,6 +9,7 @@ import (
 	"log"
 	"net"
 	"net/http"
+	"net/http/httputil"
 	"net/url"
 	"runtime/debug"
 	"strconv"
@@ -41,6 +42,7 @@ type HTTPClientConfig struct {
 	ConnectionTimeout  time.Duration
 	Timeout            time.Duration
 	ResponseBufferSize int
+	CompatibilityMode  bool
 }
 
 type HTTPClient struct {
@@ -53,6 +55,7 @@ type HTTPClient struct {
 	proxyAuth      string
 	respBuf        []byte
 	config         *HTTPClientConfig
+	goClient       *http.Client
 	redirectsCount int
 }
 
@@ -80,6 +83,13 @@ func NewHTTPClient(baseURL string, config *HTTPClientConfig) *HTTPClient {
 	client.respBuf = make([]byte, config.ResponseBufferSize)
 	client.config = config
 
+	if config.CompatibilityMode {
+		client.goClient = &http.Client{
+			// #TODO
+			// CheckRedirect: redirectPolicyFunc,
+		}
+	}
+
 	if u.User != nil {
 		client.auth = "Basic " + base64.StdEncoding.EncodeToString([]byte(u.User.String()))
 	}
@@ -194,6 +204,35 @@ func (c *HTTPClient) isAlive(readBytes *int) bool {
 	return true
 }
 
+func (c *HTTPClient) SendGoClient(data []byte) ([]byte, error) {
+	var req *http.Request
+	var resp *http.Response
+	var err error
+
+	req, err = http.ReadRequest(bufio.NewReader(bytes.NewBuffer(data)))
+	if err != nil {
+		return nil, err
+	}
+
+	if !c.config.OriginalHost {
+		req.Host = c.host
+	}
+
+	if c.auth != "" {
+		req.Header.Add("Authorization", c.auth)
+	}
+
+	req.URL, _ = url.ParseRequestURI(c.scheme + "://" + c.host + req.RequestURI)
+	req.RequestURI = ""
+
+	resp, err = c.goClient.Do(req)
+	if err != nil {
+		return nil, err
+	}
+
+	return httputil.DumpResponse(resp, true)
+}
+
 func (c *HTTPClient) Send(data []byte) (response []byte, err error) {
 	// Don't exit on panic
 	defer func() {
@@ -208,6 +247,10 @@ func (c *HTTPClient) Send(data []byte) (response []byte, err error) {
 		}
 	}()
 
+	if c.config.CompatibilityMode {
+		return c.SendGoClient(data)
+	}
+
 	var readBytes int
 	if c.conn == nil || !c.isAlive(&readBytes) {
 		Debug("[HTTPClient] Connecting:", c.baseURL)

http_client_test.go

@@ -87,6 +87,25 @@ func TestHTTPClientSend(t *testing.T) {
 	client.Send(payload("POST"))
 
 	wg.Wait()
+
+	client = NewHTTPClient(server.URL, &HTTPClientConfig{Debug: true, CompatibilityMode: true})
+
+	wg.Add(4)
+
+	if _, err := client.Send(payload("POST")); err != nil {
+		t.Fatal(err.Error())
+	}
+	if _, err := client.Send(payload("GET")); err != nil {
+		t.Fatal(err.Error())
+	}
+	if _, err := client.Send(payload("POST_CHUNKED")); err != nil {
+		t.Fatal(err.Error())
+	}
+	if _, err := client.Send(payload("POST")); err != nil {
+		t.Fatal(err.Error())
+	}
+
+	wg.Wait()
 }
 
 func TestHTTPClientResonseByClose(t *testing.T) {

middleware/README.md

@@ -69,7 +69,7 @@ gor.on("request", function(req) {
 })
 ```
 
-This middleware include `searchResponses` helper used to compare values from original and replayed responses. It may be helpful if auth system or xsrf protection returns unique tokens in headers or response, and you need to rewrite your requests based on them. Because tokens are unique, value contained in original and replayed response will differ, so you need to extract value from both responses, and rewrite requests based on those mappings.
+This middleware includes `searchResponses` helper which is used to compare value of original response with the replayed response. If authentication system or xsrf protection returns unique tokens in headers or the response, it will be helpful to rewrite your requests based on them. Because tokens are unique, and the value contained in original and replayed responses will be different. So, you need to extract value from both responses, and rewrite requests based on those mappings.
 
 `searchResponses` accepts request id, regexp pattern for searching the compared value (should include capture group), and callback which returns both original and replayed matched value.
 
@@ -112,10 +112,11 @@ Package expose following functions to process raw HTTP payloads:
 * `setHttpBodyParam` - set POST body param: `req.http = gor.setHttpBodyParam(req.http, param, value)`
 * `httpCookie` - get HTTP cookie: `gor.httpCookie(req.http, "SESSSION_ID")`
 * `setHttpCookie` - set HTTP cookie, returns modified payload: `req.http = gor.setHttpCookie(req.http, "iam", "cuckoo")`
+* `deleteHttpCookie` - delete HTTP cookie, returns modified payload: `req.http = gor.deleteHttpCookie(req.http, "iam")`
 
-Also it is totally legit to use standard `Buffer` functions like `indexOf` for processing the HTTP payload. Just do not forget that if you modify modify the body, update the `Content-Length` header with new value. And if you modify headers, line endings should be `\r\n`. Rest is up to your imagination.
+Also it is totally legit to use standard `Buffer` functions like `indexOf` for processing the HTTP payload. Just do not forget that if you modify the body, update the `Content-Length` header with a new value. And if you modify any of the headers, line endings should be `\r\n`. Rest is up to your imagination.
 
 
 ## Support
 
-Feel free to ask questions here and by sending email to [support@goreplay.org](mailto:support@goreplay.org). Commercial support available and welcomed 🙈.
+Feel free to ask questions here and by sending email to [support@goreplay.org](mailto:support@goreplay.org). Commercial support is available and welcomed 🙈.

middleware/middleware.js

@@ -381,6 +381,13 @@ function setHttpCookie(payload, name, value) {
     return setHttpHeader(payload, "Cookie", cookies.join("; "))
 }
 
+function deleteHttpCookie(payload, name) {
+    let h = httpHeader(payload, "Cookie");
+    let cookie = h ? h.value : "";
+    let cookies = cookie.split("; ").filter(function(v){ return v.indexOf(name + "=") != 0 })
+    return setHttpHeader(payload, "Cookie", cookies.join("; "))
+}
+
 function httpCookie(payload, name) {
     let h = httpHeader(payload, "Cookie");
     let cookie = h ? h.value : "";
@@ -414,6 +421,7 @@ module.exports = {
     setHttpBodyParam: setHttpBodyParam,
     httpCookie: httpCookie,
     setHttpCookie: setHttpCookie,
+    deleteHttpCookie: deleteHttpCookie,
     test: testRunner,
     benchmark: testBenchmark,
     httpHeaders: httpHeaders
@@ -423,7 +431,7 @@ module.exports = {
 // =========== Tests ==============
 
 function testRunner(){
-    ["init", "filter", "parseMessage", "httpMethod", "httpPath", "setHttpHeader", "deleteHttpHeader", "httpPathParam", "httpHeader", "httpBody", "setHttpBody", "httpBodyParam", "httpCookie", "setHttpCookie", "httpHeaders"].forEach(function(t){
+    ["init", "filter", "parseMessage", "httpMethod", "httpPath", "setHttpHeader", "deleteHttpHeader", "httpPathParam", "httpHeader", "httpBody", "setHttpBody", "httpBodyParam", "httpCookie", "setHttpCookie", "deleteHttpCookie", "httpHeaders"].forEach(function(t){
         console.log(`====== Start ${t} =======`)
         eval(`TEST_${t}()`)
         console.log(`====== End ${t} =======`)
@@ -739,6 +747,15 @@ function TEST_setHttpCookie() {
     }
 }
 
+function TEST_deleteHttpCookie() {
+    const examplePayload = "GET / HTTP/1.1\r\nCookie: a=b; test=zxc\r\n\r\n";
+    let p = deleteHttpCookie(Buffer.from(examplePayload), "a");
+    if (p != "GET / HTTP/1.1\r\nCookie: test=zxc\r\n\r\n") {
+        return fail(`Should delete cookie: ${p}`)
+    }
+}
+
+
 function TEST_httpHeaders() {
     const examplePayload = "GET / HTTP/1.1\r\nHost: localhost:3000\r\nUser-Agent: Node\r\nContent-Length:5\r\n\r\nhello";
 

middleware/package.json

@@ -1,6 +1,6 @@
 {
   "name": "goreplay_middleware",
-  "version": "0.1.19",
+  "version": "1.0.0",
   "description": "Package for writing middleware for GoReplay https://goreplay.org",
   "main": "middleware.js",
   "scripts": {

output_file.go

@@ -253,8 +253,8 @@ func (o *FileOutput) flush() {
 		if stat, err := o.file.Stat(); err == nil {
 			o.chunkSize = int(stat.Size())
 		} else {
-            log.Println("Error accessing file sats", err)
-        }
+			log.Println("Error accessing file sats", err)
+		}
 	}
 }
 

output_http.go

@@ -67,9 +67,9 @@ type HTTPOutputConfig struct {
 	stats      bool
 	workersMin int
 	workersMax int
-	statsMs int
-	workers int
-	queueLen int
+	statsMs    int
+	workers    int
+	queueLen   int
 
 	elasticSearch string
 
@@ -77,6 +77,8 @@ type HTTPOutputConfig struct {
 	OriginalHost bool
 	BufferSize   int
 
+	CompatibilityMode bool
+
 	Debug bool
 
 	TrackResponses bool
@@ -195,6 +197,7 @@ func (o *HTTPOutput) startWorker() {
 		OriginalHost:       o.config.OriginalHost,
 		Timeout:            o.config.Timeout,
 		ResponseBufferSize: o.config.BufferSize,
+		CompatibilityMode:  o.config.CompatibilityMode,
 	})
 
 	for {

settings.go

@@ -171,8 +171,9 @@ func init() {
 
 	/* outputHTTPConfig */
 	flag.IntVar(&Settings.outputHTTPConfig.BufferSize, "output-http-response-buffer", 0, "HTTP response buffer size, all data after this size will be discarded.")
+	flag.BoolVar(&Settings.outputHTTPConfig.CompatibilityMode, "output-http-compatibility-mode", false, "Use standard Go client, instead of built-in implementation. Can be slower, but more compatible.")
 
-  flag.IntVar(&Settings.outputHTTPConfig.workersMin, "output-http-workers-min", 0, "Gor uses dynamic worker scaling. Enter a number to set a minimum number of workers. default = 1.")
+	flag.IntVar(&Settings.outputHTTPConfig.workersMin, "output-http-workers-min", 0, "Gor uses dynamic worker scaling. Enter a number to set a minimum number of workers. default = 1.")
 	flag.IntVar(&Settings.outputHTTPConfig.workersMax, "output-http-workers", 0, "Gor uses dynamic worker scaling. Enter a number to set a maximum number of workers. default = 0 = unlimited.")
 	flag.IntVar(&Settings.outputHTTPConfig.queueLen, "output-http-queue-len", 1000, "Number of requests that can be queued for output, if all workers are busy. default = 1000")