Prettify http (#418)

Prettify HTTP

emitter.go

@@ -100,6 +100,10 @@ func CopyMulty(src io.Reader, writers ...io.Writer) (err error) {
 				}
 			}
 
+			if Settings.prettifyHTTP {
+				payload = prettifyHTTP(payload)
+			}
+
 			if Settings.splitOutput {
 				// Simple round robin
 				writers[wIndex].Write(payload)

http_prettifier.go

@@ -0,0 +1,68 @@
+package main
+
+import (
+    "github.com/buger/gor/proto"
+    "bytes"
+    "compress/gzip"
+    "strconv"
+    "io/ioutil"
+    "net/http/httputil"
+)
+
+func prettifyHTTP(p []byte) []byte {
+    headSize := bytes.IndexByte(p, '\n') + 1
+    head := p[:headSize]
+    body := p[headSize:]
+
+    headersPos := proto.MIMEHeadersEndPos(body)
+    headers := body[:headersPos]
+    content := body[headersPos:]
+
+    var tEnc, cEnc []byte
+    proto.ParseHeaders([][]byte{headers}, func(header, value []byte) bool {
+        if proto.HeadersEqual(header, []byte("Transfer-Encoding")) {
+            tEnc = value
+        }
+
+        if proto.HeadersEqual(header, []byte("Content-Encoding")) {
+            cEnc = value
+        }
+
+        return true
+    })
+
+    if len(tEnc) == 0 && len(cEnc) == 0 {
+        return p
+    }
+
+    if bytes.Equal(tEnc, []byte("chunked")) {
+        buf := bytes.NewBuffer(content)
+        r := httputil.NewChunkedReader(buf)
+        content, _ = ioutil.ReadAll(r)
+
+        headers = proto.DeleteHeader(headers, []byte("Transfer-Encoding"))
+
+        newLen := strconv.Itoa(len(content))
+        headers = proto.SetHeader(headers, []byte("Content-Length"), []byte(newLen))
+    }
+
+    if bytes.Equal(cEnc, []byte("gzip")) {
+        buf := bytes.NewBuffer(content)
+        g, err := gzip.NewReader(buf)
+
+        if err != nil {
+            Debug("[Prettifier] GZIP encoding error:", err)
+        }
+
+        content, _ = ioutil.ReadAll(g)
+
+        headers = proto.DeleteHeader(headers, []byte("Content-Encoding"))
+
+        newLen := strconv.Itoa(len(content))
+        headers = proto.SetHeader(headers, []byte("Content-Length"), []byte(newLen))
+    }
+
+    newPayload := append(append(head, headers...), content...)
+
+    return newPayload
+}

http_prettifier_test.go

@@ -0,0 +1,36 @@
+package main
+
+import (
+    "compress/gzip"
+    "testing"
+    "strconv"
+    "bytes"
+)
+
+func TestHTTPPrettifierGzip(t *testing.T) {
+    b := bytes.NewBufferString("")
+    w := gzip.NewWriter(b)
+    w.Write([]byte("test"))
+    w.Close()
+
+    size := strconv.Itoa(len(b.Bytes()))
+
+    payload := []byte("2 1 1\nHTTP/1.1 200 OK\r\nContent-Length: " + size + "\r\nContent-Encoding: gzip\r\n\r\n")
+    payload = append(payload, b.Bytes()...)
+
+    newPayload := prettifyHTTP(payload)
+
+    if string(newPayload) != "2 1 1\nHTTP/1.1 200 OK\r\nContent-Length: 4\r\n\r\ntest" {
+        t.Error("Payload not match:", string(newPayload))
+    }
+}
+
+func TestHTTPPrettifierChunked(t *testing.T) {
+    payload := []byte("POST / HTTP/1.1\r\nHost: www.w3.org\r\nTransfer-Encoding: chunked\r\n\r\n4\r\nWiki\r\n5\r\npedia\r\ne\r\n in\r\n\r\nchunks.\r\n0\r\n\r\n")
+
+    newPayload := prettifyHTTP(payload)
+
+    if string(newPayload) != "POST / HTTP/1.1\r\nHost: www.w3.org\r\nContent-Length: 23\r\n\r\nWikipedia in\r\n\r\nchunks." {
+        t.Error("Payload not match:", string(newPayload))
+    }
+}

limiter.go

@@ -82,7 +82,11 @@ func (l *Limiter) Write(data []byte) (n int, err error) {
 }
 
 func (l *Limiter) Read(data []byte) (n int, err error) {
-	n, err = l.plugin.(io.Reader).Read(data)
+	if r, ok := l.plugin.(io.Reader); ok {
+		n, err = r.Read(data)
+	} else {
+		return 0, nil
+	}
 
 	if l.isLimited() {
 		return 0, nil

proto/proto.go

@@ -32,7 +32,7 @@ var HeaderDelim = []byte(": ")
 
 // MIMEHeadersEndPos finds end of the Headers section, which should end with empty line.
 func MIMEHeadersEndPos(payload []byte) int {
-	return bytes.Index(payload, EmptyLine)
+	return bytes.Index(payload, EmptyLine) + 4
 }
 
 // MIMEHeadersStartPos finds start of Headers section
@@ -337,7 +337,7 @@ func DeleteHeader(payload, name []byte) []byte {
 // Body returns request/response body
 func Body(payload []byte) []byte {
 	// 4 -> len(EMPTY_LINE)
-	return payload[MIMEHeadersEndPos(payload)+4:]
+	return payload[MIMEHeadersEndPos(payload):]
 }
 
 // Path takes payload and retuns request path: Split(firstLine, ' ')[1]

settings.go

@@ -56,6 +56,8 @@ type AppSettings struct {
 	inputHTTP  MultiOption
 	outputHTTP MultiOption
 
+	prettifyHTTP bool
+
 	outputHTTPConfig HTTPOutputConfig
 	modifierConfig   HTTPModifierConfig
 
@@ -105,6 +107,8 @@ func init() {
 	flag.Var(&Settings.outputFileConfig.sizeLimit, "output-file-size-limit", "Size of each chunk. Default: 32mb")
 	flag.IntVar(&Settings.outputFileConfig.queueLimit, "output-file-queue-limit", 256, "The length of the chunk queue. Default: 256")
 
+	flag.BoolVar(&Settings.prettifyHTTP, "prettify-http", false, "If enabled, will automatically decode requests and responses with: Content-Encodning: gzip and Transfer-Encoding: chunked. Useful for debugging, in conjuction with --output-stdout")
+
 	flag.Var(&Settings.inputRAW, "input-raw", "Capture traffic from given port (use RAW sockets and require *sudo* access):\n\t# Capture traffic from 8080 port\n\tgor --input-raw :8080 --output-http staging.com")
 
 	flag.BoolVar(&Settings.inputRAWTrackResponse, "input-raw-track-response", false, "If turned on Gor will track responses in addition to requests, and they will be available to middleware and file output.")