From 0fe72cc432980fb4b2640829719584da3e313ed6 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Fri, 10 Jul 2015 17:13:11 +0500 Subject: [PATCH 01/26] Improve http-client reliability and improve debug --- emitter.go | 4 ++++ http_client.go | 32 +++++++++++++++++++++++++------- settings.go | 8 +++++--- 3 files changed, 34 insertions(+), 10 deletions(-) diff --git a/emitter.go b/emitter.go index 5e7e78e..a2373f5 100644 --- a/emitter.go +++ b/emitter.go @@ -39,6 +39,10 @@ func CopyMulty(src io.Reader, writers ...io.Writer) (err error) { } } + if Settings.debug { + Debug("[EMITTER] Sending paylod:", string(payload)) + } + if Settings.splitOutput { // Simple round robin writers[wIndex].Write(payload) diff --git a/http_client.go b/http_client.go index 4e44f77..8bf6969 100644 --- a/http_client.go +++ b/http_client.go @@ -8,6 +8,8 @@ import ( "net/url" "strings" "time" + "runtime/debug" + "log" ) var defaultPorts = map[string]string{ @@ -89,9 +91,25 @@ func (c *HTTPClient) isAlive() bool { } func (c *HTTPClient) Send(data []byte) (response []byte, err error) { + // Don't exit on panic + defer func() { + if r := recover(); r != nil { + Debug("[HTTPClient]", r, string(data)) + + if _, ok := r.(error); !ok { + log.Println("[HTTPClient] Failed to send request: ", string(data)) + fmt.Printf("PANIC: pkg: %v %s \n", r, debug.Stack()) + } + } + }() + + if c.conn == nil || !c.isAlive() { - Debug("Connecting:", c.baseURL) - c.Connect() + Debug("[HTTPClient] Connecting:", c.baseURL) + if err = c.Connect(); err != nil { + fmt.Printf("[HTTPClient] Connection error: %s\n", r) + return + } } timeout := time.Now().Add(5 * time.Second) @@ -101,11 +119,11 @@ func (c *HTTPClient) Send(data []byte) (response []byte, err error) { data = proto.SetHost(data, []byte(c.baseURL), []byte(c.host)) if c.config.Debug { - Debug("Sending:", string(data)) + Debug("[HTTPClient] Sending:", string(data)) } if _, err = c.conn.Write(data); err != nil { - Debug("Write error:", err, c.baseURL) + Debug("[HTTPClient] Write error:", err, c.baseURL) return } @@ -113,14 +131,14 @@ func (c *HTTPClient) Send(data []byte) (response []byte, err error) { n, err := c.conn.Read(c.respBuf) if err != nil { - Debug("READ ERRORR!", err, c.conn) + Debug("[HTTPClient] Response read error", err, c.conn) return } payload := c.respBuf[:n] if c.config.Debug { - Debug("Received:", string(payload)) + Debug("[HTTPClient] Received:", string(payload)) } if c.config.FollowRedirects > 0 && c.redirectsCount < c.config.FollowRedirects { @@ -134,7 +152,7 @@ func (c *HTTPClient) Send(data []byte) (response []byte, err error) { redirectPayload := []byte("GET " + string(location) + " HTTP/1.1\r\n\r\n") if c.config.Debug { - Debug("Redirecting to: " + string(location)) + Debug("[HTTPClient] Redirecting to: " + string(location)) } return c.Send(redirectPayload) diff --git a/settings.go b/settings.go index 98d9e7f..62f089d 100644 --- a/settings.go +++ b/settings.go @@ -8,7 +8,7 @@ import ( ) const ( - VERSION = "0.9.6" + VERSION = "0.9.7" ) // Allows to specify multiple flags with same name and collects all values to array @@ -25,6 +25,7 @@ func (h *MultiOption) Set(value string) error { type AppSettings struct { verbose bool + debug bool stats bool splitOutput bool @@ -59,7 +60,8 @@ func usage() { func init() { flag.Usage = usage - flag.BoolVar(&Settings.verbose, "verbose", false, "Turn on verbose/debug output") + flag.BoolVar(&Settings.verbose, "verbose", false, "Turn on more verbose output") + flag.BoolVar(&Settings.debug, "debug", false, "Turn on debug output, shows all itercepted traffic. Works only when with `verbose` flag") flag.BoolVar(&Settings.stats, "stats", false, "Turn on queue stats output") flag.BoolVar(&Settings.splitOutput, "split-output", false, "By default each output gets same traffic. If set to `true` it splits traffic equally among all outputs.") @@ -113,7 +115,7 @@ func init() { func Debug(args ...interface{}) { if Settings.verbose { - log.Print("[DEBUG] ") + fmt.Print("[DEBUG] ") log.Println(args...) } } From acaf935caea2ebdb001c29485bd8b8944899a562 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Fri, 10 Jul 2015 17:17:16 +0500 Subject: [PATCH 02/26] Fix tests --- http_client.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/http_client.go b/http_client.go index 8bf6969..fb0ec74 100644 --- a/http_client.go +++ b/http_client.go @@ -98,7 +98,7 @@ func (c *HTTPClient) Send(data []byte) (response []byte, err error) { if _, ok := r.(error); !ok { log.Println("[HTTPClient] Failed to send request: ", string(data)) - fmt.Printf("PANIC: pkg: %v %s \n", r, debug.Stack()) + log.Println("PANIC: pkg:", r, debug.Stack()) } } }() @@ -107,7 +107,7 @@ func (c *HTTPClient) Send(data []byte) (response []byte, err error) { if c.conn == nil || !c.isAlive() { Debug("[HTTPClient] Connecting:", c.baseURL) if err = c.Connect(); err != nil { - fmt.Printf("[HTTPClient] Connection error: %s\n", r) + log.Println("[HTTPClient] Connection error:", err) return } } From 6b9c9b91e324ffe867971b7b665f05108b240457 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Sat, 11 Jul 2015 09:47:26 +0500 Subject: [PATCH 03/26] Improve performance of input-raw and improve handing of large payloads --- Makefile | 2 +- byteutils/byteutils.go | 46 ++--- byteutils/byteutils_test.go | 36 ++-- emitter.go | 2 +- http_client.go | 5 +- input_raw_test.go | 59 ++++++ proto/proto.go | 186 +++++++++---------- proto/proto_test.go | 178 +++++++++--------- raw_socket_listener/listener.go | 282 ++++++++++++++++++----------- raw_socket_listener/tcp_message.go | 119 ++++++------ raw_socket_listener/tcp_packet.go | 124 ++++++------- settings.go | 2 +- test_input.go | 6 +- 13 files changed, 586 insertions(+), 461 deletions(-) diff --git a/Makefile b/Makefile index d8bc799..40fff87 100644 --- a/Makefile +++ b/Makefile @@ -21,7 +21,7 @@ dcover: go tool cover -html=coverage.out dfmt: - docker run -v `pwd`:$(SOURCE_PATH) -t -i gor go fmt + docker run -v `pwd`:$(SOURCE_PATH) -t -i gor go fmt ./... dvet: docker run -v `pwd`:$(SOURCE_PATH) -t -i gor go vet diff --git a/byteutils/byteutils.go b/byteutils/byteutils.go index 33ea49e..c65bbdb 100644 --- a/byteutils/byteutils.go +++ b/byteutils/byteutils.go @@ -1,37 +1,37 @@ package byteutils func Cut(a []byte, from, to int) []byte { - copy(a[from:], a[to:]) - a = a[:len(a)-to+from] + copy(a[from:], a[to:]) + a = a[:len(a)-to+from] - return a + return a } func Insert(a []byte, i int, b []byte) []byte { - a = append(a, make([]byte, len(b))...) - copy(a[i+len(b):], a[i:]) - copy(a[i:i+len(b)], b) + a = append(a, make([]byte, len(b))...) + copy(a[i+len(b):], a[i:]) + copy(a[i:i+len(b)], b) - return a + return a } // Unlike bytes.Replace it allows you to specify range func Replace(a []byte, from, to int, new []byte) []byte { - lenDiff := len(new) - (to - from) + lenDiff := len(new) - (to - from) - if lenDiff > 0 { - // Extend if new segment bigger - a = append(a, make([]byte, lenDiff)...) - copy(a[to+lenDiff:], a[to:]) - copy(a[from:from+len(new)], new) + if lenDiff > 0 { + // Extend if new segment bigger + a = append(a, make([]byte, lenDiff)...) + copy(a[to+lenDiff:], a[to:]) + copy(a[from:from+len(new)], new) - return a - } else if lenDiff < 0 { - copy(a[from:], new) - copy(a[from+len(new):],a[to:]) - return a[:len(a) + lenDiff] - } else { // same size - copy(a[from:], new) - return a - } -} \ No newline at end of file + return a + } else if lenDiff < 0 { + copy(a[from:], new) + copy(a[from+len(new):], a[to:]) + return a[:len(a)+lenDiff] + } else { // same size + copy(a[from:], new) + return a + } +} diff --git a/byteutils/byteutils_test.go b/byteutils/byteutils_test.go index 5b7bb04..8d242f0 100644 --- a/byteutils/byteutils_test.go +++ b/byteutils/byteutils_test.go @@ -1,32 +1,32 @@ package byteutils import ( - "testing" - "bytes" + "bytes" + "testing" ) func TestCut(t *testing.T) { - if !bytes.Equal(Cut([]byte("123456"), 2, 4), []byte("1256")) { - t.Error("Should properly cut") - } + if !bytes.Equal(Cut([]byte("123456"), 2, 4), []byte("1256")) { + t.Error("Should properly cut") + } } func TestInsert(t *testing.T) { - if !bytes.Equal(Insert([]byte("123456"), 2, []byte("abcd")), []byte("12abcd3456")) { - t.Error("Should insert into middle of slice") - } + if !bytes.Equal(Insert([]byte("123456"), 2, []byte("abcd")), []byte("12abcd3456")) { + t.Error("Should insert into middle of slice") + } } func TestReplace(t *testing.T) { - if !bytes.Equal(Replace([]byte("123456"), 2, 4, []byte("ab")), []byte("12ab56")) { - t.Error("Should replace when same length") - } + if !bytes.Equal(Replace([]byte("123456"), 2, 4, []byte("ab")), []byte("12ab56")) { + t.Error("Should replace when same length") + } - if !bytes.Equal(Replace([]byte("123456"), 2, 4, []byte("abcd")), []byte("12abcd56")) { - t.Error("Should replace when replacement length bigger") - } + if !bytes.Equal(Replace([]byte("123456"), 2, 4, []byte("abcd")), []byte("12abcd56")) { + t.Error("Should replace when replacement length bigger") + } - if !bytes.Equal(Replace([]byte("123456"), 2, 5, []byte("ab")), []byte("12ab6")) { - t.Error("Should replace when replacement length bigger") - } -} \ No newline at end of file + if !bytes.Equal(Replace([]byte("123456"), 2, 5, []byte("ab")), []byte("12ab6")) { + t.Error("Should replace when replacement length bigger") + } +} diff --git a/emitter.go b/emitter.go index a2373f5..2181331 100644 --- a/emitter.go +++ b/emitter.go @@ -40,7 +40,7 @@ func CopyMulty(src io.Reader, writers ...io.Writer) (err error) { } if Settings.debug { - Debug("[EMITTER] Sending paylod:", string(payload)) + Debug("[EMITTER] Sending payload, size:", len(payload), "First 500 bytes:", string(payload[0:500])) } if Settings.splitOutput { diff --git a/http_client.go b/http_client.go index fb0ec74..0743f04 100644 --- a/http_client.go +++ b/http_client.go @@ -4,12 +4,12 @@ import ( "crypto/tls" "github.com/buger/gor/proto" "io" + "log" "net" "net/url" + "runtime/debug" "strings" "time" - "runtime/debug" - "log" ) var defaultPorts = map[string]string{ @@ -103,7 +103,6 @@ func (c *HTTPClient) Send(data []byte) (response []byte, err error) { } }() - if c.conn == nil || !c.isAlive() { Debug("[HTTPClient] Connecting:", c.baseURL) if err = c.Connect(); err != nil { diff --git a/input_raw_test.go b/input_raw_test.go index 477eff8..9b45b95 100644 --- a/input_raw_test.go +++ b/input_raw_test.go @@ -6,6 +6,7 @@ import ( "io/ioutil" "log" "net/http" + "net/http/httptest" "net/http/httputil" "os/exec" "strings" @@ -150,3 +151,61 @@ func TestInputRAWChunkedEncoding(t *testing.T) { close(quit) } + +func TestInputRAWLargePayload(t *testing.T) { + wg := new(sync.WaitGroup) + quit := make(chan int) + + // Generate 100kb file + dd := exec.Command("dd", "if=/dev/urandom", "of=/tmp/large", "bs=1KB", "count=100") + err := dd.Run() + if err != nil { + log.Fatal("dd error:", err) + } + + // Origing and Replay server initialization + origin := startHTTP(func(req *http.Request) { + defer req.Body.Close() + body, _ := ioutil.ReadAll(req.Body) + + if len(body) != 100*1000 { + t.Error("File size should be 1mb:", len(body)) + } + + wg.Done() + }) + origin_address := strings.Replace(origin.Addr().String(), "[::]", "127.0.0.1", -1) + + input := NewRAWInput(origin_address) + + replay := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + req.Body = http.MaxBytesReader(w, req.Body, 1*1024*1024) + buf := make([]byte, 1*1024*1024) + n, _ := req.Body.Read(buf) + body := buf[0:n] + + if len(body) != 100*1000 { + t.Error("File size should be 100000 bytes:", len(body)) + } + + wg.Done() + })) + defer replay.Close() + + http_output := NewHTTPOutput(replay.URL, &HTTPOutputConfig{Debug: false}) + + Plugins.Inputs = []io.Reader{input} + Plugins.Outputs = []io.Writer{http_output} + + go Start(quit) + + wg.Add(2) + curl := exec.Command("curl", "http://"+origin_address, "--header", "Transfer-Encoding: chunked", "--data-binary", "@/tmp/large") + err = curl.Run() + if err != nil { + log.Fatal("curl error:", err) + } + + wg.Wait() + close(quit) +} diff --git a/proto/proto.go b/proto/proto.go index 626d289..ee4a1a0 100644 --- a/proto/proto.go +++ b/proto/proto.go @@ -2,9 +2,9 @@ package proto import ( - "bytes" - "github.com/buger/gor/byteutils" - _ "log" + "bytes" + "github.com/buger/gor/byteutils" + _ "log" ) var CLRF = []byte("\r\n") @@ -13,154 +13,154 @@ var HEADER_DELIM = []byte(": ") // Headers should end with empty line func MIMEHeadersEndPos(payload []byte) int { - return bytes.Index(payload, EMPTY_LINE) + return bytes.Index(payload, EMPTY_LINE) } func MIMEHeadersStartPos(payload []byte) int { - return bytes.Index(payload, CLRF) + 2 // Find first line end + return bytes.Index(payload, CLRF) + 2 // Find first line end } // Find header value or return error // Do not support multi-line headers func Header(payload []byte, name []byte) (value []byte, headerStart, valueStart, headerEnd int) { - headerStart = bytes.Index(payload, name) + headerStart = bytes.Index(payload, name) - if headerStart == -1 { - return - } + if headerStart == -1 { + return + } - valueStart = headerStart + len(name) + 1 // Skip ":" after header name - if payload[valueStart] == ' ' { // Ignore empty space after ':' - valueStart += 1 - } - headerEnd = valueStart + bytes.IndexByte(payload[valueStart:], '\r') - value = payload[valueStart:headerEnd] + valueStart = headerStart + len(name) + 1 // Skip ":" after header name + if payload[valueStart] == ' ' { // Ignore empty space after ':' + valueStart += 1 + } + headerEnd = valueStart + bytes.IndexByte(payload[valueStart:], '\r') + value = payload[valueStart:headerEnd] - return + return } func GetHeader(payload []byte, name string) []byte { - val, _, _, _ := Header(payload, []byte(name)) + val, _, _, _ := Header(payload, []byte(name)) - return val + return val } func SetHeader(payload, name, value []byte) []byte { - _, hs, vs, he := Header(payload, name) + _, hs, vs, he := Header(payload, name) - // If header found - if hs != -1 { - return byteutils.Replace(payload, vs, he, value) - } else { - return AddHeader(payload, name, value) - } + // If header found + if hs != -1 { + return byteutils.Replace(payload, vs, he, value) + } else { + return AddHeader(payload, name, value) + } } func AddHeader(payload, name, value []byte) []byte { - header := make([]byte, len(name) + 2 + len(value) + 2) - copy(header[0:], name) - copy(header[len(name):], HEADER_DELIM) - copy(header[len(name)+2:], value) - copy(header[len(header)-2:], CLRF) + header := make([]byte, len(name)+2+len(value)+2) + copy(header[0:], name) + copy(header[len(name):], HEADER_DELIM) + copy(header[len(name)+2:], value) + copy(header[len(header)-2:], CLRF) - mimeStart := MIMEHeadersStartPos(payload) + mimeStart := MIMEHeadersStartPos(payload) - return byteutils.Insert(payload, mimeStart, header) + return byteutils.Insert(payload, mimeStart, header) } func Path(payload []byte) []byte { - start := bytes.IndexByte(payload, ' ') - start += 1 + start := bytes.IndexByte(payload, ' ') + start += 1 - end := bytes.IndexByte(payload[start:], ' ') + end := bytes.IndexByte(payload[start:], ' ') - return payload[start:start+end] + return payload[start : start+end] } func SetPath(payload, path []byte) []byte { - start := bytes.IndexByte(payload, ' ') - start += 1 + start := bytes.IndexByte(payload, ' ') + start += 1 - end := bytes.IndexByte(payload[start:], ' ') + end := bytes.IndexByte(payload[start:], ' ') - return byteutils.Replace(payload, start, start+end, path) + return byteutils.Replace(payload, start, start+end, path) } func PathParam(payload, name []byte) (value []byte, valueStart, valueEnd int) { - path := Path(payload) + path := Path(payload) - if paramStart := bytes.Index(path, append(name, '=')); paramStart != -1 { - valueStart := paramStart + len(name) + 1 - paramEnd := bytes.IndexByte(path[valueStart:], '&') - if paramEnd == -1 { // It is final param - paramEnd = len(path) - } else { - paramEnd += valueStart - } + if paramStart := bytes.Index(path, append(name, '=')); paramStart != -1 { + valueStart := paramStart + len(name) + 1 + paramEnd := bytes.IndexByte(path[valueStart:], '&') + if paramEnd == -1 { // It is final param + paramEnd = len(path) + } else { + paramEnd += valueStart + } - return path[valueStart:paramEnd], valueStart, paramEnd - } else { - return []byte(""), -1, -1 - } + return path[valueStart:paramEnd], valueStart, paramEnd + } else { + return []byte(""), -1, -1 + } } func SetPathParam(payload, name, value []byte) []byte { - path := Path(payload) - _, vs, ve := PathParam(payload, name) + path := Path(payload) + _, vs, ve := PathParam(payload, name) - if vs != -1 { - newPath := make([]byte, len(path)) - copy(newPath, path) - newPath = byteutils.Replace(newPath, vs, ve, value) + if vs != -1 { + newPath := make([]byte, len(path)) + copy(newPath, path) + newPath = byteutils.Replace(newPath, vs, ve, value) - return SetPath(payload, newPath) - } else { // if param not found append to end of url - // Adding 2 because of '?' or '&' at start, and '=' in middle - newParam := make([]byte, len(name) + len(value) + 2) + return SetPath(payload, newPath) + } else { // if param not found append to end of url + // Adding 2 because of '?' or '&' at start, and '=' in middle + newParam := make([]byte, len(name)+len(value)+2) - if bytes.IndexByte(path, '?') == -1 { - newParam[0] = '?' - } else { - newParam[0] = '&' - } + if bytes.IndexByte(path, '?') == -1 { + newParam[0] = '?' + } else { + newParam[0] = '&' + } - copy(newParam[1:], name) - newParam[1+len(name)] = '=' - copy(newParam[2+len(name):], value) + copy(newParam[1:], name) + newParam[1+len(name)] = '=' + copy(newParam[2+len(name):], value) - newPath := make([]byte, len(path) + len(newParam)) - copy(newPath, path) - copy(newPath[len(path):], newParam) + newPath := make([]byte, len(path)+len(newParam)) + copy(newPath, path) + copy(newPath[len(path):], newParam) - return SetPath(payload, newPath) - } + return SetPath(payload, newPath) + } } func SetHost(payload, url, host []byte) []byte { - // If this is HTTP 1.0 traffic or proxy traffic it may include host right into path variable, so instead of setting Host header we rewrite Path - // Fix for https://github.com/buger/gor/issues/156 - if path := Path(payload); bytes.HasPrefix(path, []byte("http")) { - hostStart := bytes.IndexByte(path, ':') // : position "https?:" - hostStart += 3 // Skip 1 ':' and 2 '\' - hostEnd := hostStart + bytes.IndexByte(path[hostStart:], '/') + // If this is HTTP 1.0 traffic or proxy traffic it may include host right into path variable, so instead of setting Host header we rewrite Path + // Fix for https://github.com/buger/gor/issues/156 + if path := Path(payload); bytes.HasPrefix(path, []byte("http")) { + hostStart := bytes.IndexByte(path, ':') // : position "https?:" + hostStart += 3 // Skip 1 ':' and 2 '\' + hostEnd := hostStart + bytes.IndexByte(path[hostStart:], '/') - newPath := make([]byte, len(path)) - copy(newPath, path) - newPath = byteutils.Replace(newPath, 0, hostEnd, url) + newPath := make([]byte, len(path)) + copy(newPath, path) + newPath = byteutils.Replace(newPath, 0, hostEnd, url) - return SetPath(payload, newPath) - } else { - return SetHeader(payload, []byte("Host"), host) - } + return SetPath(payload, newPath) + } else { + return SetHeader(payload, []byte("Host"), host) + } } func Method(payload []byte) []byte { - end := bytes.IndexByte(payload, ' ') + end := bytes.IndexByte(payload, ' ') - return payload[:end] + return payload[:end] } // Status in response have same position as Path in request func Status(payload []byte) []byte { - return Path(payload) -} \ No newline at end of file + return Path(payload) +} diff --git a/proto/proto_test.go b/proto/proto_test.go index 080f023..f232ee3 100644 --- a/proto/proto_test.go +++ b/proto/proto_test.go @@ -1,152 +1,146 @@ package proto import ( - "testing" - "bytes" + "bytes" + "testing" ) func TestHeader(t *testing.T) { - var payload, val []byte - var headerStart int + var payload, val []byte + var headerStart int - payload = []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") + payload = []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") - if val, _, _, _ = Header(payload, []byte("Content-Length")); !bytes.Equal(val, []byte("7")) { - t.Error("Should find header value") - } + if val, _, _, _ = Header(payload, []byte("Content-Length")); !bytes.Equal(val, []byte("7")) { + t.Error("Should find header value") + } - payload = []byte("POST /post HTTP/1.1\r\nContent-Length:7\r\nHost: www.w3.org\r\n\r\na=1&b=2") + payload = []byte("POST /post HTTP/1.1\r\nContent-Length:7\r\nHost: www.w3.org\r\n\r\na=1&b=2") - if val, _, _, _ = Header(payload, []byte("Content-Length")); !bytes.Equal(val, []byte("7")) { - t.Error("Should find header value without space after :") - } + if val, _, _, _ = Header(payload, []byte("Content-Length")); !bytes.Equal(val, []byte("7")) { + t.Error("Should find header value without space after :") + } - if _, headerStart, _, _ = Header(payload, []byte("Not-Found")); headerStart != -1 { - t.Error("Should not found header") - } + if _, headerStart, _, _ = Header(payload, []byte("Not-Found")); headerStart != -1 { + t.Error("Should not found header") + } } func TestMIMEHeadersEndPos(t *testing.T) { - head := []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org") - payload := []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") + head := []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org") + payload := []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") - end := MIMEHeadersEndPos(payload) + end := MIMEHeadersEndPos(payload) - if !bytes.Equal(payload[:end], head) { - t.Error("Wrong headers end position:", end) - } + if !bytes.Equal(payload[:end], head) { + t.Error("Wrong headers end position:", end) + } } func TestMIMEHeadersStartPos(t *testing.T) { - headers := []byte("Content-Length: 7\r\nHost: www.w3.org") - payload := []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") + headers := []byte("Content-Length: 7\r\nHost: www.w3.org") + payload := []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") - start := MIMEHeadersStartPos(payload) - end := MIMEHeadersEndPos(payload) + start := MIMEHeadersStartPos(payload) + end := MIMEHeadersEndPos(payload) - if !bytes.Equal(payload[start:end], headers) { - t.Error("Wrong headers end position:", start, end) - } + if !bytes.Equal(payload[start:end], headers) { + t.Error("Wrong headers end position:", start, end) + } } func TestSetHeader(t *testing.T) { - var payload, payload_after []byte + var payload, payload_after []byte - payload = []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") - payload_after = []byte("POST /post HTTP/1.1\r\nContent-Length: 14\r\nHost: www.w3.org\r\n\r\na=1&b=2") + payload = []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") + payload_after = []byte("POST /post HTTP/1.1\r\nContent-Length: 14\r\nHost: www.w3.org\r\n\r\na=1&b=2") - if payload = SetHeader(payload, []byte("Content-Length"), []byte("14")); !bytes.Equal(payload, payload_after) { - t.Error("Should update header if it exists", string(payload)) - } + if payload = SetHeader(payload, []byte("Content-Length"), []byte("14")); !bytes.Equal(payload, payload_after) { + t.Error("Should update header if it exists", string(payload)) + } + payload = []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") + payload_after = []byte("POST /post HTTP/1.1\r\nUser-Agent: Gor\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") - payload = []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") - payload_after = []byte("POST /post HTTP/1.1\r\nUser-Agent: Gor\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") - - if payload = SetHeader(payload, []byte("User-Agent"), []byte("Gor")); !bytes.Equal(payload, payload_after) { - t.Error("Should add header if not found", string(payload)) - } + if payload = SetHeader(payload, []byte("User-Agent"), []byte("Gor")); !bytes.Equal(payload, payload_after) { + t.Error("Should add header if not found", string(payload)) + } } func TestPath(t *testing.T) { - var path, payload []byte + var path, payload []byte - payload = []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") + payload = []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") - if path = Path(payload); !bytes.Equal(path, []byte("/post")) { - t.Error("Should find path", string(path)) - } + if path = Path(payload); !bytes.Equal(path, []byte("/post")) { + t.Error("Should find path", string(path)) + } } func TestSetPath(t *testing.T) { - var payload, payload_after []byte + var payload, payload_after []byte - payload = []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") - payload_after = []byte("POST /new_path HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") + payload = []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") + payload_after = []byte("POST /new_path HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") - if payload = SetPath(payload, []byte("/new_path")); !bytes.Equal(payload, payload_after) { - t.Error("Should replace path", string(payload)) - } + if payload = SetPath(payload, []byte("/new_path")); !bytes.Equal(payload, payload_after) { + t.Error("Should replace path", string(payload)) + } } func TestPathParam(t *testing.T) { - var payload []byte + var payload []byte - payload = []byte("POST /post?param=test&user_id=1 HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") + payload = []byte("POST /post?param=test&user_id=1 HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") - if val, _, _ := PathParam(payload, []byte("param")); !bytes.Equal(val, []byte("test")) { - t.Error("Should detect attribute", string(val)) - } + if val, _, _ := PathParam(payload, []byte("param")); !bytes.Equal(val, []byte("test")) { + t.Error("Should detect attribute", string(val)) + } - if val, _, _ := PathParam(payload, []byte("user_id")); !bytes.Equal(val, []byte("1")) { - t.Error("Should detect attribute", string(val)) - } + if val, _, _ := PathParam(payload, []byte("user_id")); !bytes.Equal(val, []byte("1")) { + t.Error("Should detect attribute", string(val)) + } } - func TestSetPathParam(t *testing.T) { - var payload, payload_after []byte + var payload, payload_after []byte - payload = []byte("POST /post?param=test&user_id=1 HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") - payload_after = []byte("POST /post?param=new&user_id=1 HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") + payload = []byte("POST /post?param=test&user_id=1 HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") + payload_after = []byte("POST /post?param=new&user_id=1 HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") - if payload = SetPathParam(payload, []byte("param"), []byte("new")); !bytes.Equal(payload, payload_after) { - t.Error("Should replace existing value", string(payload)) - } + if payload = SetPathParam(payload, []byte("param"), []byte("new")); !bytes.Equal(payload, payload_after) { + t.Error("Should replace existing value", string(payload)) + } - payload = []byte("POST /post?param=test&user_id=1 HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") - payload_after = []byte("POST /post?param=test&user_id=2 HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") + payload = []byte("POST /post?param=test&user_id=1 HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") + payload_after = []byte("POST /post?param=test&user_id=2 HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") - if payload = SetPathParam(payload, []byte("user_id"), []byte("2")); !bytes.Equal(payload, payload_after) { - t.Error("Should replace existing value", string(payload)) - } + if payload = SetPathParam(payload, []byte("user_id"), []byte("2")); !bytes.Equal(payload, payload_after) { + t.Error("Should replace existing value", string(payload)) + } + payload = []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") + payload_after = []byte("POST /post?param=test HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") - payload = []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") - payload_after = []byte("POST /post?param=test HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") + if payload = SetPathParam(payload, []byte("param"), []byte("test")); !bytes.Equal(payload, payload_after) { + t.Error("Should set param if url have no params", string(payload)) + } - if payload = SetPathParam(payload, []byte("param"), []byte("test")); !bytes.Equal(payload, payload_after) { - t.Error("Should set param if url have no params", string(payload)) - } + payload = []byte("POST /post?param=test HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") + payload_after = []byte("POST /post?param=test&user_id=1 HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") - - payload = []byte("POST /post?param=test HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") - payload_after = []byte("POST /post?param=test&user_id=1 HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") - - if payload = SetPathParam(payload, []byte("user_id"), []byte("1")); !bytes.Equal(payload, payload_after) { - t.Error("Should set param at the end if url params", string(payload)) - } + if payload = SetPathParam(payload, []byte("user_id"), []byte("1")); !bytes.Equal(payload, payload_after) { + t.Error("Should set param at the end if url params", string(payload)) + } } - - func TestSetHostHTTP10(t *testing.T) { - var payload, payload_after []byte + var payload, payload_after []byte - payload = []byte("POST http://example.com/post HTTP/1.0\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") - payload_after = []byte("POST http://new.com/post HTTP/1.0\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") + payload = []byte("POST http://example.com/post HTTP/1.0\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") + payload_after = []byte("POST http://new.com/post HTTP/1.0\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") - if payload = SetHost(payload, []byte("http://new.com"), []byte("new.com")); !bytes.Equal(payload, payload_after) { - t.Error("Should replace host", string(payload)) - } -} \ No newline at end of file + if payload = SetHost(payload, []byte("http://new.com"), []byte("new.com")); !bytes.Equal(payload, payload_after) { + t.Error("Should replace host", string(payload)) + } +} diff --git a/raw_socket_listener/listener.go b/raw_socket_listener/listener.go index b5f7150..ef8f950 100644 --- a/raw_socket_listener/listener.go +++ b/raw_socket_listener/listener.go @@ -1,11 +1,13 @@ package raw_socket import ( - "encoding/binary" - "log" - "net" - "strconv" - "bytes" + "bytes" + "encoding/binary" + "log" + "net" + "os" + "strconv" + "syscall" ) // Capture traffic from socket using RAW_SOCKET's @@ -16,113 +18,183 @@ import ( // Since we can't use default TCP libraries RAWTCPLitener implements own TCP layer // TCP packets is parsed using tcp_packet.go, and flow control is managed by tcp_message.go type Listener struct { - messages map[string]*TCPMessage // buffer of TCPMessages waiting to be send + messages map[string]*TCPMessage // buffer of TCPMessages waiting to be send - // Expect: 100-continue request is send in 2 tcp messages - // We store ACK aliases to merge this packets together - ack_aliases map[uint32]uint32 - seq_with_data map[uint32]uint32 + // Expect: 100-continue request is send in 2 tcp messages + // We store ACK aliases to merge this packets together + ack_aliases map[uint32]uint32 + seq_with_data map[uint32]uint32 - c_packets chan *TCPPacket - c_messages chan *TCPMessage // Messages ready to be send to client + c_packets chan *TCPPacket + c_messages chan *TCPMessage // Messages ready to be send to client - c_del_message chan *TCPMessage // Used for notifications about completed or expired messages + c_del_message chan *TCPMessage // Used for notifications about completed or expired messages - addr string // IP to listen - port int // Port to listen + addr string // IP to listen + port int // Port to listen } // RAWTCPListen creates a listener to capture traffic from RAW_SOCKET func NewListener(addr string, port string) (rawListener *Listener) { - rawListener = &Listener{} + rawListener = &Listener{} - rawListener.c_packets = make(chan *TCPPacket, 10000) - rawListener.c_messages = make(chan *TCPMessage, 10000) - rawListener.c_del_message = make(chan *TCPMessage, 10000) + rawListener.c_packets = make(chan *TCPPacket, 10000) + rawListener.c_messages = make(chan *TCPMessage, 10000) + rawListener.c_del_message = make(chan *TCPMessage, 10000) - rawListener.messages = make(map[string]*TCPMessage) - rawListener.ack_aliases = make(map[uint32]uint32) - rawListener.seq_with_data = make(map[uint32]uint32) + rawListener.messages = make(map[string]*TCPMessage) + rawListener.ack_aliases = make(map[uint32]uint32) + rawListener.seq_with_data = make(map[uint32]uint32) - rawListener.addr = addr - rawListener.port, _ = strconv.Atoi(port) + rawListener.addr = addr + rawListener.port, _ = strconv.Atoi(port) - go rawListener.listen() - go rawListener.readRAWSocket() + go rawListener.listen() + go rawListener.readRAWSocket() - return + return } func (t *Listener) listen() { - for { - select { - // If message ready for deletion it means that its also complete or expired by timeout - case message := <-t.c_del_message: - t.c_messages <- message - delete(t.ack_aliases, message.packets[0].Ack) - delete(t.messages, message.ID) + for { + select { + // If message ready for deletion it means that its also complete or expired by timeout + case message := <-t.c_del_message: + t.c_messages <- message + delete(t.ack_aliases, message.Ack) + delete(t.messages, message.ID) - // We need to use channels to process each packet to avoid data races - case packet := <-t.c_packets: - t.processTCPPacket(packet) - } - } + // We need to use channels to process each packet to avoid data races + case packet := <-t.c_packets: + t.processTCPPacket(packet) + } + } +} + +// Taken from http://golang.org/src/net/sock_cloexec.go?h=sysSocket#L16 +func sysSocket(family, sotype, proto int) (int, error) { + s, err := syscall.Socket(family, sotype|syscall.SOCK_NONBLOCK|syscall.SOCK_CLOEXEC, proto) + // On Linux the SOCK_NONBLOCK and SOCK_CLOEXEC flags were + // introduced in 2.6.27 kernel and on FreeBSD both flags were + // introduced in 10 kernel. If we get an EINVAL error on Linux + // or EPROTONOSUPPORT error on FreeBSD, fall back to using + // socket without them. + if err == nil || (err != syscall.EPROTONOSUPPORT && err != syscall.EINVAL) { + return s, err + } + + // See ../syscall/exec_unix.go for description of ForkLock. + syscall.ForkLock.RLock() + s, err = syscall.Socket(family, sotype, proto) + if err == nil { + syscall.CloseOnExec(s) + } + syscall.ForkLock.RUnlock() + if err != nil { + return -1, err + } + if err = syscall.SetNonblock(s, true); err != nil { + syscall.Close(s) + return -1, err + } + return s, nil +} + +func ipToSockaddr(ip net.IP) (syscall.Sockaddr, error) { + if len(ip) == 0 { + ip = net.IPv4zero + } + if ip = ip.To4(); ip == nil { + return nil, net.InvalidAddrError("non-IPv4 address") + } + + sa := new(syscall.SockaddrInet4) + for i := 0; i < net.IPv4len; i++ { + sa.Addr[i] = ip[i] + } + sa.Port = 0 + return sa, nil } func (t *Listener) readRAWSocket() { - conn, e := net.ListenPacket("ip4:tcp", t.addr) + var n int + var sa syscall.Sockaddr + var err error - if e != nil { - log.Fatal(e) - } + addr, _ := net.ResolveIPAddr("ip4", t.addr) + sa, _ = ipToSockaddr(addr.IP) + s, e := sysSocket(syscall.AF_INET, syscall.SOCK_RAW|syscall.SOCK_NONBLOCK|syscall.SOCK_CLOEXEC, syscall.IPPROTO_TCP) - defer conn.Close() + if e != nil { + log.Fatal(e) + } - buf := make([]byte, 4096*10) + if err := syscall.Bind(s, sa); err != nil { + log.Fatal(os.NewSyscallError("bind", err)) + } - for { - // Note: ReadFrom receive messages without IP header - n, addr, err := conn.ReadFrom(buf) + defer syscall.Close(s) - if err != nil { - log.Println("Error:", err) - continue - } + for { + buf := make([]byte, 64*1024) // 64kb - if n > 0 { - t.parsePacket(addr, buf[:n]) - } - } + for { + n, sa, err = syscall.Recvfrom(s, buf, 0) + + if err != nil { + n = 0 + if err == syscall.EAGAIN { + continue + } + } + + break + } + + if err != nil { + log.Println("Error:", err) + continue + } + + if n > 0 { + // Ip header size + hsize := (int(buf[0]) & 0xf) * 4 + + if n > hsize { + go t.parsePacket(sa, buf[hsize:n]) + } + } + + } } -func (t *Listener) parsePacket(addr net.Addr, buf []byte) { - if t.isIncomingDataPacket(buf) { - new_buf := make([]byte, len(buf)) - copy(new_buf, buf) +func (t *Listener) parsePacket(sa syscall.Sockaddr, buf []byte) { + addr := &net.IPAddr{IP: sa.(*syscall.SockaddrInet4).Addr[0:]} - t.c_packets <- ParseTCPPacket(addr, new_buf) - } + if t.isIncomingDataPacket(buf) { + t.c_packets <- ParseTCPPacket(addr, buf) + } } func (t *Listener) isIncomingDataPacket(buf []byte) bool { - // To avoid full packet parsing every time, we manually parsing values needed for packet filtering - // http://en.wikipedia.org/wiki/Transmission_Control_Protocol - dest_port := binary.BigEndian.Uint16(buf[2:4]) + // To avoid full packet parsing every time, we manually parsing values needed for packet filtering + // http://en.wikipedia.org/wiki/Transmission_Control_Protocol + dest_port := binary.BigEndian.Uint16(buf[2:4]) - // Because RAW_SOCKET can't be bound to port, we have to control it by ourself - if int(dest_port) == t.port { - // Get the 'data offset' (size of the TCP header in 32-bit words) - dataOffset := (buf[12] & 0xF0) >> 4 + // Because RAW_SOCKET can't be bound to port, we have to control it by ourself + if int(dest_port) == t.port { + // Get the 'data offset' (size of the TCP header in 32-bit words) + dataOffset := (buf[12] & 0xF0) >> 4 - // We need only packets with data inside - // Check that the buffer is larger than the size of the TCP header - if len(buf) > int(dataOffset*4) { - // We should create new buffer because go slices is pointers. So buffer data shoud be immutable. - return true - } - } + // We need only packets with data inside + // Check that the buffer is larger than the size of the TCP header + if len(buf) > int(dataOffset*4) { + // We should create new buffer because go slices is pointers. So buffer data shoud be immutable. + return true + } + } - return false + return false } var bExpect100ContinueCheck = []byte("Expect: 100-continue") @@ -132,44 +204,44 @@ var bPOST = []byte("POST") // // For TCP message unique id is Acknowledgment number (see tcp_packet.go) func (t *Listener) processTCPPacket(packet *TCPPacket) { - defer func() { recover() }() + defer func() { recover() }() - var message *TCPMessage + var message *TCPMessage - parent_message_ack, parent_ok := t.seq_with_data[packet.Seq] - if parent_ok { - t.ack_aliases[packet.Ack] = parent_message_ack - delete(t.seq_with_data, packet.Seq) - } + parent_message_ack, parent_ok := t.seq_with_data[packet.Seq] + if parent_ok { + t.ack_aliases[packet.Ack] = parent_message_ack + delete(t.seq_with_data, packet.Seq) + } - ack_alias, alias_ok := t.ack_aliases[packet.Ack] - if alias_ok { - packet.Ack = ack_alias - } + ack_alias, alias_ok := t.ack_aliases[packet.Ack] + if alias_ok { + packet.Ack = ack_alias + } - m_id := packet.Addr.String() + strconv.Itoa(int(packet.Ack)) - message, ok := t.messages[m_id] + m_id := packet.Addr.String() + strconv.Itoa(int(packet.Ack)) + message, ok := t.messages[m_id] - if !ok { - // We sending c_del_message channel, so message object can communicate with Listener and notify it if message completed - message = NewTCPMessage(m_id, t.c_del_message) - t.messages[m_id] = message - } + if !ok { + // We sending c_del_message channel, so message object can communicate with Listener and notify it if message completed + message = NewTCPMessage(m_id, t.c_del_message, packet.Ack) + t.messages[m_id] = message + } - if bytes.Equal(packet.Data[0:4], bPOST) { - if bytes.Equal(packet.Data[len(packet.Data)-24:len(packet.Data)-4], bExpect100ContinueCheck) { - t.seq_with_data[packet.Seq + uint32(len(packet.Data))] = packet.Ack + if bytes.Equal(packet.Data[0:4], bPOST) { + if bytes.Equal(packet.Data[len(packet.Data)-24:len(packet.Data)-4], bExpect100ContinueCheck) { + t.seq_with_data[packet.Seq+uint32(len(packet.Data))] = packet.Ack - // Removing `Expect: 100-continue` header - packet.Data = append(packet.Data[:len(packet.Data)-24], packet.Data[len(packet.Data)-2:]...) - } - } + // Removing `Expect: 100-continue` header + packet.Data = append(packet.Data[:len(packet.Data)-24], packet.Data[len(packet.Data)-2:]...) + } + } - // Adding packet to message - message.c_packets <- packet + // Adding packet to message + message.c_packets <- packet } // Receive TCP messages from the listener channel func (t *Listener) Receive() *TCPMessage { - return <-t.c_messages + return <-t.c_messages } diff --git a/raw_socket_listener/tcp_message.go b/raw_socket_listener/tcp_message.go index 69c4b93..32fdaae 100644 --- a/raw_socket_listener/tcp_message.go +++ b/raw_socket_listener/tcp_message.go @@ -1,9 +1,9 @@ package raw_socket import ( - "log" - "sort" - "time" + "log" + "sort" + "time" ) const MSG_EXPIRE = 2000 * time.Millisecond @@ -15,91 +15,92 @@ const MSG_EXPIRE = 2000 * time.Millisecond // Message can be compiled from unique packets with same message_id which sorted by sequence // Message is received if we didn't receive any packets for 2000ms type TCPMessage struct { - ID string // Message ID - packets []*TCPPacket + ID string // Message ID + Ack uint32 + packets []*TCPPacket - timer *time.Timer // Used for expire check + timer *time.Timer // Used for expire check - c_packets chan *TCPPacket + c_packets chan *TCPPacket - c_del_message chan *TCPMessage + c_del_message chan *TCPMessage } // NewTCPMessage pointer created from a Acknowledgment number and a channel of messages readuy to be deleted -func NewTCPMessage(ID string, c_del chan *TCPMessage) (msg *TCPMessage) { - msg = &TCPMessage{ID: ID} +func NewTCPMessage(ID string, c_del chan *TCPMessage, Ack uint32) (msg *TCPMessage) { + msg = &TCPMessage{ID: ID, Ack: Ack} - msg.c_packets = make(chan *TCPPacket) - msg.c_del_message = c_del // used for notifying that message completed or expired + msg.c_packets = make(chan *TCPPacket) + msg.c_del_message = c_del // used for notifying that message completed or expired - // Every time we receive packet we reset this timer - msg.timer = time.AfterFunc(MSG_EXPIRE, msg.Timeout) + // Every time we receive packet we reset this timer + msg.timer = time.AfterFunc(MSG_EXPIRE, msg.Timeout) - go msg.listen() + go msg.listen() - return + return } func (t *TCPMessage) listen() { - for { - select { - case packet, more := <-t.c_packets: - if more { - t.AddPacket(packet) - } else { - // Stop loop if channel closed - return - } - } - } + for { + select { + case packet, more := <-t.c_packets: + if more { + t.AddPacket(packet) + } else { + // Stop loop if channel closed + return + } + } + } } // Timeout notifies message to stop listening, close channel and message ready to be sent func (t *TCPMessage) Timeout() { - select { - // In some cases Timeout can be called multiple times (do not know how yet) - // Ensure that we did not close channel 2 times - case packet, ok := <- t.c_packets: - if ok { - t.AddPacket(packet) - } else { - return - } - default: - close(t.c_packets) - t.c_del_message <- t // Notify RAWListener that message is ready to be send to replay server - } + select { + // In some cases Timeout can be called multiple times (do not know how yet) + // Ensure that we did not close channel 2 times + case packet, ok := <-t.c_packets: + if ok { + t.AddPacket(packet) + } else { + return + } + default: + close(t.c_packets) + t.c_del_message <- t // Notify RAWListener that message is ready to be send to replay server + } } // Bytes sorts packets in right orders and return message content func (t *TCPMessage) Bytes() (output []byte) { - sort.Sort(BySeq(t.packets)) + sort.Sort(BySeq(t.packets)) - for _, v := range t.packets { - output = append(output, v.Data...) - } + for _, v := range t.packets { + output = append(output, v.Data...) + } - return output + return output } // AddPacket to the message and ensure packet uniqueness // TCP allows that packet can be re-send multiple times func (t *TCPMessage) AddPacket(packet *TCPPacket) { - packetFound := false + packetFound := false - for _, pkt := range t.packets { - if packet.Seq == pkt.Seq { - packetFound = true - break - } - } + for _, pkt := range t.packets { + if packet.Seq == pkt.Seq { + packetFound = true + break + } + } - if packetFound { - log.Println("Received packet with same sequence") - } else { - t.packets = append(t.packets, packet) - } + if packetFound { + log.Println("Received packet with same sequence") + } else { + t.packets = append(t.packets, packet) + } - // Reset message timeout timer - t.timer.Reset(MSG_EXPIRE) + // Reset message timeout timer + t.timer.Reset(MSG_EXPIRE) } diff --git a/raw_socket_listener/tcp_packet.go b/raw_socket_listener/tcp_packet.go index 5af8a73..393c0e5 100644 --- a/raw_socket_listener/tcp_packet.go +++ b/raw_socket_listener/tcp_packet.go @@ -1,101 +1,101 @@ package raw_socket import ( - "encoding/binary" - "net" - "strconv" - "strings" + "encoding/binary" + "net" + "strconv" + "strings" ) // TCP Flags const ( - TCP_FIN = 1 << iota - TCP_SYN - TCP_RST - TCP_PSH - TCP_ACK - TCP_URG - TCP_ECE - TCP_CWR - TCP_NS + TCP_FIN = 1 << iota + TCP_SYN + TCP_RST + TCP_PSH + TCP_ACK + TCP_URG + TCP_ECE + TCP_CWR + TCP_NS ) // Simple TCP packet parser // // Packet structure: http://en.wikipedia.org/wiki/Transmission_Control_Protocol type TCPPacket struct { - SrcPort uint16 - DestPort uint16 - Seq uint32 - Ack uint32 - DataOffset uint8 - Flags uint16 - Window uint16 - Checksum uint16 - Urgent uint16 + SrcPort uint16 + DestPort uint16 + Seq uint32 + Ack uint32 + DataOffset uint8 + Flags uint16 + Window uint16 + Checksum uint16 + Urgent uint16 - Data []byte + Data []byte - Addr net.Addr + Addr net.Addr } func ParseTCPPacket(addr net.Addr, b []byte) (p *TCPPacket) { - p = &TCPPacket{Data: b} - p.ParseBasic() - p.Addr = addr + p = &TCPPacket{Data: b} + p.ParseBasic() + p.Addr = addr - return p + return p } // Parse TCP Packet, inspired by: https://github.com/miekg/pcap/blob/master/packet.go func (t *TCPPacket) Parse() { - t.ParseBasic() - t.SrcPort = binary.BigEndian.Uint16(t.Data[0:2]) - t.DestPort = binary.BigEndian.Uint16(t.Data[2:4]) - t.Flags = binary.BigEndian.Uint16(t.Data[12:14]) & 0x1FF - t.Window = binary.BigEndian.Uint16(t.Data[14:16]) - t.Checksum = binary.BigEndian.Uint16(t.Data[16:18]) - t.Urgent = binary.BigEndian.Uint16(t.Data[18:20]) + t.ParseBasic() + t.SrcPort = binary.BigEndian.Uint16(t.Data[0:2]) + t.DestPort = binary.BigEndian.Uint16(t.Data[2:4]) + t.Flags = binary.BigEndian.Uint16(t.Data[12:14]) & 0x1FF + t.Window = binary.BigEndian.Uint16(t.Data[14:16]) + t.Checksum = binary.BigEndian.Uint16(t.Data[16:18]) + t.Urgent = binary.BigEndian.Uint16(t.Data[18:20]) } // ParseBasic set of fields func (t *TCPPacket) ParseBasic() { - t.Seq = binary.BigEndian.Uint32(t.Data[4:8]) - t.Ack = binary.BigEndian.Uint32(t.Data[8:12]) - t.DataOffset = (t.Data[12] & 0xF0) >> 4 + t.Seq = binary.BigEndian.Uint32(t.Data[4:8]) + t.Ack = binary.BigEndian.Uint32(t.Data[8:12]) + t.DataOffset = (t.Data[12] & 0xF0) >> 4 - t.Data = t.Data[t.DataOffset*4:] + t.Data = t.Data[t.DataOffset*4:] } // String output for a TCP Packet func (t *TCPPacket) String() string { - return strings.Join([]string{ - "Source port: " + strconv.Itoa(int(t.SrcPort)), - "Dest port:" + strconv.Itoa(int(t.DestPort)), - "Sequence:" + strconv.Itoa(int(t.Seq)), - "Acknowledgment:" + strconv.Itoa(int(t.Ack)), - "Header len:" + strconv.Itoa(int(t.DataOffset)), + return strings.Join([]string{ + "Source port: " + strconv.Itoa(int(t.SrcPort)), + "Dest port:" + strconv.Itoa(int(t.DestPort)), + "Sequence:" + strconv.Itoa(int(t.Seq)), + "Acknowledgment:" + strconv.Itoa(int(t.Ack)), + "Header len:" + strconv.Itoa(int(t.DataOffset)), - "Flag ns:" + strconv.FormatBool(t.Flags&TCP_NS != 0), - "Flag crw:" + strconv.FormatBool(t.Flags&TCP_CWR != 0), - "Flag ece:" + strconv.FormatBool(t.Flags&TCP_ECE != 0), - "Flag urg:" + strconv.FormatBool(t.Flags&TCP_URG != 0), - "Flag ack:" + strconv.FormatBool(t.Flags&TCP_ACK != 0), - "Flag psh:" + strconv.FormatBool(t.Flags&TCP_PSH != 0), - "Flag rst:" + strconv.FormatBool(t.Flags&TCP_RST != 0), - "Flag syn:" + strconv.FormatBool(t.Flags&TCP_SYN != 0), - "Flag fin:" + strconv.FormatBool(t.Flags&TCP_FIN != 0), + "Flag ns:" + strconv.FormatBool(t.Flags&TCP_NS != 0), + "Flag crw:" + strconv.FormatBool(t.Flags&TCP_CWR != 0), + "Flag ece:" + strconv.FormatBool(t.Flags&TCP_ECE != 0), + "Flag urg:" + strconv.FormatBool(t.Flags&TCP_URG != 0), + "Flag ack:" + strconv.FormatBool(t.Flags&TCP_ACK != 0), + "Flag psh:" + strconv.FormatBool(t.Flags&TCP_PSH != 0), + "Flag rst:" + strconv.FormatBool(t.Flags&TCP_RST != 0), + "Flag syn:" + strconv.FormatBool(t.Flags&TCP_SYN != 0), + "Flag fin:" + strconv.FormatBool(t.Flags&TCP_FIN != 0), - "Window size:" + strconv.Itoa(int(t.Window)), - "Checksum:" + strconv.Itoa(int(t.Checksum)), + "Window size:" + strconv.Itoa(int(t.Window)), + "Checksum:" + strconv.Itoa(int(t.Checksum)), - "Data size:" + strconv.Itoa(len(t.Data)), - "Data:" + string(t.Data), - }, "\n") + "Data size:" + strconv.Itoa(len(t.Data)), + "Data:" + string(t.Data), + }, "\n") } type BySeq []*TCPPacket -func (a BySeq) Len() int { return len(a) } -func (a BySeq) Swap(i, j int) { a[i], a[j] = a[j], a[i] } -func (a BySeq) Less(i, j int) bool { return a[i].Seq < a[j].Seq } +func (a BySeq) Len() int { return len(a) } +func (a BySeq) Swap(i, j int) { a[i], a[j] = a[j], a[i] } +func (a BySeq) Less(i, j int) bool { return a[i].Seq < a[j].Seq } diff --git a/settings.go b/settings.go index 62f089d..31b4989 100644 --- a/settings.go +++ b/settings.go @@ -25,7 +25,7 @@ func (h *MultiOption) Set(value string) error { type AppSettings struct { verbose bool - debug bool + debug bool stats bool splitOutput bool diff --git a/test_input.go b/test_input.go index 2140728..8f62344 100644 --- a/test_input.go +++ b/test_input.go @@ -35,14 +35,14 @@ func (i *TestInput) EmitChunkedPOST() { i.data <- []byte("POST /pub/WWW/ HTTP/1.1\r\nHost: www.w3.org\r\nTransfer-Encoding: chunked\r\n\r\n4\r\nWiki\r\n5\r\npedia\r\ne\r\n in\r\n\r\nchunks.\r\n0\r\n\r\n") } -func (i *TestInput) EmitFile() { - size := 1024 * 1024 // 1 MB +func (i *TestInput) EmitLargePOST() { + size := 5 * 1024 * 1024 // 5 MB rb := make([]byte, size) rand.Read(rb) rs := base64.URLEncoding.EncodeToString(rb) - i.data <- []byte("POST /pub/WWW/ HTTP/1.1\nHost: www.w3.org\r\n\r\n------WebKitFormBoundaryePkpFF7tjBAqx29L\nContent-Disposition: form-data; name=\"uploadedfile\"; filename=\"hello.o\"\nContent-Type: application/x-object\n" + rs + "\r\n\r\n") + i.data <- []byte("POST / HTTP/1.1\nHost: www.w3.org\nContent-Length:5242880\r\n\r\n" + rs) } func (i *TestInput) EmitOPTIONS() { From 313c382adb8449472c7c7df20a9bf1cb70a3743b Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Sat, 11 Jul 2015 09:58:10 +0500 Subject: [PATCH 04/26] By default use all cores --- gor.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/gor.go b/gor.go index 150a990..c3c30bb 100644 --- a/gor.go +++ b/gor.go @@ -9,6 +9,7 @@ import ( "os" "runtime/debug" "runtime/pprof" + "runtime" "time" ) @@ -19,6 +20,11 @@ var ( ) func main() { + // If not set via env cariable + if len(os.Getenv("GOMAXPROCS")) == 0 { + runtime.GOMAXPROCS(runtime.NumCPU() * 2) + } + // Don't exit on panic defer func() { if r := recover(); r != nil { From d0873d14536aafa5005392ea804a973261b3fa77 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Sat, 11 Jul 2015 18:36:00 +0500 Subject: [PATCH 05/26] Try to add select --- Makefile | 2 +- http_client.go | 6 +++++ input_raw_test.go | 5 +++-- raw_socket_listener/listener.go | 39 ++++++++++++++++++++++++++++----- 4 files changed, 43 insertions(+), 9 deletions(-) diff --git a/Makefile b/Makefile index 40fff87..574ddb4 100644 --- a/Makefile +++ b/Makefile @@ -14,7 +14,7 @@ dbuild: docker build -t gor . dtest: - docker run -v `pwd`:$(SOURCE_PATH) -t -i --env GORACE="halt_on_error=1" gor go test ./... $(ARGS) -race -v -timeout 15s + docker run -v `pwd`:$(SOURCE_PATH) -t -i --env GORACE="halt_on_error=1" gor go test ./... $(ARGS) -race -v -timeout 5s dcover: docker run -v `pwd`:$(SOURCE_PATH) -t -i --env GORACE="halt_on_error=1" gor go test $(ARGS) -race -v -timeout 15s -coverprofile=coverage.out diff --git a/http_client.go b/http_client.go index 0743f04..dd6a739 100644 --- a/http_client.go +++ b/http_client.go @@ -162,3 +162,9 @@ func (c *HTTPClient) Send(data []byte) (response []byte, err error) { return payload, err } + +func (c *HTTPClient) Get(path string) (response []byte, err error) { + payload := "GET " + path + " HTTP/1.1\r\n\r\n" + + return c.Send([]byte(payload)) +} \ No newline at end of file diff --git a/input_raw_test.go b/input_raw_test.go index 9b45b95..e3ccb16 100644 --- a/input_raw_test.go +++ b/input_raw_test.go @@ -31,12 +31,13 @@ func TestRAWInput(t *testing.T) { address := strings.Replace(listener.Addr().String(), "[::]", "127.0.0.1", -1) + client := NewHTTPClient(address, &HTTPClientConfig{}) + go Start(quit) for i := 0; i < 100; i++ { wg.Add(1) - res, _ := http.Get("http://" + address) - res.Body.Close() + client.Get("/") } wg.Wait() diff --git a/raw_socket_listener/listener.go b/raw_socket_listener/listener.go index ef8f950..6e148b1 100644 --- a/raw_socket_listener/listener.go +++ b/raw_socket_listener/listener.go @@ -8,6 +8,7 @@ import ( "os" "strconv" "syscall" + "time" ) // Capture traffic from socket using RAW_SOCKET's @@ -60,6 +61,7 @@ func (t *Listener) listen() { select { // If message ready for deletion it means that its also complete or expired by timeout case message := <-t.c_del_message: + log.Println("Sending message, len:", len(message.packets)) t.c_messages <- message delete(t.ack_aliases, message.Ack) delete(t.messages, message.ID) @@ -116,34 +118,58 @@ func ipToSockaddr(ip net.IP) (syscall.Sockaddr, error) { return sa, nil } +func FD_SET(p *syscall.FdSet, i int) { + p.Bits[i/64] |= 1 << uint(i) % 64 +} + +func FD_ISSET(p *syscall.FdSet, i int) bool { + return (p.Bits[i/64] & (1 << uint(i) % 64)) != 0 +} + +func FD_ZERO(p *syscall.FdSet) { + for i := range p.Bits { + p.Bits[i] = 0 + } +} + func (t *Listener) readRAWSocket() { + var err error var n int var sa syscall.Sockaddr - var err error addr, _ := net.ResolveIPAddr("ip4", t.addr) sa, _ = ipToSockaddr(addr.IP) - s, e := sysSocket(syscall.AF_INET, syscall.SOCK_RAW|syscall.SOCK_NONBLOCK|syscall.SOCK_CLOEXEC, syscall.IPPROTO_TCP) + fd, e := sysSocket(syscall.AF_INET, syscall.SOCK_RAW|syscall.SOCK_NONBLOCK|syscall.SOCK_CLOEXEC, syscall.IPPROTO_TCP) + syscall.SetsockoptInt(fd, syscall.SOL_SOCKET, syscall.SO_BROADCAST, 1) + + syscall.SetNonblock(fd, true) if e != nil { log.Fatal(e) } - if err := syscall.Bind(s, sa); err != nil { + if err := syscall.Bind(fd, sa); err != nil { log.Fatal(os.NewSyscallError("bind", err)) } - defer syscall.Close(s) + defer syscall.Close(fd) + + rfds := &syscall.FdSet{} + timeout := syscall.NsecToTimeval(time.Second.Nanoseconds()) for { buf := make([]byte, 64*1024) // 64kb for { - n, sa, err = syscall.Recvfrom(s, buf, 0) + if _, err := syscall.Select(fd, rfds, nil, nil, &timeout); err != nil { + log.Fatal("Error", e) + } + + n, sa, err = syscall.Recvfrom(fd, buf, 0) if err != nil { - n = 0 if err == syscall.EAGAIN { + n = 0 continue } } @@ -172,6 +198,7 @@ func (t *Listener) parsePacket(sa syscall.Sockaddr, buf []byte) { addr := &net.IPAddr{IP: sa.(*syscall.SockaddrInet4).Addr[0:]} if t.isIncomingDataPacket(buf) { + log.Println("Received packet:", len(buf)) t.c_packets <- ParseTCPPacket(addr, buf) } } From bb74704f9d410d96118b8453492ca99cf3b5796d Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Sat, 11 Jul 2015 18:46:35 +0500 Subject: [PATCH 06/26] Rollback to use net package --- input_raw_test.go | 10 +-- raw_socket_listener/listener.go | 146 +++++--------------------------- 2 files changed, 26 insertions(+), 130 deletions(-) diff --git a/input_raw_test.go b/input_raw_test.go index e3ccb16..a45e5e8 100644 --- a/input_raw_test.go +++ b/input_raw_test.go @@ -157,8 +157,8 @@ func TestInputRAWLargePayload(t *testing.T) { wg := new(sync.WaitGroup) quit := make(chan int) - // Generate 100kb file - dd := exec.Command("dd", "if=/dev/urandom", "of=/tmp/large", "bs=1KB", "count=100") + // Generate 200kb file + dd := exec.Command("dd", "if=/dev/urandom", "of=/tmp/large", "bs=1KB", "count=200") err := dd.Run() if err != nil { log.Fatal("dd error:", err) @@ -169,7 +169,7 @@ func TestInputRAWLargePayload(t *testing.T) { defer req.Body.Close() body, _ := ioutil.ReadAll(req.Body) - if len(body) != 100*1000 { + if len(body) != 200*1000 { t.Error("File size should be 1mb:", len(body)) } @@ -185,8 +185,8 @@ func TestInputRAWLargePayload(t *testing.T) { n, _ := req.Body.Read(buf) body := buf[0:n] - if len(body) != 100*1000 { - t.Error("File size should be 100000 bytes:", len(body)) + if len(body) != 200*1000 { + t.Error("File size should be 200000 bytes:", len(body)) } wg.Done() diff --git a/raw_socket_listener/listener.go b/raw_socket_listener/listener.go index 6e148b1..507964c 100644 --- a/raw_socket_listener/listener.go +++ b/raw_socket_listener/listener.go @@ -5,10 +5,7 @@ import ( "encoding/binary" "log" "net" - "os" "strconv" - "syscall" - "time" ) // Capture traffic from socket using RAW_SOCKET's @@ -61,7 +58,6 @@ func (t *Listener) listen() { select { // If message ready for deletion it means that its also complete or expired by timeout case message := <-t.c_del_message: - log.Println("Sending message, len:", len(message.packets)) t.c_messages <- message delete(t.ack_aliases, message.Ack) delete(t.messages, message.ID) @@ -72,135 +68,35 @@ func (t *Listener) listen() { } } } - -// Taken from http://golang.org/src/net/sock_cloexec.go?h=sysSocket#L16 -func sysSocket(family, sotype, proto int) (int, error) { - s, err := syscall.Socket(family, sotype|syscall.SOCK_NONBLOCK|syscall.SOCK_CLOEXEC, proto) - // On Linux the SOCK_NONBLOCK and SOCK_CLOEXEC flags were - // introduced in 2.6.27 kernel and on FreeBSD both flags were - // introduced in 10 kernel. If we get an EINVAL error on Linux - // or EPROTONOSUPPORT error on FreeBSD, fall back to using - // socket without them. - if err == nil || (err != syscall.EPROTONOSUPPORT && err != syscall.EINVAL) { - return s, err - } - - // See ../syscall/exec_unix.go for description of ForkLock. - syscall.ForkLock.RLock() - s, err = syscall.Socket(family, sotype, proto) - if err == nil { - syscall.CloseOnExec(s) - } - syscall.ForkLock.RUnlock() - if err != nil { - return -1, err - } - if err = syscall.SetNonblock(s, true); err != nil { - syscall.Close(s) - return -1, err - } - return s, nil -} - -func ipToSockaddr(ip net.IP) (syscall.Sockaddr, error) { - if len(ip) == 0 { - ip = net.IPv4zero - } - if ip = ip.To4(); ip == nil { - return nil, net.InvalidAddrError("non-IPv4 address") - } - - sa := new(syscall.SockaddrInet4) - for i := 0; i < net.IPv4len; i++ { - sa.Addr[i] = ip[i] - } - sa.Port = 0 - return sa, nil -} - -func FD_SET(p *syscall.FdSet, i int) { - p.Bits[i/64] |= 1 << uint(i) % 64 -} - -func FD_ISSET(p *syscall.FdSet, i int) bool { - return (p.Bits[i/64] & (1 << uint(i) % 64)) != 0 -} - -func FD_ZERO(p *syscall.FdSet) { - for i := range p.Bits { - p.Bits[i] = 0 - } -} - func (t *Listener) readRAWSocket() { - var err error - var n int - var sa syscall.Sockaddr + conn, e := net.ListenPacket("ip4:tcp", t.addr) - addr, _ := net.ResolveIPAddr("ip4", t.addr) - sa, _ = ipToSockaddr(addr.IP) - fd, e := sysSocket(syscall.AF_INET, syscall.SOCK_RAW|syscall.SOCK_NONBLOCK|syscall.SOCK_CLOEXEC, syscall.IPPROTO_TCP) - syscall.SetsockoptInt(fd, syscall.SOL_SOCKET, syscall.SO_BROADCAST, 1) + if e != nil { + log.Fatal(e) + } - syscall.SetNonblock(fd, true) + defer conn.Close() - if e != nil { - log.Fatal(e) - } + for { + buf := make([]byte, 64*1024) // 64kb + // Note: ReadFrom receive messages without IP header + n, addr, err := conn.ReadFrom(buf) - if err := syscall.Bind(fd, sa); err != nil { - log.Fatal(os.NewSyscallError("bind", err)) - } + if err != nil { + log.Println("Error:", err) + continue + } - defer syscall.Close(fd) - - rfds := &syscall.FdSet{} - timeout := syscall.NsecToTimeval(time.Second.Nanoseconds()) - - for { - buf := make([]byte, 64*1024) // 64kb - - for { - if _, err := syscall.Select(fd, rfds, nil, nil, &timeout); err != nil { - log.Fatal("Error", e) - } - - n, sa, err = syscall.Recvfrom(fd, buf, 0) - - if err != nil { - if err == syscall.EAGAIN { - n = 0 - continue - } - } - - break - } - - if err != nil { - log.Println("Error:", err) - continue - } - - if n > 0 { - // Ip header size - hsize := (int(buf[0]) & 0xf) * 4 - - if n > hsize { - go t.parsePacket(sa, buf[hsize:n]) - } - } - - } + if n > 0 { + go t.parsePacket(addr, buf[:n]) + } + } } -func (t *Listener) parsePacket(sa syscall.Sockaddr, buf []byte) { - addr := &net.IPAddr{IP: sa.(*syscall.SockaddrInet4).Addr[0:]} - - if t.isIncomingDataPacket(buf) { - log.Println("Received packet:", len(buf)) - t.c_packets <- ParseTCPPacket(addr, buf) - } +func (t *Listener) parsePacket(addr net.Addr, buf []byte) { + if t.isIncomingDataPacket(buf) { + t.c_packets <- ParseTCPPacket(addr, buf) + } } func (t *Listener) isIncomingDataPacket(buf []byte) bool { From 34fe137d198140a50ad48ad3bae48ab6837df3fc Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Sat, 11 Jul 2015 18:54:41 +0500 Subject: [PATCH 07/26] Improve message_id to handle multiple ports --- raw_socket_listener/listener.go | 2 +- raw_socket_listener/tcp_packet.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/raw_socket_listener/listener.go b/raw_socket_listener/listener.go index 507964c..dce1dc0 100644 --- a/raw_socket_listener/listener.go +++ b/raw_socket_listener/listener.go @@ -142,7 +142,7 @@ func (t *Listener) processTCPPacket(packet *TCPPacket) { packet.Ack = ack_alias } - m_id := packet.Addr.String() + strconv.Itoa(int(packet.Ack)) + m_id := packet.Addr.String() + strconv.Itoa(int(packet.SrcPort)) + strconv.Itoa(int(packet.Ack)) message, ok := t.messages[m_id] if !ok { diff --git a/raw_socket_listener/tcp_packet.go b/raw_socket_listener/tcp_packet.go index 393c0e5..b07109b 100644 --- a/raw_socket_listener/tcp_packet.go +++ b/raw_socket_listener/tcp_packet.go @@ -50,7 +50,6 @@ func ParseTCPPacket(addr net.Addr, b []byte) (p *TCPPacket) { // Parse TCP Packet, inspired by: https://github.com/miekg/pcap/blob/master/packet.go func (t *TCPPacket) Parse() { t.ParseBasic() - t.SrcPort = binary.BigEndian.Uint16(t.Data[0:2]) t.DestPort = binary.BigEndian.Uint16(t.Data[2:4]) t.Flags = binary.BigEndian.Uint16(t.Data[12:14]) & 0x1FF t.Window = binary.BigEndian.Uint16(t.Data[14:16]) @@ -60,6 +59,7 @@ func (t *TCPPacket) Parse() { // ParseBasic set of fields func (t *TCPPacket) ParseBasic() { + t.SrcPort = binary.BigEndian.Uint16(t.Data[0:2]) t.Seq = binary.BigEndian.Uint32(t.Data[4:8]) t.Ack = binary.BigEndian.Uint32(t.Data[8:12]) t.DataOffset = (t.Data[12] & 0xF0) >> 4 From 6ba0bfc614c3ce500453dee322b7b46bc0602993 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Sat, 11 Jul 2015 18:57:00 +0500 Subject: [PATCH 08/26] Fix tests runner --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 574ddb4..40fff87 100644 --- a/Makefile +++ b/Makefile @@ -14,7 +14,7 @@ dbuild: docker build -t gor . dtest: - docker run -v `pwd`:$(SOURCE_PATH) -t -i --env GORACE="halt_on_error=1" gor go test ./... $(ARGS) -race -v -timeout 5s + docker run -v `pwd`:$(SOURCE_PATH) -t -i --env GORACE="halt_on_error=1" gor go test ./... $(ARGS) -race -v -timeout 15s dcover: docker run -v `pwd`:$(SOURCE_PATH) -t -i --env GORACE="halt_on_error=1" gor go test $(ARGS) -race -v -timeout 15s -coverprofile=coverage.out From 32d0557a28c5482718ba296e2d6f0fc1860713ef Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Sat, 11 Jul 2015 19:07:35 +0500 Subject: [PATCH 09/26] Reduce test file size --- input_raw_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/input_raw_test.go b/input_raw_test.go index a45e5e8..2f8652b 100644 --- a/input_raw_test.go +++ b/input_raw_test.go @@ -158,7 +158,7 @@ func TestInputRAWLargePayload(t *testing.T) { quit := make(chan int) // Generate 200kb file - dd := exec.Command("dd", "if=/dev/urandom", "of=/tmp/large", "bs=1KB", "count=200") + dd := exec.Command("dd", "if=/dev/urandom", "of=/tmp/large", "bs=1KB", "count=100") err := dd.Run() if err != nil { log.Fatal("dd error:", err) @@ -169,7 +169,7 @@ func TestInputRAWLargePayload(t *testing.T) { defer req.Body.Close() body, _ := ioutil.ReadAll(req.Body) - if len(body) != 200*1000 { + if len(body) != 100*1000 { t.Error("File size should be 1mb:", len(body)) } @@ -185,8 +185,8 @@ func TestInputRAWLargePayload(t *testing.T) { n, _ := req.Body.Read(buf) body := buf[0:n] - if len(body) != 200*1000 { - t.Error("File size should be 200000 bytes:", len(body)) + if len(body) != 100*1000 { + t.Error("File size should be 100000 bytes:", len(body)) } wg.Done() From db7f5c6a9cd3b3aed742bcfdd87c30e6bfd88892 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Sat, 11 Jul 2015 19:34:20 +0500 Subject: [PATCH 10/26] Update to use go 1.4.2 --- .travis.yml | 2 +- Dockerfile | 2 +- settings.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index d896eb4..a236b6a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,3 +1,3 @@ language: go -go: 1.3.3 +go: 1.4.2 script: sudo -E bash -c "source /etc/profile && eval '$(gimme 1.4)' && export GOPATH=$HOME/gopath:$GOPATH && go get && GORACE='halt_on_error=1' go test -race -v" diff --git a/Dockerfile b/Dockerfile index 3fecf83..82af609 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM google/golang +FROM google/golang:1.4 RUN cd /goroot/src/ && GOOS=linux GOARCH=386 ./make.bash --no-clean diff --git a/settings.go b/settings.go index 31b4989..8c345e7 100644 --- a/settings.go +++ b/settings.go @@ -8,7 +8,7 @@ import ( ) const ( - VERSION = "0.9.7" + VERSION = "0.9.8" ) // Allows to specify multiple flags with same name and collects all values to array From becb4b958f64f7f81d5624fdc4e860a5bfc2dfe5 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Sun, 12 Jul 2015 08:47:39 +0500 Subject: [PATCH 11/26] Delete CHANGELOG.md --- CHANGELOG.md | 25 ------------------------- 1 file changed, 25 deletions(-) delete mode 100644 CHANGELOG.md diff --git a/CHANGELOG.md b/CHANGELOG.md deleted file mode 100644 index 85b2aca..0000000 --- a/CHANGELOG.md +++ /dev/null @@ -1,25 +0,0 @@ -v0.7.0 - 31 Oct 2013 -* New modular architecture. Listener and Replay functionality merged. -* Added option to equally split traffic between multiple outputs: --split-output true -* Saving requests to file and replaying from it -* Injecting custom headers to http requests -* Advanced stats using ElasticSearch - -v0.3.5 - 15 Sep 2013 -* Significantly improved test coverage -* Fixed bug with redirect replay https://github.com/buger/gor/pull/15 -* Added limit on listener side -* Improved stability (catch and log panic, instead of exiting) -* Added License file - -v0.3.3 - 22 Jun 2013 -* Using TCP instead of UDP for communication between Listener and Replay -* Significantly improved performance -* Fixed bugs causing locking and message dropping (concurrency issues) -* Rewrote concurrency model to use more channels - -v0.3 - 10 Jun 2013 -* Use RAW_SOCKETS instead of tcpdump -* Own TCP stack -* All HTTP request types support -* Simplified request parsing From f1c341e2acef4a45f0a9290546e045583a4feec2 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Sun, 12 Jul 2015 08:52:59 +0500 Subject: [PATCH 12/26] Update README.md --- README.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index e12e2fb..37f2148 100644 --- a/README.md +++ b/README.md @@ -349,6 +349,10 @@ Basic idea is that SSL was made to protect itself from traffic interception. The More can be find here: https://github.com/buger/gor/issues/85 +# Is there a limit for size of HTTP request when using output-http? +Due to the fact that Gor can't guarantee interception of all packets, for large payloads > 200kb there is chance of missing some packets and corrupting body. Treat it as a feature and chance to test broken bodies handling :) +The only way to guarantee delivery is using `--input-http`, but you will miss some features. + ### I'm getting 'too many open files' error Typical linux shell has a small open files soft limit at 1024. You can easily raise that when you do this before starting your gor replay process: @@ -397,4 +401,4 @@ To achieve the top most performance you should tune the source server system lim * [3SCALE](http://www.3scale.net/) * [Optionlab](http://www.opinionlab.com) * [TubeMogul] (http://tubemogul.com) -* To add your company drop me a line to github.com/buger or leonsbox@gmail.com \ No newline at end of file +* To add your company drop me a line to github.com/buger or leonsbox@gmail.com From a5f2d2fd6d21e1896e13bbe41660757b4231afac Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Sun, 12 Jul 2015 08:53:59 +0500 Subject: [PATCH 13/26] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 37f2148..7f37f9d 100644 --- a/README.md +++ b/README.md @@ -349,7 +349,7 @@ Basic idea is that SSL was made to protect itself from traffic interception. The More can be find here: https://github.com/buger/gor/issues/85 -# Is there a limit for size of HTTP request when using output-http? +### Is there a limit for size of HTTP request when using output-http? Due to the fact that Gor can't guarantee interception of all packets, for large payloads > 200kb there is chance of missing some packets and corrupting body. Treat it as a feature and chance to test broken bodies handling :) The only way to guarantee delivery is using `--input-http`, but you will miss some features. From 0551b19ddd770349383c6d18369c5795cdb02046 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Sun, 12 Jul 2015 19:08:12 +0500 Subject: [PATCH 14/26] Add linter task --- Dockerfile | 1 + Makefile | 8 +++++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 82af609..f09d7f1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,4 +6,5 @@ WORKDIR /gopath/src/github.com/buger/gor/ ADD . /gopath/src/github.com/buger/gor/ +RUN go get -u github.com/golang/lint/golint RUN go get \ No newline at end of file diff --git a/Makefile b/Makefile index 40fff87..2f8b95b 100644 --- a/Makefile +++ b/Makefile @@ -13,8 +13,14 @@ release-x86: dbuild: docker build -t gor . +dlint: + docker run -v `pwd`:$(SOURCE_PATH) -t -i --env GORACE="halt_on_error=1" gor golint $(ARGS) + +drace: + docker run -v `pwd`:$(SOURCE_PATH) -t -i --env GORACE="halt_on_error=1" gor go test ./... $(ARGS) -v -race -timeout 15s + dtest: - docker run -v `pwd`:$(SOURCE_PATH) -t -i --env GORACE="halt_on_error=1" gor go test ./... $(ARGS) -race -v -timeout 15s + docker run -v `pwd`:$(SOURCE_PATH) -t -i gor go test ./... $(ARGS) -v -timeout 15s dcover: docker run -v `pwd`:$(SOURCE_PATH) -t -i --env GORACE="halt_on_error=1" gor go test $(ARGS) -race -v -timeout 15s -coverprofile=coverage.out From 6647481fe07061bc90be6db55611dd34179d7c6c Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Mon, 13 Jul 2015 09:31:11 +0500 Subject: [PATCH 15/26] Improve style and docs for proto package --- Makefile | 2 +- elasticsearch.go | 30 +++---- gor.go | 2 +- http_client.go | 4 +- http_modifier.go | 8 +- input_raw_test.go | 2 +- proto/proto.go | 134 +++++++++++++++++++----------- proto/proto_test.go | 46 +++++----- raw_socket_listener/listener.go | 40 ++++----- raw_socket_listener/tcp_packet.go | 2 +- 10 files changed, 154 insertions(+), 116 deletions(-) diff --git a/Makefile b/Makefile index 2f8b95b..cb4bb17 100644 --- a/Makefile +++ b/Makefile @@ -14,7 +14,7 @@ dbuild: docker build -t gor . dlint: - docker run -v `pwd`:$(SOURCE_PATH) -t -i --env GORACE="halt_on_error=1" gor golint $(ARGS) + docker run -v `pwd`:$(SOURCE_PATH) -t -i --env GORACE="halt_on_error=1" gor golint $(PKG) drace: docker run -v `pwd`:$(SOURCE_PATH) -t -i --env GORACE="halt_on_error=1" gor go test ./... $(ARGS) -v -race -timeout 15s diff --git a/elasticsearch.go b/elasticsearch.go index 801e75c..a930ac3 100644 --- a/elasticsearch.go +++ b/elasticsearch.go @@ -122,24 +122,24 @@ func (p *ESPlugin) ResponseAnalyze(req, resp []byte, start, stop time.Time) { esResp := ESRequestResponse{ ReqUrl: proto.Path(req), ReqMethod: proto.Method(req), - ReqUserAgent: proto.GetHeader(req, "User-Agent"), - ReqAcceptLanguage: proto.GetHeader(req, "Accept-Language"), - ReqAccept: proto.GetHeader(req, "Accept"), - ReqAcceptEncoding: proto.GetHeader(req, "Accept-Encoding"), - ReqIfModifiedSince: proto.GetHeader(req, "If-Modified-Since"), - ReqConnection: proto.GetHeader(req, "Connection"), - ReqCookies: proto.GetHeader(req, "Cookie"), + ReqUserAgent: proto.Header(req, []byte("User-Agent")), + ReqAcceptLanguage: proto.Header(req, []byte("Accept-Language")), + ReqAccept: proto.Header(req, []byte("Accept")), + ReqAcceptEncoding: proto.Header(req, []byte("Accept-Encoding")), + ReqIfModifiedSince: proto.Header(req, []byte("If-Modified-Since")), + ReqConnection: proto.Header(req, []byte("Connection")), + ReqCookies: proto.Header(req, []byte("Cookie")), RespStatus: proto.Status(resp), RespStatusCode: proto.Status(resp), RespProto: proto.Method(resp), - RespContentLength: proto.GetHeader(resp, "Content-Length"), - RespContentType: proto.GetHeader(resp, "Content-Type"), - RespTransferEncoding: proto.GetHeader(resp, "Transfer-Encoding"), - RespContentEncoding: proto.GetHeader(resp, "Content-Encoding"), - RespExpires: proto.GetHeader(resp, "Expires"), - RespCacheControl: proto.GetHeader(resp, "Cache-Control"), - RespVary: proto.GetHeader(resp, "Vary"), - RespSetCookie: proto.GetHeader(resp, "Set-Cookie"), + RespContentLength: proto.Header(resp, []byte("Content-Length")), + RespContentType: proto.Header(resp, []byte("Content-Type")), + RespTransferEncoding: proto.Header(resp, []byte("Transfer-Encoding")), + RespContentEncoding: proto.Header(resp, []byte("Content-Encoding")), + RespExpires: proto.Header(resp, []byte("Expires")), + RespCacheControl: proto.Header(resp, []byte("Cache-Control")), + RespVary: proto.Header(resp, []byte("Vary")), + RespSetCookie: proto.Header(resp, []byte("Set-Cookie")), Rtt: rtt, Timestamp: t, } diff --git a/gor.go b/gor.go index c3c30bb..af9bf5f 100644 --- a/gor.go +++ b/gor.go @@ -7,9 +7,9 @@ import ( "fmt" "log" "os" + "runtime" "runtime/debug" "runtime/pprof" - "runtime" "time" ) diff --git a/http_client.go b/http_client.go index dd6a739..c54e6a8 100644 --- a/http_client.go +++ b/http_client.go @@ -147,7 +147,7 @@ func (c *HTTPClient) Send(data []byte) (response []byte, err error) { if status[0] == '3' { c.redirectsCount += 1 - location, _, _, _ := proto.Header(payload, []byte("Location")) + location := proto.Header(payload, []byte("Location")) redirectPayload := []byte("GET " + string(location) + " HTTP/1.1\r\n\r\n") if c.config.Debug { @@ -167,4 +167,4 @@ func (c *HTTPClient) Get(path string) (response []byte, err error) { payload := "GET " + path + " HTTP/1.1\r\n\r\n" return c.Send([]byte(payload)) -} \ No newline at end of file +} diff --git a/http_modifier.go b/http_modifier.go index d820df6..96be68b 100644 --- a/http_modifier.go +++ b/http_modifier.go @@ -86,9 +86,9 @@ func (m *HTTPModifier) Rewrite(payload []byte) (response []byte) { if len(m.config.headerFilters) > 0 { for _, f := range m.config.headerFilters { - value, s, _, _ := proto.Header(payload, f.name) + value := proto.Header(payload, f.name) - if s != -1 && !f.regexp.Match(value) { + if len(value) > 0 && !f.regexp.Match(value) { return } } @@ -96,9 +96,9 @@ func (m *HTTPModifier) Rewrite(payload []byte) (response []byte) { if len(m.config.headerHashFilters) > 0 { for _, f := range m.config.headerHashFilters { - value, s, _, _ := proto.Header(payload, f.name) + value := proto.Header(payload, f.name) - if s != -1 { + if len(value) > 0 { hasher := fnv.New32a() hasher.Write(value) diff --git a/input_raw_test.go b/input_raw_test.go index 2f8652b..6262667 100644 --- a/input_raw_test.go +++ b/input_raw_test.go @@ -31,7 +31,7 @@ func TestRAWInput(t *testing.T) { address := strings.Replace(listener.Addr().String(), "[::]", "127.0.0.1", -1) - client := NewHTTPClient(address, &HTTPClientConfig{}) + client := NewHTTPClient(address, &HTTPClientConfig{}) go Start(quit) diff --git a/proto/proto.go b/proto/proto.go index ee4a1a0..f595234 100644 --- a/proto/proto.go +++ b/proto/proto.go @@ -1,28 +1,50 @@ -// Low-level interaction with HTTP request payload +/* +Package proto provides byte-level interaction with HTTP request payload. + +Example of HTTP payload for future references, new line symbols escaped: + + POST /upload HTTP/1.1\r\n + User-Agent: Gor\r\n + Content-Length: 11\r\n + \r\n + Hello world + + GET /index.html HTTP/1.1\r\n + User-Agent: Gor\r\n + \r\n + \r\n +*/ package proto import ( "bytes" "github.com/buger/gor/byteutils" - _ "log" ) +// In HTTP newline defined by 2 bytes (for both windows and *nix support) var CLRF = []byte("\r\n") -var EMPTY_LINE = []byte("\r\n\r\n") -var HEADER_DELIM = []byte(": ") -// Headers should end with empty line +// New line acts as separator: end of Headers or Body (in some cases) +var EmptyLine = []byte("\r\n\r\n") + +// Separator for Header line. Header looks like: `HeaderName: value` +var HeaderDelim = []byte(": ") + +// MIMEHeadersEndPos finds end of the Headers section, which should end with empty line. func MIMEHeadersEndPos(payload []byte) int { - return bytes.Index(payload, EMPTY_LINE) + return bytes.Index(payload, EmptyLine) } +// MIMEHeadersStartPos finds start of Headers section +// It just finds position of second line (first contains location and method). func MIMEHeadersStartPos(payload []byte) int { return bytes.Index(payload, CLRF) + 2 // Find first line end } -// Find header value or return error -// Do not support multi-line headers -func Header(payload []byte, name []byte) (value []byte, headerStart, valueStart, headerEnd int) { +// header return value and positions of header/value start/end. +// If not found, value will be blank, and headerStart will be -1 +// Do not support multi-line headers. +func header(payload []byte, name []byte) (value []byte, headerStart, valueStart, headerEnd int) { headerStart = bytes.Index(payload, name) if headerStart == -1 { @@ -31,7 +53,7 @@ func Header(payload []byte, name []byte) (value []byte, headerStart, valueStart, valueStart = headerStart + len(name) + 1 // Skip ":" after header name if payload[valueStart] == ' ' { // Ignore empty space after ':' - valueStart += 1 + valueStart++ } headerEnd = valueStart + bytes.IndexByte(payload[valueStart:], '\r') value = payload[valueStart:headerEnd] @@ -39,27 +61,32 @@ func Header(payload []byte, name []byte) (value []byte, headerStart, valueStart, return } -func GetHeader(payload []byte, name string) []byte { - val, _, _, _ := Header(payload, []byte(name)) +// Header returns header value, if header not found, value will be blank +func Header(payload, name []byte) []byte { + val, _, _, _ := header(payload, name) return val } +// SetHeader sets header value. If header not found it creates new one. +// Returns modified request payload func SetHeader(payload, name, value []byte) []byte { - _, hs, vs, he := Header(payload, name) + _, hs, vs, he := header(payload, name) - // If header found if hs != -1 { + // If header found we just repace its value return byteutils.Replace(payload, vs, he, value) - } else { - return AddHeader(payload, name, value) } + + return AddHeader(payload, name, value) } +// AddHeader takes http payload and appends new header to the start of headers section +// Returns modified request payload func AddHeader(payload, name, value []byte) []byte { header := make([]byte, len(name)+2+len(value)+2) copy(header[0:], name) - copy(header[len(name):], HEADER_DELIM) + copy(header[len(name):], HeaderDelim) copy(header[len(name)+2:], value) copy(header[len(header)-2:], CLRF) @@ -68,30 +95,31 @@ func AddHeader(payload, name, value []byte) []byte { return byteutils.Insert(payload, mimeStart, header) } +// Path takes payload and retuns request path: Split(firstLine, ' ')[1] func Path(payload []byte) []byte { - start := bytes.IndexByte(payload, ' ') - start += 1 - + start := bytes.IndexByte(payload, ' ') + 1 end := bytes.IndexByte(payload[start:], ' ') return payload[start : start+end] } +// SetPath takes payload, sets new path and returns modified payload func SetPath(payload, path []byte) []byte { - start := bytes.IndexByte(payload, ' ') - start += 1 - + start := bytes.IndexByte(payload, ' ') + 1 end := bytes.IndexByte(payload[start:], ' ') return byteutils.Replace(payload, start, start+end, path) } +// PathParam returns URL query attribute by given name, if no found: valueStart will be -1 func PathParam(payload, name []byte) (value []byte, valueStart, valueEnd int) { path := Path(payload) if paramStart := bytes.Index(path, append(name, '=')); paramStart != -1 { valueStart := paramStart + len(name) + 1 paramEnd := bytes.IndexByte(path[valueStart:], '&') + + // Param can end with '&' (another param), or end of line if paramEnd == -1 { // It is final param paramEnd = len(path) } else { @@ -99,43 +127,51 @@ func PathParam(payload, name []byte) (value []byte, valueStart, valueEnd int) { } return path[valueStart:paramEnd], valueStart, paramEnd - } else { - return []byte(""), -1, -1 } + + return []byte(""), -1, -1 } +// SetPathParam takes payload and updates path Query attribute +// If query param not found, it will append new +// Returns modified payload func SetPathParam(payload, name, value []byte) []byte { path := Path(payload) _, vs, ve := PathParam(payload, name) - if vs != -1 { + if vs != -1 { // If param found, replace its value and set new Path newPath := make([]byte, len(path)) copy(newPath, path) newPath = byteutils.Replace(newPath, vs, ve, value) - return SetPath(payload, newPath) - } else { // if param not found append to end of url - // Adding 2 because of '?' or '&' at start, and '=' in middle - newParam := make([]byte, len(name)+len(value)+2) - - if bytes.IndexByte(path, '?') == -1 { - newParam[0] = '?' - } else { - newParam[0] = '&' - } - - copy(newParam[1:], name) - newParam[1+len(name)] = '=' - copy(newParam[2+len(name):], value) - - newPath := make([]byte, len(path)+len(newParam)) - copy(newPath, path) - copy(newPath[len(path):], newParam) - return SetPath(payload, newPath) } + + // if param not found append to end of url + // Adding 2 because of '?' or '&' at start, and '=' in middle + newParam := make([]byte, len(name)+len(value)+2) + + if bytes.IndexByte(path, '?') == -1 { + newParam[0] = '?' + } else { + newParam[0] = '&' + } + + // Copy "param=value" into buffer, after it looks like "?param=value" + copy(newParam[1:], name) + newParam[1+len(name)] = '=' + copy(newParam[2+len(name):], value) + + // Append param to the end of path + newPath := make([]byte, len(path)+len(newParam)) + copy(newPath, path) + copy(newPath[len(path):], newParam) + + return SetPath(payload, newPath) } +// SetHost updates Host header for HTTP/1.1 or updates host in path for HTTP/1.0 or Proxy requests +// Returns modified payload func SetHost(payload, url, host []byte) []byte { // If this is HTTP 1.0 traffic or proxy traffic it may include host right into path variable, so instead of setting Host header we rewrite Path // Fix for https://github.com/buger/gor/issues/156 @@ -149,18 +185,20 @@ func SetHost(payload, url, host []byte) []byte { newPath = byteutils.Replace(newPath, 0, hostEnd, url) return SetPath(payload, newPath) - } else { - return SetHeader(payload, []byte("Host"), host) } + + return SetHeader(payload, []byte("Host"), host) } +// Method returns HTTP method func Method(payload []byte) []byte { end := bytes.IndexByte(payload, ' ') return payload[:end] } -// Status in response have same position as Path in request +// Status returns response status. +// It happend to be in same position as request payload path func Status(payload []byte) []byte { return Path(payload) } diff --git a/proto/proto_test.go b/proto/proto_test.go index f232ee3..80adbe2 100644 --- a/proto/proto_test.go +++ b/proto/proto_test.go @@ -11,17 +11,17 @@ func TestHeader(t *testing.T) { payload = []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") - if val, _, _, _ = Header(payload, []byte("Content-Length")); !bytes.Equal(val, []byte("7")) { + if val = Header(payload, []byte("Content-Length")); !bytes.Equal(val, []byte("7")) { t.Error("Should find header value") } payload = []byte("POST /post HTTP/1.1\r\nContent-Length:7\r\nHost: www.w3.org\r\n\r\na=1&b=2") - if val, _, _, _ = Header(payload, []byte("Content-Length")); !bytes.Equal(val, []byte("7")) { + if val = Header(payload, []byte("Content-Length")); !bytes.Equal(val, []byte("7")) { t.Error("Should find header value without space after :") } - if _, headerStart, _, _ = Header(payload, []byte("Not-Found")); headerStart != -1 { + if _, headerStart, _, _ = header(payload, []byte("Not-Found")); headerStart != -1 { t.Error("Should not found header") } } @@ -50,19 +50,19 @@ func TestMIMEHeadersStartPos(t *testing.T) { } func TestSetHeader(t *testing.T) { - var payload, payload_after []byte + var payload, payloadAfter []byte payload = []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") - payload_after = []byte("POST /post HTTP/1.1\r\nContent-Length: 14\r\nHost: www.w3.org\r\n\r\na=1&b=2") + payloadAfter = []byte("POST /post HTTP/1.1\r\nContent-Length: 14\r\nHost: www.w3.org\r\n\r\na=1&b=2") - if payload = SetHeader(payload, []byte("Content-Length"), []byte("14")); !bytes.Equal(payload, payload_after) { + if payload = SetHeader(payload, []byte("Content-Length"), []byte("14")); !bytes.Equal(payload, payloadAfter) { t.Error("Should update header if it exists", string(payload)) } payload = []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") - payload_after = []byte("POST /post HTTP/1.1\r\nUser-Agent: Gor\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") + payloadAfter = []byte("POST /post HTTP/1.1\r\nUser-Agent: Gor\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") - if payload = SetHeader(payload, []byte("User-Agent"), []byte("Gor")); !bytes.Equal(payload, payload_after) { + if payload = SetHeader(payload, []byte("User-Agent"), []byte("Gor")); !bytes.Equal(payload, payloadAfter) { t.Error("Should add header if not found", string(payload)) } } @@ -78,12 +78,12 @@ func TestPath(t *testing.T) { } func TestSetPath(t *testing.T) { - var payload, payload_after []byte + var payload, payloadAfter []byte payload = []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") - payload_after = []byte("POST /new_path HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") + payloadAfter = []byte("POST /new_path HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") - if payload = SetPath(payload, []byte("/new_path")); !bytes.Equal(payload, payload_after) { + if payload = SetPath(payload, []byte("/new_path")); !bytes.Equal(payload, payloadAfter) { t.Error("Should replace path", string(payload)) } } @@ -103,44 +103,44 @@ func TestPathParam(t *testing.T) { } func TestSetPathParam(t *testing.T) { - var payload, payload_after []byte + var payload, payloadAfter []byte payload = []byte("POST /post?param=test&user_id=1 HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") - payload_after = []byte("POST /post?param=new&user_id=1 HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") + payloadAfter = []byte("POST /post?param=new&user_id=1 HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") - if payload = SetPathParam(payload, []byte("param"), []byte("new")); !bytes.Equal(payload, payload_after) { + if payload = SetPathParam(payload, []byte("param"), []byte("new")); !bytes.Equal(payload, payloadAfter) { t.Error("Should replace existing value", string(payload)) } payload = []byte("POST /post?param=test&user_id=1 HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") - payload_after = []byte("POST /post?param=test&user_id=2 HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") + payloadAfter = []byte("POST /post?param=test&user_id=2 HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") - if payload = SetPathParam(payload, []byte("user_id"), []byte("2")); !bytes.Equal(payload, payload_after) { + if payload = SetPathParam(payload, []byte("user_id"), []byte("2")); !bytes.Equal(payload, payloadAfter) { t.Error("Should replace existing value", string(payload)) } payload = []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") - payload_after = []byte("POST /post?param=test HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") + payloadAfter = []byte("POST /post?param=test HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") - if payload = SetPathParam(payload, []byte("param"), []byte("test")); !bytes.Equal(payload, payload_after) { + if payload = SetPathParam(payload, []byte("param"), []byte("test")); !bytes.Equal(payload, payloadAfter) { t.Error("Should set param if url have no params", string(payload)) } payload = []byte("POST /post?param=test HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") - payload_after = []byte("POST /post?param=test&user_id=1 HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") + payloadAfter = []byte("POST /post?param=test&user_id=1 HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") - if payload = SetPathParam(payload, []byte("user_id"), []byte("1")); !bytes.Equal(payload, payload_after) { + if payload = SetPathParam(payload, []byte("user_id"), []byte("1")); !bytes.Equal(payload, payloadAfter) { t.Error("Should set param at the end if url params", string(payload)) } } func TestSetHostHTTP10(t *testing.T) { - var payload, payload_after []byte + var payload, payloadAfter []byte payload = []byte("POST http://example.com/post HTTP/1.0\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") - payload_after = []byte("POST http://new.com/post HTTP/1.0\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") + payloadAfter = []byte("POST http://new.com/post HTTP/1.0\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") - if payload = SetHost(payload, []byte("http://new.com"), []byte("new.com")); !bytes.Equal(payload, payload_after) { + if payload = SetHost(payload, []byte("http://new.com"), []byte("new.com")); !bytes.Equal(payload, payloadAfter) { t.Error("Should replace host", string(payload)) } } diff --git a/raw_socket_listener/listener.go b/raw_socket_listener/listener.go index dce1dc0..9fd0c65 100644 --- a/raw_socket_listener/listener.go +++ b/raw_socket_listener/listener.go @@ -69,34 +69,34 @@ func (t *Listener) listen() { } } func (t *Listener) readRAWSocket() { - conn, e := net.ListenPacket("ip4:tcp", t.addr) + conn, e := net.ListenPacket("ip4:tcp", t.addr) - if e != nil { - log.Fatal(e) - } + if e != nil { + log.Fatal(e) + } - defer conn.Close() + defer conn.Close() - for { - buf := make([]byte, 64*1024) // 64kb - // Note: ReadFrom receive messages without IP header - n, addr, err := conn.ReadFrom(buf) + for { + buf := make([]byte, 64*1024) // 64kb + // Note: ReadFrom receive messages without IP header + n, addr, err := conn.ReadFrom(buf) - if err != nil { - log.Println("Error:", err) - continue - } + if err != nil { + log.Println("Error:", err) + continue + } - if n > 0 { - go t.parsePacket(addr, buf[:n]) - } - } + if n > 0 { + go t.parsePacket(addr, buf[:n]) + } + } } func (t *Listener) parsePacket(addr net.Addr, buf []byte) { - if t.isIncomingDataPacket(buf) { - t.c_packets <- ParseTCPPacket(addr, buf) - } + if t.isIncomingDataPacket(buf) { + t.c_packets <- ParseTCPPacket(addr, buf) + } } func (t *Listener) isIncomingDataPacket(buf []byte) bool { diff --git a/raw_socket_listener/tcp_packet.go b/raw_socket_listener/tcp_packet.go index b07109b..4d3f4e6 100644 --- a/raw_socket_listener/tcp_packet.go +++ b/raw_socket_listener/tcp_packet.go @@ -59,7 +59,7 @@ func (t *TCPPacket) Parse() { // ParseBasic set of fields func (t *TCPPacket) ParseBasic() { - t.SrcPort = binary.BigEndian.Uint16(t.Data[0:2]) + t.SrcPort = binary.BigEndian.Uint16(t.Data[0:2]) t.Seq = binary.BigEndian.Uint32(t.Data[4:8]) t.Ack = binary.BigEndian.Uint32(t.Data[8:12]) t.DataOffset = (t.Data[12] & 0xF0) >> 4 From 32081b2d1791d41204b783e24993693858f3d7b0 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Mon, 13 Jul 2015 09:36:54 +0500 Subject: [PATCH 16/26] Improve byteutils --- byteutils/byteutils.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/byteutils/byteutils.go b/byteutils/byteutils.go index c65bbdb..f1c1e15 100644 --- a/byteutils/byteutils.go +++ b/byteutils/byteutils.go @@ -1,5 +1,7 @@ +// Package byteutils probvides helpers for working with byte slices package byteutils +// Cut elements from slice for a given range func Cut(a []byte, from, to int) []byte { copy(a[from:], a[to:]) a = a[:len(a)-to+from] @@ -7,6 +9,7 @@ func Cut(a []byte, from, to int) []byte { return a } +// Insert new slice at specified position func Insert(a []byte, i int, b []byte) []byte { a = append(a, make([]byte, len(b))...) copy(a[i+len(b):], a[i:]) @@ -15,7 +18,7 @@ func Insert(a []byte, i int, b []byte) []byte { return a } -// Unlike bytes.Replace it allows you to specify range +// Replace function unlike bytes.Replace allows you to specify range func Replace(a []byte, from, to int, new []byte) []byte { lenDiff := len(new) - (to - from) From 0859642b386271d757f8e2dd972ddcb45e7c90ad Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Mon, 13 Jul 2015 16:19:24 +0500 Subject: [PATCH 17/26] Update raw listener --- raw_socket_listener/listener.go | 97 +++++++++++++++++------------- raw_socket_listener/tcp_message.go | 29 ++++----- raw_socket_listener/tcp_packet.go | 50 +++++++-------- 3 files changed, 95 insertions(+), 81 deletions(-) diff --git a/raw_socket_listener/listener.go b/raw_socket_listener/listener.go index 9fd0c65..29534c2 100644 --- a/raw_socket_listener/listener.go +++ b/raw_socket_listener/listener.go @@ -1,4 +1,16 @@ -package raw_socket +/* +Package rawSocket provides traffic sniffier using RAW sockets. + +Capture traffic from socket using RAW_SOCKET's +http://en.wikipedia.org/wiki/Raw_socket + +RAW_SOCKET allow you listen for traffic on any port (e.g. sniffing) because they operate on IP level. + +Ports is TCP feature, same as flow control, reliable transmission and etc. + +This package implements own TCP layer: TCP packets is parsed using tcp_packet.go, and flow control is managed by tcp_message.go +*/ +package rawSocket import ( "bytes" @@ -8,41 +20,41 @@ import ( "strconv" ) -// Capture traffic from socket using RAW_SOCKET's -// http://en.wikipedia.org/wiki/Raw_socket -// -// RAW_SOCKET allow you listen for traffic on any port (e.g. sniffing) because they operate on IP level. -// Ports is TCP feature, same as flow control, reliable transmission and etc. -// Since we can't use default TCP libraries RAWTCPLitener implements own TCP layer -// TCP packets is parsed using tcp_packet.go, and flow control is managed by tcp_message.go +// Listener handle traffic capture type Listener struct { - messages map[string]*TCPMessage // buffer of TCPMessages waiting to be send + // buffer of TCPMessages waiting to be send + messages map[string]*TCPMessage // Expect: 100-continue request is send in 2 tcp messages // We store ACK aliases to merge this packets together - ack_aliases map[uint32]uint32 - seq_with_data map[uint32]uint32 + ackAliases map[uint32]uint32 + // To get ACK of second message we need to compute its Seq and wait for them message + seqWithData map[uint32]uint32 - c_packets chan *TCPPacket - c_messages chan *TCPMessage // Messages ready to be send to client + // Messages ready to be send to client + packetsChan chan *TCPPacket - c_del_message chan *TCPMessage // Used for notifications about completed or expired messages + // Messages ready to be send to client + messagesChan chan *TCPMessage + + // Used for notifications about completed or expired messages + messageDelChan chan *TCPMessage addr string // IP to listen port int // Port to listen } -// RAWTCPListen creates a listener to capture traffic from RAW_SOCKET +// NewListener creates and initializes new Listener object func NewListener(addr string, port string) (rawListener *Listener) { rawListener = &Listener{} - rawListener.c_packets = make(chan *TCPPacket, 10000) - rawListener.c_messages = make(chan *TCPMessage, 10000) - rawListener.c_del_message = make(chan *TCPMessage, 10000) + rawListener.packetsChan = make(chan *TCPPacket, 10000) + rawListener.messagesChan = make(chan *TCPMessage, 10000) + rawListener.messageDelChan = make(chan *TCPMessage, 10000) rawListener.messages = make(map[string]*TCPMessage) - rawListener.ack_aliases = make(map[uint32]uint32) - rawListener.seq_with_data = make(map[uint32]uint32) + rawListener.ackAliases = make(map[uint32]uint32) + rawListener.seqWithData = make(map[uint32]uint32) rawListener.addr = addr rawListener.port, _ = strconv.Atoi(port) @@ -57,13 +69,14 @@ func (t *Listener) listen() { for { select { // If message ready for deletion it means that its also complete or expired by timeout - case message := <-t.c_del_message: - t.c_messages <- message - delete(t.ack_aliases, message.Ack) + case message := <-t.messageDelChan: + log.Println("MESSAGE") + t.messagesChan <- message + delete(t.ackAliases, message.Ack) delete(t.messages, message.ID) // We need to use channels to process each packet to avoid data races - case packet := <-t.c_packets: + case packet := <-t.packetsChan: t.processTCPPacket(packet) } } @@ -94,18 +107,20 @@ func (t *Listener) readRAWSocket() { } func (t *Listener) parsePacket(addr net.Addr, buf []byte) { + log.Println("PACKET") + if t.isIncomingDataPacket(buf) { - t.c_packets <- ParseTCPPacket(addr, buf) + t.packetsChan <- ParseTCPPacket(addr, buf) } } func (t *Listener) isIncomingDataPacket(buf []byte) bool { // To avoid full packet parsing every time, we manually parsing values needed for packet filtering // http://en.wikipedia.org/wiki/Transmission_Control_Protocol - dest_port := binary.BigEndian.Uint16(buf[2:4]) + destPort := binary.BigEndian.Uint16(buf[2:4]) // Because RAW_SOCKET can't be bound to port, we have to control it by ourself - if int(dest_port) == t.port { + if int(destPort) == t.port { // Get the 'data offset' (size of the TCP header in 32-bit words) dataOffset := (buf[12] & 0xF0) >> 4 @@ -131,29 +146,27 @@ func (t *Listener) processTCPPacket(packet *TCPPacket) { var message *TCPMessage - parent_message_ack, parent_ok := t.seq_with_data[packet.Seq] - if parent_ok { - t.ack_aliases[packet.Ack] = parent_message_ack - delete(t.seq_with_data, packet.Seq) + if parentAck, ok := t.seqWithData[packet.Seq]; ok { + t.ackAliases[packet.Ack] = parentAck + delete(t.seqWithData, packet.Seq) } - ack_alias, alias_ok := t.ack_aliases[packet.Ack] - if alias_ok { - packet.Ack = ack_alias + if alias, ok := t.ackAliases[packet.Ack]; ok { + packet.Ack = alias } - m_id := packet.Addr.String() + strconv.Itoa(int(packet.SrcPort)) + strconv.Itoa(int(packet.Ack)) - message, ok := t.messages[m_id] + mID := packet.Addr.String() + strconv.Itoa(int(packet.SrcPort)) + strconv.Itoa(int(packet.Ack)) + message, ok := t.messages[mID] if !ok { - // We sending c_del_message channel, so message object can communicate with Listener and notify it if message completed - message = NewTCPMessage(m_id, t.c_del_message, packet.Ack) - t.messages[m_id] = message + // We sending messageDelChan channel, so message object can communicate with Listener and notify it if message completed + message = NewTCPMessage(mID, t.messageDelChan, packet.Ack) + t.messages[mID] = message } if bytes.Equal(packet.Data[0:4], bPOST) { if bytes.Equal(packet.Data[len(packet.Data)-24:len(packet.Data)-4], bExpect100ContinueCheck) { - t.seq_with_data[packet.Seq+uint32(len(packet.Data))] = packet.Ack + t.seqWithData[packet.Seq+uint32(len(packet.Data))] = packet.Ack // Removing `Expect: 100-continue` header packet.Data = append(packet.Data[:len(packet.Data)-24], packet.Data[len(packet.Data)-2:]...) @@ -161,10 +174,10 @@ func (t *Listener) processTCPPacket(packet *TCPPacket) { } // Adding packet to message - message.c_packets <- packet + message.packetsChan <- packet } // Receive TCP messages from the listener channel func (t *Listener) Receive() *TCPMessage { - return <-t.c_messages + return <-t.messagesChan } diff --git a/raw_socket_listener/tcp_message.go b/raw_socket_listener/tcp_message.go index 32fdaae..ddfb0f2 100644 --- a/raw_socket_listener/tcp_message.go +++ b/raw_socket_listener/tcp_message.go @@ -1,4 +1,4 @@ -package raw_socket +package rawSocket import ( "log" @@ -6,7 +6,8 @@ import ( "time" ) -const MSG_EXPIRE = 2000 * time.Millisecond +// MsgExpire specify period that message should wait before it considered as finished +const MsgExpire = 2000 * time.Millisecond // TCPMessage ensure that all TCP packets for given request is received, and processed in right sequence // Its needed because all TCP message can be fragmented or re-transmitted @@ -21,20 +22,20 @@ type TCPMessage struct { timer *time.Timer // Used for expire check - c_packets chan *TCPPacket + packetsChan chan *TCPPacket - c_del_message chan *TCPMessage + delChan chan *TCPMessage } // NewTCPMessage pointer created from a Acknowledgment number and a channel of messages readuy to be deleted -func NewTCPMessage(ID string, c_del chan *TCPMessage, Ack uint32) (msg *TCPMessage) { +func NewTCPMessage(ID string, delChan chan *TCPMessage, Ack uint32) (msg *TCPMessage) { msg = &TCPMessage{ID: ID, Ack: Ack} - msg.c_packets = make(chan *TCPPacket) - msg.c_del_message = c_del // used for notifying that message completed or expired + msg.packetsChan = make(chan *TCPPacket) + msg.delChan = delChan // used for notifying that message completed or expired // Every time we receive packet we reset this timer - msg.timer = time.AfterFunc(MSG_EXPIRE, msg.Timeout) + msg.timer = time.AfterFunc(MsgExpire, msg.Timeout) go msg.listen() @@ -44,7 +45,7 @@ func NewTCPMessage(ID string, c_del chan *TCPMessage, Ack uint32) (msg *TCPMessa func (t *TCPMessage) listen() { for { select { - case packet, more := <-t.c_packets: + case packet, more := <-t.packetsChan: if more { t.AddPacket(packet) } else { @@ -60,21 +61,21 @@ func (t *TCPMessage) Timeout() { select { // In some cases Timeout can be called multiple times (do not know how yet) // Ensure that we did not close channel 2 times - case packet, ok := <-t.c_packets: + case packet, ok := <-t.packetsChan: if ok { t.AddPacket(packet) } else { return } default: - close(t.c_packets) - t.c_del_message <- t // Notify RAWListener that message is ready to be send to replay server + close(t.packetsChan) + t.delChan <- t // Notify RAWListener that message is ready to be send to replay server } } // Bytes sorts packets in right orders and return message content func (t *TCPMessage) Bytes() (output []byte) { - sort.Sort(BySeq(t.packets)) + sort.Sort(sortBySeq(t.packets)) for _, v := range t.packets { output = append(output, v.Data...) @@ -102,5 +103,5 @@ func (t *TCPMessage) AddPacket(packet *TCPPacket) { } // Reset message timeout timer - t.timer.Reset(MSG_EXPIRE) + t.timer.Reset(MsgExpire) } diff --git a/raw_socket_listener/tcp_packet.go b/raw_socket_listener/tcp_packet.go index 4d3f4e6..3ebcf88 100644 --- a/raw_socket_listener/tcp_packet.go +++ b/raw_socket_listener/tcp_packet.go @@ -1,4 +1,4 @@ -package raw_socket +package rawSocket import ( "encoding/binary" @@ -9,19 +9,18 @@ import ( // TCP Flags const ( - TCP_FIN = 1 << iota - TCP_SYN - TCP_RST - TCP_PSH - TCP_ACK - TCP_URG - TCP_ECE - TCP_CWR - TCP_NS + fFIN = 1 << iota + fSYN + fRST + fPSH + fACK + fURG + fECE + fCWR + fNS ) -// Simple TCP packet parser -// +// TCPPacket provides tcp packet parser // Packet structure: http://en.wikipedia.org/wiki/Transmission_Control_Protocol type TCPPacket struct { SrcPort uint16 @@ -39,6 +38,7 @@ type TCPPacket struct { Addr net.Addr } +// ParseTCPPacket takes address and tcp payload and returns parsed TCPPacket func ParseTCPPacket(addr net.Addr, b []byte) (p *TCPPacket) { p = &TCPPacket{Data: b} p.ParseBasic() @@ -76,15 +76,15 @@ func (t *TCPPacket) String() string { "Acknowledgment:" + strconv.Itoa(int(t.Ack)), "Header len:" + strconv.Itoa(int(t.DataOffset)), - "Flag ns:" + strconv.FormatBool(t.Flags&TCP_NS != 0), - "Flag crw:" + strconv.FormatBool(t.Flags&TCP_CWR != 0), - "Flag ece:" + strconv.FormatBool(t.Flags&TCP_ECE != 0), - "Flag urg:" + strconv.FormatBool(t.Flags&TCP_URG != 0), - "Flag ack:" + strconv.FormatBool(t.Flags&TCP_ACK != 0), - "Flag psh:" + strconv.FormatBool(t.Flags&TCP_PSH != 0), - "Flag rst:" + strconv.FormatBool(t.Flags&TCP_RST != 0), - "Flag syn:" + strconv.FormatBool(t.Flags&TCP_SYN != 0), - "Flag fin:" + strconv.FormatBool(t.Flags&TCP_FIN != 0), + "Flag ns:" + strconv.FormatBool(t.Flags&fNS != 0), + "Flag crw:" + strconv.FormatBool(t.Flags&fCWR != 0), + "Flag ece:" + strconv.FormatBool(t.Flags&fECE != 0), + "Flag urg:" + strconv.FormatBool(t.Flags&fURG != 0), + "Flag ack:" + strconv.FormatBool(t.Flags&fACK != 0), + "Flag psh:" + strconv.FormatBool(t.Flags&fPSH != 0), + "Flag rst:" + strconv.FormatBool(t.Flags&fRST != 0), + "Flag syn:" + strconv.FormatBool(t.Flags&fSYN != 0), + "Flag fin:" + strconv.FormatBool(t.Flags&fFIN != 0), "Window size:" + strconv.Itoa(int(t.Window)), "Checksum:" + strconv.Itoa(int(t.Checksum)), @@ -94,8 +94,8 @@ func (t *TCPPacket) String() string { }, "\n") } -type BySeq []*TCPPacket +type sortBySeq []*TCPPacket -func (a BySeq) Len() int { return len(a) } -func (a BySeq) Swap(i, j int) { a[i], a[j] = a[j], a[i] } -func (a BySeq) Less(i, j int) bool { return a[i].Seq < a[j].Seq } +func (a sortBySeq) Len() int { return len(a) } +func (a sortBySeq) Swap(i, j int) { a[i], a[j] = a[j], a[i] } +func (a sortBySeq) Less(i, j int) bool { return a[i].Seq < a[j].Seq } From 6158ee30255ce6a651dd6f1712d87b4ee66b2f03 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Mon, 13 Jul 2015 16:20:57 +0500 Subject: [PATCH 18/26] Remove debug --- raw_socket_listener/listener.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/raw_socket_listener/listener.go b/raw_socket_listener/listener.go index 29534c2..280e288 100644 --- a/raw_socket_listener/listener.go +++ b/raw_socket_listener/listener.go @@ -70,7 +70,6 @@ func (t *Listener) listen() { select { // If message ready for deletion it means that its also complete or expired by timeout case message := <-t.messageDelChan: - log.Println("MESSAGE") t.messagesChan <- message delete(t.ackAliases, message.Ack) delete(t.messages, message.ID) @@ -107,8 +106,6 @@ func (t *Listener) readRAWSocket() { } func (t *Listener) parsePacket(addr net.Addr, buf []byte) { - log.Println("PACKET") - if t.isIncomingDataPacket(buf) { t.packetsChan <- ParseTCPPacket(addr, buf) } From 28540f43a11bfe0932fb1ab49a749d5bb270a9b6 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Mon, 13 Jul 2015 16:56:52 +0500 Subject: [PATCH 19/26] New portion of changes --- output_http.go | 32 +++++++++++++++++++------------- output_http_test.go | 4 ++-- output_tcp.go | 5 +++++ plugins.go | 23 +++++++++++++---------- settings.go | 9 +++++++-- test_input.go | 7 +++++++ test_output.go | 2 ++ 7 files changed, 55 insertions(+), 27 deletions(-) diff --git a/output_http.go b/output_http.go index e2a6c96..eb3e925 100644 --- a/output_http.go +++ b/output_http.go @@ -7,8 +7,9 @@ import ( "time" ) -const InitialDynamicWorkers = 10 +const initialDynamicWorkers = 10 +// HTTPOutputConfig struct for holding http output configuration type HTTPOutputConfig struct { redirectLimit int @@ -20,6 +21,9 @@ type HTTPOutputConfig struct { Debug bool } +// HTTPOutput plugin manage pool of workers which send request to replayed server +// By default workers pool is dynamic and starts with 10 workers +// You can specify fixed number of workers using `--output-http-workers` type HTTPOutput struct { // Keep this as first element of struct because it guarantees 64bit // alignment. atomic.* functions crash on 32bit machines if operand is not @@ -39,6 +43,8 @@ type HTTPOutput struct { elasticSearch *ESPlugin } +// NewHTTPOutput constructor for HTTPOutput +// Initialize workers func NewHTTPOutput(address string, config *HTTPOutputConfig) io.Writer { o := new(HTTPOutput) @@ -55,7 +61,7 @@ func NewHTTPOutput(address string, config *HTTPOutputConfig) io.Writer { // Initial workers count if o.config.workers == 0 { - o.needWorker <- InitialDynamicWorkers + o.needWorker <- initialDynamicWorkers } else { o.needWorker <- o.config.workers } @@ -65,16 +71,16 @@ func NewHTTPOutput(address string, config *HTTPOutputConfig) io.Writer { o.elasticSearch.Init(o.config.elasticSearch) } - go o.WorkerMaster() + go o.workerMaster() return o } -func (o *HTTPOutput) WorkerMaster() { +func (o *HTTPOutput) workerMaster() { for { - new_workers := <-o.needWorker - for i := 0; i < new_workers; i++ { - go o.Worker() + newWorkers := <-o.needWorker + for i := 0; i < newWorkers; i++ { + go o.startWorker() } // Disable dynamic scaling if workers poll fixed size @@ -84,13 +90,13 @@ func (o *HTTPOutput) WorkerMaster() { } } -func (o *HTTPOutput) Worker() { +func (o *HTTPOutput) startWorker() { client := NewHTTPClient(o.address, &HTTPClientConfig{ FollowRedirects: o.config.redirectLimit, Debug: o.config.Debug, }) - death_count := 0 + deathCount := 0 atomic.AddInt64(&o.activeWorkers, 1) @@ -98,19 +104,19 @@ func (o *HTTPOutput) Worker() { select { case data := <-o.queue: o.sendRequest(client, data) - death_count = 0 + deathCount = 0 case <-time.After(time.Millisecond * 100): // When dynamic scaling enabled workers die after 2s of inactivity if o.config.workers == 0 { - death_count += 1 + deathCount++ } else { continue } - if death_count > 20 { + if deathCount > 20 { workersCount := atomic.LoadInt64(&o.activeWorkers) - // At least 1 worker should be alive + // At least 1 startWorker should be alive if workersCount != 1 { atomic.AddInt64(&o.activeWorkers, -1) return diff --git a/output_http_test.go b/output_http_test.go index 7cf8c24..16f8422 100644 --- a/output_http_test.go +++ b/output_http_test.go @@ -86,10 +86,10 @@ func TestOutputHTTPSSL(t *testing.T) { })) input := NewTestInput() - http_output := NewHTTPOutput(server.URL, &HTTPOutputConfig{}) + output := NewHTTPOutput(server.URL, &HTTPOutputConfig{}) Plugins.Inputs = []io.Reader{input} - Plugins.Outputs = []io.Writer{http_output} + Plugins.Outputs = []io.Writer{output} go Start(quit) diff --git a/output_tcp.go b/output_tcp.go index f6f0e9d..dd7cdc4 100644 --- a/output_tcp.go +++ b/output_tcp.go @@ -9,6 +9,9 @@ import ( "time" ) +// TCPOutput used for sending raw tcp payloads +// Currently used for internal communication between listener and replay server +// Can be used for transfering binary payloads like protocol buffers type TCPOutput struct { address string limit int @@ -16,6 +19,8 @@ type TCPOutput struct { bufStats *GorStat } +// NewTCPOutput constructor for TCPOutput +// Initialize 10 workers which hold keep-alive connection func NewTCPOutput(address string) io.Writer { o := new(TCPOutput) diff --git a/plugins.go b/plugins.go index 88134ef..cb681ae 100644 --- a/plugins.go +++ b/plugins.go @@ -6,23 +6,25 @@ import ( "strings" ) +// InOutPlugins struct for holding references to plugins type InOutPlugins struct { Inputs []io.Reader Outputs []io.Writer } -type ReaderOrWriter interface{} - -var Plugins *InOutPlugins = new(InOutPlugins) +// Plugins holds all the plugin objects +var Plugins *InOutPlugins +// extractLimitOptions detects if plugin get called with limiter support +// Returns address and limit func extractLimitOptions(options string) (string, string) { split := strings.Split(options, "|") if len(split) > 1 { return split[0], split[1] - } else { - return split[0], "" } + + return split[0], "" } // Automatically detects type of plugin and initialize it @@ -45,23 +47,24 @@ func registerPlugin(constructor interface{}, options ...interface{}) { // Calling our constructor with list of given options plugin := vc.Call(vo)[0].Interface() - plugin_wrapper := plugin + pluginWrapper := plugin if limit != "" { - plugin_wrapper = NewLimiter(plugin, limit) + pluginWrapper = NewLimiter(plugin, limit) } else { - plugin_wrapper = plugin + pluginWrapper = plugin } if _, ok := plugin.(io.Reader); ok { - Plugins.Inputs = append(Plugins.Inputs, plugin_wrapper.(io.Reader)) + Plugins.Inputs = append(Plugins.Inputs, pluginWrapper.(io.Reader)) } if _, ok := plugin.(io.Writer); ok { - Plugins.Outputs = append(Plugins.Outputs, plugin_wrapper.(io.Writer)) + Plugins.Outputs = append(Plugins.Outputs, pluginWrapper.(io.Writer)) } } +// InitPlugins specify and initialize all available plugins func InitPlugins() { for _, options := range Settings.inputDummy { registerPlugin(NewDummyInput, options) diff --git a/settings.go b/settings.go index 8c345e7..a08a9c1 100644 --- a/settings.go +++ b/settings.go @@ -8,21 +8,24 @@ import ( ) const ( + // VERSION specifies Gor current version VERSION = "0.9.8" ) -// Allows to specify multiple flags with same name and collects all values to array +// MultiOption allows to specify multiple flags with same name and collects all values into array type MultiOption []string func (h *MultiOption) String() string { return fmt.Sprint(*h) } +// Set gets called multiple times for each flag with same name func (h *MultiOption) Set(value string) error { *h = append(*h, value) return nil } +// AppSettings is the struct of main configuration type AppSettings struct { verbose bool debug bool @@ -49,7 +52,8 @@ type AppSettings struct { modifierConfig HTTPModifierConfig } -var Settings AppSettings = AppSettings{} +// Settings holds Gor configuration +var Settings AppSettings func usage() { fmt.Printf("Gor is a simple http traffic replication tool written in Go. Its main goal is to replay traffic from production servers to staging and dev environments.\nProject page: https://github.com/buger/gor\nAuthor: leonsbox@gmail.com\nCurrent Version: %s\n\n", VERSION) @@ -113,6 +117,7 @@ func init() { flag.Var(&Settings.modifierConfig.paramHashFilters, "http-param-limiter", "Takes a fraction of requests, consistently taking or rejecting a request based on the FNV32-1A hash of a specific GET param:\n\t gor --input-raw :8080 --output-http staging.com --http-param-limiter user_id:25%") } +// Debug gets called only if --verbose flag specified func Debug(args ...interface{}) { if Settings.verbose { fmt.Print("[DEBUG] ") diff --git a/test_input.go b/test_input.go index 8f62344..a9a359a 100644 --- a/test_input.go +++ b/test_input.go @@ -5,10 +5,12 @@ import ( "encoding/base64" ) +// TestInput used for testing purpose, it allows emitting requests on demand type TestInput struct { data chan []byte } +// NewTestInput constructor for TestInput func NewTestInput() (i *TestInput) { i = new(TestInput) i.data = make(chan []byte, 100) @@ -23,18 +25,22 @@ func (i *TestInput) Read(data []byte) (int, error) { return len(buf), nil } +// EmitGET emits GET request without headers func (i *TestInput) EmitGET() { i.data <- []byte("GET / HTTP/1.1\r\n\r\n") } +// EmitPOST emits POST request with Content-Length func (i *TestInput) EmitPOST() { i.data <- []byte("POST /pub/WWW/ HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") } +// EmitChunkedPOST emits POST request with `Transfer-Encoding: chunked` and chunked body func (i *TestInput) EmitChunkedPOST() { i.data <- []byte("POST /pub/WWW/ HTTP/1.1\r\nHost: www.w3.org\r\nTransfer-Encoding: chunked\r\n\r\n4\r\nWiki\r\n5\r\npedia\r\ne\r\n in\r\n\r\nchunks.\r\n0\r\n\r\n") } +// EmitLargePOST emits POST request with large payload (5mb) func (i *TestInput) EmitLargePOST() { size := 5 * 1024 * 1024 // 5 MB rb := make([]byte, size) @@ -45,6 +51,7 @@ func (i *TestInput) EmitLargePOST() { i.data <- []byte("POST / HTTP/1.1\nHost: www.w3.org\nContent-Length:5242880\r\n\r\n" + rs) } +// EmitOPTIONS emits OPTIONS request, similar to GET func (i *TestInput) EmitOPTIONS() { i.data <- []byte("OPTIONS / HTTP/1.1\nHost: www.w3.org\r\n\r\n") } diff --git a/test_output.go b/test_output.go index 1d34da0..e76fbb6 100644 --- a/test_output.go +++ b/test_output.go @@ -2,10 +2,12 @@ package main type writeCallback func(data []byte) +// TestOutput used in testing to intercept any output into callback type TestOutput struct { cb writeCallback } +// NewTestOutput constructor for TestOutput, accepts callback which get called on each incoming Write func NewTestOutput(cb writeCallback) (i *TestOutput) { i = new(TestOutput) i.cb = cb From fa003f5fecaafc1730a1e44e8bdc12d5345335e2 Mon Sep 17 00:00:00 2001 From: Rafael Garcia Date: Fri, 17 Jul 2015 03:26:06 +0000 Subject: [PATCH 20/26] add --http-disallow-header option --- README.md | 5 +++++ http_modifier.go | 14 +++++++++++++- http_modifier_settings.go | 15 ++++++++------- http_modifier_test.go | 27 +++++++++++++++++++++++++++ settings.go | 2 ++ 5 files changed, 55 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 7f37f9d..3ffb38d 100644 --- a/README.md +++ b/README.md @@ -128,6 +128,9 @@ gor --input-raw :8080 --output-http staging.com --http-disallow-url /api ``` # only forward requests with an api version of 1.0x gor --input-raw :8080 --output-http staging.com --http-allow-header api-version:^1\.0\d + +# only forward requests NOT containing User-Agent header value "Replayed by Gor" +gor --input-raw :8080 --output-http staging.com --http-disallow-header "User-Agent: Replayed by Gor" ``` #### Filter based on http method @@ -260,6 +263,8 @@ https://github.com/buger/gor/releases ``` -http-allow-header=[]: A regexp to match a specific header against. Requests with non-matching headers will be dropped: gor --input-raw :8080 --output-http staging.com --http-allow-header api-version:^v1 + -http-disallow-header=[]: A regexp to match a specific header against. Requests with matching headers will be dropped: + gor --input-raw :8080 --output-http staging.com --http-disallow-header "User-Agent: Replayed by Gor" -http-allow-method=[]: Whitelist of HTTP methods to replay. Anything else will be dropped: gor --input-raw :8080 --output-http staging.com --http-allow-method GET --http-allow-method OPTIONS -http-allow-url=[]: A regexp to match requests against. Filter get matched agains full url with domain. Anything else will be dropped: diff --git a/http_modifier.go b/http_modifier.go index d820df6..4efa684 100644 --- a/http_modifier.go +++ b/http_modifier.go @@ -2,8 +2,9 @@ package main import ( "bytes" - "github.com/buger/gor/proto" "hash/fnv" + + "github.com/buger/gor/proto" ) type HTTPModifier struct { @@ -16,6 +17,7 @@ func NewHTTPModifier(config *HTTPModifierConfig) *HTTPModifier { len(config.urlNegativeRegexp) == 0 && len(config.urlRewrite) == 0 && len(config.headerFilters) == 0 && + len(config.headerNegativeFilters) == 0 && len(config.headerHashFilters) == 0 && len(config.paramHashFilters) == 0 && len(config.params) == 0 && @@ -94,6 +96,16 @@ func (m *HTTPModifier) Rewrite(payload []byte) (response []byte) { } } + if len(m.config.headerNegativeFilters) > 0 { + for _, f := range m.config.headerNegativeFilters { + value, s, _, _ := proto.Header(payload, f.name) + + if s != -1 && f.regexp.Match(value) { + return + } + } + } + if len(m.config.headerHashFilters) > 0 { for _, f := range m.config.headerHashFilters { value, s, _, _ := proto.Header(payload, f.name) diff --git a/http_modifier_settings.go b/http_modifier_settings.go index 854b11a..43f81d1 100644 --- a/http_modifier_settings.go +++ b/http_modifier_settings.go @@ -9,12 +9,13 @@ import ( ) type HTTPModifierConfig struct { - urlNegativeRegexp HTTPUrlRegexp - urlRegexp HTTPUrlRegexp - urlRewrite UrlRewriteMap - headerFilters HTTPHeaderFilters - headerHashFilters HTTPHashFilters - paramHashFilters HTTPHashFilters + urlNegativeRegexp HTTPUrlRegexp + urlRegexp HTTPUrlRegexp + urlRewrite UrlRewriteMap + headerFilters HTTPHeaderFilters + headerNegativeFilters HTTPHeaderFilters + headerHashFilters HTTPHashFilters + paramHashFilters HTTPHashFilters params HTTPParams headers HTTPHeaders @@ -22,7 +23,7 @@ type HTTPModifierConfig struct { } // -// Handling of --http-allow-header options +// Handling of --http-allow-header, --http-disallow-header options // type headerFilter struct { name []byte diff --git a/http_modifier_test.go b/http_modifier_test.go index 40e5ec4..d92829e 100644 --- a/http_modifier_test.go +++ b/http_modifier_test.go @@ -39,6 +39,33 @@ func TestHTTPModifierHeaderFilters(t *testing.T) { } } +func TestHTTPModifierHeaderNegativeFilters(t *testing.T) { + filters := HTTPHeaderFilters{} + filters.Set("Host:^www.w3.org$") + + modifier := NewHTTPModifier(&HTTPModifierConfig{ + headerNegativeFilters: filters, + }) + + payload := []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w4.org\r\n\r\na=1&b=2") + + if len(modifier.Rewrite(payload)) == 0 { + t.Error("Request should pass filters") + } + + filters = HTTPHeaderFilters{} + // Setting filter that not match our header + filters.Set("Host:^www.w4.org$") + + modifier = NewHTTPModifier(&HTTPModifierConfig{ + headerNegativeFilters: filters, + }) + + if len(modifier.Rewrite(payload)) != 0 { + t.Error("Request should not pass filters") + } +} + func TestHTTPModifierURLRewrite(t *testing.T) { var url, new_url []byte diff --git a/settings.go b/settings.go index 8c345e7..3e6d6d0 100644 --- a/settings.go +++ b/settings.go @@ -107,6 +107,8 @@ func init() { flag.Var(&Settings.modifierConfig.headerFilters, "http-allow-header", "A regexp to match a specific header against. Requests with non-matching headers will be dropped:\n\t gor --input-raw :8080 --output-http staging.com --http-allow-header api-version:^v1") flag.Var(&Settings.modifierConfig.headerFilters, "output-http-header-filter", "WARNING: `--output-http-header-filter` DEPRECATED, use `--http-allow-header` instead") + flag.Var(&Settings.modifierConfig.headerFilters, "http-disallow-header", "A regexp to match a specific header against. Requests with matching headers will be dropped:\n\t gor --input-raw :8080 --output-http staging.com --http-disallow-header \"User-Agent: Replayed by Gor\"") + flag.Var(&Settings.modifierConfig.headerHashFilters, "http-header-limiter", "Takes a fraction of requests, consistently taking or rejecting a request based on the FNV32-1A hash of a specific header:\n\t gor --input-raw :8080 --output-http staging.com --http-header-imiter user-id:25%") flag.Var(&Settings.modifierConfig.headerHashFilters, "output-http-header-hash-filter", "WARNING: `output-http-header-hash-filter` DEPRECATED, use `--http-header-hash-limiter` instead") From 125d9db83410c354adf72228f5936454c11022bd Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Fri, 17 Jul 2015 11:30:43 +0500 Subject: [PATCH 21/26] Next batch --- .travis.yml | 4 ++-- emitter.go | 3 ++- http_client.go | 2 +- http_client_test.go | 24 ++++++++++----------- http_modifier_settings.go | 6 ++++-- http_modifier_test.go | 22 ++++++++++---------- input_dummy.go | 2 ++ input_file.go | 6 ++++-- input_http.go | 2 ++ input_raw.go | 2 ++ input_raw_test.go | 44 +++++++++++++++++++-------------------- input_tcp.go | 6 +++--- input_tcp_test.go | 6 +++--- limiter.go | 3 +++ output_dummy.go | 2 ++ output_file.go | 8 +++++-- plugins.go | 2 +- 17 files changed, 82 insertions(+), 62 deletions(-) diff --git a/.travis.yml b/.travis.yml index a236b6a..7f2d291 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,3 +1,3 @@ -language: go +language: go go: 1.4.2 -script: sudo -E bash -c "source /etc/profile && eval '$(gimme 1.4)' && export GOPATH=$HOME/gopath:$GOPATH && go get && GORACE='halt_on_error=1' go test -race -v" +script: sudo -E bash -c "source /etc/profile && eval '$(gimme 1.4.2)' && export GOPATH=$HOME/gopath:$GOPATH && go get && GORACE='halt_on_error=1' go test -v" \ No newline at end of file diff --git a/emitter.go b/emitter.go index 2181331..1f7caa0 100644 --- a/emitter.go +++ b/emitter.go @@ -5,6 +5,7 @@ import ( "time" ) +// Start initialize loop for sending data from inputs to outputs func Start(stop chan int) { for _, in := range Plugins.Inputs { go CopyMulty(in, Plugins.Outputs...) @@ -19,7 +20,7 @@ func Start(stop chan int) { } } -// Copy from 1 reader to multiple writers +// CopyMulty copies from 1 reader to multiple writers func CopyMulty(src io.Reader, writers ...io.Writer) (err error) { buf := make([]byte, 5*1024*1024) wIndex := 0 diff --git a/http_client.go b/http_client.go index c54e6a8..f9ad04d 100644 --- a/http_client.go +++ b/http_client.go @@ -145,7 +145,7 @@ func (c *HTTPClient) Send(data []byte) (response []byte, err error) { // 3xx requests if status[0] == '3' { - c.redirectsCount += 1 + c.redirectsCount++ location := proto.Header(payload, []byte("Location")) redirectPayload := []byte("GET " + string(location) + " HTTP/1.1\r\n\r\n") diff --git a/http_client_test.go b/http_client_test.go index a5b238b..feb0326 100644 --- a/http_client_test.go +++ b/http_client_test.go @@ -132,7 +132,7 @@ func TestHTTPClientHTTPSSend(t *testing.T) { func TestHTTPClientServerInstantDisconnect(t *testing.T) { wg := new(sync.WaitGroup) - GET_payload := []byte("GET / HTTP/1.1\r\n\r\n") + GETPayload := []byte("GET / HTTP/1.1\r\n\r\n") ln, _ := net.Listen("tcp", ":0") @@ -148,8 +148,8 @@ func TestHTTPClientServerInstantDisconnect(t *testing.T) { client := NewHTTPClient(ln.Addr().String(), &HTTPClientConfig{}) wg.Add(2) - client.Send(GET_payload) - client.Send(GET_payload) + client.Send(GETPayload) + client.Send(GETPayload) wg.Wait() } @@ -157,7 +157,7 @@ func TestHTTPClientServerInstantDisconnect(t *testing.T) { func TestHTTPClientServerNoKeepAlive(t *testing.T) { wg := new(sync.WaitGroup) - GET_payload := []byte("GET / HTTP/1.1\r\n\r\n") + GETPayload := []byte("GET / HTTP/1.1\r\n\r\n") ln, _ := net.Listen("tcp", ":0") @@ -186,8 +186,8 @@ func TestHTTPClientServerNoKeepAlive(t *testing.T) { client := NewHTTPClient(ln.Addr().String(), &HTTPClientConfig{}) wg.Add(2) - client.Send(GET_payload) - client.Send(GET_payload) + client.Send(GETPayload) + client.Send(GETPayload) wg.Wait() } @@ -195,7 +195,7 @@ func TestHTTPClientServerNoKeepAlive(t *testing.T) { func TestHTTPClientRedirect(t *testing.T) { wg := new(sync.WaitGroup) - GET_payload := []byte("GET / HTTP/1.1\r\n\r\n") + GETPayload := []byte("GET / HTTP/1.1\r\n\r\n") server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -210,7 +210,7 @@ func TestHTTPClientRedirect(t *testing.T) { // Should do 2 queries wg.Add(2) - client.Send(GET_payload) + client.Send(GETPayload) wg.Wait() } @@ -218,7 +218,7 @@ func TestHTTPClientRedirect(t *testing.T) { func TestHTTPClientRedirectLimit(t *testing.T) { wg := new(sync.WaitGroup) - GET_payload := []byte("GET / HTTP/1.1\r\n\r\n") + GETPayload := []byte("GET / HTTP/1.1\r\n\r\n") server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -241,7 +241,7 @@ func TestHTTPClientRedirectLimit(t *testing.T) { // Have 3 redirects + 1 GET, but should do only 2 redirects + GET wg.Add(3) - client.Send(GET_payload) + client.Send(GETPayload) wg.Wait() } @@ -249,7 +249,7 @@ func TestHTTPClientRedirectLimit(t *testing.T) { func TestHTTPClientHandleHTTP10(t *testing.T) { wg := new(sync.WaitGroup) - GET_payload := []byte("GET http://foobar.com/path HTTP/1.0\r\n\r\n") + GETPayload := []byte("GET http://foobar.com/path HTTP/1.0\r\n\r\n") server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -263,7 +263,7 @@ func TestHTTPClientHandleHTTP10(t *testing.T) { client := NewHTTPClient(server.URL, &HTTPClientConfig{Debug: true}) wg.Add(1) - client.Send(GET_payload) + client.Send(GETPayload) wg.Wait() } diff --git a/http_modifier_settings.go b/http_modifier_settings.go index 854b11a..1a7323a 100644 --- a/http_modifier_settings.go +++ b/http_modifier_settings.go @@ -8,6 +8,7 @@ import ( "strings" ) +// HTTPModifierConfig holds configuration options for built-in traffic modifier type HTTPModifierConfig struct { urlNegativeRegexp HTTPUrlRegexp urlRegexp HTTPUrlRegexp @@ -29,6 +30,7 @@ type headerFilter struct { regexp *regexp.Regexp } +// HTTPHeaderFilters holds list of headers and their regexps type HTTPHeaderFilters []headerFilter func (h *HTTPHeaderFilters) String() string { @@ -67,7 +69,7 @@ func (h *HTTPHashFilters) String() string { func (h *HTTPHashFilters) Set(value string) error { valArr := strings.SplitN(value, ":", 2) if len(valArr) < 2 { - return errors.New("need both header and value, colon-delimited (ex. user_id:50%).") + return errors.New("need both header and value, colon-delimited (ex. user_id:50%)") } f := hashFilter{name: []byte(valArr[0])} @@ -180,7 +182,7 @@ func (r *UrlRewriteMap) String() string { func (r *UrlRewriteMap) Set(value string) error { valArr := strings.SplitN(value, ":", 2) if len(valArr) < 2 { - return errors.New("need both src and target, colon-delimited (ex. /a:/b).") + return errors.New("need both src and target, colon-delimited (ex. /a:/b)") } regexp, err := regexp.Compile(valArr[0]) if err != nil { diff --git a/http_modifier_test.go b/http_modifier_test.go index 40e5ec4..b4e9348 100644 --- a/http_modifier_test.go +++ b/http_modifier_test.go @@ -40,7 +40,7 @@ func TestHTTPModifierHeaderFilters(t *testing.T) { } func TestHTTPModifierURLRewrite(t *testing.T) { - var url, new_url []byte + var url, newURL []byte rewrites := UrlRewriteMap{} @@ -58,13 +58,13 @@ func TestHTTPModifierURLRewrite(t *testing.T) { }) url = []byte("/v1/user/joe/ping") - if new_url = proto.Path(modifier.Rewrite(payload(url))); bytes.Equal(new_url, url) { - t.Error("Request url should have been rewritten, wasn't", string(new_url)) + if newURL = proto.Path(modifier.Rewrite(payload(url))); bytes.Equal(newURL, url) { + t.Error("Request url should have been rewritten, wasn't", string(newURL)) } url = []byte("/v1/user/ping") - if new_url = proto.Path(modifier.Rewrite(payload(url))); !bytes.Equal(new_url, url) { - t.Error("Request url should have been rewritten, wasn't", string(new_url)) + if newURL = proto.Path(modifier.Rewrite(payload(url))); !bytes.Equal(newURL, url) { + t.Error("Request url should have been rewritten, wasn't", string(newURL)) } } @@ -128,9 +128,9 @@ func TestHTTPModifierHeaders(t *testing.T) { }) payload := []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") - new_payload := []byte("POST /post HTTP/1.1\r\nHeader1: 1\r\nContent-Length: 7\r\nHost: localhost\r\n\r\na=1&b=2") + newPayload := []byte("POST /post HTTP/1.1\r\nHeader1: 1\r\nContent-Length: 7\r\nHost: localhost\r\n\r\na=1&b=2") - if payload = modifier.Rewrite(payload); !bytes.Equal(payload, new_payload) { + if payload = modifier.Rewrite(payload); !bytes.Equal(payload, newPayload) { t.Error("Should update request headers", string(payload)) } } @@ -196,9 +196,9 @@ func TestHTTPModifierSetHeader(t *testing.T) { }) payload := []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") - payload_after := []byte("POST /post HTTP/1.1\r\nUser-Agent: Gor\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") + payloadAfter := []byte("POST /post HTTP/1.1\r\nUser-Agent: Gor\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") - if payload = modifier.Rewrite(payload); !bytes.Equal(payload_after, payload) { + if payload = modifier.Rewrite(payload); !bytes.Equal(payloadAfter, payload) { t.Error("Should add new header", string(payload)) } } @@ -212,9 +212,9 @@ func TestHTTPModifierSetParam(t *testing.T) { }) payload := []byte("POST /post?api_key=1234 HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") - payload_after := []byte("POST /post?api_key=1 HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") + payloadAfter := []byte("POST /post?api_key=1 HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") - if payload = modifier.Rewrite(payload); !bytes.Equal(payload_after, payload) { + if payload = modifier.Rewrite(payload); !bytes.Equal(payloadAfter, payload) { t.Error("Should override param", string(payload)) } } diff --git a/input_dummy.go b/input_dummy.go index a2a87ac..7f6a7f8 100644 --- a/input_dummy.go +++ b/input_dummy.go @@ -4,10 +4,12 @@ import ( "time" ) +// DummyInput used for debugging. It generate 1 "GET /"" request per second. type DummyInput struct { data chan []byte } +// NewDummyInput constructor for DummyInput func NewDummyInput(options string) (di *DummyInput) { di = new(DummyInput) di.data = make(chan []byte) diff --git a/input_file.go b/input_file.go index 043ca3e..60c93ec 100644 --- a/input_file.go +++ b/input_file.go @@ -7,6 +7,7 @@ import ( "time" ) +// FileInput can read requests generated by FileOutput type FileInput struct { data chan []byte path string @@ -14,19 +15,20 @@ type FileInput struct { speedFactor float64 } +// NewFileInput constructor for FileInput. Accepts file path as argument. func NewFileInput(path string) (i *FileInput) { i = new(FileInput) i.data = make(chan []byte) i.path = path i.speedFactor = 1 - i.Init(path) + i.init(path) go i.emit() return } -func (i *FileInput) Init(path string) { +func (i *FileInput) init(path string) { file, err := os.Open(path) if err != nil { diff --git a/input_http.go b/input_http.go index 0ed16eb..5150c4e 100644 --- a/input_http.go +++ b/input_http.go @@ -7,12 +7,14 @@ import ( "net/http/httputil" ) +// HTTPInput used for sending requests to Gor via http type HTTPInput struct { data chan []byte address string listener net.Listener } +// NewHTTPInput constructor for HTTPInput. Accepts address with port which he will listen on. func NewHTTPInput(address string) (i *HTTPInput) { i = new(HTTPInput) i.data = make(chan []byte) diff --git a/input_raw.go b/input_raw.go index 3a6a960..93b007d 100644 --- a/input_raw.go +++ b/input_raw.go @@ -7,11 +7,13 @@ import ( "strings" ) +// RAWInput used for intercepting traffic for given address type RAWInput struct { data chan []byte address string } +// NewRAWInput constructor for RAWInput. Accepts address with port as argument. func NewRAWInput(address string) (i *RAWInput) { i = new(RAWInput) i.data = make(chan []byte) diff --git a/input_raw_test.go b/input_raw_test.go index 6262667..7380fad 100644 --- a/input_raw_test.go +++ b/input_raw_test.go @@ -49,7 +49,7 @@ func TestInputRAW100Expect(t *testing.T) { wg := new(sync.WaitGroup) quit := make(chan int) - file_content, _ := ioutil.ReadFile("README.md") + fileContent, _ := ioutil.ReadFile("README.md") // Origing and Replay server initialization origin := startHTTP(func(req *http.Request) { @@ -59,12 +59,12 @@ func TestInputRAW100Expect(t *testing.T) { wg.Done() }) - origin_address := strings.Replace(origin.Addr().String(), "[::]", "127.0.0.1", -1) + originAddr := strings.Replace(origin.Addr().String(), "[::]", "127.0.0.1", -1) - input := NewRAWInput(origin_address) + input := NewRAWInput(originAddr) // We will use it to get content of raw HTTP request - test_output := NewTestOutput(func(data []byte) { + testOutput := NewTestOutput(func(data []byte) { if strings.Contains(string(data), "Expect: 100-continue") { t.Error("Should not contain 100-continue header") } @@ -75,24 +75,24 @@ func TestInputRAW100Expect(t *testing.T) { defer req.Body.Close() body, _ := ioutil.ReadAll(req.Body) - if !bytes.Equal(body, file_content) { + if !bytes.Equal(body, fileContent) { buf, _ := httputil.DumpRequest(req, true) t.Error("Wrong POST body:", string(buf)) } wg.Done() }) - replay_address := listener.Addr().String() + replayAddr := listener.Addr().String() - http_output := NewHTTPOutput(replay_address, &HTTPOutputConfig{}) + httpOutput := NewHTTPOutput(replayAddr, &HTTPOutputConfig{}) Plugins.Inputs = []io.Reader{input} - Plugins.Outputs = []io.Writer{test_output, http_output} + Plugins.Outputs = []io.Writer{testOutput, httpOutput} go Start(quit) wg.Add(3) - curl := exec.Command("curl", "http://"+origin_address, "--data-binary", "@README.md") + curl := exec.Command("curl", "http://"+originAddr, "--data-binary", "@README.md") err := curl.Run() if err != nil { log.Fatal(err) @@ -106,7 +106,7 @@ func TestInputRAWChunkedEncoding(t *testing.T) { wg := new(sync.WaitGroup) quit := make(chan int) - file_content, _ := ioutil.ReadFile("README.md") + fileContent, _ := ioutil.ReadFile("README.md") // Origing and Replay server initialization origin := startHTTP(func(req *http.Request) { @@ -116,33 +116,33 @@ func TestInputRAWChunkedEncoding(t *testing.T) { wg.Done() }) - origin_address := strings.Replace(origin.Addr().String(), "[::]", "127.0.0.1", -1) + originAddr := strings.Replace(origin.Addr().String(), "[::]", "127.0.0.1", -1) - input := NewRAWInput(origin_address) + input := NewRAWInput(originAddr) listener := startHTTP(func(req *http.Request) { defer req.Body.Close() body, _ := ioutil.ReadAll(req.Body) - if !bytes.Equal(body, file_content) { + if !bytes.Equal(body, fileContent) { buf, _ := httputil.DumpRequest(req, true) t.Error("Wrong POST body:", string(buf)) } wg.Done() }) - replay_address := listener.Addr().String() + replayAddr := listener.Addr().String() - http_output := NewHTTPOutput(replay_address, &HTTPOutputConfig{Debug: true}) + httpOutput := NewHTTPOutput(replayAddr, &HTTPOutputConfig{Debug: true}) Plugins.Inputs = []io.Reader{input} - Plugins.Outputs = []io.Writer{http_output} + Plugins.Outputs = []io.Writer{httpOutput} go Start(quit) wg.Add(2) - curl := exec.Command("curl", "http://"+origin_address, "--header", "Transfer-Encoding: chunked", "--data-binary", "@README.md") + curl := exec.Command("curl", "http://"+originAddr, "--header", "Transfer-Encoding: chunked", "--data-binary", "@README.md") err := curl.Run() if err != nil { log.Fatal(err) @@ -175,9 +175,9 @@ func TestInputRAWLargePayload(t *testing.T) { wg.Done() }) - origin_address := strings.Replace(origin.Addr().String(), "[::]", "127.0.0.1", -1) + originAddr := strings.Replace(origin.Addr().String(), "[::]", "127.0.0.1", -1) - input := NewRAWInput(origin_address) + input := NewRAWInput(originAddr) replay := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { req.Body = http.MaxBytesReader(w, req.Body, 1*1024*1024) @@ -193,15 +193,15 @@ func TestInputRAWLargePayload(t *testing.T) { })) defer replay.Close() - http_output := NewHTTPOutput(replay.URL, &HTTPOutputConfig{Debug: false}) + httpOutput := NewHTTPOutput(replay.URL, &HTTPOutputConfig{Debug: false}) Plugins.Inputs = []io.Reader{input} - Plugins.Outputs = []io.Writer{http_output} + Plugins.Outputs = []io.Writer{httpOutput} go Start(quit) wg.Add(2) - curl := exec.Command("curl", "http://"+origin_address, "--header", "Transfer-Encoding: chunked", "--data-binary", "@/tmp/large") + curl := exec.Command("curl", "http://"+originAddr, "--header", "Transfer-Encoding: chunked", "--data-binary", "@/tmp/large") err = curl.Run() if err != nil { log.Fatal("curl error:", err) diff --git a/input_tcp.go b/input_tcp.go index 27b7de6..9a2da90 100644 --- a/input_tcp.go +++ b/input_tcp.go @@ -9,15 +9,15 @@ import ( "os" ) -// Can be tested using nc tool: -// echo "asdad" | nc 127.0.0.1 27017 -// +// TCPInput used for internal communication +// It expected hex encoded data type TCPInput struct { data chan []byte address string listener net.Listener } +// NewTCPInput constructor for TCPInput, accepts address with port func NewTCPInput(address string) (i *TCPInput) { i = new(TCPInput) i.data = make(chan []byte) diff --git a/input_tcp_test.go b/input_tcp_test.go index 27c00fc..bbb0cc7 100644 --- a/input_tcp_test.go +++ b/input_tcp_test.go @@ -83,10 +83,10 @@ func BenchmarkTCPInput(b *testing.B) { for { data := <-dataChan - new_buf := make([]byte, len(data)+2) + buf := make([]byte, len(data)+2) data = append(data, []byte("ΒΆ")...) - copy(new_buf, data) - conn.Write(new_buf) + copy(buf, data) + conn.Write(buf) } }(conn) } diff --git a/limiter.go b/limiter.go index 3bd5acb..0e317a0 100644 --- a/limiter.go +++ b/limiter.go @@ -9,6 +9,7 @@ import ( "time" ) +// Limiter is a wrapper for input or output plugin which adds rate limiting type Limiter struct { plugin interface{} limit int @@ -30,6 +31,8 @@ func parseLimitOptions(options string) (limit int, isPercent bool) { return } +// NewLimiter constructor for Limiter, accepts plugin and options +// `options` allow to sprcify relatve or absolute limiting func NewLimiter(plugin interface{}, options string) io.ReadWriter { l := new(Limiter) l.limit, l.isPercent = parseLimitOptions(options) diff --git a/output_dummy.go b/output_dummy.go index f730c81..a033ebd 100644 --- a/output_dummy.go +++ b/output_dummy.go @@ -4,9 +4,11 @@ import ( "fmt" ) +// DummyOutput used for debugging, prints all incoming requests type DummyOutput struct { } +// NewDummyOutput constructor for DummyOutput func NewDummyOutput(options string) (di *DummyOutput) { di = new(DummyOutput) diff --git a/output_file.go b/output_file.go index ac0af4b..4f5be15 100644 --- a/output_file.go +++ b/output_file.go @@ -8,26 +8,30 @@ import ( "time" ) +// RawRequest stores original start time and request payload type RawRequest struct { Timestamp int64 Request []byte } +// FileOutput output plugin type FileOutput struct { path string encoder *gob.Encoder file *os.File } + +// NewFileOutput constructor for FileOutput, accepts path func NewFileOutput(path string) io.Writer { o := new(FileOutput) o.path = path - o.Init(path) + o.init(path) return o } -func (o *FileOutput) Init(path string) { +func (o *FileOutput) init(path string) { var err error o.file, err = os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0660) diff --git a/plugins.go b/plugins.go index cb681ae..d47012b 100644 --- a/plugins.go +++ b/plugins.go @@ -13,7 +13,7 @@ type InOutPlugins struct { } // Plugins holds all the plugin objects -var Plugins *InOutPlugins +var Plugins *InOutPlugins = new(InOutPlugins) // extractLimitOptions detects if plugin get called with limiter support // Returns address and limit From e5d9f73937ec23a0ccffd584fb3ca194ad3e0f0b Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Fri, 17 Jul 2015 11:36:18 +0500 Subject: [PATCH 22/26] Fix tests --- http_modifier.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/http_modifier.go b/http_modifier.go index f4af2e2..e2c50cb 100644 --- a/http_modifier.go +++ b/http_modifier.go @@ -98,7 +98,7 @@ func (m *HTTPModifier) Rewrite(payload []byte) (response []byte) { if len(m.config.headerNegativeFilters) > 0 { for _, f := range m.config.headerNegativeFilters { - value, s, _, _ := proto.Header(payload, f.name) + value := proto.Header(payload, f.name) if s != -1 && f.regexp.Match(value) { return From ac3e20978717b49b90bbd724d06711ab4d692fd4 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Fri, 17 Jul 2015 11:38:42 +0500 Subject: [PATCH 23/26] One more fix --- http_modifier.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/http_modifier.go b/http_modifier.go index e2c50cb..14cdf85 100644 --- a/http_modifier.go +++ b/http_modifier.go @@ -100,7 +100,7 @@ func (m *HTTPModifier) Rewrite(payload []byte) (response []byte) { for _, f := range m.config.headerNegativeFilters { value := proto.Header(payload, f.name) - if s != -1 && f.regexp.Match(value) { + if len(value) > 0 && f.regexp.Match(value) { return } } From 107982394ebc31bb18bd0cf34efc172bd2bd0759 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Fri, 17 Jul 2015 11:47:41 +0500 Subject: [PATCH 24/26] Run all tests --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 7f2d291..d86d33e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,3 +1,3 @@ language: go go: 1.4.2 -script: sudo -E bash -c "source /etc/profile && eval '$(gimme 1.4.2)' && export GOPATH=$HOME/gopath:$GOPATH && go get && GORACE='halt_on_error=1' go test -v" \ No newline at end of file +script: sudo -E bash -c "source /etc/profile && eval '$(gimme 1.4.2)' && export GOPATH=$HOME/gopath:$GOPATH && go get && GORACE='halt_on_error=1' go test ./... -v -timeout 15" \ No newline at end of file From 5c21600bf9a67d96175d922adc4d387baa3b4d14 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Fri, 17 Jul 2015 12:12:04 +0500 Subject: [PATCH 25/26] Fix timeout --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index d86d33e..d491b84 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,3 +1,3 @@ language: go go: 1.4.2 -script: sudo -E bash -c "source /etc/profile && eval '$(gimme 1.4.2)' && export GOPATH=$HOME/gopath:$GOPATH && go get && GORACE='halt_on_error=1' go test ./... -v -timeout 15" \ No newline at end of file +script: sudo -E bash -c "source /etc/profile && eval '$(gimme 1.4.2)' && export GOPATH=$HOME/gopath:$GOPATH && go get && GORACE='halt_on_error=1' go test ./... -v -timeout 15s" \ No newline at end of file From 0a3058213180f0d8a1c12422f99f035e7880736c Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Fri, 17 Jul 2015 12:57:36 +0500 Subject: [PATCH 26/26] Fix tests by adding small delay --- Makefile | 2 +- input_raw_test.go | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index cb4bb17..4e4a814 100644 --- a/Makefile +++ b/Makefile @@ -20,7 +20,7 @@ drace: docker run -v `pwd`:$(SOURCE_PATH) -t -i --env GORACE="halt_on_error=1" gor go test ./... $(ARGS) -v -race -timeout 15s dtest: - docker run -v `pwd`:$(SOURCE_PATH) -t -i gor go test ./... $(ARGS) -v -timeout 15s + docker run -v `pwd`:$(SOURCE_PATH) -t -i gor go test ./... $(ARGS) -v -timeout 5s dcover: docker run -v `pwd`:$(SOURCE_PATH) -t -i --env GORACE="halt_on_error=1" gor go test $(ARGS) -race -v -timeout 15s -coverprofile=coverage.out diff --git a/input_raw_test.go b/input_raw_test.go index 7380fad..4bcca9a 100644 --- a/input_raw_test.go +++ b/input_raw_test.go @@ -12,6 +12,7 @@ import ( "strings" "sync" "testing" + "time" ) func TestRAWInput(t *testing.T) { @@ -33,6 +34,8 @@ func TestRAWInput(t *testing.T) { client := NewHTTPClient(address, &HTTPClientConfig{}) + time.Sleep(time.Millisecond) + go Start(quit) for i := 0; i < 100; i++ {