diff --git a/Dockerfile b/Dockerfile index 914094e..3f03461 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ -FROM alpine:latest +FROM alpine:3.6 RUN apk update && apk add ca-certificates && update-ca-certificates && apk add openssl -RUN wget https://github.com/buger/goreplay/releases/download/v0.16.0.2/gor_0.16.0_x64.tar.gz -O gor.tar.gz +RUN wget https://github.com/buger/goreplay/releases/download/v0.16.1/gor_0.16.1_x64.tar.gz -O gor.tar.gz RUN tar xzf gor.tar.gz -ENTRYPOINT ./gor +ENTRYPOINT ["./goreplay"] \ No newline at end of file diff --git a/README.md b/README.md index 0b09462..227e8ff 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -[![GitHub release](https://img.shields.io/github/release/buger/gor.svg?maxAge=3600)](https://github.com/buger/goreplay/releases) [![codebeat](https://codebeat.co/badges/6427d589-a78e-416c-a546-d299b4089893)](https://codebeat.co/projects/github-com-buger-gor) [![Go Report Card](https://goreportcard.com/badge/github.com/buger/gor)](https://goreportcard.com/report/github.com/buger/gor) [![Join the chat at https://gitter.im/buger/gor](https://badges.gitter.im/buger/gor.svg)](https://gitter.im/buger/gor?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) +[![GitHub release](https://img.shields.io/github/release/buger/gor.svg?maxAge=3600)](https://github.com/buger/goreplay/releases) [![codebeat](https://codebeat.co/badges/6427d589-a78e-416c-a546-d299b4089893)](https://codebeat.co/projects/github-com-buger-gor) [![Go Report Card](https://goreportcard.com/badge/github.com/buger/gor)](https://goreportcard.com/report/github.com/buger/gor) [![Join the chat at https://gitter.im/buger/gor](https://badges.gitter.im/buger/gor.svg)](https://gitter.im/buger/gor?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) [![Reviewed by Hound](https://img.shields.io/badge/Reviewed_by-Hound-8E64B0.svg)](https://houndci.com) ![Go Replay](http://i.imgur.com/ZG2ki5n.png) @@ -10,7 +10,7 @@ GoReplay is the simplest and safest way to test your app using real traffic before you put it into production. -As your application grows, the effort required to test it also grows exponentially. GoReplay offers you the simple idea of reusing your existing traffic for testing, which makes it incredibly powerful. Our state of art technique allows to analyze and record your application traffic without affecting it. This eliminates the risks that come with putting a third party component in the critical path. +As your application grows, the effort required to test it also grows exponentially. GoReplay offers you the simple idea of reusing your existing traffic for testing, which makes it incredibly powerful. Our state of art technique allows you to analyze and record your application traffic without affecting it. This eliminates the risks that come with putting a third party component in the critical path. GoReplay increases your confidence in code deployments, configuration changes and infrastructure changes. Did we mention that no coding is required? @@ -43,7 +43,7 @@ We have created a [GoReplay PRO](https://goreplay.org/pro.html) extension which ## Problems? If you have a problem, please review the [FAQ](https://github.com/buger/goreplay/wiki/FAQ) and [Troubleshooting](https://github.com/buger/goreplay/wiki/Troubleshooting) wiki pages. Searching the [issues](https://github.com/buger/goreplay/issues) for your problem is also a good idea. -All bug-reports and suggestions should go though Github Issues or our [Google Group](https://groups.google.com/forum/#!forum/gor-users) (you can just send email to gor-users@googlegroups.com). +All bug-reports and suggestions should go through Github Issues or our [Google Group](https://groups.google.com/forum/#!forum/gor-users) (you can just send email to gor-users@googlegroups.com). If you have a private question feel free to send email to support@gortool.com. @@ -68,7 +68,7 @@ If you have a private question feel free to send email to support@gortool.com. * [Granify](http://granify.com) - AI backed SaaS solution that enables online retailers to maximise their sales * And many more! -If you are using Gor we are happy add you to the list and share your story, just write to: hello@goreplay.org +If you are using Gor, we are happy to add you to the list and share your story, just write to: hello@goreplay.org ## Author diff --git a/docs/Distributed-configuration.md b/docs/Distributed-configuration.md index 2795bf8..e08cc28 100644 --- a/docs/Distributed-configuration.md +++ b/docs/Distributed-configuration.md @@ -13,7 +13,7 @@ If you have multiple replay machines you can split traffic among them using `--s gor --input-raw :80 --split-output --output-tcp replay1.local:28020 --output-tcp replay2.local:28020 ``` -[GoReplay PRO](https://goreplay.com/pro.html) support accurate recording and replaying of tcp sessions, and when `--recognize-tcp-sessions` option is passed, instead of round-robin it will use a smarter algorithm which ensures that same sessions will be sent to the same replay instance. +[GoReplay PRO](https://goreplay.org/pro.html) support accurate recording and replaying of tcp sessions, and when `--recognize-tcp-sessions` option is passed, instead of round-robin it will use a smarter algorithm which ensures that same sessions will be sent to the same replay instance. In case if you are planning a large load testing, you may consider use separate master instance which will control Gor slaves which actually replay traffic. For example: diff --git a/docs/Middleware.md b/docs/Middleware.md index 7c98cbe..5edd111 100644 --- a/docs/Middleware.md +++ b/docs/Middleware.md @@ -26,7 +26,7 @@ Simple bash echo middleware (returns same request) will look like this: ```bash while read line; do echo $line -end +done ``` Middleware can be enabled using `--middleware` option, by specifying path to executable file: @@ -71,4 +71,4 @@ Imagine that you have auth system that randomly generate access tokens, which us *** -You may also read about [[Request filtering]], [[Rate limiting]] and [[Request rewriting]]. \ No newline at end of file +You may also read about [[Request filtering]], [[Rate limiting]] and [[Request rewriting]]. diff --git a/docs/Request-rewriting.md b/docs/Request-rewriting.md index 825610b..fb378b7 100644 --- a/docs/Request-rewriting.md +++ b/docs/Request-rewriting.md @@ -23,16 +23,16 @@ Set request header, if header already exists it will be overwritten. May be usef ``` gor --input-raw :80 --output-http "http://staging.server" \ - --http-header "User-Agent: Replayed by Gor" \ - --http-header "Enable-Feature-X: true" + --http-set-header "User-Agent: Replayed by Gor" \ + --http-set-header "Enable-Feature-X: true" ``` #### Host header -Host header gets special treatment. By default Host get set to the value specified in --output-http. If you manually set --http-header "Host: anonther.com", Gor will not override Host value. +Host header gets special treatment. By default Host get set to the value specified in --output-http. If you manually set --http-set-header "Host: anonther.com", Gor will not override Host value. If you app accepts traffic from multiple domains, and you want to keep original headers, there is specific `--http-original-host` with tells Gor do not touch Host header at all. *** -You may also read about [[Request filtering]], [[Rate limiting]] and [[Middleware]] \ No newline at end of file +You may also read about [[Request filtering]], [[Rate limiting]] and [[Middleware]] diff --git a/elasticsearch.go b/elasticsearch.go index b46f277..01935a0 100644 --- a/elasticsearch.go +++ b/elasticsearch.go @@ -1,9 +1,9 @@ package main import ( - "net/url" "encoding/json" "log" + "net/url" "strings" //"regexp" "time" @@ -72,10 +72,10 @@ func parseURI(URI string) (err error, index string) { // check URL validity by extracting host and undex values. host := parsedUrl.Host urlPathParts := strings.Split(parsedUrl.Path, "/") - index = urlPathParts[len(urlPathParts) - 1 ] + index = urlPathParts[len(urlPathParts)-1] // force index specification in uri : ie no implicit index - if (host == "" || index == "") { + if host == "" || index == "" { err = new(ESUriErorr) } diff --git a/elasticsearch_test.go b/elasticsearch_test.go index c29724d..7bddcff 100644 --- a/elasticsearch_test.go +++ b/elasticsearch_test.go @@ -6,19 +6,19 @@ import ( const expectedIndex = "gor" -func assertExpectedGorIndex (index string, t *testing.T) { +func assertExpectedGorIndex(index string, t *testing.T) { if expectedIndex != index { t.Fatalf("Expected index %s but got %s", expectedIndex, index) } } -func assertExpectedIndex (expectedIndex string, index string, t *testing.T) { +func assertExpectedIndex(expectedIndex string, index string, t *testing.T) { if expectedIndex != index { t.Fatalf("Expected index %s but got %s", expectedIndex, index) } } -func assertExpectedError (returnedError error, t *testing.T) { +func assertExpectedError(returnedError error, t *testing.T) { expectedError := new(ESUriErorr) if expectedError != returnedError { @@ -26,9 +26,9 @@ func assertExpectedError (returnedError error, t *testing.T) { } } -func assertNoError (returnedError error, t *testing.T) { +func assertNoError(returnedError error, t *testing.T) { if nil != returnedError { - t.Errorf("Expected no error got %s", returnedError) + t.Errorf("Expected no err but got %s", returnedError) } } @@ -38,7 +38,7 @@ func assertNoError (returnedError error, t *testing.T) { func TestElasticConnectionBuildFailWithoutScheme(t *testing.T) { uri := "localhost:9200/" + expectedIndex - err, _ := parseURI(uri) + err, _ := parseURI(uri) assertExpectedError(err, t) } @@ -48,7 +48,7 @@ func TestElasticConnectionBuildFailWithoutScheme(t *testing.T) { func TestElasticConnectionBuildFailWithoutIndex(t *testing.T) { uri := "http://localhost:9200" - err, index := parseURI(uri) + err, index := parseURI(uri) assertExpectedIndex("", index, t) diff --git a/examples/middleware/echo.py b/examples/middleware/echo.py index f8b7c99..a38d1af 100644 --- a/examples/middleware/echo.py +++ b/examples/middleware/echo.py @@ -55,7 +55,7 @@ def process_stdin(): request_type_id = int(raw_metadata.split(b' ')[0]) log('Request type: {}'.format({ 1: 'Request', - 2: 'Original Request', + 2: 'Original Response', 3: 'Replayed Response' }[request_type_id])) log('===================================') diff --git a/examples/middleware/echo.rb b/examples/middleware/echo.rb index 75311a8..7afe673 100755 --- a/examples/middleware/echo.rb +++ b/examples/middleware/echo.rb @@ -1,12 +1,12 @@ #!/usr/bin/env ruby # encoding: utf-8 -while data = STDIN.gets # continiously read line from STDIN +while data = STDIN.gets # continuously read line from STDIN next unless data data = data.chomp # remove end of line symbol decoded = [data].pack("H*") # decode base64 encoded request - # dedoded value is raw HTTP payload, example: + # decoded value is raw HTTP payload, example: # # POST /post HTTP/1.1 # Content-Length: 7 diff --git a/gor_stat.go b/gor_stat.go index 205640e..2273a5c 100644 --- a/gor_stat.go +++ b/gor_stat.go @@ -40,7 +40,7 @@ func (s *GorStat) Write(latest int) { s.max = latest } if latest != 0 { - s.mean = (s.mean + latest) / 2 + s.mean = ((s.mean * s.count) + latest) / (s.count + 1) } s.latest = latest s.count = s.count + 1 diff --git a/http_client.go b/http_client.go index 7efc237..6ae5eac 100644 --- a/http_client.go +++ b/http_client.go @@ -113,16 +113,10 @@ func (c *HTTPClient) Disconnect() { } } -func (c *HTTPClient) isAlive() bool { - one := make([]byte, 1) - +func (c *HTTPClient) isAlive(readBytes *int) bool { // Ready 1 byte from socket without timeout to check if it not closed c.conn.SetReadDeadline(time.Now().Add(time.Millisecond)) - _, err := c.conn.Read(one) - - if err == nil { - return true - } + n, err := c.conn.Read(c.respBuf[:1]) if err == io.EOF { Debug("[HTTPClient] connection closed, reconnecting") @@ -133,7 +127,10 @@ func (c *HTTPClient) isAlive() bool { Debug("Detected broken pipe.", err) return false } - + if n != 0 { + *readBytes += n + Debug("[HTTPClient] isAlive readBytes ", *readBytes) + } return true } @@ -153,7 +150,8 @@ func (c *HTTPClient) Send(data []byte) (response []byte, err error) { } }() - if c.conn == nil || !c.isAlive() { + var readBytes, n int + if c.conn == nil || !c.isAlive(&readBytes) { Debug("[HTTPClient] Connecting:", c.baseURL) if err = c.Connect(); err != nil { log.Println("[HTTPClient] Connection error:", err) @@ -181,10 +179,10 @@ func (c *HTTPClient) Send(data []byte) (response []byte, err error) { if _, err = c.conn.Write(data); err != nil { Debug("[HTTPClient] Write error:", err, c.baseURL) response = errorPayload(HTTP_TIMEOUT) + c.Disconnect() return } - var readBytes, n int var currentChunk []byte timeout = time.Now().Add(c.config.Timeout) chunked := false @@ -205,13 +203,21 @@ func (c *HTTPClient) Send(data []byte) (response []byte, err error) { currentContentLength += n } else { // If headers are finished - - if bytes.Contains(c.respBuf[:readBytes], proto.EmptyLine) { + var firstEmptyLine = bytes.Index(c.respBuf[:readBytes], proto.EmptyLine) + if firstEmptyLine != -1 { if bytes.Equal(proto.Header(c.respBuf[:readBytes], []byte("Transfer-Encoding")), []byte("chunked")) { chunked = true } else { status, _ := strconv.Atoi(string(proto.Status(c.respBuf[:readBytes]))) - if (status >= 100 && status < 200) || status == 204 || status == 304 { + // We want to soak up all 100 Continues received to get the real result code + if status >= 100 && status < 200 { + timeout = time.Now().Add(c.config.Timeout) + var deleteLen = firstEmptyLine + len(proto.EmptyLine) + copy(c.respBuf, c.respBuf[deleteLen:readBytes]) + readBytes -= deleteLen + chunks-- + continue + } else if status == 204 || status == 304 { contentLength = 0 break } else { @@ -298,9 +304,9 @@ func (c *HTTPClient) Send(data []byte) (response []byte, err error) { if err != nil && readBytes == 0 { maxRead := 100 - if readBytes < maxRead { - maxRead = readBytes - } + if readBytes < maxRead { + maxRead = readBytes + } Debug("[HTTPClient] Response read timeout error", err, c.conn, readBytes, string(c.respBuf[:maxRead])) response = errorPayload(HTTP_TIMEOUT) c.Disconnect() @@ -309,9 +315,9 @@ func (c *HTTPClient) Send(data []byte) (response []byte, err error) { if readBytes < 4 || string(c.respBuf[:4]) != "HTTP" { maxRead := 100 - if readBytes < maxRead { - maxRead = readBytes - } + if readBytes < maxRead { + maxRead = readBytes + } Debug("[HTTPClient] Response read unknown error", err, c.conn, readBytes, string(c.respBuf[:maxRead])) response = errorPayload(HTTP_UNKNOWN_ERROR) c.Disconnect() diff --git a/http_modifier.go b/http_modifier.go index 60c4501..b397b5a 100644 --- a/http_modifier.go +++ b/http_modifier.go @@ -2,9 +2,9 @@ package main import ( "bytes" - "strings" - "hash/fnv" "encoding/base64" + "hash/fnv" + "strings" "github.com/buger/gor-pro/proto" ) diff --git a/http_modifier_settings.go b/http_modifier_settings.go index 5ae70e6..7291162 100644 --- a/http_modifier_settings.go +++ b/http_modifier_settings.go @@ -81,7 +81,6 @@ func (h *HTTPHeaderBasicAuthFilters) Set(value string) error { return nil } - // // Handling of --http-allow-header-hash and --http-allow-param-hash options // diff --git a/http_modifier_test.go b/http_modifier_test.go index e39e0af..ac7baba 100644 --- a/http_modifier_test.go +++ b/http_modifier_test.go @@ -97,7 +97,7 @@ func TestHTTPHeaderBasicAuthFilters(t *testing.T) { payload = []byte("POST /post HTTP/1.1\r\nContent-Length: 88\r\nAuthorization: Basic Y3VzdG9tZXI2OnJlc3RAMTIzXlRFU1Q==\r\n\r\na=1&b=2") if len(modifier.Rewrite(payload)) == 0 { t.Error("Request should pass filters") - } + } filters = HTTPHeaderBasicAuthFilters{} // Setting filter that not match our header @@ -115,7 +115,7 @@ func TestHTTPHeaderBasicAuthFilters(t *testing.T) { payload = []byte("POST /post HTTP/1.1\r\nContent-Length: 88\r\nAuthorization: Basic bWlja2V5IG1vdXNlOmhhcHB5MTIz\r\n\r\na=1&b=2") if len(modifier.Rewrite(payload)) == 0 { t.Error("Request should pass filters") - } + } } func TestHTTPModifierURLRewrite(t *testing.T) { diff --git a/input_kafka.go b/input_kafka.go index e03fdbb..90c17cd 100644 --- a/input_kafka.go +++ b/input_kafka.go @@ -3,6 +3,7 @@ package main import ( "encoding/json" "log" + "strings" "github.com/Shopify/sarama" "github.com/Shopify/sarama/mocks" @@ -27,7 +28,8 @@ func NewKafkaInput(address string, config *KafkaConfig) *KafkaInput { con = config.consumer } else { var err error - con, err = sarama.NewConsumer([]string{config.host}, c) + //con, err = sarama.NewConsumer([]string{config.host}, c) + con, err = sarama.NewConsumer(strings.Split(config.host, ","), c) if err != nil { log.Fatalln("Failed to start Sarama(Kafka) consumer:", err) diff --git a/input_raw.go b/input_raw.go index 066c648..c18b83c 100644 --- a/input_raw.go +++ b/input_raw.go @@ -21,6 +21,8 @@ type RAWInput struct { listener *raw.Listener protocol raw.TCPProtocol bpfFilter string + timestampType string + bufferSize int } // Available engines for intercepting traffic @@ -31,7 +33,7 @@ const ( ) // NewRAWInput constructor for RAWInput. Accepts address with port as argument. -func NewRAWInput(address string, engine int, trackResponse bool, expire time.Duration, realIPHeader string, protocol string, bpfFilter string) (i *RAWInput) { +func NewRAWInput(address string, engine int, trackResponse bool, expire time.Duration, realIPHeader string, protocol string, bpfFilter string, timestampType string, bufferSize int) (i *RAWInput) { i = new(RAWInput) i.data = make(chan *raw.TCPMessage) i.address = address @@ -41,6 +43,8 @@ func NewRAWInput(address string, engine int, trackResponse bool, expire time.Dur i.realIPHeader = []byte(realIPHeader) i.quit = make(chan bool) i.trackResponse = trackResponse + i.timestampType = timestampType + i.bufferSize = bufferSize switch protocol { case "http": @@ -69,7 +73,7 @@ func (i *RAWInput) Read(data []byte) (int, error) { buf = proto.SetHeader(buf, i.realIPHeader, []byte(msg.IP().String())) } } else { - header = payloadHeader(ResponsePayload, msg.UUID(), msg.AssocMessage.Start.UnixNano(), msg.End.UnixNano()-msg.AssocMessage.Start.UnixNano()) + header = payloadHeader(ResponsePayload, msg.UUID(), msg.Start.UnixNano(), msg.End.UnixNano()-msg.AssocMessage.End.UnixNano()) } copy(data[0:len(header)], header) @@ -87,7 +91,7 @@ func (i *RAWInput) listen(address string) { log.Fatal("input-raw: error while parsing address", err) } - i.listener = raw.NewListener(host, port, i.engine, i.trackResponse, i.expire, i.protocol, i.bpfFilter) + i.listener = raw.NewListener(host, port, i.engine, i.trackResponse, i.expire, i.protocol, i.bpfFilter, i.timestampType, i.bufferSize) ch := i.listener.Receiver() diff --git a/input_raw_test.go b/input_raw_test.go index f4d5f01..f295901 100644 --- a/input_raw_test.go +++ b/input_raw_test.go @@ -44,7 +44,7 @@ func TestRAWInputIPv4(t *testing.T) { var respCounter, reqCounter int64 - input := NewRAWInput(originAddr, EnginePcap, true, testRawExpire, "X-Real-IP", "http", "") + input := NewRAWInput(originAddr, EnginePcap, true, testRawExpire, "X-Real-IP", "http", "", "", 0) defer input.Close() output := NewTestOutput(func(data []byte) { @@ -106,7 +106,7 @@ func TestRAWInputNoKeepAlive(t *testing.T) { originAddr := listener.Addr().String() - input := NewRAWInput(originAddr, EnginePcap, true, testRawExpire, "", "http", "") + input := NewRAWInput(originAddr, EnginePcap, true, testRawExpire, "", "http", "", "", 0) defer input.Close() output := NewTestOutput(func(data []byte) { @@ -152,7 +152,7 @@ func TestRAWInputIPv6(t *testing.T) { var respCounter, reqCounter int64 - input := NewRAWInput(originAddr, EnginePcap, true, testRawExpire, "", "http", "") + input := NewRAWInput(originAddr, EnginePcap, true, testRawExpire, "", "http", "", "", 0) defer input.Close() output := NewTestOutput(func(data []byte) { @@ -203,7 +203,7 @@ func TestInputRAW100Expect(t *testing.T) { originAddr := strings.Replace(origin.Listener.Addr().String(), "[::]", "127.0.0.1", -1) - input := NewRAWInput(originAddr, EnginePcap, true, time.Second, "", "http", "") + input := NewRAWInput(originAddr, EnginePcap, true, time.Second, "", "http", "", "", 0) defer input.Close() // We will use it to get content of raw HTTP request @@ -266,7 +266,7 @@ func TestInputRAWChunkedEncoding(t *testing.T) { })) originAddr := strings.Replace(origin.Listener.Addr().String(), "[::]", "127.0.0.1", -1) - input := NewRAWInput(originAddr, EnginePcap, true, time.Second, "", "http", "") + input := NewRAWInput(originAddr, EnginePcap, true, time.Second, "", "http", "", "", 0) defer input.Close() replay := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -330,7 +330,7 @@ func TestInputRAWLargePayload(t *testing.T) { })) originAddr := strings.Replace(origin.Listener.Addr().String(), "[::]", "127.0.0.1", -1) - input := NewRAWInput(originAddr, EnginePcap, true, testRawExpire, "", "http", "") + input := NewRAWInput(originAddr, EnginePcap, true, testRawExpire, "", "http", "", "", 0) defer input.Close() replay := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { @@ -375,7 +375,7 @@ func BenchmarkRAWInput(b *testing.B) { var respCounter, reqCounter int64 - input := NewRAWInput(originAddr, EnginePcap, true, testRawExpire, "", "http", "") + input := NewRAWInput(originAddr, EnginePcap, true, testRawExpire, "", "http", "", "", 0) defer input.Close() output := NewTestOutput(func(data []byte) { diff --git a/middleware.go b/middleware.go index 0b720e0..0fa0031 100644 --- a/middleware.go +++ b/middleware.go @@ -62,28 +62,39 @@ func (m *Middleware) ReadFrom(plugin io.Reader) { func (m *Middleware) copy(to io.Writer, from io.Reader) { buf := make([]byte, 5*1024*1024) - dst := make([]byte, len(buf)*3) + dst := make([]byte, len(buf)*4) for { nr, _ := from.Read(buf) - if nr > 0 && len(buf) > nr { - payload := buf[0:nr] + if nr == 0 || nr > len(buf) { + continue + } - if Settings.prettifyHTTP { - payload = prettifyHTTP(payload) - nr = len(payload) + payload := buf[0:nr] + + if Settings.prettifyHTTP { + payload = prettifyHTTP(payload) + nr = len(payload) + + if nr*2 > len(dst) { + continue } + } - hex.Encode(dst, payload) - dst[nr*2] = '\n' + if Settings.prettifyHTTP { + payload = prettifyHTTP(payload) + nr = len(payload) + } - m.mu.Lock() - to.Write(dst[0 : nr*2+1]) - m.mu.Unlock() + hex.Encode(dst, payload) + dst[nr*2] = '\n' - if Settings.debug { - Debug("[MIDDLEWARE-MASTER] Sending:", string(buf[0:nr]), "From:", from) - } + m.mu.Lock() + to.Write(dst[0 : nr*2+1]) + m.mu.Unlock() + + if Settings.debug { + Debug("[MIDDLEWARE-MASTER] Sending:", string(buf[0:nr]), "From:", from) } } } diff --git a/middleware/README.md b/middleware/README.md index b53a0f2..4fb231b 100644 --- a/middleware/README.md +++ b/middleware/README.md @@ -103,6 +103,7 @@ Package expose following functions to process raw HTTP payloads: * `httpPathParam` - get param from URL path: `gor.httpPathParam(req.http, queryParam)` * `setHttpPathParam` - set URL param: `req.http = gor.setHttpPathParam(req.http, queryParam, value)` * `httpStatus` - response status code +* `httpHeaders` - get all headers: `gor.httpHeaders(req.http)` * `httpHeader` - get HTTP header: `gor.httpHeader(req.http, "Content-Length")` * `setHttpHeader` - Set HTTP header, returns modified payload: `req.http = gor.setHttpHeader(req.http, "X-Replayed", "1")` * `httpBody` - get HTTP Body: `gor.httpBody(req.http)` diff --git a/middleware/middleware.js b/middleware/middleware.js index 306c17b..7964a15 100755 --- a/middleware/middleware.js +++ b/middleware/middleware.js @@ -41,12 +41,17 @@ function init() { let resp = msg; - ["message", chanPrefix, chanPrefix + "#" + msg.ID].forEach(function(chanID){ + ["message", chanPrefix, chanPrefix + "#" + msg.ID].forEach(function(chanID, idx){ if (proxy.ch[chanID]) { proxy.ch[chanID].forEach(function(ch){ let r = ch.cb(msg); if (r) resp = r; // If one of callback decided not to send response back, do not override it in global callbacks }) + + // Cleanup Individual message channels to avoid memory leaks + if (idx == 2) { + delete proxy.ch[chanID] + } } }) @@ -57,14 +62,25 @@ function init() { } // Clean up old messaged ID specific channels if they are older then 60s - setInterval(function(){ + let gc = function(gcTime){ let now = new Date(); for (k in proxy.ch) { if (k.indexOf("#") == -1) continue; - proxy.ch[k] = proxy.ch[k].filter(function(ch){ return (now - ch.created) < (60 * 1000) }) + proxy.ch[k] = proxy.ch[k].filter(function(ch){ + return (now - ch.created) < gcTime + }) + + if (proxy.ch[k].length == 0) { + delete proxy.ch[k] + } } - }, 1000) + } + proxy.gc = gc + + setInterval(function(){ + gc(10 * 1000) + }, 1000); const readline = require('readline'); const rl = readline.createInterface({ @@ -237,6 +253,21 @@ function setHttpStatus(payload, newStatus) { return setHttpPath(payload, newStatus); } +function httpHeaders(payload) { + var httpHeaderString = payload.slice(0,payload.indexOf("\r\n\r\n") + 4).toString().split("\n").slice(1); + var headers = {}; + + for (var item in httpHeaderString) { + var parts = httpHeaderString[item].split(":"); + + if (parts.length > 1) { + headers[parts[0]] = parts.slice(1).join(":").trim(); + } + } + + return headers; +} + function httpHeader(payload, name) { var currentLine = 0; var i = 0; @@ -291,6 +322,16 @@ function setHttpHeader(payload, name, value) { } } +function deleteHttpHeader(payload, name) { + let header = httpHeader(payload, name); + + if (header) { + return Buffer.concat([payload.slice(0, header.start), payload.slice(header.end+1, payload.length)]) + } + + return payload +} + function httpBody(payload) { return payload.slice(payload.indexOf("\r\n\r\n") + 4, payload.length); } @@ -364,31 +405,72 @@ module.exports = { setHttpStatus: setHttpStatus, httpHeader: httpHeader, setHttpHeader: setHttpHeader, + deleteHttpHeader: deleteHttpHeader, httpBody: httpBody, setHttpBody: setHttpBody, httpBodyParam: httpBodyParam, setHttpBodyParam: setHttpBodyParam, httpCookie: httpCookie, setHttpCookie: setHttpCookie, - test: testRunner + test: testRunner, + benchmark: testBenchmark, + httpHeaders: httpHeaders } // =========== Tests ============== function testRunner(){ - ["init", "parseMessage", "httpMethod", "httpPath", "setHttpHeader", "httpPathParam", "httpHeader", "httpBody", "setHttpBody", "httpBodyParam", "httpCookie", "setHttpCookie"].forEach(function(t){ + ["init", "parseMessage", "httpMethod", "httpPath", "setHttpHeader", "deleteHttpHeader", "httpPathParam", "httpHeader", "httpBody", "setHttpBody", "httpBodyParam", "httpCookie", "setHttpCookie", "httpHeaders"].forEach(function(t){ console.log(`====== Start ${t} =======`) eval(`TEST_${t}()`) console.log(`====== End ${t} =======`) }) } +function testBenchmark(){ + const child_process = require('child_process'); + + let gor = init(); + gor.on("message", function(){ + }); + + gor.on("request", function(){ + }); + + for (var i = 0; i<256; i++) { + let req = parseMessage(Buffer.from("1 2 3\nGET / HTTP/1.1\r\n\r\n").toString('hex')); + req.ID = +Date.now() + gor.emit(req); + + gor.on("request", req.ID+"", function(){ + gor.on("response", req.ID+"", function(){ + }) + }) + + if ( i % 3 == 0 ) { + let resp = parseMessage(Buffer.from("2 2 3\nHTTP/1.1 200 OK\r\n\r\n").toString('hex')); + resp.ID = req.ID + gor.emit(resp); + } + } + + child_process.execSync("sleep 0.01"); + + gor.gc(1) + + fail(JSON.stringify(gor.ch)) +} + // Just print in red color function fail(message) { console.error("\x1b[31m[MIDDLEWARE] %s\x1b[0m", message) } +function log(message) { + console.error(message) +} + function TEST_init() { const child_process = require('child_process'); @@ -571,6 +653,18 @@ function TEST_setHttpHeader() { } } +function TEST_deleteHttpHeader() { + const examplePayload = "GET / HTTP/1.1\r\nUser-Agent: Node\r\nContent-Length: 5\r\n\r\nhello"; + + // Adding new header + let expected = `GET / HTTP/1.1\r\nContent-Length: 5\r\n\r\nhello`; + let p = Buffer.from(examplePayload); + p = deleteHttpHeader(p, "User-Agent", "test"); + if (p != expected) { + console.error(`setHeader failed, expected delete header 'User-Agent' header: ${p}`) + } +} + function TEST_httpBody() { const examplePayload = "GET / HTTP/1.1\r\nUser-Agent: Node\r\nContent-Length: 5\r\n\r\nhello"; let body = httpBody(Buffer.from(examplePayload)); @@ -613,3 +707,25 @@ function TEST_setHttpCookie() { return fail(`Should add new cookie: ${p}`) } } + +function TEST_httpHeaders() { + const examplePayload = "GET / HTTP/1.1\r\nHost: localhost:3000\r\nUser-Agent: Node\r\nContent-Length:5\r\n\r\nhello"; + + let expectedHeaders = {"Host": "localhost:3000", "User-Agent": "Node", "Content-Length": "5"} + let payload = Buffer.from(examplePayload); + let headers = httpHeaders(payload); + + ["Host", "User-Agent", "Content-Length"].forEach(function(header){ + let actual = headers[header]; + let expected = expectedHeaders[header]; + + if (!actual) { + fail(`${header} Header was not found`); + } + + if (actual != expected) { + fail(`${header} Header not Equal to Expected: ${expected} was ${actual}`); + } + + }) +} diff --git a/middleware/package.json b/middleware/package.json index 698165f..9748460 100644 --- a/middleware/package.json +++ b/middleware/package.json @@ -1,10 +1,11 @@ { "name": "goreplay_middleware", - "version": "0.1.13", + "version": "0.1.15", "description": "Package for writing middleware for GoReplay https://goreplay.org", "main": "middleware.js", "scripts": { - "test": "node -e \"var gor = require('./middleware.js'); gor.test(); process.exit()\"" + "test": "node -e \"var gor = require('./middleware.js'); gor.test(); process.exit()\"", + "benchmark": "node -e \"var gor = require('./middleware.js'); gor.benchmark(); process.exit()\"" }, "keywords": [ "middleware", diff --git a/middleware_test.go b/middleware_test.go index 09d19d5..931eca2 100644 --- a/middleware_test.go +++ b/middleware_test.go @@ -118,7 +118,7 @@ func TestEchoMiddleware(t *testing.T) { // Catch traffic from one service fromAddr := strings.Replace(from.Listener.Addr().String(), "[::]", "127.0.0.1", -1) - input := NewRAWInput(fromAddr, EnginePcap, true, testRawExpire, "", "http", "") + input := NewRAWInput(fromAddr, EnginePcap, true, testRawExpire, "", "http", "", "", 0) defer input.Close() // And redirect to another @@ -180,7 +180,7 @@ func TestTokenMiddleware(t *testing.T) { fromAddr := strings.Replace(from.Listener.Addr().String(), "[::]", "127.0.0.1", -1) // Catch traffic from one service - input := NewRAWInput(fromAddr, EnginePcap, true, testRawExpire, "", "http", "") + input := NewRAWInput(fromAddr, EnginePcap, true, testRawExpire, "", "http", "", "", 0) defer input.Close() // And redirect to another diff --git a/output_file.go b/output_file.go index 760715c..395524c 100644 --- a/output_file.go +++ b/output_file.go @@ -192,9 +192,9 @@ func (o *FileOutput) Write(data []byte) (n int, err error) { o.currentID = meta[1] o.payloadType = meta[0] } - + o.updateName() - + if o.file == nil || o.currentName != o.file.Name() { o.mu.Lock() o.Close() diff --git a/plugins.go b/plugins.go index 95cded3..928e741 100644 --- a/plugins.go +++ b/plugins.go @@ -106,7 +106,7 @@ func InitPlugins() { } for _, options := range Settings.inputRAW { - registerPlugin(NewRAWInput, options, engine, Settings.inputRAWTrackResponse, Settings.inputRAWExpire, Settings.inputRAWRealIPHeader, Settings.inputRAWProtocol, Settings.inputRAWBpfFilter) + registerPlugin(NewRAWInput, options, engine, Settings.inputRAWTrackResponse, Settings.inputRAWExpire, Settings.inputRAWRealIPHeader, Settings.inputRAWProtocol, Settings.inputRAWBpfFilter, Settings.inputRAWTimestampType, Settings.inputRawBufferSize) } for _, options := range Settings.inputTCP { diff --git a/proto/proto.go b/proto/proto.go index 9672050..a6d148a 100644 --- a/proto/proto.go +++ b/proto/proto.go @@ -93,6 +93,11 @@ func headerIndex(payload []byte, name []byte) int { return i - len(name) } + // We are at the end + if i == len(payload) { + return -1 + } + if payload[i] != name[j] { break } @@ -338,6 +343,10 @@ func DeleteHeader(payload, name []byte) []byte { // Body returns request/response body func Body(payload []byte) []byte { // 4 -> len(EMPTY_LINE) + if len(payload) < 4 { + return []byte{} + } + return payload[MIMEHeadersEndPos(payload):] } @@ -467,7 +476,7 @@ func Status(payload []byte) []byte { } var httpMethods []string = []string{ - "GET ", "OPTI", "HEAD", "POST", "PUT ", "DELE", "TRAC", "CONN", "PATC" /* custom methods */, "BAN", "PURG", + "GET ", "OPTI", "HEAD", "POST", "PUT ", "DELE", "TRAC", "CONN", "PATC" /* custom methods */, "BAN ", "PURG", "PROP", "MKCO", "COPY", "MOVE", "LOCK", "UNLO", } func IsHTTPPayload(payload []byte) bool { diff --git a/raw_socket_listener/listener.go b/raw_socket_listener/listener.go index 8aa47ce..36d6425 100644 --- a/raw_socket_listener/listener.go +++ b/raw_socket_listener/listener.go @@ -72,7 +72,10 @@ type Listener struct { trackResponse bool messageExpire time.Duration - bpfFilter string + bpfFilter string + timestampType string + + bufferSize int conn net.PacketConn pcapHandles []*pcap.Handle @@ -96,7 +99,7 @@ const ( EnginePcapFile ) -func NewListener(addr string, port string, engine int, trackResponse bool, expire time.Duration, protocol TCPProtocol, bpfFilter string) (l *Listener) { +func NewListener(addr string, port string, engine int, trackResponse bool, expire time.Duration, protocol TCPProtocol, bpfFilter string, timestampType string, bufferSize int) (l *Listener) { l = &Listener{} l.packetsChan = make(chan *packet, 10000) @@ -112,6 +115,8 @@ func NewListener(addr string, port string, engine int, trackResponse bool, expir l.trackResponse = trackResponse l.protocol = protocol l.bpfFilter = bpfFilter + l.timestampType = timestampType + l.bufferSize = bufferSize l.addr = addr _port, _ := strconv.Atoi(port) @@ -331,12 +336,42 @@ func (t *Listener) readPcap() { for _, d := range devices { go func(device pcap.Interface) { - handle, err := pcap.OpenLive(device.Name, 65536, true, t.messageExpire) + inactive, err := pcap.NewInactiveHandle(device.Name) if err != nil { log.Println("Pcap Error while opening device", device.Name, err) wg.Done() return } + + if t.timestampType != "" { + if tt, terr := pcap.TimestampSourceFromString(t.timestampType); terr != nil { + log.Println("Supported timestamp types: ", inactive.SupportedTimestamps(), device.Name) + } else if terr := inactive.SetTimestampSource(tt); terr != nil { + log.Println("Supported timestamp types: ", inactive.SupportedTimestamps(), device.Name) + } + } + + if it, err := net.InterfaceByName(device.Name); err == nil { + // Auto-guess max length of packet to capture + inactive.SetSnapLen(it.MTU + 68*2) + } else { + inactive.SetSnapLen(65536) + } + + inactive.SetTimeout(t.messageExpire) + inactive.SetPromisc(true) + + if t.bufferSize > 0 { + inactive.SetBufferSize(t.bufferSize) + } + + handle, herr := inactive.Activate() + if herr != nil { + log.Println("PCAP Activate error:", herr) + wg.Done() + return + } + defer handle.Close() t.mu.Lock() @@ -426,12 +461,12 @@ func (t *Listener) readPcap() { of = 4 case layers.LinkTypeLoop: of = 4 - case layers.LinkTypeRaw: + case layers.LinkTypeRaw, layers.LayerTypeIPv4: of = 0 case layers.LinkTypeLinuxSLL: of = 16 default: - log.Println("Unknown packet layer", packet) + log.Println("Unknown packet layer", decoder, packet) break } @@ -649,12 +684,6 @@ func (t *Listener) readRAWSocket() { } func (t *Listener) buildPacket(packetSrcIP []byte, packetData []byte, timestamp time.Time) *packet { - copyPacketSrcIP := make([]byte, 16) - copyPacketData := make([]byte, len(packetData)) - - copy(copyPacketSrcIP, packetSrcIP) - copy(copyPacketData, packetSrcIP) - return &packet{ srcIP: packetSrcIP, data: packetData, @@ -695,27 +724,33 @@ func (t *Listener) processTCPPacket(packet *TCPPacket) { } }() + var responseRequest *TCPMessage var message *TCPMessage isIncoming := packet.DestPort == t.port if t.protocol == ProtocolHTTP { - // Seek for 100-expect chunks - if parentAck, ok := t.seqWithData[packet.Seq]; ok { + if !isIncoming { + responseRequest, _ = t.respAliases[packet.Ack] + } + + // Seek for 100-expect chunks + // `packet.Ack != parentAck` is protection for clients who send data without ignoring server 100-continue response, e.g have data chunks have same Ack + if parentAck, ok := t.seqWithData[packet.Seq]; ok && packet.Ack != parentAck { // Skip zero-length chunks https://github.com/buger/goreplay/issues/496 if len(packet.Data) == 0 { return } - // In case if non-first data chunks comes first - for _, m := range t.messages { - if m.Ack == packet.Ack && bytes.Equal(m.packets[0].Addr, packet.Addr) { - t.deleteMessage(m) - - if m.AssocMessage != nil { - m.setAssocMessage(nil) - } + // In case if non-first data chunks comes first + for _, m := range t.messages { + if m.Ack == packet.Ack && bytes.Equal(m.packets[0].Addr, packet.Addr) { + t.deleteMessage(m) + if m.AssocMessage != nil { + m.AssocMessage.setAssocMessage(nil) + m.setAssocMessage(nil) + } for _, pkt := range m.packets { // log.Println("Updating ack", parentAck, pkt.Ack) pkt.UpdateAck(parentAck) @@ -742,12 +777,6 @@ func (t *Listener) processTCPPacket(packet *TCPPacket) { packet.UpdateAck(alias) } - var responseRequest *TCPMessage - - if !isIncoming { - responseRequest, _ = t.respAliases[packet.Ack] - } - message, ok := t.messages[packet.ID] if !ok { @@ -769,8 +798,9 @@ func (t *Listener) processTCPPacket(packet *TCPPacket) { // Handling Expect: 100-continue requests if t.protocol == ProtocolHTTP && message.expectType == httpExpect100Continue && len(message.packets) == message.headerPacket+1 { - seq := packet.Seq + uint32(message.Size()) + seq := packet.Seq + uint32(len(packet.Data)) t.seqWithData[seq] = packet.Ack + message.DataSeq = seq message.complete = false @@ -796,7 +826,13 @@ func (t *Listener) processTCPPacket(packet *TCPPacket) { packet.Data = proto.DeleteHeader(packet.Data, bExpectHeader) } - // log.Println("Received message:", string(message.Bytes()), message.ID(), t.messages) + // If client do sends Expect: 100-continue but do not respect server response + if message.expectType == httpExpect100Continue && (message.headerPacket != -1 && len(message.packets) > message.headerPacket+1) { + delete(t.seqWithData, message.DataSeq) + seq := packet.Seq + uint32(len(packet.Data)) + t.seqWithData[seq] = packet.Ack + message.DataSeq = seq + } if isIncoming { // If message have multiple packets, delete previous alias diff --git a/raw_socket_listener/listener_test.go b/raw_socket_listener/listener_test.go index 4e9809c..fc736a2 100644 --- a/raw_socket_listener/listener_test.go +++ b/raw_socket_listener/listener_test.go @@ -12,7 +12,7 @@ import ( func TestRawListenerInput(t *testing.T) { var req, resp *TCPMessage - listener := NewListener("", "0", EnginePcap, true, 10*time.Millisecond, ProtocolHTTP, "") + listener := NewListener("", "0", EnginePcap, true, 10*time.Millisecond, ProtocolHTTP, "", "", 0) defer listener.Close() reqPacket := buildPacket(true, 1, 1, []byte("GET / HTTP/1.1\r\n\r\n"), time.Now()) @@ -46,79 +46,134 @@ func TestRawListenerInput(t *testing.T) { } } -func TestSingleAck100Continue(t *testing.T) { - listener := NewListener("", "0", EnginePcap, true, 10*time.Millisecond, "") - defer listener.Close() - - reqPacket1 := buildPacket( +func firstPacket(payload []byte) *TCPPacket { + return buildPacket( true, 1, 1, - []byte("POST / HTTP/1.1\r\nExpect: 100-continue\r\nContent-Length: 4\r\n\r\n"), time.Now()) + payload, + time.Now(), + ) +} - respPacket1 := buildPacket(false, - uint32(len(reqPacket1.Data)) + reqPacket1.Seq, - 1, - []byte(""), time.Now()) +func nextPacket(prev *TCPPacket, payload []byte) *TCPPacket { + return buildPacket( + prev.SrcPort == 1, + prev.Ack, + prev.Seq+uint32(len(prev.Data)), + payload, + prev.timestamp.Add(time.Millisecond), + ) +} - respPacket2 := buildPacket( false, - uint32(len(reqPacket1.Data)) + reqPacket1.Seq, - 1, - []byte("HTTP/1.1 100 Continue\r\n"), time.Now()) +func responsePacket(prev *TCPPacket, payload []byte) *TCPPacket { + return buildPacket( + !(prev.SrcPort == 1), + prev.Seq+uint32(len(prev.Data)), + prev.Ack, + payload, + prev.timestamp.Add(time.Millisecond), + ) +} - reqPacket3 := buildPacket(true, - uint32(len(reqPacket1.Data)) + respPacket1.Seq, - reqPacket1.Seq+uint32(len(reqPacket1.Data)), - []byte("DATA"), time.Now()) +func TestHEADRequestNoBody(t *testing.T) { + listener := NewListener("", "0", EnginePcap, true, 10*time.Millisecond, ProtocolHTTP, "", "", 0) + defer listener.Close() - respPacket3 := buildPacket(false, - uint32(len(reqPacket3.Data)) + reqPacket3.Seq, - respPacket1.Seq+uint32(len(respPacket1.Data)), []byte("HTTP/1.1 200 OK\r\n\r\n"), time.Now()) + reqPacket := firstPacket([]byte("HEAD / HTTP/1.1\r\nContent-Length: 0\r\n\r\n")) + respPacket := responsePacket(reqPacket, []byte("HTTP/1.1 200 OK\r\nContent-Length: 100\r\n\r\n")) + + listener.packetsChan <- reqPacket.dump() + listener.packetsChan <- respPacket.dump() + + var req, resp *TCPMessage + select { + case req = <-listener.messagesChan: + case <-time.After(time.Millisecond): + t.Error("Should return request immediately") + return + } + + if !req.IsIncoming { + t.Error("Should be request") + } + + select { + case resp = <-listener.messagesChan: + case <-time.After(20 * time.Millisecond): + t.Error("Should return response immediately") + return + } + + if resp.IsIncoming { + t.Error("Should be response") + } +} + +func TestSingleAck100Continue(t *testing.T) { + listener := NewListener("", "0", EnginePcap, true, 10*time.Millisecond, ProtocolHTTP, "", "", 0) + defer listener.Close() + + reqPacket1 := firstPacket([]byte("POST / HTTP/1.1\r\nExpect: 100-continue\r\nContent-Length: 4\r\n\r\n")) + respPacket1 := responsePacket(reqPacket1, []byte("")) + respPacket2 := responsePacket(reqPacket1, []byte("HTTP/1.1 100 Continue\r\n")) + reqPacket2 := responsePacket(respPacket2, []byte("DATA")) + respPacket3 := responsePacket(reqPacket2, []byte("HTTP/1.1 200 OK\r\n\r\n")) result := []byte("POST / HTTP/1.1\r\nContent-Length: 4\r\n\r\nDATA") testRawListener100Continue(t, listener, result, reqPacket1, respPacket1, respPacket2, - reqPacket3, - respPacket3 ) + reqPacket2, + respPacket3) } - -func TestDoubleAck100Continue(t *testing.T) { - listener := NewListener("", "0", EnginePcap, true, 10*time.Millisecond, "") +func Test100ContinueWithoutWaiting(t *testing.T) { + listener := NewListener("", "0", EnginePcap, true, 10*time.Millisecond, ProtocolHTTP, "", "", 0) defer listener.Close() - reqPacket1 := buildPacket( - true, - 1, - 1, - []byte("POST / HTTP/1.1\r\nExpect: 100-continue\r\nContent-Length: 4\r\n\r\n"), time.Now()) + req1 := firstPacket([]byte("POST / HTTP/1.1\r\nExpect: 100-continue\r\nContent-Length: 4\r\n\r\n")) + req2 := nextPacket(req1, []byte("DATA")) + resp1 := responsePacket(req1, []byte("HTTP/1.1 100 Continue\r\n")) + resp2 := responsePacket(req2, []byte("HTTP/1.1 200 OK\r\n\r\n")) - respPacket1 := buildPacket(false, - uint32(len(reqPacket1.Data)) + reqPacket1.Seq, - 1, - []byte(""), time.Now()) + result := []byte("POST / HTTP/1.1\r\nContent-Length: 4\r\n\r\nDATA") - respPacket2 := buildPacket( false, - uint32(len(reqPacket1.Data)) + reqPacket1.Seq, - 1, - []byte("HTTP/1.1 100 Continue\r\n"), time.Now()) + testRawListener100Continue(t, listener, result, + req1, req2, resp1, resp2) +} - reqPacket2 := buildPacket(true, - uint32(len(reqPacket1.Data)) + respPacket1.Seq, - reqPacket1.Seq+uint32(len(reqPacket1.Data)), - []byte(""), time.Now()) +// Client first sends data without waiting 100-continue, but once response received, generate packets based on Ack payload +func Test100ContinueMixed(t *testing.T) { + listener := NewListener("", "0", EnginePcap, true, 10*time.Millisecond, ProtocolHTTP, "", "", 0) + defer listener.Close() - reqPacket3 := buildPacket(true, - uint32(len(reqPacket1.Data)) + respPacket1.Seq, - reqPacket1.Seq+uint32(len(reqPacket1.Data)), - []byte("DATA"), time.Now()) + req1 := firstPacket([]byte("POST / HTTP/1.1\r\nExpect: 100-continue\r\nContent-Length: 12\r\n\r\n")) + req2 := nextPacket(req1, []byte("DAT1")) + resp1 := responsePacket(req1, []byte("HTTP/1.1 100 Continue\r\n\r\n")) + req3 := responsePacket(resp1, []byte("DAT2")) + req3.Seq = req2.Seq + uint32(len(req2.Data)) + req4 := nextPacket(req3, []byte("DAT3")) + resp2 := responsePacket(req4, []byte("HTTP/1.1 200 OK\r\n\r\n")) - respPacket3 := buildPacket(false, - uint32(len(reqPacket3.Data)) + reqPacket3.Seq, - respPacket1.Seq+uint32(len(respPacket1.Data)), - []byte("HTTP/1.1 200 OK\r\n\r\n"), time.Now()) + result := []byte("POST / HTTP/1.1\r\nContent-Length: 12\r\n\r\nDAT1DAT2DAT3") + + testRawListener100Continue(t, listener, result, + req1, req2, req3, req4, resp1, resp2) +} + +func TestDoubleAck100Continue(t *testing.T) { + listener := NewListener("", "0", EnginePcap, true, 10*time.Millisecond, ProtocolHTTP, "", "", 0) + defer listener.Close() + + reqPacket1 := firstPacket([]byte("POST / HTTP/1.1\r\nExpect: 100-continue\r\nContent-Length: 4\r\n\r\n")) + + respPacket1 := responsePacket(reqPacket1, []byte("")) + respPacket2 := responsePacket(reqPacket1, []byte("HTTP/1.1 100 Continue\r\n")) + reqPacket2 := responsePacket(respPacket2, []byte("")) + reqPacket3 := responsePacket(respPacket2, []byte("DATA")) + respPacket3 := responsePacket(reqPacket3, []byte("HTTP/1.1 200 OK\r\n\r\n")) result := []byte("POST / HTTP/1.1\r\nContent-Length: 4\r\n\r\nDATA") @@ -126,14 +181,13 @@ func TestDoubleAck100Continue(t *testing.T) { reqPacket1, respPacket1, respPacket2, reqPacket2, reqPacket3, - respPacket3 ) + respPacket3) } - func TestRawListenerInputResponseByClose(t *testing.T) { var req, resp *TCPMessage - listener := NewListener("", "0", EnginePcap, true, 10*time.Millisecond, "") + listener := NewListener("", "0", EnginePcap, true, 10*time.Millisecond, ProtocolHTTP, "", "", 0) defer listener.Close() reqPacket := buildPacket(true, 1, 1, []byte("GET / HTTP/1.1\r\n\r\n"), time.Now()) @@ -173,11 +227,7 @@ func TestRawListenerInputResponseByClose(t *testing.T) { func TestRawListenerInputWithoutResponse(t *testing.T) { var req *TCPMessage -<<<<<<< HEAD - listener := NewListener("", "0", EnginePcap, false, 10*time.Millisecond, ProtocolHTTP) -======= - listener := NewListener("", "0", EnginePcap, false, 10*time.Millisecond, "") ->>>>>>> d309650589a56db2bfa93c533337e3db1cfb0425 + listener := NewListener("", "0", EnginePcap, false, 10*time.Millisecond, ProtocolHTTP, "", "", 0) defer listener.Close() reqPacket := buildPacket(true, 1, 1, []byte("GET / HTTP/1.1\r\n\r\n"), time.Now()) @@ -199,15 +249,11 @@ func TestRawListenerInputWithoutResponse(t *testing.T) { func TestRawListenerResponse(t *testing.T) { var req, resp *TCPMessage -<<<<<<< HEAD - listener := NewListener("", "0", EnginePcap, true, 10*time.Millisecond, ProtocolHTTP) -======= - listener := NewListener("", "0", EnginePcap, true, 10*time.Millisecond, "") ->>>>>>> d309650589a56db2bfa93c533337e3db1cfb0425 + listener := NewListener("", "0", EnginePcap, true, 10*time.Millisecond, ProtocolHTTP, "", "", 0) defer listener.Close() - reqPacket := buildPacket(true, 1, 1, []byte("GET / HTTP/1.1\r\n\r\n"), time.Now()) - respPacket := buildPacket(false, 1+uint32(len(reqPacket.Data)), 2, []byte("HTTP/1.1 200 OK\r\n\r\n"), time.Now()) + reqPacket := firstPacket([]byte("GET / HTTP/1.1\r\n\r\n")) + respPacket := responsePacket(reqPacket, []byte("HTTP/1.1 200 OK\r\n\r\n")) // If response packet comes before request listener.packetsChan <- respPacket.dump() @@ -240,74 +286,37 @@ func TestRawListenerResponse(t *testing.T) { } } +func get100ContinuePackets() (req []*TCPPacket, resp []*TCPPacket) { + req1 := firstPacket([]byte("POST / HTTP/1.1\r\nExpect: 100-continue\r\nContent-Length: 2\r\n\r\n")) + resp1 := responsePacket(req1, []byte("HTTP/1.1 100 Continue\r\n")) + req2 := responsePacket(resp1, []byte("a")) + req3 := nextPacket(req2, []byte("b")) + resp2 := responsePacket(req3, []byte("HTTP/1.1 200 OK\r\n\r\n")) + + return []*TCPPacket{req1, req2, req3}, []*TCPPacket{resp1, resp2} +} + func TestShort100Continue(t *testing.T) { -<<<<<<< HEAD - listener := NewListener("", "0", EnginePcap, true, 10*time.Millisecond, ProtocolHTTP) -======= - listener := NewListener("", "0", EnginePcap, true, 10*time.Millisecond, "") ->>>>>>> d309650589a56db2bfa93c533337e3db1cfb0425 + listener := NewListener("", "0", EnginePcap, true, 10*time.Millisecond, ProtocolHTTP, "", "", 0) defer listener.Close() - reqPacket1 := buildPacket(true, 1, 1, []byte("POST / HTTP/1.1\r\nContent-Length: 2\r\nExpect: 100-continue\r\n\r\n"), time.Now()) - // Packet with data have different Seq - reqPacket2 := buildPacket(true, 2, reqPacket1.Seq+uint32(len(reqPacket1.Data)), []byte("a"), time.Now()) - reqPacket3 := buildPacket(true, 2, reqPacket2.Seq+1, []byte("b"), time.Now()) - - respPacket1 := buildPacket(false, 10, 3, []byte("HTTP/1.1 100 Continue\r\n\r\n"), time.Now()) - - // panic(int(uint32(len(reqPacket1.Data)) + uint32(len(reqPacket2.Data)) + uint32(len(reqPacket3.Data)))) - respPacket2 := buildPacket(false, reqPacket3.Seq+1 /* len of data */, 2, []byte("HTTP/1.1 200 OK\r\n\r\n"), time.Now()) + req, resp := get100ContinuePackets() result := []byte("POST / HTTP/1.1\r\nContent-Length: 2\r\n\r\nab") - testRawListener100Continue(t, listener, result, reqPacket1, reqPacket2, reqPacket3, respPacket1, respPacket2) + testRawListener100Continue(t, listener, result, req[0], req[1], req[2], resp[0], resp[1]) } // Response comes before Request func Test100ContinueWrongOrder(t *testing.T) { -<<<<<<< HEAD - listener := NewListener("", "0", EnginePcap, true, 10*time.Millisecond, ProtocolHTTP) -======= - listener := NewListener("", "0", EnginePcap, true, 10*time.Millisecond, "") ->>>>>>> d309650589a56db2bfa93c533337e3db1cfb0425 + listener := NewListener("", "0", EnginePcap, true, 10*time.Millisecond, ProtocolHTTP, "", "", 0) defer listener.Close() - reqPacket1 := buildPacket(true, 1, 1, []byte("POST / HTTP/1.1\r\nContent-Length: 2\r\nExpect: 100-continue\r\n\r\n"), time.Now()) - // Packet with data have different Seq - reqPacket2 := buildPacket(true, 2, reqPacket1.Seq+uint32(len(reqPacket1.Data)), []byte("a"), time.Now()) - reqPacket3 := buildPacket(true, 2, reqPacket2.Seq+1, []byte("b"), time.Now()) - - respPacket1 := buildPacket(false, 10, 3, []byte("HTTP/1.1 100 Continue\r\n"), time.Now()) - - // panic(int(uint32(len(reqPacket1.Data)) + uint32(len(reqPacket2.Data)) + uint32(len(reqPacket3.Data)))) - respPacket2 := buildPacket(false, reqPacket3.Seq+1 /* len of data */, 2, []byte("HTTP/1.1 200 OK\r\n\r\n"), time.Now()) + req, resp := get100ContinuePackets() result := []byte("POST / HTTP/1.1\r\nContent-Length: 2\r\n\r\nab") - testRawListener100Continue(t, listener, result, respPacket1, respPacket2, reqPacket1, reqPacket2, reqPacket3) -} - -func TestAlt100ContinueHeaderOrder(t *testing.T) { -<<<<<<< HEAD - listener := NewListener("", "0", EnginePcap, true, 10*time.Millisecond, ProtocolHTTP) -======= - listener := NewListener("", "0", EnginePcap, true, 10*time.Millisecond, "") ->>>>>>> d309650589a56db2bfa93c533337e3db1cfb0425 - defer listener.Close() - - reqPacket1 := buildPacket(true, 1, 1, []byte("POST / HTTP/1.1\r\nExpect: 100-continue\r\nContent-Length: 2\r\n\r\n"), time.Now()) - // Packet with data have different Seq - reqPacket2 := buildPacket(true, 2, reqPacket1.Seq+uint32(len(reqPacket1.Data)), []byte("a"), time.Now()) - reqPacket3 := buildPacket(true, 2, reqPacket2.Seq+1, []byte("b"), time.Now()) - - respPacket1 := buildPacket(false, 10, 3, []byte("HTTP/1.1 100 Continue\r\n"), time.Now()) - - // panic(int(uint32(len(reqPacket1.Data)) + uint32(len(reqPacket2.Data)) + uint32(len(reqPacket3.Data)))) - respPacket2 := buildPacket(false, reqPacket3.Seq+1 /* len of data */, 2, []byte("HTTP/1.1 200 OK\r\n\r\n"), time.Now()) - - result := []byte("POST / HTTP/1.1\r\nContent-Length: 2\r\n\r\nab") - - testRawListener100Continue(t, listener, result, reqPacket1, reqPacket2, reqPacket3, respPacket1, respPacket2) + testRawListener100Continue(t, listener, result, resp[0], resp[1], req[0], req[1], req[2]) } func testRawListener100Continue(t *testing.T, listener *Listener, result []byte, packets ...*TCPPacket) { @@ -453,29 +462,22 @@ func permutation(n int, list []*TCPPacket) []*TCPPacket { // Response comes before Request func TestRawListenerChunkedWrongOrder(t *testing.T) { -<<<<<<< HEAD - listener := NewListener("", "0", EnginePcap, true, 10*time.Millisecond, ProtocolHTTP) -======= - listener := NewListener("", "0", EnginePcap, true, 10*time.Millisecond, "") ->>>>>>> d309650589a56db2bfa93c533337e3db1cfb0425 + listener := NewListener("", "0", EnginePcap, true, 10*time.Millisecond, ProtocolHTTP, "", "", 0) defer listener.Close() - reqPacket1 := buildPacket(true, 1, 1, []byte("POST / HTTP/1.1\r\nTransfer-Encoding: chunked\r\nExpect: 100-continue\r\n\r\n"), time.Now()) - // Packet with data have different Seq - reqPacket2 := buildPacket(true, 2, reqPacket1.Seq+uint32(len(reqPacket1.Data)), []byte("1\r\na\r\n"), time.Now()) - reqPacket3 := buildPacket(true, 2, reqPacket2.Seq+uint32(len(reqPacket2.Data)), []byte("1\r\nb\r\n"), time.Now()) - reqPacket4 := buildPacket(true, 2, reqPacket3.Seq+uint32(len(reqPacket3.Data)), []byte("0\r\n\r\n"), time.Now()) + reqPacket1 := firstPacket([]byte("POST / HTTP/1.1\r\nTransfer-Encoding: chunked\r\nExpect: 100-continue\r\n\r\n")) - respPacket1 := buildPacket(false, 10, 3, []byte("HTTP/1.1 100 Continue\r\n\r\n"), time.Now()) + respPacket1 := responsePacket(reqPacket1, []byte("HTTP/1.1 100 Continue\r\n")) + reqPacket2 := responsePacket(respPacket1, []byte("1\r\na\r\n")) + reqPacket3 := nextPacket(reqPacket2, []byte("1\r\nb\r\n")) + reqPacket4 := nextPacket(reqPacket3, []byte("0\r\n\r\n")) - // panic(int(uint32(len(reqPacket1.Data)) + uint32(len(reqPacket2.Data)) + uint32(len(reqPacket3.Data)))) - respPacket2 := buildPacket(false, reqPacket4.Seq+5 /* len of data */, 2, []byte("HTTP/1.1 200 OK\r\n\r\n"), time.Now()) + respPacket2 := responsePacket(reqPacket4, []byte("HTTP/1.1 200 OK\r\n\r\n")) // Should re-construct message from all possible combinations for i := 0; i < 6*5*4*3*2*1; i++ { packets := permutation(i, []*TCPPacket{reqPacket1, reqPacket2, reqPacket3, reqPacket4, respPacket1, respPacket2}) - t.Log("permutation:", i) testChunkedSequence(t, listener, packets...) } } @@ -530,11 +532,7 @@ func getMessage() []*TCPPacket { // Response comes before Request func TestRawListenerBench(t *testing.T) { -<<<<<<< HEAD - l := NewListener("", "0", EnginePcap, true, 200*time.Millisecond, ProtocolHTTP) -======= - l := NewListener("", "0", EnginePcap, true, 200*time.Millisecond, "") ->>>>>>> d309650589a56db2bfa93c533337e3db1cfb0425 + l := NewListener("", "0", EnginePcap, true, 200*time.Millisecond, ProtocolHTTP, "", "", 0) defer l.Close() // Should re-construct message from all possible combinations @@ -582,3 +580,44 @@ func TestRawListenerBench(t *testing.T) { } } } + +func TestResponseZeroContentLength(t *testing.T) { + var req, resp *TCPMessage + listener := NewListener("", "0", EnginePcap, true, 10*time.Millisecond, ProtocolHTTP, "", "", 0) + defer listener.Close() + + reqPacket := firstPacket([]byte("POST /api/setup/install HTTP/1.1\r\nHost: localhost:22936\r\nUser-Agent: curl/7.57.0\r\nAccept: */*\r\nContent-Length: 0\r\nContent-Type: application/x-www-form-urlencoded\r\n\r\n")) + respPacket := responsePacket(reqPacket, []byte("HTTP/1.1 200\r\nDate: Fri, 11 May 2018 15:09:10 GMT\r\nServer: Kestrel\r\nCache-Control: no-cache\r\nTransfer-Encoding: chunked\r\n\r\n")) + respPacket2 := nextPacket(respPacket, []byte("0\r\n\r\n")) + + // If response packet comes before request + listener.packetsChan <- reqPacket.dump() + listener.packetsChan <- respPacket.dump() + listener.packetsChan <- respPacket2.dump() + + select { + case req = <-listener.messagesChan: + case <-time.After(time.Millisecond): + t.Error("Should return respose immediately") + return + } + + if !req.IsIncoming { + t.Error("Should be request") + } + + select { + case resp = <-listener.messagesChan: + case <-time.After(time.Millisecond): + t.Error("Should return response immediately") + return + } + + if resp.IsIncoming { + t.Error("Should be response") + } + + if !bytes.Equal(resp.UUID(), req.UUID()) { + t.Error("Resp and Req UUID should be equal") + } +} \ No newline at end of file diff --git a/raw_socket_listener/tcp_message.go b/raw_socket_listener/tcp_message.go index 1e38e68..446ca08 100644 --- a/raw_socket_listener/tcp_message.go +++ b/raw_socket_listener/tcp_message.go @@ -8,6 +8,7 @@ import ( "log" "net" "strconv" + "strings" "time" "github.com/buger/gor-pro/proto" @@ -83,7 +84,7 @@ func (t *TCPMessage) Bytes() (output []byte) { return output } -// Size returns total body size +// BodySize returns total body size func (t *TCPMessage) BodySize() (size int) { if len(t.packets) == 0 || t.headerPacket == -1 { return 0 @@ -133,20 +134,18 @@ func (t *TCPMessage) AddPacket(packet *TCPPacket) { break } } - } - if t.IsIncoming { - t.End = time.Now() - } else { - t.End = time.Now().Add(time.Millisecond) - } + if packet.OrigAck != 0 { + t.DataAck = packet.OrigAck + } - if packet.OrigAck != 0 { - t.DataAck = packet.OrigAck - } + if packet.timestamp.Before(t.Start) || t.Start.IsZero() { + t.Start = packet.timestamp + } - if packet.timestamp.Before(t.Start) { - t.Start = packet.timestamp + if packet.timestamp.After(t.End) || t.End.IsZero() { + t.End = packet.timestamp + } } t.checkSeqIntegrity() @@ -207,6 +206,7 @@ func (t *TCPMessage) checkSeqIntegrity() { } var bEmptyLine = []byte("\r\n\r\n") +var bBR = []byte("\r\n") var bChunkEnd = []byte("0\r\n\r\n") func (t *TCPMessage) updateHeadersPacket() { @@ -223,30 +223,42 @@ func (t *TCPMessage) updateHeadersPacket() { } for i, p := range t.packets { - if bytes.LastIndex(p.Data, bEmptyLine) != -1 { - t.headerPacket = i - return + if len(p.Data) >= len(bEmptyLine) { + if bytes.LastIndex(p.Data, bEmptyLine) != -1 { + t.headerPacket = i + return + } + } else if bytes.Equal(p.Data, bBR) { + if bytes.LastIndex(t.packets[i-1].Data, bBR) != -1 { + t.headerPacket = i + return + } } } return } -// isMultipart returns true if message contains from multiple tcp packets +// checkIfComplete returns true if all of the packets that compse the message arrived. func (t *TCPMessage) checkIfComplete() { if t.seqMissing || t.headerPacket == -1 { + // log.Println("Seq missing", t.seqMissing, t.packets) return } if t.methodType == httpMethodNotFound { + // log.Println("Method missing", t.methodType, t.packets) return } // Responses can be emitted only if we found request if !t.IsIncoming && t.AssocMessage == nil { + // log.Println("Assoc not found", t) return } + // log.Println("Found?", t) + switch t.bodyType { case httpBodyEmpty: t.complete = true @@ -364,9 +376,19 @@ func (t *TCPMessage) updateBodyType() { case httpMethodNotFound: return case httpMethodKnown: + + if !t.IsIncoming && + t.AssocMessage != nil && + bytes.IndexByte(t.AssocMessage.Bytes(), ' ') > -1 && + bytes.Equal([]byte("HEAD"), proto.Method(t.AssocMessage.Bytes())) { + // Need to check if this is a response to a head request, + // in which case the body has to be empty regardless. + t.bodyType = httpBodyEmpty + return + } + if len(lengthB) > 0 { t.contentLength, _ = strconv.Atoi(string(lengthB)) - if t.contentLength == 0 { t.bodyType = httpBodyEmpty } else { @@ -487,3 +509,11 @@ func (t *TCPMessage) ID() tcpID { func (t *TCPMessage) IP() net.IP { return net.IP(t.packets[0].Addr) } + +func (t *TCPMessage) String() string { + return strings.Join([]string{ + "Len packets: " + strconv.Itoa(len(t.packets)), + "Data size:" + strconv.Itoa(len(t.Bytes())), + "Data:" + string(t.Bytes()), + }, "\n") +} diff --git a/settings.go b/settings.go index fab7307..6fcb11d 100644 --- a/settings.go +++ b/settings.go @@ -56,6 +56,8 @@ type AppSettings struct { inputRAWProtocol string inputRAWExpire time.Duration inputRAWBpfFilter string + inputRAWTimestampType string + inputRawBufferSize int middleware string @@ -141,6 +143,10 @@ func init() { flag.StringVar(&Settings.inputRAWBpfFilter, "input-raw-bpf-filter", "", "BPF filter to write custom expressions. Can be useful in case of non standard network interfaces like tunneling or SPAN port. Example: --input-raw-bpf-filter 'dst port 80'") + flag.StringVar(&Settings.inputRAWTimestampType, "input-raw-timestamp-type", "", "Possible values: PCAP_TSTAMP_HOST, PCAP_TSTAMP_HOST_LOWPREC, PCAP_TSTAMP_HOST_HIPREC, PCAP_TSTAMP_ADAPTER, PCAP_TSTAMP_ADAPTER_UNSYNCED. This values not supported on all systems, GoReplay will tell you available values of you put wrong one.") + + flag.IntVar(&Settings.inputRawBufferSize, "input-raw-buffer-size", 0, "Controls size of the OS buffer (in bytes) which holds packets until they dispatched. Default value depends by system: in Linux around 2MB. If you see big package drop, increase this value.") + flag.StringVar(&Settings.middleware, "middleware", "", "Used for modifying traffic using external command") // flag.Var(&Settings.inputHTTP, "input-http", "Read requests from HTTP, should be explicitly sent from your application:\n\t# Listen for http on 9000\n\tgor --input-http :9000 --output-http staging.com") @@ -201,9 +207,9 @@ func init() { flag.Var(&Settings.modifierConfig.headerNegativeFilters, "http-disallow-header", "A regexp to match a specific header against. Requests with matching headers will be dropped:\n\t gor --input-raw :8080 --output-http staging.com --http-disallow-header \"User-Agent: Replayed by Gor\"") - flag.Var(&Settings.modifierConfig.headerBasicAuthFilters, "http-basic-auth-filter", "A regexp to match the decoded basic auth string against. Requests with non-matching headers will be dropped:\n\t gor --input-raw :8080 --output-http staging.com --http-basic-auth-filter \"^customer[0-9].*\"") + flag.Var(&Settings.modifierConfig.headerBasicAuthFilters, "http-basic-auth-filter", "A regexp to match the decoded basic auth string against. Requests with non-matching headers will be dropped:\n\t gor --input-raw :8080 --output-http staging.com --http-basic-auth-filter \"^customer[0-9].*\"") - flag.Var(&Settings.modifierConfig.headerHashFilters, "http-header-limiter", "Takes a fraction of requests, consistently taking or rejecting a request based on the FNV32-1A hash of a specific header:\n\t gor --input-raw :8080 --output-http staging.com --http-header-limiter user-id:25%") + flag.Var(&Settings.modifierConfig.headerHashFilters, "http-header-limiter", "Takes a fraction of requests, consistently taking or rejecting a request based on the FNV32-1A hash of a specific header:\n\t gor --input-raw :8080 --output-http staging.com --http-header-limiter user-id:25%") flag.Var(&Settings.modifierConfig.headerHashFilters, "output-http-header-hash-filter", "WARNING: `output-http-header-hash-filter` DEPRECATED, use `--http-header-hash-limiter` instead") diff --git a/vendor/vendor.json b/vendor/vendor.json index a6319c5..5db34eb 100644 --- a/vendor/vendor.json +++ b/vendor/vendor.json @@ -45,10 +45,10 @@ "revisionTime": "2016-05-29T05:00:41Z" }, { - "checksumSHA1": "U2Ydh7vEAKlN0Wq22n1JpefF7uY=", + "checksumSHA1": "WT6lYgJhoWbXLpnFOxPISxrL2/o=", "path": "github.com/google/gopacket", - "revision": "b09bf408520f7646e29b7033d9adb00ed779a1c4", - "revisionTime": "2016-05-12T15:06:07Z" + "revision": "60ab61cd59496fcfa4d208b265ba79b1e37c1476", + "revisionTime": "2018-05-13T17:29:36Z" }, { "checksumSHA1": "BM6ZlNJmtKy3GBoWwg2X55gnZ4A=",