From 41294f94708ce76b50e8780031d7db3e77ccffe5 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Mon, 29 Jun 2015 17:01:26 +0500 Subject: [PATCH 01/30] Fix 100-Expect requests and refactor chunked encoding --- Makefile | 3 + input_raw.go | 2 +- input_raw_test.go | 125 +++++++++++++++++++++++++++++ output_http.go | 11 +-- output_http_test.go | 43 +--------- output_tcp_test.go | 2 +- raw_socket_listener/listener.go | 43 ++++++++-- raw_socket_listener/tcp_message.go | 30 ++++++- raw_socket_listener/tcp_packet.go | 1 + 9 files changed, 202 insertions(+), 58 deletions(-) diff --git a/Makefile b/Makefile index 037e311..475e122 100644 --- a/Makefile +++ b/Makefile @@ -17,6 +17,9 @@ dtest: dfmt: docker run -v `pwd`:/gopath/src/gor -t -i gor go fmt +dvet: + docker run -v `pwd`:/gopath/src/gor -t -i gor go vet + dbench: docker run -v `pwd`:/gopath/src/gor -t -i gor go test -v -run NOT_EXISTING -bench HTTP diff --git a/input_raw.go b/input_raw.go index 3a6a960..20dd4b0 100644 --- a/input_raw.go +++ b/input_raw.go @@ -1,7 +1,7 @@ package main import ( - raw "github.com/buger/gor/raw_socket_listener" + raw "gor/raw_socket_listener" "log" "net" "strings" diff --git a/input_raw_test.go b/input_raw_test.go index b1bcca6..d446d11 100644 --- a/input_raw_test.go +++ b/input_raw_test.go @@ -1,8 +1,13 @@ package main import ( + "bytes" "io" + "io/ioutil" + "log" "net/http" + "net/http/httputil" + "os/exec" "strings" "sync" "testing" @@ -37,3 +42,123 @@ func TestRAWInput(t *testing.T) { close(quit) } + +func TestInputRAW100Expect(t *testing.T) { + wg := new(sync.WaitGroup) + quit := make(chan int) + + file_content, _ := ioutil.ReadFile("README.md") + + // Origing and Replay server initialization + origin := startHTTP(func(req *http.Request) { + defer req.Body.Close() + ioutil.ReadAll(req.Body) + + wg.Done() + }) + + origin_address := strings.Replace(origin.Addr().String(), "[::]", "127.0.0.1", -1) + + input := NewRAWInput(origin_address) + + // We will use it to get content of raw HTTP request + test_output := NewTestOutput(func(data []byte) { + if strings.Contains(string(data), "Expect: 100-continue") { + t.Error("Should not contain 100-continue header") + } + wg.Done() + }) + + listener := startHTTP(func(req *http.Request) { + defer req.Body.Close() + body, _ := ioutil.ReadAll(req.Body) + + if !bytes.Equal(body, file_content) { + buf, _ := httputil.DumpRequest(req, true) + t.Error("Wrong POST body:", string(buf)) + } + + wg.Done() + }) + replay_address := listener.Addr().String() + + headers := HTTPHeaders{HTTPHeader{"User-Agent", "Gor"}} + methods := HTTPMethods{"GET", "PUT", "POST"} + http_output := NewHTTPOutput(replay_address, headers, methods, HTTPUrlRegexp{}, HTTPHeaderFilters{}, HTTPHeaderHashFilters{}, "", UrlRewriteMap{}, 0) + + Plugins.Inputs = []io.Reader{input} + Plugins.Outputs = []io.Writer{test_output, http_output} + + go Start(quit) + + wg.Add(3) + curl := exec.Command("curl", "http://"+origin_address, "--data-binary", "@README.md") + err := curl.Run() + if err != nil { + log.Fatal(err) + } + + wg.Wait() + close(quit) +} + +func TestInputRAWChunkedEncoding(t *testing.T) { + wg := new(sync.WaitGroup) + quit := make(chan int) + + file_content, _ := ioutil.ReadFile("README.md") + + // Origing and Replay server initialization + origin := startHTTP(func(req *http.Request) { + defer req.Body.Close() + ioutil.ReadAll(req.Body) + + wg.Done() + }) + + origin_address := strings.Replace(origin.Addr().String(), "[::]", "127.0.0.1", -1) + + input := NewRAWInput(origin_address) + + // We will use it to get content of raw HTTP request + test_output := NewTestOutput(func(data []byte) { + if strings.Contains(string(data), "Expect: 100-continue") { + t.Error("Should not contain 100-continue header") + } + wg.Done() + }) + + listener := startHTTP(func(req *http.Request) { + defer req.Body.Close() + body, _ := ioutil.ReadAll(req.Body) + + if !bytes.Equal(body, file_content) { + buf, _ := httputil.DumpRequest(req, true) + t.Error("Wrong POST body:", string(buf)) + } + + wg.Done() + }) + replay_address := listener.Addr().String() + + headers := HTTPHeaders{HTTPHeader{"User-Agent", "Gor"}} + methods := HTTPMethods{"GET", "PUT", "POST"} + http_output := NewHTTPOutput(replay_address, headers, methods, HTTPUrlRegexp{}, HTTPHeaderFilters{}, HTTPHeaderHashFilters{}, "", UrlRewriteMap{}, 0) + + Plugins.Inputs = []io.Reader{input} + Plugins.Outputs = []io.Writer{test_output, http_output} + + go Start(quit) + + wg.Add(3) + + curl := exec.Command("curl", "http://"+origin_address, "--header", "Transfer-Encoding: chunked", "--data-binary", "@README.md") + err := curl.Run() + if err != nil { + log.Fatal(err) + } + + wg.Wait() + + close(quit) +} diff --git a/output_http.go b/output_http.go index 46a342a..2623641 100644 --- a/output_http.go +++ b/output_http.go @@ -7,7 +7,6 @@ import ( "io/ioutil" "log" "net/http" - "net/http/httputil" "net/url" "strings" "sync/atomic" @@ -32,9 +31,6 @@ func (o *HTTPOutput) customCheckRedirect(req *http.Request, via []*http.Request) func ParseRequest(data []byte) (request *http.Request, err error) { var body []byte - // Test if request have Transfer-Encoding: chunked - isChunked := bytes.Contains(data, []byte(": chunked\r\n")) - buf := bytes.NewBuffer(data) reader := bufio.NewReader(buf) @@ -46,12 +42,7 @@ func ParseRequest(data []byte) (request *http.Request, err error) { } if request.Method == "POST" { - // This works, because ReadRequest method modify buffer and strips all headers, leaving only body - if isChunked { - body, _ = ioutil.ReadAll(httputil.NewChunkedReader(reader)) - } else { - body, _ = ioutil.ReadAll(reader) - } + body, _ = ioutil.ReadAll(reader) bodyBuf := bytes.NewBuffer(body) diff --git a/output_http_test.go b/output_http_test.go index 2432d5c..c954cc3 100644 --- a/output_http_test.go +++ b/output_http_test.go @@ -6,7 +6,6 @@ import ( "net" "net/http" "net/http/httputil" - _ "strings" "sync" "testing" "time" @@ -51,9 +50,6 @@ func TestHTTPOutput(t *testing.T) { input := NewTestInput() - headers := HTTPHeaders{HTTPHeader{"User-Agent", "Gor"}} - methods := HTTPMethods{"GET", "PUT", "POST"} - listener := startHTTP(func(req *http.Request) { if req.Header.Get("User-Agent") != "Gor" { t.Error("Wrong header") @@ -76,6 +72,9 @@ func TestHTTPOutput(t *testing.T) { wg.Done() }) + headers := HTTPHeaders{HTTPHeader{"User-Agent", "Gor"}} + methods := HTTPMethods{"GET", "PUT", "POST"} + output := NewHTTPOutput(listener.Addr().String(), headers, methods, HTTPUrlRegexp{}, HTTPHeaderFilters{}, HTTPHeaderHashFilters{}, "", UrlRewriteMap{}, 0) Plugins.Inputs = []io.Reader{input} @@ -95,42 +94,6 @@ func TestHTTPOutput(t *testing.T) { close(quit) } -func TestHTTPOutputChunkedEncoding(t *testing.T) { - wg := new(sync.WaitGroup) - quit := make(chan int) - - input := NewTestInput() - - headers := HTTPHeaders{HTTPHeader{"User-Agent", "Gor"}} - methods := HTTPMethods{"GET", "PUT", "POST"} - - listener := startHTTP(func(req *http.Request) { - defer req.Body.Close() - body, _ := ioutil.ReadAll(req.Body) - - if string(body) != "Wikipedia in\r\n\r\nchunks." { - buf, _ := httputil.DumpRequest(req, true) - t.Error("Wrong POST body:", buf, body, []byte("Wikipedia in\r\n\r\nchunks.")) - } - - wg.Done() - }) - - output := NewHTTPOutput(listener.Addr().String(), headers, methods, HTTPUrlRegexp{}, HTTPHeaderFilters{}, HTTPHeaderHashFilters{}, "", UrlRewriteMap{}, 0) - - Plugins.Inputs = []io.Reader{input} - Plugins.Outputs = []io.Writer{output} - - go Start(quit) - - wg.Add(1) - input.EmitChunkedPOST() - - wg.Wait() - - close(quit) -} - func BenchmarkHTTPOutput(b *testing.B) { wg := new(sync.WaitGroup) quit := make(chan int) diff --git a/output_tcp_test.go b/output_tcp_test.go index 045ccfe..aaf0cf6 100644 --- a/output_tcp_test.go +++ b/output_tcp_test.go @@ -44,6 +44,7 @@ func startTCP(cb func([]byte)) net.Listener { go func() { for { conn, _ := listener.Accept() + defer conn.Close() go func() { reader := bufio.NewReader(conn) @@ -59,7 +60,6 @@ func startTCP(cb func([]byte)) net.Listener { } cb(new_buf) } - conn.Close() }() } }() diff --git a/raw_socket_listener/listener.go b/raw_socket_listener/listener.go index 625a100..b5f7150 100644 --- a/raw_socket_listener/listener.go +++ b/raw_socket_listener/listener.go @@ -5,6 +5,7 @@ import ( "log" "net" "strconv" + "bytes" ) // Capture traffic from socket using RAW_SOCKET's @@ -17,6 +18,11 @@ import ( type Listener struct { messages map[string]*TCPMessage // buffer of TCPMessages waiting to be send + // Expect: 100-continue request is send in 2 tcp messages + // We store ACK aliases to merge this packets together + ack_aliases map[uint32]uint32 + seq_with_data map[uint32]uint32 + c_packets chan *TCPPacket c_messages chan *TCPMessage // Messages ready to be send to client @@ -30,10 +36,13 @@ type Listener struct { func NewListener(addr string, port string) (rawListener *Listener) { rawListener = &Listener{} - rawListener.c_packets = make(chan *TCPPacket, 100) - rawListener.c_messages = make(chan *TCPMessage, 100) - rawListener.c_del_message = make(chan *TCPMessage, 100) + rawListener.c_packets = make(chan *TCPPacket, 10000) + rawListener.c_messages = make(chan *TCPMessage, 10000) + rawListener.c_del_message = make(chan *TCPMessage, 10000) + rawListener.messages = make(map[string]*TCPMessage) + rawListener.ack_aliases = make(map[uint32]uint32) + rawListener.seq_with_data = make(map[uint32]uint32) rawListener.addr = addr rawListener.port, _ = strconv.Atoi(port) @@ -50,6 +59,7 @@ func (t *Listener) listen() { // If message ready for deletion it means that its also complete or expired by timeout case message := <-t.c_del_message: t.c_messages <- message + delete(t.ack_aliases, message.packets[0].Ack) delete(t.messages, message.ID) // We need to use channels to process each packet to avoid data races @@ -68,7 +78,7 @@ func (t *Listener) readRAWSocket() { defer conn.Close() - buf := make([]byte, 4096*2) + buf := make([]byte, 4096*10) for { // Note: ReadFrom receive messages without IP header @@ -115,6 +125,9 @@ func (t *Listener) isIncomingDataPacket(buf []byte) bool { return false } +var bExpect100ContinueCheck = []byte("Expect: 100-continue") +var bPOST = []byte("POST") + // Trying to add packet to existing message or creating new message // // For TCP message unique id is Acknowledgment number (see tcp_packet.go) @@ -122,8 +135,19 @@ func (t *Listener) processTCPPacket(packet *TCPPacket) { defer func() { recover() }() var message *TCPMessage - m_id := packet.Addr.String() + strconv.Itoa(int(packet.Ack)) + parent_message_ack, parent_ok := t.seq_with_data[packet.Seq] + if parent_ok { + t.ack_aliases[packet.Ack] = parent_message_ack + delete(t.seq_with_data, packet.Seq) + } + + ack_alias, alias_ok := t.ack_aliases[packet.Ack] + if alias_ok { + packet.Ack = ack_alias + } + + m_id := packet.Addr.String() + strconv.Itoa(int(packet.Ack)) message, ok := t.messages[m_id] if !ok { @@ -132,6 +156,15 @@ func (t *Listener) processTCPPacket(packet *TCPPacket) { t.messages[m_id] = message } + if bytes.Equal(packet.Data[0:4], bPOST) { + if bytes.Equal(packet.Data[len(packet.Data)-24:len(packet.Data)-4], bExpect100ContinueCheck) { + t.seq_with_data[packet.Seq + uint32(len(packet.Data))] = packet.Ack + + // Removing `Expect: 100-continue` header + packet.Data = append(packet.Data[:len(packet.Data)-24], packet.Data[len(packet.Data)-2:]...) + } + } + // Adding packet to message message.c_packets <- packet } diff --git a/raw_socket_listener/tcp_message.go b/raw_socket_listener/tcp_message.go index 21ed63e..9e45a85 100644 --- a/raw_socket_listener/tcp_message.go +++ b/raw_socket_listener/tcp_message.go @@ -4,6 +4,10 @@ import ( "log" "sort" "time" + "bytes" + "net/http/httputil" + "bufio" + "io/ioutil" ) const MSG_EXPIRE = 2000 * time.Millisecond @@ -71,6 +75,30 @@ func (t *TCPMessage) Timeout() { } } +var bTransferEncodingChunked = []byte("Transfer-Encoding: chunked\r\n") +var b2xCRLF = []byte("\r\n\r\n") + +// Norimalize requests with `Transfer-Encoding: chunked` header, because they have special body format +func fixChunkedEncoding(data []byte) []byte { + if bytes.Equal(data[0:4], bPOST) { + body_idx := bytes.Index(data, b2xCRLF) + chunked_header_idx := bytes.Index(data[:body_idx], bTransferEncodingChunked) + + if chunked_header_idx != -1 { + buf := bytes.NewBuffer(data[body_idx+4:]) + // Adding 4 bytes to skip 2xCLRF + bodyReader := bufio.NewReader(buf) + body, _ := ioutil.ReadAll(httputil.NewChunkedReader(bodyReader)) + + // Exclude Transfer-Encoding header and append new body + return append(append(append(data[:chunked_header_idx], + data[chunked_header_idx+len(bTransferEncodingChunked):body_idx]...), b2xCRLF...), body...) + } + } + + return data +} + // Bytes sorts packets in right orders and return message content func (t *TCPMessage) Bytes() (output []byte) { sort.Sort(BySeq(t.packets)) @@ -79,7 +107,7 @@ func (t *TCPMessage) Bytes() (output []byte) { output = append(output, v.Data...) } - return + return fixChunkedEncoding(output) } // AddPacket to the message and ensure packet uniqueness diff --git a/raw_socket_listener/tcp_packet.go b/raw_socket_listener/tcp_packet.go index 5e81ff5..5af8a73 100644 --- a/raw_socket_listener/tcp_packet.go +++ b/raw_socket_listener/tcp_packet.go @@ -89,6 +89,7 @@ func (t *TCPPacket) String() string { "Window size:" + strconv.Itoa(int(t.Window)), "Checksum:" + strconv.Itoa(int(t.Checksum)), + "Data size:" + strconv.Itoa(len(t.Data)), "Data:" + string(t.Data), }, "\n") } From 1e7b603ae0fab66f2a41c2d05f2a60a991a08495 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Mon, 29 Jun 2015 17:14:11 +0500 Subject: [PATCH 02/30] Fix import path --- input_raw.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/input_raw.go b/input_raw.go index 20dd4b0..3a6a960 100644 --- a/input_raw.go +++ b/input_raw.go @@ -1,7 +1,7 @@ package main import ( - raw "gor/raw_socket_listener" + raw "github.com/buger/gor/raw_socket_listener" "log" "net" "strings" From 3d3c6a34b1fb474079853ae58e7b9e62f48f5f3a Mon Sep 17 00:00:00 2001 From: "Choe, Cheng-Dae" Date: Tue, 30 Jun 2015 21:34:42 +0900 Subject: [PATCH 03/30] add .dockerignore --- .dockerignore | 1 + 1 file changed, 1 insertion(+) create mode 100644 .dockerignore diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..335ec95 --- /dev/null +++ b/.dockerignore @@ -0,0 +1 @@ +*.tar.gz From 122f06f6c24dc076b948efd6e0f26f21b5aaff62 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Wed, 1 Jul 2015 19:06:49 +0500 Subject: [PATCH 04/30] Add raw http client --- emitter.go | 2 - http_client.go | 61 +++++++++++++++++++++++ http_client_test.go | 118 ++++++++++++++++++++++++++++++++++++++++++++ input_raw_test.go | 4 +- output_http.go | 6 +++ output_http_test.go | 32 ++++++++++++ test_input.go | 4 +- 7 files changed, 221 insertions(+), 6 deletions(-) create mode 100644 http_client.go create mode 100644 http_client_test.go diff --git a/emitter.go b/emitter.go index 0bc8086..867fc75 100644 --- a/emitter.go +++ b/emitter.go @@ -27,8 +27,6 @@ func CopyMulty(src io.Reader, writers ...io.Writer) (err error) { for { nr, er := src.Read(buf) if nr > 0 && len(buf) > nr { - Debug("Sending", src, ": ", string(buf[0:nr])) - if Settings.splitOutput { // Simple round robin writers[wIndex].Write(buf[0:nr]) diff --git a/http_client.go b/http_client.go new file mode 100644 index 0000000..2a1fe6e --- /dev/null +++ b/http_client.go @@ -0,0 +1,61 @@ +package main + +import ( + "net" + "net/url" + "crypto/tls" + "strings" +) + +var defaultPorts = map[string]string{ + "http": "80", + "https": "443", +} + +type HTTPClient struct { + baseURL *url.URL + conn net.Conn + buf []byte +} + +func NewHTTPClient(baseURL string) (*HTTPClient) { + client := new(HTTPClient) + client.baseURL, _ = url.Parse(baseURL) + client.buf = make([]byte, 4096*10) + + if !strings.Contains(client.baseURL.Host, ":") { + client.baseURL.Host += ":" + defaultPorts[client.baseURL.Scheme] + } + + return client +} + +func (c *HTTPClient) Connect() (err error) { + c.conn, err = net.Dial("tcp", c.baseURL.Host) + + if (c.baseURL.Scheme == "https") { + tlsConn := tls.Client(c.conn, &tls.Config{InsecureSkipVerify: true}) + err = tlsConn.Handshake() + c.conn = tlsConn + } + + return +} + +func (c *HTTPClient) Disconnect() { + c.conn.Close() + c.conn = nil +} + +func (c *HTTPClient) Send(data []byte) (response []byte, err error) { + if c.conn == nil { + c.Connect() + } + + _, err = c.conn.Write(data) + n, err := c.conn.Read(c.buf) + + Debug(string(c.buf[:n])) + + return c.buf[:n], err +} \ No newline at end of file diff --git a/http_client_test.go b/http_client_test.go new file mode 100644 index 0000000..80e234a --- /dev/null +++ b/http_client_test.go @@ -0,0 +1,118 @@ +package main + +import ( + "testing" + "net/http/httptest" + "net/http/httputil" + "net/http" + "io/ioutil" + "sync" +) + +func TestHTTPClientURLPort(t *testing.T) { + c1 := NewHTTPClient("http://example.com") + if c1.baseURL.String() != "http://example.com:80" { + t.Error("Sould add 80 port for http:", c1.baseURL.String()) + } + + c2 := NewHTTPClient("https://example.com") + if c2.baseURL.String() != "https://example.com:443" { + t.Error("Sould add 443 port for https:", c2.baseURL.String()) + } + + c3 := NewHTTPClient("https://example.com:1") + if c3.baseURL.String() != "https://example.com:1" { + t.Error("Sould use specified port:", c3.baseURL.String()) + } +} + +func TestHTTPClientSend(t *testing.T) { + wg := new(sync.WaitGroup) + + GET_payload := []byte("GET / HTTP/1.1\r\n\r\n") + + // Post request terminates by reading Content-Length without double CRLF + POST_payload := []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") + + // Chunked requests terminated with double CRLF + POST_CHUNKED_payload := []byte("POST / HTTP/1.1\r\nHost: www.w3.org\r\nTransfer-Encoding: chunked\r\n\r\n4\r\nWiki\r\n5\r\npedia\r\ne\r\n in\r\n\r\nchunks.\r\n0\r\n\r\n") + + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + + if r.Method == "POST" { + defer r.Body.Close() + body, _ := ioutil.ReadAll(r.Body) + + if len(r.TransferEncoding) > 0 && r.TransferEncoding[0] == "chunked" { + if string(body) != "Wikipedia in\r\n\r\nchunks." { + t.Error("Wrong POST body:", body, string(body)) + } + } else { + if string(body) != "a=1&b=2" { + buf, _ := httputil.DumpRequest(r, true) + t.Error("Wrong POST body:", string(body), string(buf)) + } + } + } + + wg.Done() + })) + + + client := NewHTTPClient(server.URL) + + wg.Add(4) + client.Send(POST_payload) + client.Send(GET_payload) + client.Send(POST_CHUNKED_payload) + client.Send(POST_payload) + + wg.Wait() +} + + +func TestHTTPClientHTTPSSend(t *testing.T) { + wg := new(sync.WaitGroup) + + GET_payload := []byte("GET / HTTP/1.1\r\n\r\n") + + // Post request terminates by reading Content-Length without double CRLF + POST_payload := []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") + + // Chunked requests terminated with double CRLF + POST_CHUNKED_payload := []byte("POST / HTTP/1.1\r\nHost: www.w3.org\r\nTransfer-Encoding: chunked\r\n\r\n4\r\nWiki\r\n5\r\npedia\r\ne\r\n in\r\n\r\nchunks.\r\n0\r\n\r\n") + + + server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + + if r.Method == "POST" { + defer r.Body.Close() + body, _ := ioutil.ReadAll(r.Body) + + if len(r.TransferEncoding) > 0 && r.TransferEncoding[0] == "chunked" { + if string(body) != "Wikipedia in\r\n\r\nchunks." { + t.Error("Wrong POST body:", body, string(body)) + } + } else { + if string(body) != "a=1&b=2" { + buf, _ := httputil.DumpRequest(r, true) + t.Error("Wrong POST body:", string(body), string(buf)) + } + } + } + + wg.Done() + })) + + + client := NewHTTPClient(server.URL) + + wg.Add(4) + client.Send(GET_payload) + client.Send(POST_payload) + client.Send(POST_CHUNKED_payload) + client.Send(POST_payload) + + wg.Wait() +} \ No newline at end of file diff --git a/input_raw_test.go b/input_raw_test.go index d446d11..ba9b537 100644 --- a/input_raw_test.go +++ b/input_raw_test.go @@ -122,8 +122,8 @@ func TestInputRAWChunkedEncoding(t *testing.T) { // We will use it to get content of raw HTTP request test_output := NewTestOutput(func(data []byte) { - if strings.Contains(string(data), "Expect: 100-continue") { - t.Error("Should not contain 100-continue header") + if strings.Contains(string(data), "Transfer-Encoding: chunked") { + t.Error("Should not contain chunked header") } wg.Done() }) diff --git a/output_http.go b/output_http.go index 2623641..e702a60 100644 --- a/output_http.go +++ b/output_http.go @@ -11,6 +11,7 @@ import ( "strings" "sync/atomic" "time" + "crypto/tls" ) type RedirectNotAllowed struct{} @@ -140,7 +141,12 @@ func (o *HTTPOutput) WorkerMaster() { } func (o *HTTPOutput) Worker() { + tr := &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + } + client := &http.Client{ + Transport: tr, CheckRedirect: o.customCheckRedirect, } diff --git a/output_http_test.go b/output_http_test.go index c954cc3..e3afb5c 100644 --- a/output_http_test.go +++ b/output_http_test.go @@ -6,6 +6,7 @@ import ( "net" "net/http" "net/http/httputil" + "net/http/httptest" "sync" "testing" "time" @@ -94,6 +95,37 @@ func TestHTTPOutput(t *testing.T) { close(quit) } + +func TestOutputHTTPSSL(t *testing.T) { + wg := new(sync.WaitGroup) + quit := make(chan int) + + // Origing and Replay server initialization + server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + wg.Done() + })) + + input := NewTestInput() + + headers := HTTPHeaders{HTTPHeader{"User-Agent", "Gor"}} + methods := HTTPMethods{"GET", "PUT", "POST"} + + http_output := NewHTTPOutput(server.URL, headers, methods, HTTPUrlRegexp{}, HTTPHeaderFilters{}, HTTPHeaderHashFilters{}, "", UrlRewriteMap{}, 0) + + Plugins.Inputs = []io.Reader{input} + Plugins.Outputs = []io.Writer{http_output} + + go Start(quit) + + wg.Add(2) + + input.EmitPOST() + input.EmitGET() + + wg.Wait() + close(quit) +} + func BenchmarkHTTPOutput(b *testing.B) { wg := new(sync.WaitGroup) quit := make(chan int) diff --git a/test_input.go b/test_input.go index 694736c..7de50e2 100644 --- a/test_input.go +++ b/test_input.go @@ -28,11 +28,11 @@ func (i *TestInput) EmitGET() { } func (i *TestInput) EmitPOST() { - i.data <- []byte("POST /pub/WWW/ HTTP/1.1\nHost: www.w3.org\r\n\r\na=1&b=2") + i.data <- []byte("POST /pub/WWW/ HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2\r\n\r\n") } func (i *TestInput) EmitChunkedPOST() { - i.data <- []byte("POST /pub/WWW/ HTTP/1.1\nHost: www.w3.org\nTransfer-Encoding: chunked\r\n\r\n4\r\nWiki\r\n5\r\npedia\r\ne\r\n in\r\n\r\nchunks.\r\n0\r\n\r\n") + i.data <- []byte("POST /pub/WWW/ HTTP/1.1\r\nHost: www.w3.org\r\nTransfer-Encoding: chunked\r\n\r\n4\r\nWiki\r\n5\r\npedia\r\ne\r\n in\r\n\r\nchunks.\r\n0\r\n\r\n") } func (i *TestInput) EmitFile() { From 21c544ef677cf96f60fd0f65065a0dd80f9ad996 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Wed, 1 Jul 2015 19:29:19 +0500 Subject: [PATCH 05/30] Fix tests and apply go fmt --- http_client.go | 68 +++++++++--------- http_client_test.go | 163 +++++++++++++++++++++----------------------- output_http.go | 8 +-- output_http_test.go | 6 +- test_input.go | 2 +- 5 files changed, 120 insertions(+), 127 deletions(-) diff --git a/http_client.go b/http_client.go index 2a1fe6e..ae36b2d 100644 --- a/http_client.go +++ b/http_client.go @@ -1,61 +1,61 @@ package main import ( - "net" - "net/url" - "crypto/tls" - "strings" + "crypto/tls" + "net" + "net/url" + "strings" ) var defaultPorts = map[string]string{ - "http": "80", - "https": "443", + "http": "80", + "https": "443", } type HTTPClient struct { - baseURL *url.URL - conn net.Conn - buf []byte + baseURL *url.URL + conn net.Conn + buf []byte } -func NewHTTPClient(baseURL string) (*HTTPClient) { - client := new(HTTPClient) - client.baseURL, _ = url.Parse(baseURL) - client.buf = make([]byte, 4096*10) +func NewHTTPClient(baseURL string) *HTTPClient { + client := new(HTTPClient) + client.baseURL, _ = url.Parse(baseURL) + client.buf = make([]byte, 4096*10) - if !strings.Contains(client.baseURL.Host, ":") { - client.baseURL.Host += ":" + defaultPorts[client.baseURL.Scheme] - } + if !strings.Contains(client.baseURL.Host, ":") { + client.baseURL.Host += ":" + defaultPorts[client.baseURL.Scheme] + } - return client + return client } func (c *HTTPClient) Connect() (err error) { - c.conn, err = net.Dial("tcp", c.baseURL.Host) + c.conn, err = net.Dial("tcp", c.baseURL.Host) - if (c.baseURL.Scheme == "https") { - tlsConn := tls.Client(c.conn, &tls.Config{InsecureSkipVerify: true}) - err = tlsConn.Handshake() - c.conn = tlsConn - } + if c.baseURL.Scheme == "https" { + tlsConn := tls.Client(c.conn, &tls.Config{InsecureSkipVerify: true}) + err = tlsConn.Handshake() + c.conn = tlsConn + } - return + return } func (c *HTTPClient) Disconnect() { - c.conn.Close() - c.conn = nil + c.conn.Close() + c.conn = nil } func (c *HTTPClient) Send(data []byte) (response []byte, err error) { - if c.conn == nil { - c.Connect() - } + if c.conn == nil { + c.Connect() + } - _, err = c.conn.Write(data) - n, err := c.conn.Read(c.buf) + _, err = c.conn.Write(data) + n, err := c.conn.Read(c.buf) - Debug(string(c.buf[:n])) + Debug(string(c.buf[:n])) - return c.buf[:n], err -} \ No newline at end of file + return c.buf[:n], err +} diff --git a/http_client_test.go b/http_client_test.go index 80e234a..63504b1 100644 --- a/http_client_test.go +++ b/http_client_test.go @@ -1,118 +1,113 @@ package main import ( - "testing" - "net/http/httptest" - "net/http/httputil" - "net/http" - "io/ioutil" - "sync" + "io/ioutil" + "net/http" + "net/http/httptest" + "net/http/httputil" + "sync" + "testing" ) func TestHTTPClientURLPort(t *testing.T) { - c1 := NewHTTPClient("http://example.com") - if c1.baseURL.String() != "http://example.com:80" { - t.Error("Sould add 80 port for http:", c1.baseURL.String()) - } + c1 := NewHTTPClient("http://example.com") + if c1.baseURL.String() != "http://example.com:80" { + t.Error("Sould add 80 port for http:", c1.baseURL.String()) + } - c2 := NewHTTPClient("https://example.com") - if c2.baseURL.String() != "https://example.com:443" { - t.Error("Sould add 443 port for https:", c2.baseURL.String()) - } + c2 := NewHTTPClient("https://example.com") + if c2.baseURL.String() != "https://example.com:443" { + t.Error("Sould add 443 port for https:", c2.baseURL.String()) + } - c3 := NewHTTPClient("https://example.com:1") - if c3.baseURL.String() != "https://example.com:1" { - t.Error("Sould use specified port:", c3.baseURL.String()) - } + c3 := NewHTTPClient("https://example.com:1") + if c3.baseURL.String() != "https://example.com:1" { + t.Error("Sould use specified port:", c3.baseURL.String()) + } } func TestHTTPClientSend(t *testing.T) { - wg := new(sync.WaitGroup) + wg := new(sync.WaitGroup) - GET_payload := []byte("GET / HTTP/1.1\r\n\r\n") + GET_payload := []byte("GET / HTTP/1.1\r\n\r\n") - // Post request terminates by reading Content-Length without double CRLF - POST_payload := []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") + // Post request terminates by reading Content-Length without double CRLF + POST_payload := []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") - // Chunked requests terminated with double CRLF - POST_CHUNKED_payload := []byte("POST / HTTP/1.1\r\nHost: www.w3.org\r\nTransfer-Encoding: chunked\r\n\r\n4\r\nWiki\r\n5\r\npedia\r\ne\r\n in\r\n\r\nchunks.\r\n0\r\n\r\n") + // Chunked requests terminated with double CRLF + POST_CHUNKED_payload := []byte("POST / HTTP/1.1\r\nHost: www.w3.org\r\nTransfer-Encoding: chunked\r\n\r\n4\r\nWiki\r\n5\r\npedia\r\ne\r\n in\r\n\r\nchunks.\r\n0\r\n\r\n") + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == "POST" { + defer r.Body.Close() + body, _ := ioutil.ReadAll(r.Body) - if r.Method == "POST" { - defer r.Body.Close() - body, _ := ioutil.ReadAll(r.Body) + if len(r.TransferEncoding) > 0 && r.TransferEncoding[0] == "chunked" { + if string(body) != "Wikipedia in\r\n\r\nchunks." { + t.Error("Wrong POST body:", body, string(body)) + } + } else { + if string(body) != "a=1&b=2" { + buf, _ := httputil.DumpRequest(r, true) + t.Error("Wrong POST body:", string(body), string(buf)) + } + } + } - if len(r.TransferEncoding) > 0 && r.TransferEncoding[0] == "chunked" { - if string(body) != "Wikipedia in\r\n\r\nchunks." { - t.Error("Wrong POST body:", body, string(body)) - } - } else { - if string(body) != "a=1&b=2" { - buf, _ := httputil.DumpRequest(r, true) - t.Error("Wrong POST body:", string(body), string(buf)) - } - } - } + wg.Done() + })) - wg.Done() - })) + client := NewHTTPClient(server.URL) + wg.Add(4) + client.Send(POST_payload) + client.Send(GET_payload) + client.Send(POST_CHUNKED_payload) + client.Send(POST_payload) - client := NewHTTPClient(server.URL) - - wg.Add(4) - client.Send(POST_payload) - client.Send(GET_payload) - client.Send(POST_CHUNKED_payload) - client.Send(POST_payload) - - wg.Wait() + wg.Wait() } - func TestHTTPClientHTTPSSend(t *testing.T) { - wg := new(sync.WaitGroup) + wg := new(sync.WaitGroup) - GET_payload := []byte("GET / HTTP/1.1\r\n\r\n") + GET_payload := []byte("GET / HTTP/1.1\r\n\r\n") - // Post request terminates by reading Content-Length without double CRLF - POST_payload := []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") + // Post request terminates by reading Content-Length without double CRLF + POST_payload := []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") - // Chunked requests terminated with double CRLF - POST_CHUNKED_payload := []byte("POST / HTTP/1.1\r\nHost: www.w3.org\r\nTransfer-Encoding: chunked\r\n\r\n4\r\nWiki\r\n5\r\npedia\r\ne\r\n in\r\n\r\nchunks.\r\n0\r\n\r\n") + // Chunked requests terminated with double CRLF + POST_CHUNKED_payload := []byte("POST / HTTP/1.1\r\nHost: www.w3.org\r\nTransfer-Encoding: chunked\r\n\r\n4\r\nWiki\r\n5\r\npedia\r\ne\r\n in\r\n\r\nchunks.\r\n0\r\n\r\n") + server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == "POST" { + defer r.Body.Close() + body, _ := ioutil.ReadAll(r.Body) - if r.Method == "POST" { - defer r.Body.Close() - body, _ := ioutil.ReadAll(r.Body) + if len(r.TransferEncoding) > 0 && r.TransferEncoding[0] == "chunked" { + if string(body) != "Wikipedia in\r\n\r\nchunks." { + t.Error("Wrong POST body:", body, string(body)) + } + } else { + if string(body) != "a=1&b=2" { + buf, _ := httputil.DumpRequest(r, true) + t.Error("Wrong POST body:", string(body), string(buf)) + } + } + } - if len(r.TransferEncoding) > 0 && r.TransferEncoding[0] == "chunked" { - if string(body) != "Wikipedia in\r\n\r\nchunks." { - t.Error("Wrong POST body:", body, string(body)) - } - } else { - if string(body) != "a=1&b=2" { - buf, _ := httputil.DumpRequest(r, true) - t.Error("Wrong POST body:", string(body), string(buf)) - } - } - } + wg.Done() + })) - wg.Done() - })) + client := NewHTTPClient(server.URL) + wg.Add(4) + client.Send(GET_payload) + client.Send(POST_payload) + client.Send(POST_CHUNKED_payload) + client.Send(POST_payload) - client := NewHTTPClient(server.URL) - - wg.Add(4) - client.Send(GET_payload) - client.Send(POST_payload) - client.Send(POST_CHUNKED_payload) - client.Send(POST_payload) - - wg.Wait() -} \ No newline at end of file + wg.Wait() +} diff --git a/output_http.go b/output_http.go index e702a60..4b1d08c 100644 --- a/output_http.go +++ b/output_http.go @@ -3,6 +3,7 @@ package main import ( "bufio" "bytes" + "crypto/tls" "io" "io/ioutil" "log" @@ -11,7 +12,6 @@ import ( "strings" "sync/atomic" "time" - "crypto/tls" ) type RedirectNotAllowed struct{} @@ -142,11 +142,11 @@ func (o *HTTPOutput) WorkerMaster() { func (o *HTTPOutput) Worker() { tr := &http.Transport{ - TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, - } + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + } client := &http.Client{ - Transport: tr, + Transport: tr, CheckRedirect: o.customCheckRedirect, } diff --git a/output_http_test.go b/output_http_test.go index e3afb5c..af13c1a 100644 --- a/output_http_test.go +++ b/output_http_test.go @@ -5,8 +5,8 @@ import ( "io/ioutil" "net" "net/http" - "net/http/httputil" "net/http/httptest" + _ "net/http/httputil" "sync" "testing" "time" @@ -65,8 +65,7 @@ func TestHTTPOutput(t *testing.T) { body, _ := ioutil.ReadAll(req.Body) if string(body) != "a=1&b=2" { - buf, _ := httputil.DumpRequest(req, true) - t.Error("Wrong POST body:", string(buf)) + t.Error("Wrong POST body:", string(body)) } } @@ -95,7 +94,6 @@ func TestHTTPOutput(t *testing.T) { close(quit) } - func TestOutputHTTPSSL(t *testing.T) { wg := new(sync.WaitGroup) quit := make(chan int) diff --git a/test_input.go b/test_input.go index 7de50e2..2140728 100644 --- a/test_input.go +++ b/test_input.go @@ -28,7 +28,7 @@ func (i *TestInput) EmitGET() { } func (i *TestInput) EmitPOST() { - i.data <- []byte("POST /pub/WWW/ HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2\r\n\r\n") + i.data <- []byte("POST /pub/WWW/ HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") } func (i *TestInput) EmitChunkedPOST() { From decb9483a540351f08510a37c0e6e32a590a6394 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Thu, 2 Jul 2015 10:25:47 +0500 Subject: [PATCH 06/30] Fix docker file path --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 2b174ce..3fecf83 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,8 +2,8 @@ FROM google/golang RUN cd /goroot/src/ && GOOS=linux GOARCH=386 ./make.bash --no-clean -WORKDIR /gopath/src/gor +WORKDIR /gopath/src/github.com/buger/gor/ -ADD . /gopath/src/gor +ADD . /gopath/src/github.com/buger/gor/ RUN go get \ No newline at end of file From a08ecc79fc40925033150c46351ab0d8e07f5f2b Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Thu, 2 Jul 2015 18:33:38 +0500 Subject: [PATCH 07/30] Add support for timeouts and servers closing connections --- Makefile | 18 ++++++------ http_client.go | 59 +++++++++++++++++++++++++++++++------- http_client_test.go | 70 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 129 insertions(+), 18 deletions(-) diff --git a/Makefile b/Makefile index 475e122..12db096 100644 --- a/Makefile +++ b/Makefile @@ -1,31 +1,33 @@ SOURCE = emitter.go gor.go gor_stat.go input_dummy.go input_file.go input_raw.go input_tcp.go limiter.go output_dummy.go output_file.go input_http.go output_http.go output_tcp.go plugins.go settings.go settings_header_filters.go settings_header_hash_filters.go settings_headers.go settings_methods.go settings_option.go settings_url_regexp.go test_input.go elasticsearch.go settings_url_map.go +SOURCE_PATH = /gopath/src/github.com/buger/gor/ + release: release-x86 release-x64 release-x64: - docker run -v `pwd`:/gopath/src/gor -t --env GOOS=linux --env GOARCH=amd64 --env CGO_ENABLED=0 -i gor go build && tar -czf gor_x64.tar.gz gor && rm gor + docker run -v `pwd`:$(SOURCE_PATH) -t --env GOOS=linux --env GOARCH=amd64 --env CGO_ENABLED=0 -i gor go build && tar -czf gor_x64.tar.gz gor && rm gor release-x86: - docker run -v `pwd`:/gopath/src/gor -t --env GOOS=linux --env GOARCH=386 --env CGO_ENABLED=0 -i gor go build && tar -czf gor_x86.tar.gz gor && rm gor + docker run -v `pwd`:$(SOURCE_PATH) -t --env GOOS=linux --env GOARCH=386 --env CGO_ENABLED=0 -i gor go build && tar -czf gor_x86.tar.gz gor && rm gor dbuild: docker build -t gor . dtest: - docker run -v `pwd`:/gopath/src/gor -t -i --env GORACE="halt_on_error=1" gor go test $(ARGS) -race -v + docker run -v `pwd`:$(SOURCE_PATH) -t -i --env GORACE="halt_on_error=1" gor go test $(ARGS) -race -v dfmt: - docker run -v `pwd`:/gopath/src/gor -t -i gor go fmt + docker run -v `pwd`:$(SOURCE_PATH) -t -i gor go fmt dvet: - docker run -v `pwd`:/gopath/src/gor -t -i gor go vet + docker run -v `pwd`:$(SOURCE_PATH) -t -i gor go vet dbench: - docker run -v `pwd`:/gopath/src/gor -t -i gor go test -v -run NOT_EXISTING -bench HTTP + docker run -v `pwd`:$(SOURCE_PATH) -t -i gor go test -v -run NOT_EXISTING -bench HTTP # Used mainly for debugging, because docker container do not have access to parent machine ports drun: - docker run -v `pwd`:/gopath/src/gor -t -i gor go run $(SOURCE) --input-dummy=0 --input-http=:9000 --output-http="http://localhost:9000" --verbose + docker run -v `pwd`:$(SOURCE_PATH) -t -i gor go run $(SOURCE) --input-dummy=0 --input-http=:9000 --output-http="http://localhost:9000" --verbose dbash: - docker run -v `pwd`:/gopath/src/gor -t -i gor /bin/bash \ No newline at end of file + docker run -v `pwd`:$(SOURCE_PATH) -t -i gor /bin/bash \ No newline at end of file diff --git a/http_client.go b/http_client.go index ae36b2d..b9b8e13 100644 --- a/http_client.go +++ b/http_client.go @@ -2,9 +2,11 @@ package main import ( "crypto/tls" + "io" "net" "net/url" "strings" + "time" ) var defaultPorts = map[string]string{ @@ -15,13 +17,17 @@ var defaultPorts = map[string]string{ type HTTPClient struct { baseURL *url.URL conn net.Conn - buf []byte + respBuf []byte } func NewHTTPClient(baseURL string) *HTTPClient { + if !strings.HasPrefix(baseURL, "http") { + baseURL = "http://" + baseURL + } + client := new(HTTPClient) client.baseURL, _ = url.Parse(baseURL) - client.buf = make([]byte, 4096*10) + client.respBuf = make([]byte, 4096*10) if !strings.Contains(client.baseURL.Host, ":") { client.baseURL.Host += ":" + defaultPorts[client.baseURL.Scheme] @@ -31,11 +37,17 @@ func NewHTTPClient(baseURL string) *HTTPClient { } func (c *HTTPClient) Connect() (err error) { + c.Disconnect() + c.conn, err = net.Dial("tcp", c.baseURL.Host) if c.baseURL.Scheme == "https" { tlsConn := tls.Client(c.conn, &tls.Config{InsecureSkipVerify: true}) - err = tlsConn.Handshake() + + if err = tlsConn.Handshake(); err != nil { + return + } + c.conn = tlsConn } @@ -43,19 +55,46 @@ func (c *HTTPClient) Connect() (err error) { } func (c *HTTPClient) Disconnect() { - c.conn.Close() - c.conn = nil + if c.conn != nil { + c.conn.Close() + c.conn = nil + Debug("Disconnected: ", c.baseURL) + } +} + +func (c *HTTPClient) isAlive() bool { + one := make([]byte, 1) + + // Ready 1 byte from socket without timeout to check if it not closed + c.conn.SetReadDeadline(time.Now().Add(time.Millisecond)) + if _, err := c.conn.Read(one); err == io.EOF { + return false + } + + return true } func (c *HTTPClient) Send(data []byte) (response []byte, err error) { - if c.conn == nil { + if c.conn == nil || !c.isAlive() { + Debug("Connecting:", c.baseURL) c.Connect() } - _, err = c.conn.Write(data) - n, err := c.conn.Read(c.buf) + timeout := time.Now().Add(5 * time.Second) - Debug(string(c.buf[:n])) + c.conn.SetWriteDeadline(timeout) + if _, err = c.conn.Write(data); err != nil { + Debug("Write error:", err, c.baseURL) + return + } - return c.buf[:n], err + c.conn.SetReadDeadline(timeout) + n, err := c.conn.Read(c.respBuf) + + if err != nil { + Debug("READ ERRORR!", err, c.conn) + return + } + + return c.respBuf[:n], err } diff --git a/http_client_test.go b/http_client_test.go index 63504b1..ce369c6 100644 --- a/http_client_test.go +++ b/http_client_test.go @@ -2,11 +2,13 @@ package main import ( "io/ioutil" + "net" "net/http" "net/http/httptest" "net/http/httputil" "sync" "testing" + _ "time" ) func TestHTTPClientURLPort(t *testing.T) { @@ -24,6 +26,11 @@ func TestHTTPClientURLPort(t *testing.T) { if c3.baseURL.String() != "https://example.com:1" { t.Error("Sould use specified port:", c3.baseURL.String()) } + + c4 := NewHTTPClient("example.com") + if c4.baseURL.String() != "http://example.com:80" { + t.Error("Sould add default protocol:", c4.baseURL.String()) + } } func TestHTTPClientSend(t *testing.T) { @@ -111,3 +118,66 @@ func TestHTTPClientHTTPSSend(t *testing.T) { wg.Wait() } + +func TestHTTPClientServerInstantDisconnect(t *testing.T) { + wg := new(sync.WaitGroup) + + GET_payload := []byte("GET / HTTP/1.1\r\n\r\n") + + ln, _ := net.Listen("tcp", ":0") + + go func() { + for { + conn, _ := ln.Accept() + conn.Close() + + wg.Done() + } + }() + + client := NewHTTPClient(ln.Addr().String()) + + wg.Add(2) + client.Send(GET_payload) + client.Send(GET_payload) + + wg.Wait() +} + +func TestHTTPClientServerNoKeepAlive(t *testing.T) { + wg := new(sync.WaitGroup) + + GET_payload := []byte("GET / HTTP/1.1\r\n\r\n") + + ln, _ := net.Listen("tcp", ":0") + + go func() { + for { + conn, err := ln.Accept() + if err != nil { + // handle error + } + + buf := make([]byte, 4096) + reqLen, err := conn.Read(buf) + if err != nil { + t.Error("Error reading:", err.Error()) + } + Debug("Received: ", string(buf[0:reqLen])) + conn.Write([]byte("OK")) + + // No keep-alive connections + conn.Close() + + wg.Done() + } + }() + + client := NewHTTPClient(ln.Addr().String()) + + wg.Add(2) + client.Send(GET_payload) + client.Send(GET_payload) + + wg.Wait() +} From 416586a36598d231ecb1593179b46afeb380b2f1 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Thu, 2 Jul 2015 20:41:25 +0500 Subject: [PATCH 08/30] Add support for redirects --- http_client.go | 62 ++++++++++++++++++++++++++++++++++++++- http_client_test.go | 70 +++++++++++++++++++++++++++++++++++++++------ 2 files changed, 123 insertions(+), 9 deletions(-) diff --git a/http_client.go b/http_client.go index b9b8e13..125a8a5 100644 --- a/http_client.go +++ b/http_client.go @@ -7,6 +7,9 @@ import ( "net/url" "strings" "time" + "bytes" + "bufio" + "errors" ) var defaultPorts = map[string]string{ @@ -14,13 +17,20 @@ var defaultPorts = map[string]string{ "https": "443", } +type HTTPClientConfig struct { + FollowRedirects int + Debug bool +} + type HTTPClient struct { baseURL *url.URL conn net.Conn respBuf []byte + config *HTTPClientConfig + redirectsCount int } -func NewHTTPClient(baseURL string) *HTTPClient { +func NewHTTPClient(baseURL string, config *HTTPClientConfig) *HTTPClient { if !strings.HasPrefix(baseURL, "http") { baseURL = "http://" + baseURL } @@ -28,6 +38,7 @@ func NewHTTPClient(baseURL string) *HTTPClient { client := new(HTTPClient) client.baseURL, _ = url.Parse(baseURL) client.respBuf = make([]byte, 4096*10) + client.config = config if !strings.Contains(client.baseURL.Host, ":") { client.baseURL.Host += ":" + defaultPorts[client.baseURL.Scheme] @@ -74,6 +85,26 @@ func (c *HTTPClient) isAlive() bool { return true } +func header(payload []byte, name []byte) ([]byte, error) { + buf := bytes.NewBuffer(payload) + reader := bufio.NewReader(buf) + + // Skip status line + reader.ReadLine() + + for { + line, _, err := reader.ReadLine() + + if err != nil { + return nil, errors.New("Header not found") + } + + if bytes.HasPrefix(line, name) { + return bytes.Split(line, []byte(": "))[1], nil + } + } +} + func (c *HTTPClient) Send(data []byte) (response []byte, err error) { if c.conn == nil || !c.isAlive() { Debug("Connecting:", c.baseURL) @@ -83,6 +114,11 @@ func (c *HTTPClient) Send(data []byte) (response []byte, err error) { timeout := time.Now().Add(5 * time.Second) c.conn.SetWriteDeadline(timeout) + + if c.config.Debug { + Debug("Sending:", string(data)) + } + if _, err = c.conn.Write(data); err != nil { Debug("Write error:", err, c.baseURL) return @@ -96,5 +132,29 @@ func (c *HTTPClient) Send(data []byte) (response []byte, err error) { return } + if c.config.Debug { + Debug("Received:", string(c.respBuf[:n])) + } + + if c.config.FollowRedirects > 0 && c.redirectsCount < c.config.FollowRedirects { + status := c.respBuf[9:12] + + // 3xx requests + if status[0] == '3' { + c.redirectsCount += 1 + + location, _ := header(c.respBuf[:n], []byte("Location:")) + redirectPayload := []byte("GET " + string(location) + " HTTP/1.1\r\n\r\n") + + if c.config.Debug { + Debug("Redirecting to: " + string(location)) + } + + return c.Send(redirectPayload) + } + } + + c.redirectsCount = 0 + return c.respBuf[:n], err } diff --git a/http_client_test.go b/http_client_test.go index ce369c6..25f8b4c 100644 --- a/http_client_test.go +++ b/http_client_test.go @@ -12,22 +12,22 @@ import ( ) func TestHTTPClientURLPort(t *testing.T) { - c1 := NewHTTPClient("http://example.com") + c1 := NewHTTPClient("http://example.com", &HTTPClientConfig{}) if c1.baseURL.String() != "http://example.com:80" { t.Error("Sould add 80 port for http:", c1.baseURL.String()) } - c2 := NewHTTPClient("https://example.com") + c2 := NewHTTPClient("https://example.com", &HTTPClientConfig{}) if c2.baseURL.String() != "https://example.com:443" { t.Error("Sould add 443 port for https:", c2.baseURL.String()) } - c3 := NewHTTPClient("https://example.com:1") + c3 := NewHTTPClient("https://example.com:1", &HTTPClientConfig{}) if c3.baseURL.String() != "https://example.com:1" { t.Error("Sould use specified port:", c3.baseURL.String()) } - c4 := NewHTTPClient("example.com") + c4 := NewHTTPClient("example.com", &HTTPClientConfig{}) if c4.baseURL.String() != "http://example.com:80" { t.Error("Sould add default protocol:", c4.baseURL.String()) } @@ -65,7 +65,7 @@ func TestHTTPClientSend(t *testing.T) { wg.Done() })) - client := NewHTTPClient(server.URL) + client := NewHTTPClient(server.URL, &HTTPClientConfig{}) wg.Add(4) client.Send(POST_payload) @@ -108,7 +108,7 @@ func TestHTTPClientHTTPSSend(t *testing.T) { wg.Done() })) - client := NewHTTPClient(server.URL) + client := NewHTTPClient(server.URL, &HTTPClientConfig{}) wg.Add(4) client.Send(GET_payload) @@ -135,7 +135,7 @@ func TestHTTPClientServerInstantDisconnect(t *testing.T) { } }() - client := NewHTTPClient(ln.Addr().String()) + client := NewHTTPClient(ln.Addr().String(), &HTTPClientConfig{}) wg.Add(2) client.Send(GET_payload) @@ -173,7 +173,7 @@ func TestHTTPClientServerNoKeepAlive(t *testing.T) { } }() - client := NewHTTPClient(ln.Addr().String()) + client := NewHTTPClient(ln.Addr().String(), &HTTPClientConfig{}) wg.Add(2) client.Send(GET_payload) @@ -181,3 +181,57 @@ func TestHTTPClientServerNoKeepAlive(t *testing.T) { wg.Wait() } + +func TestHTTPClientRedirect(t *testing.T) { + wg := new(sync.WaitGroup) + + GET_payload := []byte("GET / HTTP/1.1\r\n\r\n") + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + + if r.URL.Path == "/" { + http.Redirect(w, r, "/new", 301) + } + + wg.Done() + })) + + client := NewHTTPClient(server.URL, &HTTPClientConfig{FollowRedirects: 1, Debug: false}) + + // Should do 2 queries + wg.Add(2) + client.Send(GET_payload) + + wg.Wait() +} + +func TestHTTPClientRedirectLimit(t *testing.T) { + wg := new(sync.WaitGroup) + + GET_payload := []byte("GET / HTTP/1.1\r\n\r\n") + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + + if r.URL.Path == "/" { + http.Redirect(w, r, "/r1", 301) + } + + if r.URL.Path == "/r1" { + http.Redirect(w, r, "/r2", 301) + } + + if r.URL.Path == "/r2" { + http.Redirect(w, r, "/new", 301) + } + + wg.Done() + })) + + client := NewHTTPClient(server.URL, &HTTPClientConfig{FollowRedirects: 2, Debug: false}) + + // Have 3 redirects + 1 GET, but should do only 2 redirects + GET + wg.Add(3) + client.Send(GET_payload) + + wg.Wait() +} \ No newline at end of file From 481e4f2e74219326be162e4f7d5d4163665b004e Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Thu, 2 Jul 2015 23:08:31 +0500 Subject: [PATCH 09/30] Refactor http output settings --- input_raw_test.go | 8 ++--- output_http.go | 73 +++++++++++++++++++++++---------------------- output_http_test.go | 16 +++------- plugins.go | 2 +- settings.go | 22 +++++++------- 5 files changed, 56 insertions(+), 65 deletions(-) diff --git a/input_raw_test.go b/input_raw_test.go index ba9b537..60bcf49 100644 --- a/input_raw_test.go +++ b/input_raw_test.go @@ -82,9 +82,7 @@ func TestInputRAW100Expect(t *testing.T) { }) replay_address := listener.Addr().String() - headers := HTTPHeaders{HTTPHeader{"User-Agent", "Gor"}} - methods := HTTPMethods{"GET", "PUT", "POST"} - http_output := NewHTTPOutput(replay_address, headers, methods, HTTPUrlRegexp{}, HTTPHeaderFilters{}, HTTPHeaderHashFilters{}, "", UrlRewriteMap{}, 0) + http_output := NewHTTPOutput(replay_address, &HTTPOutputConfig{}) Plugins.Inputs = []io.Reader{input} Plugins.Outputs = []io.Writer{test_output, http_output} @@ -141,9 +139,7 @@ func TestInputRAWChunkedEncoding(t *testing.T) { }) replay_address := listener.Addr().String() - headers := HTTPHeaders{HTTPHeader{"User-Agent", "Gor"}} - methods := HTTPMethods{"GET", "PUT", "POST"} - http_output := NewHTTPOutput(replay_address, headers, methods, HTTPUrlRegexp{}, HTTPHeaderFilters{}, HTTPHeaderHashFilters{}, "", UrlRewriteMap{}, 0) + http_output := NewHTTPOutput(replay_address, &HTTPOutputConfig{}) Plugins.Inputs = []io.Reader{input} Plugins.Outputs = []io.Writer{test_output, http_output} diff --git a/output_http.go b/output_http.go index 4b1d08c..cd39eed 100644 --- a/output_http.go +++ b/output_http.go @@ -22,7 +22,7 @@ func (e *RedirectNotAllowed) Error() string { // customCheckRedirect disables redirects https://github.com/buger/gor/pull/15 func (o *HTTPOutput) customCheckRedirect(req *http.Request, via []*http.Request) error { - if len(via) >= o.redirectLimit { + if len(via) >= o.config.redirectLimit { return new(RedirectNotAllowed) } return nil @@ -56,6 +56,23 @@ func ParseRequest(data []byte) (request *http.Request, err error) { const InitialDynamicWorkers = 10 +type HTTPOutputConfig struct { + redirectLimit int + + urlRegexp HTTPUrlRegexp + urlRewrite UrlRewriteMap + headerFilters HTTPHeaderFilters + headerHashFilters HTTPHeaderHashFilters + + stats bool + workers int + + headers HTTPHeaders + methods HTTPMethods + + elasticSearch string +} + type HTTPOutput struct { // Keep this as first element of struct because it guarantees 64bit // alignment. atomic.* functions crash on 32bit machines if operand is not @@ -66,24 +83,16 @@ type HTTPOutput struct { limit int queue chan []byte - redirectLimit int - needWorker chan int - urlRegexp HTTPUrlRegexp - headerFilters HTTPHeaderFilters - headerHashFilters HTTPHeaderHashFilters - outputHTTPUrlRewrite UrlRewriteMap - - headers HTTPHeaders - methods HTTPMethods - - elasticSearch *ESPlugin + config *HTTPOutputConfig queueStats *GorStat + + elasticSearch *ESPlugin } -func NewHTTPOutput(address string, headers HTTPHeaders, methods HTTPMethods, urlRegexp HTTPUrlRegexp, headerFilters HTTPHeaderFilters, headerHashFilters HTTPHeaderHashFilters, elasticSearchAddr string, outputHTTPUrlRewrite UrlRewriteMap, outputHTTPRedirects int) io.Writer { +func NewHTTPOutput(address string, config *HTTPOutputConfig) io.Writer { o := new(HTTPOutput) @@ -92,33 +101,25 @@ func NewHTTPOutput(address string, headers HTTPHeaders, methods HTTPMethods, url } o.address = address - o.headers = headers - o.methods = methods + o.config = config - o.redirectLimit = Settings.outputHTTPRedirects - - o.urlRegexp = urlRegexp - o.headerFilters = headerFilters - o.headerHashFilters = headerHashFilters - o.outputHTTPUrlRewrite = outputHTTPUrlRewrite - - o.queue = make(chan []byte, 100) - if Settings.outputHTTPStats { + if o.config.stats { o.queueStats = NewGorStat("output_http") } + o.queue = make(chan []byte, 100) o.needWorker = make(chan int, 1) // Initial workers count - if Settings.outputHTTPWorkers == -1 { + if o.config.workers == 0 { o.needWorker <- InitialDynamicWorkers } else { - o.needWorker <- Settings.outputHTTPWorkers + o.needWorker <- o.config.workers } - if elasticSearchAddr != "" { + if o.config.elasticSearch != "" { o.elasticSearch = new(ESPlugin) - o.elasticSearch.Init(elasticSearchAddr) + o.elasticSearch.Init(o.config.elasticSearch) } go o.WorkerMaster() @@ -134,7 +135,7 @@ func (o *HTTPOutput) WorkerMaster() { } // Disable dynamic scaling if workers poll fixed size - if Settings.outputHTTPWorkers != -1 { + if o.config.workers != 0 { return } } @@ -161,7 +162,7 @@ func (o *HTTPOutput) Worker() { death_count = 0 case <-time.After(time.Millisecond * 100): // When dynamic scaling enabled workers die after 2s of inactivity - if Settings.outputHTTPWorkers == -1 { + if o.config.workers == 0 { death_count += 1 } else { continue @@ -186,11 +187,11 @@ func (o *HTTPOutput) Write(data []byte) (n int, err error) { o.queue <- buf - if Settings.outputHTTPStats { + if o.config.stats { o.queueStats.Write(len(o.queue)) } - if Settings.outputHTTPWorkers == -1 { + if o.config.workers == 0 { workersCount := atomic.LoadInt64(&o.activeWorkers) if len(o.queue) > int(workersCount) { @@ -209,16 +210,16 @@ func (o *HTTPOutput) sendRequest(client *http.Client, data []byte) { return } - if len(o.methods) > 0 && !o.methods.Contains(request.Method) { + if len(o.config.methods) > 0 && !o.config.methods.Contains(request.Method) { return } - if !(o.urlRegexp.Good(request) && o.headerFilters.Good(request) && o.headerHashFilters.Good(request)) { + if !(o.config.urlRegexp.Good(request) && o.config.headerFilters.Good(request) && o.config.headerHashFilters.Good(request)) { return } // Rewrite the path as necessary - request.URL.Path = o.outputHTTPUrlRewrite.Rewrite(request.URL.Path) + request.URL.Path = o.config.urlRewrite.Rewrite(request.URL.Path) // Change HOST of original request URL := o.address + request.URL.Path + "?" + request.URL.RawQuery @@ -226,7 +227,7 @@ func (o *HTTPOutput) sendRequest(client *http.Client, data []byte) { request.RequestURI = "" request.URL, _ = url.ParseRequestURI(URL) - for _, header := range o.headers { + for _, header := range o.config.headers { SetHeader(request, header.Name, header.Value) } diff --git a/output_http_test.go b/output_http_test.go index af13c1a..82c8877 100644 --- a/output_http_test.go +++ b/output_http_test.go @@ -75,7 +75,7 @@ func TestHTTPOutput(t *testing.T) { headers := HTTPHeaders{HTTPHeader{"User-Agent", "Gor"}} methods := HTTPMethods{"GET", "PUT", "POST"} - output := NewHTTPOutput(listener.Addr().String(), headers, methods, HTTPUrlRegexp{}, HTTPHeaderFilters{}, HTTPHeaderHashFilters{}, "", UrlRewriteMap{}, 0) + output := NewHTTPOutput(listener.Addr().String(), &HTTPOutputConfig{headers: headers, methods: methods}) Plugins.Inputs = []io.Reader{input} Plugins.Outputs = []io.Writer{output} @@ -104,11 +104,7 @@ func TestOutputHTTPSSL(t *testing.T) { })) input := NewTestInput() - - headers := HTTPHeaders{HTTPHeader{"User-Agent", "Gor"}} - methods := HTTPMethods{"GET", "PUT", "POST"} - - http_output := NewHTTPOutput(server.URL, headers, methods, HTTPUrlRegexp{}, HTTPHeaderFilters{}, HTTPHeaderHashFilters{}, "", UrlRewriteMap{}, 0) + http_output := NewHTTPOutput(server.URL, &HTTPOutputConfig{}) Plugins.Inputs = []io.Reader{input} Plugins.Outputs = []io.Writer{http_output} @@ -128,17 +124,13 @@ func BenchmarkHTTPOutput(b *testing.B) { wg := new(sync.WaitGroup) quit := make(chan int) - input := NewTestInput() - - headers := HTTPHeaders{HTTPHeader{"User-Agent", "Gor"}} - methods := HTTPMethods{"GET", "PUT", "POST"} - listener := startHTTP(func(req *http.Request) { time.Sleep(50 * time.Millisecond) wg.Done() }) - output := NewHTTPOutput(listener.Addr().String(), headers, methods, HTTPUrlRegexp{}, HTTPHeaderFilters{}, HTTPHeaderHashFilters{}, "", UrlRewriteMap{}, 0) + input := NewTestInput() + output := NewHTTPOutput(listener.Addr().String(), &HTTPOutputConfig{}) Plugins.Inputs = []io.Reader{input} Plugins.Outputs = []io.Writer{output} diff --git a/plugins.go b/plugins.go index 20234a0..65d7937 100644 --- a/plugins.go +++ b/plugins.go @@ -96,6 +96,6 @@ func InitPlugins() { } for _, options := range Settings.outputHTTP { - registerPlugin(NewHTTPOutput, options, Settings.outputHTTPHeaders, Settings.outputHTTPMethods, Settings.outputHTTPUrlRegexp, Settings.outputHTTPHeaderFilters, Settings.outputHTTPHeaderHashFilters, Settings.outputHTTPElasticSearch, Settings.outputHTTPUrlRewrite, Settings.outputHTTPRedirects) + registerPlugin(NewHTTPOutput, options, &HTTPOutputSettings) } } diff --git a/settings.go b/settings.go index c5a8452..6d917e9 100644 --- a/settings.go +++ b/settings.go @@ -44,6 +44,7 @@ type AppSettings struct { } var Settings AppSettings = AppSettings{} +var HTTPOutputSettings = HTTPOutputConfig{} func usage() { fmt.Printf("Gor is a simple http traffic replication tool written in Go. Its main goal is to replay traffic from production servers to staging and dev environments.\nProject page: https://github.com/buger/gor\nAuthor: leonsbox@gmail.com\nCurrent Version: %s\n\n", VERSION) @@ -74,21 +75,22 @@ func init() { flag.Var(&Settings.inputHTTP, "input-http", "Read requests from HTTP, should be explicitly sent from your application:\n\t# Listen for http on 9000\n\tgor --input-http :9000 --output-http staging.com") flag.Var(&Settings.outputHTTP, "output-http", "Forwards incoming requests to given http address.\n\t# Redirect all incoming requests to staging.com address \n\tgor --input-raw :80 --output-http http://staging.com") - flag.Var(&Settings.outputHTTPHeaders, "output-http-header", "Inject additional headers to http reqest:\n\tgor --input-raw :8080 --output-http staging.com --output-http-header 'User-Agent: Gor'") - flag.Var(&Settings.outputHTTPMethods, "output-http-method", "Whitelist of HTTP methods to replay. Anything else will be dropped:\n\tgor --input-raw :8080 --output-http staging.com --output-http-method GET --output-http-method OPTIONS") - flag.Var(&Settings.outputHTTPUrlRegexp, "output-http-url-regexp", "A regexp to match requests against. Anything else will be dropped:\n\t gor --input-raw :8080 --output-http staging.com --output-http-url-regexp ^www.") - flag.Var(&Settings.outputHTTPHeaderFilters, "output-http-header-filter", "A regexp to match a specific header against. Requests with non-matching headers will be dropped:\n\t gor --input-raw :8080 --output-http staging.com --output-http-header-filter api-version:^v1") - flag.Var(&Settings.outputHTTPHeaderHashFilters, "output-http-header-hash-filter", "Takes a fraction of requests, consistently taking or rejecting a request based on the FNV32-1A hash of a specific header. The fraction must have a denominator that is a power of two:\n\t gor --input-raw :8080 --output-http staging.com --output-http-header-hash-filter user-id:1/4") - flag.IntVar(&Settings.outputHTTPWorkers, "output-http-workers", -1, "Gor uses dynamic worker scaling by default. Enter a number to run a set number of workers.") - flag.BoolVar(&Settings.outputHTTPStats, "output-http-stats", false, "Report http output queue stats to console every 5 seconds.") + flag.Var(&HTTPOutputSettings.headers, "output-http-header", "Inject additional headers to http reqest:\n\tgor --input-raw :8080 --output-http staging.com --output-http-header 'User-Agent: Gor'") + flag.Var(&HTTPOutputSettings.methods, "output-http-method", "Whitelist of HTTP methods to replay. Anything else will be dropped:\n\tgor --input-raw :8080 --output-http staging.com --output-http-method GET --output-http-method OPTIONS") + flag.Var(&HTTPOutputSettings.urlRegexp, "output-http-url-regexp", "A regexp to match requests against. Anything else will be dropped:\n\t gor --input-raw :8080 --output-http staging.com --output-http-url-regexp ^www.") + flag.Var(&HTTPOutputSettings.headerFilters, "output-http-header-filter", "A regexp to match a specific header against. Requests with non-matching headers will be dropped:\n\t gor --input-raw :8080 --output-http staging.com --output-http-header-filter api-version:^v1") + flag.Var(&HTTPOutputSettings.headerHashFilters, "output-http-header-hash-filter", "Takes a fraction of requests, consistently taking or rejecting a request based on the FNV32-1A hash of a specific header. The fraction must have a denominator that is a power of two:\n\t gor --input-raw :8080 --output-http staging.com --output-http-header-hash-filter user-id:1/4") + flag.IntVar(&HTTPOutputSettings.workers, "output-http-workers", 0, "Gor uses dynamic worker scaling by default. Enter a number to run a set number of workers.") + flag.BoolVar(&HTTPOutputSettings.stats, "output-http-stats", false, "Report http output queue stats to console every 5 seconds.") - flag.StringVar(&Settings.outputHTTPElasticSearch, "output-http-elasticsearch", "", "Send request and response stats to ElasticSearch:\n\tgor --input-raw :8080 --output-http staging.com --output-http-elasticsearch 'es_host:api_port/index_name'") - flag.Var(&Settings.outputHTTPUrlRewrite, "output-http-rewrite-url", "Rewrite the requst url based on a mapping:\n\tgor --input-raw :8080 --output-http staging.com --output-http-rewrite-url /xml_test/interface.php:/api/service.do") - flag.IntVar(&Settings.outputHTTPRedirects, "output-http-redirects", 0, "Enable how often redirects should be followed.") + flag.StringVar(&HTTPOutputSettings.elasticSearch, "output-http-elasticsearch", "", "Send request and response stats to ElasticSearch:\n\tgor --input-raw :8080 --output-http staging.com --output-http-elasticsearch 'es_host:api_port/index_name'") + flag.Var(&HTTPOutputSettings.urlRewrite, "output-http-rewrite-url", "Rewrite the requst url based on a mapping:\n\tgor --input-raw :8080 --output-http staging.com --output-http-rewrite-url /xml_test/interface.php:/api/service.do") + flag.IntVar(&HTTPOutputSettings.redirectLimit, "output-http-redirects", 0, "Enable how often redirects should be followed.") } func Debug(args ...interface{}) { if Settings.verbose { + log.Print("[DEBUG] ") log.Println(args...) } } From 925ff151dcb43a7f3509981829681ab47a1c12d6 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Sat, 4 Jul 2015 19:59:14 +0500 Subject: [PATCH 10/30] Update README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 860ef35..0b3baea 100644 --- a/README.md +++ b/README.md @@ -380,4 +380,5 @@ To achieve the top most performance you should tune the source server system lim * [TomTom](http://www.tomtom.com/) * [3SCALE](http://www.3scale.net/) * [Optionlab](http://www.opinionlab.com) +* [TubeMogul] (http://tubemogul.com) * To add your company drop me a line to github.com/buger or leonsbox@gmail.com From 557cdbebb6c956df57dee51a195eabac43b0d71c Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Sat, 4 Jul 2015 22:05:23 +0500 Subject: [PATCH 11/30] Update README.md --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 0b3baea..4428d9d 100644 --- a/README.md +++ b/README.md @@ -27,6 +27,8 @@ sudo gor --input-raw :80 --output-tcp replay.local:28020 gor --input-tcp replay.local:28020 --output-http http://staging.com ``` +Since Gor use raw sockets to capture traffic it require `sudo` access. Alternatively you can allow access to raw sockets like this: `sudo setcap CAP_NET_RAW=ep gor` + ### Using 1 Gor instance for both listening and replaying It's recommended to use separate server for replaying traffic, but if you have enough CPU resources you can use single Gor instance. From 72b15db93ff18884165468964d2e648ee6132c4d Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Sun, 5 Jul 2015 09:42:59 +0500 Subject: [PATCH 12/30] Add utilities for modifying HTTP payload --- Makefile | 2 +- byteutils/byteutils.go | 37 +++++++++++++++ byteutils/byteutils_test.go | 32 +++++++++++++ http_client.go | 34 +++----------- proto/proto.go | 81 +++++++++++++++++++++++++++++++++ proto/proto_test.go | 90 +++++++++++++++++++++++++++++++++++++ 6 files changed, 248 insertions(+), 28 deletions(-) create mode 100644 byteutils/byteutils.go create mode 100644 byteutils/byteutils_test.go create mode 100644 proto/proto.go create mode 100644 proto/proto_test.go diff --git a/Makefile b/Makefile index 12db096..9ff6068 100644 --- a/Makefile +++ b/Makefile @@ -14,7 +14,7 @@ dbuild: docker build -t gor . dtest: - docker run -v `pwd`:$(SOURCE_PATH) -t -i --env GORACE="halt_on_error=1" gor go test $(ARGS) -race -v + docker run -v `pwd`:$(SOURCE_PATH) -t -i --env GORACE="halt_on_error=1" gor go test ./... $(ARGS) -race -v dfmt: docker run -v `pwd`:$(SOURCE_PATH) -t -i gor go fmt diff --git a/byteutils/byteutils.go b/byteutils/byteutils.go new file mode 100644 index 0000000..33ea49e --- /dev/null +++ b/byteutils/byteutils.go @@ -0,0 +1,37 @@ +package byteutils + +func Cut(a []byte, from, to int) []byte { + copy(a[from:], a[to:]) + a = a[:len(a)-to+from] + + return a +} + +func Insert(a []byte, i int, b []byte) []byte { + a = append(a, make([]byte, len(b))...) + copy(a[i+len(b):], a[i:]) + copy(a[i:i+len(b)], b) + + return a +} + +// Unlike bytes.Replace it allows you to specify range +func Replace(a []byte, from, to int, new []byte) []byte { + lenDiff := len(new) - (to - from) + + if lenDiff > 0 { + // Extend if new segment bigger + a = append(a, make([]byte, lenDiff)...) + copy(a[to+lenDiff:], a[to:]) + copy(a[from:from+len(new)], new) + + return a + } else if lenDiff < 0 { + copy(a[from:], new) + copy(a[from+len(new):],a[to:]) + return a[:len(a) + lenDiff] + } else { // same size + copy(a[from:], new) + return a + } +} \ No newline at end of file diff --git a/byteutils/byteutils_test.go b/byteutils/byteutils_test.go new file mode 100644 index 0000000..5b7bb04 --- /dev/null +++ b/byteutils/byteutils_test.go @@ -0,0 +1,32 @@ +package byteutils + +import ( + "testing" + "bytes" +) + +func TestCut(t *testing.T) { + if !bytes.Equal(Cut([]byte("123456"), 2, 4), []byte("1256")) { + t.Error("Should properly cut") + } +} + +func TestInsert(t *testing.T) { + if !bytes.Equal(Insert([]byte("123456"), 2, []byte("abcd")), []byte("12abcd3456")) { + t.Error("Should insert into middle of slice") + } +} + +func TestReplace(t *testing.T) { + if !bytes.Equal(Replace([]byte("123456"), 2, 4, []byte("ab")), []byte("12ab56")) { + t.Error("Should replace when same length") + } + + if !bytes.Equal(Replace([]byte("123456"), 2, 4, []byte("abcd")), []byte("12abcd56")) { + t.Error("Should replace when replacement length bigger") + } + + if !bytes.Equal(Replace([]byte("123456"), 2, 5, []byte("ab")), []byte("12ab6")) { + t.Error("Should replace when replacement length bigger") + } +} \ No newline at end of file diff --git a/http_client.go b/http_client.go index 125a8a5..4f78e9d 100644 --- a/http_client.go +++ b/http_client.go @@ -7,9 +7,7 @@ import ( "net/url" "strings" "time" - "bytes" - "bufio" - "errors" + "github.com/buger/gor/proto" ) var defaultPorts = map[string]string{ @@ -85,26 +83,6 @@ func (c *HTTPClient) isAlive() bool { return true } -func header(payload []byte, name []byte) ([]byte, error) { - buf := bytes.NewBuffer(payload) - reader := bufio.NewReader(buf) - - // Skip status line - reader.ReadLine() - - for { - line, _, err := reader.ReadLine() - - if err != nil { - return nil, errors.New("Header not found") - } - - if bytes.HasPrefix(line, name) { - return bytes.Split(line, []byte(": "))[1], nil - } - } -} - func (c *HTTPClient) Send(data []byte) (response []byte, err error) { if c.conn == nil || !c.isAlive() { Debug("Connecting:", c.baseURL) @@ -132,18 +110,20 @@ func (c *HTTPClient) Send(data []byte) (response []byte, err error) { return } + payload := c.respBuf[:n] + if c.config.Debug { - Debug("Received:", string(c.respBuf[:n])) + Debug("Received:", string(payload)) } if c.config.FollowRedirects > 0 && c.redirectsCount < c.config.FollowRedirects { - status := c.respBuf[9:12] + status := payload[9:12] // 3xx requests if status[0] == '3' { c.redirectsCount += 1 - location, _ := header(c.respBuf[:n], []byte("Location:")) + location, _, _, _ := proto.Header(payload, []byte("Location")) redirectPayload := []byte("GET " + string(location) + " HTTP/1.1\r\n\r\n") if c.config.Debug { @@ -156,5 +136,5 @@ func (c *HTTPClient) Send(data []byte) (response []byte, err error) { c.redirectsCount = 0 - return c.respBuf[:n], err + return payload, err } diff --git a/proto/proto.go b/proto/proto.go new file mode 100644 index 0000000..8a68949 --- /dev/null +++ b/proto/proto.go @@ -0,0 +1,81 @@ +// Low-level interaction with HTTP request payload +package proto + +import ( + "bytes" + "github.com/buger/gor/byteutils" + _ "log" +) + +var CLRF = []byte("\r\n") +var EMPTY_LINE = []byte("\r\n\r\n") +var HEADER_DELIM = []byte(": ") + +// Headers should end with empty line +func MIMEHeadersEndPos(payload []byte) int { + return bytes.Index(payload, EMPTY_LINE) +} + +func MIMEHeadersStartPos(payload []byte) int { + return bytes.Index(payload, CLRF) + 2 // Find first line end +} + +// Find header value or return error +// Do not support multi-line headers +func Header(payload []byte, name []byte) (value []byte, headerStart, valueStart, headerEnd int) { + headerStart = bytes.Index(payload, name) + + if headerStart == -1 { + return + } + + valueStart = headerStart + len(name) + 1 // Skip ":" after header name + if payload[valueStart] == ' ' { // Ignore empty space after ':' + valueStart += 1 + } + headerEnd = valueStart + bytes.IndexByte(payload[valueStart:], '\r') + value = payload[valueStart:headerEnd] + + return +} + +func SetHeader(payload, name, value []byte) []byte { + _, hs, vs, he := Header(payload, name) + + // If header found + if hs != -1 { + return byteutils.Replace(payload, vs, he, value) + } else { + return AddHeader(payload, name, value) + } +} + +func AddHeader(payload, name, value []byte) []byte { + header := make([]byte, len(name) + 2 + len(value) + 2) + copy(header[0:], name) + copy(header[len(name):], HEADER_DELIM) + copy(header[len(name)+2:], value) + copy(header[len(header)-2:], CLRF) + + mimeStart := MIMEHeadersStartPos(payload) + + return byteutils.Insert(payload, mimeStart, header) +} + +func Path(payload []byte) []byte { + start := bytes.IndexByte(payload, ' ') + start += 1 + + end := bytes.IndexByte(payload[start:], ' ') + + return payload[start:start+end] +} + +func SetPath(payload, path []byte) []byte { + start := bytes.IndexByte(payload, ' ') + start += 1 + + end := bytes.IndexByte(payload[start:], ' ') + + return byteutils.Replace(payload, start, start+end, path) +} diff --git a/proto/proto_test.go b/proto/proto_test.go new file mode 100644 index 0000000..bc7dcaf --- /dev/null +++ b/proto/proto_test.go @@ -0,0 +1,90 @@ +package proto + +import ( + "testing" + "bytes" +) + +func TestHeader(t *testing.T) { + var payload, val []byte + var headerStart int + + payload = []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") + + if val, _, _, _ = Header(payload, []byte("Content-Length")); !bytes.Equal(val, []byte("7")) { + t.Error("Should find header value") + } + + payload = []byte("POST /post HTTP/1.1\r\nContent-Length:7\r\nHost: www.w3.org\r\n\r\na=1&b=2") + + if val, _, _, _ = Header(payload, []byte("Content-Length")); !bytes.Equal(val, []byte("7")) { + t.Error("Should find header value without space after :") + } + + if _, headerStart, _, _ = Header(payload, []byte("Not-Found")); headerStart != -1 { + t.Error("Should not found header") + } +} + +func TestMIMEHeadersEndPos(t *testing.T) { + head := []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org") + payload := []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") + + end := MIMEHeadersEndPos(payload) + + if !bytes.Equal(payload[:end], head) { + t.Error("Wrong headers end position:", end) + } +} + +func TestMIMEHeadersStartPos(t *testing.T) { + headers := []byte("Content-Length: 7\r\nHost: www.w3.org") + payload := []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") + + start := MIMEHeadersStartPos(payload) + end := MIMEHeadersEndPos(payload) + + if !bytes.Equal(payload[start:end], headers) { + t.Error("Wrong headers end position:", start, end) + } +} + +func TestSetHeader(t *testing.T) { + var payload, payload_after []byte + + payload = []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") + payload_after = []byte("POST /post HTTP/1.1\r\nContent-Length: 14\r\nHost: www.w3.org\r\n\r\na=1&b=2") + + if payload = SetHeader(payload, []byte("Content-Length"), []byte("14")); !bytes.Equal(payload, payload_after) { + t.Error("Should update header if it exists", string(payload)) + } + + + payload = []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") + payload_after = []byte("POST /post HTTP/1.1\r\nUser-Agent: Gor\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") + + if payload = SetHeader(payload, []byte("User-Agent"), []byte("Gor")); !bytes.Equal(payload, payload_after) { + t.Error("Should add header if not found", string(payload)) + } +} + +func TestPath(t *testing.T) { + var path, payload []byte + + payload = []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") + + if path = Path(payload); !bytes.Equal(path, []byte("/post")) { + t.Error("Should find path", string(path)) + } +} + +func TestSetPath(t *testing.T) { + var payload, payload_after []byte + + payload = []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") + payload_after = []byte("POST /new_path HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") + + if payload = SetPath(payload, []byte("/new_path")); !bytes.Equal(payload, payload_after) { + t.Error("Should replace path", string(payload)) + } +} \ No newline at end of file From 96fd71345338451d82f9c27ae50bc41a15736705 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Sun, 5 Jul 2015 10:25:44 +0500 Subject: [PATCH 13/30] Use separate config for modifier --- http_modifier.go | 11 +++++++++++ output_http.go | 16 +++++----------- output_http_test.go | 3 ++- plugins.go | 2 +- settings.go | 33 ++++++++++++--------------------- 5 files changed, 31 insertions(+), 34 deletions(-) create mode 100644 http_modifier.go diff --git a/http_modifier.go b/http_modifier.go new file mode 100644 index 0000000..73b5203 --- /dev/null +++ b/http_modifier.go @@ -0,0 +1,11 @@ +package main + +type HTTPModifierConfig struct { + urlRegexp HTTPUrlRegexp + urlRewrite UrlRewriteMap + headerFilters HTTPHeaderFilters + headerHashFilters HTTPHeaderHashFilters + + headers HTTPHeaders + methods HTTPMethods +} \ No newline at end of file diff --git a/output_http.go b/output_http.go index cd39eed..14b630e 100644 --- a/output_http.go +++ b/output_http.go @@ -59,16 +59,10 @@ const InitialDynamicWorkers = 10 type HTTPOutputConfig struct { redirectLimit int - urlRegexp HTTPUrlRegexp - urlRewrite UrlRewriteMap - headerFilters HTTPHeaderFilters - headerHashFilters HTTPHeaderHashFilters - stats bool workers int - headers HTTPHeaders - methods HTTPMethods + modifier HTTPModifierConfig elasticSearch string } @@ -210,16 +204,16 @@ func (o *HTTPOutput) sendRequest(client *http.Client, data []byte) { return } - if len(o.config.methods) > 0 && !o.config.methods.Contains(request.Method) { + if len(o.config.modifier.methods) > 0 && !o.config.modifier.methods.Contains(request.Method) { return } - if !(o.config.urlRegexp.Good(request) && o.config.headerFilters.Good(request) && o.config.headerHashFilters.Good(request)) { + if !(o.config.modifier.urlRegexp.Good(request) && o.config.modifier.headerFilters.Good(request) && o.config.modifier.headerHashFilters.Good(request)) { return } // Rewrite the path as necessary - request.URL.Path = o.config.urlRewrite.Rewrite(request.URL.Path) + request.URL.Path = o.config.modifier.urlRewrite.Rewrite(request.URL.Path) // Change HOST of original request URL := o.address + request.URL.Path + "?" + request.URL.RawQuery @@ -227,7 +221,7 @@ func (o *HTTPOutput) sendRequest(client *http.Client, data []byte) { request.RequestURI = "" request.URL, _ = url.ParseRequestURI(URL) - for _, header := range o.config.headers { + for _, header := range o.config.modifier.headers { SetHeader(request, header.Name, header.Value) } diff --git a/output_http_test.go b/output_http_test.go index 82c8877..589cab4 100644 --- a/output_http_test.go +++ b/output_http_test.go @@ -74,8 +74,9 @@ func TestHTTPOutput(t *testing.T) { headers := HTTPHeaders{HTTPHeader{"User-Agent", "Gor"}} methods := HTTPMethods{"GET", "PUT", "POST"} + modifierConfig := HTTPModifierConfig{headers: headers, methods: methods} - output := NewHTTPOutput(listener.Addr().String(), &HTTPOutputConfig{headers: headers, methods: methods}) + output := NewHTTPOutput(listener.Addr().String(), &HTTPOutputConfig{modifier: modifierConfig}) Plugins.Inputs = []io.Reader{input} Plugins.Outputs = []io.Writer{output} diff --git a/plugins.go b/plugins.go index 65d7937..88134ef 100644 --- a/plugins.go +++ b/plugins.go @@ -96,6 +96,6 @@ func InitPlugins() { } for _, options := range Settings.outputHTTP { - registerPlugin(NewHTTPOutput, options, &HTTPOutputSettings) + registerPlugin(NewHTTPOutput, options, &Settings.outputHTTPConfig) } } diff --git a/settings.go b/settings.go index 6d917e9..669d145 100644 --- a/settings.go +++ b/settings.go @@ -31,20 +31,11 @@ type AppSettings struct { inputHTTP MultiOption outputHTTP MultiOption - outputHTTPHeaders HTTPHeaders - outputHTTPMethods HTTPMethods - outputHTTPUrlRegexp HTTPUrlRegexp - outputHTTPUrlRewrite UrlRewriteMap - outputHTTPHeaderFilters HTTPHeaderFilters - outputHTTPHeaderHashFilters HTTPHeaderHashFilters - outputHTTPElasticSearch string - outputHTTPWorkers int - outputHTTPStats bool - outputHTTPRedirects int + + outputHTTPConfig HTTPOutputConfig } var Settings AppSettings = AppSettings{} -var HTTPOutputSettings = HTTPOutputConfig{} func usage() { fmt.Printf("Gor is a simple http traffic replication tool written in Go. Its main goal is to replay traffic from production servers to staging and dev environments.\nProject page: https://github.com/buger/gor\nAuthor: leonsbox@gmail.com\nCurrent Version: %s\n\n", VERSION) @@ -75,17 +66,17 @@ func init() { flag.Var(&Settings.inputHTTP, "input-http", "Read requests from HTTP, should be explicitly sent from your application:\n\t# Listen for http on 9000\n\tgor --input-http :9000 --output-http staging.com") flag.Var(&Settings.outputHTTP, "output-http", "Forwards incoming requests to given http address.\n\t# Redirect all incoming requests to staging.com address \n\tgor --input-raw :80 --output-http http://staging.com") - flag.Var(&HTTPOutputSettings.headers, "output-http-header", "Inject additional headers to http reqest:\n\tgor --input-raw :8080 --output-http staging.com --output-http-header 'User-Agent: Gor'") - flag.Var(&HTTPOutputSettings.methods, "output-http-method", "Whitelist of HTTP methods to replay. Anything else will be dropped:\n\tgor --input-raw :8080 --output-http staging.com --output-http-method GET --output-http-method OPTIONS") - flag.Var(&HTTPOutputSettings.urlRegexp, "output-http-url-regexp", "A regexp to match requests against. Anything else will be dropped:\n\t gor --input-raw :8080 --output-http staging.com --output-http-url-regexp ^www.") - flag.Var(&HTTPOutputSettings.headerFilters, "output-http-header-filter", "A regexp to match a specific header against. Requests with non-matching headers will be dropped:\n\t gor --input-raw :8080 --output-http staging.com --output-http-header-filter api-version:^v1") - flag.Var(&HTTPOutputSettings.headerHashFilters, "output-http-header-hash-filter", "Takes a fraction of requests, consistently taking or rejecting a request based on the FNV32-1A hash of a specific header. The fraction must have a denominator that is a power of two:\n\t gor --input-raw :8080 --output-http staging.com --output-http-header-hash-filter user-id:1/4") - flag.IntVar(&HTTPOutputSettings.workers, "output-http-workers", 0, "Gor uses dynamic worker scaling by default. Enter a number to run a set number of workers.") - flag.BoolVar(&HTTPOutputSettings.stats, "output-http-stats", false, "Report http output queue stats to console every 5 seconds.") + flag.IntVar(&Settings.outputHTTPConfig.workers, "output-http-workers", 0, "Gor uses dynamic worker scaling by default. Enter a number to run a set number of workers.") + flag.IntVar(&Settings.outputHTTPConfig.redirectLimit, "output-http-redirects", 0, "Enable how often redirects should be followed.") + flag.Var(&Settings.outputHTTPConfig.modifier.headers, "output-http-header", "Inject additional headers to http reqest:\n\tgor --input-raw :8080 --output-http staging.com --output-http-header 'User-Agent: Gor'") + flag.Var(&Settings.outputHTTPConfig.modifier.methods, "output-http-method", "Whitelist of HTTP methods to replay. Anything else will be dropped:\n\tgor --input-raw :8080 --output-http staging.com --output-http-method GET --output-http-method OPTIONS") + flag.Var(&Settings.outputHTTPConfig.modifier.urlRegexp, "output-http-url-regexp", "A regexp to match requests against. Anything else will be dropped:\n\t gor --input-raw :8080 --output-http staging.com --output-http-url-regexp ^www.") + flag.Var(&Settings.outputHTTPConfig.modifier.urlRewrite, "output-http-rewrite-url", "Rewrite the requst url based on a mapping:\n\tgor --input-raw :8080 --output-http staging.com --output-http-rewrite-url /xml_test/interface.php:/api/service.do") + flag.Var(&Settings.outputHTTPConfig.modifier.headerFilters, "output-http-header-filter", "A regexp to match a specific header against. Requests with non-matching headers will be dropped:\n\t gor --input-raw :8080 --output-http staging.com --output-http-header-filter api-version:^v1") + flag.Var(&Settings.outputHTTPConfig.modifier.headerHashFilters, "output-http-header-hash-filter", "Takes a fraction of requests, consistently taking or rejecting a request based on the FNV32-1A hash of a specific header. The fraction must have a denominator that is a power of two:\n\t gor --input-raw :8080 --output-http staging.com --output-http-header-hash-filter user-id:1/4") + flag.BoolVar(&Settings.outputHTTPConfig.stats, "output-http-stats", false, "Report http output queue stats to console every 5 seconds.") - flag.StringVar(&HTTPOutputSettings.elasticSearch, "output-http-elasticsearch", "", "Send request and response stats to ElasticSearch:\n\tgor --input-raw :8080 --output-http staging.com --output-http-elasticsearch 'es_host:api_port/index_name'") - flag.Var(&HTTPOutputSettings.urlRewrite, "output-http-rewrite-url", "Rewrite the requst url based on a mapping:\n\tgor --input-raw :8080 --output-http staging.com --output-http-rewrite-url /xml_test/interface.php:/api/service.do") - flag.IntVar(&HTTPOutputSettings.redirectLimit, "output-http-redirects", 0, "Enable how often redirects should be followed.") + flag.StringVar(&Settings.outputHTTPConfig.elasticSearch, "output-http-elasticsearch", "", "Send request and response stats to ElasticSearch:\n\tgor --input-raw :8080 --output-http staging.com --output-http-elasticsearch 'es_host:api_port/index_name'") } func Debug(args ...interface{}) { From 2b17989fd91addc200b2947d2d47c5dad5cadec8 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Sun, 5 Jul 2015 23:17:57 +0500 Subject: [PATCH 14/30] Move HTTP modifier from HTTP output plugin --- Makefile | 4 +- emitter.go | 16 ++++- http_modifier.go | 87 ++++++++++++++++++++++++++++ http_modifier_test.go | 38 ++++++++++++ output_http.go | 17 ------ output_http_test.go | 31 ++-------- proto/proto.go | 6 ++ settings.go | 13 +++-- settings_header_filters.go | 16 +---- settings_header_filters_test.go | 42 +++++--------- settings_header_hash_filters.go | 22 +------ settings_header_hash_filters_test.go | 21 +------ settings_methods.go | 9 +-- settings_methods_test.go | 6 +- settings_url_map.go | 15 +---- settings_url_map_test.go | 53 +---------------- 16 files changed, 196 insertions(+), 200 deletions(-) create mode 100644 http_modifier_test.go diff --git a/Makefile b/Makefile index 9ff6068..07d3f7c 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -SOURCE = emitter.go gor.go gor_stat.go input_dummy.go input_file.go input_raw.go input_tcp.go limiter.go output_dummy.go output_file.go input_http.go output_http.go output_tcp.go plugins.go settings.go settings_header_filters.go settings_header_hash_filters.go settings_headers.go settings_methods.go settings_option.go settings_url_regexp.go test_input.go elasticsearch.go settings_url_map.go +SOURCE = emitter.go gor.go gor_stat.go input_dummy.go input_file.go input_raw.go input_tcp.go limiter.go output_dummy.go output_file.go input_http.go output_http.go output_tcp.go plugins.go settings.go settings_header_filters.go settings_header_hash_filters.go settings_headers.go settings_methods.go settings_option.go settings_url_regexp.go test_input.go elasticsearch.go settings_url_map.go http_modifier.go SOURCE_PATH = /gopath/src/github.com/buger/gor/ @@ -27,7 +27,7 @@ dbench: # Used mainly for debugging, because docker container do not have access to parent machine ports drun: - docker run -v `pwd`:$(SOURCE_PATH) -t -i gor go run $(SOURCE) --input-dummy=0 --input-http=:9000 --output-http="http://localhost:9000" --verbose + docker run -v `pwd`:$(SOURCE_PATH) -t -i gor go run $(SOURCE) --input-dummy=0 --output-http="http://localhost:9000" --verbose dbash: docker run -v `pwd`:$(SOURCE_PATH) -t -i gor /bin/bash \ No newline at end of file diff --git a/emitter.go b/emitter.go index 867fc75..5e7e78e 100644 --- a/emitter.go +++ b/emitter.go @@ -23,13 +23,25 @@ func Start(stop chan int) { func CopyMulty(src io.Reader, writers ...io.Writer) (err error) { buf := make([]byte, 5*1024*1024) wIndex := 0 + modifier := NewHTTPModifier(&Settings.modifierConfig) for { nr, er := src.Read(buf) if nr > 0 && len(buf) > nr { + payload := buf[0:nr] + + if modifier != nil { + payload = modifier.Rewrite(payload) + + // If modifier tells to skip request + if len(payload) == 0 { + continue + } + } + if Settings.splitOutput { // Simple round robin - writers[wIndex].Write(buf[0:nr]) + writers[wIndex].Write(payload) wIndex++ @@ -38,7 +50,7 @@ func CopyMulty(src io.Reader, writers ...io.Writer) (err error) { } } else { for _, dst := range writers { - dst.Write(buf[0:nr]) + dst.Write(payload) } } diff --git a/http_modifier.go b/http_modifier.go index 73b5203..95fb819 100644 --- a/http_modifier.go +++ b/http_modifier.go @@ -1,5 +1,11 @@ package main +import ( + "github.com/buger/gor/proto" + "hash/fnv" +) + + type HTTPModifierConfig struct { urlRegexp HTTPUrlRegexp urlRewrite UrlRewriteMap @@ -8,4 +14,85 @@ type HTTPModifierConfig struct { headers HTTPHeaders methods HTTPMethods +} + +type HTTPModifier struct { + config *HTTPModifierConfig +} + +func NewHTTPModifier(config *HTTPModifierConfig) *HTTPModifier { + // Optimization to skip modifier completely if we do not need it + if config.urlRegexp.regexp == nil && + len(config.urlRewrite) == 0 && + len(config.headerFilters) == 0 && + len(config.headerHashFilters) == 0 && + len(config.headers) == 0 && + len(config.methods) == 0 { + return nil + } + + return &HTTPModifier{config: config} +} + +func (m *HTTPModifier) Rewrite(payload []byte) (response []byte) { + if len(m.config.methods) > 0 && !m.config.methods.Contains(proto.Method(payload)) { + return + } + + if m.config.urlRegexp.regexp != nil { + host, _, _, _ := proto.Header(payload, []byte("Host")) + fullPath := append(host, proto.Path(payload)...) + + if !m.config.urlRegexp.regexp.Match(fullPath) { + return + } + } + + if len(m.config.headerFilters) > 0 { + for _, f := range m.config.headerFilters { + value, s, _, _ := proto.Header(payload, f.name) + + if s != -1 && !f.regexp.Match(value) { + return + } + } + } + + if len(m.config.headerHashFilters) > 0 { + for _, f := range m.config.headerHashFilters { + value, s, _, _ := proto.Header(payload, f.name) + + if s == -1 { + return + } + + hasher := fnv.New32a() + hasher.Write(value) + if hasher.Sum32() > f.maxHash { + return + } + } + } + + if len(m.config.urlRewrite) > 0 { + path := proto.Path(payload) + + for _, f := range m.config.urlRewrite { + if f.src.Match(path) { + path = f.src.ReplaceAll(path, f.target) + payload = proto.SetPath(payload, path) + + break + } + } + } + + if len(m.config.headers) > 0 { + for _, header := range m.config.headers { + payload = proto.SetHeader(payload, []byte(header.Name), []byte(header.Value)) + } + } + + + return payload } \ No newline at end of file diff --git a/http_modifier_test.go b/http_modifier_test.go new file mode 100644 index 0000000..6ca055d --- /dev/null +++ b/http_modifier_test.go @@ -0,0 +1,38 @@ +package main + +import ( + "testing" +) + +func TestHTTPModifierWithoutConfig(t *testing.T) { + if NewHTTPModifier(&HTTPModifierConfig{}) != nil { + t.Error("If no config specified should not be initialized") + } +} + +func TestHTTPModifierHeaderFilters(t *testing.T) { + filters := HTTPHeaderFilters{} + filters.Set("Host:^www.w3.org$") + + modifier := NewHTTPModifier(&HTTPModifierConfig{ + headerFilters: filters, + }) + + payload := []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") + + if len(modifier.Rewrite(payload)) == 0 { + t.Error("Request should pass filters") + } + + filters = HTTPHeaderFilters{} + // Setting filter that not match our header + filters.Set("Host:^www.w4.org$") + + modifier = NewHTTPModifier(&HTTPModifierConfig{ + headerFilters: filters, + }) + + if len(modifier.Rewrite(payload)) != 0 { + t.Error("Request should not pass filters") + } +} diff --git a/output_http.go b/output_http.go index 14b630e..c1b42f9 100644 --- a/output_http.go +++ b/output_http.go @@ -62,8 +62,6 @@ type HTTPOutputConfig struct { stats bool workers int - modifier HTTPModifierConfig - elasticSearch string } @@ -204,27 +202,12 @@ func (o *HTTPOutput) sendRequest(client *http.Client, data []byte) { return } - if len(o.config.modifier.methods) > 0 && !o.config.modifier.methods.Contains(request.Method) { - return - } - - if !(o.config.modifier.urlRegexp.Good(request) && o.config.modifier.headerFilters.Good(request) && o.config.modifier.headerHashFilters.Good(request)) { - return - } - - // Rewrite the path as necessary - request.URL.Path = o.config.modifier.urlRewrite.Rewrite(request.URL.Path) - // Change HOST of original request URL := o.address + request.URL.Path + "?" + request.URL.RawQuery request.RequestURI = "" request.URL, _ = url.ParseRequestURI(URL) - for _, header := range o.config.modifier.headers { - SetHeader(request, header.Name, header.Value) - } - start := time.Now() resp, err := client.Do(request) stop := time.Now() diff --git a/output_http_test.go b/output_http_test.go index 589cab4..7cf8c24 100644 --- a/output_http_test.go +++ b/output_http_test.go @@ -24,27 +24,6 @@ func startHTTP(cb func(*http.Request)) net.Listener { return listener } -func TestSetHeader(t *testing.T) { - - req := &http.Request{ - Header: make(map[string][]string), - } - req.Host = "test.com" - - SetHeader(req, "Host", "test2.com") - - if req.Host != "test2.com" { - t.Error("Expected test2.com - got ", req.Host) - } - - SetHeader(req, "test_header", "test_value") - - if req.Header.Get("test_header") != "test_value" { - t.Error("Wrong header value found") - } - -} - func TestHTTPOutput(t *testing.T) { wg := new(sync.WaitGroup) quit := make(chan int) @@ -73,10 +52,10 @@ func TestHTTPOutput(t *testing.T) { }) headers := HTTPHeaders{HTTPHeader{"User-Agent", "Gor"}} - methods := HTTPMethods{"GET", "PUT", "POST"} - modifierConfig := HTTPModifierConfig{headers: headers, methods: methods} + methods := HTTPMethods{[]byte("GET"), []byte("PUT"), []byte("POST")} + Settings.modifierConfig = HTTPModifierConfig{headers: headers, methods: methods} - output := NewHTTPOutput(listener.Addr().String(), &HTTPOutputConfig{modifier: modifierConfig}) + output := NewHTTPOutput(listener.Addr().String(), &HTTPOutputConfig{}) Plugins.Inputs = []io.Reader{input} Plugins.Outputs = []io.Writer{output} @@ -84,7 +63,7 @@ func TestHTTPOutput(t *testing.T) { go Start(quit) for i := 0; i < 100; i++ { - wg.Add(2) + wg.Add(2) // OPTIONS should be ignored input.EmitPOST() input.EmitOPTIONS() input.EmitGET() @@ -93,6 +72,8 @@ func TestHTTPOutput(t *testing.T) { wg.Wait() close(quit) + + Settings.modifierConfig = HTTPModifierConfig{} } func TestOutputHTTPSSL(t *testing.T) { diff --git a/proto/proto.go b/proto/proto.go index 8a68949..f38645a 100644 --- a/proto/proto.go +++ b/proto/proto.go @@ -79,3 +79,9 @@ func SetPath(payload, path []byte) []byte { return byteutils.Replace(payload, start, start+end, path) } + +func Method(payload []byte) []byte { + end := bytes.IndexByte(payload, ' ') + + return payload[:end] +} \ No newline at end of file diff --git a/settings.go b/settings.go index 669d145..411ca62 100644 --- a/settings.go +++ b/settings.go @@ -33,6 +33,7 @@ type AppSettings struct { outputHTTP MultiOption outputHTTPConfig HTTPOutputConfig + modifierConfig HTTPModifierConfig } var Settings AppSettings = AppSettings{} @@ -68,12 +69,12 @@ func init() { flag.Var(&Settings.outputHTTP, "output-http", "Forwards incoming requests to given http address.\n\t# Redirect all incoming requests to staging.com address \n\tgor --input-raw :80 --output-http http://staging.com") flag.IntVar(&Settings.outputHTTPConfig.workers, "output-http-workers", 0, "Gor uses dynamic worker scaling by default. Enter a number to run a set number of workers.") flag.IntVar(&Settings.outputHTTPConfig.redirectLimit, "output-http-redirects", 0, "Enable how often redirects should be followed.") - flag.Var(&Settings.outputHTTPConfig.modifier.headers, "output-http-header", "Inject additional headers to http reqest:\n\tgor --input-raw :8080 --output-http staging.com --output-http-header 'User-Agent: Gor'") - flag.Var(&Settings.outputHTTPConfig.modifier.methods, "output-http-method", "Whitelist of HTTP methods to replay. Anything else will be dropped:\n\tgor --input-raw :8080 --output-http staging.com --output-http-method GET --output-http-method OPTIONS") - flag.Var(&Settings.outputHTTPConfig.modifier.urlRegexp, "output-http-url-regexp", "A regexp to match requests against. Anything else will be dropped:\n\t gor --input-raw :8080 --output-http staging.com --output-http-url-regexp ^www.") - flag.Var(&Settings.outputHTTPConfig.modifier.urlRewrite, "output-http-rewrite-url", "Rewrite the requst url based on a mapping:\n\tgor --input-raw :8080 --output-http staging.com --output-http-rewrite-url /xml_test/interface.php:/api/service.do") - flag.Var(&Settings.outputHTTPConfig.modifier.headerFilters, "output-http-header-filter", "A regexp to match a specific header against. Requests with non-matching headers will be dropped:\n\t gor --input-raw :8080 --output-http staging.com --output-http-header-filter api-version:^v1") - flag.Var(&Settings.outputHTTPConfig.modifier.headerHashFilters, "output-http-header-hash-filter", "Takes a fraction of requests, consistently taking or rejecting a request based on the FNV32-1A hash of a specific header. The fraction must have a denominator that is a power of two:\n\t gor --input-raw :8080 --output-http staging.com --output-http-header-hash-filter user-id:1/4") + flag.Var(&Settings.modifierConfig.headers, "output-http-header", "Inject additional headers to http reqest:\n\tgor --input-raw :8080 --output-http staging.com --output-http-header 'User-Agent: Gor'") + flag.Var(&Settings.modifierConfig.methods, "output-http-method", "Whitelist of HTTP methods to replay. Anything else will be dropped:\n\tgor --input-raw :8080 --output-http staging.com --output-http-method GET --output-http-method OPTIONS") + flag.Var(&Settings.modifierConfig.urlRegexp, "output-http-url-regexp", "A regexp to match requests against. Anything else will be dropped:\n\t gor --input-raw :8080 --output-http staging.com --output-http-url-regexp ^www.") + flag.Var(&Settings.modifierConfig.urlRewrite, "output-http-rewrite-url", "Rewrite the requst url based on a mapping:\n\tgor --input-raw :8080 --output-http staging.com --output-http-rewrite-url /xml_test/interface.php:/api/service.do") + flag.Var(&Settings.modifierConfig.headerFilters, "output-http-header-filter", "A regexp to match a specific header against. Requests with non-matching headers will be dropped:\n\t gor --input-raw :8080 --output-http staging.com --output-http-header-filter api-version:^v1") + flag.Var(&Settings.modifierConfig.headerHashFilters, "output-http-header-hash-filter", "Takes a fraction of requests, consistently taking or rejecting a request based on the FNV32-1A hash of a specific header. The fraction must have a denominator that is a power of two:\n\t gor --input-raw :8080 --output-http staging.com --output-http-header-hash-filter user-id:1/4") flag.BoolVar(&Settings.outputHTTPConfig.stats, "output-http-stats", false, "Report http output queue stats to console every 5 seconds.") flag.StringVar(&Settings.outputHTTPConfig.elasticSearch, "output-http-elasticsearch", "", "Send request and response stats to ElasticSearch:\n\tgor --input-raw :8080 --output-http staging.com --output-http-elasticsearch 'es_host:api_port/index_name'") diff --git a/settings_header_filters.go b/settings_header_filters.go index ea2b034..7a017a6 100644 --- a/settings_header_filters.go +++ b/settings_header_filters.go @@ -3,13 +3,12 @@ package main import ( "errors" "fmt" - "net/http" "regexp" "strings" ) type headerFilter struct { - name string + name []byte regexp *regexp.Regexp } @@ -29,16 +28,7 @@ func (h *HTTPHeaderFilters) Set(value string) error { return err } - *h = append(*h, headerFilter{name: valArr[0], regexp: r}) + *h = append(*h, headerFilter{name: []byte(valArr[0]), regexp: r}) return nil -} - -func (h *HTTPHeaderFilters) Good(req *http.Request) bool { - for _, f := range *h { - if !f.regexp.MatchString(req.Header.Get(f.name)) { - return false - } - } - return true -} +} \ No newline at end of file diff --git a/settings_header_filters_test.go b/settings_header_filters_test.go index 2d0af19..499f910 100644 --- a/settings_header_filters_test.go +++ b/settings_header_filters_test.go @@ -1,35 +1,25 @@ package main import ( - "net/http" - "testing" + "testing" ) func TestHTTPHeaderFilters(t *testing.T) { - filters := HTTPHeaderFilters{} + filters := HTTPHeaderFilters{} - err := filters.Set("Header1:^$") - if err != nil { - t.Error("Should not error on Header1:^$") - } + err := filters.Set("Header1:^$") + if err != nil { + t.Error("Should not error on Header1:^$") + } - err = filters.Set("Header2:^:$") - if err != nil { - t.Error("Should not error on Header2:^:$") - } + err = filters.Set("Header2:^:$") + if err != nil { + t.Error("Should not error on Header2:^:$") + } - err = filters.Set("Header3-^$") - if err == nil { - t.Error("Should error on Header2:^:$") - } - - req := http.Request{} - req.Header = make(map[string][]string) - req.Header.Add("Header1", "") - req.Header.Add("Header2", ":") - req.Header.Add("Header3", "Irrelevant") - - if !filters.Good(&req) { - t.Error("Request should pass filters") - } -} + // Missing colon + err = filters.Set("Header3-^$") + if err == nil { + t.Error("Should error on Header2:^:$") + } +} \ No newline at end of file diff --git a/settings_header_hash_filters.go b/settings_header_hash_filters.go index f76aa9e..0dc9801 100644 --- a/settings_header_hash_filters.go +++ b/settings_header_hash_filters.go @@ -3,14 +3,12 @@ package main import ( "errors" "fmt" - "hash/fnv" - "net/http" "strconv" "strings" ) type headerHashFilter struct { - name string + name []byte maxHash uint32 } @@ -44,23 +42,9 @@ func (h *HTTPHeaderHashFilters) Set(value string) error { } var f headerHashFilter - f.name = valArr[0] + f.name = []byte(valArr[0]) f.maxHash = (uint32)(num * (((uint64)(2 << 31)) / den)) *h = append(*h, f) return nil -} - -func (h *HTTPHeaderHashFilters) Good(req *http.Request) bool { - for _, f := range *h { - if req.Header.Get(f.name) == "" { - return false - } - hasher := fnv.New32a() - hasher.Write([]byte(req.Header.Get(f.name))) - if hasher.Sum32() > f.maxHash { - return false - } - } - return true -} +} \ No newline at end of file diff --git a/settings_header_hash_filters_test.go b/settings_header_hash_filters_test.go index f725761..171b69b 100644 --- a/settings_header_hash_filters_test.go +++ b/settings_header_hash_filters_test.go @@ -1,7 +1,6 @@ package main import ( - "net/http" "testing" ) @@ -18,31 +17,15 @@ func TestHTTPHeaderHashFilters(t *testing.T) { t.Error("Should not error on Header2:^:$") } + // Denominator must be power of 2 err = filters.Set("HeaderIrrelevant:1/3") if err == nil { t.Error("Should error on HeaderIrrelevant:1/3") } + // Denominator must be power of 2 err = filters.Set("Pow2Denom:1/31") if err == nil { t.Error("Should error on Pow2Denom:1/31") } - - req := http.Request{} - req.Header = make(map[string][]string) - req.Header.Add("Header1", "test3414") - - if filters.Good(&req) { - t.Error("Request should not pass filters, Header2 does not exist") - } - - req.Header.Add("Header2", "test2") - if filters.Good(&req) { - t.Error("Request should not pass filters, Header2 hash too high") - } - - req.Header.Set("Header2", "test3414") - if !filters.Good(&req) { - t.Error("Request should pass filters") - } } diff --git a/settings_methods.go b/settings_methods.go index e83109f..cd5a977 100644 --- a/settings_methods.go +++ b/settings_methods.go @@ -3,22 +3,23 @@ package main import ( "fmt" "strings" + "bytes" ) -type HTTPMethods []string +type HTTPMethods [][]byte func (h *HTTPMethods) String() string { return fmt.Sprint(*h) } func (h *HTTPMethods) Set(value string) error { - *h = append(*h, strings.ToUpper(value)) + *h = append(*h, []byte(strings.ToUpper(value))) return nil } -func (h *HTTPMethods) Contains(value string) bool { +func (h *HTTPMethods) Contains(value []byte) bool { for _, method := range *h { - if value == method { + if bytes.Equal(value, method) { return true } } diff --git a/settings_methods_test.go b/settings_methods_test.go index 774e952..4c56ed2 100644 --- a/settings_methods_test.go +++ b/settings_methods_test.go @@ -10,15 +10,15 @@ func TestHTTPMethods(t *testing.T) { methods.Set("lower") methods.Set("UPPER") - if !methods.Contains("LOWER") { + if !methods.Contains([]byte("LOWER")) { t.Error("Does not contain LOWER") } - if !methods.Contains("UPPER") { + if !methods.Contains([]byte("UPPER")) { t.Error("Does not contain UPPER") } - if methods.Contains("ABSENT") { + if methods.Contains([]byte("ABSENT")) { t.Error("Does contain ABSENT") } } diff --git a/settings_url_map.go b/settings_url_map.go index 5453610..d07a313 100644 --- a/settings_url_map.go +++ b/settings_url_map.go @@ -9,7 +9,7 @@ import ( type urlRewrite struct { src *regexp.Regexp - target string + target []byte } type UrlRewriteMap []urlRewrite @@ -27,15 +27,6 @@ func (r *UrlRewriteMap) Set(value string) error { if err != nil { return err } - *r = append(*r, urlRewrite{src: regexp, target: valArr[1]}) + *r = append(*r, urlRewrite{src: regexp, target: []byte(valArr[1]) }) return nil -} - -func (r *UrlRewriteMap) Rewrite(path string) string { - for _, f := range *r { - if f.src.MatchString(path) { - path = f.src.ReplaceAllString(path, f.target) - } - } - return path -} +} \ No newline at end of file diff --git a/settings_url_map_test.go b/settings_url_map_test.go index ed5ed21..85f0393 100644 --- a/settings_url_map_test.go +++ b/settings_url_map_test.go @@ -1,52 +1 @@ -package main - -import ( - "testing" -) - -func TestUrlRewriteMap_1(t *testing.T) { - var url string - - rewrites := UrlRewriteMap{} - - err := rewrites.Set("/abc:/123") - if err != nil { - t.Error("Should not error on /abc:/123") - } - - url = "/abc" - if rewrites.Rewrite(url) == url { - t.Error("Request url should have been rewritten, wasn't") - } - - url = "/wibble" - if rewrites.Rewrite(url) != url { - t.Error("Request url should not have been rewritten, was") - } -} - -func TestUrlRewriteMap_2(t *testing.T) { - var url string - - rewrites := UrlRewriteMap{} - - err := rewrites.Set("/v1/user/([^\\/]+)/ping:/v2/user/$1/ping") - if err != nil { - t.Error("Should not error on /v1/user/([^\\/]+)/ping:/v2/user/$1/ping") - } - - url = "/v1/user/joe/ping" - if rewrites.Rewrite(url) == url { - t.Error("Request url should have been rewritten, wasn't") - } - - url = "/v1/user/joe/ping" - if rewrites.Rewrite(url) != "/v2/user/joe/ping" { - t.Error("Request url should have been rewritten, wasn't") - } - - url = "/v1/user/ping" - if rewrites.Rewrite(url) != url { - t.Error("Request url should not have been rewritten, was") - } -} +package main \ No newline at end of file From a859ae02ad26c8a319ae1ab2900e846a97154ce9 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Mon, 6 Jul 2015 18:36:12 +0500 Subject: [PATCH 15/30] More tests and simplify hashFilter --- http_modifier.go | 3 +- http_modifier_test.go | 57 +++++++++++++++++++++++++++++++++ settings_header_hash_filters.go | 12 ++----- settings_url_map_test.go | 19 ++++++++++- 4 files changed, 79 insertions(+), 12 deletions(-) diff --git a/http_modifier.go b/http_modifier.go index 95fb819..2f1dc3d 100644 --- a/http_modifier.go +++ b/http_modifier.go @@ -68,7 +68,8 @@ func (m *HTTPModifier) Rewrite(payload []byte) (response []byte) { hasher := fnv.New32a() hasher.Write(value) - if hasher.Sum32() > f.maxHash { + + if (hasher.Sum32() % 100) >= f.percent { return } } diff --git a/http_modifier_test.go b/http_modifier_test.go index 6ca055d..6803587 100644 --- a/http_modifier_test.go +++ b/http_modifier_test.go @@ -2,6 +2,8 @@ package main import ( "testing" + "github.com/buger/gor/proto" + "bytes" ) func TestHTTPModifierWithoutConfig(t *testing.T) { @@ -36,3 +38,58 @@ func TestHTTPModifierHeaderFilters(t *testing.T) { t.Error("Request should not pass filters") } } + + +func TestHTTPModifierURLRegexp(t *testing.T) { + var url, new_url []byte + + rewrites := UrlRewriteMap{} + + payload := func(url []byte) []byte { + return []byte("POST " + string(url) + " HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") + } + + err := rewrites.Set("/v1/user/([^\\/]+)/ping:/v2/user/$1/ping") + if err != nil { + t.Error("Should not error on /v1/user/([^\\/]+)/ping:/v2/user/$1/ping") + } + + modifier := NewHTTPModifier(&HTTPModifierConfig{ + urlRewrite: rewrites, + }) + + url = []byte("/v1/user/joe/ping") + if new_url = proto.Path(modifier.Rewrite(payload(url))); bytes.Equal(new_url, url) { + t.Error("Request url should have been rewritten, wasn't", string(new_url)) + } + + url = []byte("/v1/user/ping") + if new_url = proto.Path(modifier.Rewrite(payload(url))); !bytes.Equal(new_url, url) { + t.Error("Request url should have been rewritten, wasn't", string(new_url)) + } +} + +func TestHTTPModifierHeaderHashFilters(t *testing.T) { + filters := HTTPHeaderHashFilters{} + filters.Set("Header2:1/2") + + modifier := NewHTTPModifier(&HTTPModifierConfig{ + headerHashFilters: filters, + }) + + payload := func(header []byte) []byte { + return []byte("POST / HTTP/1.1\r\n" + string(header) + "Content-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") + } + + if p := modifier.Rewrite(payload([]byte(""))); len(p) > 0 { + t.Error("Request should not pass filters, Header2 does not exist") + } + + if p := modifier.Rewrite(payload([]byte("Header2: 3\r\n"))); len(p) > 0 { + t.Error("Request should not pass filters, Header2 hash too high") + } + + if p := modifier.Rewrite(payload([]byte("Header2: 1\r\n"))); len(p) == 0 { + t.Error("Request should pass filters") + } +} diff --git a/settings_header_hash_filters.go b/settings_header_hash_filters.go index 0dc9801..5c848fe 100644 --- a/settings_header_hash_filters.go +++ b/settings_header_hash_filters.go @@ -9,7 +9,7 @@ import ( type headerHashFilter struct { name []byte - maxHash uint32 + percent uint32 } type HTTPHeaderHashFilters []headerHashFilter @@ -33,17 +33,9 @@ func (h *HTTPHeaderHashFilters) Set(value string) error { num, _ = strconv.ParseUint(fracArr[0], 10, 64) den, _ = strconv.ParseUint(fracArr[1], 10, 64) - if num < 1 || den < 1 || num > den { - panic("need positive numerators and denominators, with the former less than the latter.") - } - - if den&(den-1) != 0 { - return errors.New("must have a denominator which is a power of two.") - } - var f headerHashFilter f.name = []byte(valArr[0]) - f.maxHash = (uint32)(num * (((uint64)(2 << 31)) / den)) + f.percent = uint32((float64(num) / float64(den)) * 100) *h = append(*h, f) return nil diff --git a/settings_url_map_test.go b/settings_url_map_test.go index 85f0393..de7ad7d 100644 --- a/settings_url_map_test.go +++ b/settings_url_map_test.go @@ -1 +1,18 @@ -package main \ No newline at end of file +package main + +import ( + "testing" +) + +func TestUrlRewriteMap(t *testing.T) { + var err error + rewrites := UrlRewriteMap{} + + if err = rewrites.Set("/v1/user/([^\\/]+)/ping:/v2/user/$1/ping"); err != nil { + t.Error("Should set mapping", err) + } + + if err = rewrites.Set("/v1/user/([^\\/]+)/ping"); err == nil { + t.Error("Should not set mapping without :") + } +} \ No newline at end of file From 53c27cf37420b8e001e892df9113c6da529a88b6 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Mon, 6 Jul 2015 18:53:54 +0500 Subject: [PATCH 16/30] Add Header Modifier tests --- http_modifier_test.go | 19 ++++++++++++++++++- settings_url_regexp.go | 10 +--------- settings_url_regexp_test.go | 16 ---------------- 3 files changed, 19 insertions(+), 26 deletions(-) diff --git a/http_modifier_test.go b/http_modifier_test.go index 6803587..ad06719 100644 --- a/http_modifier_test.go +++ b/http_modifier_test.go @@ -40,7 +40,7 @@ func TestHTTPModifierHeaderFilters(t *testing.T) { } -func TestHTTPModifierURLRegexp(t *testing.T) { +func TestHTTPModifierURLRewrite(t *testing.T) { var url, new_url []byte rewrites := UrlRewriteMap{} @@ -93,3 +93,20 @@ func TestHTTPModifierHeaderHashFilters(t *testing.T) { t.Error("Request should pass filters") } } + +func TestHTTPModifierHeaders(t *testing.T) { + headers := HTTPHeaders{} + headers.Set("Header1:1") + headers.Set("Host:localhost") + + modifier := NewHTTPModifier(&HTTPModifierConfig{ + headers: headers, + }) + + payload := []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") + new_payload := []byte("POST /post HTTP/1.1\r\nHeader1: 1\r\nContent-Length: 7\r\nHost: localhost\r\n\r\na=1&b=2") + + if payload = modifier.Rewrite(payload); !bytes.Equal(payload, new_payload) { + t.Error("Should update request headers", string(payload)) + } +} diff --git a/settings_url_regexp.go b/settings_url_regexp.go index 27900bf..bea2003 100644 --- a/settings_url_regexp.go +++ b/settings_url_regexp.go @@ -1,7 +1,6 @@ package main import ( - "net/http" "regexp" ) @@ -20,11 +19,4 @@ func (r *HTTPUrlRegexp) Set(value string) error { regexp, err := regexp.Compile(value) r.regexp = regexp return err -} - -func (r *HTTPUrlRegexp) Good(req *http.Request) bool { - if r.regexp == nil { - return true - } - return r.regexp.MatchString(req.Host + req.URL.String()) -} +} \ No newline at end of file diff --git a/settings_url_regexp_test.go b/settings_url_regexp_test.go index 483c401..cb4b825 100644 --- a/settings_url_regexp_test.go +++ b/settings_url_regexp_test.go @@ -1,26 +1,10 @@ package main import ( - "net/http" - "net/url" "testing" ) func TestHTTPUrlRegexp(t *testing.T) { filter := HTTPUrlRegexp{} - filter.Set("^www.google.com/admin/") - - req := http.Request{} - req.Host = "www.google.com" - var err error - req.URL, err = url.Parse("/admin/testpage1") - if !filter.Good(&req) || err != nil { - t.Error("Request should pass filters") - } - - req.URL, err = url.Parse("/user/testpage2") - if filter.Good(&req) || err != nil { - t.Error("Request should not pass filters") - } } From bc48e1ad49b768e8b50944e2fb73cbb7c5409d66 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Mon, 6 Jul 2015 18:56:50 +0500 Subject: [PATCH 17/30] Fix tests --- settings_header_hash_filters_test.go | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/settings_header_hash_filters_test.go b/settings_header_hash_filters_test.go index 171b69b..29c34ff 100644 --- a/settings_header_hash_filters_test.go +++ b/settings_header_hash_filters_test.go @@ -12,20 +12,8 @@ func TestHTTPHeaderHashFilters(t *testing.T) { t.Error("Should not error on Header1:^$") } - err = filters.Set("Header2:1/2") - if err != nil { - t.Error("Should not error on Header2:^:$") - } - - // Denominator must be power of 2 - err = filters.Set("HeaderIrrelevant:1/3") + err = filters.Set("Header2:1") if err == nil { - t.Error("Should error on HeaderIrrelevant:1/3") - } - - // Denominator must be power of 2 - err = filters.Set("Pow2Denom:1/31") - if err == nil { - t.Error("Should error on Pow2Denom:1/31") + t.Error("Should error on Header2:^:$") } } From 9a0d9166c310717d7b86e6c58800e63c197b75fb Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Mon, 6 Jul 2015 21:08:34 +0500 Subject: [PATCH 18/30] Yay! http.Client replaced by own simple http client --- Makefile | 2 +- elasticsearch.go | 86 +++++++++++------------ http_client.go | 2 + http_client_test.go | 52 ++++++++------ input_raw_test.go | 14 +--- output_http.go | 108 +++-------------------------- proto/proto.go | 11 +++ raw_socket_listener/tcp_message.go | 30 +------- 8 files changed, 102 insertions(+), 203 deletions(-) diff --git a/Makefile b/Makefile index 07d3f7c..c75b5f2 100644 --- a/Makefile +++ b/Makefile @@ -14,7 +14,7 @@ dbuild: docker build -t gor . dtest: - docker run -v `pwd`:$(SOURCE_PATH) -t -i --env GORACE="halt_on_error=1" gor go test ./... $(ARGS) -race -v + docker run -v `pwd`:$(SOURCE_PATH) -t -i --env GORACE="halt_on_error=1" gor go test ./... $(ARGS) -race -v -timeout 15s dfmt: docker run -v `pwd`:$(SOURCE_PATH) -t -i gor go fmt diff --git a/elasticsearch.go b/elasticsearch.go index 2ef8ea4..1819c9d 100644 --- a/elasticsearch.go +++ b/elasticsearch.go @@ -4,8 +4,8 @@ import ( "encoding/json" "github.com/buger/elastigo/api" "github.com/buger/elastigo/core" + "github.com/buger/gor/proto" "log" - "net/http" "regexp" "time" ) @@ -26,26 +26,26 @@ type ESPlugin struct { } type ESRequestResponse struct { - ReqUrl string `json:"Req_URL"` - ReqMethod string `json:"Req_Method"` - ReqUserAgent string `json:"Req_User-Agent"` - ReqAcceptLanguage string `json:"Req_Accept-Language,omitempty"` - ReqAccept string `json:"Req_Accept,omitempty"` - ReqAcceptEncoding string `json:"Req_Accept-Encoding,omitempty"` - ReqIfModifiedSince string `json:"Req_If-Modified-Since,omitempty"` - ReqConnection string `json:"Req_Connection,omitempty"` - ReqCookies []*http.Cookie `json:"Req_Cookies,omitempty"` - RespStatus string `json:"Resp_Status"` - RespStatusCode int `json:"Resp_Status-Code"` - RespProto string `json:"Resp_Proto,omitempty"` - RespContentLength int64 `json:"Resp_Content-Length,omitempty"` - RespContentType string `json:"Resp_Content-Type,omitempty"` - RespTransferEncoding []string `json:"Resp_Transfer-Encoding,omitempty"` - RespContentEncoding string `json:"Resp_Content-Encoding,omitempty"` - RespExpires string `json:"Resp_Expires,omitempty"` - RespCacheControl string `json:"Resp_Cache-Control,omitempty"` - RespVary string `json:"Resp_Vary,omitempty"` - RespSetCookie string `json:"Resp_Set-Cookie,omitempty"` + ReqUrl []byte `json:"Req_URL"` + ReqMethod []byte `json:"Req_Method"` + ReqUserAgent []byte `json:"Req_User-Agent"` + ReqAcceptLanguage []byte `json:"Req_Accept-Language,omitempty"` + ReqAccept []byte `json:"Req_Accept,omitempty"` + ReqAcceptEncoding []byte `json:"Req_Accept-Encoding,omitempty"` + ReqIfModifiedSince []byte `json:"Req_If-Modified-Since,omitempty"` + ReqConnection []byte `json:"Req_Connection,omitempty"` + ReqCookies []byte `json:"Req_Cookies,omitempty"` + RespStatus []byte `json:"Resp_Status"` + RespStatusCode []byte `json:"Resp_Status-Code"` + RespProto []byte `json:"Resp_Proto,omitempty"` + RespContentLength []byte `json:"Resp_Content-Length,omitempty"` + RespContentType []byte `json:"Resp_Content-Type,omitempty"` + RespTransferEncoding []byte `json:"Resp_Transfer-Encoding,omitempty"` + RespContentEncoding []byte `json:"Resp_Content-Encoding,omitempty"` + RespExpires []byte `json:"Resp_Expires,omitempty"` + RespCacheControl []byte `json:"Resp_Cache-Control,omitempty"` + RespVary []byte `json:"Resp_Vary,omitempty"` + RespSetCookie []byte `json:"Resp_Set-Cookie,omitempty"` Rtt int64 `json:"RTT"` Timestamp time.Time } @@ -111,8 +111,8 @@ func (p *ESPlugin) RttDurationToMs(d time.Duration) int64 { return int64(fl) } -func (p *ESPlugin) ResponseAnalyze(req *http.Request, resp *http.Response, start, stop time.Time) { - if resp == nil { +func (p *ESPlugin) ResponseAnalyze(req, resp []byte, start, stop time.Time) { + if len(resp) == 0 { // nil http response - skipped elasticsearch export for this request return } @@ -120,26 +120,26 @@ func (p *ESPlugin) ResponseAnalyze(req *http.Request, resp *http.Response, start rtt := p.RttDurationToMs(stop.Sub(start)) esResp := ESRequestResponse{ - ReqUrl: req.URL.String(), - ReqMethod: req.Method, - ReqUserAgent: req.UserAgent(), - ReqAcceptLanguage: req.Header.Get("Accept-Language"), - ReqAccept: req.Header.Get("Accept"), - ReqAcceptEncoding: req.Header.Get("Accept-Encoding"), - ReqIfModifiedSince: req.Header.Get("If-Modified-Since"), - ReqConnection: req.Header.Get("Connection"), - ReqCookies: req.Cookies(), - RespStatus: resp.Status, - RespStatusCode: resp.StatusCode, - RespProto: resp.Proto, - RespContentLength: resp.ContentLength, - RespContentType: resp.Header.Get("Content-Type"), - RespTransferEncoding: resp.TransferEncoding, - RespContentEncoding: resp.Header.Get("Content-Encoding"), - RespExpires: resp.Header.Get("Expires"), - RespCacheControl: resp.Header.Get("Cache-Control"), - RespVary: resp.Header.Get("Vary"), - RespSetCookie: resp.Header.Get("Set-Cookie"), + ReqUrl: proto.Path(req), + ReqMethod: proto.Method(req), + ReqUserAgent: proto.GetHeader(req, "User-Agent"), + ReqAcceptLanguage: proto.GetHeader(req, "Accept-Language"), + ReqAccept: proto.GetHeader(req, "Accept"), + ReqAcceptEncoding: proto.GetHeader(req, "Accept-Encoding"), + ReqIfModifiedSince: proto.GetHeader(req, "If-Modified-Since"), + ReqConnection: proto.GetHeader(req, "Connection"), + ReqCookies: proto.GetHeader(req, "Cookie"), + RespStatus: proto.Status(resp), + RespStatusCode: proto.Status(resp), + RespProto: proto.Method(resp), + RespContentLength: proto.GetHeader(resp, "Content-Length"), + RespContentType: proto.GetHeader(resp, "Content-Type"), + RespTransferEncoding: proto.GetHeader(resp, "Transfer-Encoding"), + RespContentEncoding: proto.GetHeader(resp, "Content-Encoding"), + RespExpires: proto.GetHeader(resp, "Expires"), + RespCacheControl: proto.GetHeader(resp, "Cache-Control"), + RespVary: proto.GetHeader(resp, "Vary"), + RespSetCookie: proto.GetHeader(resp, "Set-Cookie"), Rtt: rtt, Timestamp: t, } diff --git a/http_client.go b/http_client.go index 4f78e9d..e5f859d 100644 --- a/http_client.go +++ b/http_client.go @@ -93,6 +93,8 @@ func (c *HTTPClient) Send(data []byte) (response []byte, err error) { c.conn.SetWriteDeadline(timeout) + data = proto.SetHeader(data, []byte("Host"), []byte(c.baseURL.Host)) + if c.config.Debug { Debug("Sending:", string(data)) } diff --git a/http_client_test.go b/http_client_test.go index 25f8b4c..45a3ba9 100644 --- a/http_client_test.go +++ b/http_client_test.go @@ -36,13 +36,18 @@ func TestHTTPClientURLPort(t *testing.T) { func TestHTTPClientSend(t *testing.T) { wg := new(sync.WaitGroup) - GET_payload := []byte("GET / HTTP/1.1\r\n\r\n") + payload := func(reqType string) []byte { + switch reqType { + case "GET": + return []byte("GET / HTTP/1.1\r\n\r\n") + case "POST": + return []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") + case "POST_CHUNKED": + return []byte("POST / HTTP/1.1\r\nHost: www.w3.org\r\nTransfer-Encoding: chunked\r\n\r\n4\r\nWiki\r\n5\r\npedia\r\ne\r\n in\r\n\r\nchunks.\r\n0\r\n\r\n") + } - // Post request terminates by reading Content-Length without double CRLF - POST_payload := []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") - - // Chunked requests terminated with double CRLF - POST_CHUNKED_payload := []byte("POST / HTTP/1.1\r\nHost: www.w3.org\r\nTransfer-Encoding: chunked\r\n\r\n4\r\nWiki\r\n5\r\npedia\r\ne\r\n in\r\n\r\nchunks.\r\n0\r\n\r\n") + return []byte("") + } server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -65,13 +70,13 @@ func TestHTTPClientSend(t *testing.T) { wg.Done() })) - client := NewHTTPClient(server.URL, &HTTPClientConfig{}) + client := NewHTTPClient(server.URL, &HTTPClientConfig{Debug: false}) wg.Add(4) - client.Send(POST_payload) - client.Send(GET_payload) - client.Send(POST_CHUNKED_payload) - client.Send(POST_payload) + client.Send(payload("POST")) + client.Send(payload("GET")) + client.Send(payload("POST_CHUNKED")) + client.Send(payload("POST")) wg.Wait() } @@ -79,13 +84,18 @@ func TestHTTPClientSend(t *testing.T) { func TestHTTPClientHTTPSSend(t *testing.T) { wg := new(sync.WaitGroup) - GET_payload := []byte("GET / HTTP/1.1\r\n\r\n") + payload := func(reqType string) []byte { + switch reqType { + case "GET": + return []byte("GET / HTTP/1.1\r\n\r\n") + case "POST": + return []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") + case "POST_CHUNKED": + return []byte("POST / HTTP/1.1\r\nHost: www.w3.org\r\nTransfer-Encoding: chunked\r\n\r\n4\r\nWiki\r\n5\r\npedia\r\ne\r\n in\r\n\r\nchunks.\r\n0\r\n\r\n") + } - // Post request terminates by reading Content-Length without double CRLF - POST_payload := []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") - - // Chunked requests terminated with double CRLF - POST_CHUNKED_payload := []byte("POST / HTTP/1.1\r\nHost: www.w3.org\r\nTransfer-Encoding: chunked\r\n\r\n4\r\nWiki\r\n5\r\npedia\r\ne\r\n in\r\n\r\nchunks.\r\n0\r\n\r\n") + return []byte("") + } server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -111,10 +121,10 @@ func TestHTTPClientHTTPSSend(t *testing.T) { client := NewHTTPClient(server.URL, &HTTPClientConfig{}) wg.Add(4) - client.Send(GET_payload) - client.Send(POST_payload) - client.Send(POST_CHUNKED_payload) - client.Send(POST_payload) + client.Send(payload("POST")) + client.Send(payload("GET")) + client.Send(payload("POST_CHUNKED")) + client.Send(payload("POST")) wg.Wait() } diff --git a/input_raw_test.go b/input_raw_test.go index 60bcf49..477eff8 100644 --- a/input_raw_test.go +++ b/input_raw_test.go @@ -118,14 +118,6 @@ func TestInputRAWChunkedEncoding(t *testing.T) { input := NewRAWInput(origin_address) - // We will use it to get content of raw HTTP request - test_output := NewTestOutput(func(data []byte) { - if strings.Contains(string(data), "Transfer-Encoding: chunked") { - t.Error("Should not contain chunked header") - } - wg.Done() - }) - listener := startHTTP(func(req *http.Request) { defer req.Body.Close() body, _ := ioutil.ReadAll(req.Body) @@ -139,14 +131,14 @@ func TestInputRAWChunkedEncoding(t *testing.T) { }) replay_address := listener.Addr().String() - http_output := NewHTTPOutput(replay_address, &HTTPOutputConfig{}) + http_output := NewHTTPOutput(replay_address, &HTTPOutputConfig{Debug: true}) Plugins.Inputs = []io.Reader{input} - Plugins.Outputs = []io.Writer{test_output, http_output} + Plugins.Outputs = []io.Writer{http_output} go Start(quit) - wg.Add(3) + wg.Add(2) curl := exec.Command("curl", "http://"+origin_address, "--header", "Transfer-Encoding: chunked", "--data-binary", "@README.md") err := curl.Run() diff --git a/output_http.go b/output_http.go index c1b42f9..9b77e37 100644 --- a/output_http.go +++ b/output_http.go @@ -1,59 +1,12 @@ package main import ( - "bufio" - "bytes" - "crypto/tls" "io" - "io/ioutil" "log" - "net/http" - "net/url" - "strings" "sync/atomic" "time" ) -type RedirectNotAllowed struct{} - -func (e *RedirectNotAllowed) Error() string { - return "Redirects not allowed" -} - -// customCheckRedirect disables redirects https://github.com/buger/gor/pull/15 -func (o *HTTPOutput) customCheckRedirect(req *http.Request, via []*http.Request) error { - if len(via) >= o.config.redirectLimit { - return new(RedirectNotAllowed) - } - return nil -} - -// ParseRequest in []byte returns a http request or an error -func ParseRequest(data []byte) (request *http.Request, err error) { - var body []byte - - buf := bytes.NewBuffer(data) - reader := bufio.NewReader(buf) - - // ReadRequest does not read POST bodies, we have to do it by ourseves - request, err = http.ReadRequest(reader) - - if err != nil { - return - } - - if request.Method == "POST" { - body, _ = ioutil.ReadAll(reader) - - bodyBuf := bytes.NewBuffer(body) - - request.Body = ioutil.NopCloser(bodyBuf) - request.ContentLength = int64(bodyBuf.Len()) - } - - return -} - const InitialDynamicWorkers = 10 type HTTPOutputConfig struct { @@ -63,6 +16,8 @@ type HTTPOutputConfig struct { workers int elasticSearch string + + Debug bool } type HTTPOutput struct { @@ -88,10 +43,6 @@ func NewHTTPOutput(address string, config *HTTPOutputConfig) io.Writer { o := new(HTTPOutput) - if !strings.HasPrefix(address, "http") { - address = "http://" + address - } - o.address = address o.config = config @@ -134,14 +85,10 @@ func (o *HTTPOutput) WorkerMaster() { } func (o *HTTPOutput) Worker() { - tr := &http.Transport{ - TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, - } - - client := &http.Client{ - Transport: tr, - CheckRedirect: o.customCheckRedirect, - } + client := NewHTTPClient(o.address, &HTTPClientConfig{ + FollowRedirects: o.config.redirectLimit, + Debug: o.config.Debug, + }) death_count := 0 @@ -194,55 +141,20 @@ func (o *HTTPOutput) Write(data []byte) (n int, err error) { return len(data), nil } -func (o *HTTPOutput) sendRequest(client *http.Client, data []byte) { - request, err := ParseRequest(data) - - if err != nil { - log.Println("Cannot parse request", string(data), err) - return - } - - // Change HOST of original request - URL := o.address + request.URL.Path + "?" + request.URL.RawQuery - - request.RequestURI = "" - request.URL, _ = url.ParseRequestURI(URL) - +func (o *HTTPOutput) sendRequest(client *HTTPClient, request []byte) { start := time.Now() - resp, err := client.Do(request) + resp, err := client.Send(request) stop := time.Now() - // We should not count Redirect as errors - if urlErr, ok := err.(*url.Error); ok { - if _, ok := urlErr.Err.(*RedirectNotAllowed); ok { - err = nil - } - } - - if err == nil { - defer resp.Body.Close() - } else { + if err != nil { log.Println("Request error:", err) } if o.elasticSearch != nil { - o.elasticSearch.ResponseAnalyze(request, resp, start, stop) + o.elasticSearch.ResponseAnalyze(request, resp, start, stop) } } -func SetHeader(request *http.Request, name string, value string) { - // Need to check here for the Host header as it needs to be set on the request and not as a separate header - // http.ReadRequest sets it by default to the URL Host of the request being read - if name == "Host" { - request.Host = value - } else { - request.Header.Set(name, value) - } - - return - -} - func (o *HTTPOutput) String() string { return "HTTP output: " + o.address } diff --git a/proto/proto.go b/proto/proto.go index f38645a..07622db 100644 --- a/proto/proto.go +++ b/proto/proto.go @@ -39,6 +39,12 @@ func Header(payload []byte, name []byte) (value []byte, headerStart, valueStart, return } +func GetHeader(payload []byte, name string) []byte { + val, _, _, _ := Header(payload, []byte(name)) + + return val +} + func SetHeader(payload, name, value []byte) []byte { _, hs, vs, he := Header(payload, name) @@ -84,4 +90,9 @@ func Method(payload []byte) []byte { end := bytes.IndexByte(payload, ' ') return payload[:end] +} + +// Status in response have same position as Path in request +func Status(payload []byte) []byte { + return Path(payload) } \ No newline at end of file diff --git a/raw_socket_listener/tcp_message.go b/raw_socket_listener/tcp_message.go index 9e45a85..69c4b93 100644 --- a/raw_socket_listener/tcp_message.go +++ b/raw_socket_listener/tcp_message.go @@ -4,10 +4,6 @@ import ( "log" "sort" "time" - "bytes" - "net/http/httputil" - "bufio" - "io/ioutil" ) const MSG_EXPIRE = 2000 * time.Millisecond @@ -75,30 +71,6 @@ func (t *TCPMessage) Timeout() { } } -var bTransferEncodingChunked = []byte("Transfer-Encoding: chunked\r\n") -var b2xCRLF = []byte("\r\n\r\n") - -// Norimalize requests with `Transfer-Encoding: chunked` header, because they have special body format -func fixChunkedEncoding(data []byte) []byte { - if bytes.Equal(data[0:4], bPOST) { - body_idx := bytes.Index(data, b2xCRLF) - chunked_header_idx := bytes.Index(data[:body_idx], bTransferEncodingChunked) - - if chunked_header_idx != -1 { - buf := bytes.NewBuffer(data[body_idx+4:]) - // Adding 4 bytes to skip 2xCLRF - bodyReader := bufio.NewReader(buf) - body, _ := ioutil.ReadAll(httputil.NewChunkedReader(bodyReader)) - - // Exclude Transfer-Encoding header and append new body - return append(append(append(data[:chunked_header_idx], - data[chunked_header_idx+len(bTransferEncodingChunked):body_idx]...), b2xCRLF...), body...) - } - } - - return data -} - // Bytes sorts packets in right orders and return message content func (t *TCPMessage) Bytes() (output []byte) { sort.Sort(BySeq(t.packets)) @@ -107,7 +79,7 @@ func (t *TCPMessage) Bytes() (output []byte) { output = append(output, v.Data...) } - return fixChunkedEncoding(output) + return output } // AddPacket to the message and ensure packet uniqueness From 4f114f067b7fef993a5a4c9956f2e2d08b71ba08 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Mon, 6 Jul 2015 21:09:06 +0500 Subject: [PATCH 19/30] Apply fmt --- elasticsearch.go | 42 ++++----- http_client.go | 12 +-- http_client_test.go | 2 +- http_modifier.go | 136 +++++++++++++++--------------- http_modifier_test.go | 145 ++++++++++++++++---------------- output_http.go | 6 +- settings.go | 6 +- settings_header_filters.go | 2 +- settings_header_filters_test.go | 32 +++---- settings_header_hash_filters.go | 2 +- settings_methods.go | 2 +- settings_url_map.go | 4 +- settings_url_map_test.go | 20 ++--- settings_url_regexp.go | 2 +- 14 files changed, 205 insertions(+), 208 deletions(-) diff --git a/elasticsearch.go b/elasticsearch.go index 1819c9d..801e75c 100644 --- a/elasticsearch.go +++ b/elasticsearch.go @@ -26,27 +26,27 @@ type ESPlugin struct { } type ESRequestResponse struct { - ReqUrl []byte `json:"Req_URL"` - ReqMethod []byte `json:"Req_Method"` - ReqUserAgent []byte `json:"Req_User-Agent"` - ReqAcceptLanguage []byte `json:"Req_Accept-Language,omitempty"` - ReqAccept []byte `json:"Req_Accept,omitempty"` - ReqAcceptEncoding []byte `json:"Req_Accept-Encoding,omitempty"` - ReqIfModifiedSince []byte `json:"Req_If-Modified-Since,omitempty"` - ReqConnection []byte `json:"Req_Connection,omitempty"` - ReqCookies []byte `json:"Req_Cookies,omitempty"` - RespStatus []byte `json:"Resp_Status"` - RespStatusCode []byte `json:"Resp_Status-Code"` - RespProto []byte `json:"Resp_Proto,omitempty"` - RespContentLength []byte `json:"Resp_Content-Length,omitempty"` - RespContentType []byte `json:"Resp_Content-Type,omitempty"` - RespTransferEncoding []byte `json:"Resp_Transfer-Encoding,omitempty"` - RespContentEncoding []byte `json:"Resp_Content-Encoding,omitempty"` - RespExpires []byte `json:"Resp_Expires,omitempty"` - RespCacheControl []byte `json:"Resp_Cache-Control,omitempty"` - RespVary []byte `json:"Resp_Vary,omitempty"` - RespSetCookie []byte `json:"Resp_Set-Cookie,omitempty"` - Rtt int64 `json:"RTT"` + ReqUrl []byte `json:"Req_URL"` + ReqMethod []byte `json:"Req_Method"` + ReqUserAgent []byte `json:"Req_User-Agent"` + ReqAcceptLanguage []byte `json:"Req_Accept-Language,omitempty"` + ReqAccept []byte `json:"Req_Accept,omitempty"` + ReqAcceptEncoding []byte `json:"Req_Accept-Encoding,omitempty"` + ReqIfModifiedSince []byte `json:"Req_If-Modified-Since,omitempty"` + ReqConnection []byte `json:"Req_Connection,omitempty"` + ReqCookies []byte `json:"Req_Cookies,omitempty"` + RespStatus []byte `json:"Resp_Status"` + RespStatusCode []byte `json:"Resp_Status-Code"` + RespProto []byte `json:"Resp_Proto,omitempty"` + RespContentLength []byte `json:"Resp_Content-Length,omitempty"` + RespContentType []byte `json:"Resp_Content-Type,omitempty"` + RespTransferEncoding []byte `json:"Resp_Transfer-Encoding,omitempty"` + RespContentEncoding []byte `json:"Resp_Content-Encoding,omitempty"` + RespExpires []byte `json:"Resp_Expires,omitempty"` + RespCacheControl []byte `json:"Resp_Cache-Control,omitempty"` + RespVary []byte `json:"Resp_Vary,omitempty"` + RespSetCookie []byte `json:"Resp_Set-Cookie,omitempty"` + Rtt int64 `json:"RTT"` Timestamp time.Time } diff --git a/http_client.go b/http_client.go index e5f859d..3d8d5bf 100644 --- a/http_client.go +++ b/http_client.go @@ -2,12 +2,12 @@ package main import ( "crypto/tls" + "github.com/buger/gor/proto" "io" "net" "net/url" "strings" "time" - "github.com/buger/gor/proto" ) var defaultPorts = map[string]string{ @@ -17,14 +17,14 @@ var defaultPorts = map[string]string{ type HTTPClientConfig struct { FollowRedirects int - Debug bool + Debug bool } type HTTPClient struct { - baseURL *url.URL - conn net.Conn - respBuf []byte - config *HTTPClientConfig + baseURL *url.URL + conn net.Conn + respBuf []byte + config *HTTPClientConfig redirectsCount int } diff --git a/http_client_test.go b/http_client_test.go index 45a3ba9..b22eabd 100644 --- a/http_client_test.go +++ b/http_client_test.go @@ -244,4 +244,4 @@ func TestHTTPClientRedirectLimit(t *testing.T) { client.Send(GET_payload) wg.Wait() -} \ No newline at end of file +} diff --git a/http_modifier.go b/http_modifier.go index 2f1dc3d..5d2b3e3 100644 --- a/http_modifier.go +++ b/http_modifier.go @@ -1,99 +1,97 @@ package main import ( - "github.com/buger/gor/proto" - "hash/fnv" + "github.com/buger/gor/proto" + "hash/fnv" ) - type HTTPModifierConfig struct { - urlRegexp HTTPUrlRegexp - urlRewrite UrlRewriteMap - headerFilters HTTPHeaderFilters - headerHashFilters HTTPHeaderHashFilters + urlRegexp HTTPUrlRegexp + urlRewrite UrlRewriteMap + headerFilters HTTPHeaderFilters + headerHashFilters HTTPHeaderHashFilters - headers HTTPHeaders - methods HTTPMethods + headers HTTPHeaders + methods HTTPMethods } type HTTPModifier struct { - config *HTTPModifierConfig + config *HTTPModifierConfig } func NewHTTPModifier(config *HTTPModifierConfig) *HTTPModifier { - // Optimization to skip modifier completely if we do not need it - if config.urlRegexp.regexp == nil && - len(config.urlRewrite) == 0 && - len(config.headerFilters) == 0 && - len(config.headerHashFilters) == 0 && - len(config.headers) == 0 && - len(config.methods) == 0 { - return nil - } + // Optimization to skip modifier completely if we do not need it + if config.urlRegexp.regexp == nil && + len(config.urlRewrite) == 0 && + len(config.headerFilters) == 0 && + len(config.headerHashFilters) == 0 && + len(config.headers) == 0 && + len(config.methods) == 0 { + return nil + } - return &HTTPModifier{config: config} + return &HTTPModifier{config: config} } func (m *HTTPModifier) Rewrite(payload []byte) (response []byte) { - if len(m.config.methods) > 0 && !m.config.methods.Contains(proto.Method(payload)) { - return - } + if len(m.config.methods) > 0 && !m.config.methods.Contains(proto.Method(payload)) { + return + } - if m.config.urlRegexp.regexp != nil { - host, _, _, _ := proto.Header(payload, []byte("Host")) - fullPath := append(host, proto.Path(payload)...) + if m.config.urlRegexp.regexp != nil { + host, _, _, _ := proto.Header(payload, []byte("Host")) + fullPath := append(host, proto.Path(payload)...) - if !m.config.urlRegexp.regexp.Match(fullPath) { - return - } - } + if !m.config.urlRegexp.regexp.Match(fullPath) { + return + } + } - if len(m.config.headerFilters) > 0 { - for _, f := range m.config.headerFilters { - value, s, _, _ := proto.Header(payload, f.name) + if len(m.config.headerFilters) > 0 { + for _, f := range m.config.headerFilters { + value, s, _, _ := proto.Header(payload, f.name) - if s != -1 && !f.regexp.Match(value) { - return - } - } - } + if s != -1 && !f.regexp.Match(value) { + return + } + } + } - if len(m.config.headerHashFilters) > 0 { - for _, f := range m.config.headerHashFilters { - value, s, _, _ := proto.Header(payload, f.name) + if len(m.config.headerHashFilters) > 0 { + for _, f := range m.config.headerHashFilters { + value, s, _, _ := proto.Header(payload, f.name) - if s == -1 { - return - } + if s == -1 { + return + } - hasher := fnv.New32a() - hasher.Write(value) + hasher := fnv.New32a() + hasher.Write(value) - if (hasher.Sum32() % 100) >= f.percent { - return - } - } - } + if (hasher.Sum32() % 100) >= f.percent { + return + } + } + } - if len(m.config.urlRewrite) > 0 { - path := proto.Path(payload) + if len(m.config.urlRewrite) > 0 { + path := proto.Path(payload) - for _, f := range m.config.urlRewrite { - if f.src.Match(path) { - path = f.src.ReplaceAll(path, f.target) - payload = proto.SetPath(payload, path) + for _, f := range m.config.urlRewrite { + if f.src.Match(path) { + path = f.src.ReplaceAll(path, f.target) + payload = proto.SetPath(payload, path) - break - } - } - } + break + } + } + } - if len(m.config.headers) > 0 { - for _, header := range m.config.headers { - payload = proto.SetHeader(payload, []byte(header.Name), []byte(header.Value)) - } - } + if len(m.config.headers) > 0 { + for _, header := range m.config.headers { + payload = proto.SetHeader(payload, []byte(header.Name), []byte(header.Value)) + } + } - - return payload -} \ No newline at end of file + return payload +} diff --git a/http_modifier_test.go b/http_modifier_test.go index ad06719..4689aca 100644 --- a/http_modifier_test.go +++ b/http_modifier_test.go @@ -1,112 +1,111 @@ package main import ( - "testing" - "github.com/buger/gor/proto" - "bytes" + "bytes" + "github.com/buger/gor/proto" + "testing" ) func TestHTTPModifierWithoutConfig(t *testing.T) { - if NewHTTPModifier(&HTTPModifierConfig{}) != nil { - t.Error("If no config specified should not be initialized") - } + if NewHTTPModifier(&HTTPModifierConfig{}) != nil { + t.Error("If no config specified should not be initialized") + } } func TestHTTPModifierHeaderFilters(t *testing.T) { - filters := HTTPHeaderFilters{} - filters.Set("Host:^www.w3.org$") + filters := HTTPHeaderFilters{} + filters.Set("Host:^www.w3.org$") - modifier := NewHTTPModifier(&HTTPModifierConfig{ - headerFilters: filters, - }) + modifier := NewHTTPModifier(&HTTPModifierConfig{ + headerFilters: filters, + }) - payload := []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") + payload := []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") - if len(modifier.Rewrite(payload)) == 0 { - t.Error("Request should pass filters") - } + if len(modifier.Rewrite(payload)) == 0 { + t.Error("Request should pass filters") + } - filters = HTTPHeaderFilters{} - // Setting filter that not match our header - filters.Set("Host:^www.w4.org$") + filters = HTTPHeaderFilters{} + // Setting filter that not match our header + filters.Set("Host:^www.w4.org$") - modifier = NewHTTPModifier(&HTTPModifierConfig{ - headerFilters: filters, - }) + modifier = NewHTTPModifier(&HTTPModifierConfig{ + headerFilters: filters, + }) - if len(modifier.Rewrite(payload)) != 0 { - t.Error("Request should not pass filters") - } + if len(modifier.Rewrite(payload)) != 0 { + t.Error("Request should not pass filters") + } } - func TestHTTPModifierURLRewrite(t *testing.T) { - var url, new_url []byte + var url, new_url []byte - rewrites := UrlRewriteMap{} + rewrites := UrlRewriteMap{} - payload := func(url []byte) []byte { - return []byte("POST " + string(url) + " HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") - } + payload := func(url []byte) []byte { + return []byte("POST " + string(url) + " HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") + } - err := rewrites.Set("/v1/user/([^\\/]+)/ping:/v2/user/$1/ping") - if err != nil { - t.Error("Should not error on /v1/user/([^\\/]+)/ping:/v2/user/$1/ping") - } + err := rewrites.Set("/v1/user/([^\\/]+)/ping:/v2/user/$1/ping") + if err != nil { + t.Error("Should not error on /v1/user/([^\\/]+)/ping:/v2/user/$1/ping") + } - modifier := NewHTTPModifier(&HTTPModifierConfig{ - urlRewrite: rewrites, - }) + modifier := NewHTTPModifier(&HTTPModifierConfig{ + urlRewrite: rewrites, + }) - url = []byte("/v1/user/joe/ping") - if new_url = proto.Path(modifier.Rewrite(payload(url))); bytes.Equal(new_url, url) { - t.Error("Request url should have been rewritten, wasn't", string(new_url)) - } + url = []byte("/v1/user/joe/ping") + if new_url = proto.Path(modifier.Rewrite(payload(url))); bytes.Equal(new_url, url) { + t.Error("Request url should have been rewritten, wasn't", string(new_url)) + } - url = []byte("/v1/user/ping") - if new_url = proto.Path(modifier.Rewrite(payload(url))); !bytes.Equal(new_url, url) { - t.Error("Request url should have been rewritten, wasn't", string(new_url)) - } + url = []byte("/v1/user/ping") + if new_url = proto.Path(modifier.Rewrite(payload(url))); !bytes.Equal(new_url, url) { + t.Error("Request url should have been rewritten, wasn't", string(new_url)) + } } func TestHTTPModifierHeaderHashFilters(t *testing.T) { - filters := HTTPHeaderHashFilters{} - filters.Set("Header2:1/2") + filters := HTTPHeaderHashFilters{} + filters.Set("Header2:1/2") - modifier := NewHTTPModifier(&HTTPModifierConfig{ - headerHashFilters: filters, - }) + modifier := NewHTTPModifier(&HTTPModifierConfig{ + headerHashFilters: filters, + }) - payload := func(header []byte) []byte { - return []byte("POST / HTTP/1.1\r\n" + string(header) + "Content-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") - } + payload := func(header []byte) []byte { + return []byte("POST / HTTP/1.1\r\n" + string(header) + "Content-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") + } - if p := modifier.Rewrite(payload([]byte(""))); len(p) > 0 { - t.Error("Request should not pass filters, Header2 does not exist") - } + if p := modifier.Rewrite(payload([]byte(""))); len(p) > 0 { + t.Error("Request should not pass filters, Header2 does not exist") + } - if p := modifier.Rewrite(payload([]byte("Header2: 3\r\n"))); len(p) > 0 { - t.Error("Request should not pass filters, Header2 hash too high") - } + if p := modifier.Rewrite(payload([]byte("Header2: 3\r\n"))); len(p) > 0 { + t.Error("Request should not pass filters, Header2 hash too high") + } - if p := modifier.Rewrite(payload([]byte("Header2: 1\r\n"))); len(p) == 0 { - t.Error("Request should pass filters") - } + if p := modifier.Rewrite(payload([]byte("Header2: 1\r\n"))); len(p) == 0 { + t.Error("Request should pass filters") + } } func TestHTTPModifierHeaders(t *testing.T) { - headers := HTTPHeaders{} - headers.Set("Header1:1") - headers.Set("Host:localhost") + headers := HTTPHeaders{} + headers.Set("Header1:1") + headers.Set("Host:localhost") - modifier := NewHTTPModifier(&HTTPModifierConfig{ - headers: headers, - }) + modifier := NewHTTPModifier(&HTTPModifierConfig{ + headers: headers, + }) - payload := []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") - new_payload := []byte("POST /post HTTP/1.1\r\nHeader1: 1\r\nContent-Length: 7\r\nHost: localhost\r\n\r\na=1&b=2") + payload := []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") + new_payload := []byte("POST /post HTTP/1.1\r\nHeader1: 1\r\nContent-Length: 7\r\nHost: localhost\r\n\r\na=1&b=2") - if payload = modifier.Rewrite(payload); !bytes.Equal(payload, new_payload) { - t.Error("Should update request headers", string(payload)) - } + if payload = modifier.Rewrite(payload); !bytes.Equal(payload, new_payload) { + t.Error("Should update request headers", string(payload)) + } } diff --git a/output_http.go b/output_http.go index 9b77e37..e2a6c96 100644 --- a/output_http.go +++ b/output_http.go @@ -12,7 +12,7 @@ const InitialDynamicWorkers = 10 type HTTPOutputConfig struct { redirectLimit int - stats bool + stats bool workers int elasticSearch string @@ -87,7 +87,7 @@ func (o *HTTPOutput) WorkerMaster() { func (o *HTTPOutput) Worker() { client := NewHTTPClient(o.address, &HTTPClientConfig{ FollowRedirects: o.config.redirectLimit, - Debug: o.config.Debug, + Debug: o.config.Debug, }) death_count := 0 @@ -151,7 +151,7 @@ func (o *HTTPOutput) sendRequest(client *HTTPClient, request []byte) { } if o.elasticSearch != nil { - o.elasticSearch.ResponseAnalyze(request, resp, start, stop) + o.elasticSearch.ResponseAnalyze(request, resp, start, stop) } } diff --git a/settings.go b/settings.go index 411ca62..3b6a7d0 100644 --- a/settings.go +++ b/settings.go @@ -29,11 +29,11 @@ type AppSettings struct { inputRAW MultiOption - inputHTTP MultiOption - outputHTTP MultiOption + inputHTTP MultiOption + outputHTTP MultiOption outputHTTPConfig HTTPOutputConfig - modifierConfig HTTPModifierConfig + modifierConfig HTTPModifierConfig } var Settings AppSettings = AppSettings{} diff --git a/settings_header_filters.go b/settings_header_filters.go index 7a017a6..bf10aaa 100644 --- a/settings_header_filters.go +++ b/settings_header_filters.go @@ -31,4 +31,4 @@ func (h *HTTPHeaderFilters) Set(value string) error { *h = append(*h, headerFilter{name: []byte(valArr[0]), regexp: r}) return nil -} \ No newline at end of file +} diff --git a/settings_header_filters_test.go b/settings_header_filters_test.go index 499f910..bbffecb 100644 --- a/settings_header_filters_test.go +++ b/settings_header_filters_test.go @@ -1,25 +1,25 @@ package main import ( - "testing" + "testing" ) func TestHTTPHeaderFilters(t *testing.T) { - filters := HTTPHeaderFilters{} + filters := HTTPHeaderFilters{} - err := filters.Set("Header1:^$") - if err != nil { - t.Error("Should not error on Header1:^$") - } + err := filters.Set("Header1:^$") + if err != nil { + t.Error("Should not error on Header1:^$") + } - err = filters.Set("Header2:^:$") - if err != nil { - t.Error("Should not error on Header2:^:$") - } + err = filters.Set("Header2:^:$") + if err != nil { + t.Error("Should not error on Header2:^:$") + } - // Missing colon - err = filters.Set("Header3-^$") - if err == nil { - t.Error("Should error on Header2:^:$") - } -} \ No newline at end of file + // Missing colon + err = filters.Set("Header3-^$") + if err == nil { + t.Error("Should error on Header2:^:$") + } +} diff --git a/settings_header_hash_filters.go b/settings_header_hash_filters.go index 5c848fe..e8a2072 100644 --- a/settings_header_hash_filters.go +++ b/settings_header_hash_filters.go @@ -39,4 +39,4 @@ func (h *HTTPHeaderHashFilters) Set(value string) error { *h = append(*h, f) return nil -} \ No newline at end of file +} diff --git a/settings_methods.go b/settings_methods.go index cd5a977..8d84521 100644 --- a/settings_methods.go +++ b/settings_methods.go @@ -1,9 +1,9 @@ package main import ( + "bytes" "fmt" "strings" - "bytes" ) type HTTPMethods [][]byte diff --git a/settings_url_map.go b/settings_url_map.go index d07a313..d90ac6f 100644 --- a/settings_url_map.go +++ b/settings_url_map.go @@ -27,6 +27,6 @@ func (r *UrlRewriteMap) Set(value string) error { if err != nil { return err } - *r = append(*r, urlRewrite{src: regexp, target: []byte(valArr[1]) }) + *r = append(*r, urlRewrite{src: regexp, target: []byte(valArr[1])}) return nil -} \ No newline at end of file +} diff --git a/settings_url_map_test.go b/settings_url_map_test.go index de7ad7d..b822387 100644 --- a/settings_url_map_test.go +++ b/settings_url_map_test.go @@ -1,18 +1,18 @@ package main import ( - "testing" + "testing" ) func TestUrlRewriteMap(t *testing.T) { - var err error - rewrites := UrlRewriteMap{} + var err error + rewrites := UrlRewriteMap{} - if err = rewrites.Set("/v1/user/([^\\/]+)/ping:/v2/user/$1/ping"); err != nil { - t.Error("Should set mapping", err) - } + if err = rewrites.Set("/v1/user/([^\\/]+)/ping:/v2/user/$1/ping"); err != nil { + t.Error("Should set mapping", err) + } - if err = rewrites.Set("/v1/user/([^\\/]+)/ping"); err == nil { - t.Error("Should not set mapping without :") - } -} \ No newline at end of file + if err = rewrites.Set("/v1/user/([^\\/]+)/ping"); err == nil { + t.Error("Should not set mapping without :") + } +} diff --git a/settings_url_regexp.go b/settings_url_regexp.go index bea2003..e676860 100644 --- a/settings_url_regexp.go +++ b/settings_url_regexp.go @@ -19,4 +19,4 @@ func (r *HTTPUrlRegexp) Set(value string) error { regexp, err := regexp.Compile(value) r.regexp = regexp return err -} \ No newline at end of file +} From 31fa612fef39cc21c94cd86da184506088475e23 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Tue, 7 Jul 2015 08:29:49 +0500 Subject: [PATCH 20/30] Add go tool coverage make task --- .gitignore | 2 ++ Makefile | 6 +++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index dfabfec..9d54abc 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,4 @@ *.swp *.gor + +*.out diff --git a/Makefile b/Makefile index c75b5f2..ca1a5f5 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,7 @@ release: release-x86 release-x64 release-x64: docker run -v `pwd`:$(SOURCE_PATH) -t --env GOOS=linux --env GOARCH=amd64 --env CGO_ENABLED=0 -i gor go build && tar -czf gor_x64.tar.gz gor && rm gor - + release-x86: docker run -v `pwd`:$(SOURCE_PATH) -t --env GOOS=linux --env GOARCH=386 --env CGO_ENABLED=0 -i gor go build && tar -czf gor_x86.tar.gz gor && rm gor @@ -16,6 +16,10 @@ dbuild: dtest: docker run -v `pwd`:$(SOURCE_PATH) -t -i --env GORACE="halt_on_error=1" gor go test ./... $(ARGS) -race -v -timeout 15s +dcover: + docker run -v `pwd`:$(SOURCE_PATH) -t -i --env GORACE="halt_on_error=1" gor go test $(ARGS) -race -v -timeout 15s -coverprofile=coverage.out + go tool cover -html=coverage.out + dfmt: docker run -v `pwd`:$(SOURCE_PATH) -t -i gor go fmt From 07c65117ade7a9a137856617fd78d4732700a72a Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Wed, 8 Jul 2015 18:26:14 +0500 Subject: [PATCH 21/30] Handle proxy request format and HTTP 1.0 requests --- http_client.go | 24 +++++++++++++++--------- http_client_test.go | 40 +++++++++++++++++++++++++++++++--------- proto/proto.go | 18 ++++++++++++++++++ proto/proto_test.go | 11 +++++++++++ 4 files changed, 75 insertions(+), 18 deletions(-) diff --git a/http_client.go b/http_client.go index 3d8d5bf..50237ec 100644 --- a/http_client.go +++ b/http_client.go @@ -21,7 +21,9 @@ type HTTPClientConfig struct { } type HTTPClient struct { - baseURL *url.URL + baseURL string + scheme string + host string conn net.Conn respBuf []byte config *HTTPClientConfig @@ -33,24 +35,28 @@ func NewHTTPClient(baseURL string, config *HTTPClientConfig) *HTTPClient { baseURL = "http://" + baseURL } + u, _ := url.Parse(baseURL) + if !strings.Contains(u.Host, ":") { + u.Host += ":" + defaultPorts[u.Scheme] + } + + client := new(HTTPClient) - client.baseURL, _ = url.Parse(baseURL) + client.baseURL = u.String() + client.host = u.Host + client.scheme = u.Scheme client.respBuf = make([]byte, 4096*10) client.config = config - if !strings.Contains(client.baseURL.Host, ":") { - client.baseURL.Host += ":" + defaultPorts[client.baseURL.Scheme] - } - return client } func (c *HTTPClient) Connect() (err error) { c.Disconnect() - c.conn, err = net.Dial("tcp", c.baseURL.Host) + c.conn, err = net.Dial("tcp", c.host) - if c.baseURL.Scheme == "https" { + if c.scheme == "https" { tlsConn := tls.Client(c.conn, &tls.Config{InsecureSkipVerify: true}) if err = tlsConn.Handshake(); err != nil { @@ -93,7 +99,7 @@ func (c *HTTPClient) Send(data []byte) (response []byte, err error) { c.conn.SetWriteDeadline(timeout) - data = proto.SetHeader(data, []byte("Host"), []byte(c.baseURL.Host)) + data = proto.SetHost(data, []byte(c.baseURL), []byte(c.host)) if c.config.Debug { Debug("Sending:", string(data)) diff --git a/http_client_test.go b/http_client_test.go index b22eabd..a5b238b 100644 --- a/http_client_test.go +++ b/http_client_test.go @@ -13,23 +13,23 @@ import ( func TestHTTPClientURLPort(t *testing.T) { c1 := NewHTTPClient("http://example.com", &HTTPClientConfig{}) - if c1.baseURL.String() != "http://example.com:80" { - t.Error("Sould add 80 port for http:", c1.baseURL.String()) + if c1.baseURL != "http://example.com:80" { + t.Error("Sould add 80 port for http:", c1.baseURL) } c2 := NewHTTPClient("https://example.com", &HTTPClientConfig{}) - if c2.baseURL.String() != "https://example.com:443" { - t.Error("Sould add 443 port for https:", c2.baseURL.String()) + if c2.baseURL != "https://example.com:443" { + t.Error("Sould add 443 port for https:", c2.baseURL) } c3 := NewHTTPClient("https://example.com:1", &HTTPClientConfig{}) - if c3.baseURL.String() != "https://example.com:1" { - t.Error("Sould use specified port:", c3.baseURL.String()) + if c3.baseURL != "https://example.com:1" { + t.Error("Sould use specified port:", c3.baseURL) } c4 := NewHTTPClient("example.com", &HTTPClientConfig{}) - if c4.baseURL.String() != "http://example.com:80" { - t.Error("Sould add default protocol:", c4.baseURL.String()) + if c4.baseURL != "http://example.com:80" { + t.Error("Sould add default protocol:", c4.baseURL) } } @@ -70,7 +70,7 @@ func TestHTTPClientSend(t *testing.T) { wg.Done() })) - client := NewHTTPClient(server.URL, &HTTPClientConfig{Debug: false}) + client := NewHTTPClient(server.URL, &HTTPClientConfig{Debug: true}) wg.Add(4) client.Send(payload("POST")) @@ -245,3 +245,25 @@ func TestHTTPClientRedirectLimit(t *testing.T) { wg.Wait() } + +func TestHTTPClientHandleHTTP10(t *testing.T) { + wg := new(sync.WaitGroup) + + GET_payload := []byte("GET http://foobar.com/path HTTP/1.0\r\n\r\n") + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + + if r.URL.Path != "/path" { + t.Error("Path not match:", r.URL.Path) + } + + wg.Done() + })) + + client := NewHTTPClient(server.URL, &HTTPClientConfig{Debug: true}) + + wg.Add(1) + client.Send(GET_payload) + + wg.Wait() +} diff --git a/proto/proto.go b/proto/proto.go index 07622db..330115b 100644 --- a/proto/proto.go +++ b/proto/proto.go @@ -86,6 +86,24 @@ func SetPath(payload, path []byte) []byte { return byteutils.Replace(payload, start, start+end, path) } +func SetHost(payload, url, host []byte) []byte { + // If this is HTTP 1.0 traffic or proxy traffic it may include host right into path variable, so instead of setting Host header we rewrite Path + // Fix for https://github.com/buger/gor/issues/156 + if path := Path(payload); bytes.HasPrefix(path, []byte("http")) { + hostStart := bytes.IndexByte(path, ':') // : position "https?:" + hostStart += 3 // Skip 1 ':' and 2 '\' + hostEnd := hostStart + bytes.IndexByte(path[hostStart:], '/') + + newPath := make([]byte, len(path)) + copy(newPath, path) + newPath = byteutils.Replace(newPath, 0, hostEnd, url) + + return SetPath(payload, newPath) + } else { + return SetHeader(payload, []byte("Host"), host) + } +} + func Method(payload []byte) []byte { end := bytes.IndexByte(payload, ' ') diff --git a/proto/proto_test.go b/proto/proto_test.go index bc7dcaf..3ada427 100644 --- a/proto/proto_test.go +++ b/proto/proto_test.go @@ -87,4 +87,15 @@ func TestSetPath(t *testing.T) { if payload = SetPath(payload, []byte("/new_path")); !bytes.Equal(payload, payload_after) { t.Error("Should replace path", string(payload)) } +} + +func TestSetHostHTTP10(t *testing.T) { + var payload, payload_after []byte + + payload = []byte("POST http://example.com/post HTTP/1.0\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") + payload_after = []byte("POST http://new.com/post HTTP/1.0\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") + + if payload = SetHost(payload, []byte("http://new.com"), []byte("new.com")); !bytes.Equal(payload, payload_after) { + t.Error("Should replace host", string(payload)) + } } \ No newline at end of file From 4d00ff87572bcb42877acdfd4cf5909938c4b740 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Wed, 8 Jul 2015 19:07:50 +0500 Subject: [PATCH 22/30] Change internal message format --- input_tcp.go | 32 ++++++++++++++------------------ input_tcp_test.go | 9 +++++---- output_tcp.go | 10 ++++++---- output_tcp_test.go | 20 +++++++++----------- 4 files changed, 34 insertions(+), 37 deletions(-) diff --git a/input_tcp.go b/input_tcp.go index 7eeb488..2e26c62 100644 --- a/input_tcp.go +++ b/input_tcp.go @@ -2,9 +2,11 @@ package main import ( "bufio" - "io" "log" "net" + "encoding/hex" + "fmt" + "os" ) // Can be tested using nc tool: @@ -59,24 +61,18 @@ func (i *TCPInput) handleConnection(conn net.Conn) { defer conn.Close() reader := bufio.NewReader(conn) + scanner := bufio.NewScanner(reader) - for { - buf, err := reader.ReadBytes('¶') - if err == io.EOF { - return - } else if err != nil { - log.Println("Unexpected error in input tcp connection", err) - return - } - buf_len := len(buf) - if buf_len > 0 { - new_buf_len := len(buf) - 2 - if new_buf_len > 0 { - new_buf := make([]byte, new_buf_len) - copy(new_buf, buf[:new_buf_len]) - i.data <- new_buf - } - } + for scanner.Scan() { + encodedPayload := scanner.Bytes() + // Hex encoding always 2x number of bytes + decoded := make([]byte, len(encodedPayload)/2) + hex.Decode(decoded, encodedPayload) + i.data <- decoded + } + + if err := scanner.Err(); err != nil { + fmt.Fprintln(os.Stderr, "Unexpected error in input tcp connection:", err) } } diff --git a/input_tcp_test.go b/input_tcp_test.go index fb8a2d1..0f76e4c 100644 --- a/input_tcp_test.go +++ b/input_tcp_test.go @@ -6,6 +6,7 @@ import ( "net" "sync" "testing" + "encoding/hex" ) func TestTCPInput(t *testing.T) { @@ -38,10 +39,10 @@ func TestTCPInput(t *testing.T) { for i := 0; i < 100; i++ { wg.Add(1) - new_buf := make([]byte, len(msg)+2) - msg = append(msg, []byte("¶")...) - copy(new_buf, msg) - conn.Write(new_buf) + + encoded := make([]byte, len(msg)*2 + 1) + hex.Encode(encoded, msg) + conn.Write(append(encoded, '\n')) } wg.Wait() diff --git a/output_tcp.go b/output_tcp.go index 53eaa7f..fb06025 100644 --- a/output_tcp.go +++ b/output_tcp.go @@ -6,6 +6,7 @@ import ( "log" "net" "time" + "encoding/hex" ) type TCPOutput struct { @@ -51,10 +52,11 @@ func (o *TCPOutput) worker() { } func (o *TCPOutput) Write(data []byte) (n int, err error) { - new_buf := make([]byte, len(data)+2) - data = append(data, []byte("¶")...) - copy(new_buf, data) - o.buf <- new_buf + // Hex encoding always 2x number of bytes + encoded := make([]byte, len(data)*2 + 1) + hex.Encode(encoded, data) + o.buf <- append(encoded, '\n') + if Settings.outputTCPStats { o.bufStats.Write(len(o.buf)) } diff --git a/output_tcp_test.go b/output_tcp_test.go index aaf0cf6..e42f14a 100644 --- a/output_tcp_test.go +++ b/output_tcp_test.go @@ -7,6 +7,7 @@ import ( "net" "sync" "testing" + "encoding/hex" ) func TestTCPOutput(t *testing.T) { @@ -48,17 +49,14 @@ func startTCP(cb func([]byte)) net.Listener { go func() { reader := bufio.NewReader(conn) - for { - buf, err := reader.ReadBytes('¶') - new_buf_len := len(buf) - 2 - new_buf := make([]byte, new_buf_len) - copy(new_buf, buf[:new_buf_len]) - if err != nil { - if err != io.EOF { - log.Printf("error: %s\n", err) - } - } - cb(new_buf) + scanner := bufio.NewScanner(reader) + + for scanner.Scan() { + encodedPayload := scanner.Bytes() + // Hex encoding always 2x number of bytes + decoded := make([]byte, len(encodedPayload)/2) + hex.Decode(decoded, encodedPayload) + cb(decoded) } }() } From 686d7436aed322a1989e9b96ef0d3509d90873f1 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Wed, 8 Jul 2015 20:09:43 +0500 Subject: [PATCH 23/30] Merge setting files --- http_modifier_settings.go | 175 +++++++++++++++++++++++++++ http_modifier_settings_test.go | 68 +++++++++++ settings.go | 42 ++++++- settings_header_filters.go | 34 ------ settings_header_filters_test.go | 25 ---- settings_header_hash_filters.go | 42 ------- settings_header_hash_filters_test.go | 19 --- settings_headers.go | 32 ----- settings_methods.go | 27 ----- settings_methods_test.go | 24 ---- settings_option.go | 16 --- settings_url_map.go | 32 ----- settings_url_map_test.go | 18 --- settings_url_regexp.go | 22 ---- settings_url_regexp_test.go | 10 -- 15 files changed, 279 insertions(+), 307 deletions(-) create mode 100644 http_modifier_settings.go create mode 100644 http_modifier_settings_test.go delete mode 100644 settings_header_filters.go delete mode 100644 settings_header_filters_test.go delete mode 100644 settings_header_hash_filters.go delete mode 100644 settings_header_hash_filters_test.go delete mode 100644 settings_headers.go delete mode 100644 settings_methods.go delete mode 100644 settings_methods_test.go delete mode 100644 settings_option.go delete mode 100644 settings_url_map.go delete mode 100644 settings_url_map_test.go delete mode 100644 settings_url_regexp.go delete mode 100644 settings_url_regexp_test.go diff --git a/http_modifier_settings.go b/http_modifier_settings.go new file mode 100644 index 0000000..52a8ac7 --- /dev/null +++ b/http_modifier_settings.go @@ -0,0 +1,175 @@ +package main + +import ( + "errors" + "fmt" + "regexp" + "strings" + "strconv" + "bytes" +) + +// +// Handling of --http-allow-header options +// +type headerFilter struct { + name []byte + regexp *regexp.Regexp +} + +type HTTPHeaderFilters []headerFilter + +func (h *HTTPHeaderFilters) String() string { + return fmt.Sprint(*h) +} + +func (h *HTTPHeaderFilters) Set(value string) error { + valArr := strings.SplitN(value, ":", 2) + if len(valArr) < 2 { + return errors.New("need both header and value, colon-delimited (ex. user_id:^169$).") + } + r, err := regexp.Compile(valArr[1]) + if err != nil { + return err + } + + *h = append(*h, headerFilter{name: []byte(valArr[0]), regexp: r}) + + return nil +} + +// +// Handling of --http-allow-header-hash options +// +type headerHashFilter struct { + name []byte + percent uint32 +} + +type HTTPHeaderHashFilters []headerHashFilter + +func (h *HTTPHeaderHashFilters) String() string { + return fmt.Sprint(*h) +} + +func (h *HTTPHeaderHashFilters) Set(value string) error { + valArr := strings.SplitN(value, ":", 2) + if len(valArr) < 2 { + return errors.New("need both header and value, colon-delimited (ex. user_id:1/2).") + } + + fracArr := strings.Split(valArr[1], "/") + if len(fracArr) < 2 { + return errors.New("need both a numerator and denominator specified, slash-delimited (ex. user_id:1/4).") + } + + var num, den uint64 + num, _ = strconv.ParseUint(fracArr[0], 10, 64) + den, _ = strconv.ParseUint(fracArr[1], 10, 64) + + var f headerHashFilter + f.name = []byte(valArr[0]) + f.percent = uint32((float64(num) / float64(den)) * 100) + *h = append(*h, f) + + return nil +} + + +// +// Handling of --http-set-header option +// +type HTTPHeaders []HTTPHeader +type HTTPHeader struct { + Name string + Value string +} + +func (h *HTTPHeaders) String() string { + return fmt.Sprint(*h) +} + +func (h *HTTPHeaders) Set(value string) error { + v := strings.SplitN(value, ":", 2) + if len(v) != 2 { + return errors.New("Expected `Key: Value`") + } + + header := HTTPHeader{ + strings.TrimSpace(v[0]), + strings.TrimSpace(v[1]), + } + + *h = append(*h, header) + return nil +} + +// +// Handling of --http-allow-method option +// +type HTTPMethods [][]byte + +func (h *HTTPMethods) String() string { + return fmt.Sprint(*h) +} + +func (h *HTTPMethods) Set(value string) error { + *h = append(*h, []byte(value)) + return nil +} + +func (h *HTTPMethods) Contains(value []byte) bool { + for _, method := range *h { + if bytes.Equal(value, method) { + return true + } + } + return false +} + +// +// Handling of --http-rewrite-url option +// +type urlRewrite struct { + src *regexp.Regexp + target []byte +} + +type UrlRewriteMap []urlRewrite + +func (r *UrlRewriteMap) String() string { + return fmt.Sprint(*r) +} + +func (r *UrlRewriteMap) Set(value string) error { + valArr := strings.SplitN(value, ":", 2) + if len(valArr) < 2 { + return errors.New("need both src and target, colon-delimited (ex. /a:/b).") + } + regexp, err := regexp.Compile(valArr[0]) + if err != nil { + return err + } + *r = append(*r, urlRewrite{src: regexp, target: []byte(valArr[1])}) + return nil +} + +// +// Handling of --http-allow-url option +// +type HTTPUrlRegexp struct { + regexp *regexp.Regexp +} + +func (r *HTTPUrlRegexp) String() string { + if r.regexp == nil { + return "" + } + return r.regexp.String() +} + +func (r *HTTPUrlRegexp) Set(value string) error { + regexp, err := regexp.Compile(value) + r.regexp = regexp + return err +} diff --git a/http_modifier_settings_test.go b/http_modifier_settings_test.go new file mode 100644 index 0000000..badf30e --- /dev/null +++ b/http_modifier_settings_test.go @@ -0,0 +1,68 @@ +package main + +import ( + "testing" +) + +func TestHTTPHeaderFilters(t *testing.T) { + filters := HTTPHeaderFilters{} + + err := filters.Set("Header1:^$") + if err != nil { + t.Error("Should not error on Header1:^$") + } + + err = filters.Set("Header2:^:$") + if err != nil { + t.Error("Should not error on Header2:^:$") + } + + // Missing colon + err = filters.Set("Header3-^$") + if err == nil { + t.Error("Should error on Header2:^:$") + } +} + +func TestHTTPHeaderHashFilters(t *testing.T) { + filters := HTTPHeaderHashFilters{} + + err := filters.Set("Header1:1/2") + if err != nil { + t.Error("Should not error on Header1:^$") + } + + err = filters.Set("Header2:1") + if err == nil { + t.Error("Should error on Header2:^:$") + } +} + +func TestHTTPMethods(t *testing.T) { + methods := HTTPMethods{} + + methods.Set("GET") + methods.Set("POST") + + if !methods.Contains([]byte("GET")) { + t.Error("Does not contain GET") + } + + if !methods.Contains([]byte("POST")) { + t.Error("Does not contain POST") + } +} + +func TestUrlRewriteMap(t *testing.T) { + var err error + rewrites := UrlRewriteMap{} + + if err = rewrites.Set("/v1/user/([^\\/]+)/ping:/v2/user/$1/ping"); err != nil { + t.Error("Should set mapping", err) + } + + if err = rewrites.Set("/v1/user/([^\\/]+)/ping"); err == nil { + t.Error("Should not set mapping without :") + } +} + diff --git a/settings.go b/settings.go index 3b6a7d0..f540cf1 100644 --- a/settings.go +++ b/settings.go @@ -11,6 +11,19 @@ const ( VERSION = "0.9.4" ) +// Allows to specify multiple flags with same name and collects all values to array +type MultiOption []string + +func (h *MultiOption) String() string { + return fmt.Sprint(*h) +} + +func (h *MultiOption) Set(value string) error { + *h = append(*h, value) + return nil +} + + type AppSettings struct { verbose bool stats bool @@ -69,15 +82,32 @@ func init() { flag.Var(&Settings.outputHTTP, "output-http", "Forwards incoming requests to given http address.\n\t# Redirect all incoming requests to staging.com address \n\tgor --input-raw :80 --output-http http://staging.com") flag.IntVar(&Settings.outputHTTPConfig.workers, "output-http-workers", 0, "Gor uses dynamic worker scaling by default. Enter a number to run a set number of workers.") flag.IntVar(&Settings.outputHTTPConfig.redirectLimit, "output-http-redirects", 0, "Enable how often redirects should be followed.") - flag.Var(&Settings.modifierConfig.headers, "output-http-header", "Inject additional headers to http reqest:\n\tgor --input-raw :8080 --output-http staging.com --output-http-header 'User-Agent: Gor'") - flag.Var(&Settings.modifierConfig.methods, "output-http-method", "Whitelist of HTTP methods to replay. Anything else will be dropped:\n\tgor --input-raw :8080 --output-http staging.com --output-http-method GET --output-http-method OPTIONS") - flag.Var(&Settings.modifierConfig.urlRegexp, "output-http-url-regexp", "A regexp to match requests against. Anything else will be dropped:\n\t gor --input-raw :8080 --output-http staging.com --output-http-url-regexp ^www.") - flag.Var(&Settings.modifierConfig.urlRewrite, "output-http-rewrite-url", "Rewrite the requst url based on a mapping:\n\tgor --input-raw :8080 --output-http staging.com --output-http-rewrite-url /xml_test/interface.php:/api/service.do") - flag.Var(&Settings.modifierConfig.headerFilters, "output-http-header-filter", "A regexp to match a specific header against. Requests with non-matching headers will be dropped:\n\t gor --input-raw :8080 --output-http staging.com --output-http-header-filter api-version:^v1") - flag.Var(&Settings.modifierConfig.headerHashFilters, "output-http-header-hash-filter", "Takes a fraction of requests, consistently taking or rejecting a request based on the FNV32-1A hash of a specific header. The fraction must have a denominator that is a power of two:\n\t gor --input-raw :8080 --output-http staging.com --output-http-header-hash-filter user-id:1/4") + flag.BoolVar(&Settings.outputHTTPConfig.stats, "output-http-stats", false, "Report http output queue stats to console every 5 seconds.") flag.StringVar(&Settings.outputHTTPConfig.elasticSearch, "output-http-elasticsearch", "", "Send request and response stats to ElasticSearch:\n\tgor --input-raw :8080 --output-http staging.com --output-http-elasticsearch 'es_host:api_port/index_name'") + + flag.Var(&Settings.modifierConfig.headers, "http-set-header", "Inject additional headers to http reqest:\n\tgor --input-raw :8080 --output-http staging.com --http-set-header 'User-Agent: Gor'") + flag.Var(&Settings.modifierConfig.headers, "output-http-header", "WARNING: `--output-http-header` DEPRECATED, use `--http-set-header` instead") + + + flag.Var(&Settings.modifierConfig.methods, "http-allow-method", "Whitelist of HTTP methods to replay. Anything else will be dropped:\n\tgor --input-raw :8080 --output-http staging.com --http-allow-method GET --http-allow-method OPTIONS") + flag.Var(&Settings.modifierConfig.methods, "output-http-method", "WARNING: `--output-http-method` DEPRECATED, use `--http-allow-method` instead") + + + flag.Var(&Settings.modifierConfig.urlRegexp, "http-allow-url", "A regexp to match requests against. Filter get matched agains full url with domain. Anything else will be dropped:\n\t gor --input-raw :8080 --output-http staging.com --http-filter-url ^www.") + flag.Var(&Settings.modifierConfig.urlRegexp, "output-http-url-regexp", "WARNING: `--output-http-url-regexp` DEPRECATED, use `--http-allow-url` instead") + + + flag.Var(&Settings.modifierConfig.urlRewrite, "http-rewrite-url", "Rewrite the requst url based on a mapping:\n\tgor --input-raw :8080 --output-http staging.com --http-rewrite-url /v1/user/([^\\/]+)/ping:/v2/user/$1/ping") + flag.Var(&Settings.modifierConfig.urlRewrite, "output-http-rewrite-url", "WARNING: `--output-http-rewrite-url` DEPRECATED, use `--http-rewrite-url` instead") + + flag.Var(&Settings.modifierConfig.headerFilters, "http-allow-header", "A regexp to match a specific header against. Requests with non-matching headers will be dropped:\n\t gor --input-raw :8080 --output-http staging.com --http-allow-header api-version:^v1") + flag.Var(&Settings.modifierConfig.headerFilters, "output-http-header-filter", "WARNING: `--output-http-header-filter` DEPRECATED, use `--http-allow-header` instead") + + + flag.Var(&Settings.modifierConfig.headerHashFilters, "http-allow-header-hash", "Takes a fraction of requests, consistently taking or rejecting a request based on the FNV32-1A hash of a specific header:\n\t gor --input-raw :8080 --output-http staging.com --http-allow-header-hash user-id:1/4") + flag.Var(&Settings.modifierConfig.headerHashFilters, "output-http-header-hash-filter", "WARNING: `output-http-header-hash-filter` DEPRECATED, use `--http-allow-header-hash` instead") } func Debug(args ...interface{}) { diff --git a/settings_header_filters.go b/settings_header_filters.go deleted file mode 100644 index bf10aaa..0000000 --- a/settings_header_filters.go +++ /dev/null @@ -1,34 +0,0 @@ -package main - -import ( - "errors" - "fmt" - "regexp" - "strings" -) - -type headerFilter struct { - name []byte - regexp *regexp.Regexp -} - -type HTTPHeaderFilters []headerFilter - -func (h *HTTPHeaderFilters) String() string { - return fmt.Sprint(*h) -} - -func (h *HTTPHeaderFilters) Set(value string) error { - valArr := strings.SplitN(value, ":", 2) - if len(valArr) < 2 { - return errors.New("need both header and value, colon-delimited (ex. user_id:^169$).") - } - r, err := regexp.Compile(valArr[1]) - if err != nil { - return err - } - - *h = append(*h, headerFilter{name: []byte(valArr[0]), regexp: r}) - - return nil -} diff --git a/settings_header_filters_test.go b/settings_header_filters_test.go deleted file mode 100644 index bbffecb..0000000 --- a/settings_header_filters_test.go +++ /dev/null @@ -1,25 +0,0 @@ -package main - -import ( - "testing" -) - -func TestHTTPHeaderFilters(t *testing.T) { - filters := HTTPHeaderFilters{} - - err := filters.Set("Header1:^$") - if err != nil { - t.Error("Should not error on Header1:^$") - } - - err = filters.Set("Header2:^:$") - if err != nil { - t.Error("Should not error on Header2:^:$") - } - - // Missing colon - err = filters.Set("Header3-^$") - if err == nil { - t.Error("Should error on Header2:^:$") - } -} diff --git a/settings_header_hash_filters.go b/settings_header_hash_filters.go deleted file mode 100644 index e8a2072..0000000 --- a/settings_header_hash_filters.go +++ /dev/null @@ -1,42 +0,0 @@ -package main - -import ( - "errors" - "fmt" - "strconv" - "strings" -) - -type headerHashFilter struct { - name []byte - percent uint32 -} - -type HTTPHeaderHashFilters []headerHashFilter - -func (h *HTTPHeaderHashFilters) String() string { - return fmt.Sprint(*h) -} - -func (h *HTTPHeaderHashFilters) Set(value string) error { - valArr := strings.SplitN(value, ":", 2) - if len(valArr) < 2 { - return errors.New("need both header and value, colon-delimited (ex. user_id:1/2).") - } - - fracArr := strings.Split(valArr[1], "/") - if len(fracArr) < 2 { - return errors.New("need both a numerator and denominator specified, slash-delimited (ex. user_id:1/4).") - } - - var num, den uint64 - num, _ = strconv.ParseUint(fracArr[0], 10, 64) - den, _ = strconv.ParseUint(fracArr[1], 10, 64) - - var f headerHashFilter - f.name = []byte(valArr[0]) - f.percent = uint32((float64(num) / float64(den)) * 100) - *h = append(*h, f) - - return nil -} diff --git a/settings_header_hash_filters_test.go b/settings_header_hash_filters_test.go deleted file mode 100644 index 29c34ff..0000000 --- a/settings_header_hash_filters_test.go +++ /dev/null @@ -1,19 +0,0 @@ -package main - -import ( - "testing" -) - -func TestHTTPHeaderHashFilters(t *testing.T) { - filters := HTTPHeaderHashFilters{} - - err := filters.Set("Header1:1/2") - if err != nil { - t.Error("Should not error on Header1:^$") - } - - err = filters.Set("Header2:1") - if err == nil { - t.Error("Should error on Header2:^:$") - } -} diff --git a/settings_headers.go b/settings_headers.go deleted file mode 100644 index 1bee807..0000000 --- a/settings_headers.go +++ /dev/null @@ -1,32 +0,0 @@ -package main - -import ( - "errors" - "fmt" - "strings" -) - -type HTTPHeaders []HTTPHeader -type HTTPHeader struct { - Name string - Value string -} - -func (h *HTTPHeaders) String() string { - return fmt.Sprint(*h) -} - -func (h *HTTPHeaders) Set(value string) error { - v := strings.SplitN(value, ":", 2) - if len(v) != 2 { - return errors.New("Expected `Key: Value`") - } - - header := HTTPHeader{ - strings.TrimSpace(v[0]), - strings.TrimSpace(v[1]), - } - - *h = append(*h, header) - return nil -} diff --git a/settings_methods.go b/settings_methods.go deleted file mode 100644 index 8d84521..0000000 --- a/settings_methods.go +++ /dev/null @@ -1,27 +0,0 @@ -package main - -import ( - "bytes" - "fmt" - "strings" -) - -type HTTPMethods [][]byte - -func (h *HTTPMethods) String() string { - return fmt.Sprint(*h) -} - -func (h *HTTPMethods) Set(value string) error { - *h = append(*h, []byte(strings.ToUpper(value))) - return nil -} - -func (h *HTTPMethods) Contains(value []byte) bool { - for _, method := range *h { - if bytes.Equal(value, method) { - return true - } - } - return false -} diff --git a/settings_methods_test.go b/settings_methods_test.go deleted file mode 100644 index 4c56ed2..0000000 --- a/settings_methods_test.go +++ /dev/null @@ -1,24 +0,0 @@ -package main - -import ( - "testing" -) - -func TestHTTPMethods(t *testing.T) { - methods := HTTPMethods{} - - methods.Set("lower") - methods.Set("UPPER") - - if !methods.Contains([]byte("LOWER")) { - t.Error("Does not contain LOWER") - } - - if !methods.Contains([]byte("UPPER")) { - t.Error("Does not contain UPPER") - } - - if methods.Contains([]byte("ABSENT")) { - t.Error("Does contain ABSENT") - } -} diff --git a/settings_option.go b/settings_option.go deleted file mode 100644 index c14d712..0000000 --- a/settings_option.go +++ /dev/null @@ -1,16 +0,0 @@ -package main - -import ( - "fmt" -) - -type MultiOption []string - -func (h *MultiOption) String() string { - return fmt.Sprint(*h) -} - -func (h *MultiOption) Set(value string) error { - *h = append(*h, value) - return nil -} diff --git a/settings_url_map.go b/settings_url_map.go deleted file mode 100644 index d90ac6f..0000000 --- a/settings_url_map.go +++ /dev/null @@ -1,32 +0,0 @@ -package main - -import ( - "errors" - "fmt" - "regexp" - "strings" -) - -type urlRewrite struct { - src *regexp.Regexp - target []byte -} - -type UrlRewriteMap []urlRewrite - -func (r *UrlRewriteMap) String() string { - return fmt.Sprint(*r) -} - -func (r *UrlRewriteMap) Set(value string) error { - valArr := strings.SplitN(value, ":", 2) - if len(valArr) < 2 { - return errors.New("need both src and target, colon-delimited (ex. /a:/b).") - } - regexp, err := regexp.Compile(valArr[0]) - if err != nil { - return err - } - *r = append(*r, urlRewrite{src: regexp, target: []byte(valArr[1])}) - return nil -} diff --git a/settings_url_map_test.go b/settings_url_map_test.go deleted file mode 100644 index b822387..0000000 --- a/settings_url_map_test.go +++ /dev/null @@ -1,18 +0,0 @@ -package main - -import ( - "testing" -) - -func TestUrlRewriteMap(t *testing.T) { - var err error - rewrites := UrlRewriteMap{} - - if err = rewrites.Set("/v1/user/([^\\/]+)/ping:/v2/user/$1/ping"); err != nil { - t.Error("Should set mapping", err) - } - - if err = rewrites.Set("/v1/user/([^\\/]+)/ping"); err == nil { - t.Error("Should not set mapping without :") - } -} diff --git a/settings_url_regexp.go b/settings_url_regexp.go deleted file mode 100644 index e676860..0000000 --- a/settings_url_regexp.go +++ /dev/null @@ -1,22 +0,0 @@ -package main - -import ( - "regexp" -) - -type HTTPUrlRegexp struct { - regexp *regexp.Regexp -} - -func (r *HTTPUrlRegexp) String() string { - if r.regexp == nil { - return "" - } - return r.regexp.String() -} - -func (r *HTTPUrlRegexp) Set(value string) error { - regexp, err := regexp.Compile(value) - r.regexp = regexp - return err -} diff --git a/settings_url_regexp_test.go b/settings_url_regexp_test.go deleted file mode 100644 index cb4b825..0000000 --- a/settings_url_regexp_test.go +++ /dev/null @@ -1,10 +0,0 @@ -package main - -import ( - "testing" -) - -func TestHTTPUrlRegexp(t *testing.T) { - filter := HTTPUrlRegexp{} - filter.Set("^www.google.com/admin/") -} From 413f50129f3b485a8b2e3224469b3f039491bb89 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Wed, 8 Jul 2015 20:27:35 +0500 Subject: [PATCH 24/30] Add support for restricting certain urls --- http_modifier.go | 19 +++++++++---------- http_modifier_settings.go | 11 +++++++++++ settings.go | 4 +++- 3 files changed, 23 insertions(+), 11 deletions(-) diff --git a/http_modifier.go b/http_modifier.go index 5d2b3e3..b6f994c 100644 --- a/http_modifier.go +++ b/http_modifier.go @@ -5,16 +5,6 @@ import ( "hash/fnv" ) -type HTTPModifierConfig struct { - urlRegexp HTTPUrlRegexp - urlRewrite UrlRewriteMap - headerFilters HTTPHeaderFilters - headerHashFilters HTTPHeaderHashFilters - - headers HTTPHeaders - methods HTTPMethods -} - type HTTPModifier struct { config *HTTPModifierConfig } @@ -47,6 +37,15 @@ func (m *HTTPModifier) Rewrite(payload []byte) (response []byte) { } } + if m.config.urlNegativeRegexp.regexp != nil { + host, _, _, _ := proto.Header(payload, []byte("Host")) + fullPath := append(host, proto.Path(payload)...) + + if m.config.urlNegativeRegexp.regexp.Match(fullPath) { + return + } + } + if len(m.config.headerFilters) > 0 { for _, f := range m.config.headerFilters { value, s, _, _ := proto.Header(payload, f.name) diff --git a/http_modifier_settings.go b/http_modifier_settings.go index 52a8ac7..5ea8601 100644 --- a/http_modifier_settings.go +++ b/http_modifier_settings.go @@ -9,6 +9,17 @@ import ( "bytes" ) +type HTTPModifierConfig struct { + urlNegativeRegexp HTTPUrlRegexp + urlRegexp HTTPUrlRegexp + urlRewrite UrlRewriteMap + headerFilters HTTPHeaderFilters + headerHashFilters HTTPHeaderHashFilters + + headers HTTPHeaders + methods HTTPMethods +} + // // Handling of --http-allow-header options // diff --git a/settings.go b/settings.go index f540cf1..58b8400 100644 --- a/settings.go +++ b/settings.go @@ -95,9 +95,11 @@ func init() { flag.Var(&Settings.modifierConfig.methods, "output-http-method", "WARNING: `--output-http-method` DEPRECATED, use `--http-allow-method` instead") - flag.Var(&Settings.modifierConfig.urlRegexp, "http-allow-url", "A regexp to match requests against. Filter get matched agains full url with domain. Anything else will be dropped:\n\t gor --input-raw :8080 --output-http staging.com --http-filter-url ^www.") + flag.Var(&Settings.modifierConfig.urlRegexp, "http-allow-url", "A regexp to match requests against. Filter get matched agains full url with domain. Anything else will be dropped:\n\t gor --input-raw :8080 --output-http staging.com --http-allow-url ^www.") flag.Var(&Settings.modifierConfig.urlRegexp, "output-http-url-regexp", "WARNING: `--output-http-url-regexp` DEPRECATED, use `--http-allow-url` instead") + flag.Var(&Settings.modifierConfig.urlNegativeRegexp, "http-diallow-url", "A regexp to match requests against. Filter get matched agains full url with domain. Anything else will be dropped:\n\t gor --input-raw :8080 --output-http staging.com --http-disallow-url ^www.") + flag.Var(&Settings.modifierConfig.urlRewrite, "http-rewrite-url", "Rewrite the requst url based on a mapping:\n\tgor --input-raw :8080 --output-http staging.com --http-rewrite-url /v1/user/([^\\/]+)/ping:/v2/user/$1/ping") flag.Var(&Settings.modifierConfig.urlRewrite, "output-http-rewrite-url", "WARNING: `--output-http-rewrite-url` DEPRECATED, use `--http-rewrite-url` instead") From 5a23573cfed073d392ecd0d9d88a315775c8373f Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Thu, 9 Jul 2015 15:30:06 +0500 Subject: [PATCH 25/30] Add --http-allow-param-hash --- http_modifier.go | 26 ++++++++++++++++++++------ http_modifier_settings.go | 15 ++++++++------- http_modifier_settings_test.go | 4 ++-- http_modifier_test.go | 32 +++++++++++++++++++++++++++++--- proto/proto.go | 18 ++++++++++++++++++ proto/proto_test.go | 15 +++++++++++++++ settings.go | 2 ++ 7 files changed, 94 insertions(+), 18 deletions(-) diff --git a/http_modifier.go b/http_modifier.go index b6f994c..3aa0476 100644 --- a/http_modifier.go +++ b/http_modifier.go @@ -15,6 +15,7 @@ func NewHTTPModifier(config *HTTPModifierConfig) *HTTPModifier { len(config.urlRewrite) == 0 && len(config.headerFilters) == 0 && len(config.headerHashFilters) == 0 && + len(config.paramHashFilters) == 0 && len(config.headers) == 0 && len(config.methods) == 0 { return nil @@ -60,15 +61,28 @@ func (m *HTTPModifier) Rewrite(payload []byte) (response []byte) { for _, f := range m.config.headerHashFilters { value, s, _, _ := proto.Header(payload, f.name) - if s == -1 { - return + if s != -1 { + hasher := fnv.New32a() + hasher.Write(value) + + if (hasher.Sum32() % 100) >= f.percent { + return + } } + } + } - hasher := fnv.New32a() - hasher.Write(value) + if len(m.config.paramHashFilters) > 0 { + for _, f := range m.config.paramHashFilters { + value, s, _ := proto.PathParam(payload, f.name) - if (hasher.Sum32() % 100) >= f.percent { - return + if s != -1 { + hasher := fnv.New32a() + hasher.Write(value) + + if (hasher.Sum32() % 100) >= f.percent { + return + } } } } diff --git a/http_modifier_settings.go b/http_modifier_settings.go index 5ea8601..391854a 100644 --- a/http_modifier_settings.go +++ b/http_modifier_settings.go @@ -14,7 +14,8 @@ type HTTPModifierConfig struct { urlRegexp HTTPUrlRegexp urlRewrite UrlRewriteMap headerFilters HTTPHeaderFilters - headerHashFilters HTTPHeaderHashFilters + headerHashFilters HTTPHashFilters + paramHashFilters HTTPHashFilters headers HTTPHeaders methods HTTPMethods @@ -50,20 +51,20 @@ func (h *HTTPHeaderFilters) Set(value string) error { } // -// Handling of --http-allow-header-hash options +// Handling of --http-allow-header-hash and --http-allow-param-hash options // -type headerHashFilter struct { +type hashFilter struct { name []byte percent uint32 } -type HTTPHeaderHashFilters []headerHashFilter +type HTTPHashFilters []hashFilter -func (h *HTTPHeaderHashFilters) String() string { +func (h *HTTPHashFilters) String() string { return fmt.Sprint(*h) } -func (h *HTTPHeaderHashFilters) Set(value string) error { +func (h *HTTPHashFilters) Set(value string) error { valArr := strings.SplitN(value, ":", 2) if len(valArr) < 2 { return errors.New("need both header and value, colon-delimited (ex. user_id:1/2).") @@ -78,7 +79,7 @@ func (h *HTTPHeaderHashFilters) Set(value string) error { num, _ = strconv.ParseUint(fracArr[0], 10, 64) den, _ = strconv.ParseUint(fracArr[1], 10, 64) - var f headerHashFilter + var f hashFilter f.name = []byte(valArr[0]) f.percent = uint32((float64(num) / float64(den)) * 100) *h = append(*h, f) diff --git a/http_modifier_settings_test.go b/http_modifier_settings_test.go index badf30e..0b1701d 100644 --- a/http_modifier_settings_test.go +++ b/http_modifier_settings_test.go @@ -24,8 +24,8 @@ func TestHTTPHeaderFilters(t *testing.T) { } } -func TestHTTPHeaderHashFilters(t *testing.T) { - filters := HTTPHeaderHashFilters{} +func TestHTTPHashFilters(t *testing.T) { + filters := HTTPHashFilters{} err := filters.Set("Header1:1/2") if err != nil { diff --git a/http_modifier_test.go b/http_modifier_test.go index 4689aca..80a3bb3 100644 --- a/http_modifier_test.go +++ b/http_modifier_test.go @@ -69,7 +69,7 @@ func TestHTTPModifierURLRewrite(t *testing.T) { } func TestHTTPModifierHeaderHashFilters(t *testing.T) { - filters := HTTPHeaderHashFilters{} + filters := HTTPHashFilters{} filters.Set("Header2:1/2") modifier := NewHTTPModifier(&HTTPModifierConfig{ @@ -80,8 +80,8 @@ func TestHTTPModifierHeaderHashFilters(t *testing.T) { return []byte("POST / HTTP/1.1\r\n" + string(header) + "Content-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") } - if p := modifier.Rewrite(payload([]byte(""))); len(p) > 0 { - t.Error("Request should not pass filters, Header2 does not exist") + if p := modifier.Rewrite(payload([]byte(""))); len(p) == 0 { + t.Error("Request should pass filters if Header does not exist") } if p := modifier.Rewrite(payload([]byte("Header2: 3\r\n"))); len(p) > 0 { @@ -93,6 +93,32 @@ func TestHTTPModifierHeaderHashFilters(t *testing.T) { } } + +func TestHTTPModifierParamHashFilters(t *testing.T) { + filters := HTTPHashFilters{} + filters.Set("user_id:1/2") + + modifier := NewHTTPModifier(&HTTPModifierConfig{ + paramHashFilters: filters, + }) + + payload := func(value []byte) []byte { + return []byte("POST /" + string(value) + " HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") + } + + if p := modifier.Rewrite(payload([]byte(""))); len(p) == 0 { + t.Error("Request should pass filters if param does not exist") + } + + if p := modifier.Rewrite(payload([]byte("?user_id=3"))); len(p) > 0 { + t.Error("Request should not pass filters", string(p)) + } + + if p := modifier.Rewrite(payload([]byte("?user_id=1"))); len(p) == 0 { + t.Error("Request should pass filters") + } +} + func TestHTTPModifierHeaders(t *testing.T) { headers := HTTPHeaders{} headers.Set("Header1:1") diff --git a/proto/proto.go b/proto/proto.go index 330115b..9b0e5cf 100644 --- a/proto/proto.go +++ b/proto/proto.go @@ -86,6 +86,24 @@ func SetPath(payload, path []byte) []byte { return byteutils.Replace(payload, start, start+end, path) } +func PathParam(payload, name []byte) (value []byte, valueStart, valueEnd int) { + path := Path(payload) + + if paramStart := bytes.Index(path, append(name, '=')); paramStart != -1 { + valueStart := paramStart + len(name) + 1 + paramEnd := bytes.IndexByte(path[valueStart:], '&') + if paramEnd == -1 { // It is final param + paramEnd = len(path) + } else { + paramEnd += valueStart + } + + return path[valueStart:paramEnd], valueStart, paramEnd + } else { + return []byte(""), -1, -1 + } +} + func SetHost(payload, url, host []byte) []byte { // If this is HTTP 1.0 traffic or proxy traffic it may include host right into path variable, so instead of setting Host header we rewrite Path // Fix for https://github.com/buger/gor/issues/156 diff --git a/proto/proto_test.go b/proto/proto_test.go index 3ada427..12227e2 100644 --- a/proto/proto_test.go +++ b/proto/proto_test.go @@ -89,6 +89,21 @@ func TestSetPath(t *testing.T) { } } +func TestPathParam(t *testing.T) { + var payload []byte + + payload = []byte("POST /post?param=test&user_id=1 HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") + + if val, _, _ := PathParam(payload, []byte("param")); !bytes.Equal(val, []byte("test")) { + t.Error("Should detect attribute", string(val)) + } + + if val, _, _ := PathParam(payload, []byte("user_id")); !bytes.Equal(val, []byte("1")) { + t.Error("Should detect attribute", string(val)) + } +} + + func TestSetHostHTTP10(t *testing.T) { var payload, payload_after []byte diff --git a/settings.go b/settings.go index 58b8400..aa9cb6d 100644 --- a/settings.go +++ b/settings.go @@ -110,6 +110,8 @@ func init() { flag.Var(&Settings.modifierConfig.headerHashFilters, "http-allow-header-hash", "Takes a fraction of requests, consistently taking or rejecting a request based on the FNV32-1A hash of a specific header:\n\t gor --input-raw :8080 --output-http staging.com --http-allow-header-hash user-id:1/4") flag.Var(&Settings.modifierConfig.headerHashFilters, "output-http-header-hash-filter", "WARNING: `output-http-header-hash-filter` DEPRECATED, use `--http-allow-header-hash` instead") + + flag.Var(&Settings.modifierConfig.paramHashFilters, "http-allow-param-hash", "Takes a fraction of requests, consistently taking or rejecting a request based on the FNV32-1A hash of a specific GET param:\n\t gor --input-raw :8080 --output-http staging.com --http-allow-param-hash user_id:1/4") } func Debug(args ...interface{}) { From 1ce10249beec40580f1425ac96210e053c81d8e5 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Thu, 9 Jul 2015 15:44:15 +0500 Subject: [PATCH 26/30] Percent based syntax for hash filters --- http_modifier_settings.go | 28 +++++++++++++++++----------- http_modifier_settings_test.go | 17 +++++++++++++++-- settings.go | 4 ++-- 3 files changed, 34 insertions(+), 15 deletions(-) diff --git a/http_modifier_settings.go b/http_modifier_settings.go index 391854a..a3a9148 100644 --- a/http_modifier_settings.go +++ b/http_modifier_settings.go @@ -67,21 +67,27 @@ func (h *HTTPHashFilters) String() string { func (h *HTTPHashFilters) Set(value string) error { valArr := strings.SplitN(value, ":", 2) if len(valArr) < 2 { - return errors.New("need both header and value, colon-delimited (ex. user_id:1/2).") + return errors.New("need both header and value, colon-delimited (ex. user_id:50%).") } - fracArr := strings.Split(valArr[1], "/") - if len(fracArr) < 2 { - return errors.New("need both a numerator and denominator specified, slash-delimited (ex. user_id:1/4).") + f := hashFilter{ name: []byte(valArr[0]) } + + if strings.Contains(valArr[1], "%") { + p, _ := strconv.ParseInt(valArr[1][:len(valArr[1])-1], 0, 0) + f.percent = uint32(p) + } else if strings.Contains(valArr[1], "/") { + // DEPRECATED format + var num, den uint64 + + fracArr := strings.Split(valArr[1], "/") + num, _ = strconv.ParseUint(fracArr[0], 10, 64) + den, _ = strconv.ParseUint(fracArr[1], 10, 64) + + f.percent = uint32((float64(num) / float64(den)) * 100) + } else { + return errors.New("Value should be percent and contain '%'") } - var num, den uint64 - num, _ = strconv.ParseUint(fracArr[0], 10, 64) - den, _ = strconv.ParseUint(fracArr[1], 10, 64) - - var f hashFilter - f.name = []byte(valArr[0]) - f.percent = uint32((float64(num) / float64(den)) * 100) *h = append(*h, f) return nil diff --git a/http_modifier_settings_test.go b/http_modifier_settings_test.go index 0b1701d..1298115 100644 --- a/http_modifier_settings_test.go +++ b/http_modifier_settings_test.go @@ -29,12 +29,25 @@ func TestHTTPHashFilters(t *testing.T) { err := filters.Set("Header1:1/2") if err != nil { - t.Error("Should not error on Header1:^$") + t.Error("Should support old syntax") + } + + if filters[0].percent != 50 { + t.Error("Wrong percentage", filters[0].percent) } err = filters.Set("Header2:1") if err == nil { - t.Error("Should error on Header2:^:$") + t.Error("Should error on Header2 because no % symbol") + } + + err = filters.Set("Header2:10%") + if err != nil { + t.Error("Should pass") + } + + if filters[1].percent != 10 { + t.Error("Wrong percentage", filters[1].percent) } } diff --git a/settings.go b/settings.go index aa9cb6d..c39a2a1 100644 --- a/settings.go +++ b/settings.go @@ -108,10 +108,10 @@ func init() { flag.Var(&Settings.modifierConfig.headerFilters, "output-http-header-filter", "WARNING: `--output-http-header-filter` DEPRECATED, use `--http-allow-header` instead") - flag.Var(&Settings.modifierConfig.headerHashFilters, "http-allow-header-hash", "Takes a fraction of requests, consistently taking or rejecting a request based on the FNV32-1A hash of a specific header:\n\t gor --input-raw :8080 --output-http staging.com --http-allow-header-hash user-id:1/4") + flag.Var(&Settings.modifierConfig.headerHashFilters, "http-allow-header-hash", "Takes a fraction of requests, consistently taking or rejecting a request based on the FNV32-1A hash of a specific header:\n\t gor --input-raw :8080 --output-http staging.com --http-allow-header-hash user-id:25%") flag.Var(&Settings.modifierConfig.headerHashFilters, "output-http-header-hash-filter", "WARNING: `output-http-header-hash-filter` DEPRECATED, use `--http-allow-header-hash` instead") - flag.Var(&Settings.modifierConfig.paramHashFilters, "http-allow-param-hash", "Takes a fraction of requests, consistently taking or rejecting a request based on the FNV32-1A hash of a specific GET param:\n\t gor --input-raw :8080 --output-http staging.com --http-allow-param-hash user_id:1/4") + flag.Var(&Settings.modifierConfig.paramHashFilters, "http-allow-param-hash", "Takes a fraction of requests, consistently taking or rejecting a request based on the FNV32-1A hash of a specific GET param:\n\t gor --input-raw :8080 --output-http staging.com --http-allow-param-hash user_id:25%") } func Debug(args ...interface{}) { From 29035d4518fe06c506816deffca34baf3bd86ad1 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Thu, 9 Jul 2015 16:05:00 +0500 Subject: [PATCH 27/30] Allow multiple url regexp and apply fmt --- http_client.go | 1 - http_modifier.go | 29 +++-- http_modifier_settings.go | 190 ++++++++++++++++----------------- http_modifier_settings_test.go | 103 +++++++++--------- http_modifier_test.go | 53 ++++++++- input_tcp.go | 4 +- input_tcp_test.go | 4 +- output_tcp.go | 4 +- output_tcp_test.go | 2 +- settings.go | 5 - 10 files changed, 224 insertions(+), 171 deletions(-) diff --git a/http_client.go b/http_client.go index 50237ec..4e44f77 100644 --- a/http_client.go +++ b/http_client.go @@ -40,7 +40,6 @@ func NewHTTPClient(baseURL string, config *HTTPClientConfig) *HTTPClient { u.Host += ":" + defaultPorts[u.Scheme] } - client := new(HTTPClient) client.baseURL = u.String() client.host = u.Host diff --git a/http_modifier.go b/http_modifier.go index 3aa0476..5023008 100644 --- a/http_modifier.go +++ b/http_modifier.go @@ -11,7 +11,8 @@ type HTTPModifier struct { func NewHTTPModifier(config *HTTPModifierConfig) *HTTPModifier { // Optimization to skip modifier completely if we do not need it - if config.urlRegexp.regexp == nil && + if len(config.urlRegexp) == 0 && + len(config.urlNegativeRegexp) == 0 && len(config.urlRewrite) == 0 && len(config.headerFilters) == 0 && len(config.headerHashFilters) == 0 && @@ -29,21 +30,29 @@ func (m *HTTPModifier) Rewrite(payload []byte) (response []byte) { return } - if m.config.urlRegexp.regexp != nil { - host, _, _, _ := proto.Header(payload, []byte("Host")) - fullPath := append(host, proto.Path(payload)...) + if len(m.config.urlRegexp) > 0 { + path := proto.Path(payload) - if !m.config.urlRegexp.regexp.Match(fullPath) { + matched := false + + for _, f := range m.config.urlRegexp { + if f.regexp.Match(path) { + matched = true + } + } + + if !matched { return } } - if m.config.urlNegativeRegexp.regexp != nil { - host, _, _, _ := proto.Header(payload, []byte("Host")) - fullPath := append(host, proto.Path(payload)...) + if len(m.config.urlNegativeRegexp) > 0 { + path := proto.Path(payload) - if m.config.urlNegativeRegexp.regexp.Match(fullPath) { - return + for _, f := range m.config.urlNegativeRegexp { + if f.regexp.Match(path) { + return + } } } diff --git a/http_modifier_settings.go b/http_modifier_settings.go index a3a9148..cfbc82e 100644 --- a/http_modifier_settings.go +++ b/http_modifier_settings.go @@ -1,125 +1,124 @@ package main import ( - "errors" - "fmt" - "regexp" - "strings" - "strconv" - "bytes" + "bytes" + "errors" + "fmt" + "regexp" + "strconv" + "strings" ) type HTTPModifierConfig struct { - urlNegativeRegexp HTTPUrlRegexp - urlRegexp HTTPUrlRegexp - urlRewrite UrlRewriteMap - headerFilters HTTPHeaderFilters - headerHashFilters HTTPHashFilters - paramHashFilters HTTPHashFilters + urlNegativeRegexp HTTPUrlRegexp + urlRegexp HTTPUrlRegexp + urlRewrite UrlRewriteMap + headerFilters HTTPHeaderFilters + headerHashFilters HTTPHashFilters + paramHashFilters HTTPHashFilters - headers HTTPHeaders - methods HTTPMethods + headers HTTPHeaders + methods HTTPMethods } // // Handling of --http-allow-header options // type headerFilter struct { - name []byte - regexp *regexp.Regexp + name []byte + regexp *regexp.Regexp } type HTTPHeaderFilters []headerFilter func (h *HTTPHeaderFilters) String() string { - return fmt.Sprint(*h) + return fmt.Sprint(*h) } func (h *HTTPHeaderFilters) Set(value string) error { - valArr := strings.SplitN(value, ":", 2) - if len(valArr) < 2 { - return errors.New("need both header and value, colon-delimited (ex. user_id:^169$).") - } - r, err := regexp.Compile(valArr[1]) - if err != nil { - return err - } + valArr := strings.SplitN(value, ":", 2) + if len(valArr) < 2 { + return errors.New("need both header and value, colon-delimited (ex. user_id:^169$).") + } + r, err := regexp.Compile(valArr[1]) + if err != nil { + return err + } - *h = append(*h, headerFilter{name: []byte(valArr[0]), regexp: r}) + *h = append(*h, headerFilter{name: []byte(valArr[0]), regexp: r}) - return nil + return nil } // // Handling of --http-allow-header-hash and --http-allow-param-hash options // type hashFilter struct { - name []byte - percent uint32 + name []byte + percent uint32 } type HTTPHashFilters []hashFilter func (h *HTTPHashFilters) String() string { - return fmt.Sprint(*h) + return fmt.Sprint(*h) } func (h *HTTPHashFilters) Set(value string) error { - valArr := strings.SplitN(value, ":", 2) - if len(valArr) < 2 { - return errors.New("need both header and value, colon-delimited (ex. user_id:50%).") - } + valArr := strings.SplitN(value, ":", 2) + if len(valArr) < 2 { + return errors.New("need both header and value, colon-delimited (ex. user_id:50%).") + } - f := hashFilter{ name: []byte(valArr[0]) } + f := hashFilter{name: []byte(valArr[0])} - if strings.Contains(valArr[1], "%") { - p, _ := strconv.ParseInt(valArr[1][:len(valArr[1])-1], 0, 0) - f.percent = uint32(p) - } else if strings.Contains(valArr[1], "/") { - // DEPRECATED format - var num, den uint64 + if strings.Contains(valArr[1], "%") { + p, _ := strconv.ParseInt(valArr[1][:len(valArr[1])-1], 0, 0) + f.percent = uint32(p) + } else if strings.Contains(valArr[1], "/") { + // DEPRECATED format + var num, den uint64 - fracArr := strings.Split(valArr[1], "/") - num, _ = strconv.ParseUint(fracArr[0], 10, 64) - den, _ = strconv.ParseUint(fracArr[1], 10, 64) + fracArr := strings.Split(valArr[1], "/") + num, _ = strconv.ParseUint(fracArr[0], 10, 64) + den, _ = strconv.ParseUint(fracArr[1], 10, 64) - f.percent = uint32((float64(num) / float64(den)) * 100) - } else { - return errors.New("Value should be percent and contain '%'") - } + f.percent = uint32((float64(num) / float64(den)) * 100) + } else { + return errors.New("Value should be percent and contain '%'") + } - *h = append(*h, f) + *h = append(*h, f) - return nil + return nil } - // // Handling of --http-set-header option // type HTTPHeaders []HTTPHeader type HTTPHeader struct { - Name string - Value string + Name string + Value string } func (h *HTTPHeaders) String() string { - return fmt.Sprint(*h) + return fmt.Sprint(*h) } func (h *HTTPHeaders) Set(value string) error { - v := strings.SplitN(value, ":", 2) - if len(v) != 2 { - return errors.New("Expected `Key: Value`") - } + v := strings.SplitN(value, ":", 2) + if len(v) != 2 { + return errors.New("Expected `Key: Value`") + } - header := HTTPHeader{ - strings.TrimSpace(v[0]), - strings.TrimSpace(v[1]), - } + header := HTTPHeader{ + strings.TrimSpace(v[0]), + strings.TrimSpace(v[1]), + } - *h = append(*h, header) - return nil + *h = append(*h, header) + return nil } // @@ -128,66 +127,67 @@ func (h *HTTPHeaders) Set(value string) error { type HTTPMethods [][]byte func (h *HTTPMethods) String() string { - return fmt.Sprint(*h) + return fmt.Sprint(*h) } func (h *HTTPMethods) Set(value string) error { - *h = append(*h, []byte(value)) - return nil + *h = append(*h, []byte(value)) + return nil } func (h *HTTPMethods) Contains(value []byte) bool { - for _, method := range *h { - if bytes.Equal(value, method) { - return true - } - } - return false + for _, method := range *h { + if bytes.Equal(value, method) { + return true + } + } + return false } // // Handling of --http-rewrite-url option // type urlRewrite struct { - src *regexp.Regexp - target []byte + src *regexp.Regexp + target []byte } type UrlRewriteMap []urlRewrite func (r *UrlRewriteMap) String() string { - return fmt.Sprint(*r) + return fmt.Sprint(*r) } func (r *UrlRewriteMap) Set(value string) error { - valArr := strings.SplitN(value, ":", 2) - if len(valArr) < 2 { - return errors.New("need both src and target, colon-delimited (ex. /a:/b).") - } - regexp, err := regexp.Compile(valArr[0]) - if err != nil { - return err - } - *r = append(*r, urlRewrite{src: regexp, target: []byte(valArr[1])}) - return nil + valArr := strings.SplitN(value, ":", 2) + if len(valArr) < 2 { + return errors.New("need both src and target, colon-delimited (ex. /a:/b).") + } + regexp, err := regexp.Compile(valArr[0]) + if err != nil { + return err + } + *r = append(*r, urlRewrite{src: regexp, target: []byte(valArr[1])}) + return nil } // // Handling of --http-allow-url option // -type HTTPUrlRegexp struct { - regexp *regexp.Regexp +type urlRegexp struct { + regexp *regexp.Regexp } +type HTTPUrlRegexp []urlRegexp + func (r *HTTPUrlRegexp) String() string { - if r.regexp == nil { - return "" - } - return r.regexp.String() + return fmt.Sprint(*r) } func (r *HTTPUrlRegexp) Set(value string) error { - regexp, err := regexp.Compile(value) - r.regexp = regexp - return err + regexp, err := regexp.Compile(value) + + *r = append(*r, urlRegexp{regexp: regexp}) + + return err } diff --git a/http_modifier_settings_test.go b/http_modifier_settings_test.go index 1298115..5e135b6 100644 --- a/http_modifier_settings_test.go +++ b/http_modifier_settings_test.go @@ -1,81 +1,80 @@ package main import ( - "testing" + "testing" ) func TestHTTPHeaderFilters(t *testing.T) { - filters := HTTPHeaderFilters{} + filters := HTTPHeaderFilters{} - err := filters.Set("Header1:^$") - if err != nil { - t.Error("Should not error on Header1:^$") - } + err := filters.Set("Header1:^$") + if err != nil { + t.Error("Should not error on Header1:^$") + } - err = filters.Set("Header2:^:$") - if err != nil { - t.Error("Should not error on Header2:^:$") - } + err = filters.Set("Header2:^:$") + if err != nil { + t.Error("Should not error on Header2:^:$") + } - // Missing colon - err = filters.Set("Header3-^$") - if err == nil { - t.Error("Should error on Header2:^:$") - } + // Missing colon + err = filters.Set("Header3-^$") + if err == nil { + t.Error("Should error on Header2:^:$") + } } func TestHTTPHashFilters(t *testing.T) { - filters := HTTPHashFilters{} + filters := HTTPHashFilters{} - err := filters.Set("Header1:1/2") - if err != nil { - t.Error("Should support old syntax") - } + err := filters.Set("Header1:1/2") + if err != nil { + t.Error("Should support old syntax") + } - if filters[0].percent != 50 { - t.Error("Wrong percentage", filters[0].percent) - } + if filters[0].percent != 50 { + t.Error("Wrong percentage", filters[0].percent) + } - err = filters.Set("Header2:1") - if err == nil { - t.Error("Should error on Header2 because no % symbol") - } + err = filters.Set("Header2:1") + if err == nil { + t.Error("Should error on Header2 because no % symbol") + } - err = filters.Set("Header2:10%") - if err != nil { - t.Error("Should pass") - } + err = filters.Set("Header2:10%") + if err != nil { + t.Error("Should pass") + } - if filters[1].percent != 10 { - t.Error("Wrong percentage", filters[1].percent) - } + if filters[1].percent != 10 { + t.Error("Wrong percentage", filters[1].percent) + } } func TestHTTPMethods(t *testing.T) { - methods := HTTPMethods{} + methods := HTTPMethods{} - methods.Set("GET") - methods.Set("POST") + methods.Set("GET") + methods.Set("POST") - if !methods.Contains([]byte("GET")) { - t.Error("Does not contain GET") - } + if !methods.Contains([]byte("GET")) { + t.Error("Does not contain GET") + } - if !methods.Contains([]byte("POST")) { - t.Error("Does not contain POST") - } + if !methods.Contains([]byte("POST")) { + t.Error("Does not contain POST") + } } func TestUrlRewriteMap(t *testing.T) { - var err error - rewrites := UrlRewriteMap{} + var err error + rewrites := UrlRewriteMap{} - if err = rewrites.Set("/v1/user/([^\\/]+)/ping:/v2/user/$1/ping"); err != nil { - t.Error("Should set mapping", err) - } + if err = rewrites.Set("/v1/user/([^\\/]+)/ping:/v2/user/$1/ping"); err != nil { + t.Error("Should set mapping", err) + } - if err = rewrites.Set("/v1/user/([^\\/]+)/ping"); err == nil { - t.Error("Should not set mapping without :") - } + if err = rewrites.Set("/v1/user/([^\\/]+)/ping"); err == nil { + t.Error("Should not set mapping without :") + } } - diff --git a/http_modifier_test.go b/http_modifier_test.go index 80a3bb3..55d9a19 100644 --- a/http_modifier_test.go +++ b/http_modifier_test.go @@ -93,7 +93,6 @@ func TestHTTPModifierHeaderHashFilters(t *testing.T) { } } - func TestHTTPModifierParamHashFilters(t *testing.T) { filters := HTTPHashFilters{} filters.Set("user_id:1/2") @@ -135,3 +134,55 @@ func TestHTTPModifierHeaders(t *testing.T) { t.Error("Should update request headers", string(payload)) } } + +func TestHTTPModifierURLRegexp(t *testing.T) { + filters := HTTPUrlRegexp{} + filters.Set("/v1/app") + filters.Set("/v1/api") + + modifier := NewHTTPModifier(&HTTPModifierConfig{ + urlRegexp: filters, + }) + + payload := func(url string) []byte { + return []byte("POST " + url + " HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") + } + + if len(modifier.Rewrite(payload("/v1/app/test"))) == 0 { + t.Error("Should pass url") + } + + if len(modifier.Rewrite(payload("/v1/api/test"))) == 0 { + t.Error("Should pass url") + } + + if len(modifier.Rewrite(payload("/other"))) > 0 { + t.Error("Should not pass url") + } +} + +func TestHTTPModifierURLNegativeRegexp(t *testing.T) { + filters := HTTPUrlRegexp{} + filters.Set("/restricted1") + filters.Set("/some/restricted2") + + modifier := NewHTTPModifier(&HTTPModifierConfig{ + urlNegativeRegexp: filters, + }) + + payload := func(url string) []byte { + return []byte("POST " + url + " HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") + } + + if len(modifier.Rewrite(payload("/v1/app/test"))) == 0 { + t.Error("Should pass url") + } + + if len(modifier.Rewrite(payload("/restricted1"))) > 0 { + t.Error("Should not pass url") + } + + if len(modifier.Rewrite(payload("/some/restricted2"))) > 0 { + t.Error("Should not pass url") + } +} diff --git a/input_tcp.go b/input_tcp.go index 2e26c62..27b7de6 100644 --- a/input_tcp.go +++ b/input_tcp.go @@ -2,10 +2,10 @@ package main import ( "bufio" - "log" - "net" "encoding/hex" "fmt" + "log" + "net" "os" ) diff --git a/input_tcp_test.go b/input_tcp_test.go index 0f76e4c..27c00fc 100644 --- a/input_tcp_test.go +++ b/input_tcp_test.go @@ -1,12 +1,12 @@ package main import ( + "encoding/hex" "io" "log" "net" "sync" "testing" - "encoding/hex" ) func TestTCPInput(t *testing.T) { @@ -40,7 +40,7 @@ func TestTCPInput(t *testing.T) { for i := 0; i < 100; i++ { wg.Add(1) - encoded := make([]byte, len(msg)*2 + 1) + encoded := make([]byte, len(msg)*2+1) hex.Encode(encoded, msg) conn.Write(append(encoded, '\n')) } diff --git a/output_tcp.go b/output_tcp.go index fb06025..f6f0e9d 100644 --- a/output_tcp.go +++ b/output_tcp.go @@ -1,12 +1,12 @@ package main import ( + "encoding/hex" "fmt" "io" "log" "net" "time" - "encoding/hex" ) type TCPOutput struct { @@ -53,7 +53,7 @@ func (o *TCPOutput) worker() { func (o *TCPOutput) Write(data []byte) (n int, err error) { // Hex encoding always 2x number of bytes - encoded := make([]byte, len(data)*2 + 1) + encoded := make([]byte, len(data)*2+1) hex.Encode(encoded, data) o.buf <- append(encoded, '\n') diff --git a/output_tcp_test.go b/output_tcp_test.go index e42f14a..84c66b8 100644 --- a/output_tcp_test.go +++ b/output_tcp_test.go @@ -2,12 +2,12 @@ package main import ( "bufio" + "encoding/hex" "io" "log" "net" "sync" "testing" - "encoding/hex" ) func TestTCPOutput(t *testing.T) { diff --git a/settings.go b/settings.go index c39a2a1..8fd2127 100644 --- a/settings.go +++ b/settings.go @@ -23,7 +23,6 @@ func (h *MultiOption) Set(value string) error { return nil } - type AppSettings struct { verbose bool stats bool @@ -90,24 +89,20 @@ func init() { flag.Var(&Settings.modifierConfig.headers, "http-set-header", "Inject additional headers to http reqest:\n\tgor --input-raw :8080 --output-http staging.com --http-set-header 'User-Agent: Gor'") flag.Var(&Settings.modifierConfig.headers, "output-http-header", "WARNING: `--output-http-header` DEPRECATED, use `--http-set-header` instead") - flag.Var(&Settings.modifierConfig.methods, "http-allow-method", "Whitelist of HTTP methods to replay. Anything else will be dropped:\n\tgor --input-raw :8080 --output-http staging.com --http-allow-method GET --http-allow-method OPTIONS") flag.Var(&Settings.modifierConfig.methods, "output-http-method", "WARNING: `--output-http-method` DEPRECATED, use `--http-allow-method` instead") - flag.Var(&Settings.modifierConfig.urlRegexp, "http-allow-url", "A regexp to match requests against. Filter get matched agains full url with domain. Anything else will be dropped:\n\t gor --input-raw :8080 --output-http staging.com --http-allow-url ^www.") flag.Var(&Settings.modifierConfig.urlRegexp, "output-http-url-regexp", "WARNING: `--output-http-url-regexp` DEPRECATED, use `--http-allow-url` instead") flag.Var(&Settings.modifierConfig.urlNegativeRegexp, "http-diallow-url", "A regexp to match requests against. Filter get matched agains full url with domain. Anything else will be dropped:\n\t gor --input-raw :8080 --output-http staging.com --http-disallow-url ^www.") - flag.Var(&Settings.modifierConfig.urlRewrite, "http-rewrite-url", "Rewrite the requst url based on a mapping:\n\tgor --input-raw :8080 --output-http staging.com --http-rewrite-url /v1/user/([^\\/]+)/ping:/v2/user/$1/ping") flag.Var(&Settings.modifierConfig.urlRewrite, "output-http-rewrite-url", "WARNING: `--output-http-rewrite-url` DEPRECATED, use `--http-rewrite-url` instead") flag.Var(&Settings.modifierConfig.headerFilters, "http-allow-header", "A regexp to match a specific header against. Requests with non-matching headers will be dropped:\n\t gor --input-raw :8080 --output-http staging.com --http-allow-header api-version:^v1") flag.Var(&Settings.modifierConfig.headerFilters, "output-http-header-filter", "WARNING: `--output-http-header-filter` DEPRECATED, use `--http-allow-header` instead") - flag.Var(&Settings.modifierConfig.headerHashFilters, "http-allow-header-hash", "Takes a fraction of requests, consistently taking or rejecting a request based on the FNV32-1A hash of a specific header:\n\t gor --input-raw :8080 --output-http staging.com --http-allow-header-hash user-id:25%") flag.Var(&Settings.modifierConfig.headerHashFilters, "output-http-header-hash-filter", "WARNING: `output-http-header-hash-filter` DEPRECATED, use `--http-allow-header-hash` instead") From 48492dc8cb9204a73aa0e5ae87cb389d2a0f4205 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Thu, 9 Jul 2015 16:58:42 +0500 Subject: [PATCH 28/30] Allow url param rewriting --- http_modifier.go | 38 ++++++++++++++++++++++++++------- http_modifier_settings.go | 39 +++++++++++++++++++++++++--------- http_modifier_settings_test.go | 15 ------------- http_modifier_test.go | 32 ++++++++++++++++++++++++++++ proto/proto.go | 32 ++++++++++++++++++++++++++++ proto/proto_test.go | 36 +++++++++++++++++++++++++++++++ settings.go | 4 +++- 7 files changed, 162 insertions(+), 34 deletions(-) diff --git a/http_modifier.go b/http_modifier.go index 5023008..d820df6 100644 --- a/http_modifier.go +++ b/http_modifier.go @@ -1,6 +1,7 @@ package main import ( + "bytes" "github.com/buger/gor/proto" "hash/fnv" ) @@ -17,6 +18,7 @@ func NewHTTPModifier(config *HTTPModifierConfig) *HTTPModifier { len(config.headerFilters) == 0 && len(config.headerHashFilters) == 0 && len(config.paramHashFilters) == 0 && + len(config.params) == 0 && len(config.headers) == 0 && len(config.methods) == 0 { return nil @@ -26,8 +28,33 @@ func NewHTTPModifier(config *HTTPModifierConfig) *HTTPModifier { } func (m *HTTPModifier) Rewrite(payload []byte) (response []byte) { - if len(m.config.methods) > 0 && !m.config.methods.Contains(proto.Method(payload)) { - return + if len(m.config.methods) > 0 { + method := proto.Method(payload) + + matched := false + + for _, m := range m.config.methods { + if bytes.Equal(method, m) { + matched = true + break + } + } + + if !matched { + return + } + } + + if len(m.config.headers) > 0 { + for _, header := range m.config.headers { + payload = proto.SetHeader(payload, []byte(header.Name), []byte(header.Value)) + } + } + + if len(m.config.params) > 0 { + for _, param := range m.config.params { + payload = proto.SetPathParam(payload, param.Name, param.Value) + } } if len(m.config.urlRegexp) > 0 { @@ -38,6 +65,7 @@ func (m *HTTPModifier) Rewrite(payload []byte) (response []byte) { for _, f := range m.config.urlRegexp { if f.regexp.Match(path) { matched = true + break } } @@ -109,11 +137,5 @@ func (m *HTTPModifier) Rewrite(payload []byte) (response []byte) { } } - if len(m.config.headers) > 0 { - for _, header := range m.config.headers { - payload = proto.SetHeader(payload, []byte(header.Name), []byte(header.Value)) - } - } - return payload } diff --git a/http_modifier_settings.go b/http_modifier_settings.go index cfbc82e..854b11a 100644 --- a/http_modifier_settings.go +++ b/http_modifier_settings.go @@ -1,7 +1,6 @@ package main import ( - "bytes" "errors" "fmt" "regexp" @@ -17,6 +16,7 @@ type HTTPModifierConfig struct { headerHashFilters HTTPHashFilters paramHashFilters HTTPHashFilters + params HTTPParams headers HTTPHeaders methods HTTPMethods } @@ -121,6 +121,34 @@ func (h *HTTPHeaders) Set(value string) error { return nil } +// +// Handling of --http-set-param option +// +type HTTPParams []HTTPParam +type HTTPParam struct { + Name []byte + Value []byte +} + +func (h *HTTPParams) String() string { + return fmt.Sprint(*h) +} + +func (h *HTTPParams) Set(value string) error { + v := strings.SplitN(value, "=", 2) + if len(v) != 2 { + return errors.New("Expected `Key=Value`") + } + + param := HTTPParam{ + []byte(strings.TrimSpace(v[0])), + []byte(strings.TrimSpace(v[1])), + } + + *h = append(*h, param) + return nil +} + // // Handling of --http-allow-method option // @@ -135,15 +163,6 @@ func (h *HTTPMethods) Set(value string) error { return nil } -func (h *HTTPMethods) Contains(value []byte) bool { - for _, method := range *h { - if bytes.Equal(value, method) { - return true - } - } - return false -} - // // Handling of --http-rewrite-url option // diff --git a/http_modifier_settings_test.go b/http_modifier_settings_test.go index 5e135b6..60f382a 100644 --- a/http_modifier_settings_test.go +++ b/http_modifier_settings_test.go @@ -51,21 +51,6 @@ func TestHTTPHashFilters(t *testing.T) { } } -func TestHTTPMethods(t *testing.T) { - methods := HTTPMethods{} - - methods.Set("GET") - methods.Set("POST") - - if !methods.Contains([]byte("GET")) { - t.Error("Does not contain GET") - } - - if !methods.Contains([]byte("POST")) { - t.Error("Does not contain POST") - } -} - func TestUrlRewriteMap(t *testing.T) { var err error rewrites := UrlRewriteMap{} diff --git a/http_modifier_test.go b/http_modifier_test.go index 55d9a19..40e5ec4 100644 --- a/http_modifier_test.go +++ b/http_modifier_test.go @@ -186,3 +186,35 @@ func TestHTTPModifierURLNegativeRegexp(t *testing.T) { t.Error("Should not pass url") } } + +func TestHTTPModifierSetHeader(t *testing.T) { + filters := HTTPHeaders{} + filters.Set("User-Agent:Gor") + + modifier := NewHTTPModifier(&HTTPModifierConfig{ + headers: filters, + }) + + payload := []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") + payload_after := []byte("POST /post HTTP/1.1\r\nUser-Agent: Gor\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") + + if payload = modifier.Rewrite(payload); !bytes.Equal(payload_after, payload) { + t.Error("Should add new header", string(payload)) + } +} + +func TestHTTPModifierSetParam(t *testing.T) { + filters := HTTPParams{} + filters.Set("api_key=1") + + modifier := NewHTTPModifier(&HTTPModifierConfig{ + params: filters, + }) + + payload := []byte("POST /post?api_key=1234 HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") + payload_after := []byte("POST /post?api_key=1 HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") + + if payload = modifier.Rewrite(payload); !bytes.Equal(payload_after, payload) { + t.Error("Should override param", string(payload)) + } +} diff --git a/proto/proto.go b/proto/proto.go index 9b0e5cf..626d289 100644 --- a/proto/proto.go +++ b/proto/proto.go @@ -104,6 +104,38 @@ func PathParam(payload, name []byte) (value []byte, valueStart, valueEnd int) { } } +func SetPathParam(payload, name, value []byte) []byte { + path := Path(payload) + _, vs, ve := PathParam(payload, name) + + if vs != -1 { + newPath := make([]byte, len(path)) + copy(newPath, path) + newPath = byteutils.Replace(newPath, vs, ve, value) + + return SetPath(payload, newPath) + } else { // if param not found append to end of url + // Adding 2 because of '?' or '&' at start, and '=' in middle + newParam := make([]byte, len(name) + len(value) + 2) + + if bytes.IndexByte(path, '?') == -1 { + newParam[0] = '?' + } else { + newParam[0] = '&' + } + + copy(newParam[1:], name) + newParam[1+len(name)] = '=' + copy(newParam[2+len(name):], value) + + newPath := make([]byte, len(path) + len(newParam)) + copy(newPath, path) + copy(newPath[len(path):], newParam) + + return SetPath(payload, newPath) + } +} + func SetHost(payload, url, host []byte) []byte { // If this is HTTP 1.0 traffic or proxy traffic it may include host right into path variable, so instead of setting Host header we rewrite Path // Fix for https://github.com/buger/gor/issues/156 diff --git a/proto/proto_test.go b/proto/proto_test.go index 12227e2..080f023 100644 --- a/proto/proto_test.go +++ b/proto/proto_test.go @@ -104,6 +104,42 @@ func TestPathParam(t *testing.T) { } +func TestSetPathParam(t *testing.T) { + var payload, payload_after []byte + + payload = []byte("POST /post?param=test&user_id=1 HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") + payload_after = []byte("POST /post?param=new&user_id=1 HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") + + if payload = SetPathParam(payload, []byte("param"), []byte("new")); !bytes.Equal(payload, payload_after) { + t.Error("Should replace existing value", string(payload)) + } + + payload = []byte("POST /post?param=test&user_id=1 HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") + payload_after = []byte("POST /post?param=test&user_id=2 HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") + + if payload = SetPathParam(payload, []byte("user_id"), []byte("2")); !bytes.Equal(payload, payload_after) { + t.Error("Should replace existing value", string(payload)) + } + + + payload = []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") + payload_after = []byte("POST /post?param=test HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") + + if payload = SetPathParam(payload, []byte("param"), []byte("test")); !bytes.Equal(payload, payload_after) { + t.Error("Should set param if url have no params", string(payload)) + } + + + payload = []byte("POST /post?param=test HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") + payload_after = []byte("POST /post?param=test&user_id=1 HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") + + if payload = SetPathParam(payload, []byte("user_id"), []byte("1")); !bytes.Equal(payload, payload_after) { + t.Error("Should set param at the end if url params", string(payload)) + } +} + + + func TestSetHostHTTP10(t *testing.T) { var payload, payload_after []byte diff --git a/settings.go b/settings.go index 8fd2127..68e8f3a 100644 --- a/settings.go +++ b/settings.go @@ -89,6 +89,8 @@ func init() { flag.Var(&Settings.modifierConfig.headers, "http-set-header", "Inject additional headers to http reqest:\n\tgor --input-raw :8080 --output-http staging.com --http-set-header 'User-Agent: Gor'") flag.Var(&Settings.modifierConfig.headers, "output-http-header", "WARNING: `--output-http-header` DEPRECATED, use `--http-set-header` instead") + flag.Var(&Settings.modifierConfig.params, "http-set-param", "Set request url param, if param already exists it will be overwritten:\n\tgor --input-raw :8080 --output-http staging.com --http-set-param api_key=1") + flag.Var(&Settings.modifierConfig.methods, "http-allow-method", "Whitelist of HTTP methods to replay. Anything else will be dropped:\n\tgor --input-raw :8080 --output-http staging.com --http-allow-method GET --http-allow-method OPTIONS") flag.Var(&Settings.modifierConfig.methods, "output-http-method", "WARNING: `--output-http-method` DEPRECATED, use `--http-allow-method` instead") @@ -97,7 +99,7 @@ func init() { flag.Var(&Settings.modifierConfig.urlNegativeRegexp, "http-diallow-url", "A regexp to match requests against. Filter get matched agains full url with domain. Anything else will be dropped:\n\t gor --input-raw :8080 --output-http staging.com --http-disallow-url ^www.") - flag.Var(&Settings.modifierConfig.urlRewrite, "http-rewrite-url", "Rewrite the requst url based on a mapping:\n\tgor --input-raw :8080 --output-http staging.com --http-rewrite-url /v1/user/([^\\/]+)/ping:/v2/user/$1/ping") + flag.Var(&Settings.modifierConfig.urlRewrite, "http-rewrite-url", "Rewrite the request url based on a mapping:\n\tgor --input-raw :8080 --output-http staging.com --http-rewrite-url /v1/user/([^\\/]+)/ping:/v2/user/$1/ping") flag.Var(&Settings.modifierConfig.urlRewrite, "output-http-rewrite-url", "WARNING: `--output-http-rewrite-url` DEPRECATED, use `--http-rewrite-url` instead") flag.Var(&Settings.modifierConfig.headerFilters, "http-allow-header", "A regexp to match a specific header against. Requests with non-matching headers will be dropped:\n\t gor --input-raw :8080 --output-http staging.com --http-allow-header api-version:^v1") From fd5388b6c9ed36cf0940351d73657d6e4c892454 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Thu, 9 Jul 2015 18:39:07 +0500 Subject: [PATCH 29/30] Improve README --- Makefile | 4 +- README.md | 330 +++++++++++++++++++++++++++------------------------- settings.go | 6 +- 3 files changed, 177 insertions(+), 163 deletions(-) diff --git a/Makefile b/Makefile index ca1a5f5..d8bc799 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -SOURCE = emitter.go gor.go gor_stat.go input_dummy.go input_file.go input_raw.go input_tcp.go limiter.go output_dummy.go output_file.go input_http.go output_http.go output_tcp.go plugins.go settings.go settings_header_filters.go settings_header_hash_filters.go settings_headers.go settings_methods.go settings_option.go settings_url_regexp.go test_input.go elasticsearch.go settings_url_map.go http_modifier.go +SOURCE = emitter.go gor.go gor_stat.go input_dummy.go input_file.go input_raw.go input_tcp.go limiter.go output_dummy.go output_file.go input_http.go output_http.go output_tcp.go plugins.go settings.go test_input.go elasticsearch.go http_modifier.go http_modifier_settings.go http_client.go SOURCE_PATH = /gopath/src/github.com/buger/gor/ @@ -31,7 +31,7 @@ dbench: # Used mainly for debugging, because docker container do not have access to parent machine ports drun: - docker run -v `pwd`:$(SOURCE_PATH) -t -i gor go run $(SOURCE) --input-dummy=0 --output-http="http://localhost:9000" --verbose + docker run -v `pwd`:$(SOURCE_PATH) -t -i gor go run $(SOURCE) --input-dummy=0 --output-http="http://localhost:9000" --verbose -h dbash: docker run -v `pwd`:$(SOURCE_PATH) -t -i gor /bin/bash \ No newline at end of file diff --git a/README.md b/README.md index 4428d9d..e12e2fb 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,3 @@ -[![Stories in Ready](https://badge.waffle.io/buger/gor.png?label=ready)](https://waffle.io/buger/gor) [![Build Status](https://travis-ci.org/buger/gor.png?branch=master)](https://travis-ci.org/buger/gor) ## About @@ -10,7 +9,7 @@ Its main goal is to replay traffic from production servers to staging and dev en Now you can test your code on real user sessions in an automated and repeatable fashion. **No more falling down in production!** -Here is basic workflow: The listener server catches http traffic and sends it to the replay server or saves to file.The replay server forwards traffic to a given address. +Here is basic workflow: The listener server catches http traffic and sends it to the replay server or saves to file. The replay server forwards traffic to a given address. ![Diagram](http://i.imgur.com/9mqj2SK.png) @@ -45,26 +44,43 @@ sudo gor --input-http :28019 --output-http "http://staging.com" Then in your application you should send copy (e.g. like reverse proxy) all incoming requests to Gor http input. -### Following redirects -If you have a scenario where following redirects is usefull you can do it like with: +## Configuration + +### Forward to multiple addresses + +You can forward traffic to multiple endpoints. Just add multiple --output-* arguments. +``` +gor --input-tcp :28020 --output-http "http://staging.com" --output-http "http://dev.com" +``` + +#### Splitting traffic +By default it will send same traffic to all outputs, but you have options to equally split it: ``` -gor --input-tcp replay.local:28020 --output-http http://staging.com --output-http-redirects 10 +gor --input-tcp :28020 --output-http "http://staging.com" --output-http "http://dev.com" --split-output true ``` -The given example will follow up to 10 redirects per request. -## Advanced use +### HTTP output workers +By default Gor creates dynamic pull of workers: it starts with 10 and create more http output workers when the http output queue length is greater than 10. The number of workers created (N) is equal to the queue length at the time which it is checked and found to have a length greater than 10. The queue length is checked every time a message is written to the http output queue. No more workers will be spawned until that request to spawn N workers is satisfied. If a dynamic worker cannot process a message at that time, it will sleep for 100 milliseconds. If a dynamic worker cannot process a message for 2 seconds it dies. +You may specify fixed number of workers using `--output-http-workers=20` option. + +### Follow redirects +By default Gor will ignore all redirects since they are handled by clients using your app, but in scenarios when your replayed environment introduce new redirects, you can enable them like this: +``` +gor --input-tcp replay.local:28020 --output-http http://staging.com --output-http-redirects 2 +``` +The given example will follow up to 2 redirects per request. ### Rate limiting -Every input and output support rate limiting. It can be useful if you want -forward only part of production traffic and not overload your staging -environment. +Rate limiting can be useful if you want forward only part of production traffic and not overload your staging environment. There is 2 strategies: dropping random requests or dropping fraction of requests based on Header or URL param value. +#### Dropping random requests +Every input and output support random rate limiting. There are 2 limiting algorithms: absolute or percentage based. Absolute: If for current second it reached specified requests limit - disregard the rest, on next second counter reseted. -Percentage: For input-file it will slowdown or speedup request execution, for the rest it will use random generator to decide if request pass or not based on weight you specified. +Percentage: For input-file it will slowdown or speedup request execution, for the rest it will use random generator to decide if request pass or not based on chance you specified. You can specify your desired limit using the "|" operator after the server address: @@ -82,51 +98,72 @@ gor --input-tcp :28020 --output-http "http://staging.com|10" gor --input-raw :80 --output-tcp "replay.local:28020|10%" ``` -### Load testing - -Currently it supported only by `input-file` and only when using percentage based limiter. Unlike default limiter for `input-file` instead of dropping requests it will slowdown or speedup request emitting. - -``` -# Replay from file on 2x speed -gor --input-file "requests.gor|200%" --output-http "staging.com" +#### Limiting based on Header or URL param value +If you have unique user id (like API key) stored in header or URL you can consistently forward specified percent of traffic only for fraction of this users. +Basic formula looks like this: `FNV32-1A_hashing(value) % 100 >= chance`. Examples: ``` +# Limit based on header value +gor --input-raw :80 --output-tcp "replay.local:28020|10%" --http-header-limiter "X-API-KEY: 10%" +# Limit based on header value +gor --input-raw :80 --output-tcp "replay.local:28020|10%" --http-param-limiter "api_key: 10%" +``` +Only percentage based limiting supported. ### Filtering -#### Match on regexp of url +#### Allow url regexp ``` -# only forward requests being sent to the api... domains -gor --input-raw :8080 --output-http staging.com --output-http-url-regexp ^www. +# only forward requests being sent to the /api endpoint +gor --input-raw :8080 --output-http staging.com --http-allow-url /api ``` +#### Disallow url regexp +``` +# only forward requests NOT being sent to the /api... endpoint +gor --input-raw :8080 --output-http staging.com --http-disallow-url /api +``` #### Filter based on regexp of header + ``` # only forward requests with an api version of 1.0x -gor --input-raw :8080 --output-http staging.com --output-http-header-filter api-version:^1\.0\d +gor --input-raw :8080 --output-http staging.com --http-allow-header api-version:^1\.0\d ``` -#### Filter based on hash of header -``` -# send 1/32 of all users consistently to staging -gor --input-raw :8080 --output-http staging.com --output-http-header-hash-filter user-id:1/32 -``` - -### Forward to multiple addresses - -You can forward traffic to multiple endpoints. Just add multiple --output-* arguments. -``` -gor --input-tcp :28020 --output-http "http://staging.com" --output-http "http://dev.com" -``` - -#### Splitting traffic -By default it will send same traffic to all outputs, but you have options to equally split it: +#### Filter based on http method +Requests not matching a specified whitelist can be filtered out. For example to strip non-nullipotent requests: ``` -gor --input-tcp :28020 --output-http "http://staging.com" --output-http "http://dev.com" --split-output true +gor --input-raw :80 --output-http "http://staging.server" \ + --http-allow-method GET \ + --http-allow-method OPTIONS ``` -### Saving requests to file +### Rewriting original request +Gor supports built-in basic rewriting support, for complex logic see https://github.com/buger/gor/pull/162 + +#### Rewrite URL based on a mapping +``` +# rewrite url to match the following +gor --input-raw :8080 --output-http staging.com --http-rewrite-url /v1/user/([^\\/]+)/ping:/v2/user/$1/ping +``` + +#### Set URL param +Set request url param, if param already exists it will be overwritten +``` +gor --input-raw :8080 --output-http staging.com --http-set-param api_key=1 +``` + +#### Set Header +Set request header, if header already exists it will be overwritten. This may be useful if you need to identify requests generated by Gor or enable feature flagged functionality in an application: + +``` +gor --input-raw :80 --output-http "http://staging.server" \ + --http-header "User-Agent: Replayed by Gor" \ + --http-header "Enable-Feature-X: true" +``` + +### Saving requests to file and replaying them You can save requests to file, and replay them later: ``` # write to file @@ -138,24 +175,13 @@ gor --input-file requests.gor --output-http "http://staging.com" **Note:** Replay will preserve the original time differences between requests. -### Injecting headers +### Load testing -Additional headers can be injected/overwritten into requests during replay. This may be useful if you need to identify requests generated by Gor or enable feature flagged functionality in an application: +Currently it supported only by `input-file` and only when using percentage based limiter. Unlike default limiter for `input-file` instead of dropping requests it will slowdown or speedup request emitting. Note that unlike examples above limiter is applied to input: ``` -gor --input-raw :80 --output-http "http://staging.server" \ - --output-http-header "User-Agent: Replayed by Gor" \ - --output-http-header "Enable-Feature-X: true" -``` - -## Filtering HTTP methods - -Requests not matching a specified whitelist can be filtered out. For example to strip non-nullipotent requests: - -``` -gor --input-raw :80 --output-http "http://staging.server" \ - --output-http-method GET \ - --output-http-method OPTIONS +# Replay from file on 2x speed +gor --input-file "requests.gor|200%" --output-http "staging.com" ``` ### Basic Auth @@ -168,14 +194,52 @@ gor --input-raw :80 --output-http "http://user:pass@staging .com" Note: This will overwrite any Authorization headers in the original request. -#### Rewrite the target urls based on a mapping -``` -# rewrite url to match the following -gor --input-raw :8080 --output-http staging.com --output-http-rewrite-url /xml_test/interface.php:/api/service.do -``` - ## Stats + +Gor can report stats on the `output-tcp` and `output-http` request queues. Stats are reported to the console every 5 seconds in the form `latest,mean,max,count,count/second` by using the `--output-http-stats` and `--output-tcp-stats` options. + +Examples: + +``` +2014/04/23 21:17:50 output_tcp:latest,mean,max,count,count/second +2014/04/23 21:17:50 output_tcp:0,0,0,0,0 +2014/04/23 21:17:55 output_tcp:1,1,2,68,13 +2014/04/23 21:18:00 output_tcp:1,1,2,92,18 +2014/04/23 21:18:05 output_tcp:1,1,2,119,23 +``` + +``` +Version: 0.8 +2014/04/23 21:19:46 output_http:latest,mean,max,count,count/second +2014/04/23 21:19:46 output_http:0,0,0,0,0 +2014/04/23 21:19:51 output_http:0,0,0,0,0 +2014/04/23 21:19:56 output_http:0,0,0,0,0 +2014/04/23 21:20:01 output_http:1,0,1,50,10 +2014/04/23 21:20:06 output_http:1,1,4,72,14 +2014/04/23 21:20:11 output_http:1,0,1,179,35 +2014/04/23 21:20:16 output_http:1,0,1,148,29 +2014/04/23 21:20:21 output_http:1,1,2,91,18 +2014/04/23 21:20:26 output_http:1,1,2,150,30 +2014/04/23 21:18:15 output_http:100,99,100,70,14 +2014/04/23 21:18:21 output_http:100,99,100,55,11 +``` + +### How can I tell if I have bottlenecks? +Key areas that sometimes experience bottlenecks are the output-tcp and output-http functions which have internal queues for requests. Each queue has an upper limit of 100. Enable stats reporting to see if any queues are experiencing bottleneck behavior. + +#### output-http bottlenecks +When running a Gor replay the output-http feature may bottleneck if: + + * the replay has inadequate bandwidth. If the replay is receiving or sending more messages than its network adapter can handle the output-http-stats may report that the output-http queue is filling up. See if there is a way to upgrade the replay's bandwidth. + * with `--output-http-workers` set to anything other than `-1` the `-output-http` target is unable to respond to messages in a timely manner. The http output workers which take messages off the output-http queue, process the request, and ensure that the request did not result in an error may not be able to keep up with the number of incoming requests. If the replay is not using dynamic worker scaling (`--output-http-workers=-1`) The optimal number of output-http-workers can be determined with the formula `output-workers = (Average number of requests per second)/(Average target response time per second)`. + +#### output-tcp bottlenecks +When using the Gor listener the output-tcp feature may bottleneck if: + + * the replay is unable to accept and process more requests than the listener is able generate. Prior to troubleshooting the output-tcp bottleneck, ensure that the replay target is not experiencing any bottlenecks. + * the replay target has inadequate bandwidth to handle all its incoming requests. If a replay target's incoming bandwidth is maxed out the output-tcp-stats may report that the output-tcp queue is filling up. See if there is a way to upgrade the replay's bandwidth. + ### ElasticSearch For deep response analyze based on url, cookie, user-agent and etc. you can export response metadata to ElasticSearch. See [ELASTICSEARCH.md](ELASTICSEARCH.md) for more details. @@ -194,46 +258,51 @@ https://github.com/buger/gor/releases ## Command line reference `gor -h` output: ``` - -cpuprofile="": write cpu profile to file + -http-allow-header=[]: A regexp to match a specific header against. Requests with non-matching headers will be dropped: + gor --input-raw :8080 --output-http staging.com --http-allow-header api-version:^v1 + -http-allow-method=[]: Whitelist of HTTP methods to replay. Anything else will be dropped: + gor --input-raw :8080 --output-http staging.com --http-allow-method GET --http-allow-method OPTIONS + -http-allow-url=[]: A regexp to match requests against. Filter get matched agains full url with domain. Anything else will be dropped: + gor --input-raw :8080 --output-http staging.com --http-allow-url ^www. + -http-diallow-url=[]: A regexp to match requests against. Filter get matched agains full url with domain. Anything else will be dropped: + gor --input-raw :8080 --output-http staging.com --http-disallow-url ^www. + -http-header-limiter=[]: Takes a fraction of requests, consistently taking or rejecting a request based on the FNV32-1A hash of a specific header: + gor --input-raw :8080 --output-http staging.com --http-header-imiter user-id:25% + -http-param-limiter=[]: Takes a fraction of requests, consistently taking or rejecting a request based on the FNV32-1A hash of a specific GET param: + gor --input-raw :8080 --output-http staging.com --http-param-limiter user_id:25% + -http-rewrite-url=[]: Rewrite the request url based on a mapping: + gor --input-raw :8080 --output-http staging.com --http-rewrite-url /v1/user/([^\/]+)/ping:/v2/user/$1/ping + -http-set-header=[]: Inject additional headers to http reqest: + gor --input-raw :8080 --output-http staging.com --http-set-header 'User-Agent: Gor' + -http-set-param=[]: Set request url param, if param already exists it will be overwritten: + gor --input-raw :8080 --output-http staging.com --http-set-param api_key=1 -input-dummy=[]: Used for testing outputs. Emits 'Get /' request every 1s - -input-file=[]: Read requests from file: - gor --input-file ./requests.gor --output-http staging.com + -input-file=[]: Read requests from file: + gor --input-file ./requests.gor --output-http staging.com -input-http=[]: Read requests from HTTP, should be explicitly sent from your application: - # Listen for http on 9000 - gor --input-http :9000 --output-http staging.com + # Listen for http on 9000 + gor --input-http :9000 --output-http staging.com -input-raw=[]: Capture traffic from given port (use RAW sockets and require *sudo* access): - # Capture traffic from 8080 port - gor --input-raw :8080 --output-http staging.com - -input-tcp=[]: Used for internal communication between Gor instances. Example: - # Receive requests from other Gor instances on 28020 port, and redirect output to staging - gor --input-tcp :28020 --output-http staging.com + # Capture traffic from 8080 port + gor --input-raw :8080 --output-http staging.com + -input-tcp=[]: Used for internal communication between Gor instances. Example: + # Receive requests from other Gor instances on 28020 port, and redirect output to staging + gor --input-tcp :28020 --output-http staging.com -memprofile="": write memory profile to this file -output-dummy=[]: Used for testing inputs. Just prints data coming from inputs. - -output-file=[]: Write incoming requests to file: - gor --input-raw :80 --output-file ./requests.gor + -output-file=[]: Write incoming requests to file: + gor --input-raw :80 --output-file ./requests.gor -output-http=[]: Forwards incoming requests to given http address. - # Redirect all incoming requests to staging.com address - gor --input-raw :80 --output-http http://staging.com + # Redirect all incoming requests to staging.com address + gor --input-raw :80 --output-http http://staging.com -output-http-elasticsearch="": Send request and response stats to ElasticSearch: - gor --input-raw :8080 --output-http staging.com --output-http-elasticsearch 'es_host:api_port/index_name' - -output-http-header=[]: Inject additional headers to http reqest: - gor --input-raw :8080 --output-http staging.com --output-http-header 'User-Agent: Gor' - -output-http-header-filter=[]: A regexp to match a specific header against. Requests with non-matching headers will be dropped: - gor --input-raw :8080 --output-http staging.com --output-http-header-filter api-version:^v1 - -output-http-header-hash-filter=[]: Takes a fraction of requests, consistently taking or rejecting a request based on the FNV32-1A hash of a specific header. The fraction must have a denominator that is a power of two: - gor --input-raw :8080 --output-http staging.com --output-http-header-hash-filter user-id:1/4 - -output-http-method=[]: Whitelist of HTTP methods to replay. Anything else will be dropped: - gor --input-raw :8080 --output-http staging.com --output-http-method GET --output-http-method OPTIONS - -output-http-redirects=0: Enable how often redirects should be followed. - -output-http-rewrite-url=[]: Rewrite the requst url based on a mapping: - gor --input-raw :8080 --output-http staging.com --output-http-rewrite-url /xml_test/interface.php:/api/service.do + gor --input-raw :8080 --output-http staging.com --output-http-elasticsearch 'es_host:api_port/index_name' + -output-http-header-filter=[]: WARNING: `--output-http-header-filter` DEPRECATED, use `--http-allow-header` instead -output-http-redirects=0: Enable how often redirects should be followed. -output-http-stats=false: Report http output queue stats to console every 5 seconds. - -output-http-url-regexp=: A regexp to match requests against. Anything else will be dropped: - gor --input-raw :8080 --output-http staging.com --output-http-url-regexp ^www. - -output-http-workers=-1: Gor uses dynamic worker scaling by default. Enter a number to run a set number of workers. - -output-tcp=[]: Used for internal communication between Gor instances. Example: - # Listen for requests on 80 port and forward them to other Gor instance on 28020 port - gor --input-raw :80 --output-tcp replay.local:28020 + -output-http-workers=0: Gor uses dynamic worker scaling by default. Enter a number to run a set number of workers. + -output-tcp=[]: Used for internal communication between Gor instances. Example: + # Listen for requests on 80 port and forward them to other Gor instance on 28020 port + gor --input-raw :80 --output-tcp replay.local:28020 -output-tcp-stats=false: Report TCP output queue stats to console every 5 seconds. -split-output=false: By default each output gets same traffic. If set to `true` it splits traffic equally among all outputs. -stats=false: Turn on queue stats output @@ -270,6 +339,16 @@ For now only Linux based. *BSD (including MacOS is not supported yet, check http Listener works by sniffing traffic from a given port. It's accessible only by using sudo or root access. +### How do you deal with user session to replay the traffic correctly? +You can rewrite session related headers/params to match your staging environment. If you require custom logic (e.g random token based auth) follow this discussion: https://github.com/buger/gor/issues/154 + +### Can i use Gor to intercept SSL traffic? +Basic idea is that SSL was made to protect itself from traffic interception. There 2 options: +1. Move SSL handling to proxy like Nginx or Amazon ELB. And allow Gor to listen on upstreams. +2. Use `--input-http` so you can duplicate request payload directly from your app to Gor, but it will require your app modifications. + +More can be find here: https://github.com/buger/gor/issues/85 + ### I'm getting 'too many open files' error Typical linux shell has a small open files soft limit at 1024. You can easily raise that when you do this before starting your gor replay process: @@ -277,73 +356,8 @@ Typical linux shell has a small open files soft limit at 1024. You can easily ra More about ulimit: http://blog.thecodingmachine.com/content/solving-too-many-open-files-exception-red5-or-any-other-application -### What do the stats commands do? -Gor can report stats on the output-tcp and output-http request queues. Stats are reported to the console every 5 seconds in the form `latest,mean,max,count,count/second` by using the `-output-http-stats` and `-output-tcp-stats` options. - -Examples: - -``` -2014/04/23 21:17:50 output_tcp:latest,mean,max,count,count/second -2014/04/23 21:17:50 output_tcp:0,0,0,0,0 -2014/04/23 21:17:55 output_tcp:1,1,2,68,13 -2014/04/23 21:18:00 output_tcp:1,1,2,92,18 -2014/04/23 21:18:05 output_tcp:1,1,2,119,23 -2014/04/23 21:18:10 output_tcp:1,0,1,95,19 -2014/04/23 21:18:15 output_tcp:1,1,2,92,18 -2014/04/23 21:18:20 output_tcp:1,1,2,108,21 -2014/04/23 21:18:25 output_tcp:1,1,2,117,23 -2014/04/23 21:18:30 output_tcp:1,1,2,113,22 -2014/04/23 21:18:35 output_tcp:21,20,21,132,26 -2014/04/23 21:18:40 output_tcp:100,99,100,99,19 -``` - -``` -Version: 0.8 -2014/04/23 21:19:46 output_http:latest,mean,max,count,count/second -2014/04/23 21:19:46 output_http:0,0,0,0,0 -2014/04/23 21:19:51 output_http:0,0,0,0,0 -2014/04/23 21:19:56 output_http:0,0,0,0,0 -2014/04/23 21:20:01 output_http:1,0,1,50,10 -2014/04/23 21:20:06 output_http:1,1,4,72,14 -2014/04/23 21:20:11 output_http:1,0,1,179,35 -2014/04/23 21:20:16 output_http:1,0,1,148,29 -2014/04/23 21:20:21 output_http:1,1,2,91,18 -2014/04/23 21:20:26 output_http:1,1,2,150,30 -2014/04/23 21:18:15 output_http:100,99,100,70,14 -2014/04/23 21:18:21 output_http:100,99,100,55,11 -2014/04/23 21:18:28 output_http:100,99,100,55,11 -2014/04/23 21:18:34 output_http:100,99,100,57,11 -2014/04/23 21:18:41 output_http:100,99,100,61,12 -2014/04/23 21:18:48 output_http:100,99,100,56,11 -2014/04/23 21:18:56 output_http:100,99,100,58,11 -2014/04/23 21:19:01 output_http:100,99,100,31,6 -2014/04/23 21:19:08 output_http:100,99,100,61,12 -2014/04/23 21:19:15 output_http:100,99,100,64,12 -2014/04/23 21:19:21 output_http:100,99,100,70,14 -2014/04/23 21:19:28 output_http:100,99,100,61,12 -2014/04/23 21:19:35 output_http:100,99,100,56,11 -``` - -### How can I tell if I have bottlenecks? -Key areas that sometimes experience bottlenecks are the output-tcp and output-http functions which have internal queues for requests. Each queue has an upper limit of 100. Enable stats reporting to see if any queues are experiencing bottleneck behavior. - -#### output-http bottlenecks -When running a Gor replay the output-http feature may bottleneck if: - - * the replay has inadequate bandwidth. If the replay is receiving or sending more messages than its network adapter can handle the output-http-stats may report that the output-http queue is filling up. See if there is a way to upgrade the replay's bandwidth. - * with `--output-http-workers` set to anything other than `-1` the `-output-http` target is unable to respond to messages in a timely manner. The http output workers which take messages off the output-http queue, process the request, and ensure that the request did not result in an error may not be able to keep up with the number of incoming requests. If the replay is not using dynamic worker scaling (`--output-http-workers=-1`) The optimal number of output-http-workers can be determined with the formula `output-workers = (Average number of requests per second)/(Average target response time per second)`. - -#### output-tcp bottlenecks -When using the Gor listener the output-tcp feature may bottleneck if: - - * the replay is unable to accept and process more requests than the listener is able generate. Prior to troubleshooting the output-tcp bottleneck, ensure that the replay target is not experiencing any bottlenecks. - * the replay target has inadequate bandwidth to handle all its incoming requests. If a replay target's incoming bandwidth is maxed out the output-tcp-stats may report that the output-tcp queue is filling up. See if there is a way to upgrade the replay's bandwidth. - ### The CPU average across my load-balanced targets is higher than the source If you are replaying traffic from multiple listeners to a load-balanced target and you use sticky sessions, you may observe that the target servers have a higher CPU load than the listener servers. This may be because the sticky session cookie of the original load balancer is not honored by the target load balancer thus resulting in requests that would normally hit the same target server hitting different servers on the backend thus reducing some caching benefits gained via the load balancing. Try running just one listener against one replay target and see if the CPU utilization comparison is more accurate. - -### How does dynamic http worker scaling work? -By using the Gor setting `--output-http-workers=-1` Gor will create more http output workers when the http output queue length is greater than 10. The number of workers created (N) is equal to the queue length at the time which it is checked and found to have a length greater than 10. The queue length is checked every time a message is written to the http output queue. No more workers will be spawned until that request to spawn N workers is satisfied. If a dynamic worker cannot process a message at that time, it will sleep for 100 milliseconds. If a dynamic worker cannot process a message for 2 seconds it dies. ## Tuning @@ -383,4 +397,4 @@ To achieve the top most performance you should tune the source server system lim * [3SCALE](http://www.3scale.net/) * [Optionlab](http://www.opinionlab.com) * [TubeMogul] (http://tubemogul.com) -* To add your company drop me a line to github.com/buger or leonsbox@gmail.com +* To add your company drop me a line to github.com/buger or leonsbox@gmail.com \ No newline at end of file diff --git a/settings.go b/settings.go index 68e8f3a..b182df3 100644 --- a/settings.go +++ b/settings.go @@ -105,10 +105,10 @@ func init() { flag.Var(&Settings.modifierConfig.headerFilters, "http-allow-header", "A regexp to match a specific header against. Requests with non-matching headers will be dropped:\n\t gor --input-raw :8080 --output-http staging.com --http-allow-header api-version:^v1") flag.Var(&Settings.modifierConfig.headerFilters, "output-http-header-filter", "WARNING: `--output-http-header-filter` DEPRECATED, use `--http-allow-header` instead") - flag.Var(&Settings.modifierConfig.headerHashFilters, "http-allow-header-hash", "Takes a fraction of requests, consistently taking or rejecting a request based on the FNV32-1A hash of a specific header:\n\t gor --input-raw :8080 --output-http staging.com --http-allow-header-hash user-id:25%") - flag.Var(&Settings.modifierConfig.headerHashFilters, "output-http-header-hash-filter", "WARNING: `output-http-header-hash-filter` DEPRECATED, use `--http-allow-header-hash` instead") + flag.Var(&Settings.modifierConfig.headerHashFilters, "http-header-limiter", "Takes a fraction of requests, consistently taking or rejecting a request based on the FNV32-1A hash of a specific header:\n\t gor --input-raw :8080 --output-http staging.com --http-header-imiter user-id:25%") + flag.Var(&Settings.modifierConfig.headerHashFilters, "output-http-header-hash-filter", "WARNING: `output-http-header-hash-filter` DEPRECATED, use `--http-header-hash-limiter` instead") - flag.Var(&Settings.modifierConfig.paramHashFilters, "http-allow-param-hash", "Takes a fraction of requests, consistently taking or rejecting a request based on the FNV32-1A hash of a specific GET param:\n\t gor --input-raw :8080 --output-http staging.com --http-allow-param-hash user_id:25%") + flag.Var(&Settings.modifierConfig.paramHashFilters, "http-param-limiter", "Takes a fraction of requests, consistently taking or rejecting a request based on the FNV32-1A hash of a specific GET param:\n\t gor --input-raw :8080 --output-http staging.com --http-param-limiter user_id:25%") } func Debug(args ...interface{}) { From b38cb458fff401f4fe1d9db714f4f6ba33646aee Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Thu, 9 Jul 2015 18:45:39 +0500 Subject: [PATCH 30/30] Update version --- settings.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/settings.go b/settings.go index b182df3..98d9e7f 100644 --- a/settings.go +++ b/settings.go @@ -8,7 +8,7 @@ import ( ) const ( - VERSION = "0.9.4" + VERSION = "0.9.6" ) // Allows to specify multiple flags with same name and collects all values to array