From 208faec7d298a7a95958bc21ef9427966a7d1d59 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Tue, 16 Dec 2014 16:25:17 +0500 Subject: [PATCH 01/64] Initial support for input modifiers --- Dockerfile | 4 +- Makefile | 8 +-- examples/echo_modifier/echo_modifier.go | 44 +++++++++++++++++ plugins.go | 5 ++ pong_modifier.rb | 13 +++++ settings.go | 4 ++ traffic_modifier.go | 65 +++++++++++++++++++++++++ 7 files changed, 138 insertions(+), 5 deletions(-) create mode 100644 examples/echo_modifier/echo_modifier.go create mode 100755 pong_modifier.rb create mode 100644 traffic_modifier.go diff --git a/Dockerfile b/Dockerfile index 2b174ce..7352439 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,8 +2,10 @@ FROM google/golang RUN cd /goroot/src/ && GOOS=linux GOARCH=386 ./make.bash --no-clean +RUN apt-get install ruby -y + WORKDIR /gopath/src/gor ADD . /gopath/src/gor -RUN go get \ No newline at end of file +RUN go get diff --git a/Makefile b/Makefile index 037e311..4b68a9e 100644 --- a/Makefile +++ b/Makefile @@ -1,10 +1,10 @@ -SOURCE = emitter.go gor.go gor_stat.go input_dummy.go input_file.go input_raw.go input_tcp.go limiter.go output_dummy.go output_file.go input_http.go output_http.go output_tcp.go plugins.go settings.go settings_header_filters.go settings_header_hash_filters.go settings_headers.go settings_methods.go settings_option.go settings_url_regexp.go test_input.go elasticsearch.go settings_url_map.go +SOURCE = emitter.go gor.go traffic_modifier.go gor_stat.go input_dummy.go input_file.go input_raw.go input_tcp.go limiter.go output_dummy.go output_file.go input_http.go output_http.go output_tcp.go plugins.go settings.go settings_header_filters.go settings_header_hash_filters.go settings_headers.go settings_methods.go settings_option.go settings_url_regexp.go test_input.go elasticsearch.go settings_url_map.go release: release-x86 release-x64 release-x64: docker run -v `pwd`:/gopath/src/gor -t --env GOOS=linux --env GOARCH=amd64 --env CGO_ENABLED=0 -i gor go build && tar -czf gor_x64.tar.gz gor && rm gor - + release-x86: docker run -v `pwd`:/gopath/src/gor -t --env GOOS=linux --env GOARCH=386 --env CGO_ENABLED=0 -i gor go build && tar -czf gor_x86.tar.gz gor && rm gor @@ -22,7 +22,7 @@ dbench: # Used mainly for debugging, because docker container do not have access to parent machine ports drun: - docker run -v `pwd`:/gopath/src/gor -t -i gor go run $(SOURCE) --input-dummy=0 --input-http=:9000 --output-http="http://localhost:9000" --verbose + docker run -v `pwd`:/gopath/src/gor -t -i gor go run $(SOURCE) --input-modifier="./pong_modifier.rb" --input-dummy=0 --input-http=:9000 --output-http="http://localhost:9000" --verbose -dbash: +dbash: docker run -v `pwd`:/gopath/src/gor -t -i gor /bin/bash \ No newline at end of file diff --git a/examples/echo_modifier/echo_modifier.go b/examples/echo_modifier/echo_modifier.go new file mode 100644 index 0000000..9dd0ea6 --- /dev/null +++ b/examples/echo_modifier/echo_modifier.go @@ -0,0 +1,44 @@ +package main + +import ( + "os" + "bufio" +) + +func main() { + reader := bufio.NewReader(os.Stdin) + data := make(chan []byte) + + go ReadStdin(data) + + for { + os.Stdout.Print(<- data, '¶') + } +} + +func ReadStdin(data chan []byte){ + for { + buf, err := reader.ReadBytes('¶') + buf_len := len(buf) + if buf_len > 0 { + new_buf_len := len(buf) - 2 + if new_buf_len > 0 { + new_buf := make([]byte, new_buf_len) + copy(new_buf, buf[:new_buf_len]) + data <- new_buf + if err != nil { + if err != io.EOF { + log.Printf("error: %s\n", err) + } + } + } + } + } +} + +while data = STDIN.gets(separator) + STDERR.puts "==== Start ====" + STDERR.puts data + puts data + STDERR.puts "==== End ====" +end \ No newline at end of file diff --git a/plugins.go b/plugins.go index 7f96ae0..0cc8ab5 100644 --- a/plugins.go +++ b/plugins.go @@ -9,6 +9,8 @@ import ( type InOutPlugins struct { Inputs []io.Reader Outputs []io.Writer + + Modifiers []TrafficModifier } type ReaderOrWriter interface{} @@ -54,6 +56,9 @@ func registerPlugin(constructor interface{}, options ...interface{}) { } if _, ok := plugin.(io.Reader); ok { + for _, options := range Settings.inputModifier { + plugin_wrapper = NewTrafficModifier(plugin_wrapper, options) + } Plugins.Inputs = append(Plugins.Inputs, plugin_wrapper.(io.Reader)) } diff --git a/pong_modifier.rb b/pong_modifier.rb new file mode 100755 index 0000000..e702507 --- /dev/null +++ b/pong_modifier.rb @@ -0,0 +1,13 @@ +#!/usr/bin/env ruby +# encoding: utf-8 +require "base64" + +STDERR.puts "Starting modifier" +puts "Starting modifier" + +while data = STDIN.gets.chomp + STDERR.puts "==== Start ====" + STDERR.puts Base64.encode64(data) + puts data + STDERR.puts "==== End ====" +end \ No newline at end of file diff --git a/settings.go b/settings.go index f7973c1..e59a43a 100644 --- a/settings.go +++ b/settings.go @@ -29,6 +29,8 @@ type AppSettings struct { inputRAW MultiOption + inputModifier MultiOption + inputHTTP MultiOption outputHTTP MultiOption outputHTTPHeaders HTTPHeaders @@ -70,6 +72,8 @@ func init() { flag.Var(&Settings.inputRAW, "input-raw", "Capture traffic from given port (use RAW sockets and require *sudo* access):\n\t# Capture traffic from 8080 port\n\tgor --input-raw :8080 --output-http staging.com") + flag.Var(&Settings.inputModifier, "input-modifier", "Used for modifying input traffic using external command") + flag.Var(&Settings.inputHTTP, "input-http", "Read requests from HTTP, should be explicitly sent from your application:\n\t# Listen for http on 9000\n\tgor --input-http :9000 --output-http staging.com") flag.Var(&Settings.outputHTTP, "output-http", "Forwards incoming requests to given http address.\n\t# Redirect all incoming requests to staging.com address \n\tgor --input-raw :80 --output-http http://staging.com") diff --git a/traffic_modifier.go b/traffic_modifier.go new file mode 100644 index 0000000..0e8a09a --- /dev/null +++ b/traffic_modifier.go @@ -0,0 +1,65 @@ +package main + +import ( + "fmt" + "log" + "io" + "os/exec" + "os" + "encoding/base64" +) + +type TrafficModifier struct { + plugin interface{} + command string + + data chan []byte + + Stdin io.Writer + Stdout io.Reader +} + +func NewTrafficModifier(plugin interface{}, command string) io.Reader { + m := new(TrafficModifier) + m.plugin = plugin + m.command = command + + cmd := exec.Command("bash", "-c", command) + cmd.Stderr = os.Stderr + + m.Stdout, _ = cmd.StdoutPipe() + m.Stdin, _ = cmd.StdinPipe() + + m.Stdout = base64.NewDecoder(base64.StdEncoding, m.Stdout) + + go m.copy(m.Stdin, m.plugin.(io.Reader)) + + err := cmd.Run() + + if (err != nil) { + log.Fatal(err) + } + + return m +} + +func (m *TrafficModifier) copy(to io.Writer, from io.Reader) { + buf := make([]byte, 5*1024*1024) + + for { + nr, er := from.Read(buf) + if nr > 0 && len(buf) > nr { + to.Write(base64.StdEncoding.Encode(buf)) + } + } +} + +func (m *TrafficModifier) Read(data []byte) (n int, err error) { + n, err = m.Stdout.Read(data) + + return +} + +func (m *TrafficModifier) String() string { + return fmt.Sprintf("Modifying traffic for %s using '%s' command", m.plugin, m.command) +} From 4f03f81d049a381a35c46bf61138d1bb7e9b1866 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Sat, 27 Jun 2015 21:09:06 +0500 Subject: [PATCH 02/64] Working modifier implementation with examples --- Dockerfile | 2 +- Makefile | 2 +- emitter.go | 1 + examples/echo_modifier.rb | 16 +++++++ examples/echo_modifier.sh | 10 +++++ examples/echo_modifier/echo_modifier.go | 44 ------------------- input_dummy.go | 2 +- plugins.go | 1 + pong_modifier.rb | 13 ------ traffic_modifier.go | 58 +++++++++++++++++++------ 10 files changed, 76 insertions(+), 73 deletions(-) create mode 100755 examples/echo_modifier.rb create mode 100755 examples/echo_modifier.sh delete mode 100644 examples/echo_modifier/echo_modifier.go delete mode 100755 pong_modifier.rb diff --git a/Dockerfile b/Dockerfile index 7352439..7691f7a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,7 @@ FROM google/golang RUN cd /goroot/src/ && GOOS=linux GOARCH=386 ./make.bash --no-clean -RUN apt-get install ruby -y +RUN apt-get update && apt-get install ruby vim-common -y WORKDIR /gopath/src/gor diff --git a/Makefile b/Makefile index 4b68a9e..18b69e5 100644 --- a/Makefile +++ b/Makefile @@ -22,7 +22,7 @@ dbench: # Used mainly for debugging, because docker container do not have access to parent machine ports drun: - docker run -v `pwd`:/gopath/src/gor -t -i gor go run $(SOURCE) --input-modifier="./pong_modifier.rb" --input-dummy=0 --input-http=:9000 --output-http="http://localhost:9000" --verbose + docker run -v `pwd`:/gopath/src/gor -t -i gor go run $(SOURCE) --input-modifier="./examples/echo_modifier.sh" --input-dummy=0 --output-http="http://localhost:9000" --verbose dbash: docker run -v `pwd`:/gopath/src/gor -t -i gor /bin/bash \ No newline at end of file diff --git a/emitter.go b/emitter.go index 0bc8086..84c0acb 100644 --- a/emitter.go +++ b/emitter.go @@ -26,6 +26,7 @@ func CopyMulty(src io.Reader, writers ...io.Writer) (err error) { for { nr, er := src.Read(buf) + if nr > 0 && len(buf) > nr { Debug("Sending", src, ": ", string(buf[0:nr])) diff --git a/examples/echo_modifier.rb b/examples/echo_modifier.rb new file mode 100755 index 0000000..e3a489d --- /dev/null +++ b/examples/echo_modifier.rb @@ -0,0 +1,16 @@ +#!/usr/bin/env ruby +# encoding: utf-8 +while data = STDIN.gets + next unless data + data = data.chomp + + decoded = [data].pack("H*") + encoded = decoded.unpack("H*").first + + STDOUT.puts encoded + + + STDERR.puts "[DEBUG] Original data: #{data}" + STDERR.puts "[DEBUG] Decoded request: #{decoded}" + STDERR.puts "[DEBUG] Encoded data: #{encoded}" +end \ No newline at end of file diff --git a/examples/echo_modifier.sh b/examples/echo_modifier.sh new file mode 100755 index 0000000..1d56cdc --- /dev/null +++ b/examples/echo_modifier.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +while read line; do + decoded=$(echo "$line" | xxd -r -p) + encoded=$(echo "$decoded" | xxd -p | tr -d "\\n") + echo "$encoded" + + >&2 echo "[DEBUG] Original data: $line" + >&2 echo "[DEBUG] Decoded request: $decoded" + >&2 echo "[DEBUG] Encoded data: $encoded" +done; diff --git a/examples/echo_modifier/echo_modifier.go b/examples/echo_modifier/echo_modifier.go deleted file mode 100644 index 9dd0ea6..0000000 --- a/examples/echo_modifier/echo_modifier.go +++ /dev/null @@ -1,44 +0,0 @@ -package main - -import ( - "os" - "bufio" -) - -func main() { - reader := bufio.NewReader(os.Stdin) - data := make(chan []byte) - - go ReadStdin(data) - - for { - os.Stdout.Print(<- data, '¶') - } -} - -func ReadStdin(data chan []byte){ - for { - buf, err := reader.ReadBytes('¶') - buf_len := len(buf) - if buf_len > 0 { - new_buf_len := len(buf) - 2 - if new_buf_len > 0 { - new_buf := make([]byte, new_buf_len) - copy(new_buf, buf[:new_buf_len]) - data <- new_buf - if err != nil { - if err != io.EOF { - log.Printf("error: %s\n", err) - } - } - } - } - } -} - -while data = STDIN.gets(separator) - STDERR.puts "==== Start ====" - STDERR.puts data - puts data - STDERR.puts "==== End ====" -end \ No newline at end of file diff --git a/input_dummy.go b/input_dummy.go index a2a87ac..846e86a 100644 --- a/input_dummy.go +++ b/input_dummy.go @@ -30,7 +30,7 @@ func (i *DummyInput) emit() { for { select { case <-ticker.C: - i.data <- []byte("GET / HTTP/1.1\r\n\r\n") + i.data <- []byte("POST /pub/WWW/å HTTP/1.1\nHost: www.w3.org\r\n\r\na=1&b=2") } } } diff --git a/plugins.go b/plugins.go index 62bce40..ecafbfa 100644 --- a/plugins.go +++ b/plugins.go @@ -62,6 +62,7 @@ func registerPlugin(constructor interface{}, options ...interface{}) { Plugins.Inputs = append(Plugins.Inputs, plugin_wrapper.(io.Reader)) } + if _, ok := plugin.(io.Writer); ok { Plugins.Outputs = append(Plugins.Outputs, plugin_wrapper.(io.Writer)) } diff --git a/pong_modifier.rb b/pong_modifier.rb deleted file mode 100755 index e702507..0000000 --- a/pong_modifier.rb +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/env ruby -# encoding: utf-8 -require "base64" - -STDERR.puts "Starting modifier" -puts "Starting modifier" - -while data = STDIN.gets.chomp - STDERR.puts "==== Start ====" - STDERR.puts Base64.encode64(data) - puts data - STDERR.puts "==== End ====" -end \ No newline at end of file diff --git a/traffic_modifier.go b/traffic_modifier.go index 0e8a09a..ddc92d1 100644 --- a/traffic_modifier.go +++ b/traffic_modifier.go @@ -6,7 +6,8 @@ import ( "io" "os/exec" "os" - "encoding/base64" + "bufio" + "encoding/hex" ) type TrafficModifier struct { @@ -23,43 +24,74 @@ func NewTrafficModifier(plugin interface{}, command string) io.Reader { m := new(TrafficModifier) m.plugin = plugin m.command = command + m.data = make(chan []byte) - cmd := exec.Command("bash", "-c", command) - cmd.Stderr = os.Stderr + cmd := exec.Command(command) m.Stdout, _ = cmd.StdoutPipe() m.Stdin, _ = cmd.StdinPipe() - - m.Stdout = base64.NewDecoder(base64.StdEncoding, m.Stdout) + cmd.Stderr = os.Stderr go m.copy(m.Stdin, m.plugin.(io.Reader)) + go m.read(m.Stdout) - err := cmd.Run() + go func(){ + err := cmd.Start() - if (err != nil) { - log.Fatal(err) - } + if (err != nil) { + log.Fatal(err) + } + }() + + defer cmd.Wait() return m } func (m *TrafficModifier) copy(to io.Writer, from io.Reader) { buf := make([]byte, 5*1024*1024) + dst := make([]byte, len(buf)*2) for { - nr, er := from.Read(buf) + nr, _ := from.Read(buf) if nr > 0 && len(buf) > nr { - to.Write(base64.StdEncoding.Encode(buf)) + hex.Encode(dst, buf[0:nr]) + to.Write(dst[0:nr*2]) + to.Write([]byte("\r\n")) } } } -func (m *TrafficModifier) Read(data []byte) (n int, err error) { - n, err = m.Stdout.Read(data) +func (m *TrafficModifier) read(from io.Reader) { + buf := make([]byte, 5*1024*1024) + + scanner := bufio.NewScanner(from) + + for scanner.Scan() { + bytes := scanner.Bytes() + hex.Decode(buf, bytes) + + Debug("Received:", buf[0:len(bytes)/2]) + + m.data <- buf[0:len(bytes)/2] + } + + if err := scanner.Err(); err != nil { + fmt.Fprintln(os.Stderr, "Traffic modifier command failed:", err) + } return } +func (m *TrafficModifier) Read(data []byte) (int, error) { + Debug("Trying to read channel!") + buf := <- m.data + copy(data, buf) + + return len(buf), nil +} + + func (m *TrafficModifier) String() string { return fmt.Sprintf("Modifying traffic for %s using '%s' command", m.plugin, m.command) } From 674ffc620241cd0c2562ba7a917fa9ce322a463a Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Sun, 28 Jun 2015 00:16:19 +0500 Subject: [PATCH 03/64] Start adding tests for traffic modifier --- input_raw_test.go | 2 +- output_http_test.go | 10 ++-- plugins.go | 1 - traffic_modifier.go | 115 +++++++++++++++++++-------------------- traffic_modifier_test.go | 74 +++++++++++++++++++++++++ 5 files changed, 137 insertions(+), 65 deletions(-) create mode 100644 traffic_modifier_test.go diff --git a/input_raw_test.go b/input_raw_test.go index b1bcca6..2eeb6d9 100644 --- a/input_raw_test.go +++ b/input_raw_test.go @@ -13,7 +13,7 @@ func TestRAWInput(t *testing.T) { wg := new(sync.WaitGroup) quit := make(chan int) - listener := startHTTP(func(req *http.Request) {}) + listener := startHTTP(func(w http.ResponseWriter, req *http.Request) {}) input := NewRAWInput(listener.Addr().String()) output := NewTestOutput(func(data []byte) { diff --git a/output_http_test.go b/output_http_test.go index 2432d5c..b24f442 100644 --- a/output_http_test.go +++ b/output_http_test.go @@ -12,9 +12,9 @@ import ( "time" ) -func startHTTP(cb func(*http.Request)) net.Listener { +func startHTTP(cb func(http.ResponseWriter, *http.Request)) net.Listener { handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - cb(r) + cb(w, r) }) listener, _ := net.Listen("tcp", ":0") @@ -54,7 +54,7 @@ func TestHTTPOutput(t *testing.T) { headers := HTTPHeaders{HTTPHeader{"User-Agent", "Gor"}} methods := HTTPMethods{"GET", "PUT", "POST"} - listener := startHTTP(func(req *http.Request) { + listener := startHTTP(func(w http.ResponseWriter, req *http.Request) { if req.Header.Get("User-Agent") != "Gor" { t.Error("Wrong header") } @@ -104,7 +104,7 @@ func TestHTTPOutputChunkedEncoding(t *testing.T) { headers := HTTPHeaders{HTTPHeader{"User-Agent", "Gor"}} methods := HTTPMethods{"GET", "PUT", "POST"} - listener := startHTTP(func(req *http.Request) { + listener := startHTTP(func(w http.ResponseWriter, req *http.Request) { defer req.Body.Close() body, _ := ioutil.ReadAll(req.Body) @@ -140,7 +140,7 @@ func BenchmarkHTTPOutput(b *testing.B) { headers := HTTPHeaders{HTTPHeader{"User-Agent", "Gor"}} methods := HTTPMethods{"GET", "PUT", "POST"} - listener := startHTTP(func(req *http.Request) { + listener := startHTTP(func(w http.ResponseWriter, req *http.Request) { time.Sleep(50 * time.Millisecond) wg.Done() }) diff --git a/plugins.go b/plugins.go index ecafbfa..62bce40 100644 --- a/plugins.go +++ b/plugins.go @@ -62,7 +62,6 @@ func registerPlugin(constructor interface{}, options ...interface{}) { Plugins.Inputs = append(Plugins.Inputs, plugin_wrapper.(io.Reader)) } - if _, ok := plugin.(io.Writer); ok { Plugins.Outputs = append(Plugins.Outputs, plugin_wrapper.(io.Writer)) } diff --git a/traffic_modifier.go b/traffic_modifier.go index ddc92d1..7a34bb9 100644 --- a/traffic_modifier.go +++ b/traffic_modifier.go @@ -1,97 +1,96 @@ package main import ( - "fmt" - "log" - "io" - "os/exec" - "os" - "bufio" - "encoding/hex" + "bufio" + "encoding/hex" + "fmt" + "io" + "log" + "os" + "os/exec" ) type TrafficModifier struct { - plugin interface{} - command string + plugin interface{} + command string - data chan []byte + data chan []byte - Stdin io.Writer - Stdout io.Reader + Stdin io.Writer + Stdout io.Reader } func NewTrafficModifier(plugin interface{}, command string) io.Reader { - m := new(TrafficModifier) - m.plugin = plugin - m.command = command - m.data = make(chan []byte) + m := new(TrafficModifier) + m.plugin = plugin + m.command = command + m.data = make(chan []byte) - cmd := exec.Command(command) + cmd := exec.Command(command) - m.Stdout, _ = cmd.StdoutPipe() - m.Stdin, _ = cmd.StdinPipe() - cmd.Stderr = os.Stderr + m.Stdout, _ = cmd.StdoutPipe() + m.Stdin, _ = cmd.StdinPipe() + cmd.Stderr = os.Stderr - go m.copy(m.Stdin, m.plugin.(io.Reader)) - go m.read(m.Stdout) + go m.copy(m.Stdin, m.plugin.(io.Reader)) + go m.read(m.Stdout) - go func(){ - err := cmd.Start() + go func() { + err := cmd.Start() - if (err != nil) { - log.Fatal(err) - } - }() + if err != nil { + log.Fatal(err) + } + }() - defer cmd.Wait() + defer cmd.Wait() - return m + return m } func (m *TrafficModifier) copy(to io.Writer, from io.Reader) { - buf := make([]byte, 5*1024*1024) - dst := make([]byte, len(buf)*2) + buf := make([]byte, 5*1024*1024) + dst := make([]byte, len(buf)*2) - for { - nr, _ := from.Read(buf) - if nr > 0 && len(buf) > nr { - hex.Encode(dst, buf[0:nr]) - to.Write(dst[0:nr*2]) - to.Write([]byte("\r\n")) - } - } + for { + nr, _ := from.Read(buf) + if nr > 0 && len(buf) > nr { + hex.Encode(dst, buf[0:nr]) + to.Write(dst[0 : nr*2]) + to.Write([]byte("\r\n")) + } + } } func (m *TrafficModifier) read(from io.Reader) { - buf := make([]byte, 5*1024*1024) + buf := make([]byte, 5*1024*1024) - scanner := bufio.NewScanner(from) + scanner := bufio.NewScanner(from) - for scanner.Scan() { - bytes := scanner.Bytes() - hex.Decode(buf, bytes) + for scanner.Scan() { + bytes := scanner.Bytes() + hex.Decode(buf, bytes) - Debug("Received:", buf[0:len(bytes)/2]) + Debug("Received:", buf[0:len(bytes)/2]) - m.data <- buf[0:len(bytes)/2] - } + m.data <- buf[0 : len(bytes)/2] + } - if err := scanner.Err(); err != nil { - fmt.Fprintln(os.Stderr, "Traffic modifier command failed:", err) - } + if err := scanner.Err(); err != nil { + fmt.Fprintln(os.Stderr, "Traffic modifier command failed:", err) + } - return + return } func (m *TrafficModifier) Read(data []byte) (int, error) { - Debug("Trying to read channel!") - buf := <- m.data - copy(data, buf) + Debug("Trying to read channel!") + buf := <-m.data + copy(data, buf) - return len(buf), nil + return len(buf), nil } - func (m *TrafficModifier) String() string { - return fmt.Sprintf("Modifying traffic for %s using '%s' command", m.plugin, m.command) + return fmt.Sprintf("Modifying traffic for %s using '%s' command", m.plugin, m.command) } diff --git a/traffic_modifier_test.go b/traffic_modifier_test.go new file mode 100644 index 0000000..547876e --- /dev/null +++ b/traffic_modifier_test.go @@ -0,0 +1,74 @@ +package main + +import ( + _ "bufio" + "bytes" + "crypto/rand" + _ "io" + "io/ioutil" + _ "log" + _ "net" + "net/http" + "sync" + "testing" +) + +// Simple service that generate token on request, and require this token for accesing to secure area +func NewFakeSecureService(wg *sync.WaitGroup) string { + active_tokens := make([][]byte, 0) + + listener := startHTTP(func(w http.ResponseWriter, req *http.Request) { + switch req.URL.Path { + case "/token": + // Generate random token + token_length := 10 + token := make([]byte, token_length) + rand.Read(token) + + w.Write(token) + + active_tokens = append(active_tokens, token) + case "/secure": + token := []byte(req.URL.Query().Get("token")) + + for _, t := range active_tokens { + if bytes.Equal(t, token) { + w.WriteHeader(http.StatusAccepted) + } else { + w.WriteHeader(http.StatusForbidden) + } + } + } + + wg.Done() + }) + + return "http://" + listener.Addr().String() +} + +func TestFakeSecureService(t *testing.T) { + var resp *http.Response + + wg := new(sync.WaitGroup) + + addr := NewFakeSecureService(wg) + + wg.Add(3) + + resp, _ = http.Get(addr + "/token") + token, _ := ioutil.ReadAll(resp.Body) + + // Right token + resp, _ = http.Get(addr + "/secure?token=" + string(token)) + if resp.StatusCode != http.StatusAccepted { + t.Error("Valid token should returns wrong status:", resp.StatusCode) + } + + // Wrong tokens forbidden + resp, _ = http.Get(addr + "/secure?token=wrong") + if resp.StatusCode != http.StatusForbidden { + t.Error("Wrong tokens should be forbidden, instead:", resp.StatusCode) + } + + wg.Wait() +} From 98f941b69ac01ce85e8c3a0593b1a405bae28022 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Sun, 28 Jun 2015 00:23:21 +0500 Subject: [PATCH 04/64] Add support for complex commands --- Makefile | 2 +- traffic_modifier.go | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 18b69e5..693a80f 100644 --- a/Makefile +++ b/Makefile @@ -22,7 +22,7 @@ dbench: # Used mainly for debugging, because docker container do not have access to parent machine ports drun: - docker run -v `pwd`:/gopath/src/gor -t -i gor go run $(SOURCE) --input-modifier="./examples/echo_modifier.sh" --input-dummy=0 --output-http="http://localhost:9000" --verbose + docker run -v `pwd`:/gopath/src/gor -t -i gor go run $(SOURCE) --input-modifier="bash ./examples/echo_modifier.sh" --input-dummy=0 --output-http="http://localhost:9000" --verbose dbash: docker run -v `pwd`:/gopath/src/gor -t -i gor /bin/bash \ No newline at end of file diff --git a/traffic_modifier.go b/traffic_modifier.go index 7a34bb9..6a4a5b6 100644 --- a/traffic_modifier.go +++ b/traffic_modifier.go @@ -8,6 +8,7 @@ import ( "log" "os" "os/exec" + "strings" ) type TrafficModifier struct { @@ -26,7 +27,8 @@ func NewTrafficModifier(plugin interface{}, command string) io.Reader { m.command = command m.data = make(chan []byte) - cmd := exec.Command(command) + commands := strings.Split(command, " ") + cmd := exec.Command(commands[0], commands[1:]...) m.Stdout, _ = cmd.StdoutPipe() m.Stdin, _ = cmd.StdinPipe() From 3e86456e36e94f8d18c10238f17736d577e4c696 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Sun, 28 Jun 2015 22:00:55 +0500 Subject: [PATCH 05/64] Add more debug --- Makefile | 2 +- input_raw.go | 2 ++ output_http.go | 3 +- traffic_modifier_test.go | 71 +++++++++++++++++++++++++++++++++------- 4 files changed, 65 insertions(+), 13 deletions(-) diff --git a/Makefile b/Makefile index 693a80f..b79497d 100644 --- a/Makefile +++ b/Makefile @@ -12,7 +12,7 @@ dbuild: docker build -t gor . dtest: - docker run -v `pwd`:/gopath/src/gor -t -i --env GORACE="halt_on_error=1" gor go test $(ARGS) -race -v + docker run -v `pwd`:/gopath/src/gor -t -i --env GORACE="halt_on_error=1" gor go test $(ARGS) -race -v --verbose dfmt: docker run -v `pwd`:/gopath/src/gor -t -i gor go fmt diff --git a/input_raw.go b/input_raw.go index 3a6a960..9653559 100644 --- a/input_raw.go +++ b/input_raw.go @@ -32,6 +32,8 @@ func (i *RAWInput) Read(data []byte) (int, error) { func (i *RAWInput) listen(address string) { address = strings.Replace(address, "[::]", "127.0.0.1", -1) + Debug("Listening for traffic on: " + address) + host, port, err := net.SplitHostPort(address) if err != nil { diff --git a/output_http.go b/output_http.go index 46a342a..ba1afa2 100644 --- a/output_http.go +++ b/output_http.go @@ -73,6 +73,7 @@ type HTTPOutput struct { address string limit int queue chan []byte + responses chan []byte redirectLimit int @@ -91,7 +92,7 @@ type HTTPOutput struct { queueStats *GorStat } -func NewHTTPOutput(address string, headers HTTPHeaders, methods HTTPMethods, urlRegexp HTTPUrlRegexp, headerFilters HTTPHeaderFilters, headerHashFilters HTTPHeaderHashFilters, elasticSearchAddr string, outputHTTPUrlRewrite UrlRewriteMap, outputHTTPRedirects int) io.Writer { +func NewHTTPOutput(address string, headers HTTPHeaders, methods HTTPMethods, urlRegexp HTTPUrlRegexp, headerFilters HTTPHeaderFilters, headerHashFilters HTTPHeaderHashFilters, elasticSearchAddr string, outputHTTPUrlRewrite UrlRewriteMap, outputHTTPRedirects int) io.ReadWriter { o := new(HTTPOutput) diff --git a/traffic_modifier_test.go b/traffic_modifier_test.go index 547876e..82e786e 100644 --- a/traffic_modifier_test.go +++ b/traffic_modifier_test.go @@ -4,13 +4,14 @@ import ( _ "bufio" "bytes" "crypto/rand" - _ "io" + "io" "io/ioutil" _ "log" _ "net" "net/http" "sync" "testing" + "strings" ) // Simple service that generate token on request, and require this token for accesing to secure area @@ -18,32 +19,40 @@ func NewFakeSecureService(wg *sync.WaitGroup) string { active_tokens := make([][]byte, 0) listener := startHTTP(func(w http.ResponseWriter, req *http.Request) { + Debug("Received request: " + req.URL.String()) + switch req.URL.Path { case "/token": // Generate random token token_length := 10 token := make([]byte, token_length) rand.Read(token) - - w.Write(token) - active_tokens = append(active_tokens, token) + + w.Write(token) case "/secure": token := []byte(req.URL.Query().Get("token")) + token_found := false for _, t := range active_tokens { if bytes.Equal(t, token) { - w.WriteHeader(http.StatusAccepted) - } else { - w.WriteHeader(http.StatusForbidden) + token_found = true + break } } + + if token_found { + w.WriteHeader(http.StatusAccepted) + } else { + w.WriteHeader(http.StatusForbidden) + } } wg.Done() }) - return "http://" + listener.Addr().String() + address := strings.Replace(listener.Addr().String(), "[::]", "127.0.0.1", -1) + return address } func TestFakeSecureService(t *testing.T) { @@ -55,20 +64,60 @@ func TestFakeSecureService(t *testing.T) { wg.Add(3) - resp, _ = http.Get(addr + "/token") + resp, _ = http.Get("http://" + addr + "/token") token, _ := ioutil.ReadAll(resp.Body) // Right token - resp, _ = http.Get(addr + "/secure?token=" + string(token)) + resp, _ = http.Get("http://" + addr + "/secure?token=" + string(token)) if resp.StatusCode != http.StatusAccepted { t.Error("Valid token should returns wrong status:", resp.StatusCode) } // Wrong tokens forbidden - resp, _ = http.Get(addr + "/secure?token=wrong") + resp, _ = http.Get("http://" + addr + "/secure?token=wrong") if resp.StatusCode != http.StatusForbidden { t.Error("Wrong tokens should be forbidden, instead:", resp.StatusCode) } wg.Wait() } + +func TestTrafficModifier(t *testing.T) { + var resp *http.Response + + wg := new(sync.WaitGroup) + + from := NewFakeSecureService(wg) + to := NewFakeSecureService(wg) + + quit := make(chan int) + + // Catch traffic from one service + input := NewRAWInput(from) + + // And redirect to another + headers := HTTPHeaders{HTTPHeader{"User-Agent", "Gor"}} + methods := HTTPMethods{"GET", "PUT", "POST"} + output := NewHTTPOutput(to, headers, methods, HTTPUrlRegexp{}, HTTPHeaderFilters{}, HTTPHeaderHashFilters{}, "", UrlRewriteMap{}, 0) + + Plugins.Inputs = []io.Reader{input} + Plugins.Outputs = []io.Writer{output} + + // Start Gor + go Start(quit) + + // Should receive 2 requests from original + 2 from replayed + wg.Add(4) + + // Sending traffic to original service + resp, _ = http.Get("http://" + from + "/token") + token, _ := ioutil.ReadAll(resp.Body) + + resp, _ = http.Get("http://" + from + "/secure?token=" + string(token)) + if resp.StatusCode != http.StatusAccepted { + t.Error("Valid token should returns wrong status:", resp.StatusCode) + } + + wg.Wait() + close(quit) +} From e64aac9364c5aee4d7d3436081671cbb3cf06cc0 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Thu, 9 Jul 2015 20:24:27 +0500 Subject: [PATCH 06/64] Rename input modifier to middleware --- traffic_modifier.go => middleware.go | 14 +++++++------- traffic_modifier_test.go => middleware_test.go | 2 +- plugins.go | 6 +++--- settings.go | 4 ++-- 4 files changed, 13 insertions(+), 13 deletions(-) rename traffic_modifier.go => middleware.go (80%) rename traffic_modifier_test.go => middleware_test.go (98%) diff --git a/traffic_modifier.go b/middleware.go similarity index 80% rename from traffic_modifier.go rename to middleware.go index 6a4a5b6..5b20f77 100644 --- a/traffic_modifier.go +++ b/middleware.go @@ -11,7 +11,7 @@ import ( "strings" ) -type TrafficModifier struct { +type Middleware struct { plugin interface{} command string @@ -21,8 +21,8 @@ type TrafficModifier struct { Stdout io.Reader } -func NewTrafficModifier(plugin interface{}, command string) io.Reader { - m := new(TrafficModifier) +func NewMiddleware(plugin interface{}, command string) io.Reader { + m := new(Middleware) m.plugin = plugin m.command = command m.data = make(chan []byte) @@ -50,7 +50,7 @@ func NewTrafficModifier(plugin interface{}, command string) io.Reader { return m } -func (m *TrafficModifier) copy(to io.Writer, from io.Reader) { +func (m *Middleware) copy(to io.Writer, from io.Reader) { buf := make([]byte, 5*1024*1024) dst := make([]byte, len(buf)*2) @@ -64,7 +64,7 @@ func (m *TrafficModifier) copy(to io.Writer, from io.Reader) { } } -func (m *TrafficModifier) read(from io.Reader) { +func (m *Middleware) read(from io.Reader) { buf := make([]byte, 5*1024*1024) scanner := bufio.NewScanner(from) @@ -85,7 +85,7 @@ func (m *TrafficModifier) read(from io.Reader) { return } -func (m *TrafficModifier) Read(data []byte) (int, error) { +func (m *Middleware) Read(data []byte) (int, error) { Debug("Trying to read channel!") buf := <-m.data copy(data, buf) @@ -93,6 +93,6 @@ func (m *TrafficModifier) Read(data []byte) (int, error) { return len(buf), nil } -func (m *TrafficModifier) String() string { +func (m *Middleware) String() string { return fmt.Sprintf("Modifying traffic for %s using '%s' command", m.plugin, m.command) } diff --git a/traffic_modifier_test.go b/middleware_test.go similarity index 98% rename from traffic_modifier_test.go rename to middleware_test.go index 5307256..347453c 100644 --- a/traffic_modifier_test.go +++ b/middleware_test.go @@ -82,7 +82,7 @@ func TestFakeSecureService(t *testing.T) { wg.Wait() } -func TestTrafficModifier(t *testing.T) { +func TestMiddleware(t *testing.T) { var resp *http.Response wg := new(sync.WaitGroup) diff --git a/plugins.go b/plugins.go index b06b8cc..e83a7f2 100644 --- a/plugins.go +++ b/plugins.go @@ -10,7 +10,7 @@ type InOutPlugins struct { Inputs []io.Reader Outputs []io.Writer - Modifiers []TrafficModifier + Middleware []Middleware } type ReaderOrWriter interface{} @@ -56,8 +56,8 @@ func registerPlugin(constructor interface{}, options ...interface{}) { } if _, ok := plugin.(io.Reader); ok { - for _, options := range Settings.inputModifier { - plugin_wrapper = NewTrafficModifier(plugin_wrapper, options) + for _, options := range Settings.middleware { + plugin_wrapper = NewMiddleware(plugin_wrapper, options) } Plugins.Inputs = append(Plugins.Inputs, plugin_wrapper.(io.Reader)) } diff --git a/settings.go b/settings.go index f7e168c..537b97a 100644 --- a/settings.go +++ b/settings.go @@ -41,7 +41,7 @@ type AppSettings struct { inputRAW MultiOption - inputModifier MultiOption + middleware MultiOption inputHTTP MultiOption outputHTTP MultiOption @@ -78,7 +78,7 @@ func init() { flag.Var(&Settings.inputRAW, "input-raw", "Capture traffic from given port (use RAW sockets and require *sudo* access):\n\t# Capture traffic from 8080 port\n\tgor --input-raw :8080 --output-http staging.com") - flag.Var(&Settings.inputModifier, "input-modifier", "Used for modifying input traffic using external command") + flag.Var(&Settings.middleware, "middleware", "Used for modifying input traffic using external command") flag.Var(&Settings.inputHTTP, "input-http", "Read requests from HTTP, should be explicitly sent from your application:\n\t# Listen for http on 9000\n\tgor --input-http :9000 --output-http staging.com") From f812120dfc2152d4f9747f3234581ccaac0f0d09 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Fri, 10 Jul 2015 10:24:00 +0500 Subject: [PATCH 07/64] Support only 1 middleware and rewrite tests --- emitter.go | 2 ++ http_client.go | 20 ++++++++++------ middleware_test.go | 60 ++++++++++++++++++++++++---------------------- plugins.go | 4 ++-- proto/proto.go | 5 ++++ settings.go | 6 ++--- 6 files changed, 56 insertions(+), 41 deletions(-) diff --git a/emitter.go b/emitter.go index 174cfc8..3fdbac4 100644 --- a/emitter.go +++ b/emitter.go @@ -31,6 +31,8 @@ func CopyMulty(src io.Reader, writers ...io.Writer) (err error) { if nr > 0 && len(buf) > nr { payload := buf[0:nr] + Debug("[EMITTER] Received payload:", string(payload)) + if modifier != nil { payload = modifier.Rewrite(payload) diff --git a/http_client.go b/http_client.go index 4e44f77..d142e21 100644 --- a/http_client.go +++ b/http_client.go @@ -72,7 +72,7 @@ func (c *HTTPClient) Disconnect() { if c.conn != nil { c.conn.Close() c.conn = nil - Debug("Disconnected: ", c.baseURL) + Debug("[HTTP] Disconnected: ", c.baseURL) } } @@ -90,7 +90,7 @@ func (c *HTTPClient) isAlive() bool { func (c *HTTPClient) Send(data []byte) (response []byte, err error) { if c.conn == nil || !c.isAlive() { - Debug("Connecting:", c.baseURL) + Debug("[HTTP] Connecting:", c.baseURL) c.Connect() } @@ -101,11 +101,11 @@ func (c *HTTPClient) Send(data []byte) (response []byte, err error) { data = proto.SetHost(data, []byte(c.baseURL), []byte(c.host)) if c.config.Debug { - Debug("Sending:", string(data)) + Debug("[HTTP] Sending:", string(data)) } if _, err = c.conn.Write(data); err != nil { - Debug("Write error:", err, c.baseURL) + Debug("[HTTP] Write error:", err, c.baseURL) return } @@ -113,14 +113,14 @@ func (c *HTTPClient) Send(data []byte) (response []byte, err error) { n, err := c.conn.Read(c.respBuf) if err != nil { - Debug("READ ERRORR!", err, c.conn) + Debug("[HTTP] READ ERRORR!", err, c.conn) return } payload := c.respBuf[:n] if c.config.Debug { - Debug("Received:", string(payload)) + Debug("[HTTP] Received:", string(payload)) } if c.config.FollowRedirects > 0 && c.redirectsCount < c.config.FollowRedirects { @@ -134,7 +134,7 @@ func (c *HTTPClient) Send(data []byte) (response []byte, err error) { redirectPayload := []byte("GET " + string(location) + " HTTP/1.1\r\n\r\n") if c.config.Debug { - Debug("Redirecting to: " + string(location)) + Debug("[HTTP] Redirecting to: " + string(location)) } return c.Send(redirectPayload) @@ -145,3 +145,9 @@ func (c *HTTPClient) Send(data []byte) (response []byte, err error) { return payload, err } + +func (c *HTTPClient) Get(path string) (response []byte, err error) { + payload := "GET " + path + " HTTP/1.1\r\n\r\n" + + return c.Send([]byte(payload)) +} \ No newline at end of file diff --git a/middleware_test.go b/middleware_test.go index 347453c..7ce251a 100644 --- a/middleware_test.go +++ b/middleware_test.go @@ -1,22 +1,20 @@ package main import ( - _ "bufio" "bytes" "crypto/rand" "io" - "io/ioutil" - _ "log" - _ "net" "net/http" "sync" "testing" "strings" + "github.com/buger/gor/proto" + "encoding/hex" ) // Simple service that generate token on request, and require this token for accesing to secure area func NewFakeSecureService(wg *sync.WaitGroup) string { - active_tokens := make([][]byte, 0) + active_tokens := make([]string, 0) listener := startHTTP(func(w http.ResponseWriter, req *http.Request) { Debug("Received request: " + req.URL.String()) @@ -25,17 +23,18 @@ func NewFakeSecureService(wg *sync.WaitGroup) string { case "/token": // Generate random token token_length := 10 - token := make([]byte, token_length) - rand.Read(token) + buf := make([]byte, token_length) + rand.Read(buf) + token := hex.EncodeToString(buf) active_tokens = append(active_tokens, token) - w.Write(token) + w.Write([]byte(token)) case "/secure": - token := []byte(req.URL.Query().Get("token")) + token := req.URL.Query().Get("token") token_found := false for _, t := range active_tokens { - if bytes.Equal(t, token) { + if t == token { token_found = true break } @@ -56,7 +55,7 @@ func NewFakeSecureService(wg *sync.WaitGroup) string { } func TestFakeSecureService(t *testing.T) { - var resp *http.Response + var resp, token []byte wg := new(sync.WaitGroup) @@ -64,26 +63,27 @@ func TestFakeSecureService(t *testing.T) { wg.Add(3) - resp, _ = http.Get("http://" + addr + "/token") - token, _ := ioutil.ReadAll(resp.Body) + client := NewHTTPClient("http://" + addr, &HTTPClientConfig{Debug: true}) + resp, _ = client.Get("/token") + token = proto.Body(resp) - // Right token - resp, _ = http.Get("http://" + addr + "/secure?token=" + string(token)) - if resp.StatusCode != http.StatusAccepted { - t.Error("Valid token should returns wrong status:", resp.StatusCode) - } + // Right token + resp, _ = client.Get("/secure?token=" + string(token)) + if !bytes.Equal(proto.Status(resp), []byte("202")) { + t.Error("Valid token should return status 202:", string(proto.Status(resp))) + } - // Wrong tokens forbidden - resp, _ = http.Get("http://" + addr + "/secure?token=wrong") - if resp.StatusCode != http.StatusForbidden { - t.Error("Wrong tokens should be forbidden, instead:", resp.StatusCode) + // Wrong tokens forbidden + resp, _ = client.Get("/secure?token=wrong") + if !bytes.Equal(proto.Status(resp), []byte("403")) { + t.Error("Wrong token should returns status 403:", string(proto.Status(resp))) } wg.Wait() } func TestMiddleware(t *testing.T) { - var resp *http.Response + var resp, token []byte wg := new(sync.WaitGroup) @@ -107,13 +107,15 @@ func TestMiddleware(t *testing.T) { // Should receive 2 requests from original + 2 from replayed wg.Add(4) - // Sending traffic to original service - resp, _ = http.Get("http://" + from + "/token") - token, _ := ioutil.ReadAll(resp.Body) + client := NewHTTPClient("http://" + from, &HTTPClientConfig{Debug: true}) - resp, _ = http.Get("http://" + from + "/secure?token=" + string(token)) - if resp.StatusCode != http.StatusAccepted { - t.Error("Valid token should returns wrong status:", resp.StatusCode) + // Sending traffic to original service + resp, _ = client.Get("/token") + token = proto.Body(resp) + + resp, _ = client.Get("/secure?token=" + string(token)) + if !bytes.Equal(proto.Status(resp), []byte("202")) { + t.Error("Valid token should return 202:", proto.Status(resp)) } wg.Wait() diff --git a/plugins.go b/plugins.go index e83a7f2..adfc242 100644 --- a/plugins.go +++ b/plugins.go @@ -56,8 +56,8 @@ func registerPlugin(constructor interface{}, options ...interface{}) { } if _, ok := plugin.(io.Reader); ok { - for _, options := range Settings.middleware { - plugin_wrapper = NewMiddleware(plugin_wrapper, options) + if len(Settings.middleware) > 0 { + plugin_wrapper = NewMiddleware(plugin_wrapper, Settings.middleware) } Plugins.Inputs = append(Plugins.Inputs, plugin_wrapper.(io.Reader)) } diff --git a/proto/proto.go b/proto/proto.go index 626d289..9ca60e5 100644 --- a/proto/proto.go +++ b/proto/proto.go @@ -68,6 +68,11 @@ func AddHeader(payload, name, value []byte) []byte { return byteutils.Insert(payload, mimeStart, header) } +func Body(payload []byte) []byte { + // 4 -> len(EMPTY_LINE) + return payload[MIMEHeadersEndPos(payload) + 4:] +} + func Path(payload []byte) []byte { start := bytes.IndexByte(payload, ' ') start += 1 diff --git a/settings.go b/settings.go index 537b97a..5c34f46 100644 --- a/settings.go +++ b/settings.go @@ -41,7 +41,7 @@ type AppSettings struct { inputRAW MultiOption - middleware MultiOption + middleware string inputHTTP MultiOption outputHTTP MultiOption @@ -78,7 +78,7 @@ func init() { flag.Var(&Settings.inputRAW, "input-raw", "Capture traffic from given port (use RAW sockets and require *sudo* access):\n\t# Capture traffic from 8080 port\n\tgor --input-raw :8080 --output-http staging.com") - flag.Var(&Settings.middleware, "middleware", "Used for modifying input traffic using external command") + flag.StringVar(&Settings.middleware, "middleware", "", "Used for modifying traffic using external command") flag.Var(&Settings.inputHTTP, "input-http", "Read requests from HTTP, should be explicitly sent from your application:\n\t# Listen for http on 9000\n\tgor --input-http :9000 --output-http staging.com") @@ -117,7 +117,7 @@ func init() { func Debug(args ...interface{}) { if Settings.verbose { - log.Print("[DEBUG] ") + fmt.Print("[DEBUG] ") log.Println(args...) } } From 5e6f7e03a763bdb97b343d8e67afe802c2b2ec15 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Fri, 17 Jul 2015 20:58:31 +0500 Subject: [PATCH 08/64] First middleware tests! :dance: --- emitter.go | 31 ++++++++++++-- examples/echo_modifier.rb | 6 +-- examples/echo_modifier.sh | 6 +-- middleware.go | 19 ++++----- middleware_test.go | 85 ++++++++++++++++++++++++++++++++++----- plugins.go | 7 ---- 6 files changed, 120 insertions(+), 34 deletions(-) diff --git a/emitter.go b/emitter.go index 3fdbac4..8237f6f 100644 --- a/emitter.go +++ b/emitter.go @@ -3,11 +3,36 @@ package main import ( "io" "time" + "crypto/rand" ) +func uuid() []byte { + b := make([]byte, 16) + rand.Read(b) + return b +} + + func Start(stop chan int) { - for _, in := range Plugins.Inputs { - go CopyMulty(in, Plugins.Outputs...) + if Settings.middleware != "" { + middleware := NewMiddleware(Settings.middleware) + + for _, in := range Plugins.Inputs { + middleware.ReadFrom(in) + } + + // We going only to read responses, so using same ReadFrom method + for _, out := range Plugins.Outputs { + if r, ok := out.(io.Reader); ok { + middleware.ReadFrom(r) + } + } + + go CopyMulty(middleware, Plugins.Outputs...) + } else { + for _, in := range Plugins.Inputs { + go CopyMulty(in, Plugins.Outputs...) + } } for { @@ -31,7 +56,7 @@ func CopyMulty(src io.Reader, writers ...io.Writer) (err error) { if nr > 0 && len(buf) > nr { payload := buf[0:nr] - Debug("[EMITTER] Received payload:", string(payload)) + Debug("[EMITTER] input:", string(payload)) if modifier != nil { payload = modifier.Rewrite(payload) diff --git a/examples/echo_modifier.rb b/examples/echo_modifier.rb index e3a489d..a20beb8 100755 --- a/examples/echo_modifier.rb +++ b/examples/echo_modifier.rb @@ -10,7 +10,7 @@ while data = STDIN.gets STDOUT.puts encoded - STDERR.puts "[DEBUG] Original data: #{data}" - STDERR.puts "[DEBUG] Decoded request: #{decoded}" - STDERR.puts "[DEBUG] Encoded data: #{encoded}" + STDERR.puts "[DEBUG][MIDDLEWARE] Original data: #{data}" + STDERR.puts "[DEBUG][MIDDLEWARE] Decoded request: #{decoded}" + STDERR.puts "[DEBUG][MIDDLEWARE] Encoded data: #{encoded}" end \ No newline at end of file diff --git a/examples/echo_modifier.sh b/examples/echo_modifier.sh index 1d56cdc..22c7386 100755 --- a/examples/echo_modifier.sh +++ b/examples/echo_modifier.sh @@ -4,7 +4,7 @@ while read line; do encoded=$(echo "$decoded" | xxd -p | tr -d "\\n") echo "$encoded" - >&2 echo "[DEBUG] Original data: $line" - >&2 echo "[DEBUG] Decoded request: $decoded" - >&2 echo "[DEBUG] Encoded data: $encoded" + >&2 echo "[DEBUG][MIDDLEWARE] Original data: $line" + >&2 echo "[DEBUG][MIDDLEWARE] Decoded request: $decoded" + >&2 echo "[DEBUG][MIDDLEWARE] Encoded data: $encoded" done; diff --git a/middleware.go b/middleware.go index 5b20f77..dc387eb 100644 --- a/middleware.go +++ b/middleware.go @@ -12,7 +12,6 @@ import ( ) type Middleware struct { - plugin interface{} command string data chan []byte @@ -21,11 +20,10 @@ type Middleware struct { Stdout io.Reader } -func NewMiddleware(plugin interface{}, command string) io.Reader { +func NewMiddleware(command string) *Middleware { m := new(Middleware) - m.plugin = plugin m.command = command - m.data = make(chan []byte) + m.data = make(chan []byte, 1000) commands := strings.Split(command, " ") cmd := exec.Command(commands[0], commands[1:]...) @@ -34,7 +32,6 @@ func NewMiddleware(plugin interface{}, command string) io.Reader { m.Stdin, _ = cmd.StdinPipe() cmd.Stderr = os.Stderr - go m.copy(m.Stdin, m.plugin.(io.Reader)) go m.read(m.Stdout) go func() { @@ -43,13 +40,17 @@ func NewMiddleware(plugin interface{}, command string) io.Reader { if err != nil { log.Fatal(err) } + + cmd.Wait() }() - defer cmd.Wait() - return m } +func (m *Middleware) ReadFrom(plugin io.Reader) { + go m.copy(m.Stdin, plugin) +} + func (m *Middleware) copy(to io.Writer, from io.Reader) { buf := make([]byte, 5*1024*1024) dst := make([]byte, len(buf)*2) @@ -59,7 +60,7 @@ func (m *Middleware) copy(to io.Writer, from io.Reader) { if nr > 0 && len(buf) > nr { hex.Encode(dst, buf[0:nr]) to.Write(dst[0 : nr*2]) - to.Write([]byte("\r\n")) + to.Write([]byte("\n")) } } } @@ -94,5 +95,5 @@ func (m *Middleware) Read(data []byte) (int, error) { } func (m *Middleware) String() string { - return fmt.Sprintf("Modifying traffic for %s using '%s' command", m.plugin, m.command) + return fmt.Sprintf("Modifying traffic using '%s' command", m.command) } diff --git a/middleware_test.go b/middleware_test.go index 7ce251a..e069d23 100644 --- a/middleware_test.go +++ b/middleware_test.go @@ -9,14 +9,18 @@ import ( "testing" "strings" "github.com/buger/gor/proto" + "net/http/httptest" "encoding/hex" + "time" ) +type fakeServiceCb func(string, int, []byte) + // Simple service that generate token on request, and require this token for accesing to secure area -func NewFakeSecureService(wg *sync.WaitGroup) string { +func NewFakeSecureService(wg *sync.WaitGroup, cb fakeServiceCb) string { active_tokens := make([]string, 0) - listener := startHTTP(func(w http.ResponseWriter, req *http.Request) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { Debug("Received request: " + req.URL.String()) switch req.URL.Path { @@ -29,6 +33,8 @@ func NewFakeSecureService(wg *sync.WaitGroup) string { active_tokens = append(active_tokens, token) w.Write([]byte(token)) + + cb(req.URL.Path, 200, []byte(token)) case "/secure": token := req.URL.Query().Get("token") token_found := false @@ -42,15 +48,17 @@ func NewFakeSecureService(wg *sync.WaitGroup) string { if token_found { w.WriteHeader(http.StatusAccepted) + cb(req.URL.Path, 202, []byte(nil)) } else { w.WriteHeader(http.StatusForbidden) + cb(req.URL.Path, 403, []byte(nil)) } } wg.Done() - }) + })) - address := strings.Replace(listener.Addr().String(), "[::]", "127.0.0.1", -1) + address := strings.Replace(server.Listener.Addr().String(), "[::]", "127.0.0.1", -1) return address } @@ -59,7 +67,9 @@ func TestFakeSecureService(t *testing.T) { wg := new(sync.WaitGroup) - addr := NewFakeSecureService(wg) + addr := NewFakeSecureService(wg, func(path string, status int, resp []byte){ + + }) wg.Add(3) @@ -82,13 +92,66 @@ func TestFakeSecureService(t *testing.T) { wg.Wait() } -func TestMiddleware(t *testing.T) { +func TestEchoMiddleware(t *testing.T) { + wg := new(sync.WaitGroup) + + from := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + wg.Done() + })) + to := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + wg.Done() + })) + + quit := make(chan int) + + // Catch traffic from one service + input := NewRAWInput(from.Listener.Addr().String()) + + // And redirect to another + output := NewHTTPOutput(to.URL, &HTTPOutputConfig{}) + + Plugins.Inputs = []io.Reader{input} + Plugins.Outputs = []io.Writer{output} + Settings.middleware = "./examples/echo_modifier.sh" + + // Start Gor + go Start(quit) + + time.Sleep(time.Millisecond) + + // Should receive 2 requests from original + 2 from replayed + wg.Add(4) + + client := NewHTTPClient(from.URL, &HTTPClientConfig{Debug: true}) + + // Request should be echoed + client.Get("/") + client.Get("/") + + wg.Wait() + close(quit) + Settings.middleware = "" +} + +func TestTokenMiddleware(t *testing.T) { var resp, token []byte wg := new(sync.WaitGroup) - from := NewFakeSecureService(wg) - to := NewFakeSecureService(wg) + from := NewFakeSecureService(wg, func(path string, status int, tok []byte){ + }) + to := NewFakeSecureService(wg, func(path string, status int, tok []byte){ + switch path { + case "/token": + if bytes.Equal(token, tok) { + t.Error("Tokens should not match") + } + case "/secure": + if status != 202 { + // t.Error("Server should receive valid rewritten token") + } + } + }) quit := make(chan int) @@ -100,10 +163,13 @@ func TestMiddleware(t *testing.T) { Plugins.Inputs = []io.Reader{input} Plugins.Outputs = []io.Writer{output} + // Settings.middleware = "./examples/echo_modifier.sh" // Start Gor go Start(quit) + time.Sleep(time.Millisecond) + // Should receive 2 requests from original + 2 from replayed wg.Add(4) @@ -120,4 +186,5 @@ func TestMiddleware(t *testing.T) { wg.Wait() close(quit) -} + Settings.middleware = "" +} \ No newline at end of file diff --git a/plugins.go b/plugins.go index adfc242..321317e 100644 --- a/plugins.go +++ b/plugins.go @@ -9,12 +9,8 @@ import ( type InOutPlugins struct { Inputs []io.Reader Outputs []io.Writer - - Middleware []Middleware } -type ReaderOrWriter interface{} - var Plugins *InOutPlugins = new(InOutPlugins) func extractLimitOptions(options string) (string, string) { @@ -56,9 +52,6 @@ func registerPlugin(constructor interface{}, options ...interface{}) { } if _, ok := plugin.(io.Reader); ok { - if len(Settings.middleware) > 0 { - plugin_wrapper = NewMiddleware(plugin_wrapper, Settings.middleware) - } Plugins.Inputs = append(Plugins.Inputs, plugin_wrapper.(io.Reader)) } From d62eccb655f048ac8434094e2b41ba2b20564b3e Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Tue, 21 Jul 2015 10:06:10 +0500 Subject: [PATCH 09/64] Make raw message expiration configurable to improve tests time --- input_raw.go | 7 +++++-- input_raw_test.go | 10 ++++++---- middleware_test.go | 4 ++-- raw_socket_listener/listener.go | 13 +++++++++++-- raw_socket_listener/tcp_message.go | 13 ++++++------- 5 files changed, 30 insertions(+), 17 deletions(-) diff --git a/input_raw.go b/input_raw.go index c4f3c34..613cbbe 100644 --- a/input_raw.go +++ b/input_raw.go @@ -5,19 +5,22 @@ import ( "log" "net" "strings" + "time" ) // RAWInput used for intercepting traffic for given address type RAWInput struct { data chan []byte address string + expire time.Duration } // NewRAWInput constructor for RAWInput. Accepts address with port as argument. -func NewRAWInput(address string) (i *RAWInput) { +func NewRAWInput(address string, expire time.Duration) (i *RAWInput) { i = new(RAWInput) i.data = make(chan []byte) i.address = address + i.expire = expire go i.listen(address) @@ -42,7 +45,7 @@ func (i *RAWInput) listen(address string) { log.Fatal("input-raw: error while parsing address", err) } - listener := raw.NewListener(host, port) + listener := raw.NewListener(host, port, i.expire) for { // Receiving TCPMessage object diff --git a/input_raw_test.go b/input_raw_test.go index 0f7b0c3..a145845 100644 --- a/input_raw_test.go +++ b/input_raw_test.go @@ -15,6 +15,8 @@ import ( "time" ) +const testRawExpire = time.Millisecond * 100 + func TestRAWInput(t *testing.T) { wg := new(sync.WaitGroup) @@ -22,7 +24,7 @@ func TestRAWInput(t *testing.T) { listener := startHTTP(func(w http.ResponseWriter, req *http.Request) {}) - input := NewRAWInput(listener.Addr().String()) + input := NewRAWInput(listener.Addr().String(), testRawExpire) output := NewTestOutput(func(data []byte) { wg.Done() }) @@ -64,7 +66,7 @@ func TestInputRAW100Expect(t *testing.T) { originAddr := strings.Replace(origin.Addr().String(), "[::]", "127.0.0.1", -1) - input := NewRAWInput(originAddr) + input := NewRAWInput(originAddr, testRawExpire) // We will use it to get content of raw HTTP request testOutput := NewTestOutput(func(data []byte) { @@ -121,7 +123,7 @@ func TestInputRAWChunkedEncoding(t *testing.T) { originAddr := strings.Replace(origin.Addr().String(), "[::]", "127.0.0.1", -1) - input := NewRAWInput(originAddr) + input := NewRAWInput(originAddr, testRawExpire) listener := startHTTP(func(w http.ResponseWriter, req *http.Request) { defer req.Body.Close() @@ -179,7 +181,7 @@ func TestInputRAWLargePayload(t *testing.T) { })) originAddr := strings.Replace(origin.Listener.Addr().String(), "[::]", "127.0.0.1", -1) - input := NewRAWInput(originAddr) + input := NewRAWInput(originAddr, testRawExpire) replay := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { req.Body = http.MaxBytesReader(w, req.Body, 1*1024*1024) diff --git a/middleware_test.go b/middleware_test.go index cfea8b4..cb7319a 100644 --- a/middleware_test.go +++ b/middleware_test.go @@ -105,7 +105,7 @@ func TestEchoMiddleware(t *testing.T) { quit := make(chan int) // Catch traffic from one service - input := NewRAWInput(from.Listener.Addr().String()) + input := NewRAWInput(from.Listener.Addr().String(), testRawExpire) // And redirect to another output := NewHTTPOutput(to.URL, &HTTPOutputConfig{}) @@ -156,7 +156,7 @@ func TestTokenMiddleware(t *testing.T) { quit := make(chan int) // Catch traffic from one service - input := NewRAWInput(from) + input := NewRAWInput(from, testRawExpire) // And redirect to another output := NewHTTPOutput(to, &HTTPOutputConfig{}) diff --git a/raw_socket_listener/listener.go b/raw_socket_listener/listener.go index f82ae6f..53857ac 100644 --- a/raw_socket_listener/listener.go +++ b/raw_socket_listener/listener.go @@ -18,6 +18,7 @@ import ( "log" "net" "strconv" + "time" ) // Listener handle traffic capture @@ -42,10 +43,12 @@ type Listener struct { addr string // IP to listen port int // Port to listen + + messageExpire time.Duration } // NewListener creates and initializes new Listener object -func NewListener(addr string, port string) (rawListener *Listener) { +func NewListener(addr string, port string, expire time.Duration) (rawListener *Listener) { rawListener = &Listener{} rawListener.packetsChan = make(chan *TCPPacket, 10000) @@ -59,6 +62,12 @@ func NewListener(addr string, port string) (rawListener *Listener) { rawListener.addr = addr rawListener.port, _ = strconv.Atoi(port) + if expire.Nanoseconds() == 0 { + expire = 2000 * time.Millisecond + } + + rawListener.messageExpire = expire + go rawListener.listen() go rawListener.readRAWSocket() @@ -157,7 +166,7 @@ func (t *Listener) processTCPPacket(packet *TCPPacket) { if !ok { // We sending messageDelChan channel, so message object can communicate with Listener and notify it if message completed - message = NewTCPMessage(mID, t.messageDelChan, packet.Ack) + message = NewTCPMessage(mID, t.messageDelChan, packet.Ack, &t.messageExpire) t.messages[mID] = message } diff --git a/raw_socket_listener/tcp_message.go b/raw_socket_listener/tcp_message.go index ddfb0f2..e6e31bd 100644 --- a/raw_socket_listener/tcp_message.go +++ b/raw_socket_listener/tcp_message.go @@ -6,9 +6,6 @@ import ( "time" ) -// MsgExpire specify period that message should wait before it considered as finished -const MsgExpire = 2000 * time.Millisecond - // TCPMessage ensure that all TCP packets for given request is received, and processed in right sequence // Its needed because all TCP message can be fragmented or re-transmitted // @@ -25,17 +22,19 @@ type TCPMessage struct { packetsChan chan *TCPPacket delChan chan *TCPMessage + + expire *time.Duration } // NewTCPMessage pointer created from a Acknowledgment number and a channel of messages readuy to be deleted -func NewTCPMessage(ID string, delChan chan *TCPMessage, Ack uint32) (msg *TCPMessage) { - msg = &TCPMessage{ID: ID, Ack: Ack} +func NewTCPMessage(ID string, delChan chan *TCPMessage, Ack uint32, expire *time.Duration) (msg *TCPMessage) { + msg = &TCPMessage{ID: ID, Ack: Ack, expire: expire} msg.packetsChan = make(chan *TCPPacket) msg.delChan = delChan // used for notifying that message completed or expired // Every time we receive packet we reset this timer - msg.timer = time.AfterFunc(MsgExpire, msg.Timeout) + msg.timer = time.AfterFunc(*msg.expire, msg.Timeout) go msg.listen() @@ -103,5 +102,5 @@ func (t *TCPMessage) AddPacket(packet *TCPPacket) { } // Reset message timeout timer - t.timer.Reset(MsgExpire) + t.timer.Reset(*t.expire) } From 35854196f05e0b8f7b2969c121a28c2ebd21b627 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Mon, 3 Aug 2015 20:07:28 +0300 Subject: [PATCH 10/64] Fix tests --- output_http_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/output_http_test.go b/output_http_test.go index ddc9110..a428c38 100644 --- a/output_http_test.go +++ b/output_http_test.go @@ -82,7 +82,7 @@ func TestHTTPOutputKeepOriginalHost(t *testing.T) { input := NewTestInput() - listener := startHTTP(func(req *http.Request) { + listener := startHTTP(func(w http.ResponseWriter, req *http.Request) { if req.Host != "custom-host.com" { t.Error("Wrong header", req.Host) } From 9ab8b3c11372848e76fa3ff87566f421649c24e6 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Mon, 10 Aug 2015 22:57:18 +0300 Subject: [PATCH 11/64] Add header to request and response --- emitter.go | 10 ++++++---- examples/echo_modifier.sh | 23 +++++++++++++++++++++-- input_raw.go | 9 ++++++++- middleware.go | 2 +- middleware_test.go | 7 +++++-- output_http.go | 20 ++++++++++++++++++++ 6 files changed, 61 insertions(+), 10 deletions(-) diff --git a/emitter.go b/emitter.go index 63453a5..ec5065c 100644 --- a/emitter.go +++ b/emitter.go @@ -56,7 +56,9 @@ func CopyMulty(src io.Reader, writers ...io.Writer) (err error) { if nr > 0 && len(buf) > nr { payload := buf[0:nr] - Debug("[EMITTER] input:", string(payload)) + if Settings.debug { + Debug("[EMITTER] input:", string(payload[0:500])) + } if modifier != nil { payload = modifier.Rewrite(payload) @@ -65,10 +67,10 @@ func CopyMulty(src io.Reader, writers ...io.Writer) (err error) { if len(payload) == 0 { continue } - } - if Settings.debug { - Debug("[EMITTER] Sending payload, size:", len(payload), "First 500 bytes:", string(payload[0:500])) + if Settings.debug { + Debug("[EMITTER] Rewrittern input:", len(payload), "First 500 bytes:", string(payload[0:500])) + } } if Settings.splitOutput { diff --git a/examples/echo_modifier.sh b/examples/echo_modifier.sh index 22c7386..9aded36 100755 --- a/examples/echo_modifier.sh +++ b/examples/echo_modifier.sh @@ -1,8 +1,27 @@ #!/usr/bin/env bash while read line; do decoded=$(echo "$line" | xxd -r -p) - encoded=$(echo "$decoded" | xxd -p | tr -d "\\n") - echo "$encoded" + + header=$(echo "$decoded" | head -n +1) + payload=$(echo "$decoded" | tail -n +2) + + encoded=$(echo -e "$header\n$payload" | xxd -p | tr -d "\\n") + + >&2 echo "" + >&2 echo "[DEBUG][MIDDLEWARE] ===================================" + + case ${header:0:1} in + "2") + >&2 echo "[DEBUG][MIDDLEWARE] Request type: Replayed Response" + ;; + "1") + >&2 echo "[DEBUG][MIDDLEWARE] Request type: Request" + echo "$encoded" + ;; + *) + >&2 echo "[DEBUG][MIDDLEWARE] Unknown request type $header" + esac + >&2 echo "[DEBUG][MIDDLEWARE] ===================================" >&2 echo "[DEBUG][MIDDLEWARE] Original data: $line" >&2 echo "[DEBUG][MIDDLEWARE] Decoded request: $decoded" diff --git a/input_raw.go b/input_raw.go index 613cbbe..48775fa 100644 --- a/input_raw.go +++ b/input_raw.go @@ -29,7 +29,14 @@ func NewRAWInput(address string, expire time.Duration) (i *RAWInput) { func (i *RAWInput) Read(data []byte) (int, error) { buf := <-i.data - copy(data, buf) + + if len(Settings.middleware) > 0 { + header := []byte("1\n") + copy(data[0:len(header)], header) + copy(data[len(header):], buf) + } else { + copy(data, buf) + } return len(buf), nil } diff --git a/middleware.go b/middleware.go index 20714bf..89080e2 100644 --- a/middleware.go +++ b/middleware.go @@ -74,7 +74,7 @@ func (m *Middleware) read(from io.Reader) { bytes := scanner.Bytes() hex.Decode(buf, bytes) - Debug("Received:", buf[0:len(bytes)/2]) + Debug("[MIDDLEWARE-MASTER] Received:", string(buf[0:len(bytes)/2])) m.data <- buf[0 : len(bytes)/2] } diff --git a/middleware_test.go b/middleware_test.go index cb7319a..d9274bc 100644 --- a/middleware_test.go +++ b/middleware_test.go @@ -99,6 +99,7 @@ func TestEchoMiddleware(t *testing.T) { wg.Done() })) to := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + panic("Asdasd") wg.Done() })) @@ -108,7 +109,7 @@ func TestEchoMiddleware(t *testing.T) { input := NewRAWInput(from.Listener.Addr().String(), testRawExpire) // And redirect to another - output := NewHTTPOutput(to.URL, &HTTPOutputConfig{}) + output := NewHTTPOutput(to.URL, &HTTPOutputConfig{Debug: true}) Plugins.Inputs = []io.Reader{input} Plugins.Outputs = []io.Writer{output} @@ -122,7 +123,7 @@ func TestEchoMiddleware(t *testing.T) { // Should receive 2 requests from original + 2 from replayed wg.Add(4) - client := NewHTTPClient(from.URL, &HTTPClientConfig{Debug: true}) + client := NewHTTPClient(from.URL, &HTTPClientConfig{Debug: false}) // Request should be echoed client.Get("/") @@ -131,6 +132,8 @@ func TestEchoMiddleware(t *testing.T) { wg.Wait() close(quit) Settings.middleware = "" + + time.Sleep(10*time.Millisecond) } func TestTokenMiddleware(t *testing.T) { diff --git a/output_http.go b/output_http.go index 43bab96..3288c30 100644 --- a/output_http.go +++ b/output_http.go @@ -60,6 +60,7 @@ func NewHTTPOutput(address string, config *HTTPOutputConfig) io.Writer { } o.queue = make(chan []byte, 100) + o.responses = make(chan []byte, 100) o.needWorker = make(chan int, 1) // Initial workers count @@ -152,15 +153,34 @@ func (o *HTTPOutput) Write(data []byte) (n int, err error) { return len(data), nil } +func (o *HTTPOutput) Read(data []byte) (int, error) { + buf := <- o.responses + header := []byte("2\n") + copy(data[0:2], header) + copy(data[2:], buf) + + return len(buf) + len(header), nil +} + func (o *HTTPOutput) sendRequest(client *HTTPClient, request []byte) { + if len(Settings.middleware) > 0 { + request = request[2:] + } + start := time.Now() resp, err := client.Send(request) stop := time.Now() + panic(string(resp)) + if err != nil { log.Println("Request error:", err) } + if len(Settings.middleware) > 0 { + o.responses <- resp + } + if o.elasticSearch != nil { o.elasticSearch.ResponseAnalyze(request, resp, start, stop) } From 5098e388a24a3b09871928b223b07e186a4cf981 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Tue, 11 Aug 2015 11:06:25 +0300 Subject: [PATCH 12/64] Fix tests, now properly receive responses --- examples/echo_modifier.sh | 22 +++++++++++----------- input_raw.go | 6 ++++-- middleware.go | 9 ++++++++- middleware_test.go | 3 +-- output_http.go | 2 -- 5 files changed, 24 insertions(+), 18 deletions(-) diff --git a/examples/echo_modifier.sh b/examples/echo_modifier.sh index 9aded36..e3f7dd9 100755 --- a/examples/echo_modifier.sh +++ b/examples/echo_modifier.sh @@ -1,29 +1,29 @@ #!/usr/bin/env bash while read line; do - decoded=$(echo "$line" | xxd -r -p) + decoded=$(echo -e "$line" | xxd -r -p) - header=$(echo "$decoded" | head -n +1) - payload=$(echo "$decoded" | tail -n +2) + header=$(echo -e "$decoded" | head -n +1) + payload=$(echo -e "$decoded" | tail -n +2) encoded=$(echo -e "$header\n$payload" | xxd -p | tr -d "\\n") >&2 echo "" - >&2 echo "[DEBUG][MIDDLEWARE] ===================================" + >&2 echo "[DEBUG][ECHO] ===================================" case ${header:0:1} in "2") - >&2 echo "[DEBUG][MIDDLEWARE] Request type: Replayed Response" + >&2 echo "[DEBUG][ECHO] Request type: Replayed Response" ;; "1") - >&2 echo "[DEBUG][MIDDLEWARE] Request type: Request" + >&2 echo "[DEBUG][ECHO] Request type: Request" echo "$encoded" ;; *) - >&2 echo "[DEBUG][MIDDLEWARE] Unknown request type $header" + >&2 echo "[DEBUG][ECHO] Unknown request type $header" esac - >&2 echo "[DEBUG][MIDDLEWARE] ===================================" + >&2 echo "[DEBUG][ECHO] ===================================" - >&2 echo "[DEBUG][MIDDLEWARE] Original data: $line" - >&2 echo "[DEBUG][MIDDLEWARE] Decoded request: $decoded" - >&2 echo "[DEBUG][MIDDLEWARE] Encoded data: $encoded" + >&2 echo "[DEBUG][ECHO] Original data: $line" + >&2 echo "[DEBUG][ECHO] Decoded request: $decoded" + >&2 echo "[DEBUG][ECHO] Encoded data: $encoded" done; diff --git a/input_raw.go b/input_raw.go index 48775fa..e29a273 100644 --- a/input_raw.go +++ b/input_raw.go @@ -34,11 +34,13 @@ func (i *RAWInput) Read(data []byte) (int, error) { header := []byte("1\n") copy(data[0:len(header)], header) copy(data[len(header):], buf) + + return len(buf)+len(header), nil } else { copy(data, buf) - } - return len(buf), nil + return len(buf), nil + } } func (i *RAWInput) listen(address string) { diff --git a/middleware.go b/middleware.go index 89080e2..c642cd2 100644 --- a/middleware.go +++ b/middleware.go @@ -58,9 +58,14 @@ func (m *Middleware) copy(to io.Writer, from io.Reader) { for { nr, _ := from.Read(buf) if nr > 0 && len(buf) > nr { + hex.Encode(dst, buf[0:nr]) to.Write(dst[0 : nr*2]) to.Write([]byte("\n")) + + if Settings.debug { + Debug("[MIDDLEWARE-MASTER] Sending:", string(buf[0:nr]), "From:", from) + } } } } @@ -74,7 +79,9 @@ func (m *Middleware) read(from io.Reader) { bytes := scanner.Bytes() hex.Decode(buf, bytes) - Debug("[MIDDLEWARE-MASTER] Received:", string(buf[0:len(bytes)/2])) + if Settings.debug { + Debug("[MIDDLEWARE-MASTER] Received:", string(buf[0:len(bytes)/2])) + } m.data <- buf[0 : len(bytes)/2] } diff --git a/middleware_test.go b/middleware_test.go index d9274bc..a75d6bc 100644 --- a/middleware_test.go +++ b/middleware_test.go @@ -99,7 +99,6 @@ func TestEchoMiddleware(t *testing.T) { wg.Done() })) to := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - panic("Asdasd") wg.Done() })) @@ -133,7 +132,7 @@ func TestEchoMiddleware(t *testing.T) { close(quit) Settings.middleware = "" - time.Sleep(10*time.Millisecond) + time.Sleep(100*time.Millisecond) } func TestTokenMiddleware(t *testing.T) { diff --git a/output_http.go b/output_http.go index 3288c30..a66027c 100644 --- a/output_http.go +++ b/output_http.go @@ -171,8 +171,6 @@ func (o *HTTPOutput) sendRequest(client *HTTPClient, request []byte) { resp, err := client.Send(request) stop := time.Now() - panic(string(resp)) - if err != nil { log.Println("Request error:", err) } From 0009555d96dd32efe34e36c7b0625bb3aa6e6105 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Tue, 11 Aug 2015 11:08:14 +0300 Subject: [PATCH 13/64] Apply fmt --- http_client.go | 14 +++++++------- input_raw.go | 2 +- middleware_test.go | 2 +- output_http.go | 2 +- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/http_client.go b/http_client.go index 6a8632c..37b2691 100644 --- a/http_client.go +++ b/http_client.go @@ -18,11 +18,11 @@ var defaultPorts = map[string]string{ } type HTTPClientConfig struct { - FollowRedirects int - Debug bool - OriginalHost bool - Timeout time.Duration - ResponseBufferSize int + FollowRedirects int + Debug bool + OriginalHost bool + Timeout time.Duration + ResponseBufferSize int } type HTTPClient struct { @@ -52,7 +52,7 @@ func NewHTTPClient(baseURL string, config *HTTPClientConfig) *HTTPClient { } if config.ResponseBufferSize == 0 { - config.ResponseBufferSize = 512*1024 // 500kb + config.ResponseBufferSize = 512 * 1024 // 500kb } client := new(HTTPClient) @@ -69,7 +69,7 @@ func (c *HTTPClient) Connect() (err error) { c.Disconnect() if !strings.Contains(c.host, ":") { - c.conn, err = net.Dial("tcp", c.host + ":80") + c.conn, err = net.Dial("tcp", c.host+":80") } else { c.conn, err = net.Dial("tcp", c.host) } diff --git a/input_raw.go b/input_raw.go index e29a273..6fdb9ee 100644 --- a/input_raw.go +++ b/input_raw.go @@ -35,7 +35,7 @@ func (i *RAWInput) Read(data []byte) (int, error) { copy(data[0:len(header)], header) copy(data[len(header):], buf) - return len(buf)+len(header), nil + return len(buf) + len(header), nil } else { copy(data, buf) diff --git a/middleware_test.go b/middleware_test.go index a75d6bc..71748a8 100644 --- a/middleware_test.go +++ b/middleware_test.go @@ -132,7 +132,7 @@ func TestEchoMiddleware(t *testing.T) { close(quit) Settings.middleware = "" - time.Sleep(100*time.Millisecond) + time.Sleep(100 * time.Millisecond) } func TestTokenMiddleware(t *testing.T) { diff --git a/output_http.go b/output_http.go index a66027c..7be477f 100644 --- a/output_http.go +++ b/output_http.go @@ -154,7 +154,7 @@ func (o *HTTPOutput) Write(data []byte) (n int, err error) { } func (o *HTTPOutput) Read(data []byte) (int, error) { - buf := <- o.responses + buf := <-o.responses header := []byte("2\n") copy(data[0:2], header) copy(data[2:], buf) From 8046468ea47701e21fd0c56616349452ef6c546b Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Tue, 11 Aug 2015 11:28:22 +0300 Subject: [PATCH 14/64] Fix tests --- plugins.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/plugins.go b/plugins.go index cb189a4..d0ba71a 100644 --- a/plugins.go +++ b/plugins.go @@ -55,11 +55,14 @@ func registerPlugin(constructor interface{}, options ...interface{}) { pluginWrapper = plugin } - if _, ok := plugin.(io.Reader); ok { + _, isR := plugin.(io.Reader) + _, isW := plugin.(io.Writer) + + if isR && !isW { Plugins.Inputs = append(Plugins.Inputs, pluginWrapper.(io.Reader)) } - if _, ok := plugin.(io.Writer); ok { + if isW { Plugins.Outputs = append(Plugins.Outputs, pluginWrapper.(io.Writer)) } } From 2990fab6355868038572ff6d371e72a657a9460c Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Tue, 11 Aug 2015 17:37:37 +0300 Subject: [PATCH 15/64] Track original responses --- emitter.go | 9 +++++-- examples/echo_modifier.sh | 3 +++ input_raw.go | 39 +++++++++++++++++++--------- input_raw_test.go | 41 +++++++++++++++++++++++++++--- middleware_test.go | 6 ++--- plugins.go | 1 + raw_socket_listener/listener.go | 38 ++++++++++++++++++++------- raw_socket_listener/tcp_message.go | 24 ++++++++++++++--- raw_socket_listener/tcp_packet.go | 2 +- 9 files changed, 128 insertions(+), 35 deletions(-) diff --git a/emitter.go b/emitter.go index ec5065c..0769295 100644 --- a/emitter.go +++ b/emitter.go @@ -56,8 +56,13 @@ func CopyMulty(src io.Reader, writers ...io.Writer) (err error) { if nr > 0 && len(buf) > nr { payload := buf[0:nr] + _maxN := nr + if nr > 500 { + _maxN = 500 + } + if Settings.debug { - Debug("[EMITTER] input:", string(payload[0:500])) + Debug("[EMITTER] input:", string(payload[0:_maxN])) } if modifier != nil { @@ -69,7 +74,7 @@ func CopyMulty(src io.Reader, writers ...io.Writer) (err error) { } if Settings.debug { - Debug("[EMITTER] Rewrittern input:", len(payload), "First 500 bytes:", string(payload[0:500])) + Debug("[EMITTER] Rewrittern input:", len(payload), "First 500 bytes:", string(payload[0:_maxN])) } } diff --git a/examples/echo_modifier.sh b/examples/echo_modifier.sh index e3f7dd9..105b4bb 100755 --- a/examples/echo_modifier.sh +++ b/examples/echo_modifier.sh @@ -11,6 +11,9 @@ while read line; do >&2 echo "[DEBUG][ECHO] ===================================" case ${header:0:1} in + "3") + >&2 echo "[DEBUG][ECHO] Request type: Original Response" + ;; "2") >&2 echo "[DEBUG][ECHO] Request type: Replayed Response" ;; diff --git a/input_raw.go b/input_raw.go index 6fdb9ee..762655c 100644 --- a/input_raw.go +++ b/input_raw.go @@ -10,17 +10,21 @@ import ( // RAWInput used for intercepting traffic for given address type RAWInput struct { - data chan []byte + requests chan []byte + responses chan []byte address string expire time.Duration + captureResponse bool } // NewRAWInput constructor for RAWInput. Accepts address with port as argument. -func NewRAWInput(address string, expire time.Duration) (i *RAWInput) { +func NewRAWInput(address string, expire time.Duration, captureResponse bool) (i *RAWInput) { i = new(RAWInput) - i.data = make(chan []byte) + i.requests = make(chan []byte) + i.responses = make(chan []byte) i.address = address i.expire = expire + i.captureResponse = captureResponse go i.listen(address) @@ -28,18 +32,25 @@ func NewRAWInput(address string, expire time.Duration) (i *RAWInput) { } func (i *RAWInput) Read(data []byte) (int, error) { - buf := <-i.data + select { + case buf := <- i.requests: + if i.captureResponse { + header := []byte("1\n") + copy(data[0:len(header)], header) + copy(data[len(header):], buf) - if len(Settings.middleware) > 0 { - header := []byte("1\n") + return len(buf) + len(header), nil + } else { + copy(data, buf) + + return len(buf), nil + } + case buf := <- i.responses: + header := []byte("3\n") copy(data[0:len(header)], header) copy(data[len(header):], buf) return len(buf) + len(header), nil - } else { - copy(data, buf) - - return len(buf), nil } } @@ -54,13 +65,17 @@ func (i *RAWInput) listen(address string) { log.Fatal("input-raw: error while parsing address", err) } - listener := raw.NewListener(host, port, i.expire) + listener := raw.NewListener(host, port, i.expire, i.captureResponse) for { // Receiving TCPMessage object m := listener.Receive() - i.data <- m.Bytes() + i.requests <- m.RequestBytes() + + if i.captureResponse { + i.responses <- m.ResponseBytes() + } } } diff --git a/input_raw_test.go b/input_raw_test.go index a145845..c822029 100644 --- a/input_raw_test.go +++ b/input_raw_test.go @@ -24,7 +24,7 @@ func TestRAWInput(t *testing.T) { listener := startHTTP(func(w http.ResponseWriter, req *http.Request) {}) - input := NewRAWInput(listener.Addr().String(), testRawExpire) + input := NewRAWInput(listener.Addr().String(), testRawExpire, false) output := NewTestOutput(func(data []byte) { wg.Done() }) @@ -66,7 +66,7 @@ func TestInputRAW100Expect(t *testing.T) { originAddr := strings.Replace(origin.Addr().String(), "[::]", "127.0.0.1", -1) - input := NewRAWInput(originAddr, testRawExpire) + input := NewRAWInput(originAddr, testRawExpire, false) // We will use it to get content of raw HTTP request testOutput := NewTestOutput(func(data []byte) { @@ -123,7 +123,7 @@ func TestInputRAWChunkedEncoding(t *testing.T) { originAddr := strings.Replace(origin.Addr().String(), "[::]", "127.0.0.1", -1) - input := NewRAWInput(originAddr, testRawExpire) + input := NewRAWInput(originAddr, testRawExpire, false) listener := startHTTP(func(w http.ResponseWriter, req *http.Request) { defer req.Body.Close() @@ -181,7 +181,7 @@ func TestInputRAWLargePayload(t *testing.T) { })) originAddr := strings.Replace(origin.Listener.Addr().String(), "[::]", "127.0.0.1", -1) - input := NewRAWInput(originAddr, testRawExpire) + input := NewRAWInput(originAddr, testRawExpire, false) replay := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { req.Body = http.MaxBytesReader(w, req.Body, 1*1024*1024) @@ -214,3 +214,36 @@ func TestInputRAWLargePayload(t *testing.T) { wg.Wait() close(quit) } + +func TestInputRAWResponse(t *testing.T) { + wg := new(sync.WaitGroup) + quit := make(chan int) + + listener := startHTTP(func(w http.ResponseWriter, req *http.Request) {}) + + input := NewRAWInput(listener.Addr().String(), testRawExpire, true) + output := NewTestOutput(func(data []byte) { + wg.Done() + }) + + Plugins.Inputs = []io.Reader{input} + Plugins.Outputs = []io.Writer{output} + + address := strings.Replace(listener.Addr().String(), "[::]", "127.0.0.1", -1) + + client := NewHTTPClient(address, &HTTPClientConfig{}) + + time.Sleep(time.Millisecond) + go Start(quit) + + for i := 0; i < 100; i++ { + // 2 because we track both request and response + wg.Add(2) + client.Get("/") + } + + wg.Wait() + close(quit) + + time.Sleep(100*time.Millisecond) +} \ No newline at end of file diff --git a/middleware_test.go b/middleware_test.go index 71748a8..e323fd9 100644 --- a/middleware_test.go +++ b/middleware_test.go @@ -105,7 +105,7 @@ func TestEchoMiddleware(t *testing.T) { quit := make(chan int) // Catch traffic from one service - input := NewRAWInput(from.Listener.Addr().String(), testRawExpire) + input := NewRAWInput(from.Listener.Addr().String(), testRawExpire, true) // And redirect to another output := NewHTTPOutput(to.URL, &HTTPOutputConfig{Debug: true}) @@ -158,14 +158,14 @@ func TestTokenMiddleware(t *testing.T) { quit := make(chan int) // Catch traffic from one service - input := NewRAWInput(from, testRawExpire) + input := NewRAWInput(from, testRawExpire, true) // And redirect to another output := NewHTTPOutput(to, &HTTPOutputConfig{}) Plugins.Inputs = []io.Reader{input} Plugins.Outputs = []io.Writer{output} - // Settings.middleware = "./examples/echo_modifier.sh" + Settings.middleware = "./examples/echo_modifier.sh" // Start Gor go Start(quit) diff --git a/plugins.go b/plugins.go index d0ba71a..c92435c 100644 --- a/plugins.go +++ b/plugins.go @@ -58,6 +58,7 @@ func registerPlugin(constructor interface{}, options ...interface{}) { _, isR := plugin.(io.Reader) _, isW := plugin.(io.Writer) + // Some of the output can be Readers as well because return responses if isR && !isW { Plugins.Inputs = append(Plugins.Inputs, pluginWrapper.(io.Reader)) } diff --git a/raw_socket_listener/listener.go b/raw_socket_listener/listener.go index 53857ac..ef8b283 100644 --- a/raw_socket_listener/listener.go +++ b/raw_socket_listener/listener.go @@ -32,6 +32,8 @@ type Listener struct { // To get ACK of second message we need to compute its Seq and wait for them message seqWithData map[uint32]uint32 + respAliases map[uint32]uint32 + // Messages ready to be send to client packetsChan chan *TCPPacket @@ -42,14 +44,16 @@ type Listener struct { messageDelChan chan *TCPMessage addr string // IP to listen - port int // Port to listen + port uint16 // Port to listen messageExpire time.Duration + + captureResponse bool } // NewListener creates and initializes new Listener object -func NewListener(addr string, port string, expire time.Duration) (rawListener *Listener) { - rawListener = &Listener{} +func NewListener(addr string, port string, expire time.Duration, captureResponse bool) (rawListener *Listener) { + rawListener = &Listener{captureResponse: captureResponse} rawListener.packetsChan = make(chan *TCPPacket, 10000) rawListener.messagesChan = make(chan *TCPMessage, 10000) @@ -58,9 +62,11 @@ func NewListener(addr string, port string, expire time.Duration) (rawListener *L rawListener.messages = make(map[string]*TCPMessage) rawListener.ackAliases = make(map[uint32]uint32) rawListener.seqWithData = make(map[uint32]uint32) + rawListener.respAliases = make(map[uint32]uint32) rawListener.addr = addr - rawListener.port, _ = strconv.Atoi(port) + _port, _ := strconv.Atoi(port) + rawListener.port = uint16(_port) if expire.Nanoseconds() == 0 { expire = 2000 * time.Millisecond @@ -115,18 +121,19 @@ func (t *Listener) readRAWSocket() { } func (t *Listener) parsePacket(addr net.Addr, buf []byte) { - if t.isIncomingDataPacket(buf) { + if t.isValidPacket(buf) { t.packetsChan <- ParseTCPPacket(addr, buf) } } -func (t *Listener) isIncomingDataPacket(buf []byte) bool { +func (t *Listener) isValidPacket(buf []byte) bool { // To avoid full packet parsing every time, we manually parsing values needed for packet filtering // http://en.wikipedia.org/wiki/Transmission_Control_Protocol destPort := binary.BigEndian.Uint16(buf[2:4]) + srcPort := binary.BigEndian.Uint16(buf[0:2]) // Because RAW_SOCKET can't be bound to port, we have to control it by ourself - if int(destPort) == t.port { + if destPort == t.port || (t.captureResponse && srcPort == t.port) { // Get the 'data offset' (size of the TCP header in 32-bit words) dataOffset := (buf[12] & 0xF0) >> 4 @@ -161,12 +168,20 @@ func (t *Listener) processTCPPacket(packet *TCPPacket) { packet.Ack = alias } - mID := packet.Addr.String() + strconv.Itoa(int(packet.SrcPort)) + strconv.Itoa(int(packet.Ack)) + // if response + if t.captureResponse && packet.DestPort != t.port { + if alias, ok := t.respAliases[packet.Ack]; ok { + packet.Ack = alias + } + } + + + mID := packet.Addr.String() + strconv.Itoa(int(packet.Ack)) message, ok := t.messages[mID] if !ok { // We sending messageDelChan channel, so message object can communicate with Listener and notify it if message completed - message = NewTCPMessage(mID, t.messageDelChan, packet.Ack, &t.messageExpire) + message = NewTCPMessage(mID, t.messageDelChan, packet.Ack, &t.messageExpire, t.port) t.messages[mID] = message } @@ -179,6 +194,11 @@ func (t *Listener) processTCPPacket(packet *TCPPacket) { } } + if t.captureResponse { + // Response tracking + t.respAliases[packet.Seq+uint32(len(packet.Data))] = packet.Ack + } + // Adding packet to message message.packetsChan <- packet } diff --git a/raw_socket_listener/tcp_message.go b/raw_socket_listener/tcp_message.go index e6e31bd..5e0f8a6 100644 --- a/raw_socket_listener/tcp_message.go +++ b/raw_socket_listener/tcp_message.go @@ -15,6 +15,7 @@ import ( type TCPMessage struct { ID string // Message ID Ack uint32 + port uint16 packets []*TCPPacket timer *time.Timer // Used for expire check @@ -27,8 +28,8 @@ type TCPMessage struct { } // NewTCPMessage pointer created from a Acknowledgment number and a channel of messages readuy to be deleted -func NewTCPMessage(ID string, delChan chan *TCPMessage, Ack uint32, expire *time.Duration) (msg *TCPMessage) { - msg = &TCPMessage{ID: ID, Ack: Ack, expire: expire} +func NewTCPMessage(ID string, delChan chan *TCPMessage, Ack uint32, expire *time.Duration, port uint16) (msg *TCPMessage) { + msg = &TCPMessage{ID: ID, Ack: Ack, expire: expire, port: port} msg.packetsChan = make(chan *TCPPacket) msg.delChan = delChan // used for notifying that message completed or expired @@ -73,11 +74,26 @@ func (t *TCPMessage) Timeout() { } // Bytes sorts packets in right orders and return message content -func (t *TCPMessage) Bytes() (output []byte) { +func (t *TCPMessage) RequestBytes() (output []byte) { sort.Sort(sortBySeq(t.packets)) for _, v := range t.packets { - output = append(output, v.Data...) + if v.DestPort == t.port { + output = append(output, v.Data...) + } + } + + return output +} + +// Bytes sorts packets in right orders and return message content +func (t *TCPMessage) ResponseBytes() (output []byte) { + sort.Sort(sortBySeq(t.packets)) + + for _, v := range t.packets { + if v.DestPort != t.port { + output = append(output, v.Data...) + } } return output diff --git a/raw_socket_listener/tcp_packet.go b/raw_socket_listener/tcp_packet.go index 3ebcf88..998eeb4 100644 --- a/raw_socket_listener/tcp_packet.go +++ b/raw_socket_listener/tcp_packet.go @@ -50,7 +50,6 @@ func ParseTCPPacket(addr net.Addr, b []byte) (p *TCPPacket) { // Parse TCP Packet, inspired by: https://github.com/miekg/pcap/blob/master/packet.go func (t *TCPPacket) Parse() { t.ParseBasic() - t.DestPort = binary.BigEndian.Uint16(t.Data[2:4]) t.Flags = binary.BigEndian.Uint16(t.Data[12:14]) & 0x1FF t.Window = binary.BigEndian.Uint16(t.Data[14:16]) t.Checksum = binary.BigEndian.Uint16(t.Data[16:18]) @@ -59,6 +58,7 @@ func (t *TCPPacket) Parse() { // ParseBasic set of fields func (t *TCPPacket) ParseBasic() { + t.DestPort = binary.BigEndian.Uint16(t.Data[2:4]) t.SrcPort = binary.BigEndian.Uint16(t.Data[0:2]) t.Seq = binary.BigEndian.Uint32(t.Data[4:8]) t.Ack = binary.BigEndian.Uint32(t.Data[8:12]) From 491541c2df1d3f3518aeb618c122bf7a1b6e4434 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Thu, 13 Aug 2015 22:11:23 +0300 Subject: [PATCH 16/64] Response object should be separate message --- input_raw.go | 40 ++++++++++-------------- input_raw_test.go | 4 +-- raw_socket_listener/listener.go | 50 +++++++++++++++++++++--------- raw_socket_listener/tcp_message.go | 38 ++++++++++------------- 4 files changed, 72 insertions(+), 60 deletions(-) diff --git a/input_raw.go b/input_raw.go index 762655c..5201f2d 100644 --- a/input_raw.go +++ b/input_raw.go @@ -10,18 +10,16 @@ import ( // RAWInput used for intercepting traffic for given address type RAWInput struct { - requests chan []byte - responses chan []byte - address string - expire time.Duration + data chan *raw.TCPMessage + address string + expire time.Duration captureResponse bool } // NewRAWInput constructor for RAWInput. Accepts address with port as argument. func NewRAWInput(address string, expire time.Duration, captureResponse bool) (i *RAWInput) { i = new(RAWInput) - i.requests = make(chan []byte) - i.responses = make(chan []byte) + i.data = make(chan *raw.TCPMessage) i.address = address i.expire = expire i.captureResponse = captureResponse @@ -32,25 +30,25 @@ func NewRAWInput(address string, expire time.Duration, captureResponse bool) (i } func (i *RAWInput) Read(data []byte) (int, error) { - select { - case buf := <- i.requests: - if i.captureResponse { - header := []byte("1\n") - copy(data[0:len(header)], header) - copy(data[len(header):], buf) + msg := <-i.data + buf := msg.Bytes() - return len(buf) + len(header), nil + if i.captureResponse { + var header []byte + + if msg.IsIncoming { + header = []byte("1\n") } else { - copy(data, buf) - - return len(buf), nil + header = []byte("3\n") } - case buf := <- i.responses: - header := []byte("3\n") + copy(data[0:len(header)], header) copy(data[len(header):], buf) return len(buf) + len(header), nil + } else { + copy(data, buf) + return len(buf), nil } } @@ -71,11 +69,7 @@ func (i *RAWInput) listen(address string) { // Receiving TCPMessage object m := listener.Receive() - i.requests <- m.RequestBytes() - - if i.captureResponse { - i.responses <- m.ResponseBytes() - } + i.data <- m } } diff --git a/input_raw_test.go b/input_raw_test.go index c822029..6a2b117 100644 --- a/input_raw_test.go +++ b/input_raw_test.go @@ -245,5 +245,5 @@ func TestInputRAWResponse(t *testing.T) { wg.Wait() close(quit) - time.Sleep(100*time.Millisecond) -} \ No newline at end of file + time.Sleep(100 * time.Millisecond) +} diff --git a/raw_socket_listener/listener.go b/raw_socket_listener/listener.go index ef8b283..fa208d4 100644 --- a/raw_socket_listener/listener.go +++ b/raw_socket_listener/listener.go @@ -17,6 +17,7 @@ import ( "encoding/binary" "log" "net" + "runtime/debug" "strconv" "time" ) @@ -32,7 +33,7 @@ type Listener struct { // To get ACK of second message we need to compute its Seq and wait for them message seqWithData map[uint32]uint32 - respAliases map[uint32]uint32 + respAliases map[uint32]*request // Messages ready to be send to client packetsChan chan *TCPPacket @@ -44,13 +45,18 @@ type Listener struct { messageDelChan chan *TCPMessage addr string // IP to listen - port uint16 // Port to listen + port uint16 // Port to listen messageExpire time.Duration captureResponse bool } +type request struct { + start int64 + ack uint32 +} + // NewListener creates and initializes new Listener object func NewListener(addr string, port string, expire time.Duration, captureResponse bool) (rawListener *Listener) { rawListener = &Listener{captureResponse: captureResponse} @@ -62,7 +68,7 @@ func NewListener(addr string, port string, expire time.Duration, captureResponse rawListener.messages = make(map[string]*TCPMessage) rawListener.ackAliases = make(map[uint32]uint32) rawListener.seqWithData = make(map[uint32]uint32) - rawListener.respAliases = make(map[uint32]uint32) + rawListener.respAliases = make(map[uint32]*request) rawListener.addr = addr _port, _ := strconv.Atoi(port) @@ -155,10 +161,17 @@ var bPOST = []byte("POST") // // For TCP message unique id is Acknowledgment number (see tcp_packet.go) func (t *Listener) processTCPPacket(packet *TCPPacket) { - defer func() { recover() }() + // Don't exit on panic + defer func() { + if r := recover(); r != nil { + log.Println("PANIC: pkg:", r, string(debug.Stack())) + } + }() var message *TCPMessage + isIncoming := packet.DestPort == t.port + if parentAck, ok := t.seqWithData[packet.Seq]; ok { t.ackAliases[packet.Ack] = parentAck delete(t.seqWithData, packet.Seq) @@ -168,23 +181,27 @@ func (t *Listener) processTCPPacket(packet *TCPPacket) { packet.Ack = alias } - // if response - if t.captureResponse && packet.DestPort != t.port { - if alias, ok := t.respAliases[packet.Ack]; ok { - packet.Ack = alias - } + var responseRequest *request + + if t.captureResponse && !isIncoming { + responseRequest, _ = t.respAliases[packet.Ack] } + mID := packet.Addr.String() + strconv.Itoa(int(packet.DestPort)) + strconv.Itoa(int(packet.Ack)) - mID := packet.Addr.String() + strconv.Itoa(int(packet.Ack)) message, ok := t.messages[mID] if !ok { // We sending messageDelChan channel, so message object can communicate with Listener and notify it if message completed - message = NewTCPMessage(mID, t.messageDelChan, packet.Ack, &t.messageExpire, t.port) + message = NewTCPMessage(mID, t.messageDelChan, packet.Ack, &t.messageExpire, isIncoming) t.messages[mID] = message + + if !isIncoming && responseRequest != nil { + message.RequestStart = responseRequest.start + } } + // Handling Expect: 100-continue requests if bytes.Equal(packet.Data[0:4], bPOST) { if bytes.Equal(packet.Data[len(packet.Data)-24:len(packet.Data)-4], bExpect100ContinueCheck) { t.seqWithData[packet.Seq+uint32(len(packet.Data))] = packet.Ack @@ -194,9 +211,14 @@ func (t *Listener) processTCPPacket(packet *TCPPacket) { } } - if t.captureResponse { - // Response tracking - t.respAliases[packet.Seq+uint32(len(packet.Data))] = packet.Ack + if t.captureResponse && isIncoming { + // If message have multiple packets, delete previous alias + if len(message.packets) > 0 { + delete(t.respAliases, uint32(message.Size())) + } + + responseAck := packet.Seq + uint32(message.Size()+len(packet.Data)) + t.respAliases[responseAck] = &request{message.Start, message.Ack} } // Adding packet to message diff --git a/raw_socket_listener/tcp_message.go b/raw_socket_listener/tcp_message.go index 5e0f8a6..d06ad19 100644 --- a/raw_socket_listener/tcp_message.go +++ b/raw_socket_listener/tcp_message.go @@ -13,10 +13,12 @@ import ( // Message can be compiled from unique packets with same message_id which sorted by sequence // Message is received if we didn't receive any packets for 2000ms type TCPMessage struct { - ID string // Message ID - Ack uint32 - port uint16 - packets []*TCPPacket + ID string // Message ID + Ack uint32 + RequestStart int64 + Start int64 + IsIncoming bool + packets []*TCPPacket timer *time.Timer // Used for expire check @@ -28,9 +30,9 @@ type TCPMessage struct { } // NewTCPMessage pointer created from a Acknowledgment number and a channel of messages readuy to be deleted -func NewTCPMessage(ID string, delChan chan *TCPMessage, Ack uint32, expire *time.Duration, port uint16) (msg *TCPMessage) { - msg = &TCPMessage{ID: ID, Ack: Ack, expire: expire, port: port} - +func NewTCPMessage(ID string, delChan chan *TCPMessage, Ack uint32, expire *time.Duration, IsIncoming bool) (msg *TCPMessage) { + msg = &TCPMessage{ID: ID, Ack: Ack, expire: expire, IsIncoming: IsIncoming} + msg.Start = time.Now().UnixNano() msg.packetsChan = make(chan *TCPPacket) msg.delChan = delChan // used for notifying that message completed or expired @@ -74,29 +76,23 @@ func (t *TCPMessage) Timeout() { } // Bytes sorts packets in right orders and return message content -func (t *TCPMessage) RequestBytes() (output []byte) { +func (t *TCPMessage) Bytes() (output []byte) { sort.Sort(sortBySeq(t.packets)) - for _, v := range t.packets { - if v.DestPort == t.port { - output = append(output, v.Data...) - } + for _, p := range t.packets { + output = append(output, p.Data...) } return output } -// Bytes sorts packets in right orders and return message content -func (t *TCPMessage) ResponseBytes() (output []byte) { - sort.Sort(sortBySeq(t.packets)) - - for _, v := range t.packets { - if v.DestPort != t.port { - output = append(output, v.Data...) - } +// Size returns total size of message +func (t *TCPMessage) Size() (size int) { + for _, p := range t.packets { + size += len(p.Data) } - return output + return } // AddPacket to the message and ensure packet uniqueness From bb319b5fd24bd147d28b1604ca78da04d7091527 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Fri, 14 Aug 2015 20:42:23 +0300 Subject: [PATCH 17/64] Add request ID and token modifier middleware --- examples/echo_modifier.sh | 4 +- examples/token_modifier.go | 97 ++++++++++++++++++++++++++++++ gor.go | 18 +++--- input_raw.go | 36 +++++++++-- middleware.go | 29 ++++++--- middleware_test.go | 31 ++++++---- output_http.go | 29 ++++++--- raw_socket_listener/listener.go | 1 + raw_socket_listener/tcp_message.go | 22 +++++++ 9 files changed, 220 insertions(+), 47 deletions(-) create mode 100644 examples/token_modifier.go diff --git a/examples/echo_modifier.sh b/examples/echo_modifier.sh index 105b4bb..13d2074 100755 --- a/examples/echo_modifier.sh +++ b/examples/echo_modifier.sh @@ -11,10 +11,10 @@ while read line; do >&2 echo "[DEBUG][ECHO] ===================================" case ${header:0:1} in - "3") + "2") >&2 echo "[DEBUG][ECHO] Request type: Original Response" ;; - "2") + "3") >&2 echo "[DEBUG][ECHO] Request type: Replayed Response" ;; "1") diff --git a/examples/token_modifier.go b/examples/token_modifier.go new file mode 100644 index 0000000..7ca37dd --- /dev/null +++ b/examples/token_modifier.go @@ -0,0 +1,97 @@ +package main + +import ( + "os" + "bufio" + "encoding/hex" + "github.com/buger/gor/proto" + "bytes" + "fmt" +) + +// requestID -> originalToken +var originalTokens map[string][]byte + +// originalToken -> replayedToken +var tokenAliases map[string][]byte + +func main() { + originalTokens = make(map[string][]byte) + tokenAliases = make(map[string][]byte) + + scanner := bufio.NewScanner(os.Stdin) + + for scanner.Scan() { + encoded := scanner.Bytes() + buf := make([]byte, len(encoded)/2) + hex.Decode(buf, encoded) + + go process(buf) + } +} + +func process(buf []byte) { + // First byte indicate payload type, possible values: + // 1 - Request + // 2 - Response + // 3 - ReplayedResponse + payloadType := buf[0] + headerSize := 42 + header := buf[:headerSize] + // For each request you should receive 3 payloads (request, response, replayed response) with same request id + reqID := string(header[2:headerSize]) + payload := buf[headerSize:] + + Debug("Received payload:", string(buf)) + + switch payloadType { + case '1': + if bytes.Equal(proto.Path(payload), []byte("/token")) { + originalTokens[reqID] = []byte{} + Debug("Found token request:", reqID) + } else { + tokenVal, vs, _ := proto.PathParam(payload, []byte("token")) + + if vs != -1 { // If there is GET token param + if alias, ok := tokenAliases[string(tokenVal)]; ok { + // Rewrite original token to alias + payload = proto.SetPathParam(payload, []byte("token"), alias) + + // Copy modified payload to our buffer + copy(buf[headerSize:], payload) + } + } + } + + // Re-compute length in case if payload was modified + bufLen := len(header) + len(payload) + // Encoding request and sending it back + dst := make([]byte, bufLen*2+1) + hex.Encode(dst, buf[:bufLen]) + dst[len(dst)-1] = '\n' + + os.Stdout.Write(dst) + + return + case '2': // Original response + if _, ok := originalTokens[reqID]; ok { + // Token is inside response body + secureToken := proto.Body(payload) + originalTokens[reqID] = secureToken + Debug("Remember origial token:", string(secureToken)) + } + case '3': // Replayed response + if originalToken, ok := originalTokens[reqID]; ok { + delete(originalTokens, reqID) + secureToken := proto.Body(payload) + tokenAliases[string(originalToken)] = secureToken + + Debug("Create alias for new token token, was:", string(originalToken), "now:", string(secureToken)) + } + } +} + +func Debug(args ...interface{}) { + fmt.Fprint(os.Stderr, "[DEBUG][TOKEN-MOD] ") + fmt.Fprintln(os.Stderr, args...) +} \ No newline at end of file diff --git a/gor.go b/gor.go index af9bf5f..4cd8596 100644 --- a/gor.go +++ b/gor.go @@ -8,7 +8,7 @@ import ( "log" "os" "runtime" - "runtime/debug" + _ "runtime/debug" "runtime/pprof" "time" ) @@ -20,20 +20,18 @@ var ( ) func main() { + // // Don't exit on panic + // defer func() { + // if r := recover(); r != nil { + // fmt.Printf("PANIC: pkg: %v %s \n", r, debug.Stack()) + // } + // }() + // If not set via env cariable if len(os.Getenv("GOMAXPROCS")) == 0 { runtime.GOMAXPROCS(runtime.NumCPU() * 2) } - // Don't exit on panic - defer func() { - if r := recover(); r != nil { - if _, ok := r.(error); !ok { - fmt.Printf("PANIC: pkg: %v %s \n", r, debug.Stack()) - } - } - }() - fmt.Println("Version:", VERSION) flag.Parse() diff --git a/input_raw.go b/input_raw.go index 5201f2d..9d786b3 100644 --- a/input_raw.go +++ b/input_raw.go @@ -29,19 +29,43 @@ func NewRAWInput(address string, expire time.Duration, captureResponse bool) (i return } +const ( + RequestPayload = 1 << iota + ResponsePayload + ReplayedResponsePayload +) + +func payloadHeader(payloadType int, uuid []byte) (header []byte) { + header = make([]byte, 43) + header[1] = ' ' + header[len(header)-1] = '\n' + + switch payloadType { + case RequestPayload: + header[0] = '1' + case ResponsePayload: + header[0] = '2' + case ReplayedResponsePayload: + header[0] = '3' + } + + copy(header[2:], uuid) + + return header +} + func (i *RAWInput) Read(data []byte) (int, error) { msg := <-i.data buf := msg.Bytes() if i.captureResponse { - var header []byte - - if msg.IsIncoming { - header = []byte("1\n") - } else { - header = []byte("3\n") + payloadType := RequestPayload + if !msg.IsIncoming { + payloadType = ResponsePayload } + header := payloadHeader(payloadType, msg.UUID()) + copy(data[0:len(header)], header) copy(data[len(header):], buf) diff --git a/middleware.go b/middleware.go index c642cd2..f1d072d 100644 --- a/middleware.go +++ b/middleware.go @@ -9,6 +9,7 @@ import ( "os" "os/exec" "strings" + "sync" ) type Middleware struct { @@ -16,6 +17,8 @@ type Middleware struct { data chan []byte + mu sync.Mutex + Stdin io.Writer Stdout io.Reader } @@ -48,6 +51,7 @@ func NewMiddleware(command string) *Middleware { } func (m *Middleware) ReadFrom(plugin io.Reader) { + Debug("[MIDDLEWARE-MASTER] Starting reading from", plugin) go m.copy(m.Stdin, plugin) } @@ -60,8 +64,11 @@ func (m *Middleware) copy(to io.Writer, from io.Reader) { if nr > 0 && len(buf) > nr { hex.Encode(dst, buf[0:nr]) - to.Write(dst[0 : nr*2]) - to.Write([]byte("\n")) + dst[nr*2] = '\n' + + m.mu.Lock() + to.Write(dst[0 : nr*2+1]) + m.mu.Unlock() if Settings.debug { Debug("[MIDDLEWARE-MASTER] Sending:", string(buf[0:nr]), "From:", from) @@ -71,19 +78,23 @@ func (m *Middleware) copy(to io.Writer, from io.Reader) { } func (m *Middleware) read(from io.Reader) { - buf := make([]byte, 5*1024*1024) - scanner := bufio.NewScanner(from) for scanner.Scan() { bytes := scanner.Bytes() - hex.Decode(buf, bytes) - - if Settings.debug { - Debug("[MIDDLEWARE-MASTER] Received:", string(buf[0:len(bytes)/2])) + buf := make([]byte, len(bytes)/2) + if _, err := hex.Decode(buf, bytes); err != nil { + fmt.Fprintln(os.Stderr, "Failed to decode input payload", err, len(bytes)) } - m.data <- buf[0 : len(bytes)/2] + if Settings.debug { + Debug("[MIDDLEWARE-MASTER] Received:", string(buf)) + } + + // We should accept only request payloads + if buf[0] == '1' { + m.data <- buf + } } if err := scanner.Err(); err != nil { diff --git a/middleware_test.go b/middleware_test.go index e323fd9..c691e58 100644 --- a/middleware_test.go +++ b/middleware_test.go @@ -96,23 +96,28 @@ func TestEchoMiddleware(t *testing.T) { wg := new(sync.WaitGroup) from := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Env", "prod") + w.Header().Set("RequestPath", r.URL.Path) wg.Done() })) to := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Env", "test") + w.Header().Set("RequestPath", r.URL.Path) wg.Done() })) quit := make(chan int) + Settings.middleware = "./examples/echo_modifier.sh" + // Catch traffic from one service input := NewRAWInput(from.Listener.Addr().String(), testRawExpire, true) // And redirect to another - output := NewHTTPOutput(to.URL, &HTTPOutputConfig{Debug: true}) + output := NewHTTPOutput(to.URL, &HTTPOutputConfig{Debug: false}) Plugins.Inputs = []io.Reader{input} Plugins.Outputs = []io.Writer{output} - Settings.middleware = "./examples/echo_modifier.sh" // Start Gor go Start(quit) @@ -125,14 +130,14 @@ func TestEchoMiddleware(t *testing.T) { client := NewHTTPClient(from.URL, &HTTPClientConfig{Debug: false}) // Request should be echoed - client.Get("/") - client.Get("/") + client.Get("/a") + client.Get("/b") wg.Wait() close(quit) - Settings.middleware = "" - time.Sleep(100 * time.Millisecond) + + Settings.middleware = "" } func TestTokenMiddleware(t *testing.T) { @@ -150,7 +155,7 @@ func TestTokenMiddleware(t *testing.T) { } case "/secure": if status != 202 { - // t.Error("Server should receive valid rewritten token") + t.Error("Server should receive valid rewritten token") } } }) @@ -161,26 +166,29 @@ func TestTokenMiddleware(t *testing.T) { input := NewRAWInput(from, testRawExpire, true) // And redirect to another - output := NewHTTPOutput(to, &HTTPOutputConfig{}) + output := NewHTTPOutput(to, &HTTPOutputConfig{Debug: true}) Plugins.Inputs = []io.Reader{input} Plugins.Outputs = []io.Writer{output} - Settings.middleware = "./examples/echo_modifier.sh" + Settings.middleware = "go run ./examples/token_modifier.go" // Start Gor go Start(quit) - time.Sleep(time.Millisecond) + // Wait for middleware to initialize + time.Sleep(500 * time.Millisecond) // Should receive 2 requests from original + 2 from replayed wg.Add(4) - client := NewHTTPClient("http://"+from, &HTTPClientConfig{Debug: true}) + client := NewHTTPClient("http://"+from, &HTTPClientConfig{Debug: false}) // Sending traffic to original service resp, _ = client.Get("/token") token = proto.Body(resp) + time.Sleep(50*time.Millisecond) + resp, _ = client.Get("/secure?token=" + string(token)) if !bytes.Equal(proto.Status(resp), []byte("202")) { t.Error("Valid token should return 202:", proto.Status(resp)) @@ -188,5 +196,6 @@ func TestTokenMiddleware(t *testing.T) { wg.Wait() close(quit) + time.Sleep(100 * time.Millisecond) Settings.middleware = "" } diff --git a/output_http.go b/output_http.go index 7be477f..38c423f 100644 --- a/output_http.go +++ b/output_http.go @@ -9,6 +9,11 @@ import ( const initialDynamicWorkers = 10 +type response struct { + payload []byte + uuid []byte +} + // HTTPOutputConfig struct for holding http output configuration type HTTPOutputConfig struct { redirectLimit int @@ -36,7 +41,7 @@ type HTTPOutput struct { address string limit int queue chan []byte - responses chan []byte + responses chan response needWorker chan int @@ -60,7 +65,7 @@ func NewHTTPOutput(address string, config *HTTPOutputConfig) io.Writer { } o.queue = make(chan []byte, 100) - o.responses = make(chan []byte, 100) + o.responses = make(chan response, 100) o.needWorker = make(chan int, 1) // Initial workers count @@ -154,17 +159,23 @@ func (o *HTTPOutput) Write(data []byte) (n int, err error) { } func (o *HTTPOutput) Read(data []byte) (int, error) { - buf := <-o.responses - header := []byte("2\n") - copy(data[0:2], header) - copy(data[2:], buf) + resp := <-o.responses - return len(buf) + len(header), nil + Debug("[OUTPUT-HTTP] Received response", string(resp.payload)) + + header := payloadHeader(ReplayedResponsePayload, resp.uuid) + copy(data[0:len(header)], header) + copy(data[len(header):], resp.payload) + + return len(resp.payload) + len(header), nil } func (o *HTTPOutput) sendRequest(client *HTTPClient, request []byte) { + var uuid []byte + if len(Settings.middleware) > 0 { - request = request[2:] + uuid = request[2:42] + request = request[43:] } start := time.Now() @@ -176,7 +187,7 @@ func (o *HTTPOutput) sendRequest(client *HTTPClient, request []byte) { } if len(Settings.middleware) > 0 { - o.responses <- resp + o.responses <- response{resp, uuid} } if o.elasticSearch != nil { diff --git a/raw_socket_listener/listener.go b/raw_socket_listener/listener.go index fa208d4..3c7af46 100644 --- a/raw_socket_listener/listener.go +++ b/raw_socket_listener/listener.go @@ -198,6 +198,7 @@ func (t *Listener) processTCPPacket(packet *TCPPacket) { if !isIncoming && responseRequest != nil { message.RequestStart = responseRequest.start + message.RequestAck = responseRequest.ack } } diff --git a/raw_socket_listener/tcp_message.go b/raw_socket_listener/tcp_message.go index d06ad19..14e70d9 100644 --- a/raw_socket_listener/tcp_message.go +++ b/raw_socket_listener/tcp_message.go @@ -4,6 +4,9 @@ import ( "log" "sort" "time" + "crypto/sha1" + "encoding/hex" + "strconv" ) // TCPMessage ensure that all TCP packets for given request is received, and processed in right sequence @@ -16,6 +19,7 @@ type TCPMessage struct { ID string // Message ID Ack uint32 RequestStart int64 + RequestAck uint32 Start int64 IsIncoming bool packets []*TCPPacket @@ -116,3 +120,21 @@ func (t *TCPMessage) AddPacket(packet *TCPPacket) { // Reset message timeout timer t.timer.Reset(*t.expire) } + +func (t *TCPMessage) UUID() []byte { + var key []byte + + if t.IsIncoming { + key = strconv.AppendInt(key, t.Start, 10) + key = strconv.AppendUint(key, uint64(t.Ack), 10) + } else { + key = strconv.AppendInt(key, t.RequestStart, 10) + key = strconv.AppendUint(key, uint64(t.RequestAck), 10) + } + + uuid := make([]byte, 40) + sha := sha1.Sum(key) + hex.Encode(uuid, sha[:20]) + + return uuid +} From 70e7d8dd292839c43f4c120c58385ee2ae012d37 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Fri, 14 Aug 2015 23:07:46 +0300 Subject: [PATCH 18/64] Reduce delay of raw input --- examples/token_modifier.go | 136 ++++++++++++++--------------- input_raw_test.go | 2 +- middleware.go | 2 +- middleware_test.go | 2 +- output_http.go | 2 +- raw_socket_listener/listener.go | 1 + raw_socket_listener/tcp_message.go | 77 ++++++++++++++-- 7 files changed, 141 insertions(+), 81 deletions(-) diff --git a/examples/token_modifier.go b/examples/token_modifier.go index 7ca37dd..89ec995 100644 --- a/examples/token_modifier.go +++ b/examples/token_modifier.go @@ -1,12 +1,12 @@ package main import ( - "os" - "bufio" - "encoding/hex" - "github.com/buger/gor/proto" - "bytes" - "fmt" + "bufio" + "bytes" + "encoding/hex" + "fmt" + "github.com/buger/gor/proto" + "os" ) // requestID -> originalToken @@ -16,82 +16,82 @@ var originalTokens map[string][]byte var tokenAliases map[string][]byte func main() { - originalTokens = make(map[string][]byte) - tokenAliases = make(map[string][]byte) + originalTokens = make(map[string][]byte) + tokenAliases = make(map[string][]byte) - scanner := bufio.NewScanner(os.Stdin) + scanner := bufio.NewScanner(os.Stdin) - for scanner.Scan() { - encoded := scanner.Bytes() - buf := make([]byte, len(encoded)/2) - hex.Decode(buf, encoded) + for scanner.Scan() { + encoded := scanner.Bytes() + buf := make([]byte, len(encoded)/2) + hex.Decode(buf, encoded) - go process(buf) - } + go process(buf) + } } func process(buf []byte) { - // First byte indicate payload type, possible values: - // 1 - Request - // 2 - Response - // 3 - ReplayedResponse - payloadType := buf[0] - headerSize := 42 - header := buf[:headerSize] - // For each request you should receive 3 payloads (request, response, replayed response) with same request id - reqID := string(header[2:headerSize]) - payload := buf[headerSize:] + // First byte indicate payload type, possible values: + // 1 - Request + // 2 - Response + // 3 - ReplayedResponse + payloadType := buf[0] + headerSize := 42 + header := buf[:headerSize] + // For each request you should receive 3 payloads (request, response, replayed response) with same request id + reqID := string(header[2:headerSize]) + payload := buf[headerSize:] - Debug("Received payload:", string(buf)) + Debug("Received payload:", string(buf)) - switch payloadType { - case '1': - if bytes.Equal(proto.Path(payload), []byte("/token")) { - originalTokens[reqID] = []byte{} - Debug("Found token request:", reqID) - } else { - tokenVal, vs, _ := proto.PathParam(payload, []byte("token")) + switch payloadType { + case '1': + if bytes.Equal(proto.Path(payload), []byte("/token")) { + originalTokens[reqID] = []byte{} + Debug("Found token request:", reqID) + } else { + tokenVal, vs, _ := proto.PathParam(payload, []byte("token")) - if vs != -1 { // If there is GET token param - if alias, ok := tokenAliases[string(tokenVal)]; ok { - // Rewrite original token to alias - payload = proto.SetPathParam(payload, []byte("token"), alias) + if vs != -1 { // If there is GET token param + if alias, ok := tokenAliases[string(tokenVal)]; ok { + // Rewrite original token to alias + payload = proto.SetPathParam(payload, []byte("token"), alias) - // Copy modified payload to our buffer - copy(buf[headerSize:], payload) - } - } - } + // Copy modified payload to our buffer + copy(buf[headerSize:], payload) + } + } + } - // Re-compute length in case if payload was modified - bufLen := len(header) + len(payload) - // Encoding request and sending it back - dst := make([]byte, bufLen*2+1) - hex.Encode(dst, buf[:bufLen]) - dst[len(dst)-1] = '\n' + // Re-compute length in case if payload was modified + bufLen := len(header) + len(payload) + // Encoding request and sending it back + dst := make([]byte, bufLen*2+1) + hex.Encode(dst, buf[:bufLen]) + dst[len(dst)-1] = '\n' - os.Stdout.Write(dst) + os.Stdout.Write(dst) - return - case '2': // Original response - if _, ok := originalTokens[reqID]; ok { - // Token is inside response body - secureToken := proto.Body(payload) - originalTokens[reqID] = secureToken - Debug("Remember origial token:", string(secureToken)) - } - case '3': // Replayed response - if originalToken, ok := originalTokens[reqID]; ok { - delete(originalTokens, reqID) - secureToken := proto.Body(payload) - tokenAliases[string(originalToken)] = secureToken + return + case '2': // Original response + if _, ok := originalTokens[reqID]; ok { + // Token is inside response body + secureToken := proto.Body(payload) + originalTokens[reqID] = secureToken + Debug("Remember origial token:", string(secureToken)) + } + case '3': // Replayed response + if originalToken, ok := originalTokens[reqID]; ok { + delete(originalTokens, reqID) + secureToken := proto.Body(payload) + tokenAliases[string(originalToken)] = secureToken - Debug("Create alias for new token token, was:", string(originalToken), "now:", string(secureToken)) - } - } + Debug("Create alias for new token token, was:", string(originalToken), "now:", string(secureToken)) + } + } } func Debug(args ...interface{}) { - fmt.Fprint(os.Stderr, "[DEBUG][TOKEN-MOD] ") - fmt.Fprintln(os.Stderr, args...) -} \ No newline at end of file + fmt.Fprint(os.Stderr, "[DEBUG][TOKEN-MOD] ") + fmt.Fprintln(os.Stderr, args...) +} diff --git a/input_raw_test.go b/input_raw_test.go index 6a2b117..3276e7a 100644 --- a/input_raw_test.go +++ b/input_raw_test.go @@ -138,7 +138,7 @@ func TestInputRAWChunkedEncoding(t *testing.T) { }) replayAddr := listener.Addr().String() - httpOutput := NewHTTPOutput(replayAddr, &HTTPOutputConfig{Debug: true}) + httpOutput := NewHTTPOutput(replayAddr, &HTTPOutputConfig{Debug: false}) Plugins.Inputs = []io.Reader{input} Plugins.Outputs = []io.Writer{httpOutput} diff --git a/middleware.go b/middleware.go index f1d072d..1057173 100644 --- a/middleware.go +++ b/middleware.go @@ -17,7 +17,7 @@ type Middleware struct { data chan []byte - mu sync.Mutex + mu sync.Mutex Stdin io.Writer Stdout io.Reader diff --git a/middleware_test.go b/middleware_test.go index c691e58..f80e330 100644 --- a/middleware_test.go +++ b/middleware_test.go @@ -187,7 +187,7 @@ func TestTokenMiddleware(t *testing.T) { resp, _ = client.Get("/token") token = proto.Body(resp) - time.Sleep(50*time.Millisecond) + time.Sleep(10 * time.Millisecond) resp, _ = client.Get("/secure?token=" + string(token)) if !bytes.Equal(proto.Status(resp), []byte("202")) { diff --git a/output_http.go b/output_http.go index 38c423f..c3e0450 100644 --- a/output_http.go +++ b/output_http.go @@ -11,7 +11,7 @@ const initialDynamicWorkers = 10 type response struct { payload []byte - uuid []byte + uuid []byte } // HTTPOutputConfig struct for holding http output configuration diff --git a/raw_socket_listener/listener.go b/raw_socket_listener/listener.go index 3c7af46..eee05f4 100644 --- a/raw_socket_listener/listener.go +++ b/raw_socket_listener/listener.go @@ -204,6 +204,7 @@ func (t *Listener) processTCPPacket(packet *TCPPacket) { // Handling Expect: 100-continue requests if bytes.Equal(packet.Data[0:4], bPOST) { + // reading last 20 bytes (not counting CRLF): last header value (if no body presented) if bytes.Equal(packet.Data[len(packet.Data)-24:len(packet.Data)-4], bExpect100ContinueCheck) { t.seqWithData[packet.Seq+uint32(len(packet.Data))] = packet.Ack diff --git a/raw_socket_listener/tcp_message.go b/raw_socket_listener/tcp_message.go index 14e70d9..bba9d29 100644 --- a/raw_socket_listener/tcp_message.go +++ b/raw_socket_listener/tcp_message.go @@ -1,12 +1,14 @@ package rawSocket import ( - "log" - "sort" - "time" + "bytes" "crypto/sha1" "encoding/hex" + "github.com/buger/gor/proto" + "log" + "sort" "strconv" + "time" ) // TCPMessage ensure that all TCP packets for given request is received, and processed in right sequence @@ -22,7 +24,8 @@ type TCPMessage struct { RequestAck uint32 Start int64 IsIncoming bool - packets []*TCPPacket + + packets []*TCPPacket timer *time.Timer // Used for expire check @@ -40,9 +43,6 @@ func NewTCPMessage(ID string, delChan chan *TCPMessage, Ack uint32, expire *time msg.packetsChan = make(chan *TCPPacket) msg.delChan = delChan // used for notifying that message completed or expired - // Every time we receive packet we reset this timer - msg.timer = time.AfterFunc(*msg.expire, msg.Timeout) - go msg.listen() return @@ -64,6 +64,10 @@ func (t *TCPMessage) listen() { // Timeout notifies message to stop listening, close channel and message ready to be sent func (t *TCPMessage) Timeout() { + if t.timer != nil { + t.timer.Stop() + } + select { // In some cases Timeout can be called multiple times (do not know how yet) // Ensure that we did not close channel 2 times @@ -117,8 +121,63 @@ func (t *TCPMessage) AddPacket(packet *TCPPacket) { t.packets = append(t.packets, packet) } - // Reset message timeout timer - t.timer.Reset(*t.expire) + if !t.isMultipart() { + log.Println("MESSAGE NOT MULTIPART", string(packet.Data)) + t.Timeout() + } else { + log.Println("MESSAGE MULTIPART", string(packet.Data)) + // If more then 1 packet, wait for more, and set expiration + if len(t.packets) == 1 { + // Every time we receive packet we reset this timer + t.timer = time.AfterFunc(*t.expire, t.Timeout) + } else { + // Reset message timeout timer + t.timer.Reset(*t.expire) + } + } +} + +// isMultipart returns true if message contains from multiple tcp packets +func (t *TCPMessage) isMultipart() bool { + if len(t.packets) > 1 { + return true + } + + payload := t.packets[0].Data + m := payload[:3] + + if t.IsIncoming { + // If one GET, OPTIONS, or HEAD request + if bytes.Equal(m, []byte("GET")) || bytes.Equal(m, []byte("OPT")) || bytes.Equal(m, []byte("HEA")) { + return false + } else { + if length := proto.Header(payload, []byte("Content-Length")); len(length) > 0 { + l, _ := strconv.Atoi(string(length)) + + log.Println("Content-Length", l, "Body length:", len(proto.Body(payload))) + // If content-length equal current body length + if l > 0 && l == len(proto.Body(payload)) { + return false + } + } + } + } else { + if length := proto.Header(payload, []byte("Content-Length")); len(length) > 0 { + if length[0] == '0' { + return false + } + + l, _ := strconv.Atoi(string(length)) + + log.Println("Content-Length", l, "Body length:", len(proto.Body(payload))) + // If content-length equal current body length + if l > 0 && l == len(proto.Body(payload)) { + return false + } + } + } + + return true } func (t *TCPMessage) UUID() []byte { From 56f0a3fcc0a9c564d3aa29d84b88f2bab01220c8 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Sat, 15 Aug 2015 07:57:25 +0300 Subject: [PATCH 19/64] Remove too verbose output --- raw_socket_listener/tcp_message.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/raw_socket_listener/tcp_message.go b/raw_socket_listener/tcp_message.go index bba9d29..4ff9975 100644 --- a/raw_socket_listener/tcp_message.go +++ b/raw_socket_listener/tcp_message.go @@ -154,7 +154,6 @@ func (t *TCPMessage) isMultipart() bool { if length := proto.Header(payload, []byte("Content-Length")); len(length) > 0 { l, _ := strconv.Atoi(string(length)) - log.Println("Content-Length", l, "Body length:", len(proto.Body(payload))) // If content-length equal current body length if l > 0 && l == len(proto.Body(payload)) { return false @@ -169,7 +168,6 @@ func (t *TCPMessage) isMultipart() bool { l, _ := strconv.Atoi(string(length)) - log.Println("Content-Length", l, "Body length:", len(proto.Body(payload))) // If content-length equal current body length if l > 0 && l == len(proto.Body(payload)) { return false From 4f2341b2d3c2c4885def60067574c9fb26509cbb Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Sat, 15 Aug 2015 08:06:13 +0300 Subject: [PATCH 20/64] Remove more debug --- http_client.go | 2 -- raw_socket_listener/tcp_message.go | 2 -- 2 files changed, 4 deletions(-) diff --git a/http_client.go b/http_client.go index 37b2691..e3d6c1e 100644 --- a/http_client.go +++ b/http_client.go @@ -165,8 +165,6 @@ func (c *HTTPClient) Send(data []byte) (response []byte, err error) { payload := c.respBuf[:n] - Debug("[HTTPClient] Received:", n) - if c.config.Debug { Debug("[HTTPClient] Received:", string(payload)) } diff --git a/raw_socket_listener/tcp_message.go b/raw_socket_listener/tcp_message.go index 4ff9975..359e62e 100644 --- a/raw_socket_listener/tcp_message.go +++ b/raw_socket_listener/tcp_message.go @@ -122,10 +122,8 @@ func (t *TCPMessage) AddPacket(packet *TCPPacket) { } if !t.isMultipart() { - log.Println("MESSAGE NOT MULTIPART", string(packet.Data)) t.Timeout() } else { - log.Println("MESSAGE MULTIPART", string(packet.Data)) // If more then 1 packet, wait for more, and set expiration if len(t.packets) == 1 { // Every time we receive packet we reset this timer From 41c1421a1b4ad92557510381073263514529b3ff Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Sat, 15 Aug 2015 08:36:13 +0300 Subject: [PATCH 21/64] Simulate some real-life server by adding small response delay --- middleware_test.go | 3 +++ settings.go | 16 +++++++++++++--- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/middleware_test.go b/middleware_test.go index f80e330..b99ccae 100644 --- a/middleware_test.go +++ b/middleware_test.go @@ -146,6 +146,7 @@ func TestTokenMiddleware(t *testing.T) { wg := new(sync.WaitGroup) from := NewFakeSecureService(wg, func(path string, status int, tok []byte) { + time.Sleep(10 * time.Millisecond) }) to := NewFakeSecureService(wg, func(path string, status int, tok []byte) { switch path { @@ -158,6 +159,8 @@ func TestTokenMiddleware(t *testing.T) { t.Error("Server should receive valid rewritten token") } } + + time.Sleep(10 * time.Millisecond) }) quit := make(chan int) diff --git a/settings.go b/settings.go index 528adcd..37ac211 100644 --- a/settings.go +++ b/settings.go @@ -3,8 +3,9 @@ package main import ( "flag" "fmt" - "log" "os" + "sync" + "time" ) var VERSION string @@ -122,10 +123,19 @@ func init() { flag.Var(&Settings.modifierConfig.paramHashFilters, "http-param-limiter", "Takes a fraction of requests, consistently taking or rejecting a request based on the FNV32-1A hash of a specific GET param:\n\t gor --input-raw :8080 --output-http staging.com --http-param-limiter user_id:25%") } +var previousDebugTime int64 +var debugMutex sync.Mutex + // Debug gets called only if --verbose flag specified func Debug(args ...interface{}) { if Settings.verbose { - fmt.Printf("[DEBUG][PID %d] ", os.Getpid()) - log.Println(args...) + debugMutex.Lock() + now := time.Now() + diff := float64(now.UnixNano()-previousDebugTime) / 1000000 + previousDebugTime = now.UnixNano() + debugMutex.Unlock() + + fmt.Printf("[DEBUG][PID %d][%d][%fms] ", os.Getpid(), now.UnixNano(), diff) + fmt.Println(args...) } } From 1a36374c2dc66fe92afa1304bb7d2656b5e38b07 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Sat, 15 Aug 2015 08:44:27 +0300 Subject: [PATCH 22/64] Add some comments --- middleware_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/middleware_test.go b/middleware_test.go index b99ccae..6dd5b29 100644 --- a/middleware_test.go +++ b/middleware_test.go @@ -190,6 +190,8 @@ func TestTokenMiddleware(t *testing.T) { resp, _ = client.Get("/token") token = proto.Body(resp) + // When delay is too smal, middleware does not always rewrite requests in time + // Hopefuly client will have delay more then 10ms :) time.Sleep(10 * time.Millisecond) resp, _ = client.Get("/secure?token=" + string(token)) From a18079fa840112678ac1e3a1269871aa7bdfbdbf Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Sat, 15 Aug 2015 10:33:50 +0300 Subject: [PATCH 23/64] Improving docs --- README.md | 23 +++++++++++- examples/echo_modifier.sh | 32 ---------------- examples/{ => middleware}/echo_modifier.rb | 0 examples/middleware/echo_modifier.sh | 41 +++++++++++++++++++++ examples/{ => middleware}/token_modifier.go | 2 +- input_raw_test.go | 2 - middleware_test.go | 5 ++- 7 files changed, 66 insertions(+), 39 deletions(-) delete mode 100755 examples/echo_modifier.sh rename examples/{ => middleware}/echo_modifier.rb (100%) create mode 100755 examples/middleware/echo_modifier.sh rename examples/{ => middleware}/token_modifier.go (99%) diff --git a/README.md b/README.md index 2ea19c3..67f02e6 100644 --- a/README.md +++ b/README.md @@ -61,7 +61,7 @@ gor --input-tcp :28020 --output-http "http://staging.com" --output-http "http:/ ``` ### HTTP output workers -By default Gor creates dynamic pull of workers: it starts with 10 and create more http output workers when the http output queue length is greater than 10. The number of workers created (N) is equal to the queue length at the time which it is checked and found to have a length greater than 10. The queue length is checked every time a message is written to the http output queue. No more workers will be spawned until that request to spawn N workers is satisfied. If a dynamic worker cannot process a message at that time, it will sleep for 100 milliseconds. If a dynamic worker cannot process a message for 2 seconds it dies. +By default Gor creates dynamic pull of workers: it starts with 10 and create more http output workers when the http output queue length is greater than 10. The number of workers created (N) is equal to the queue length at the time which it is checked and found to have a length greater than 10. The queue length is checked every time a message is written to the http output queue. No more workers will be spawned until that request to spawn N workers is satisfied. If a dynamic worker cannot process a message at that time, it will sleep for 100 milliseconds. If a dynamic worker cannot process a message for 2 seconds it dies. You may specify fixed number of workers using `--output-http-workers=20` option. ### Follow redirects @@ -149,7 +149,7 @@ gor --input-raw :80 --output-http "http://staging.server" \ ``` ### Rewriting original request -Gor supports built-in basic rewriting support, for complex logic see https://github.com/buger/gor/pull/162 +Gor supports some basic request rewriting support. For complex logic you can use middleware, see below. #### Rewrite URL based on a mapping ``` @@ -177,6 +177,25 @@ Host header gets special treatment. By default Host get set to the value specifi If you app accepts traffic from multiple domain, and you want to keep original headers, there is specific `--http-original-host` with tells Gor do not touch Host header at all. +### Middleware +Middleware is a programm that accepts request payload and response at STDIN and emits modified requests at STDOUT. + +``` + Original request +--------------+ ++-------------+----------STDIN---------->+ | +| Gor input | | Middleware | ++-------------+----------STDIN---------->+ | + Original response +------+---+---+ + | ^ ++-------------+ Modified request v | +| Gor output +<---------STDOUT-----------------+ | ++-----+-------+ | + | | + | Replayed response | + +------------------STDIN----------------->----+ +``` + + ### Saving requests to file and replaying them You can save requests to file, and replay them later: ``` diff --git a/examples/echo_modifier.sh b/examples/echo_modifier.sh deleted file mode 100755 index 13d2074..0000000 --- a/examples/echo_modifier.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/usr/bin/env bash -while read line; do - decoded=$(echo -e "$line" | xxd -r -p) - - header=$(echo -e "$decoded" | head -n +1) - payload=$(echo -e "$decoded" | tail -n +2) - - encoded=$(echo -e "$header\n$payload" | xxd -p | tr -d "\\n") - - >&2 echo "" - >&2 echo "[DEBUG][ECHO] ===================================" - - case ${header:0:1} in - "2") - >&2 echo "[DEBUG][ECHO] Request type: Original Response" - ;; - "3") - >&2 echo "[DEBUG][ECHO] Request type: Replayed Response" - ;; - "1") - >&2 echo "[DEBUG][ECHO] Request type: Request" - echo "$encoded" - ;; - *) - >&2 echo "[DEBUG][ECHO] Unknown request type $header" - esac - >&2 echo "[DEBUG][ECHO] ===================================" - - >&2 echo "[DEBUG][ECHO] Original data: $line" - >&2 echo "[DEBUG][ECHO] Decoded request: $decoded" - >&2 echo "[DEBUG][ECHO] Encoded data: $encoded" -done; diff --git a/examples/echo_modifier.rb b/examples/middleware/echo_modifier.rb similarity index 100% rename from examples/echo_modifier.rb rename to examples/middleware/echo_modifier.rb diff --git a/examples/middleware/echo_modifier.sh b/examples/middleware/echo_modifier.sh new file mode 100755 index 0000000..96dc121 --- /dev/null +++ b/examples/middleware/echo_modifier.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +# +# `xxd` utility included into vim-common package +# It allow hex decoding/encoding + +function log { + # Logging to stderr, because stdout/stdin used for data transfer + >&2 echo "[DEBUG][ECHO] $1" +} + +while read line; do + decoded=$(echo -e "$line" | xxd -r -p) + + header=$(echo -e "$decoded" | head -n +1) + payload=$(echo -e "$decoded" | tail -n +2) + + encoded=$(echo -e "$header\n$payload" | xxd -p | tr -d "\\n") + + log "" + log "===================================" + + case ${header:0:1} in + "1") + log "Request type: Request" + echo "$encoded" + ;; + "2") + log "Request type: Original Response" + ;; + "3") + log "Request type: Replayed Response" + ;; + *) + log "Unknown request type $header" + esac + log "===================================" + + log "Original data: $line" + log "Decoded request: $decoded" + log "Encoded data: $encoded" +done; diff --git a/examples/token_modifier.go b/examples/middleware/token_modifier.go similarity index 99% rename from examples/token_modifier.go rename to examples/middleware/token_modifier.go index 89ec995..1245c5f 100644 --- a/examples/token_modifier.go +++ b/examples/middleware/token_modifier.go @@ -45,7 +45,7 @@ func process(buf []byte) { Debug("Received payload:", string(buf)) switch payloadType { - case '1': + case '1': // Request if bytes.Equal(proto.Path(payload), []byte("/token")) { originalTokens[reqID] = []byte{} Debug("Found token request:", reqID) diff --git a/input_raw_test.go b/input_raw_test.go index 3276e7a..3273035 100644 --- a/input_raw_test.go +++ b/input_raw_test.go @@ -244,6 +244,4 @@ func TestInputRAWResponse(t *testing.T) { wg.Wait() close(quit) - - time.Sleep(100 * time.Millisecond) } diff --git a/middleware_test.go b/middleware_test.go index 6dd5b29..25d3374 100644 --- a/middleware_test.go +++ b/middleware_test.go @@ -108,7 +108,7 @@ func TestEchoMiddleware(t *testing.T) { quit := make(chan int) - Settings.middleware = "./examples/echo_modifier.sh" + Settings.middleware = "./examples/middleware/echo_modifier.sh" // Catch traffic from one service input := NewRAWInput(from.Listener.Addr().String(), testRawExpire, true) @@ -173,12 +173,13 @@ func TestTokenMiddleware(t *testing.T) { Plugins.Inputs = []io.Reader{input} Plugins.Outputs = []io.Writer{output} - Settings.middleware = "go run ./examples/token_modifier.go" + Settings.middleware = "go run ./examples/middleware/token_modifier.go" // Start Gor go Start(quit) // Wait for middleware to initialize + // Give go compiller time to build programm time.Sleep(500 * time.Millisecond) // Should receive 2 requests from original + 2 from replayed From ddd519062213863878e987082df92ddb8d537c23 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Sat, 15 Aug 2015 21:05:10 +0300 Subject: [PATCH 24/64] Improve docs --- README.md | 25 ++++++++++++++++++++++++- examples/middleware/token_modifier.go | 23 +++++++++++++++++++++-- middleware.go | 5 ++++- middleware_test.go | 1 - 4 files changed, 49 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 67f02e6..5ccad35 100644 --- a/README.md +++ b/README.md @@ -178,7 +178,7 @@ Host header gets special treatment. By default Host get set to the value specifi If you app accepts traffic from multiple domain, and you want to keep original headers, there is specific `--http-original-host` with tells Gor do not touch Host header at all. ### Middleware -Middleware is a programm that accepts request payload and response at STDIN and emits modified requests at STDOUT. +Middleware is a programm that accepts request and response payload at STDIN and emits modified requests at STDOUT. You can implement any custom logic like stripping private data, advanced rewriting, support for oAuth and etc. ``` Original request +--------------+ @@ -195,6 +195,29 @@ Middleware is a programm that accepts request payload and response at STDIN and +------------------STDIN----------------->----+ ``` +Middleware can be written in any language, see `examples/middleware` folder for examples. +Middleware programm should accept the fact that all communication with Gor is asyncronious, there is no guarantee that original request and response messages will come one after each other. Your app should take care of the state if logic depends on original or replayed response, see `examples/middleware/token_modifier.go` as example. + +#### Communication protocol +All messages should be hex encoded, new line character specifieds the end of the message, eg. new message per line. + +Decoded payload consist of 2 parts: header and HTTP payload, separated by new line character. Example: +``` +1 932079936fa4306fc308d67588178d17d823647c +GET /a HTTP/1.1 +Host: 127.0.0.1 + +``` + +First header byte `1` represent payload type, possible values: `1` - request, `2` - original response, `3` - replayed response +After empty spaces, goes request id `932079936fa4306fc308d67588178d17d823647c`, which is always 40 characters length. Request id unique among all requests (sha1 of time and Ack), but remain same for original and replayed response, so you can create associations between request and responses. + +HTTP payload is unmodified HTTP requests/responses intercepted from network. You can read more about request format [here](http://www.jmarshall.com/easy/http/), [here](https://en.wikipedia.org/wiki/Hypertext_Transfer_Protocol) and [here](http://www.w3.org/Protocols/rfc2616/rfc2616.html). You can operate with payload as you want, add headers, change path, and etc. Basically you just editing a string, just ensure that it is RCF compliant. + +At the end modified (or untouched) request should be emitted back to STDOUT, keeping original header, and hex-encoded. If you want to filter request, just not send it. Emitting responses back is optional, and does not affect anyting at the moment. + +#### Advanced example +Imagine that you have auth system that randomly generate access tokens, which used later for accessing secure content. Since there is no pre-defined token value, naive approach without middleware (or if middleware use only request payloads) will fail, because replayed server have own tokens, not synced with origin. To fix this, our middleware should take in account responses of replayed and origin server, store `originalToken -> replayedToken` aliases and rewrite all requests using this token to use replayed alias. See `examples/middleware/token_modifier.go` and `middleware_test.go#TestTokenMiddleware` as example of described scheme. ### Saving requests to file and replaying them You can save requests to file, and replay them later: diff --git a/examples/middleware/token_modifier.go b/examples/middleware/token_modifier.go index 1245c5f..3b871e2 100644 --- a/examples/middleware/token_modifier.go +++ b/examples/middleware/token_modifier.go @@ -1,3 +1,22 @@ +/* +This middleware made for auth system that randomly generate access tokens, which used later for accessing secure content. Since there is no pre-defined token value, naive approach without middleware (or if middleware use only request payloads) will fail, because replayed server have own tokens, not synced with origin. To fix this, our middleware should take in account responses of replayed and origin server, store `originalToken -> replayedToken` aliases and rewrite all requests using this token to use replayed alias. See `middleware_test.go#TestTokenMiddleware` test for examples of using this middleware. + +How middleware works: + + Original request +--------------+ ++-------------+----------STDIN---------->+ | +| Gor input | | Middleware | ++-------------+----------STDIN---------->+ | + Original response +------+---+---+ + | ^ ++-------------+ Modified request v | +| Gor output +<---------STDOUT-----------------+ | ++-----+-------+ | + | | + | Replayed response | + +------------------STDIN----------------->----+ +*/ + package main import ( @@ -50,10 +69,10 @@ func process(buf []byte) { originalTokens[reqID] = []byte{} Debug("Found token request:", reqID) } else { - tokenVal, vs, _ := proto.PathParam(payload, []byte("token")) + token, vs, _ := proto.PathParam(payload, []byte("token")) if vs != -1 { // If there is GET token param - if alias, ok := tokenAliases[string(tokenVal)]; ok { + if alias, ok := tokenAliases[string(token)]; ok { // Rewrite original token to alias payload = proto.SetPathParam(payload, []byte("token"), alias) diff --git a/middleware.go b/middleware.go index 1057173..cb4ea12 100644 --- a/middleware.go +++ b/middleware.go @@ -33,7 +33,10 @@ func NewMiddleware(command string) *Middleware { m.Stdout, _ = cmd.StdoutPipe() m.Stdin, _ = cmd.StdinPipe() - cmd.Stderr = os.Stderr + + if Settings.verbose { + cmd.Stderr = os.Stderr + } go m.read(m.Stdout) diff --git a/middleware_test.go b/middleware_test.go index 25d3374..05bcb4a 100644 --- a/middleware_test.go +++ b/middleware_test.go @@ -68,7 +68,6 @@ func TestFakeSecureService(t *testing.T) { wg := new(sync.WaitGroup) addr := NewFakeSecureService(wg, func(path string, status int, resp []byte) { - }) wg.Add(3) From 879ed48b80a4a9ac8af2f24b6c0fafae9af7c42b Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Sat, 15 Aug 2015 21:05:51 +0300 Subject: [PATCH 25/64] Doc changes --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 5ccad35..33d68a4 100644 --- a/README.md +++ b/README.md @@ -210,7 +210,7 @@ Host: 127.0.0.1 ``` First header byte `1` represent payload type, possible values: `1` - request, `2` - original response, `3` - replayed response -After empty spaces, goes request id `932079936fa4306fc308d67588178d17d823647c`, which is always 40 characters length. Request id unique among all requests (sha1 of time and Ack), but remain same for original and replayed response, so you can create associations between request and responses. +After empty spaces, goes request id `932079936fa4306fc308d67588178d17d823647c`. Request id unique among all requests (sha1 of time and Ack), but remain same for original and replayed response, so you can create associations between request and responses. HTTP payload is unmodified HTTP requests/responses intercepted from network. You can read more about request format [here](http://www.jmarshall.com/easy/http/), [here](https://en.wikipedia.org/wiki/Hypertext_Transfer_Protocol) and [here](http://www.w3.org/Protocols/rfc2616/rfc2616.html). You can operate with payload as you want, add headers, change path, and etc. Basically you just editing a string, just ensure that it is RCF compliant. From 0d445722b90a8419af8dab4828a8ca4a08c6639b Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Sat, 15 Aug 2015 21:27:20 +0300 Subject: [PATCH 26/64] Some fixes --- Makefile | 4 ++-- README.md | 18 +++++++++++++++--- input_raw.go | 25 ------------------------- payload_format.go | 26 ++++++++++++++++++++++++++ plugins.go | 8 +++++++- 5 files changed, 50 insertions(+), 31 deletions(-) create mode 100644 payload_format.go diff --git a/Makefile b/Makefile index 51ebf17..6f43cab 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -SOURCE = emitter.go gor.go gor_stat.go input_dummy.go input_file.go input_raw.go input_tcp.go limiter.go output_dummy.go output_file.go input_http.go output_http.go output_tcp.go plugins.go settings.go test_input.go elasticsearch.go http_modifier.go http_modifier_settings.go http_client.go traffic_modifier.go +SOURCE = emitter.go gor.go gor_stat.go input_dummy.go input_file.go input_raw.go input_tcp.go limiter.go output_dummy.go output_file.go input_http.go output_http.go output_tcp.go plugins.go settings.go test_input.go elasticsearch.go http_modifier.go http_modifier_settings.go http_client.go middleware.go payload_format.go SOURCE_PATH = /gopath/src/github.com/buger/gor/ @@ -37,7 +37,7 @@ dbench: # Used mainly for debugging, because docker container do not have access to parent machine ports drun: - docker run -v `pwd`:$(SOURCE_PATH) -t -i gor go run $(SOURCE) --input-dummy=0 --output-http="http://localhost:9000" --verbose + docker run -v `pwd`:$(SOURCE_PATH) -t -i gor go run $(SOURCE) --input-dummy=0 --output-http="http://localhost:9000" --input-http :9000 --verbose drecord: docker run -v `pwd`:$(SOURCE_PATH) -t -i gor go run $(SOURCE) --input-dummy=0 --output-file=requests.bin --verbose diff --git a/README.md b/README.md index 33d68a4..8e5dbf0 100644 --- a/README.md +++ b/README.md @@ -178,7 +178,7 @@ Host header gets special treatment. By default Host get set to the value specifi If you app accepts traffic from multiple domain, and you want to keep original headers, there is specific `--http-original-host` with tells Gor do not touch Host header at all. ### Middleware -Middleware is a programm that accepts request and response payload at STDIN and emits modified requests at STDOUT. You can implement any custom logic like stripping private data, advanced rewriting, support for oAuth and etc. +Middleware is a program that accepts request and response payload at STDIN and emits modified requests at STDOUT. You can implement any custom logic like stripping private data, advanced rewriting, support for oAuth and etc. ``` Original request +--------------+ @@ -196,7 +196,19 @@ Middleware is a programm that accepts request and response payload at STDIN and ``` Middleware can be written in any language, see `examples/middleware` folder for examples. -Middleware programm should accept the fact that all communication with Gor is asyncronious, there is no guarantee that original request and response messages will come one after each other. Your app should take care of the state if logic depends on original or replayed response, see `examples/middleware/token_modifier.go` as example. +Middleware program should accept the fact that all communication with Gor is asynchronous, there is no guarantee that original request and response messages will come one after each other. Your app should take care of the state if logic depends on original or replayed response, see `examples/middleware/token_modifier.go` as example. + +Simple bash echo middleware (returns same request) will look like this: +```bash +while read line; do + echo $line +end +``` + +Middleware can be enabled using `--middleware` option, by specifying path to executable file: +``` +gor --input-raw :80 --middleware "/opt/middleware_executable" --output-http "http://staging.server" +``` #### Communication protocol All messages should be hex encoded, new line character specifieds the end of the message, eg. new message per line. @@ -214,7 +226,7 @@ After empty spaces, goes request id `932079936fa4306fc308d67588178d17d823647c`. HTTP payload is unmodified HTTP requests/responses intercepted from network. You can read more about request format [here](http://www.jmarshall.com/easy/http/), [here](https://en.wikipedia.org/wiki/Hypertext_Transfer_Protocol) and [here](http://www.w3.org/Protocols/rfc2616/rfc2616.html). You can operate with payload as you want, add headers, change path, and etc. Basically you just editing a string, just ensure that it is RCF compliant. -At the end modified (or untouched) request should be emitted back to STDOUT, keeping original header, and hex-encoded. If you want to filter request, just not send it. Emitting responses back is optional, and does not affect anyting at the moment. +At the end modified (or untouched) request should be emitted back to STDOUT, keeping original header, and hex-encoded. If you want to filter request, just not send it. Emitting responses back is optional, and does not affect anything at the moment. #### Advanced example Imagine that you have auth system that randomly generate access tokens, which used later for accessing secure content. Since there is no pre-defined token value, naive approach without middleware (or if middleware use only request payloads) will fail, because replayed server have own tokens, not synced with origin. To fix this, our middleware should take in account responses of replayed and origin server, store `originalToken -> replayedToken` aliases and rewrite all requests using this token to use replayed alias. See `examples/middleware/token_modifier.go` and `middleware_test.go#TestTokenMiddleware` as example of described scheme. diff --git a/input_raw.go b/input_raw.go index 9d786b3..cfdfc8b 100644 --- a/input_raw.go +++ b/input_raw.go @@ -29,31 +29,6 @@ func NewRAWInput(address string, expire time.Duration, captureResponse bool) (i return } -const ( - RequestPayload = 1 << iota - ResponsePayload - ReplayedResponsePayload -) - -func payloadHeader(payloadType int, uuid []byte) (header []byte) { - header = make([]byte, 43) - header[1] = ' ' - header[len(header)-1] = '\n' - - switch payloadType { - case RequestPayload: - header[0] = '1' - case ResponsePayload: - header[0] = '2' - case ReplayedResponsePayload: - header[0] = '3' - } - - copy(header[2:], uuid) - - return header -} - func (i *RAWInput) Read(data []byte) (int, error) { msg := <-i.data buf := msg.Bytes() diff --git a/payload_format.go b/payload_format.go new file mode 100644 index 0000000..f26b103 --- /dev/null +++ b/payload_format.go @@ -0,0 +1,26 @@ +package main + +const ( + RequestPayload = 1 << iota + ResponsePayload + ReplayedResponsePayload +) + +func payloadHeader(payloadType int, uuid []byte) (header []byte) { + header = make([]byte, 43) + header[1] = ' ' + header[len(header)-1] = '\n' + + switch payloadType { + case RequestPayload: + header[0] = '1' + case ResponsePayload: + header[0] = '2' + case ReplayedResponsePayload: + header[0] = '3' + } + + copy(header[2:], uuid) + + return header +} diff --git a/plugins.go b/plugins.go index c92435c..ce8e7a1 100644 --- a/plugins.go +++ b/plugins.go @@ -79,7 +79,13 @@ func InitPlugins() { } for _, options := range Settings.inputRAW { - registerPlugin(NewRAWInput, options) + captureResponse := false + + if len(Settings.middleware) > 0 { + captureResponse = true + } + + registerPlugin(NewRAWInput, options, 0, captureResponse) } for _, options := range Settings.inputTCP { From d1091102ba8bc043cdc1eb19654fd41c49cd2e39 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Sat, 15 Aug 2015 21:46:09 +0300 Subject: [PATCH 27/64] Ignore all release files --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index c59f5ba..3c34e46 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,5 @@ *.out *.bin + +*.gz From 211be0cdddf91a7ffbb15a1902382f2195f6e7cf Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Mon, 17 Aug 2015 17:10:43 +0300 Subject: [PATCH 28/64] Add request time and round-trip response time --- Makefile | 2 +- README.md | 22 +++++++++++++--- examples/middleware/token_modifier.go | 11 +++++--- input_raw.go | 11 ++++---- middleware.go | 1 - output_http.go | 16 ++++++----- payload_format.go | 26 ------------------ protocol.go | 38 +++++++++++++++++++++++++++ raw_socket_listener/tcp_message.go | 2 ++ 9 files changed, 82 insertions(+), 47 deletions(-) delete mode 100644 payload_format.go create mode 100644 protocol.go diff --git a/Makefile b/Makefile index 6f43cab..11ce2c6 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -SOURCE = emitter.go gor.go gor_stat.go input_dummy.go input_file.go input_raw.go input_tcp.go limiter.go output_dummy.go output_file.go input_http.go output_http.go output_tcp.go plugins.go settings.go test_input.go elasticsearch.go http_modifier.go http_modifier_settings.go http_client.go middleware.go payload_format.go +SOURCE = emitter.go gor.go gor_stat.go input_dummy.go input_file.go input_raw.go input_tcp.go limiter.go output_dummy.go output_file.go input_http.go output_http.go output_tcp.go plugins.go settings.go test_input.go elasticsearch.go http_modifier.go http_modifier_settings.go http_client.go middleware.go protocol.go SOURCE_PATH = /gopath/src/github.com/buger/gor/ diff --git a/README.md b/README.md index 8e5dbf0..e341d98 100644 --- a/README.md +++ b/README.md @@ -213,16 +213,30 @@ gor --input-raw :80 --middleware "/opt/middleware_executable" --output-http "htt #### Communication protocol All messages should be hex encoded, new line character specifieds the end of the message, eg. new message per line. -Decoded payload consist of 2 parts: header and HTTP payload, separated by new line character. Example: +Decoded payload consist of 2 parts: header and HTTP payload, separated by new line character. + +Example request payload: + ``` -1 932079936fa4306fc308d67588178d17d823647c +1 932079936fa4306fc308d67588178d17d823647c 1439818823587396305 GET /a HTTP/1.1 Host: 127.0.0.1 ``` -First header byte `1` represent payload type, possible values: `1` - request, `2` - original response, `3` - replayed response -After empty spaces, goes request id `932079936fa4306fc308d67588178d17d823647c`. Request id unique among all requests (sha1 of time and Ack), but remain same for original and replayed response, so you can create associations between request and responses. +Example response payload: + +``` +2 8e091765ae902fef8a2b7d9dd960e9d52222bd8c 2782013 +HTTP/1.1 200 OK +Date: Mon, 17 Aug 2015 13:40:23 GMT +Content-Length: 0 +Content-Type: text/plain; charset=utf-8 + +``` + +Header contains request meta information separated by spaces. First value is payload type, possible values: `1` - request, `2` - original response, `3` - replayed response. +Next goes request id: unique among all requests (sha1 of time and Ack), but remain same for original and replayed response, so you can create associations between request and responses. Third argument varies depending on payload type: for request - start time, for responses - round-trip time. HTTP payload is unmodified HTTP requests/responses intercepted from network. You can read more about request format [here](http://www.jmarshall.com/easy/http/), [here](https://en.wikipedia.org/wiki/Hypertext_Transfer_Protocol) and [here](http://www.w3.org/Protocols/rfc2616/rfc2616.html). You can operate with payload as you want, add headers, change path, and etc. Basically you just editing a string, just ensure that it is RCF compliant. diff --git a/examples/middleware/token_modifier.go b/examples/middleware/token_modifier.go index 3b871e2..9da3fc8 100644 --- a/examples/middleware/token_modifier.go +++ b/examples/middleware/token_modifier.go @@ -55,10 +55,13 @@ func process(buf []byte) { // 2 - Response // 3 - ReplayedResponse payloadType := buf[0] - headerSize := 42 - header := buf[:headerSize] + headerSize := bytes.IndexByte(buf, '\n') + 1 + header := buf[:headerSize-1] + + // Header contains space separated values of: request type, request id, and request start time (or round-trip time for responses) + meta := bytes.Split(header, []byte(" ")) // For each request you should receive 3 payloads (request, response, replayed response) with same request id - reqID := string(header[2:headerSize]) + reqID := string(meta[1]) payload := buf[headerSize:] Debug("Received payload:", string(buf)) @@ -83,7 +86,7 @@ func process(buf []byte) { } // Re-compute length in case if payload was modified - bufLen := len(header) + len(payload) + bufLen := headerSize + len(payload) // Encoding request and sending it back dst := make([]byte, bufLen*2+1) hex.Encode(dst, buf[:bufLen]) diff --git a/input_raw.go b/input_raw.go index cfdfc8b..75a37ec 100644 --- a/input_raw.go +++ b/input_raw.go @@ -34,12 +34,13 @@ func (i *RAWInput) Read(data []byte) (int, error) { buf := msg.Bytes() if i.captureResponse { - payloadType := RequestPayload - if !msg.IsIncoming { - payloadType = ResponsePayload - } + var header []byte - header := payloadHeader(payloadType, msg.UUID()) + if msg.IsIncoming { + header = payloadHeader(RequestPayload, msg.UUID(), msg.Start) + } else { + header = payloadHeader(ResponsePayload, msg.UUID(), msg.End-msg.RequestStart) + } copy(data[0:len(header)], header) copy(data[len(header):], buf) diff --git a/middleware.go b/middleware.go index cb4ea12..e88be4d 100644 --- a/middleware.go +++ b/middleware.go @@ -108,7 +108,6 @@ func (m *Middleware) read(from io.Reader) { } func (m *Middleware) Read(data []byte) (int, error) { - Debug("Trying to read channel!") buf := <-m.data copy(data, buf) diff --git a/output_http.go b/output_http.go index c3e0450..a44c7fd 100644 --- a/output_http.go +++ b/output_http.go @@ -1,6 +1,7 @@ package main import ( + "bytes" "io" "log" "sync/atomic" @@ -10,8 +11,9 @@ import ( const initialDynamicWorkers = 10 type response struct { - payload []byte - uuid []byte + payload []byte + uuid []byte + roundTripTime int64 } // HTTPOutputConfig struct for holding http output configuration @@ -163,7 +165,7 @@ func (o *HTTPOutput) Read(data []byte) (int, error) { Debug("[OUTPUT-HTTP] Received response", string(resp.payload)) - header := payloadHeader(ReplayedResponsePayload, resp.uuid) + header := payloadHeader(ReplayedResponsePayload, resp.uuid, resp.roundTripTime) copy(data[0:len(header)], header) copy(data[len(header):], resp.payload) @@ -174,8 +176,10 @@ func (o *HTTPOutput) sendRequest(client *HTTPClient, request []byte) { var uuid []byte if len(Settings.middleware) > 0 { - uuid = request[2:42] - request = request[43:] + headerSize := bytes.IndexByte(request, '\n') + meta := bytes.Split(request[:headerSize], []byte{' '}) + uuid = meta[1] + request = request[headerSize+1:] } start := time.Now() @@ -187,7 +191,7 @@ func (o *HTTPOutput) sendRequest(client *HTTPClient, request []byte) { } if len(Settings.middleware) > 0 { - o.responses <- response{resp, uuid} + o.responses <- response{resp, uuid, stop.UnixNano() - start.UnixNano()} } if o.elasticSearch != nil { diff --git a/payload_format.go b/payload_format.go deleted file mode 100644 index f26b103..0000000 --- a/payload_format.go +++ /dev/null @@ -1,26 +0,0 @@ -package main - -const ( - RequestPayload = 1 << iota - ResponsePayload - ReplayedResponsePayload -) - -func payloadHeader(payloadType int, uuid []byte) (header []byte) { - header = make([]byte, 43) - header[1] = ' ' - header[len(header)-1] = '\n' - - switch payloadType { - case RequestPayload: - header[0] = '1' - case ResponsePayload: - header[0] = '2' - case ReplayedResponsePayload: - header[0] = '3' - } - - copy(header[2:], uuid) - - return header -} diff --git a/protocol.go b/protocol.go new file mode 100644 index 0000000..a92ac80 --- /dev/null +++ b/protocol.go @@ -0,0 +1,38 @@ +package main + +import ( + "strconv" +) + +const ( + RequestPayload = 1 << iota + ResponsePayload + ReplayedResponsePayload +) + +// Timing is request start or round-trip time, depending on payloadType +func payloadHeader(payloadType int, uuid []byte, timing int64) (header []byte) { + sTime := strconv.FormatInt(timing, 10) + + //Example: + // 3 f45590522cd1838b4a0d5c5aab80b77929dea3b3 1231\n + // `+ 1` indicates space characters or end of line + header = make([]byte, 1+1+len(uuid)+1+len(sTime)+1) + header[1] = ' ' + header[2+len(uuid)] = ' ' + header[len(header)-1] = '\n' + + switch payloadType { + case RequestPayload: + header[0] = '1' + case ResponsePayload: + header[0] = '2' + case ReplayedResponsePayload: + header[0] = '3' + } + + copy(header[2:], uuid) + copy(header[3+len(uuid):], sTime) + + return header +} diff --git a/raw_socket_listener/tcp_message.go b/raw_socket_listener/tcp_message.go index 359e62e..956f4ec 100644 --- a/raw_socket_listener/tcp_message.go +++ b/raw_socket_listener/tcp_message.go @@ -23,6 +23,7 @@ type TCPMessage struct { RequestStart int64 RequestAck uint32 Start int64 + End int64 IsIncoming bool packets []*TCPPacket @@ -119,6 +120,7 @@ func (t *TCPMessage) AddPacket(packet *TCPPacket) { log.Println("Received packet with same sequence") } else { t.packets = append(t.packets, packet) + t.End = time.Now().UnixNano() } if !t.isMultipart() { From 69424bf6700fc592157c9e264af5a3575152913c Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Mon, 17 Aug 2015 17:13:55 +0300 Subject: [PATCH 29/64] Fix hound --- examples/middleware/echo_modifier.rb | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/examples/middleware/echo_modifier.rb b/examples/middleware/echo_modifier.rb index a20beb8..084acc1 100755 --- a/examples/middleware/echo_modifier.rb +++ b/examples/middleware/echo_modifier.rb @@ -1,16 +1,15 @@ #!/usr/bin/env ruby # encoding: utf-8 while data = STDIN.gets - next unless data - data = data.chomp + next unless data + data = data.chomp - decoded = [data].pack("H*") - encoded = decoded.unpack("H*").first + decoded = [data].pack("H*") + encoded = decoded.unpack("H*").first - STDOUT.puts encoded + STDOUT.puts encoded - - STDERR.puts "[DEBUG][MIDDLEWARE] Original data: #{data}" - STDERR.puts "[DEBUG][MIDDLEWARE] Decoded request: #{decoded}" - STDERR.puts "[DEBUG][MIDDLEWARE] Encoded data: #{encoded}" -end \ No newline at end of file + STDERR.puts "[DEBUG][MIDDLEWARE] Original data: #{data}" + STDERR.puts "[DEBUG][MIDDLEWARE] Decoded request: #{decoded}" + STDERR.puts "[DEBUG][MIDDLEWARE] Encoded data: #{encoded}" +end From d96e3393ec8cb059f9b4cc21be8753b2e24e5864 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Mon, 17 Aug 2015 20:36:59 +0300 Subject: [PATCH 30/64] Handle HTTP error codes --- http_client.go | 33 +++++++++++++++- http_client_test.go | 91 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 122 insertions(+), 2 deletions(-) diff --git a/http_client.go b/http_client.go index e3d6c1e..6ec1227 100644 --- a/http_client.go +++ b/http_client.go @@ -21,6 +21,7 @@ type HTTPClientConfig struct { FollowRedirects int Debug bool OriginalHost bool + ConnectionTimeout time.Duration Timeout time.Duration ResponseBufferSize int } @@ -69,9 +70,9 @@ func (c *HTTPClient) Connect() (err error) { c.Disconnect() if !strings.Contains(c.host, ":") { - c.conn, err = net.Dial("tcp", c.host+":80") + c.conn, err = net.DialTimeout("tcp", c.host+":80", c.config.ConnectionTimeout) } else { - c.conn, err = net.Dial("tcp", c.host) + c.conn, err = net.DialTimeout("tcp", c.host, c.config.ConnectionTimeout) } if c.scheme == "https" { @@ -124,6 +125,7 @@ func (c *HTTPClient) Send(data []byte) (response []byte, err error) { Debug("[HTTPClient] Connecting:", c.baseURL) if err = c.Connect(); err != nil { log.Println("[HTTPClient] Connection error:", err) + response = errorPayload(HTTP_CONNECTION_ERROR) return } } @@ -142,6 +144,7 @@ func (c *HTTPClient) Send(data []byte) (response []byte, err error) { if _, err = c.conn.Write(data); err != nil { Debug("[HTTPClient] Write error:", err, c.baseURL) + response = errorPayload(HTTP_TIMEOUT) return } @@ -160,6 +163,7 @@ func (c *HTTPClient) Send(data []byte) (response []byte, err error) { if err != nil { Debug("[HTTPClient] Response read error", err, c.conn) + response = errorPayload(HTTP_TIMEOUT) return } @@ -197,3 +201,28 @@ func (c *HTTPClient) Get(path string) (response []byte, err error) { return c.Send([]byte(payload)) } + +const ( + // https://support.cloudflare.com/hc/en-us/articles/200171936-Error-520-Web-server-is-returning-an-unknown-error + HTTP_UNKNOWN_ERROR = "520" + // https://support.cloudflare.com/hc/en-us/articles/200171916-Error-521-Web-server-is-down + HTTP_CONNECTION_ERROR = "521" + // https://support.cloudflare.com/hc/en-us/articles/200171906-Error-522-Connection-timed-out + HTTP_CONNECTION_TIMEOUT = "522" + // https://support.cloudflare.com/hc/en-us/articles/200171946-Error-523-Origin-is-unreachable + HTTP_UNREACHABLE = "523" + // https://support.cloudflare.com/hc/en-us/articles/200171926-Error-524-A-timeout-occurred + HTTP_TIMEOUT = "524" +) + +var errorPayloadTemplate = "HTTP/1.1 202 Accepted\r\nDate: Mon, 17 Aug 2015 14:10:11 GMT\r\nContent-Length: 0\r\nContent-Type: text/plain; charset=utf-8\r\n\r\n" + +func errorPayload(errorCode string) []byte { + payload := make([]byte, len(errorPayloadTemplate)) + copy(payload, errorPayloadTemplate) + + copy(payload[29:58], []byte(time.Now().Format(time.RFC1123))) + copy(payload[9:12], errorCode) + + return payload +} diff --git a/http_client_test.go b/http_client_test.go index 6109a90..3f13be2 100644 --- a/http_client_test.go +++ b/http_client_test.go @@ -8,7 +8,11 @@ import ( "net/http" "net/http/httptest" "net/http/httputil" + "github.com/buger/gor/proto" "sync" + "time" + "log" + _ "reflect" "testing" _ "time" ) @@ -299,3 +303,90 @@ func TestHTTPClientHandleHTTP10(t *testing.T) { wg.Wait() } + +func TestHTTPClientErrors(t *testing.T) { + req := []byte("GET http://foobar.com/path HTTP/1.0\r\n\r\n") + + // Port not exists + client := NewHTTPClient("http://127.0.0.1:1", &HTTPClientConfig{Debug: true}) + if resp, err := client.Send(req); err != nil { + if s := proto.Status(resp); !bytes.Equal(s, []byte("521")) { + t.Error("Should return status 521 for connection refused, instead:", string(s)) + } + } else { + t.Error("Should throw error") + } + + client = NewHTTPClient("http://not.existing", &HTTPClientConfig{Debug: true}) + if resp, err := client.Send(req); err != nil { + if s := proto.Status(resp); !bytes.Equal(s, []byte("521")) { + t.Error("Should return status 521 for no such host, instead:", string(s)) + } + } else { + t.Error("Should throw error") + } + + // Non routable IP address to simulate connection timeout + client = NewHTTPClient("http://10.255.255.1", &HTTPClientConfig{Debug: true, ConnectionTimeout: 100 * time.Millisecond }) + + if resp, err := client.Send(req); err != nil { + if s := proto.Status(resp); !bytes.Equal(s, []byte("521")) { + t.Error("Should return status 521 for io/timeout:", string(s)) + } + } else { + t.Error("Should throw error") + } + + // Connecting but io timeout on read + ln, _ := net.Listen("tcp", ":0") + client = NewHTTPClient("http://" + ln.Addr().String(), &HTTPClientConfig{Debug: true, Timeout: 10 * time.Millisecond}) + + if resp, err := client.Send(req); err != nil { + if s := proto.Status(resp); !bytes.Equal(s, []byte("524")) { + t.Error("Should return status 524 for io read, instead:", string(s)) + } + } else { + t.Error("Should throw error") + } + + // Response read error read tcp [::1]:51128: connection reset by peer &{{0xc20802a000}} + ln1, _ := net.Listen("tcp", ":0") + go func(){ + ln1.Accept() + }() + + client = NewHTTPClient("http://" + ln1.Addr().String(), &HTTPClientConfig{Debug: true, Timeout: 10 * time.Millisecond}) + + if resp, err := client.Send(req); err != nil { + if s := proto.Status(resp); !bytes.Equal(s, []byte("524")) { + t.Error("Should return status 524 for connection reset by peer, instead:", string(s)) + } + } else { + t.Error("Should throw error") + } + + ln2, _ := net.Listen("tcp", ":0") + go func(){ + for { + buf := make([]byte, 64*1024) + conn, err := ln2.Accept() + + if err != nil { + log.Println("Error while Accept()", err) + continue + } + + conn.Read(buf) + } + }() + + client = NewHTTPClient("http://" + ln2.Addr().String(), &HTTPClientConfig{Debug: true, Timeout: 10 * time.Millisecond}) + + if resp, err := client.Send(req); err != nil { + if s := proto.Status(resp); !bytes.Equal(s, []byte("524")) { + t.Error("Should return status 524 for connection reset by peer, instead:", string(s)) + } + } else { + t.Error("Should throw error") + } +} From ea94dd0162ad0fa844a6a681fe51b125304c90db Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Tue, 18 Aug 2015 08:20:08 +0300 Subject: [PATCH 31/64] fmt changes --- http_client_test.go | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/http_client_test.go b/http_client_test.go index 3f13be2..d1fe315 100644 --- a/http_client_test.go +++ b/http_client_test.go @@ -3,17 +3,17 @@ package main import ( "bytes" "crypto/rand" + "github.com/buger/gor/proto" "io/ioutil" + "log" "net" "net/http" "net/http/httptest" "net/http/httputil" - "github.com/buger/gor/proto" - "sync" - "time" - "log" _ "reflect" + "sync" "testing" + "time" _ "time" ) @@ -327,7 +327,7 @@ func TestHTTPClientErrors(t *testing.T) { } // Non routable IP address to simulate connection timeout - client = NewHTTPClient("http://10.255.255.1", &HTTPClientConfig{Debug: true, ConnectionTimeout: 100 * time.Millisecond }) + client = NewHTTPClient("http://10.255.255.1", &HTTPClientConfig{Debug: true, ConnectionTimeout: 100 * time.Millisecond}) if resp, err := client.Send(req); err != nil { if s := proto.Status(resp); !bytes.Equal(s, []byte("521")) { @@ -339,7 +339,7 @@ func TestHTTPClientErrors(t *testing.T) { // Connecting but io timeout on read ln, _ := net.Listen("tcp", ":0") - client = NewHTTPClient("http://" + ln.Addr().String(), &HTTPClientConfig{Debug: true, Timeout: 10 * time.Millisecond}) + client = NewHTTPClient("http://"+ln.Addr().String(), &HTTPClientConfig{Debug: true, Timeout: 10 * time.Millisecond}) if resp, err := client.Send(req); err != nil { if s := proto.Status(resp); !bytes.Equal(s, []byte("524")) { @@ -351,11 +351,11 @@ func TestHTTPClientErrors(t *testing.T) { // Response read error read tcp [::1]:51128: connection reset by peer &{{0xc20802a000}} ln1, _ := net.Listen("tcp", ":0") - go func(){ + go func() { ln1.Accept() }() - client = NewHTTPClient("http://" + ln1.Addr().String(), &HTTPClientConfig{Debug: true, Timeout: 10 * time.Millisecond}) + client = NewHTTPClient("http://"+ln1.Addr().String(), &HTTPClientConfig{Debug: true, Timeout: 10 * time.Millisecond}) if resp, err := client.Send(req); err != nil { if s := proto.Status(resp); !bytes.Equal(s, []byte("524")) { @@ -366,21 +366,21 @@ func TestHTTPClientErrors(t *testing.T) { } ln2, _ := net.Listen("tcp", ":0") - go func(){ + go func() { for { buf := make([]byte, 64*1024) conn, err := ln2.Accept() if err != nil { log.Println("Error while Accept()", err) - continue - } + continue + } conn.Read(buf) } }() - client = NewHTTPClient("http://" + ln2.Addr().String(), &HTTPClientConfig{Debug: true, Timeout: 10 * time.Millisecond}) + client = NewHTTPClient("http://"+ln2.Addr().String(), &HTTPClientConfig{Debug: true, Timeout: 10 * time.Millisecond}) if resp, err := client.Send(req); err != nil { if s := proto.Status(resp); !bytes.Equal(s, []byte("524")) { From 3e763c43cf255b7d6da9620970ad70c587429d65 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Tue, 18 Aug 2015 14:32:52 +0300 Subject: [PATCH 32/64] Unify all input/output plugins --- README.md | 2 +- emitter.go | 21 +++++----- examples/middleware/echo_modifier.sh | 3 +- examples/middleware/token_modifier.go | 22 +++++----- input_raw.go | 29 +++++--------- input_raw_test.go | 58 ++++++++------------------- middleware.go | 5 +-- middleware_test.go | 4 +- output_file.go | 4 ++ output_http.go | 17 ++++---- output_http_test.go | 2 +- output_tcp.go | 4 ++ protocol.go | 37 +++++++++++++++++ raw_socket_listener/listener.go | 12 ++++-- raw_socket_listener/tcp_message.go | 7 +++- raw_socket_listener/tcp_packet.go | 7 +++- test_input.go | 8 +++- 17 files changed, 136 insertions(+), 106 deletions(-) diff --git a/README.md b/README.md index e341d98..11415f4 100644 --- a/README.md +++ b/README.md @@ -240,7 +240,7 @@ Next goes request id: unique among all requests (sha1 of time and Ack), but rema HTTP payload is unmodified HTTP requests/responses intercepted from network. You can read more about request format [here](http://www.jmarshall.com/easy/http/), [here](https://en.wikipedia.org/wiki/Hypertext_Transfer_Protocol) and [here](http://www.w3.org/Protocols/rfc2616/rfc2616.html). You can operate with payload as you want, add headers, change path, and etc. Basically you just editing a string, just ensure that it is RCF compliant. -At the end modified (or untouched) request should be emitted back to STDOUT, keeping original header, and hex-encoded. If you want to filter request, just not send it. Emitting responses back is optional, and does not affect anything at the moment. +At the end modified (or untouched) request should be emitted back to STDOUT, keeping original header, and hex-encoded. If you want to filter request, just not send it. Emitting responses back is required, even if you did not touch them. #### Advanced example Imagine that you have auth system that randomly generate access tokens, which used later for accessing secure content. Since there is no pre-defined token value, naive approach without middleware (or if middleware use only request payloads) will fail, because replayed server have own tokens, not synced with origin. To fix this, our middleware should take in account responses of replayed and origin server, store `originalToken -> replayedToken` aliases and rewrite all requests using this token to use replayed alias. See `examples/middleware/token_modifier.go` and `middleware_test.go#TestTokenMiddleware` as example of described scheme. diff --git a/emitter.go b/emitter.go index 0769295..40875bc 100644 --- a/emitter.go +++ b/emitter.go @@ -1,17 +1,11 @@ package main import ( - "crypto/rand" "io" "time" + "bytes" ) -func uuid() []byte { - b := make([]byte, 16) - rand.Read(b) - return b -} - // Start initialize loop for sending data from inputs to outputs func Start(stop chan int) { if Settings.middleware != "" { @@ -65,14 +59,21 @@ func CopyMulty(src io.Reader, writers ...io.Writer) (err error) { Debug("[EMITTER] input:", string(payload[0:_maxN])) } - if modifier != nil { - payload = modifier.Rewrite(payload) + if modifier != nil && isRequestPayload(payload) { + headSize := bytes.IndexByte(payload, '\n') + 1 + body := payload[headSize:] + originalBodyLen := len(body) + body = modifier.Rewrite(body) // If modifier tells to skip request - if len(payload) == 0 { + if len(body) == 0 { continue } + if originalBodyLen != len(body) { + payload = append(payload[:headSize], body...) + } + if Settings.debug { Debug("[EMITTER] Rewrittern input:", len(payload), "First 500 bytes:", string(payload[0:_maxN])) } diff --git a/examples/middleware/echo_modifier.sh b/examples/middleware/echo_modifier.sh index 96dc121..ec946ee 100755 --- a/examples/middleware/echo_modifier.sh +++ b/examples/middleware/echo_modifier.sh @@ -22,7 +22,6 @@ while read line; do case ${header:0:1} in "1") log "Request type: Request" - echo "$encoded" ;; "2") log "Request type: Original Response" @@ -33,6 +32,8 @@ while read line; do *) log "Unknown request type $header" esac + echo "$encoded" + log "===================================" log "Original data: $line" diff --git a/examples/middleware/token_modifier.go b/examples/middleware/token_modifier.go index 9da3fc8..021408d 100644 --- a/examples/middleware/token_modifier.go +++ b/examples/middleware/token_modifier.go @@ -80,21 +80,13 @@ func process(buf []byte) { payload = proto.SetPathParam(payload, []byte("token"), alias) // Copy modified payload to our buffer - copy(buf[headerSize:], payload) + buf = append(buf[:headerSize], payload...) } } } - // Re-compute length in case if payload was modified - bufLen := headerSize + len(payload) - // Encoding request and sending it back - dst := make([]byte, bufLen*2+1) - hex.Encode(dst, buf[:bufLen]) - dst[len(dst)-1] = '\n' - - os.Stdout.Write(dst) - - return + // Emitting data back + os.Stdout.Write(encode(buf)) case '2': // Original response if _, ok := originalTokens[reqID]; ok { // Token is inside response body @@ -113,6 +105,14 @@ func process(buf []byte) { } } +func encode(buf []byte) []byte { + dst := make([]byte, len(buf)*2+1) + hex.Encode(dst, buf) + dst[len(dst)-1] = '\n' + + return dst +} + func Debug(args ...interface{}) { fmt.Fprint(os.Stderr, "[DEBUG][TOKEN-MOD] ") fmt.Fprintln(os.Stderr, args...) diff --git a/input_raw.go b/input_raw.go index 75a37ec..3bf882b 100644 --- a/input_raw.go +++ b/input_raw.go @@ -13,16 +13,14 @@ type RAWInput struct { data chan *raw.TCPMessage address string expire time.Duration - captureResponse bool } // NewRAWInput constructor for RAWInput. Accepts address with port as argument. -func NewRAWInput(address string, expire time.Duration, captureResponse bool) (i *RAWInput) { +func NewRAWInput(address string, expire time.Duration) (i *RAWInput) { i = new(RAWInput) i.data = make(chan *raw.TCPMessage) i.address = address i.expire = expire - i.captureResponse = captureResponse go i.listen(address) @@ -33,23 +31,18 @@ func (i *RAWInput) Read(data []byte) (int, error) { msg := <-i.data buf := msg.Bytes() - if i.captureResponse { - var header []byte + var header []byte - if msg.IsIncoming { - header = payloadHeader(RequestPayload, msg.UUID(), msg.Start) - } else { - header = payloadHeader(ResponsePayload, msg.UUID(), msg.End-msg.RequestStart) - } - - copy(data[0:len(header)], header) - copy(data[len(header):], buf) - - return len(buf) + len(header), nil + if msg.IsIncoming { + header = payloadHeader(RequestPayload, msg.UUID(), msg.Start) } else { - copy(data, buf) - return len(buf), nil + header = payloadHeader(ResponsePayload, msg.UUID(), msg.End-msg.RequestStart) } + + copy(data[0:len(header)], header) + copy(data[len(header):], buf) + + return len(buf) + len(header), nil } func (i *RAWInput) listen(address string) { @@ -63,7 +56,7 @@ func (i *RAWInput) listen(address string) { log.Fatal("input-raw: error while parsing address", err) } - listener := raw.NewListener(host, port, i.expire, i.captureResponse) + listener := raw.NewListener(host, port, i.expire, true) for { // Receiving TCPMessage object diff --git a/input_raw_test.go b/input_raw_test.go index 3273035..1f5abe9 100644 --- a/input_raw_test.go +++ b/input_raw_test.go @@ -24,7 +24,7 @@ func TestRAWInput(t *testing.T) { listener := startHTTP(func(w http.ResponseWriter, req *http.Request) {}) - input := NewRAWInput(listener.Addr().String(), testRawExpire, false) + input := NewRAWInput(listener.Addr().String(), testRawExpire) output := NewTestOutput(func(data []byte) { wg.Done() }) @@ -41,7 +41,8 @@ func TestRAWInput(t *testing.T) { go Start(quit) for i := 0; i < 100; i++ { - wg.Add(1) + // request + response + wg.Add(2) client.Get("/") } @@ -66,14 +67,19 @@ func TestInputRAW100Expect(t *testing.T) { originAddr := strings.Replace(origin.Addr().String(), "[::]", "127.0.0.1", -1) - input := NewRAWInput(originAddr, testRawExpire, false) + input := NewRAWInput(originAddr, testRawExpire) // We will use it to get content of raw HTTP request testOutput := NewTestOutput(func(data []byte) { - if strings.Contains(string(data), "Expect: 100-continue") { - t.Error("Should not contain 100-continue header") + switch data[0] { + case '1': + if strings.Contains(string(data), "Expect: 100-continue") { + t.Error("Should not contain 100-continue header") + } + wg.Done() + case '2': + wg.Done() } - wg.Done() }) listener := startHTTP(func(w http.ResponseWriter, req *http.Request) { @@ -96,7 +102,8 @@ func TestInputRAW100Expect(t *testing.T) { go Start(quit) - wg.Add(3) + // Origin + Response/Request Test Output + Request Http Output + wg.Add(4) curl := exec.Command("curl", "http://"+originAddr, "--data-binary", "@README.md") err := curl.Run() if err != nil { @@ -123,7 +130,7 @@ func TestInputRAWChunkedEncoding(t *testing.T) { originAddr := strings.Replace(origin.Addr().String(), "[::]", "127.0.0.1", -1) - input := NewRAWInput(originAddr, testRawExpire, false) + input := NewRAWInput(originAddr, testRawExpire) listener := startHTTP(func(w http.ResponseWriter, req *http.Request) { defer req.Body.Close() @@ -138,7 +145,7 @@ func TestInputRAWChunkedEncoding(t *testing.T) { }) replayAddr := listener.Addr().String() - httpOutput := NewHTTPOutput(replayAddr, &HTTPOutputConfig{Debug: false}) + httpOutput := NewHTTPOutput(replayAddr, &HTTPOutputConfig{Debug: true}) Plugins.Inputs = []io.Reader{input} Plugins.Outputs = []io.Writer{httpOutput} @@ -181,7 +188,7 @@ func TestInputRAWLargePayload(t *testing.T) { })) originAddr := strings.Replace(origin.Listener.Addr().String(), "[::]", "127.0.0.1", -1) - input := NewRAWInput(originAddr, testRawExpire, false) + input := NewRAWInput(originAddr, testRawExpire) replay := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { req.Body = http.MaxBytesReader(w, req.Body, 1*1024*1024) @@ -214,34 +221,3 @@ func TestInputRAWLargePayload(t *testing.T) { wg.Wait() close(quit) } - -func TestInputRAWResponse(t *testing.T) { - wg := new(sync.WaitGroup) - quit := make(chan int) - - listener := startHTTP(func(w http.ResponseWriter, req *http.Request) {}) - - input := NewRAWInput(listener.Addr().String(), testRawExpire, true) - output := NewTestOutput(func(data []byte) { - wg.Done() - }) - - Plugins.Inputs = []io.Reader{input} - Plugins.Outputs = []io.Writer{output} - - address := strings.Replace(listener.Addr().String(), "[::]", "127.0.0.1", -1) - - client := NewHTTPClient(address, &HTTPClientConfig{}) - - time.Sleep(time.Millisecond) - go Start(quit) - - for i := 0; i < 100; i++ { - // 2 because we track both request and response - wg.Add(2) - client.Get("/") - } - - wg.Wait() - close(quit) -} diff --git a/middleware.go b/middleware.go index e88be4d..8655e85 100644 --- a/middleware.go +++ b/middleware.go @@ -94,10 +94,7 @@ func (m *Middleware) read(from io.Reader) { Debug("[MIDDLEWARE-MASTER] Received:", string(buf)) } - // We should accept only request payloads - if buf[0] == '1' { - m.data <- buf - } + m.data <- buf } if err := scanner.Err(); err != nil { diff --git a/middleware_test.go b/middleware_test.go index 05bcb4a..bf8203c 100644 --- a/middleware_test.go +++ b/middleware_test.go @@ -110,7 +110,7 @@ func TestEchoMiddleware(t *testing.T) { Settings.middleware = "./examples/middleware/echo_modifier.sh" // Catch traffic from one service - input := NewRAWInput(from.Listener.Addr().String(), testRawExpire, true) + input := NewRAWInput(from.Listener.Addr().String(), testRawExpire) // And redirect to another output := NewHTTPOutput(to.URL, &HTTPOutputConfig{Debug: false}) @@ -165,7 +165,7 @@ func TestTokenMiddleware(t *testing.T) { quit := make(chan int) // Catch traffic from one service - input := NewRAWInput(from, testRawExpire, true) + input := NewRAWInput(from, testRawExpire) // And redirect to another output := NewHTTPOutput(to, &HTTPOutputConfig{Debug: true}) diff --git a/output_file.go b/output_file.go index f6a81ae..fab2210 100644 --- a/output_file.go +++ b/output_file.go @@ -43,6 +43,10 @@ func (o *FileOutput) init(path string) { } func (o *FileOutput) Write(data []byte) (n int, err error) { + if !isOriginPayload(data) { + return len(data), nil + } + raw := RawRequest{time.Now().UnixNano(), data} o.encoder.Encode(raw) diff --git a/output_http.go b/output_http.go index a44c7fd..cc75192 100644 --- a/output_http.go +++ b/output_http.go @@ -1,7 +1,6 @@ package main import ( - "bytes" "io" "log" "sync/atomic" @@ -140,6 +139,10 @@ func (o *HTTPOutput) startWorker() { } func (o *HTTPOutput) Write(data []byte) (n int, err error) { + if !isRequestPayload(data) { + return len(data), nil + } + buf := make([]byte, len(data)) copy(buf, data) @@ -173,17 +176,11 @@ func (o *HTTPOutput) Read(data []byte) (int, error) { } func (o *HTTPOutput) sendRequest(client *HTTPClient, request []byte) { - var uuid []byte - - if len(Settings.middleware) > 0 { - headerSize := bytes.IndexByte(request, '\n') - meta := bytes.Split(request[:headerSize], []byte{' '}) - uuid = meta[1] - request = request[headerSize+1:] - } + meta := payloadMeta(request) + uuid := meta[1] start := time.Now() - resp, err := client.Send(request) + resp, err := client.Send(payloadBody(request)) stop := time.Now() if err != nil { diff --git a/output_http_test.go b/output_http_test.go index a428c38..f4a4e6c 100644 --- a/output_http_test.go +++ b/output_http_test.go @@ -55,7 +55,7 @@ func TestHTTPOutput(t *testing.T) { methods := HTTPMethods{[]byte("GET"), []byte("PUT"), []byte("POST")} Settings.modifierConfig = HTTPModifierConfig{headers: headers, methods: methods} - output := NewHTTPOutput(listener.Addr().String(), &HTTPOutputConfig{}) + output := NewHTTPOutput(listener.Addr().String(), &HTTPOutputConfig{Debug: true}) Plugins.Inputs = []io.Reader{input} Plugins.Outputs = []io.Writer{output} diff --git a/output_tcp.go b/output_tcp.go index 2656a25..fdbe579 100644 --- a/output_tcp.go +++ b/output_tcp.go @@ -57,6 +57,10 @@ func (o *TCPOutput) worker() { } func (o *TCPOutput) Write(data []byte) (n int, err error) { + if !isOriginPayload(data) { + return len(data), nil + } + // Hex encoding always 2x number of bytes encoded := make([]byte, len(data)*2+1) hex.Encode(encoded, data) diff --git a/protocol.go b/protocol.go index a92ac80..9b8cc1a 100644 --- a/protocol.go +++ b/protocol.go @@ -2,6 +2,9 @@ package main import ( "strconv" + "bytes" + "encoding/hex" + "crypto/rand" ) const ( @@ -10,6 +13,17 @@ const ( ReplayedResponsePayload ) +func uuid() []byte { + b := make([]byte, 20) + rand.Read(b) + + uuid := make([]byte, 40) + hex.Encode(uuid, b) + + return uuid +} + + // Timing is request start or round-trip time, depending on payloadType func payloadHeader(payloadType int, uuid []byte, timing int64) (header []byte) { sTime := strconv.FormatInt(timing, 10) @@ -36,3 +50,26 @@ func payloadHeader(payloadType int, uuid []byte, timing int64) (header []byte) { return header } + +func payloadBody(payload []byte) []byte { + headerSize := bytes.IndexByte(payload, '\n') + return payload[headerSize+1:] +} + +func payloadMeta(payload []byte) [][]byte { + headerSize := bytes.IndexByte(payload, '\n') + return bytes.Split(payload[:headerSize], []byte{' '}) +} + +func isOriginPayload(payload []byte) bool { + switch payload[0] { + case '1', '2': + return true + default: + return false + } +} + +func isRequestPayload(payload []byte) bool { + return payload[0] == '1' +} \ No newline at end of file diff --git a/raw_socket_listener/listener.go b/raw_socket_listener/listener.go index eee05f4..eefb3e0 100644 --- a/raw_socket_listener/listener.go +++ b/raw_socket_listener/listener.go @@ -91,10 +91,15 @@ func (t *Listener) listen() { select { // If message ready for deletion it means that its also complete or expired by timeout case message := <-t.messageDelChan: - t.messagesChan <- message delete(t.ackAliases, message.Ack) delete(t.messages, message.ID) + if !message.IsIncoming { + delete(t.respAliases, message.Ack) + } + + t.messagesChan <- message + // We need to use channels to process each packet to avoid data races case packet := <-t.packetsChan: t.processTCPPacket(packet) @@ -216,11 +221,12 @@ func (t *Listener) processTCPPacket(packet *TCPPacket) { if t.captureResponse && isIncoming { // If message have multiple packets, delete previous alias if len(message.packets) > 0 { - delete(t.respAliases, uint32(message.Size())) + delete(t.respAliases, message.ResponseAck) } - responseAck := packet.Seq + uint32(message.Size()+len(packet.Data)) + responseAck := packet.Seq + uint32(len(packet.Data)) t.respAliases[responseAck] = &request{message.Start, message.Ack} + message.ResponseAck = responseAck } // Adding packet to message diff --git a/raw_socket_listener/tcp_message.go b/raw_socket_listener/tcp_message.go index 956f4ec..a0b8f88 100644 --- a/raw_socket_listener/tcp_message.go +++ b/raw_socket_listener/tcp_message.go @@ -20,6 +20,7 @@ import ( type TCPMessage struct { ID string // Message ID Ack uint32 + ResponseAck uint32 RequestStart int64 RequestAck uint32 Start int64 @@ -80,7 +81,11 @@ func (t *TCPMessage) Timeout() { } default: close(t.packetsChan) - t.delChan <- t // Notify RAWListener that message is ready to be send to replay server + // Notify RAWListener that message is ready to be send to replay server + // Responses without requests gets discarded + if t.IsIncoming || t.RequestStart != 0 { + t.delChan <- t + } } } diff --git a/raw_socket_listener/tcp_packet.go b/raw_socket_listener/tcp_packet.go index 998eeb4..e1ef358 100644 --- a/raw_socket_listener/tcp_packet.go +++ b/raw_socket_listener/tcp_packet.go @@ -69,6 +69,11 @@ func (t *TCPPacket) ParseBasic() { // String output for a TCP Packet func (t *TCPPacket) String() string { + maxLen := len(t.Data) + if maxLen > 500 { + maxLen = 500 + } + return strings.Join([]string{ "Source port: " + strconv.Itoa(int(t.SrcPort)), "Dest port:" + strconv.Itoa(int(t.DestPort)), @@ -90,7 +95,7 @@ func (t *TCPPacket) String() string { "Checksum:" + strconv.Itoa(int(t.Checksum)), "Data size:" + strconv.Itoa(len(t.Data)), - "Data:" + string(t.Data), + "Data:" + string(t.Data[:maxLen]), }, "\n") } diff --git a/test_input.go b/test_input.go index a9a359a..c699dff 100644 --- a/test_input.go +++ b/test_input.go @@ -3,6 +3,7 @@ package main import ( "crypto/rand" "encoding/base64" + "time" ) // TestInput used for testing purpose, it allows emitting requests on demand @@ -20,9 +21,12 @@ func NewTestInput() (i *TestInput) { func (i *TestInput) Read(data []byte) (int, error) { buf := <-i.data - copy(data, buf) - return len(buf), nil + header := payloadHeader(RequestPayload, uuid(), time.Now().UnixNano()) + copy(data[0:len(header)], header) + copy(data[len(header):], buf) + + return len(buf) + len(header), nil } // EmitGET emits GET request without headers From a971d8a8430557c595c731a2452ad93a96ed3860 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Tue, 18 Aug 2015 15:48:42 +0300 Subject: [PATCH 33/64] Change file format to be text based --- Makefile | 2 +- input_dummy.go | 7 +++++-- input_file.go | 50 ++++++++++++++++++++++++++++++++++---------------- output_file.go | 18 ++++-------------- 4 files changed, 44 insertions(+), 33 deletions(-) diff --git a/Makefile b/Makefile index 11ce2c6..c051a84 100644 --- a/Makefile +++ b/Makefile @@ -40,7 +40,7 @@ drun: docker run -v `pwd`:$(SOURCE_PATH) -t -i gor go run $(SOURCE) --input-dummy=0 --output-http="http://localhost:9000" --input-http :9000 --verbose drecord: - docker run -v `pwd`:$(SOURCE_PATH) -t -i gor go run $(SOURCE) --input-dummy=0 --output-file=requests.bin --verbose + docker run -v `pwd`:$(SOURCE_PATH) -t -i gor go run $(SOURCE) --input-dummy=0 --output-file=requests.gor --verbose dreplay: docker run -v `pwd`:$(SOURCE_PATH) -t -i gor go run $(SOURCE) --input-file=requests.bin --output-tcp=:9000 --verbose -h diff --git a/input_dummy.go b/input_dummy.go index f439179..4c14830 100644 --- a/input_dummy.go +++ b/input_dummy.go @@ -21,9 +21,12 @@ func NewDummyInput(options string) (di *DummyInput) { func (i *DummyInput) Read(data []byte) (int, error) { buf := <-i.data - copy(data, buf) - return len(buf), nil + header := payloadHeader(RequestPayload, uuid(), time.Now().UnixNano()) + copy(data[0:len(header)], header) + copy(data[len(header):], buf) + + return len(buf) + len(header), nil } func (i *DummyInput) emit() { diff --git a/input_file.go b/input_file.go index 60c93ec..0a38428 100644 --- a/input_file.go +++ b/input_file.go @@ -1,9 +1,11 @@ package main import ( - "encoding/gob" "log" "os" + "bufio" + "bytes" + "strconv" "time" ) @@ -11,7 +13,7 @@ import ( type FileInput struct { data chan []byte path string - decoder *gob.Decoder + file *os.File speedFactor float64 } @@ -35,7 +37,7 @@ func (i *FileInput) init(path string) { log.Fatal(i, "Cannot open file %q. Error: %s", path, err) } - i.decoder = gob.NewDecoder(file) + i.file = file } func (i *FileInput) Read(data []byte) (int, error) { @@ -49,30 +51,46 @@ func (i *FileInput) String() string { return "File input: " + i.path } +func scanSeparator(data []byte, atEOF bool) (advance int, token []byte, err error) { + if atEOF && len(data) == 0 { + return 0, nil, nil + } + + if i := bytes.Index(data, []byte(fileSeparator)); i >= 0 { + // We have a full newline-terminated line. + return i + len(fileSeparator), data[0:i], nil + } + + if atEOF { + return len(data), data, nil + } + return 0, nil, nil +} + func (i *FileInput) emit() { var lastTime int64 - for { - raw := new(RawRequest) - err := i.decoder.Decode(raw) + // reader := bufio.NewReader(conn) + scanner := bufio.NewScanner(i.file) + scanner.Split(scanSeparator) - if err != nil { - return - } + for scanner.Scan() { + buf := scanner.Bytes() + meta := payloadMeta(buf) - if lastTime != 0 { - timeDiff := raw.Timestamp - lastTime + if meta[0][0] == '1' && lastTime != 0 { + ts, _ := strconv.ParseInt(string(meta[2]), 10, 64) + timeDiff := ts - lastTime - // We can speedup or slowdown execution based on speedFactor if i.speedFactor != 1 { - timeDiff = int64(float64(raw.Timestamp-lastTime) / i.speedFactor) + timeDiff = int64(float64(timeDiff) / i.speedFactor) } time.Sleep(time.Duration(timeDiff)) + + lastTime = ts } - lastTime = raw.Timestamp - - i.data <- raw.Request + i.data <- buf } } diff --git a/output_file.go b/output_file.go index fab2210..d01f05f 100644 --- a/output_file.go +++ b/output_file.go @@ -1,23 +1,14 @@ package main import ( - "encoding/gob" "io" "log" "os" - "time" ) -// RawRequest stores original start time and request payload -type RawRequest struct { - Timestamp int64 - Request []byte -} - // FileOutput output plugin type FileOutput struct { path string - encoder *gob.Encoder file *os.File } @@ -38,18 +29,17 @@ func (o *FileOutput) init(path string) { if err != nil { log.Fatal(o, "Cannot open file %q. Error: %s", path, err) } - - o.encoder = gob.NewEncoder(o.file) } +var fileSeparator = "\n🐵🙈🙉\n" + func (o *FileOutput) Write(data []byte) (n int, err error) { if !isOriginPayload(data) { return len(data), nil } - raw := RawRequest{time.Now().UnixNano(), data} - - o.encoder.Encode(raw) + o.file.Write(data) + o.file.Write([]byte(fileSeparator)) return len(data), nil } From 7790b33ceabedf2ffa1473699ab4f61e46178bbe Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Tue, 18 Aug 2015 15:52:23 +0300 Subject: [PATCH 34/64] Refactor protocol scanner --- input_file.go | 19 +------------------ output_file.go | 4 +--- protocol.go | 18 ++++++++++++++++++ 3 files changed, 20 insertions(+), 21 deletions(-) diff --git a/input_file.go b/input_file.go index 0a38428..1fdf0f1 100644 --- a/input_file.go +++ b/input_file.go @@ -4,7 +4,6 @@ import ( "log" "os" "bufio" - "bytes" "strconv" "time" ) @@ -51,28 +50,12 @@ func (i *FileInput) String() string { return "File input: " + i.path } -func scanSeparator(data []byte, atEOF bool) (advance int, token []byte, err error) { - if atEOF && len(data) == 0 { - return 0, nil, nil - } - - if i := bytes.Index(data, []byte(fileSeparator)); i >= 0 { - // We have a full newline-terminated line. - return i + len(fileSeparator), data[0:i], nil - } - - if atEOF { - return len(data), data, nil - } - return 0, nil, nil -} - func (i *FileInput) emit() { var lastTime int64 // reader := bufio.NewReader(conn) scanner := bufio.NewScanner(i.file) - scanner.Split(scanSeparator) + scanner.Split(payloadScanner) for scanner.Scan() { buf := scanner.Bytes() diff --git a/output_file.go b/output_file.go index d01f05f..e8b0d86 100644 --- a/output_file.go +++ b/output_file.go @@ -31,15 +31,13 @@ func (o *FileOutput) init(path string) { } } -var fileSeparator = "\n🐵🙈🙉\n" - func (o *FileOutput) Write(data []byte) (n int, err error) { if !isOriginPayload(data) { return len(data), nil } o.file.Write(data) - o.file.Write([]byte(fileSeparator)) + o.file.Write([]byte(payloadSeparator)) return len(data), nil } diff --git a/protocol.go b/protocol.go index 9b8cc1a..a55f6b3 100644 --- a/protocol.go +++ b/protocol.go @@ -23,6 +23,24 @@ func uuid() []byte { return uuid } +var payloadSeparator = "\n🐵🙈🙉\n" + +func payloadScanner(data []byte, atEOF bool) (advance int, token []byte, err error) { + if atEOF && len(data) == 0 { + return 0, nil, nil + } + + if i := bytes.Index(data, []byte(payloadSeparator)); i >= 0 { + // We have a full newline-terminated line. + return i + len([]byte(payloadSeparator)), data[0:i], nil + } + + if atEOF { + return len(data), data, nil + } + return 0, nil, nil +} + // Timing is request start or round-trip time, depending on payloadType func payloadHeader(payloadType int, uuid []byte, timing int64) (header []byte) { From 8da4e54cbb7f61d170f3a1ee41a20d89c8c75f78 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Tue, 18 Aug 2015 16:02:03 +0300 Subject: [PATCH 35/64] Tcp communication should use same protocol as file based --- input_tcp.go | 12 ++---------- input_tcp_test.go | 9 +++------ output_tcp.go | 11 ++++------- output_tcp_test.go | 8 ++------ 4 files changed, 11 insertions(+), 29 deletions(-) diff --git a/input_tcp.go b/input_tcp.go index 47d2c3d..15dffa5 100644 --- a/input_tcp.go +++ b/input_tcp.go @@ -2,7 +2,6 @@ package main import ( "bufio" - "encoding/hex" "fmt" "log" "net" @@ -10,7 +9,6 @@ import ( ) // TCPInput used for internal communication -// It expected hex encoded data type TCPInput struct { data chan []byte address string @@ -62,16 +60,10 @@ func (i *TCPInput) handleConnection(conn net.Conn) { reader := bufio.NewReader(conn) scanner := bufio.NewScanner(reader) + scanner.Split(payloadScanner) for scanner.Scan() { - encodedPayload := scanner.Bytes() - // Hex encoding always 2x number of bytes - decoded := make([]byte, len(encodedPayload)/2) - _, err := hex.Decode(decoded, encodedPayload) - if err != nil { - log.Println("[TCPInput] failed to hex decode TCP payload:", err) - } - i.data <- decoded + i.data <- scanner.Bytes() } if err := scanner.Err(); err != nil { diff --git a/input_tcp_test.go b/input_tcp_test.go index f440c57..d844023 100644 --- a/input_tcp_test.go +++ b/input_tcp_test.go @@ -1,7 +1,6 @@ package main import ( - "encoding/hex" "io" "log" "net" @@ -35,14 +34,12 @@ func TestTCPInput(t *testing.T) { log.Fatal(err) } - msg := []byte("GET / HTTP/1.1\r\n\r\n") + msg := []byte("1 1 1\nGET / HTTP/1.1\r\n\r\n") for i := 0; i < 100; i++ { wg.Add(1) - - encoded := make([]byte, len(msg)*2) - hex.Encode(encoded, msg) - conn.Write(append(encoded, '\n')) + conn.Write(msg) + conn.Write([]byte(payloadSeparator)) } wg.Wait() diff --git a/output_tcp.go b/output_tcp.go index fdbe579..96295dc 100644 --- a/output_tcp.go +++ b/output_tcp.go @@ -1,7 +1,6 @@ package main import ( - "encoding/hex" "fmt" "io" "log" @@ -47,7 +46,9 @@ func (o *TCPOutput) worker() { defer conn.Close() for { - _, err := conn.Write(<-o.buf) + conn.Write(<-o.buf) + _, err := conn.Write([]byte(payloadSeparator)) + if err != nil { log.Println("Worker failed on write, exitings and starting new worker") go o.worker() @@ -61,11 +62,7 @@ func (o *TCPOutput) Write(data []byte) (n int, err error) { return len(data), nil } - // Hex encoding always 2x number of bytes - encoded := make([]byte, len(data)*2+1) - hex.Encode(encoded, data) - encoded[len(encoded)-1] = '\n' - o.buf <- encoded + o.buf <- data if Settings.outputTCPStats { o.bufStats.Write(len(o.buf)) diff --git a/output_tcp_test.go b/output_tcp_test.go index 84c66b8..b295d7a 100644 --- a/output_tcp_test.go +++ b/output_tcp_test.go @@ -2,7 +2,6 @@ package main import ( "bufio" - "encoding/hex" "io" "log" "net" @@ -50,13 +49,10 @@ func startTCP(cb func([]byte)) net.Listener { go func() { reader := bufio.NewReader(conn) scanner := bufio.NewScanner(reader) + scanner.Split(payloadScanner) for scanner.Scan() { - encodedPayload := scanner.Bytes() - // Hex encoding always 2x number of bytes - decoded := make([]byte, len(encodedPayload)/2) - hex.Decode(decoded, encodedPayload) - cb(decoded) + cb(scanner.Bytes()) } }() } From ae35f88004eefa9525a14ef252290571f9eb67cf Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Tue, 18 Aug 2015 17:18:02 +0300 Subject: [PATCH 36/64] Remove magic numbers --- input_file.go | 2 +- input_raw_test.go | 4 ++-- protocol.go | 22 +++++++--------------- 3 files changed, 10 insertions(+), 18 deletions(-) diff --git a/input_file.go b/input_file.go index 1fdf0f1..f1a057b 100644 --- a/input_file.go +++ b/input_file.go @@ -61,7 +61,7 @@ func (i *FileInput) emit() { buf := scanner.Bytes() meta := payloadMeta(buf) - if meta[0][0] == '1' && lastTime != 0 { + if meta[0][0] == RequestPayload && lastTime != 0 { ts, _ := strconv.ParseInt(string(meta[2]), 10, 64) timeDiff := ts - lastTime diff --git a/input_raw_test.go b/input_raw_test.go index 1f5abe9..0f30afb 100644 --- a/input_raw_test.go +++ b/input_raw_test.go @@ -72,12 +72,12 @@ func TestInputRAW100Expect(t *testing.T) { // We will use it to get content of raw HTTP request testOutput := NewTestOutput(func(data []byte) { switch data[0] { - case '1': + case RequestPayload: if strings.Contains(string(data), "Expect: 100-continue") { t.Error("Should not contain 100-continue header") } wg.Done() - case '2': + case ResponsePayload: wg.Done() } }) diff --git a/protocol.go b/protocol.go index a55f6b3..5d97eb7 100644 --- a/protocol.go +++ b/protocol.go @@ -8,9 +8,9 @@ import ( ) const ( - RequestPayload = 1 << iota - ResponsePayload - ReplayedResponsePayload + RequestPayload = '1' + ResponsePayload = '2' + ReplayedResponsePayload = '3' ) func uuid() []byte { @@ -43,26 +43,18 @@ func payloadScanner(data []byte, atEOF bool) (advance int, token []byte, err err // Timing is request start or round-trip time, depending on payloadType -func payloadHeader(payloadType int, uuid []byte, timing int64) (header []byte) { +func payloadHeader(payloadType byte, uuid []byte, timing int64) (header []byte) { sTime := strconv.FormatInt(timing, 10) //Example: // 3 f45590522cd1838b4a0d5c5aab80b77929dea3b3 1231\n // `+ 1` indicates space characters or end of line header = make([]byte, 1+1+len(uuid)+1+len(sTime)+1) + header[0] = payloadType header[1] = ' ' header[2+len(uuid)] = ' ' header[len(header)-1] = '\n' - switch payloadType { - case RequestPayload: - header[0] = '1' - case ResponsePayload: - header[0] = '2' - case ReplayedResponsePayload: - header[0] = '3' - } - copy(header[2:], uuid) copy(header[3+len(uuid):], sTime) @@ -81,7 +73,7 @@ func payloadMeta(payload []byte) [][]byte { func isOriginPayload(payload []byte) bool { switch payload[0] { - case '1', '2': + case RequestPayload, ResponsePayload: return true default: return false @@ -89,5 +81,5 @@ func isOriginPayload(payload []byte) bool { } func isRequestPayload(payload []byte) bool { - return payload[0] == '1' + return payload[0] == RequestPayload } \ No newline at end of file From d6c776fa0c06c86d6010c385d743d4c9131bbc6a Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Tue, 18 Aug 2015 17:27:26 +0300 Subject: [PATCH 37/64] Verbose output for travis --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index d491b84..63ecff5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,3 +1,3 @@ language: go go: 1.4.2 -script: sudo -E bash -c "source /etc/profile && eval '$(gimme 1.4.2)' && export GOPATH=$HOME/gopath:$GOPATH && go get && GORACE='halt_on_error=1' go test ./... -v -timeout 15s" \ No newline at end of file +script: sudo -E bash -c "source /etc/profile && eval '$(gimme 1.4.2)' && export GOPATH=$HOME/gopath:$GOPATH && go get && GORACE='halt_on_error=1' go test ./... -v -timeout 15s --verbose" \ No newline at end of file From b53fba041547399f2eacdda8582795cfec0c9487 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Tue, 18 Aug 2015 17:28:09 +0300 Subject: [PATCH 38/64] Fix formatting issues --- emitter.go | 2 +- input_file.go | 2 +- input_raw.go | 6 +++--- output_file.go | 4 ++-- protocol.go | 19 +++++++++---------- 5 files changed, 16 insertions(+), 17 deletions(-) diff --git a/emitter.go b/emitter.go index 40875bc..00ba070 100644 --- a/emitter.go +++ b/emitter.go @@ -1,9 +1,9 @@ package main import ( + "bytes" "io" "time" - "bytes" ) // Start initialize loop for sending data from inputs to outputs diff --git a/input_file.go b/input_file.go index f1a057b..840c5d7 100644 --- a/input_file.go +++ b/input_file.go @@ -1,9 +1,9 @@ package main import ( + "bufio" "log" "os" - "bufio" "strconv" "time" ) diff --git a/input_raw.go b/input_raw.go index 3bf882b..8c096c6 100644 --- a/input_raw.go +++ b/input_raw.go @@ -10,9 +10,9 @@ import ( // RAWInput used for intercepting traffic for given address type RAWInput struct { - data chan *raw.TCPMessage - address string - expire time.Duration + data chan *raw.TCPMessage + address string + expire time.Duration } // NewRAWInput constructor for RAWInput. Accepts address with port as argument. diff --git a/output_file.go b/output_file.go index e8b0d86..5ccc93c 100644 --- a/output_file.go +++ b/output_file.go @@ -8,8 +8,8 @@ import ( // FileOutput output plugin type FileOutput struct { - path string - file *os.File + path string + file *os.File } // NewFileOutput constructor for FileOutput, accepts path diff --git a/protocol.go b/protocol.go index 5d97eb7..1705f58 100644 --- a/protocol.go +++ b/protocol.go @@ -1,15 +1,15 @@ package main import ( - "strconv" "bytes" - "encoding/hex" "crypto/rand" + "encoding/hex" + "strconv" ) const ( - RequestPayload = '1' - ResponsePayload = '2' + RequestPayload = '1' + ResponsePayload = '2' ReplayedResponsePayload = '3' ) @@ -27,8 +27,8 @@ var payloadSeparator = "\n🐵🙈🙉\n" func payloadScanner(data []byte, atEOF bool) (advance int, token []byte, err error) { if atEOF && len(data) == 0 { - return 0, nil, nil - } + return 0, nil, nil + } if i := bytes.Index(data, []byte(payloadSeparator)); i >= 0 { // We have a full newline-terminated line. @@ -36,12 +36,11 @@ func payloadScanner(data []byte, atEOF bool) (advance int, token []byte, err err } if atEOF { - return len(data), data, nil + return len(data), data, nil } - return 0, nil, nil + return 0, nil, nil } - // Timing is request start or round-trip time, depending on payloadType func payloadHeader(payloadType byte, uuid []byte, timing int64) (header []byte) { sTime := strconv.FormatInt(timing, 10) @@ -82,4 +81,4 @@ func isOriginPayload(payload []byte) bool { func isRequestPayload(payload []byte) bool { return payload[0] == RequestPayload -} \ No newline at end of file +} From 7cff78174ae2fab06513bd64c85452cc87e5c06a Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Tue, 18 Aug 2015 17:35:54 +0300 Subject: [PATCH 39/64] Remove verbose flag --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 63ecff5..d491b84 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,3 +1,3 @@ language: go go: 1.4.2 -script: sudo -E bash -c "source /etc/profile && eval '$(gimme 1.4.2)' && export GOPATH=$HOME/gopath:$GOPATH && go get && GORACE='halt_on_error=1' go test ./... -v -timeout 15s --verbose" \ No newline at end of file +script: sudo -E bash -c "source /etc/profile && eval '$(gimme 1.4.2)' && export GOPATH=$HOME/gopath:$GOPATH && go get && GORACE='halt_on_error=1' go test ./... -v -timeout 15s" \ No newline at end of file From c4037e0c5727978a4ba27bc4186970a5d2897d2f Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Tue, 18 Aug 2015 17:42:55 +0300 Subject: [PATCH 40/64] Increase travis timeout time --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index d491b84..544d2ff 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,3 +1,3 @@ language: go go: 1.4.2 -script: sudo -E bash -c "source /etc/profile && eval '$(gimme 1.4.2)' && export GOPATH=$HOME/gopath:$GOPATH && go get && GORACE='halt_on_error=1' go test ./... -v -timeout 15s" \ No newline at end of file +script: sudo -E bash -c "source /etc/profile && eval '$(gimme 1.4.2)' && export GOPATH=$HOME/gopath:$GOPATH && go get && GORACE='halt_on_error=1' go test ./... -v -timeout 30s" \ No newline at end of file From 78456d6cd11e8422449c410fc3e801dd2ee464c4 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Tue, 18 Aug 2015 17:46:57 +0300 Subject: [PATCH 41/64] More timeout --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 544d2ff..4736e5b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,3 +1,3 @@ language: go go: 1.4.2 -script: sudo -E bash -c "source /etc/profile && eval '$(gimme 1.4.2)' && export GOPATH=$HOME/gopath:$GOPATH && go get && GORACE='halt_on_error=1' go test ./... -v -timeout 30s" \ No newline at end of file +script: sudo -E bash -c "source /etc/profile && eval '$(gimme 1.4.2)' && export GOPATH=$HOME/gopath:$GOPATH && go get && GORACE='halt_on_error=1' go test ./... -v -timeout 60s" \ No newline at end of file From a6e29c684cb0ae3cb4a58ee3d77c5184b1a22b90 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Tue, 18 Aug 2015 21:25:56 +0300 Subject: [PATCH 42/64] Close connection --- http_client_test.go | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/http_client_test.go b/http_client_test.go index d1fe315..2500c73 100644 --- a/http_client_test.go +++ b/http_client_test.go @@ -340,6 +340,7 @@ func TestHTTPClientErrors(t *testing.T) { // Connecting but io timeout on read ln, _ := net.Listen("tcp", ":0") client = NewHTTPClient("http://"+ln.Addr().String(), &HTTPClientConfig{Debug: true, Timeout: 10 * time.Millisecond}) + defer ln.Close() if resp, err := client.Send(req); err != nil { if s := proto.Status(resp); !bytes.Equal(s, []byte("524")) { @@ -354,6 +355,7 @@ func TestHTTPClientErrors(t *testing.T) { go func() { ln1.Accept() }() + defer ln1.Close() client = NewHTTPClient("http://"+ln1.Addr().String(), &HTTPClientConfig{Debug: true, Timeout: 10 * time.Millisecond}) @@ -367,18 +369,18 @@ func TestHTTPClientErrors(t *testing.T) { ln2, _ := net.Listen("tcp", ":0") go func() { - for { - buf := make([]byte, 64*1024) - conn, err := ln2.Accept() + buf := make([]byte, 64*1024) + conn, err := ln2.Accept() - if err != nil { - log.Println("Error while Accept()", err) - continue - } - - conn.Read(buf) + if err != nil { + log.Println("Error while Accept()", err) + continue } + + conn.Read(buf) + defer conn.Close() }() + defer ln2.Close() client = NewHTTPClient("http://"+ln2.Addr().String(), &HTTPClientConfig{Debug: true, Timeout: 10 * time.Millisecond}) From 5d2f6eceea399c1a196d07d49bf416fce02eda46 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Wed, 19 Aug 2015 09:19:00 +0300 Subject: [PATCH 43/64] Fixes --- Makefile | 2 +- http_client_test.go | 20 ++++++++++---------- middleware_test.go | 7 ++----- output_http.go | 9 ++++++++- raw_socket_listener/listener.go | 3 +++ raw_socket_listener/tcp_message.go | 20 +++++++++++++++----- 6 files changed, 39 insertions(+), 22 deletions(-) diff --git a/Makefile b/Makefile index c051a84..f2bba5c 100644 --- a/Makefile +++ b/Makefile @@ -20,7 +20,7 @@ drace: docker run -v `pwd`:$(SOURCE_PATH) -t -i --env GORACE="halt_on_error=1" gor go test ./... $(ARGS) -v -race -timeout 15s dtest: - docker run -v `pwd`:$(SOURCE_PATH) -t -i gor go test ./... $(ARGS) -v -timeout 10s + docker run -v `pwd`:$(SOURCE_PATH) -t -i gor go test ./... $(ARGS) -v -timeout 60s dcover: docker run -v `pwd`:$(SOURCE_PATH) -t -i --env GORACE="halt_on_error=1" gor go test $(ARGS) -race -v -timeout 15s -coverprofile=coverage.out diff --git a/http_client_test.go b/http_client_test.go index 2500c73..d970381 100644 --- a/http_client_test.go +++ b/http_client_test.go @@ -5,7 +5,7 @@ import ( "crypto/rand" "github.com/buger/gor/proto" "io/ioutil" - "log" + _ "log" "net" "net/http" "net/http/httptest" @@ -171,12 +171,16 @@ func TestHTTPClientServerInstantDisconnect(t *testing.T) { GETPayload := []byte("GET / HTTP/1.1\r\n\r\n") ln, _ := net.Listen("tcp", ":0") + defer ln.Close() go func() { for { - conn, _ := ln.Accept() - conn.Close() + conn, err := ln.Accept() + if err != nil { + break + } + conn.Close() wg.Done() } }() @@ -196,12 +200,13 @@ func TestHTTPClientServerNoKeepAlive(t *testing.T) { GETPayload := []byte("GET / HTTP/1.1\r\n\r\n") ln, _ := net.Listen("tcp", ":0") + defer ln.Close() go func() { for { conn, err := ln.Accept() if err != nil { - // handle error + break } buf := make([]byte, 4096) @@ -370,12 +375,7 @@ func TestHTTPClientErrors(t *testing.T) { ln2, _ := net.Listen("tcp", ":0") go func() { buf := make([]byte, 64*1024) - conn, err := ln2.Accept() - - if err != nil { - log.Println("Error while Accept()", err) - continue - } + conn, _ := ln2.Accept() conn.Read(buf) defer conn.Close() diff --git a/middleware_test.go b/middleware_test.go index bf8203c..aab936e 100644 --- a/middleware_test.go +++ b/middleware_test.go @@ -149,10 +149,6 @@ func TestTokenMiddleware(t *testing.T) { }) to := NewFakeSecureService(wg, func(path string, status int, tok []byte) { switch path { - case "/token": - if bytes.Equal(token, tok) { - t.Error("Tokens should not match") - } case "/secure": if status != 202 { t.Error("Server should receive valid rewritten token") @@ -164,6 +160,8 @@ func TestTokenMiddleware(t *testing.T) { quit := make(chan int) + Settings.middleware = "go run ./examples/middleware/token_modifier.go" + // Catch traffic from one service input := NewRAWInput(from, testRawExpire) @@ -172,7 +170,6 @@ func TestTokenMiddleware(t *testing.T) { Plugins.Inputs = []io.Reader{input} Plugins.Outputs = []io.Writer{output} - Settings.middleware = "go run ./examples/middleware/token_modifier.go" // Start Gor go Start(quit) diff --git a/output_http.go b/output_http.go index cc75192..0f57ea9 100644 --- a/output_http.go +++ b/output_http.go @@ -28,6 +28,8 @@ type HTTPOutputConfig struct { OriginalHost bool Debug bool + + TrackResponses bool } // HTTPOutput plugin manage pool of workers which send request to replayed server @@ -42,6 +44,7 @@ type HTTPOutput struct { address string limit int queue chan []byte + responses chan response needWorker chan int @@ -81,6 +84,10 @@ func NewHTTPOutput(address string, config *HTTPOutputConfig) io.Writer { o.elasticSearch.Init(o.config.elasticSearch) } + if len(Settings.middleware) > 0 { + o.config.TrackResponses = true + } + go o.workerMaster() return o @@ -187,7 +194,7 @@ func (o *HTTPOutput) sendRequest(client *HTTPClient, request []byte) { log.Println("Request error:", err) } - if len(Settings.middleware) > 0 { + if o.config.TrackResponses { o.responses <- response{resp, uuid, stop.UnixNano() - start.UnixNano()} } diff --git a/raw_socket_listener/listener.go b/raw_socket_listener/listener.go index eefb3e0..fc3de75 100644 --- a/raw_socket_listener/listener.go +++ b/raw_socket_listener/listener.go @@ -102,6 +102,7 @@ func (t *Listener) listen() { // We need to use channels to process each packet to avoid data races case packet := <-t.packetsChan: + // log.Println("Received packet:", packet) t.processTCPPacket(packet) } } @@ -219,10 +220,12 @@ func (t *Listener) processTCPPacket(packet *TCPPacket) { } if t.captureResponse && isIncoming { + message.mu.Lock() // If message have multiple packets, delete previous alias if len(message.packets) > 0 { delete(t.respAliases, message.ResponseAck) } + message.mu.Unlock() responseAck := packet.Seq + uint32(len(packet.Data)) t.respAliases[responseAck] = &request{message.Start, message.Ack} diff --git a/raw_socket_listener/tcp_message.go b/raw_socket_listener/tcp_message.go index a0b8f88..2ef8327 100644 --- a/raw_socket_listener/tcp_message.go +++ b/raw_socket_listener/tcp_message.go @@ -9,6 +9,7 @@ import ( "sort" "strconv" "time" + "sync" ) // TCPMessage ensure that all TCP packets for given request is received, and processed in right sequence @@ -36,6 +37,8 @@ type TCPMessage struct { delChan chan *TCPMessage expire *time.Duration + + mu sync.Mutex } // NewTCPMessage pointer created from a Acknowledgment number and a channel of messages readuy to be deleted @@ -44,6 +47,7 @@ func NewTCPMessage(ID string, delChan chan *TCPMessage, Ack uint32, expire *time msg.Start = time.Now().UnixNano() msg.packetsChan = make(chan *TCPPacket) msg.delChan = delChan // used for notifying that message completed or expired + msg.timer = time.NewTimer(0) go msg.listen() @@ -66,16 +70,13 @@ func (t *TCPMessage) listen() { // Timeout notifies message to stop listening, close channel and message ready to be sent func (t *TCPMessage) Timeout() { - if t.timer != nil { - t.timer.Stop() - } - select { // In some cases Timeout can be called multiple times (do not know how yet) // Ensure that we did not close channel 2 times case packet, ok := <-t.packetsChan: if ok { t.AddPacket(packet) + t.Timeout() } else { return } @@ -112,6 +113,9 @@ func (t *TCPMessage) Size() (size int) { // AddPacket to the message and ensure packet uniqueness // TCP allows that packet can be re-send multiple times func (t *TCPMessage) AddPacket(packet *TCPPacket) { + t.mu.Lock() + defer t.mu.Unlock() + packetFound := false for _, pkt := range t.packets { @@ -124,7 +128,13 @@ func (t *TCPMessage) AddPacket(packet *TCPPacket) { if packetFound { log.Println("Received packet with same sequence") } else { - t.packets = append(t.packets, packet) + // Packets not always captured in same Seq order, and sometimes we need to prepend + if len(t.packets) == 0 || packet.Seq > t.packets[len(t.packets)-1].Seq { + t.packets = append(t.packets, packet) + } else { + t.packets = append([]*TCPPacket{packet}, t.packets...) + } + t.End = time.Now().UnixNano() } From e54f77f22906275a94e4534391dc13eb3c22969b Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Wed, 19 Aug 2015 09:27:16 +0300 Subject: [PATCH 44/64] Try to return all responses --- raw_socket_listener/tcp_message.go | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/raw_socket_listener/tcp_message.go b/raw_socket_listener/tcp_message.go index 2ef8327..5c4cf53 100644 --- a/raw_socket_listener/tcp_message.go +++ b/raw_socket_listener/tcp_message.go @@ -6,7 +6,6 @@ import ( "encoding/hex" "github.com/buger/gor/proto" "log" - "sort" "strconv" "time" "sync" @@ -84,16 +83,14 @@ func (t *TCPMessage) Timeout() { close(t.packetsChan) // Notify RAWListener that message is ready to be send to replay server // Responses without requests gets discarded - if t.IsIncoming || t.RequestStart != 0 { + if t.IsIncoming { t.delChan <- t } } } -// Bytes sorts packets in right orders and return message content +// Bytes return message content func (t *TCPMessage) Bytes() (output []byte) { - sort.Sort(sortBySeq(t.packets)) - for _, p := range t.packets { output = append(output, p.Data...) } From 6cb4c265ddfa946cb7e159d855a0b0a1e16761f5 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Wed, 19 Aug 2015 21:46:45 +0300 Subject: [PATCH 45/64] Some debugging and race fixes --- input_raw_test.go | 12 +++++++++++- middleware_test.go | 3 +++ raw_socket_listener/tcp_message.go | 19 +++++++++++-------- 3 files changed, 25 insertions(+), 9 deletions(-) diff --git a/input_raw_test.go b/input_raw_test.go index 0f30afb..38126f1 100644 --- a/input_raw_test.go +++ b/input_raw_test.go @@ -11,6 +11,7 @@ import ( "os/exec" "strings" "sync" + "sync/atomic" "testing" "time" ) @@ -18,14 +19,23 @@ import ( const testRawExpire = time.Millisecond * 100 func TestRAWInput(t *testing.T) { - wg := new(sync.WaitGroup) quit := make(chan int) listener := startHTTP(func(w http.ResponseWriter, req *http.Request) {}) + var respCounter, reqCounter int64 + input := NewRAWInput(listener.Addr().String(), testRawExpire) output := NewTestOutput(func(data []byte) { + if data[0] == '1' { + atomic.AddInt64(&reqCounter, 1) + } else { + atomic.AddInt64(&respCounter, 1) + } + + log.Println(reqCounter, respCounter) + wg.Done() }) diff --git a/middleware_test.go b/middleware_test.go index aab936e..4697d09 100644 --- a/middleware_test.go +++ b/middleware_test.go @@ -19,8 +19,11 @@ type fakeServiceCb func(string, int, []byte) // Simple service that generate token on request, and require this token for accesing to secure area func NewFakeSecureService(wg *sync.WaitGroup, cb fakeServiceCb) string { active_tokens := make([]string, 0) + var mu sync.Mutex server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + mu.Lock() + defer mu.Unlock() Debug("Received request: " + req.URL.String()) switch req.URL.Path { diff --git a/raw_socket_listener/tcp_message.go b/raw_socket_listener/tcp_message.go index 5c4cf53..5800534 100644 --- a/raw_socket_listener/tcp_message.go +++ b/raw_socket_listener/tcp_message.go @@ -83,7 +83,7 @@ func (t *TCPMessage) Timeout() { close(t.packetsChan) // Notify RAWListener that message is ready to be send to replay server // Responses without requests gets discarded - if t.IsIncoming { + if t.IsIncoming || t.RequestStart != 0 { t.delChan <- t } } @@ -156,19 +156,22 @@ func (t *TCPMessage) isMultipart() bool { } payload := t.packets[0].Data - m := payload[:3] + m := payload[:4] if t.IsIncoming { // If one GET, OPTIONS, or HEAD request - if bytes.Equal(m, []byte("GET")) || bytes.Equal(m, []byte("OPT")) || bytes.Equal(m, []byte("HEA")) { + if bytes.Equal(m, []byte("GET ")) || bytes.Equal(m, []byte("OPTI")) || bytes.Equal(m, []byte("HEAD")) { return false } else { - if length := proto.Header(payload, []byte("Content-Length")); len(length) > 0 { - l, _ := strconv.Atoi(string(length)) + // Sometimes header comes after the body :( + if bytes.Equal(m, []byte("POST")) || bytes.Equal(m, []byte("PUT ")) || bytes.Equal(m, []byte("PATC")) { + if length := proto.Header(payload, []byte("Content-Length")); len(length) > 0 { + l, _ := strconv.Atoi(string(length)) - // If content-length equal current body length - if l > 0 && l == len(proto.Body(payload)) { - return false + // If content-length equal current body length + if l > 0 && l == len(proto.Body(payload)) { + return false + } } } } From 9550b31fdee530b1704fca57eb4905c64f6f4c08 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Wed, 19 Aug 2015 22:38:24 +0300 Subject: [PATCH 46/64] Fix plugin registration --- Makefile | 2 +- plugins.go | 9 ++------- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/Makefile b/Makefile index f2bba5c..3addd1a 100644 --- a/Makefile +++ b/Makefile @@ -37,7 +37,7 @@ dbench: # Used mainly for debugging, because docker container do not have access to parent machine ports drun: - docker run -v `pwd`:$(SOURCE_PATH) -t -i gor go run $(SOURCE) --input-dummy=0 --output-http="http://localhost:9000" --input-http :9000 --verbose + docker run -v `pwd`:$(SOURCE_PATH) -t -i gor go run $(SOURCE) --input-dummy=0 --output-http="http://localhost:9000" --input-raw :9000 --input-http :9000 --verbose --debug drecord: docker run -v `pwd`:$(SOURCE_PATH) -t -i gor go run $(SOURCE) --input-dummy=0 --output-file=requests.gor --verbose diff --git a/plugins.go b/plugins.go index ce8e7a1..8c08e2f 100644 --- a/plugins.go +++ b/plugins.go @@ -4,6 +4,7 @@ import ( "io" "reflect" "strings" + "time" ) // InOutPlugins struct for holding references to plugins @@ -79,13 +80,7 @@ func InitPlugins() { } for _, options := range Settings.inputRAW { - captureResponse := false - - if len(Settings.middleware) > 0 { - captureResponse = true - } - - registerPlugin(NewRAWInput, options, 0, captureResponse) + registerPlugin(NewRAWInput, options, time.Duration(0)) } for _, options := range Settings.inputTCP { From 33cdcd4d1386441c16b8345e75b715a699d570eb Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Thu, 20 Aug 2015 07:28:13 +0300 Subject: [PATCH 47/64] Use httptest package and properly close servers --- http_client_test.go | 6 +++++ input_raw_test.go | 61 +++++++++++++++++++++++---------------------- middleware_test.go | 23 +++++++++++------ output_http_test.go | 34 +++++++++---------------- 4 files changed, 64 insertions(+), 60 deletions(-) diff --git a/http_client_test.go b/http_client_test.go index d970381..e91284e 100644 --- a/http_client_test.go +++ b/http_client_test.go @@ -75,6 +75,7 @@ func TestHTTPClientSend(t *testing.T) { wg.Done() })) + defer server.Close() client := NewHTTPClient(server.URL, &HTTPClientConfig{Debug: true}) @@ -103,6 +104,7 @@ func TestHTTPClientResponseBuffer(t *testing.T) { wg.Done() })) + defer server.Close() client := NewHTTPClient(server.URL, &HTTPClientConfig{Debug: false, ResponseBufferSize: 1024}) @@ -153,6 +155,7 @@ func TestHTTPClientHTTPSSend(t *testing.T) { wg.Done() })) + defer server.Close() client := NewHTTPClient(server.URL, &HTTPClientConfig{}) @@ -246,6 +249,7 @@ func TestHTTPClientRedirect(t *testing.T) { wg.Done() })) + defer server.Close() client := NewHTTPClient(server.URL, &HTTPClientConfig{FollowRedirects: 1, Debug: false}) @@ -277,6 +281,7 @@ func TestHTTPClientRedirectLimit(t *testing.T) { wg.Done() })) + defer server.Close() client := NewHTTPClient(server.URL, &HTTPClientConfig{FollowRedirects: 2, Debug: false}) @@ -300,6 +305,7 @@ func TestHTTPClientHandleHTTP10(t *testing.T) { wg.Done() })) + defer server.Close() client := NewHTTPClient(server.URL, &HTTPClientConfig{Debug: true}) diff --git a/input_raw_test.go b/input_raw_test.go index 38126f1..608c06e 100644 --- a/input_raw_test.go +++ b/input_raw_test.go @@ -22,11 +22,14 @@ func TestRAWInput(t *testing.T) { wg := new(sync.WaitGroup) quit := make(chan int) - listener := startHTTP(func(w http.ResponseWriter, req *http.Request) {}) + origin := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {})) + defer origin.Close() + originAddr := strings.Replace(origin.Listener.Addr().String(), "[::]", "127.0.0.1", -1) + var respCounter, reqCounter int64 - input := NewRAWInput(listener.Addr().String(), testRawExpire) + input := NewRAWInput(originAddr, testRawExpire) output := NewTestOutput(func(data []byte) { if data[0] == '1' { atomic.AddInt64(&reqCounter, 1) @@ -42,9 +45,7 @@ func TestRAWInput(t *testing.T) { Plugins.Inputs = []io.Reader{input} Plugins.Outputs = []io.Writer{output} - address := strings.Replace(listener.Addr().String(), "[::]", "127.0.0.1", -1) - - client := NewHTTPClient(address, &HTTPClientConfig{}) + client := NewHTTPClient(origin.URL, &HTTPClientConfig{}) time.Sleep(time.Millisecond) @@ -68,14 +69,15 @@ func TestInputRAW100Expect(t *testing.T) { fileContent, _ := ioutil.ReadFile("README.md") // Origing and Replay server initialization - origin := startHTTP(func(w http.ResponseWriter, req *http.Request) { - defer req.Body.Close() - ioutil.ReadAll(req.Body) + origin := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + defer r.Body.Close() + ioutil.ReadAll(r.Body) wg.Done() - }) + })) + defer origin.Close() - originAddr := strings.Replace(origin.Addr().String(), "[::]", "127.0.0.1", -1) + originAddr := strings.Replace(origin.Listener.Addr().String(), "[::]", "127.0.0.1", -1) input := NewRAWInput(originAddr, testRawExpire) @@ -92,20 +94,20 @@ func TestInputRAW100Expect(t *testing.T) { } }) - listener := startHTTP(func(w http.ResponseWriter, req *http.Request) { - defer req.Body.Close() - body, _ := ioutil.ReadAll(req.Body) + replay := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + defer r.Body.Close() + body, _ := ioutil.ReadAll(r.Body) if !bytes.Equal(body, fileContent) { - buf, _ := httputil.DumpRequest(req, true) + buf, _ := httputil.DumpRequest(r, true) t.Error("Wrong POST body:", string(buf)) } wg.Done() - }) - replayAddr := listener.Addr().String() + })) + defer replay.Close() - httpOutput := NewHTTPOutput(replayAddr, &HTTPOutputConfig{}) + httpOutput := NewHTTPOutput(replay.URL, &HTTPOutputConfig{}) Plugins.Inputs = []io.Reader{input} Plugins.Outputs = []io.Writer{testOutput, httpOutput} @@ -131,31 +133,30 @@ func TestInputRAWChunkedEncoding(t *testing.T) { fileContent, _ := ioutil.ReadFile("README.md") // Origing and Replay server initialization - origin := startHTTP(func(w http.ResponseWriter, req *http.Request) { - defer req.Body.Close() - ioutil.ReadAll(req.Body) + origin := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + defer r.Body.Close() + ioutil.ReadAll(r.Body) wg.Done() - }) - - originAddr := strings.Replace(origin.Addr().String(), "[::]", "127.0.0.1", -1) + })) + originAddr := strings.Replace(origin.Listener.Addr().String(), "[::]", "127.0.0.1", -1) input := NewRAWInput(originAddr, testRawExpire) - listener := startHTTP(func(w http.ResponseWriter, req *http.Request) { - defer req.Body.Close() - body, _ := ioutil.ReadAll(req.Body) + replay := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + defer r.Body.Close() + body, _ := ioutil.ReadAll(r.Body) if !bytes.Equal(body, fileContent) { - buf, _ := httputil.DumpRequest(req, true) + buf, _ := httputil.DumpRequest(r, true) t.Error("Wrong POST body:", string(buf)) } wg.Done() - }) - replayAddr := listener.Addr().String() + })) + defer replay.Close() - httpOutput := NewHTTPOutput(replayAddr, &HTTPOutputConfig{Debug: true}) + httpOutput := NewHTTPOutput(replay.URL, &HTTPOutputConfig{Debug: true}) Plugins.Inputs = []io.Reader{input} Plugins.Outputs = []io.Writer{httpOutput} diff --git a/middleware_test.go b/middleware_test.go index 4697d09..8458654 100644 --- a/middleware_test.go +++ b/middleware_test.go @@ -17,7 +17,7 @@ import ( type fakeServiceCb func(string, int, []byte) // Simple service that generate token on request, and require this token for accesing to secure area -func NewFakeSecureService(wg *sync.WaitGroup, cb fakeServiceCb) string { +func NewFakeSecureService(wg *sync.WaitGroup, cb fakeServiceCb) *httptest.Server { active_tokens := make([]string, 0) var mu sync.Mutex @@ -61,8 +61,7 @@ func NewFakeSecureService(wg *sync.WaitGroup, cb fakeServiceCb) string { wg.Done() })) - address := strings.Replace(server.Listener.Addr().String(), "[::]", "127.0.0.1", -1) - return address + return server } func TestFakeSecureService(t *testing.T) { @@ -70,12 +69,13 @@ func TestFakeSecureService(t *testing.T) { wg := new(sync.WaitGroup) - addr := NewFakeSecureService(wg, func(path string, status int, resp []byte) { + server := NewFakeSecureService(wg, func(path string, status int, resp []byte) { }) + defer server.Close() wg.Add(3) - client := NewHTTPClient("http://"+addr, &HTTPClientConfig{Debug: true}) + client := NewHTTPClient(server.URL, &HTTPClientConfig{Debug: true}) resp, _ = client.Get("/token") token = proto.Body(resp) @@ -102,11 +102,14 @@ func TestEchoMiddleware(t *testing.T) { w.Header().Set("RequestPath", r.URL.Path) wg.Done() })) + defer from.Close() + to := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Env", "test") w.Header().Set("RequestPath", r.URL.Path) wg.Done() })) + defer to.Close() quit := make(chan int) @@ -150,6 +153,8 @@ func TestTokenMiddleware(t *testing.T) { from := NewFakeSecureService(wg, func(path string, status int, tok []byte) { time.Sleep(10 * time.Millisecond) }) + defer from.Close() + to := NewFakeSecureService(wg, func(path string, status int, tok []byte) { switch path { case "/secure": @@ -160,16 +165,18 @@ func TestTokenMiddleware(t *testing.T) { time.Sleep(10 * time.Millisecond) }) + defer to.Close() quit := make(chan int) Settings.middleware = "go run ./examples/middleware/token_modifier.go" + fromAddr := strings.Replace(from.Listener.Addr().String(), "[::]", "127.0.0.1", -1) // Catch traffic from one service - input := NewRAWInput(from, testRawExpire) + input := NewRAWInput(fromAddr, testRawExpire) // And redirect to another - output := NewHTTPOutput(to, &HTTPOutputConfig{Debug: true}) + output := NewHTTPOutput(to.URL, &HTTPOutputConfig{Debug: true}) Plugins.Inputs = []io.Reader{input} Plugins.Outputs = []io.Writer{output} @@ -184,7 +191,7 @@ func TestTokenMiddleware(t *testing.T) { // Should receive 2 requests from original + 2 from replayed wg.Add(4) - client := NewHTTPClient("http://"+from, &HTTPClientConfig{Debug: false}) + client := NewHTTPClient(from.URL, &HTTPClientConfig{Debug: false}) // Sending traffic to original service resp, _ = client.Get("/token") diff --git a/output_http_test.go b/output_http_test.go index f4a4e6c..b97399e 100644 --- a/output_http_test.go +++ b/output_http_test.go @@ -3,7 +3,6 @@ package main import ( "io" "io/ioutil" - "net" "net/http" "net/http/httptest" _ "net/http/httputil" @@ -12,25 +11,13 @@ import ( "time" ) -func startHTTP(cb func(http.ResponseWriter, *http.Request)) net.Listener { - handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - cb(w, r) - }) - - listener, _ := net.Listen("tcp", ":0") - - go http.Serve(listener, handler) - - return listener -} - func TestHTTPOutput(t *testing.T) { wg := new(sync.WaitGroup) quit := make(chan int) input := NewTestInput() - listener := startHTTP(func(w http.ResponseWriter, req *http.Request) { + server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { if req.Header.Get("User-Agent") != "Gor" { t.Error("Wrong header") } @@ -49,13 +36,14 @@ func TestHTTPOutput(t *testing.T) { } wg.Done() - }) + })) + defer server.Close() headers := HTTPHeaders{HTTPHeader{"User-Agent", "Gor"}} methods := HTTPMethods{[]byte("GET"), []byte("PUT"), []byte("POST")} Settings.modifierConfig = HTTPModifierConfig{headers: headers, methods: methods} - output := NewHTTPOutput(listener.Addr().String(), &HTTPOutputConfig{Debug: true}) + output := NewHTTPOutput(server.URL, &HTTPOutputConfig{Debug: true}) Plugins.Inputs = []io.Reader{input} Plugins.Outputs = []io.Writer{output} @@ -82,18 +70,19 @@ func TestHTTPOutputKeepOriginalHost(t *testing.T) { input := NewTestInput() - listener := startHTTP(func(w http.ResponseWriter, req *http.Request) { + server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { if req.Host != "custom-host.com" { t.Error("Wrong header", req.Host) } wg.Done() - }) + })) + defer server.Close() headers := HTTPHeaders{HTTPHeader{"Host", "custom-host.com"}} Settings.modifierConfig = HTTPModifierConfig{headers: headers} - output := NewHTTPOutput(listener.Addr().String(), &HTTPOutputConfig{Debug: false, OriginalHost: true}) + output := NewHTTPOutput(server.URL, &HTTPOutputConfig{Debug: false, OriginalHost: true}) Plugins.Inputs = []io.Reader{input} Plugins.Outputs = []io.Writer{output} @@ -140,13 +129,14 @@ func BenchmarkHTTPOutput(b *testing.B) { wg := new(sync.WaitGroup) quit := make(chan int) - listener := startHTTP(func(w http.ResponseWriter, req *http.Request) { + server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { time.Sleep(50 * time.Millisecond) wg.Done() - }) + })) + defer server.Close() input := NewTestInput() - output := NewHTTPOutput(listener.Addr().String(), &HTTPOutputConfig{}) + output := NewHTTPOutput(server.URL, &HTTPOutputConfig{}) Plugins.Inputs = []io.Reader{input} Plugins.Outputs = []io.Writer{output} From 6974a6e278a0a0cb28c1b5cbbac97e8981cc1cc3 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Thu, 20 Aug 2015 08:56:10 +0300 Subject: [PATCH 48/64] Ensure that raw input gets closed --- emitter.go | 13 +++++++- input_raw.go | 18 +++++++++-- input_raw_test.go | 11 +++++-- middleware_test.go | 2 ++ raw_socket_listener/listener.go | 55 ++++++++++++++++++++++----------- 5 files changed, 75 insertions(+), 24 deletions(-) diff --git a/emitter.go b/emitter.go index 00ba070..8ad053c 100644 --- a/emitter.go +++ b/emitter.go @@ -32,8 +32,19 @@ func Start(stop chan int) { for { select { case <-stop: + for _, in := range Plugins.Inputs { + if c, ok := in.(io.Closer); ok { + c.Close() + } + } + + for _, out := range Plugins.Outputs { + if c, ok := out.(io.Closer); ok { + c.Close() + } + } return - case <-time.After(1 * time.Second): + case <-time.After(time.Second): } } } diff --git a/input_raw.go b/input_raw.go index 8c096c6..7ce6a30 100644 --- a/input_raw.go +++ b/input_raw.go @@ -13,6 +13,8 @@ type RAWInput struct { data chan *raw.TCPMessage address string expire time.Duration + quit chan bool + listener *raw.Listener } // NewRAWInput constructor for RAWInput. Accepts address with port as argument. @@ -21,6 +23,7 @@ func NewRAWInput(address string, expire time.Duration) (i *RAWInput) { i.data = make(chan *raw.TCPMessage) i.address = address i.expire = expire + i.quit = make(chan bool) go i.listen(address) @@ -56,11 +59,17 @@ func (i *RAWInput) listen(address string) { log.Fatal("input-raw: error while parsing address", err) } - listener := raw.NewListener(host, port, i.expire, true) + i.listener = raw.NewListener(host, port, i.expire, true) for { + select { + case <-i.quit: + return + default: + } + // Receiving TCPMessage object - m := listener.Receive() + m := i.listener.Receive() i.data <- m } @@ -69,3 +78,8 @@ func (i *RAWInput) listen(address string) { func (i *RAWInput) String() string { return "RAW Socket input: " + i.address } + +func (i *RAWInput) Close() { + i.listener.Close() + close(i.quit) +} diff --git a/input_raw_test.go b/input_raw_test.go index 608c06e..4c7c1ad 100644 --- a/input_raw_test.go +++ b/input_raw_test.go @@ -28,8 +28,13 @@ func TestRAWInput(t *testing.T) { var respCounter, reqCounter int64 + defer func(){ + log.Println(reqCounter, respCounter) + }() input := NewRAWInput(originAddr, testRawExpire) + defer input.Close() + output := NewTestOutput(func(data []byte) { if data[0] == '1' { atomic.AddInt64(&reqCounter, 1) @@ -37,8 +42,6 @@ func TestRAWInput(t *testing.T) { atomic.AddInt64(&respCounter, 1) } - log.Println(reqCounter, respCounter) - wg.Done() }) @@ -58,7 +61,6 @@ func TestRAWInput(t *testing.T) { } wg.Wait() - close(quit) } @@ -80,6 +82,7 @@ func TestInputRAW100Expect(t *testing.T) { originAddr := strings.Replace(origin.Listener.Addr().String(), "[::]", "127.0.0.1", -1) input := NewRAWInput(originAddr, testRawExpire) + defer input.Close() // We will use it to get content of raw HTTP request testOutput := NewTestOutput(func(data []byte) { @@ -142,6 +145,7 @@ func TestInputRAWChunkedEncoding(t *testing.T) { originAddr := strings.Replace(origin.Listener.Addr().String(), "[::]", "127.0.0.1", -1) input := NewRAWInput(originAddr, testRawExpire) + defer input.Close() replay := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { defer r.Body.Close() @@ -200,6 +204,7 @@ func TestInputRAWLargePayload(t *testing.T) { originAddr := strings.Replace(origin.Listener.Addr().String(), "[::]", "127.0.0.1", -1) input := NewRAWInput(originAddr, testRawExpire) + defer input.Close() replay := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { req.Body = http.MaxBytesReader(w, req.Body, 1*1024*1024) diff --git a/middleware_test.go b/middleware_test.go index 8458654..c94154d 100644 --- a/middleware_test.go +++ b/middleware_test.go @@ -117,6 +117,7 @@ func TestEchoMiddleware(t *testing.T) { // Catch traffic from one service input := NewRAWInput(from.Listener.Addr().String(), testRawExpire) + defer input.Close() // And redirect to another output := NewHTTPOutput(to.URL, &HTTPOutputConfig{Debug: false}) @@ -174,6 +175,7 @@ func TestTokenMiddleware(t *testing.T) { fromAddr := strings.Replace(from.Listener.Addr().String(), "[::]", "127.0.0.1", -1) // Catch traffic from one service input := NewRAWInput(fromAddr, testRawExpire) + defer input.Close() // And redirect to another output := NewHTTPOutput(to.URL, &HTTPOutputConfig{Debug: true}) diff --git a/raw_socket_listener/listener.go b/raw_socket_listener/listener.go index fc3de75..65b1899 100644 --- a/raw_socket_listener/listener.go +++ b/raw_socket_listener/listener.go @@ -20,6 +20,7 @@ import ( "runtime/debug" "strconv" "time" + "strings" ) // Listener handle traffic capture @@ -50,6 +51,9 @@ type Listener struct { messageExpire time.Duration captureResponse bool + + conn net.PacketConn + quit chan bool } type request struct { @@ -58,30 +62,31 @@ type request struct { } // NewListener creates and initializes new Listener object -func NewListener(addr string, port string, expire time.Duration, captureResponse bool) (rawListener *Listener) { - rawListener = &Listener{captureResponse: captureResponse} +func NewListener(addr string, port string, expire time.Duration, captureResponse bool) (l *Listener) { + l = &Listener{captureResponse: captureResponse} - rawListener.packetsChan = make(chan *TCPPacket, 10000) - rawListener.messagesChan = make(chan *TCPMessage, 10000) - rawListener.messageDelChan = make(chan *TCPMessage, 10000) + l.packetsChan = make(chan *TCPPacket, 10000) + l.messagesChan = make(chan *TCPMessage, 10000) + l.messageDelChan = make(chan *TCPMessage, 10000) + l.quit = make(chan bool) - rawListener.messages = make(map[string]*TCPMessage) - rawListener.ackAliases = make(map[uint32]uint32) - rawListener.seqWithData = make(map[uint32]uint32) - rawListener.respAliases = make(map[uint32]*request) + l.messages = make(map[string]*TCPMessage) + l.ackAliases = make(map[uint32]uint32) + l.seqWithData = make(map[uint32]uint32) + l.respAliases = make(map[uint32]*request) - rawListener.addr = addr + l.addr = addr _port, _ := strconv.Atoi(port) - rawListener.port = uint16(_port) + l.port = uint16(_port) if expire.Nanoseconds() == 0 { expire = 2000 * time.Millisecond } - rawListener.messageExpire = expire + l.messageExpire = expire - go rawListener.listen() - go rawListener.readRAWSocket() + go l.listen() + go l.readRAWSocket() return } @@ -89,6 +94,9 @@ func NewListener(addr string, port string, expire time.Duration, captureResponse func (t *Listener) listen() { for { select { + case <-t.quit: + t.conn.Close() + return // If message ready for deletion it means that its also complete or expired by timeout case message := <-t.messageDelChan: delete(t.ackAliases, message.Ack) @@ -109,21 +117,26 @@ func (t *Listener) listen() { } func (t *Listener) readRAWSocket() { conn, e := net.ListenPacket("ip4:tcp", t.addr) + t.conn = conn if e != nil { log.Fatal(e) } - defer conn.Close() + defer t.conn.Close() for { buf := make([]byte, 64*1024) // 64kb // Note: ReadFrom receive messages without IP header - n, addr, err := conn.ReadFrom(buf) + n, addr, err := t.conn.ReadFrom(buf) if err != nil { - log.Println("Error:", err) - continue + if strings.HasSuffix(err.Error(), "closed network connection") { + return + } else { + log.Println("Raw listener error:", err) + continue + } } if n > 0 { @@ -240,3 +253,9 @@ func (t *Listener) processTCPPacket(packet *TCPPacket) { func (t *Listener) Receive() *TCPMessage { return <-t.messagesChan } + +func (t *Listener) Close() { + close(t.quit) + t.conn.Close() + return +} From 3c45595a96c4cfaacea0713a0ce34c9e6c022946 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Thu, 20 Aug 2015 09:27:44 +0300 Subject: [PATCH 49/64] More race fixes --- emitter.go | 11 ----------- input_file.go | 6 +++++- output_tcp.go | 6 +++++- 3 files changed, 10 insertions(+), 13 deletions(-) diff --git a/emitter.go b/emitter.go index 8ad053c..d767d45 100644 --- a/emitter.go +++ b/emitter.go @@ -32,17 +32,6 @@ func Start(stop chan int) { for { select { case <-stop: - for _, in := range Plugins.Inputs { - if c, ok := in.(io.Closer); ok { - c.Close() - } - } - - for _, out := range Plugins.Outputs { - if c, ok := out.(io.Closer); ok { - c.Close() - } - } return case <-time.After(time.Second): } diff --git a/input_file.go b/input_file.go index 840c5d7..24ebd2d 100644 --- a/input_file.go +++ b/input_file.go @@ -74,6 +74,10 @@ func (i *FileInput) emit() { lastTime = ts } - i.data <- buf + // scanner returs only pointer, so to remove data-race we have to allocate new array + newBuf := make([]byte, len(buf)) + copy(newBuf, buf) + + i.data <- newBuf } } diff --git a/output_tcp.go b/output_tcp.go index 96295dc..9d46fa7 100644 --- a/output_tcp.go +++ b/output_tcp.go @@ -62,7 +62,11 @@ func (o *TCPOutput) Write(data []byte) (n int, err error) { return len(data), nil } - o.buf <- data + // We have to copy, because sending data in multiple threads + newBuf := make([]byte, len(data)) + copy(newBuf, data) + + o.buf <- newBuf if Settings.outputTCPStats { o.bufStats.Write(len(o.buf)) From 81d35086539f0303d93c711360c4ea009e34fd6c Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Thu, 20 Aug 2015 09:29:51 +0300 Subject: [PATCH 50/64] Enable race testing for travis --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 4736e5b..a1eaa09 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,3 +1,3 @@ language: go go: 1.4.2 -script: sudo -E bash -c "source /etc/profile && eval '$(gimme 1.4.2)' && export GOPATH=$HOME/gopath:$GOPATH && go get && GORACE='halt_on_error=1' go test ./... -v -timeout 60s" \ No newline at end of file +script: sudo -E bash -c "source /etc/profile && eval '$(gimme 1.4.2)' && export GOPATH=$HOME/gopath:$GOPATH && go get && GORACE='halt_on_error=1' go test ./... -v -timeout 60s -race" \ No newline at end of file From 76d5e22ba18b799771ebf106143889044989bbcf Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Thu, 20 Aug 2015 09:33:51 +0300 Subject: [PATCH 51/64] Enable packet debugging --- raw_socket_listener/listener.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/raw_socket_listener/listener.go b/raw_socket_listener/listener.go index 65b1899..b35443a 100644 --- a/raw_socket_listener/listener.go +++ b/raw_socket_listener/listener.go @@ -110,7 +110,7 @@ func (t *Listener) listen() { // We need to use channels to process each packet to avoid data races case packet := <-t.packetsChan: - // log.Println("Received packet:", packet) + log.Println("Received packet:", packet) t.processTCPPacket(packet) } } From 9309fb86ef7cd4917f29d6ee81c43980f9fff970 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Thu, 20 Aug 2015 09:45:59 +0300 Subject: [PATCH 52/64] Reduce debugging --- raw_socket_listener/listener.go | 1 - 1 file changed, 1 deletion(-) diff --git a/raw_socket_listener/listener.go b/raw_socket_listener/listener.go index b35443a..5459019 100644 --- a/raw_socket_listener/listener.go +++ b/raw_socket_listener/listener.go @@ -110,7 +110,6 @@ func (t *Listener) listen() { // We need to use channels to process each packet to avoid data races case packet := <-t.packetsChan: - log.Println("Received packet:", packet) t.processTCPPacket(packet) } } From 6533b50ab08ea94d53a2be7055919d6d645881aa Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Thu, 20 Aug 2015 09:51:44 +0300 Subject: [PATCH 53/64] Travis should use 1.5 --- .travis.yml | 4 ++-- Dockerfile | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index a1eaa09..82c3aec 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,3 +1,3 @@ language: go -go: 1.4.2 -script: sudo -E bash -c "source /etc/profile && eval '$(gimme 1.4.2)' && export GOPATH=$HOME/gopath:$GOPATH && go get && GORACE='halt_on_error=1' go test ./... -v -timeout 60s -race" \ No newline at end of file +go: 1.5 +script: sudo -E bash -c "source /etc/profile && eval '$(gimme 1.5)' && export GOPATH=$HOME/gopath:$GOPATH && go get && GORACE='halt_on_error=1' go test ./... -v -timeout 60s -race" \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 5cfb783..38c5228 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ -FROM google/golang:1.4 +FROM golang:1.5 -RUN cd /goroot/src/ && GOOS=linux GOARCH=386 ./make.bash --no-clean +RUN cd /go/src/ && GOOS=linux GOARCH=386 ./make.bash --no-clean RUN apt-get update && apt-get install ruby vim-common -y From 16d7f23d980fc52db9972a90691359a48178a706 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Sat, 22 Aug 2015 09:41:22 +0300 Subject: [PATCH 54/64] New fixes --- Dockerfile | 11 +++++-- Makefile | 7 +++-- examples/middleware/echo.java | 30 +++++++++++++++++++ .../middleware/{echo_modifier.rb => echo.rb} | 0 .../middleware/{echo_modifier.sh => echo.sh} | 0 input_dummy.go | 13 ++++---- input_file.go | 2 ++ input_http.go | 9 ++++-- input_raw.go | 8 ++--- input_raw_test.go | 3 +- middleware_test.go | 2 +- output_http.go | 8 ++--- raw_socket_listener/listener.go | 2 +- raw_socket_listener/tcp_message.go | 2 +- 14 files changed, 73 insertions(+), 24 deletions(-) create mode 100644 examples/middleware/echo.java rename examples/middleware/{echo_modifier.rb => echo.rb} (100%) rename examples/middleware/{echo_modifier.sh => echo.sh} (100%) diff --git a/Dockerfile b/Dockerfile index 38c5228..678c431 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,9 +1,16 @@ -FROM golang:1.5 +FROM google/golang:1.4 -RUN cd /go/src/ && GOOS=linux GOARCH=386 ./make.bash --no-clean +RUN cd /goroot/src/ && GOOS=linux GOARCH=386 ./make.bash --no-clean RUN apt-get update && apt-get install ruby vim-common -y +# Install Java +# RUN apt-get install -y software-properties-common python-software-properties +# RUN add-apt-repository -y ppa:webupd8team/java +# RUN apt-get update -y +# RUN echo oracle-java8-installer shared/accepted-oracle-license-v1-1 select true | sudo /usr/bin/debconf-set-selections +# RUN apt-get install -y oracle-java8-installer + WORKDIR /gopath/src/github.com/buger/gor/ ADD . /gopath/src/github.com/buger/gor/ diff --git a/Makefile b/Makefile index 3addd1a..29de44d 100644 --- a/Makefile +++ b/Makefile @@ -37,10 +37,13 @@ dbench: # Used mainly for debugging, because docker container do not have access to parent machine ports drun: - docker run -v `pwd`:$(SOURCE_PATH) -t -i gor go run $(SOURCE) --input-dummy=0 --output-http="http://localhost:9000" --input-raw :9000 --input-http :9000 --verbose --debug + docker run -v `pwd`:$(SOURCE_PATH) -t -i gor go run $(SOURCE) --input-dummy=0 --output-http="http://localhost:9000" --input-raw :9000 --input-http :9000 --verbose --debug --middleware "./examples/middleware/echo.sh" + +drun-2: + docker run -v `pwd`:$(SOURCE_PATH) -t -i gor go run $(SOURCE) --input-file="./fixtures/requests.gor" --output-dummy --verbose --debug --middleware "./examples/middleware/echo.sh" drecord: - docker run -v `pwd`:$(SOURCE_PATH) -t -i gor go run $(SOURCE) --input-dummy=0 --output-file=requests.gor --verbose + docker run -v `pwd`:$(SOURCE_PATH) -t -i gor go run $(SOURCE) --input-dummy=0 --output-file=requests.gor --verbose --debug dreplay: docker run -v `pwd`:$(SOURCE_PATH) -t -i gor go run $(SOURCE) --input-file=requests.bin --output-tcp=:9000 --verbose -h diff --git a/examples/middleware/echo.java b/examples/middleware/echo.java new file mode 100644 index 0000000..e37cf72 --- /dev/null +++ b/examples/middleware/echo.java @@ -0,0 +1,30 @@ +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; + +import org.apache.commons.io.IOUtils; + +public class Echo { + public static void main(String[] args) { + if(args != null){ + for(String arg : args){ + System.out.println(arg); + } + + } + + BufferedReader stdin = new BufferedReader(new InputStreamReader( + System.in)); + String line = null; + + try { + while ((line = stdin.readLine()) != null) { + + System.out.println(line); + + } + } catch (IOException e) { + IOUtils.closeQuietly(stdin); + } + } +} \ No newline at end of file diff --git a/examples/middleware/echo_modifier.rb b/examples/middleware/echo.rb similarity index 100% rename from examples/middleware/echo_modifier.rb rename to examples/middleware/echo.rb diff --git a/examples/middleware/echo_modifier.sh b/examples/middleware/echo.sh similarity index 100% rename from examples/middleware/echo_modifier.sh rename to examples/middleware/echo.sh diff --git a/input_dummy.go b/input_dummy.go index 4c14830..17907d3 100644 --- a/input_dummy.go +++ b/input_dummy.go @@ -22,11 +22,9 @@ func NewDummyInput(options string) (di *DummyInput) { func (i *DummyInput) Read(data []byte) (int, error) { buf := <-i.data - header := payloadHeader(RequestPayload, uuid(), time.Now().UnixNano()) - copy(data[0:len(header)], header) - copy(data[len(header):], buf) + copy(data, buf) - return len(buf) + len(header), nil + return len(buf), nil } func (i *DummyInput) emit() { @@ -35,7 +33,12 @@ func (i *DummyInput) emit() { for { select { case <-ticker.C: - i.data <- []byte("POST /pub/WWW/å HTTP/1.1\nHost: www.w3.org\r\n\r\na=1&b=2") + uuid := uuid() + reqh := payloadHeader(RequestPayload, uuid, time.Now().UnixNano()) + i.data <- append(reqh, []byte("POST /pub/WWW/å HTTP/1.1\nHost: www.w3.org\r\nContent-Length: 7\r\n\r\na=1&b=2")...) + + resh := payloadHeader(ResponsePayload, uuid, 1) + i.data <- append(resh, []byte("HTTP/1.1 200 OK\r\n\r\n")...) } } } diff --git a/input_file.go b/input_file.go index 24ebd2d..60ec958 100644 --- a/input_file.go +++ b/input_file.go @@ -80,4 +80,6 @@ func (i *FileInput) emit() { i.data <- newBuf } + + log.Println("FileInput: end of file") } diff --git a/input_http.go b/input_http.go index 5150c4e..f7717c5 100644 --- a/input_http.go +++ b/input_http.go @@ -5,6 +5,7 @@ import ( "net" "net/http" "net/http/httputil" + "time" ) // HTTPInput used for sending requests to Gor via http @@ -27,9 +28,13 @@ func NewHTTPInput(address string) (i *HTTPInput) { func (i *HTTPInput) Read(data []byte) (int, error) { buf := <-i.data - copy(data, buf) - return len(buf), nil + header := payloadHeader(RequestPayload, uuid(), time.Now().UnixNano()) + + copy(data[0:len(header)], header) + copy(data[len(header):], buf) + + return len(buf) + len(header), nil } func (i *HTTPInput) handler(w http.ResponseWriter, r *http.Request) { diff --git a/input_raw.go b/input_raw.go index 7ce6a30..ed14707 100644 --- a/input_raw.go +++ b/input_raw.go @@ -10,10 +10,10 @@ import ( // RAWInput used for intercepting traffic for given address type RAWInput struct { - data chan *raw.TCPMessage - address string - expire time.Duration - quit chan bool + data chan *raw.TCPMessage + address string + expire time.Duration + quit chan bool listener *raw.Listener } diff --git a/input_raw_test.go b/input_raw_test.go index 4c7c1ad..fbe47e4 100644 --- a/input_raw_test.go +++ b/input_raw_test.go @@ -26,9 +26,8 @@ func TestRAWInput(t *testing.T) { defer origin.Close() originAddr := strings.Replace(origin.Listener.Addr().String(), "[::]", "127.0.0.1", -1) - var respCounter, reqCounter int64 - defer func(){ + defer func() { log.Println(reqCounter, respCounter) }() diff --git a/middleware_test.go b/middleware_test.go index c94154d..8474eeb 100644 --- a/middleware_test.go +++ b/middleware_test.go @@ -113,7 +113,7 @@ func TestEchoMiddleware(t *testing.T) { quit := make(chan int) - Settings.middleware = "./examples/middleware/echo_modifier.sh" + Settings.middleware = "./examples/middleware/echo.sh" // Catch traffic from one service input := NewRAWInput(from.Listener.Addr().String(), testRawExpire) diff --git a/output_http.go b/output_http.go index 0f57ea9..0528e38 100644 --- a/output_http.go +++ b/output_http.go @@ -41,9 +41,9 @@ type HTTPOutput struct { // aligned at 64bit. See https://github.com/golang/go/issues/599 activeWorkers int64 - address string - limit int - queue chan []byte + address string + limit int + queue chan []byte responses chan response @@ -173,7 +173,7 @@ func (o *HTTPOutput) Write(data []byte) (n int, err error) { func (o *HTTPOutput) Read(data []byte) (int, error) { resp := <-o.responses - Debug("[OUTPUT-HTTP] Received response", string(resp.payload)) + Debug("[OUTPUT-HTTP] Received response:", string(resp.payload)) header := payloadHeader(ReplayedResponsePayload, resp.uuid, resp.roundTripTime) copy(data[0:len(header)], header) diff --git a/raw_socket_listener/listener.go b/raw_socket_listener/listener.go index 5459019..4fe232d 100644 --- a/raw_socket_listener/listener.go +++ b/raw_socket_listener/listener.go @@ -19,8 +19,8 @@ import ( "net" "runtime/debug" "strconv" - "time" "strings" + "time" ) // Listener handle traffic capture diff --git a/raw_socket_listener/tcp_message.go b/raw_socket_listener/tcp_message.go index 5800534..cabac1d 100644 --- a/raw_socket_listener/tcp_message.go +++ b/raw_socket_listener/tcp_message.go @@ -7,8 +7,8 @@ import ( "github.com/buger/gor/proto" "log" "strconv" - "time" "sync" + "time" ) // TCPMessage ensure that all TCP packets for given request is received, and processed in right sequence From cbc27bd662f03a365eaf8bbfa96e444819d70d1e Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Sat, 22 Aug 2015 09:48:10 +0300 Subject: [PATCH 55/64] More debugging --- input_raw_test.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/input_raw_test.go b/input_raw_test.go index fbe47e4..8c3dab5 100644 --- a/input_raw_test.go +++ b/input_raw_test.go @@ -27,9 +27,6 @@ func TestRAWInput(t *testing.T) { originAddr := strings.Replace(origin.Listener.Addr().String(), "[::]", "127.0.0.1", -1) var respCounter, reqCounter int64 - defer func() { - log.Println(reqCounter, respCounter) - }() input := NewRAWInput(originAddr, testRawExpire) defer input.Close() @@ -41,6 +38,8 @@ func TestRAWInput(t *testing.T) { atomic.AddInt64(&respCounter, 1) } + log.Println(reqCounter, respCounter) + wg.Done() }) From 02bf7a86de102473bc4566ebdc2f56618c082e8f Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Sat, 22 Aug 2015 09:58:52 +0300 Subject: [PATCH 56/64] Rollback to 1.4.2 in travis --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 82c3aec..a1eaa09 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,3 +1,3 @@ language: go -go: 1.5 -script: sudo -E bash -c "source /etc/profile && eval '$(gimme 1.5)' && export GOPATH=$HOME/gopath:$GOPATH && go get && GORACE='halt_on_error=1' go test ./... -v -timeout 60s -race" \ No newline at end of file +go: 1.4.2 +script: sudo -E bash -c "source /etc/profile && eval '$(gimme 1.4.2)' && export GOPATH=$HOME/gopath:$GOPATH && go get && GORACE='halt_on_error=1' go test ./... -v -timeout 60s -race" \ No newline at end of file From d5a1eb0b9df45faa6d6abb076e37d2762874ebe9 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Sat, 22 Aug 2015 11:37:01 +0300 Subject: [PATCH 57/64] Increase input raw timeout --- input_raw_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/input_raw_test.go b/input_raw_test.go index 8c3dab5..ad2a690 100644 --- a/input_raw_test.go +++ b/input_raw_test.go @@ -16,7 +16,7 @@ import ( "time" ) -const testRawExpire = time.Millisecond * 100 +const testRawExpire = time.Millisecond * 200 func TestRAWInput(t *testing.T) { wg := new(sync.WaitGroup) From f3ea30c6df9e7dd9100eff322a15ea3a8296cbe3 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Sun, 23 Aug 2015 12:45:28 +0300 Subject: [PATCH 58/64] Add example of java echo middleware --- .gitignore | 2 ++ Dockerfile | 19 ++++++++++++------- Makefile | 2 +- examples/middleware/echo.java | 2 +- 4 files changed, 16 insertions(+), 9 deletions(-) diff --git a/.gitignore b/.gitignore index 3c34e46..0acc83e 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,5 @@ *.bin *.gz + +*.class diff --git a/Dockerfile b/Dockerfile index 678c431..bf86e03 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,16 +4,21 @@ RUN cd /goroot/src/ && GOOS=linux GOARCH=386 ./make.bash --no-clean RUN apt-get update && apt-get install ruby vim-common -y -# Install Java -# RUN apt-get install -y software-properties-common python-software-properties -# RUN add-apt-repository -y ppa:webupd8team/java -# RUN apt-get update -y -# RUN echo oracle-java8-installer shared/accepted-oracle-license-v1-1 select true | sudo /usr/bin/debconf-set-selections -# RUN apt-get install -y oracle-java8-installer +# Install Java for middleware testing +RUN echo "deb http://ppa.launchpad.net/webupd8team/java/ubuntu trusty main" | tee /etc/apt/sources.list.d/webupd8team-java.list +RUN echo "deb-src http://ppa.launchpad.net/webupd8team/java/ubuntu trusty main" | tee -a /etc/apt/sources.list.d/webupd8team-java.list +RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys EEA14886 +RUN apt-get update -y +RUN echo oracle-java7-installer shared/accepted-oracle-license-v1-1 select true | /usr/bin/debconf-set-selections +RUN apt-get install oracle-java8-installer -y + +RUN wget http://apache-mirror.rbc.ru/pub/apache//commons/io/binaries/commons-io-2.4-bin.tar.gz -P /tmp +RUN tar xzf /tmp/commons-io-2.4-bin.tar.gz -C /tmp WORKDIR /gopath/src/github.com/buger/gor/ - ADD . /gopath/src/github.com/buger/gor/ +RUN javac -cp /tmp/commons-io-2.4/commons-io-2.4.jar ./examples/middleware/echo.java + RUN go get -u github.com/golang/lint/golint RUN go get \ No newline at end of file diff --git a/Makefile b/Makefile index 29de44d..339df03 100644 --- a/Makefile +++ b/Makefile @@ -40,7 +40,7 @@ drun: docker run -v `pwd`:$(SOURCE_PATH) -t -i gor go run $(SOURCE) --input-dummy=0 --output-http="http://localhost:9000" --input-raw :9000 --input-http :9000 --verbose --debug --middleware "./examples/middleware/echo.sh" drun-2: - docker run -v `pwd`:$(SOURCE_PATH) -t -i gor go run $(SOURCE) --input-file="./fixtures/requests.gor" --output-dummy --verbose --debug --middleware "./examples/middleware/echo.sh" + docker run -v `pwd`:$(SOURCE_PATH) -t -i gor go run $(SOURCE) --input-file="./fixtures/requests.gor" --output-dummy=0 --verbose --debug --middleware "java -cp ./examples/middleware echo" drecord: docker run -v `pwd`:$(SOURCE_PATH) -t -i gor go run $(SOURCE) --input-dummy=0 --output-file=requests.gor --verbose --debug diff --git a/examples/middleware/echo.java b/examples/middleware/echo.java index e37cf72..707193e 100644 --- a/examples/middleware/echo.java +++ b/examples/middleware/echo.java @@ -4,7 +4,7 @@ import java.io.InputStreamReader; import org.apache.commons.io.IOUtils; -public class Echo { +public class echo { public static void main(String[] args) { if(args != null){ for(String arg : args){ From f0b96af420a636aeae36b9afc918d9645a417793 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Sun, 23 Aug 2015 13:07:29 +0300 Subject: [PATCH 59/64] Add protection for malformed requests --- http_modifier.go | 4 ++++ output_http.go | 8 +++++++- proto/proto.go | 15 +++++++++++++++ 3 files changed, 26 insertions(+), 1 deletion(-) diff --git a/http_modifier.go b/http_modifier.go index 14cdf85..fbca0c3 100644 --- a/http_modifier.go +++ b/http_modifier.go @@ -30,6 +30,10 @@ func NewHTTPModifier(config *HTTPModifierConfig) *HTTPModifier { } func (m *HTTPModifier) Rewrite(payload []byte) (response []byte) { + if !proto.IsHTTPPayload(payload) { + return payload + } + if len(m.config.methods) > 0 { method := proto.Method(payload) diff --git a/output_http.go b/output_http.go index 0528e38..cafe637 100644 --- a/output_http.go +++ b/output_http.go @@ -5,6 +5,7 @@ import ( "log" "sync/atomic" "time" + "github.com/buger/gor/proto" ) const initialDynamicWorkers = 10 @@ -186,8 +187,13 @@ func (o *HTTPOutput) sendRequest(client *HTTPClient, request []byte) { meta := payloadMeta(request) uuid := meta[1] + body := payloadBody(request) + if !proto.IsHTTPPayload(body) { + return + } + start := time.Now() - resp, err := client.Send(payloadBody(request)) + resp, err := client.Send(body) stop := time.Now() if err != nil { diff --git a/proto/proto.go b/proto/proto.go index da53cc0..549cec3 100644 --- a/proto/proto.go +++ b/proto/proto.go @@ -208,3 +208,18 @@ func Method(payload []byte) []byte { func Status(payload []byte) []byte { return Path(payload) } + +var httpMethods []string = []string{ + "GET ", "OPTI", "HEAD", "POST", "PUT ", "DELE", "TRAC", "CONN", +} + +func IsHTTPPayload(payload []byte) bool { + method := string(payload[0:4]) + + for _, m := range httpMethods { + if method == m { + return true + } + } + return false +} From 929e02790745042fb92aad88716a442d5021b565 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Sun, 23 Aug 2015 13:08:51 +0300 Subject: [PATCH 60/64] small formatting change --- output_http.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/output_http.go b/output_http.go index cafe637..9f82d37 100644 --- a/output_http.go +++ b/output_http.go @@ -1,11 +1,11 @@ package main import ( + "github.com/buger/gor/proto" "io" "log" "sync/atomic" "time" - "github.com/buger/gor/proto" ) const initialDynamicWorkers = 10 From 3c81723ac32ae103a3f09f42fb711fb9a8deeb0c Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Mon, 24 Aug 2015 08:58:57 +0300 Subject: [PATCH 61/64] Increase timeout for large payload --- input_raw_test.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/input_raw_test.go b/input_raw_test.go index ad2a690..79b04b1 100644 --- a/input_raw_test.go +++ b/input_raw_test.go @@ -179,6 +179,9 @@ func TestInputRAWChunkedEncoding(t *testing.T) { } func TestInputRAWLargePayload(t *testing.T) { + Settings.verbose = true + Settings.debug = true + wg := new(sync.WaitGroup) quit := make(chan int) @@ -201,7 +204,7 @@ func TestInputRAWLargePayload(t *testing.T) { })) originAddr := strings.Replace(origin.Listener.Addr().String(), "[::]", "127.0.0.1", -1) - input := NewRAWInput(originAddr, testRawExpire) + input := NewRAWInput(originAddr, time.Second) defer input.Close() replay := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { From fe91a20629d8dd78c0ebecd3e63a8bbc084cc874 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Mon, 24 Aug 2015 09:24:31 +0300 Subject: [PATCH 62/64] fix race --- input_raw_test.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/input_raw_test.go b/input_raw_test.go index 79b04b1..9305724 100644 --- a/input_raw_test.go +++ b/input_raw_test.go @@ -179,9 +179,6 @@ func TestInputRAWChunkedEncoding(t *testing.T) { } func TestInputRAWLargePayload(t *testing.T) { - Settings.verbose = true - Settings.debug = true - wg := new(sync.WaitGroup) quit := make(chan int) From 43262f333677c7af044528382f589317ec455eff Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Mon, 24 Aug 2015 09:32:31 +0300 Subject: [PATCH 63/64] Increase timeouts --- input_raw_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/input_raw_test.go b/input_raw_test.go index 9305724..7a13373 100644 --- a/input_raw_test.go +++ b/input_raw_test.go @@ -79,7 +79,7 @@ func TestInputRAW100Expect(t *testing.T) { originAddr := strings.Replace(origin.Listener.Addr().String(), "[::]", "127.0.0.1", -1) - input := NewRAWInput(originAddr, testRawExpire) + input := NewRAWInput(originAddr, time.Second) defer input.Close() // We will use it to get content of raw HTTP request @@ -142,7 +142,7 @@ func TestInputRAWChunkedEncoding(t *testing.T) { })) originAddr := strings.Replace(origin.Listener.Addr().String(), "[::]", "127.0.0.1", -1) - input := NewRAWInput(originAddr, testRawExpire) + input := NewRAWInput(originAddr, time.Second) defer input.Close() replay := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { From abde6ecfcb1c852a992b04422a2f34e0204c877b Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Mon, 24 Aug 2015 09:40:15 +0300 Subject: [PATCH 64/64] less verbose output --- input_raw_test.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/input_raw_test.go b/input_raw_test.go index 7a13373..0d75443 100644 --- a/input_raw_test.go +++ b/input_raw_test.go @@ -38,7 +38,9 @@ func TestRAWInput(t *testing.T) { atomic.AddInt64(&respCounter, 1) } - log.Println(reqCounter, respCounter) + if Settings.debug { + log.Println(reqCounter, respCounter) + } wg.Done() })