From 80572f5f18590ed3c0ff5792774e81b5649d1804 Mon Sep 17 00:00:00 2001 From: Jan Willies Date: Tue, 28 Oct 2014 14:52:55 +0100 Subject: [PATCH 01/16] add regular expression matching to --output-http-rewrite-url --- settings_url_map.go | 13 +++++++++---- settings_url_map_test.go | 24 ++++++++++++++++++++++-- 2 files changed, 31 insertions(+), 6 deletions(-) diff --git a/settings_url_map.go b/settings_url_map.go index 501b1a8..37c6173 100644 --- a/settings_url_map.go +++ b/settings_url_map.go @@ -3,11 +3,12 @@ package main import ( "errors" "fmt" + "regexp" "strings" ) type urlRewrite struct { - src string + src *regexp.Regexp target string } @@ -22,14 +23,18 @@ func (r *UrlRewriteMap) Set(value string) error { if len(valArr) < 2 { return errors.New("need both src and target, colon-delimited (ex. /a:/b).") } - *r = append(*r, urlRewrite{src: valArr[0], target: valArr[1]}) + regexp, err := regexp.Compile(valArr[0]) + if err != nil { + return err + } + *r = append(*r, urlRewrite{src: regexp, target: valArr[1]}) return nil } func (r *UrlRewriteMap) Rewrite(path string) string { for _, f := range *r { - if f.src == path { - return f.target + if f.src.MatchString(path) { + return f.src.ReplaceAllLiteralString(path, f.target) } } return path diff --git a/settings_url_map_test.go b/settings_url_map_test.go index 18cc920..23e3a5f 100644 --- a/settings_url_map_test.go +++ b/settings_url_map_test.go @@ -4,7 +4,7 @@ import ( "testing" ) -func TestUrlRewriteMap(t *testing.T) { +func TestUrlRewriteMap_1(t *testing.T) { var url string rewrites := UrlRewriteMap{} @@ -15,7 +15,6 @@ func TestUrlRewriteMap(t *testing.T) { } url = "/abc" - if rewrites.Rewrite(url) == url { t.Error("Request url should have been rewritten, wasn't") } @@ -25,3 +24,24 @@ func TestUrlRewriteMap(t *testing.T) { t.Error("Request url should not have been rewritten, was") } } + +func TestUrlRewriteMap_2(t *testing.T) { + var url string + + rewrites := UrlRewriteMap{} + + err := rewrites.Set("/abc?\\d{4}5$:/123") + if err != nil { + t.Error("Should not error on /abc?\\d{4}:/123") + } + + url = "/ab12345" + if rewrites.Rewrite(url) == url { + t.Error("Request url should have been rewritten, wasn't") + } + + url = "/ab" + if rewrites.Rewrite(url) != url { + t.Error("Request url should not have been rewritten, was") + } +} From 7f49314ec7ee9af0a9b0cdb79e0a5425f612f7c9 Mon Sep 17 00:00:00 2001 From: Jan Willies Date: Tue, 28 Oct 2014 18:12:21 +0100 Subject: [PATCH 02/16] allow matched parts in -output-http-rewrite-url --- settings_url_map.go | 2 +- settings_url_map_test.go | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/settings_url_map.go b/settings_url_map.go index 37c6173..e8fa48a 100644 --- a/settings_url_map.go +++ b/settings_url_map.go @@ -34,7 +34,7 @@ func (r *UrlRewriteMap) Set(value string) error { func (r *UrlRewriteMap) Rewrite(path string) string { for _, f := range *r { if f.src.MatchString(path) { - return f.src.ReplaceAllLiteralString(path, f.target) + return f.src.ReplaceAllString(path, f.target) } } return path diff --git a/settings_url_map_test.go b/settings_url_map_test.go index 23e3a5f..e1ce8d1 100644 --- a/settings_url_map_test.go +++ b/settings_url_map_test.go @@ -30,17 +30,17 @@ func TestUrlRewriteMap_2(t *testing.T) { rewrites := UrlRewriteMap{} - err := rewrites.Set("/abc?\\d{4}5$:/123") + err := rewrites.Set("/v1/user/([^\\/]+)/ping:/v2/user/$1/ping") if err != nil { - t.Error("Should not error on /abc?\\d{4}:/123") + t.Error("Should not error on /v1/user/([^\\/]+)/ping:/v2/user/$1/ping") } - url = "/ab12345" + url = "/v1/user/joe/ping" if rewrites.Rewrite(url) == url { t.Error("Request url should have been rewritten, wasn't") } - url = "/ab" + url = "/v1/user/ping" if rewrites.Rewrite(url) != url { t.Error("Request url should not have been rewritten, was") } From 15417894a97911e2436fedec0654dcf24cd6b56c Mon Sep 17 00:00:00 2001 From: Jan Willies Date: Tue, 28 Oct 2014 18:42:38 +0100 Subject: [PATCH 03/16] add test that explicitly checks if rewrites.Rewrite(url) == /v2/user/joe/ping --- settings_url_map_test.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/settings_url_map_test.go b/settings_url_map_test.go index e1ce8d1..ed5ed21 100644 --- a/settings_url_map_test.go +++ b/settings_url_map_test.go @@ -40,6 +40,11 @@ func TestUrlRewriteMap_2(t *testing.T) { t.Error("Request url should have been rewritten, wasn't") } + url = "/v1/user/joe/ping" + if rewrites.Rewrite(url) != "/v2/user/joe/ping" { + t.Error("Request url should have been rewritten, wasn't") + } + url = "/v1/user/ping" if rewrites.Rewrite(url) != url { t.Error("Request url should not have been rewritten, was") From eb7885cd56a4419a360fb8cc679694a1b1a5f463 Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Wed, 26 Nov 2014 16:30:17 +0500 Subject: [PATCH 04/16] Patch from @jcw9930 See https://github.com/buger/gor/pull/127#issuecomment-64538157 --- settings_url_map.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/settings_url_map.go b/settings_url_map.go index e8fa48a..5453610 100644 --- a/settings_url_map.go +++ b/settings_url_map.go @@ -34,7 +34,7 @@ func (r *UrlRewriteMap) Set(value string) error { func (r *UrlRewriteMap) Rewrite(path string) string { for _, f := range *r { if f.src.MatchString(path) { - return f.src.ReplaceAllString(path, f.target) + path = f.src.ReplaceAllString(path, f.target) } } return path From ae293320ad33fc7acb03a682f42118af2018b25c Mon Sep 17 00:00:00 2001 From: Callum Macdonald Date: Sun, 7 Dec 2014 16:20:49 +0100 Subject: [PATCH 05/16] Minor typo correction --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 2f075df..2fe9cda 100644 --- a/README.md +++ b/README.md @@ -34,7 +34,7 @@ It's recommended to use separate server for replaying traffic, but if you have e sudo gor --input-raw :80 --output-http "http://staging.com" ``` -### Guarante of replay and HTTP input +### Guarantee of replay and HTTP input Due to how traffic interception works, there is chance of missing requests. If you want guarantee that requests will be replayed you can use http input, but it will require changes in your app as well. ``` From ed5c2215f578970e43839782cac329615b299cab Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Tue, 6 Jan 2015 17:50:36 +0500 Subject: [PATCH 06/16] Fix POST requests --- output_http.go | 8 ++++++++ output_http_test.go | 15 ++++++++++++++- 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/output_http.go b/output_http.go index a905646..4295517 100644 --- a/output_http.go +++ b/output_http.go @@ -10,6 +10,7 @@ import ( "strings" "sync/atomic" "time" + "io/ioutil" ) type RedirectNotAllowed struct{} @@ -33,6 +34,13 @@ func ParseRequest(data []byte) (request *http.Request, err error) { request, err = http.ReadRequest(reader) + if request.Method == "POST" { + body, _ := ioutil.ReadAll(reader) + bodyBuf := bytes.NewBuffer(body) + request.Body = ioutil.NopCloser(bodyBuf) + request.ContentLength = int64(bodyBuf.Len()) + } + return } diff --git a/output_http_test.go b/output_http_test.go index 788878a..eec343a 100644 --- a/output_http_test.go +++ b/output_http_test.go @@ -7,11 +7,14 @@ import ( "sync" "testing" "time" + "io/ioutil" + "net/http/httputil" + _ "strings" ) func startHTTP(cb func(*http.Request)) net.Listener { handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - go cb(r) + cb(r) }) listener, _ := net.Listen("tcp", ":0") @@ -60,6 +63,16 @@ func TestHTTPOutput(t *testing.T) { t.Error("Wrong method") } + if req.Method == "POST" { + defer req.Body.Close() + body, _ := ioutil.ReadAll(req.Body) + + if string(body) != "a=1&b=2\r\n\r\n" { + buf, _ := httputil.DumpRequest(req, true) + t.Error("Wrong POST body:", string(buf)) + } + } + wg.Done() }) From b6351a8525c1f5d2e3727108153fc678755675fa Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Sun, 22 Feb 2015 14:19:48 +0500 Subject: [PATCH 07/16] Fix travis test runner --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 3294e35..d896eb4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,3 +1,3 @@ language: go go: 1.3.3 -script: sudo -E bash -c "source /etc/profile && gvm use go1.3.3 && export GOPATH=$HOME/gopath:$GOPATH && go get && GORACE='halt_on_error=1' go test -race -v" +script: sudo -E bash -c "source /etc/profile && eval '$(gimme 1.4)' && export GOPATH=$HOME/gopath:$GOPATH && go get && GORACE='halt_on_error=1' go test -race -v" From 7c68920a3bdd891df5c5515687ad6718e5f47906 Mon Sep 17 00:00:00 2001 From: Will Moss Date: Fri, 20 Feb 2015 10:55:05 -0800 Subject: [PATCH 08/16] Make TCPOutput more robust to failures This should allow for running a gor listener on a production box and bringing up and down the replay node without having to go restart the listener. --- output_tcp.go | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/output_tcp.go b/output_tcp.go index 77a5e1e..53eaa7f 100644 --- a/output_tcp.go +++ b/output_tcp.go @@ -5,6 +5,7 @@ import ( "io" "log" "net" + "time" ) type TCPOutput struct { @@ -32,11 +33,20 @@ func NewTCPOutput(address string) io.Writer { } func (o *TCPOutput) worker() { - conn, _ := o.connect(o.address) + conn, err := o.connect(o.address) + for ; err != nil; conn, err = o.connect(o.address) { + time.Sleep(2 * time.Second) + } + defer conn.Close() for { - conn.Write(<-o.buf) + _, err := conn.Write(<-o.buf) + if err != nil { + log.Println("Worker failed on write, exitings and starting new worker") + go o.worker() + break + } } } From c1c87ed4e2137a790b98935b96e859fb9383c561 Mon Sep 17 00:00:00 2001 From: Will Moss Date: Thu, 26 Feb 2015 16:07:52 -0800 Subject: [PATCH 09/16] Properly return from errors from ReadRequest --- output_http.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/output_http.go b/output_http.go index 4295517..b74f3b8 100644 --- a/output_http.go +++ b/output_http.go @@ -34,6 +34,10 @@ func ParseRequest(data []byte) (request *http.Request, err error) { request, err = http.ReadRequest(reader) + if (err != nil) { + return + } + if request.Method == "POST" { body, _ := ioutil.ReadAll(reader) bodyBuf := bytes.NewBuffer(body) From 85fbfff3084a4c3aadf21d47a156c12a4fa6cd8b Mon Sep 17 00:00:00 2001 From: Will Moss Date: Mon, 2 Mar 2015 21:47:18 -0800 Subject: [PATCH 10/16] Properly handle errors on input sockets `buf` is of length 0, so you never make it into the loop to handle the errors. This means you end up in a tight loop calling `ReadBytes` and always getting back `io.EOF` and 0 bytes. --- input_tcp.go | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/input_tcp.go b/input_tcp.go index 3f31978..7eeb488 100644 --- a/input_tcp.go +++ b/input_tcp.go @@ -62,6 +62,12 @@ func (i *TCPInput) handleConnection(conn net.Conn) { for { buf, err := reader.ReadBytes('ΒΆ') + if err == io.EOF { + return + } else if err != nil { + log.Println("Unexpected error in input tcp connection", err) + return + } buf_len := len(buf) if buf_len > 0 { new_buf_len := len(buf) - 2 @@ -69,11 +75,6 @@ func (i *TCPInput) handleConnection(conn net.Conn) { new_buf := make([]byte, new_buf_len) copy(new_buf, buf[:new_buf_len]) i.data <- new_buf - if err != nil { - if err != io.EOF { - log.Printf("error: %s\n", err) - } - } } } } From cef8b75496f59c3d350428c6e6edc86cb1dace55 Mon Sep 17 00:00:00 2001 From: Marc Falzon Date: Thu, 5 Mar 2015 21:58:52 +0100 Subject: [PATCH 11/16] Prevent crash if no permissions on port This fix prevents Gor from crashing at startup if run without proper permissions for listening on specified port. --- raw_socket_listener/listener.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/raw_socket_listener/listener.go b/raw_socket_listener/listener.go index 4bf3d0b..625a100 100644 --- a/raw_socket_listener/listener.go +++ b/raw_socket_listener/listener.go @@ -61,12 +61,13 @@ func (t *Listener) listen() { func (t *Listener) readRAWSocket() { conn, e := net.ListenPacket("ip4:tcp", t.addr) - defer conn.Close() if e != nil { log.Fatal(e) } + defer conn.Close() + buf := make([]byte, 4096*2) for { From d05bc0d5e9cb5a8762dd3ceff8f7787543a7fd28 Mon Sep 17 00:00:00 2001 From: Tobias Breitwieser Date: Fri, 13 Mar 2015 18:42:29 +0100 Subject: [PATCH 12/16] Introduce possibility to enable redirects. --- README.md | 75 ++++++++++++++++++++++----------------------- output_http.go | 16 ++++++---- output_http_test.go | 10 +++--- plugins.go | 2 +- settings.go | 2 ++ 5 files changed, 54 insertions(+), 51 deletions(-) diff --git a/README.md b/README.md index 2fe9cda..860ef35 100644 --- a/README.md +++ b/README.md @@ -43,6 +43,13 @@ sudo gor --input-http :28019 --output-http "http://staging.com" Then in your application you should send copy (e.g. like reverse proxy) all incoming requests to Gor http input. +### Following redirects +If you have a scenario where following redirects is usefull you can do it like with: + +``` +gor --input-tcp replay.local:28020 --output-http http://staging.com --output-http-redirects 10 +``` +The given example will follow up to 10 redirects per request. ## Advanced use @@ -186,59 +193,49 @@ https://github.com/buger/gor/releases `gor -h` output: ``` -cpuprofile="": write cpu profile to file - -memprofile="": write memory profile to this file - -input-dummy=[]: Used for testing outputs. Emits 'Get /' request every 1s - -input-file=[]: Read requests from file: - gor --input-file ./requests.gor --output-http staging.com - + gor --input-file ./requests.gor --output-http staging.com + -input-http=[]: Read requests from HTTP, should be explicitly sent from your application: + # Listen for http on 9000 + gor --input-http :9000 --output-http staging.com -input-raw=[]: Capture traffic from given port (use RAW sockets and require *sudo* access): - # Capture traffic from 8080 port - gor --input-raw :8080 --output-http staging.com - + # Capture traffic from 8080 port + gor --input-raw :8080 --output-http staging.com -input-tcp=[]: Used for internal communication between Gor instances. Example: - # Receive requests from other Gor instances on 28020 port, and redirect output to staging - gor --input-tcp :28020 --output-http staging.com - + # Receive requests from other Gor instances on 28020 port, and redirect output to staging + gor --input-tcp :28020 --output-http staging.com + -memprofile="": write memory profile to this file -output-dummy=[]: Used for testing inputs. Just prints data coming from inputs. - -output-file=[]: Write incoming requests to file: - gor --input-raw :80 --output-file ./requests.gor - + gor --input-raw :80 --output-file ./requests.gor -output-http=[]: Forwards incoming requests to given http address. - # Redirect all incoming requests to staging.com address - gor --input-raw :80 --output-http http://staging.com - + # Redirect all incoming requests to staging.com address + gor --input-raw :80 --output-http http://staging.com -output-http-elasticsearch="": Send request and response stats to ElasticSearch: - gor --input-raw :8080 --output-http staging.com --output-http-elasticsearch 'es_host:api_port/index_name' - + gor --input-raw :8080 --output-http staging.com --output-http-elasticsearch 'es_host:api_port/index_name' -output-http-header=[]: Inject additional headers to http reqest: - gor --input-raw :8080 --output-http staging.com --output-http-header 'User-Agent: Gor' - + gor --input-raw :8080 --output-http staging.com --output-http-header 'User-Agent: Gor' -output-http-header-filter=[]: A regexp to match a specific header against. Requests with non-matching headers will be dropped: - gor --input-raw :8080 --output-http staging.com --output-http-header-filter api-version:^v1 - + gor --input-raw :8080 --output-http staging.com --output-http-header-filter api-version:^v1 -output-http-header-hash-filter=[]: Takes a fraction of requests, consistently taking or rejecting a request based on the FNV32-1A hash of a specific header. The fraction must have a denominator that is a power of two: - gor --input-raw :8080 --output-http staging.com --output-http-header-hash-filter user-id:1/4 - + gor --input-raw :8080 --output-http staging.com --output-http-header-hash-filter user-id:1/4 + -output-http-method=[]: Whitelist of HTTP methods to replay. Anything else will be dropped: + gor --input-raw :8080 --output-http staging.com --output-http-method GET --output-http-method OPTIONS + -output-http-redirects=0: Enable how often redirects should be followed. + -output-http-rewrite-url=[]: Rewrite the requst url based on a mapping: + gor --input-raw :8080 --output-http staging.com --output-http-rewrite-url /xml_test/interface.php:/api/service.do + -output-http-stats=false: Report http output queue stats to console every 5 seconds. -output-http-url-regexp=: A regexp to match requests against. Anything else will be dropped: - gor --input-raw :8080 --output-http staging.com --output-http-url-regexp ^www. - - -output-http-workers=-1: Number of http output workers desired. Use default -1 for dynamic worker scaling. Gor will add http workers if its work queue starts getting too full and kill them . - - -output-http-stats=false: If set to `true` it gives out queuing stats for the HTTP output every 5 seconds in the form latest,mean,max,count,count/second. - + gor --input-raw :8080 --output-http staging.com --output-http-url-regexp ^www. + -output-http-workers=-1: Gor uses dynamic worker scaling by default. Enter a number to run a set number of workers. -output-tcp=[]: Used for internal communication between Gor instances. Example: - # Listen for requests on 80 port and forward them to other Gor instance on 28020 port - gor --input-raw :80 --output-tcp replay.local:28020 - - -output-tcp-stats=false: If set to `true` it gives out queuing stats for the TCP output every 5 seconds in the form latest,mean,max,count,count/second. - + # Listen for requests on 80 port and forward them to other Gor instance on 28020 port + gor --input-raw :80 --output-tcp replay.local:28020 + -output-tcp-stats=false: Report TCP output queue stats to console every 5 seconds. -split-output=false: By default each output gets same traffic. If set to `true` it splits traffic equally among all outputs. - - -output-http-rewrite-url=[]: Rewrites the url in the request based on a mapping - gor --input-raw :8080 --output-http staging.com --output-http-rewrite-url /xml_test/interface.php:/api/service.do + -stats=false: Turn on queue stats output + -verbose=false: Turn on verbose/debug output ``` ## Building from source diff --git a/output_http.go b/output_http.go index b74f3b8..eaecec6 100644 --- a/output_http.go +++ b/output_http.go @@ -4,13 +4,13 @@ import ( "bufio" "bytes" "io" + "io/ioutil" "log" "net/http" "net/url" "strings" "sync/atomic" "time" - "io/ioutil" ) type RedirectNotAllowed struct{} @@ -20,8 +20,8 @@ func (e *RedirectNotAllowed) Error() string { } // customCheckRedirect disables redirects https://github.com/buger/gor/pull/15 -func customCheckRedirect(req *http.Request, via []*http.Request) error { - if len(via) >= 0 { +func (o *HTTPOutput) customCheckRedirect(req *http.Request, via []*http.Request) error { + if len(via) >= o.redirectLimit { return new(RedirectNotAllowed) } return nil @@ -34,7 +34,7 @@ func ParseRequest(data []byte) (request *http.Request, err error) { request, err = http.ReadRequest(reader) - if (err != nil) { + if err != nil { return } @@ -55,6 +55,8 @@ type HTTPOutput struct { limit int queue chan []byte + redirectLimit int + activeWorkers int64 needWorker chan int @@ -71,7 +73,7 @@ type HTTPOutput struct { queueStats *GorStat } -func NewHTTPOutput(address string, headers HTTPHeaders, methods HTTPMethods, urlRegexp HTTPUrlRegexp, headerFilters HTTPHeaderFilters, headerHashFilters HTTPHeaderHashFilters, elasticSearchAddr string, outputHTTPUrlRewrite UrlRewriteMap) io.Writer { +func NewHTTPOutput(address string, headers HTTPHeaders, methods HTTPMethods, urlRegexp HTTPUrlRegexp, headerFilters HTTPHeaderFilters, headerHashFilters HTTPHeaderHashFilters, elasticSearchAddr string, outputHTTPUrlRewrite UrlRewriteMap, outputHTTPRedirects int) io.Writer { o := new(HTTPOutput) @@ -83,6 +85,8 @@ func NewHTTPOutput(address string, headers HTTPHeaders, methods HTTPMethods, url o.headers = headers o.methods = methods + o.redirectLimit = Settings.outputHTTPRedirects + o.urlRegexp = urlRegexp o.headerFilters = headerFilters o.headerHashFilters = headerHashFilters @@ -128,7 +132,7 @@ func (o *HTTPOutput) WorkerMaster() { func (o *HTTPOutput) Worker() { client := &http.Client{ - CheckRedirect: customCheckRedirect, + CheckRedirect: o.customCheckRedirect, } death_count := 0 diff --git a/output_http_test.go b/output_http_test.go index eec343a..a9065e9 100644 --- a/output_http_test.go +++ b/output_http_test.go @@ -2,14 +2,14 @@ package main import ( "io" + "io/ioutil" "net" "net/http" + "net/http/httputil" + _ "strings" "sync" "testing" "time" - "io/ioutil" - "net/http/httputil" - _ "strings" ) func startHTTP(cb func(*http.Request)) net.Listener { @@ -76,7 +76,7 @@ func TestHTTPOutput(t *testing.T) { wg.Done() }) - output := NewHTTPOutput(listener.Addr().String(), headers, methods, HTTPUrlRegexp{}, HTTPHeaderFilters{}, HTTPHeaderHashFilters{}, "", UrlRewriteMap{}) + output := NewHTTPOutput(listener.Addr().String(), headers, methods, HTTPUrlRegexp{}, HTTPHeaderFilters{}, HTTPHeaderHashFilters{}, "", UrlRewriteMap{}, 0) Plugins.Inputs = []io.Reader{input} Plugins.Outputs = []io.Writer{output} @@ -109,7 +109,7 @@ func BenchmarkHTTPOutput(b *testing.B) { wg.Done() }) - output := NewHTTPOutput(listener.Addr().String(), headers, methods, HTTPUrlRegexp{}, HTTPHeaderFilters{}, HTTPHeaderHashFilters{}, "", UrlRewriteMap{}) + output := NewHTTPOutput(listener.Addr().String(), headers, methods, HTTPUrlRegexp{}, HTTPHeaderFilters{}, HTTPHeaderHashFilters{}, "", UrlRewriteMap{}, 0) Plugins.Inputs = []io.Reader{input} Plugins.Outputs = []io.Writer{output} diff --git a/plugins.go b/plugins.go index 7f96ae0..20234a0 100644 --- a/plugins.go +++ b/plugins.go @@ -96,6 +96,6 @@ func InitPlugins() { } for _, options := range Settings.outputHTTP { - registerPlugin(NewHTTPOutput, options, Settings.outputHTTPHeaders, Settings.outputHTTPMethods, Settings.outputHTTPUrlRegexp, Settings.outputHTTPHeaderFilters, Settings.outputHTTPHeaderHashFilters, Settings.outputHTTPElasticSearch, Settings.outputHTTPUrlRewrite) + registerPlugin(NewHTTPOutput, options, Settings.outputHTTPHeaders, Settings.outputHTTPMethods, Settings.outputHTTPUrlRegexp, Settings.outputHTTPHeaderFilters, Settings.outputHTTPHeaderHashFilters, Settings.outputHTTPElasticSearch, Settings.outputHTTPUrlRewrite, Settings.outputHTTPRedirects) } } diff --git a/settings.go b/settings.go index f7973c1..a3011a8 100644 --- a/settings.go +++ b/settings.go @@ -40,6 +40,7 @@ type AppSettings struct { outputHTTPElasticSearch string outputHTTPWorkers int outputHTTPStats bool + outputHTTPRedirects int } var Settings AppSettings = AppSettings{} @@ -83,6 +84,7 @@ func init() { flag.StringVar(&Settings.outputHTTPElasticSearch, "output-http-elasticsearch", "", "Send request and response stats to ElasticSearch:\n\tgor --input-raw :8080 --output-http staging.com --output-http-elasticsearch 'es_host:api_port/index_name'") flag.Var(&Settings.outputHTTPUrlRewrite, "output-http-rewrite-url", "Rewrite the requst url based on a mapping:\n\tgor --input-raw :8080 --output-http staging.com --output-http-rewrite-url /xml_test/interface.php:/api/service.do") + flag.IntVar(&Settings.outputHTTPRedirects, "output-http-redirects", 0, "Enable how often redirects should be followed.") } func Debug(args ...interface{}) { From 2b497e61779d95be09f2de0b88b3a582bab94397 Mon Sep 17 00:00:00 2001 From: Martin Nowak Date: Thu, 19 Mar 2015 11:08:49 +0100 Subject: [PATCH 13/16] simplify power of 2 test --- settings_header_hash_filters.go | 6 ++---- settings_header_hash_filters_test.go | 5 +++++ 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/settings_header_hash_filters.go b/settings_header_hash_filters.go index c4147bd..76a196f 100644 --- a/settings_header_hash_filters.go +++ b/settings_header_hash_filters.go @@ -39,10 +39,8 @@ func (h *HTTPHeaderHashFilters) Set(value string) error { panic("need positive numerators and denominators, with the former less than the latter.") } - for test := den; test != 1; test /= 2 { - if test%2 == 1 { - return errors.New("must have a denominator which is a power of two.") - } + if den & (den - 1) != 0 { + return errors.New("must have a denominator which is a power of two.") } var f headerHashFilter diff --git a/settings_header_hash_filters_test.go b/settings_header_hash_filters_test.go index ec17739..f725761 100644 --- a/settings_header_hash_filters_test.go +++ b/settings_header_hash_filters_test.go @@ -23,6 +23,11 @@ func TestHTTPHeaderHashFilters(t *testing.T) { t.Error("Should error on HeaderIrrelevant:1/3") } + err = filters.Set("Pow2Denom:1/31") + if err == nil { + t.Error("Should error on Pow2Denom:1/31") + } + req := http.Request{} req.Header = make(map[string][]string) req.Header.Add("Header1", "test3414") From b2e1eea6bf8387a97691721bd6075d01bde45d18 Mon Sep 17 00:00:00 2001 From: Markus Kern Date: Sun, 22 Mar 2015 21:22:44 +0000 Subject: [PATCH 14/16] Fix alignment crash on i386 --- output_http.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/output_http.go b/output_http.go index eaecec6..e4d69aa 100644 --- a/output_http.go +++ b/output_http.go @@ -51,13 +51,17 @@ func ParseRequest(data []byte) (request *http.Request, err error) { const InitialDynamicWorkers = 10 type HTTPOutput struct { + // Keep this as first element of struct because it guarantees 64bit + // alignment. atomic.* functions crash on 32bit machines if operand is not + // aligned at 64bit. See https://github.com/golang/go/issues/599 + activeWorkers int64 + address string limit int queue chan []byte redirectLimit int - activeWorkers int64 needWorker chan int urlRegexp HTTPUrlRegexp From 390f75fd3c1ade1a8150d023e288484d40cae79d Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Fri, 26 Jun 2015 09:08:16 +0500 Subject: [PATCH 15/16] Handle Transfer-Encoding: chunked --- output_http.go | 16 +++++++++++++++- output_http_test.go | 38 +++++++++++++++++++++++++++++++++++++- test_input.go | 6 +++++- 3 files changed, 57 insertions(+), 3 deletions(-) diff --git a/output_http.go b/output_http.go index 4295517..e254ff4 100644 --- a/output_http.go +++ b/output_http.go @@ -11,6 +11,7 @@ import ( "sync/atomic" "time" "io/ioutil" + "net/http/httputil" ) type RedirectNotAllowed struct{} @@ -29,14 +30,27 @@ func customCheckRedirect(req *http.Request, via []*http.Request) error { // ParseRequest in []byte returns a http request or an error func ParseRequest(data []byte) (request *http.Request, err error) { + var body []byte + + // Test if request have Transfer-Encoding: chunked + isChunked := bytes.Contains(data, []byte(": chunked\r\n")); + buf := bytes.NewBuffer(data) reader := bufio.NewReader(buf) + // ReadRequest does not read POST bodies, we have to do it by ourseves request, err = http.ReadRequest(reader) if request.Method == "POST" { - body, _ := ioutil.ReadAll(reader) + // This works, because ReadRequest method modify buffer and strips all headers, leaving only body + if isChunked { + body, _ = ioutil.ReadAll(httputil.NewChunkedReader(reader)) + } else { + body, _ = ioutil.ReadAll(reader) + } + bodyBuf := bytes.NewBuffer(body) + request.Body = ioutil.NopCloser(bodyBuf) request.ContentLength = int64(bodyBuf.Len()) } diff --git a/output_http_test.go b/output_http_test.go index eec343a..5aef0e2 100644 --- a/output_http_test.go +++ b/output_http_test.go @@ -67,7 +67,7 @@ func TestHTTPOutput(t *testing.T) { defer req.Body.Close() body, _ := ioutil.ReadAll(req.Body) - if string(body) != "a=1&b=2\r\n\r\n" { + if string(body) != "a=1&b=2" { buf, _ := httputil.DumpRequest(req, true) t.Error("Wrong POST body:", string(buf)) } @@ -95,6 +95,42 @@ func TestHTTPOutput(t *testing.T) { close(quit) } +func TestHTTPOutputChunkedEncoding(t *testing.T) { + wg := new(sync.WaitGroup) + quit := make(chan int) + + input := NewTestInput() + + headers := HTTPHeaders{HTTPHeader{"User-Agent", "Gor"}} + methods := HTTPMethods{"GET", "PUT", "POST"} + + listener := startHTTP(func(req *http.Request) { + defer req.Body.Close() + body, _ := ioutil.ReadAll(req.Body) + + if string(body) != "Wikipedia in\r\n\r\nchunks." { + buf, _ := httputil.DumpRequest(req, true) + t.Error("Wrong POST body:", buf, body, []byte("Wikipedia in\r\n\r\nchunks.")) + } + + wg.Done() + }) + + output := NewHTTPOutput(listener.Addr().String(), headers, methods, HTTPUrlRegexp{}, HTTPHeaderFilters{}, HTTPHeaderHashFilters{}, "", UrlRewriteMap{}) + + Plugins.Inputs = []io.Reader{input} + Plugins.Outputs = []io.Writer{output} + + go Start(quit) + + wg.Add(1) + input.EmitChunkedPOST() + + wg.Wait() + + close(quit) +} + func BenchmarkHTTPOutput(b *testing.B) { wg := new(sync.WaitGroup) quit := make(chan int) diff --git a/test_input.go b/test_input.go index 9982551..694736c 100644 --- a/test_input.go +++ b/test_input.go @@ -28,7 +28,11 @@ func (i *TestInput) EmitGET() { } func (i *TestInput) EmitPOST() { - i.data <- []byte("POST /pub/WWW/ HTTP/1.1\nHost: www.w3.org\r\n\r\na=1&b=2\r\n\r\n") + i.data <- []byte("POST /pub/WWW/ HTTP/1.1\nHost: www.w3.org\r\n\r\na=1&b=2") +} + +func (i *TestInput) EmitChunkedPOST() { + i.data <- []byte("POST /pub/WWW/ HTTP/1.1\nHost: www.w3.org\nTransfer-Encoding: chunked\r\n\r\n4\r\nWiki\r\n5\r\npedia\r\ne\r\n in\r\n\r\nchunks.\r\n0\r\n\r\n") } func (i *TestInput) EmitFile() { From 451a1cd11a7c11796f47ea13176b695b8745b48d Mon Sep 17 00:00:00 2001 From: Leonid Bugaev Date: Fri, 26 Jun 2015 10:02:31 +0500 Subject: [PATCH 16/16] Update version number --- settings.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/settings.go b/settings.go index a3011a8..c5a8452 100644 --- a/settings.go +++ b/settings.go @@ -8,7 +8,7 @@ import ( ) const ( - VERSION = "0.9.2" + VERSION = "0.9.4" ) type AppSettings struct {