From 7c68920a3bdd891df5c5515687ad6718e5f47906 Mon Sep 17 00:00:00 2001 From: Will Moss Date: Fri, 20 Feb 2015 10:55:05 -0800 Subject: [PATCH 1/7] Make TCPOutput more robust to failures This should allow for running a gor listener on a production box and bringing up and down the replay node without having to go restart the listener. --- output_tcp.go | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/output_tcp.go b/output_tcp.go index 77a5e1e..53eaa7f 100644 --- a/output_tcp.go +++ b/output_tcp.go @@ -5,6 +5,7 @@ import ( "io" "log" "net" + "time" ) type TCPOutput struct { @@ -32,11 +33,20 @@ func NewTCPOutput(address string) io.Writer { } func (o *TCPOutput) worker() { - conn, _ := o.connect(o.address) + conn, err := o.connect(o.address) + for ; err != nil; conn, err = o.connect(o.address) { + time.Sleep(2 * time.Second) + } + defer conn.Close() for { - conn.Write(<-o.buf) + _, err := conn.Write(<-o.buf) + if err != nil { + log.Println("Worker failed on write, exitings and starting new worker") + go o.worker() + break + } } } From c1c87ed4e2137a790b98935b96e859fb9383c561 Mon Sep 17 00:00:00 2001 From: Will Moss Date: Thu, 26 Feb 2015 16:07:52 -0800 Subject: [PATCH 2/7] Properly return from errors from ReadRequest --- output_http.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/output_http.go b/output_http.go index 4295517..b74f3b8 100644 --- a/output_http.go +++ b/output_http.go @@ -34,6 +34,10 @@ func ParseRequest(data []byte) (request *http.Request, err error) { request, err = http.ReadRequest(reader) + if (err != nil) { + return + } + if request.Method == "POST" { body, _ := ioutil.ReadAll(reader) bodyBuf := bytes.NewBuffer(body) From 85fbfff3084a4c3aadf21d47a156c12a4fa6cd8b Mon Sep 17 00:00:00 2001 From: Will Moss Date: Mon, 2 Mar 2015 21:47:18 -0800 Subject: [PATCH 3/7] Properly handle errors on input sockets `buf` is of length 0, so you never make it into the loop to handle the errors. This means you end up in a tight loop calling `ReadBytes` and always getting back `io.EOF` and 0 bytes. --- input_tcp.go | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/input_tcp.go b/input_tcp.go index 3f31978..7eeb488 100644 --- a/input_tcp.go +++ b/input_tcp.go @@ -62,6 +62,12 @@ func (i *TCPInput) handleConnection(conn net.Conn) { for { buf, err := reader.ReadBytes('ΒΆ') + if err == io.EOF { + return + } else if err != nil { + log.Println("Unexpected error in input tcp connection", err) + return + } buf_len := len(buf) if buf_len > 0 { new_buf_len := len(buf) - 2 @@ -69,11 +75,6 @@ func (i *TCPInput) handleConnection(conn net.Conn) { new_buf := make([]byte, new_buf_len) copy(new_buf, buf[:new_buf_len]) i.data <- new_buf - if err != nil { - if err != io.EOF { - log.Printf("error: %s\n", err) - } - } } } } From cef8b75496f59c3d350428c6e6edc86cb1dace55 Mon Sep 17 00:00:00 2001 From: Marc Falzon Date: Thu, 5 Mar 2015 21:58:52 +0100 Subject: [PATCH 4/7] Prevent crash if no permissions on port This fix prevents Gor from crashing at startup if run without proper permissions for listening on specified port. --- raw_socket_listener/listener.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/raw_socket_listener/listener.go b/raw_socket_listener/listener.go index 4bf3d0b..625a100 100644 --- a/raw_socket_listener/listener.go +++ b/raw_socket_listener/listener.go @@ -61,12 +61,13 @@ func (t *Listener) listen() { func (t *Listener) readRAWSocket() { conn, e := net.ListenPacket("ip4:tcp", t.addr) - defer conn.Close() if e != nil { log.Fatal(e) } + defer conn.Close() + buf := make([]byte, 4096*2) for { From d05bc0d5e9cb5a8762dd3ceff8f7787543a7fd28 Mon Sep 17 00:00:00 2001 From: Tobias Breitwieser Date: Fri, 13 Mar 2015 18:42:29 +0100 Subject: [PATCH 5/7] Introduce possibility to enable redirects. --- README.md | 75 ++++++++++++++++++++++----------------------- output_http.go | 16 ++++++---- output_http_test.go | 10 +++--- plugins.go | 2 +- settings.go | 2 ++ 5 files changed, 54 insertions(+), 51 deletions(-) diff --git a/README.md b/README.md index 2fe9cda..860ef35 100644 --- a/README.md +++ b/README.md @@ -43,6 +43,13 @@ sudo gor --input-http :28019 --output-http "http://staging.com" Then in your application you should send copy (e.g. like reverse proxy) all incoming requests to Gor http input. +### Following redirects +If you have a scenario where following redirects is usefull you can do it like with: + +``` +gor --input-tcp replay.local:28020 --output-http http://staging.com --output-http-redirects 10 +``` +The given example will follow up to 10 redirects per request. ## Advanced use @@ -186,59 +193,49 @@ https://github.com/buger/gor/releases `gor -h` output: ``` -cpuprofile="": write cpu profile to file - -memprofile="": write memory profile to this file - -input-dummy=[]: Used for testing outputs. Emits 'Get /' request every 1s - -input-file=[]: Read requests from file: - gor --input-file ./requests.gor --output-http staging.com - + gor --input-file ./requests.gor --output-http staging.com + -input-http=[]: Read requests from HTTP, should be explicitly sent from your application: + # Listen for http on 9000 + gor --input-http :9000 --output-http staging.com -input-raw=[]: Capture traffic from given port (use RAW sockets and require *sudo* access): - # Capture traffic from 8080 port - gor --input-raw :8080 --output-http staging.com - + # Capture traffic from 8080 port + gor --input-raw :8080 --output-http staging.com -input-tcp=[]: Used for internal communication between Gor instances. Example: - # Receive requests from other Gor instances on 28020 port, and redirect output to staging - gor --input-tcp :28020 --output-http staging.com - + # Receive requests from other Gor instances on 28020 port, and redirect output to staging + gor --input-tcp :28020 --output-http staging.com + -memprofile="": write memory profile to this file -output-dummy=[]: Used for testing inputs. Just prints data coming from inputs. - -output-file=[]: Write incoming requests to file: - gor --input-raw :80 --output-file ./requests.gor - + gor --input-raw :80 --output-file ./requests.gor -output-http=[]: Forwards incoming requests to given http address. - # Redirect all incoming requests to staging.com address - gor --input-raw :80 --output-http http://staging.com - + # Redirect all incoming requests to staging.com address + gor --input-raw :80 --output-http http://staging.com -output-http-elasticsearch="": Send request and response stats to ElasticSearch: - gor --input-raw :8080 --output-http staging.com --output-http-elasticsearch 'es_host:api_port/index_name' - + gor --input-raw :8080 --output-http staging.com --output-http-elasticsearch 'es_host:api_port/index_name' -output-http-header=[]: Inject additional headers to http reqest: - gor --input-raw :8080 --output-http staging.com --output-http-header 'User-Agent: Gor' - + gor --input-raw :8080 --output-http staging.com --output-http-header 'User-Agent: Gor' -output-http-header-filter=[]: A regexp to match a specific header against. Requests with non-matching headers will be dropped: - gor --input-raw :8080 --output-http staging.com --output-http-header-filter api-version:^v1 - + gor --input-raw :8080 --output-http staging.com --output-http-header-filter api-version:^v1 -output-http-header-hash-filter=[]: Takes a fraction of requests, consistently taking or rejecting a request based on the FNV32-1A hash of a specific header. The fraction must have a denominator that is a power of two: - gor --input-raw :8080 --output-http staging.com --output-http-header-hash-filter user-id:1/4 - + gor --input-raw :8080 --output-http staging.com --output-http-header-hash-filter user-id:1/4 + -output-http-method=[]: Whitelist of HTTP methods to replay. Anything else will be dropped: + gor --input-raw :8080 --output-http staging.com --output-http-method GET --output-http-method OPTIONS + -output-http-redirects=0: Enable how often redirects should be followed. + -output-http-rewrite-url=[]: Rewrite the requst url based on a mapping: + gor --input-raw :8080 --output-http staging.com --output-http-rewrite-url /xml_test/interface.php:/api/service.do + -output-http-stats=false: Report http output queue stats to console every 5 seconds. -output-http-url-regexp=: A regexp to match requests against. Anything else will be dropped: - gor --input-raw :8080 --output-http staging.com --output-http-url-regexp ^www. - - -output-http-workers=-1: Number of http output workers desired. Use default -1 for dynamic worker scaling. Gor will add http workers if its work queue starts getting too full and kill them . - - -output-http-stats=false: If set to `true` it gives out queuing stats for the HTTP output every 5 seconds in the form latest,mean,max,count,count/second. - + gor --input-raw :8080 --output-http staging.com --output-http-url-regexp ^www. + -output-http-workers=-1: Gor uses dynamic worker scaling by default. Enter a number to run a set number of workers. -output-tcp=[]: Used for internal communication between Gor instances. Example: - # Listen for requests on 80 port and forward them to other Gor instance on 28020 port - gor --input-raw :80 --output-tcp replay.local:28020 - - -output-tcp-stats=false: If set to `true` it gives out queuing stats for the TCP output every 5 seconds in the form latest,mean,max,count,count/second. - + # Listen for requests on 80 port and forward them to other Gor instance on 28020 port + gor --input-raw :80 --output-tcp replay.local:28020 + -output-tcp-stats=false: Report TCP output queue stats to console every 5 seconds. -split-output=false: By default each output gets same traffic. If set to `true` it splits traffic equally among all outputs. - - -output-http-rewrite-url=[]: Rewrites the url in the request based on a mapping - gor --input-raw :8080 --output-http staging.com --output-http-rewrite-url /xml_test/interface.php:/api/service.do + -stats=false: Turn on queue stats output + -verbose=false: Turn on verbose/debug output ``` ## Building from source diff --git a/output_http.go b/output_http.go index b74f3b8..eaecec6 100644 --- a/output_http.go +++ b/output_http.go @@ -4,13 +4,13 @@ import ( "bufio" "bytes" "io" + "io/ioutil" "log" "net/http" "net/url" "strings" "sync/atomic" "time" - "io/ioutil" ) type RedirectNotAllowed struct{} @@ -20,8 +20,8 @@ func (e *RedirectNotAllowed) Error() string { } // customCheckRedirect disables redirects https://github.com/buger/gor/pull/15 -func customCheckRedirect(req *http.Request, via []*http.Request) error { - if len(via) >= 0 { +func (o *HTTPOutput) customCheckRedirect(req *http.Request, via []*http.Request) error { + if len(via) >= o.redirectLimit { return new(RedirectNotAllowed) } return nil @@ -34,7 +34,7 @@ func ParseRequest(data []byte) (request *http.Request, err error) { request, err = http.ReadRequest(reader) - if (err != nil) { + if err != nil { return } @@ -55,6 +55,8 @@ type HTTPOutput struct { limit int queue chan []byte + redirectLimit int + activeWorkers int64 needWorker chan int @@ -71,7 +73,7 @@ type HTTPOutput struct { queueStats *GorStat } -func NewHTTPOutput(address string, headers HTTPHeaders, methods HTTPMethods, urlRegexp HTTPUrlRegexp, headerFilters HTTPHeaderFilters, headerHashFilters HTTPHeaderHashFilters, elasticSearchAddr string, outputHTTPUrlRewrite UrlRewriteMap) io.Writer { +func NewHTTPOutput(address string, headers HTTPHeaders, methods HTTPMethods, urlRegexp HTTPUrlRegexp, headerFilters HTTPHeaderFilters, headerHashFilters HTTPHeaderHashFilters, elasticSearchAddr string, outputHTTPUrlRewrite UrlRewriteMap, outputHTTPRedirects int) io.Writer { o := new(HTTPOutput) @@ -83,6 +85,8 @@ func NewHTTPOutput(address string, headers HTTPHeaders, methods HTTPMethods, url o.headers = headers o.methods = methods + o.redirectLimit = Settings.outputHTTPRedirects + o.urlRegexp = urlRegexp o.headerFilters = headerFilters o.headerHashFilters = headerHashFilters @@ -128,7 +132,7 @@ func (o *HTTPOutput) WorkerMaster() { func (o *HTTPOutput) Worker() { client := &http.Client{ - CheckRedirect: customCheckRedirect, + CheckRedirect: o.customCheckRedirect, } death_count := 0 diff --git a/output_http_test.go b/output_http_test.go index eec343a..a9065e9 100644 --- a/output_http_test.go +++ b/output_http_test.go @@ -2,14 +2,14 @@ package main import ( "io" + "io/ioutil" "net" "net/http" + "net/http/httputil" + _ "strings" "sync" "testing" "time" - "io/ioutil" - "net/http/httputil" - _ "strings" ) func startHTTP(cb func(*http.Request)) net.Listener { @@ -76,7 +76,7 @@ func TestHTTPOutput(t *testing.T) { wg.Done() }) - output := NewHTTPOutput(listener.Addr().String(), headers, methods, HTTPUrlRegexp{}, HTTPHeaderFilters{}, HTTPHeaderHashFilters{}, "", UrlRewriteMap{}) + output := NewHTTPOutput(listener.Addr().String(), headers, methods, HTTPUrlRegexp{}, HTTPHeaderFilters{}, HTTPHeaderHashFilters{}, "", UrlRewriteMap{}, 0) Plugins.Inputs = []io.Reader{input} Plugins.Outputs = []io.Writer{output} @@ -109,7 +109,7 @@ func BenchmarkHTTPOutput(b *testing.B) { wg.Done() }) - output := NewHTTPOutput(listener.Addr().String(), headers, methods, HTTPUrlRegexp{}, HTTPHeaderFilters{}, HTTPHeaderHashFilters{}, "", UrlRewriteMap{}) + output := NewHTTPOutput(listener.Addr().String(), headers, methods, HTTPUrlRegexp{}, HTTPHeaderFilters{}, HTTPHeaderHashFilters{}, "", UrlRewriteMap{}, 0) Plugins.Inputs = []io.Reader{input} Plugins.Outputs = []io.Writer{output} diff --git a/plugins.go b/plugins.go index 7f96ae0..20234a0 100644 --- a/plugins.go +++ b/plugins.go @@ -96,6 +96,6 @@ func InitPlugins() { } for _, options := range Settings.outputHTTP { - registerPlugin(NewHTTPOutput, options, Settings.outputHTTPHeaders, Settings.outputHTTPMethods, Settings.outputHTTPUrlRegexp, Settings.outputHTTPHeaderFilters, Settings.outputHTTPHeaderHashFilters, Settings.outputHTTPElasticSearch, Settings.outputHTTPUrlRewrite) + registerPlugin(NewHTTPOutput, options, Settings.outputHTTPHeaders, Settings.outputHTTPMethods, Settings.outputHTTPUrlRegexp, Settings.outputHTTPHeaderFilters, Settings.outputHTTPHeaderHashFilters, Settings.outputHTTPElasticSearch, Settings.outputHTTPUrlRewrite, Settings.outputHTTPRedirects) } } diff --git a/settings.go b/settings.go index f7973c1..a3011a8 100644 --- a/settings.go +++ b/settings.go @@ -40,6 +40,7 @@ type AppSettings struct { outputHTTPElasticSearch string outputHTTPWorkers int outputHTTPStats bool + outputHTTPRedirects int } var Settings AppSettings = AppSettings{} @@ -83,6 +84,7 @@ func init() { flag.StringVar(&Settings.outputHTTPElasticSearch, "output-http-elasticsearch", "", "Send request and response stats to ElasticSearch:\n\tgor --input-raw :8080 --output-http staging.com --output-http-elasticsearch 'es_host:api_port/index_name'") flag.Var(&Settings.outputHTTPUrlRewrite, "output-http-rewrite-url", "Rewrite the requst url based on a mapping:\n\tgor --input-raw :8080 --output-http staging.com --output-http-rewrite-url /xml_test/interface.php:/api/service.do") + flag.IntVar(&Settings.outputHTTPRedirects, "output-http-redirects", 0, "Enable how often redirects should be followed.") } func Debug(args ...interface{}) { From 2b497e61779d95be09f2de0b88b3a582bab94397 Mon Sep 17 00:00:00 2001 From: Martin Nowak Date: Thu, 19 Mar 2015 11:08:49 +0100 Subject: [PATCH 6/7] simplify power of 2 test --- settings_header_hash_filters.go | 6 ++---- settings_header_hash_filters_test.go | 5 +++++ 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/settings_header_hash_filters.go b/settings_header_hash_filters.go index c4147bd..76a196f 100644 --- a/settings_header_hash_filters.go +++ b/settings_header_hash_filters.go @@ -39,10 +39,8 @@ func (h *HTTPHeaderHashFilters) Set(value string) error { panic("need positive numerators and denominators, with the former less than the latter.") } - for test := den; test != 1; test /= 2 { - if test%2 == 1 { - return errors.New("must have a denominator which is a power of two.") - } + if den & (den - 1) != 0 { + return errors.New("must have a denominator which is a power of two.") } var f headerHashFilter diff --git a/settings_header_hash_filters_test.go b/settings_header_hash_filters_test.go index ec17739..f725761 100644 --- a/settings_header_hash_filters_test.go +++ b/settings_header_hash_filters_test.go @@ -23,6 +23,11 @@ func TestHTTPHeaderHashFilters(t *testing.T) { t.Error("Should error on HeaderIrrelevant:1/3") } + err = filters.Set("Pow2Denom:1/31") + if err == nil { + t.Error("Should error on Pow2Denom:1/31") + } + req := http.Request{} req.Header = make(map[string][]string) req.Header.Add("Header1", "test3414") From b2e1eea6bf8387a97691721bd6075d01bde45d18 Mon Sep 17 00:00:00 2001 From: Markus Kern Date: Sun, 22 Mar 2015 21:22:44 +0000 Subject: [PATCH 7/7] Fix alignment crash on i386 --- output_http.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/output_http.go b/output_http.go index eaecec6..e4d69aa 100644 --- a/output_http.go +++ b/output_http.go @@ -51,13 +51,17 @@ func ParseRequest(data []byte) (request *http.Request, err error) { const InitialDynamicWorkers = 10 type HTTPOutput struct { + // Keep this as first element of struct because it guarantees 64bit + // alignment. atomic.* functions crash on 32bit machines if operand is not + // aligned at 64bit. See https://github.com/golang/go/issues/599 + activeWorkers int64 + address string limit int queue chan []byte redirectLimit int - activeWorkers int64 needWorker chan int urlRegexp HTTPUrlRegexp