From d05bc0d5e9cb5a8762dd3ceff8f7787543a7fd28 Mon Sep 17 00:00:00 2001 From: Tobias Breitwieser Date: Fri, 13 Mar 2015 18:42:29 +0100 Subject: [PATCH] Introduce possibility to enable redirects. --- README.md | 75 ++++++++++++++++++++++----------------------- output_http.go | 16 ++++++---- output_http_test.go | 10 +++--- plugins.go | 2 +- settings.go | 2 ++ 5 files changed, 54 insertions(+), 51 deletions(-) diff --git a/README.md b/README.md index 2fe9cda..860ef35 100644 --- a/README.md +++ b/README.md @@ -43,6 +43,13 @@ sudo gor --input-http :28019 --output-http "http://staging.com" Then in your application you should send copy (e.g. like reverse proxy) all incoming requests to Gor http input. +### Following redirects +If you have a scenario where following redirects is usefull you can do it like with: + +``` +gor --input-tcp replay.local:28020 --output-http http://staging.com --output-http-redirects 10 +``` +The given example will follow up to 10 redirects per request. ## Advanced use @@ -186,59 +193,49 @@ https://github.com/buger/gor/releases `gor -h` output: ``` -cpuprofile="": write cpu profile to file - -memprofile="": write memory profile to this file - -input-dummy=[]: Used for testing outputs. Emits 'Get /' request every 1s - -input-file=[]: Read requests from file: - gor --input-file ./requests.gor --output-http staging.com - + gor --input-file ./requests.gor --output-http staging.com + -input-http=[]: Read requests from HTTP, should be explicitly sent from your application: + # Listen for http on 9000 + gor --input-http :9000 --output-http staging.com -input-raw=[]: Capture traffic from given port (use RAW sockets and require *sudo* access): - # Capture traffic from 8080 port - gor --input-raw :8080 --output-http staging.com - + # Capture traffic from 8080 port + gor --input-raw :8080 --output-http staging.com -input-tcp=[]: Used for internal communication between Gor instances. Example: - # Receive requests from other Gor instances on 28020 port, and redirect output to staging - gor --input-tcp :28020 --output-http staging.com - + # Receive requests from other Gor instances on 28020 port, and redirect output to staging + gor --input-tcp :28020 --output-http staging.com + -memprofile="": write memory profile to this file -output-dummy=[]: Used for testing inputs. Just prints data coming from inputs. - -output-file=[]: Write incoming requests to file: - gor --input-raw :80 --output-file ./requests.gor - + gor --input-raw :80 --output-file ./requests.gor -output-http=[]: Forwards incoming requests to given http address. - # Redirect all incoming requests to staging.com address - gor --input-raw :80 --output-http http://staging.com - + # Redirect all incoming requests to staging.com address + gor --input-raw :80 --output-http http://staging.com -output-http-elasticsearch="": Send request and response stats to ElasticSearch: - gor --input-raw :8080 --output-http staging.com --output-http-elasticsearch 'es_host:api_port/index_name' - + gor --input-raw :8080 --output-http staging.com --output-http-elasticsearch 'es_host:api_port/index_name' -output-http-header=[]: Inject additional headers to http reqest: - gor --input-raw :8080 --output-http staging.com --output-http-header 'User-Agent: Gor' - + gor --input-raw :8080 --output-http staging.com --output-http-header 'User-Agent: Gor' -output-http-header-filter=[]: A regexp to match a specific header against. Requests with non-matching headers will be dropped: - gor --input-raw :8080 --output-http staging.com --output-http-header-filter api-version:^v1 - + gor --input-raw :8080 --output-http staging.com --output-http-header-filter api-version:^v1 -output-http-header-hash-filter=[]: Takes a fraction of requests, consistently taking or rejecting a request based on the FNV32-1A hash of a specific header. The fraction must have a denominator that is a power of two: - gor --input-raw :8080 --output-http staging.com --output-http-header-hash-filter user-id:1/4 - + gor --input-raw :8080 --output-http staging.com --output-http-header-hash-filter user-id:1/4 + -output-http-method=[]: Whitelist of HTTP methods to replay. Anything else will be dropped: + gor --input-raw :8080 --output-http staging.com --output-http-method GET --output-http-method OPTIONS + -output-http-redirects=0: Enable how often redirects should be followed. + -output-http-rewrite-url=[]: Rewrite the requst url based on a mapping: + gor --input-raw :8080 --output-http staging.com --output-http-rewrite-url /xml_test/interface.php:/api/service.do + -output-http-stats=false: Report http output queue stats to console every 5 seconds. -output-http-url-regexp=: A regexp to match requests against. Anything else will be dropped: - gor --input-raw :8080 --output-http staging.com --output-http-url-regexp ^www. - - -output-http-workers=-1: Number of http output workers desired. Use default -1 for dynamic worker scaling. Gor will add http workers if its work queue starts getting too full and kill them . - - -output-http-stats=false: If set to `true` it gives out queuing stats for the HTTP output every 5 seconds in the form latest,mean,max,count,count/second. - + gor --input-raw :8080 --output-http staging.com --output-http-url-regexp ^www. + -output-http-workers=-1: Gor uses dynamic worker scaling by default. Enter a number to run a set number of workers. -output-tcp=[]: Used for internal communication between Gor instances. Example: - # Listen for requests on 80 port and forward them to other Gor instance on 28020 port - gor --input-raw :80 --output-tcp replay.local:28020 - - -output-tcp-stats=false: If set to `true` it gives out queuing stats for the TCP output every 5 seconds in the form latest,mean,max,count,count/second. - + # Listen for requests on 80 port and forward them to other Gor instance on 28020 port + gor --input-raw :80 --output-tcp replay.local:28020 + -output-tcp-stats=false: Report TCP output queue stats to console every 5 seconds. -split-output=false: By default each output gets same traffic. If set to `true` it splits traffic equally among all outputs. - - -output-http-rewrite-url=[]: Rewrites the url in the request based on a mapping - gor --input-raw :8080 --output-http staging.com --output-http-rewrite-url /xml_test/interface.php:/api/service.do + -stats=false: Turn on queue stats output + -verbose=false: Turn on verbose/debug output ``` ## Building from source diff --git a/output_http.go b/output_http.go index b74f3b8..eaecec6 100644 --- a/output_http.go +++ b/output_http.go @@ -4,13 +4,13 @@ import ( "bufio" "bytes" "io" + "io/ioutil" "log" "net/http" "net/url" "strings" "sync/atomic" "time" - "io/ioutil" ) type RedirectNotAllowed struct{} @@ -20,8 +20,8 @@ func (e *RedirectNotAllowed) Error() string { } // customCheckRedirect disables redirects https://github.com/buger/gor/pull/15 -func customCheckRedirect(req *http.Request, via []*http.Request) error { - if len(via) >= 0 { +func (o *HTTPOutput) customCheckRedirect(req *http.Request, via []*http.Request) error { + if len(via) >= o.redirectLimit { return new(RedirectNotAllowed) } return nil @@ -34,7 +34,7 @@ func ParseRequest(data []byte) (request *http.Request, err error) { request, err = http.ReadRequest(reader) - if (err != nil) { + if err != nil { return } @@ -55,6 +55,8 @@ type HTTPOutput struct { limit int queue chan []byte + redirectLimit int + activeWorkers int64 needWorker chan int @@ -71,7 +73,7 @@ type HTTPOutput struct { queueStats *GorStat } -func NewHTTPOutput(address string, headers HTTPHeaders, methods HTTPMethods, urlRegexp HTTPUrlRegexp, headerFilters HTTPHeaderFilters, headerHashFilters HTTPHeaderHashFilters, elasticSearchAddr string, outputHTTPUrlRewrite UrlRewriteMap) io.Writer { +func NewHTTPOutput(address string, headers HTTPHeaders, methods HTTPMethods, urlRegexp HTTPUrlRegexp, headerFilters HTTPHeaderFilters, headerHashFilters HTTPHeaderHashFilters, elasticSearchAddr string, outputHTTPUrlRewrite UrlRewriteMap, outputHTTPRedirects int) io.Writer { o := new(HTTPOutput) @@ -83,6 +85,8 @@ func NewHTTPOutput(address string, headers HTTPHeaders, methods HTTPMethods, url o.headers = headers o.methods = methods + o.redirectLimit = Settings.outputHTTPRedirects + o.urlRegexp = urlRegexp o.headerFilters = headerFilters o.headerHashFilters = headerHashFilters @@ -128,7 +132,7 @@ func (o *HTTPOutput) WorkerMaster() { func (o *HTTPOutput) Worker() { client := &http.Client{ - CheckRedirect: customCheckRedirect, + CheckRedirect: o.customCheckRedirect, } death_count := 0 diff --git a/output_http_test.go b/output_http_test.go index eec343a..a9065e9 100644 --- a/output_http_test.go +++ b/output_http_test.go @@ -2,14 +2,14 @@ package main import ( "io" + "io/ioutil" "net" "net/http" + "net/http/httputil" + _ "strings" "sync" "testing" "time" - "io/ioutil" - "net/http/httputil" - _ "strings" ) func startHTTP(cb func(*http.Request)) net.Listener { @@ -76,7 +76,7 @@ func TestHTTPOutput(t *testing.T) { wg.Done() }) - output := NewHTTPOutput(listener.Addr().String(), headers, methods, HTTPUrlRegexp{}, HTTPHeaderFilters{}, HTTPHeaderHashFilters{}, "", UrlRewriteMap{}) + output := NewHTTPOutput(listener.Addr().String(), headers, methods, HTTPUrlRegexp{}, HTTPHeaderFilters{}, HTTPHeaderHashFilters{}, "", UrlRewriteMap{}, 0) Plugins.Inputs = []io.Reader{input} Plugins.Outputs = []io.Writer{output} @@ -109,7 +109,7 @@ func BenchmarkHTTPOutput(b *testing.B) { wg.Done() }) - output := NewHTTPOutput(listener.Addr().String(), headers, methods, HTTPUrlRegexp{}, HTTPHeaderFilters{}, HTTPHeaderHashFilters{}, "", UrlRewriteMap{}) + output := NewHTTPOutput(listener.Addr().String(), headers, methods, HTTPUrlRegexp{}, HTTPHeaderFilters{}, HTTPHeaderHashFilters{}, "", UrlRewriteMap{}, 0) Plugins.Inputs = []io.Reader{input} Plugins.Outputs = []io.Writer{output} diff --git a/plugins.go b/plugins.go index 7f96ae0..20234a0 100644 --- a/plugins.go +++ b/plugins.go @@ -96,6 +96,6 @@ func InitPlugins() { } for _, options := range Settings.outputHTTP { - registerPlugin(NewHTTPOutput, options, Settings.outputHTTPHeaders, Settings.outputHTTPMethods, Settings.outputHTTPUrlRegexp, Settings.outputHTTPHeaderFilters, Settings.outputHTTPHeaderHashFilters, Settings.outputHTTPElasticSearch, Settings.outputHTTPUrlRewrite) + registerPlugin(NewHTTPOutput, options, Settings.outputHTTPHeaders, Settings.outputHTTPMethods, Settings.outputHTTPUrlRegexp, Settings.outputHTTPHeaderFilters, Settings.outputHTTPHeaderHashFilters, Settings.outputHTTPElasticSearch, Settings.outputHTTPUrlRewrite, Settings.outputHTTPRedirects) } } diff --git a/settings.go b/settings.go index f7973c1..a3011a8 100644 --- a/settings.go +++ b/settings.go @@ -40,6 +40,7 @@ type AppSettings struct { outputHTTPElasticSearch string outputHTTPWorkers int outputHTTPStats bool + outputHTTPRedirects int } var Settings AppSettings = AppSettings{} @@ -83,6 +84,7 @@ func init() { flag.StringVar(&Settings.outputHTTPElasticSearch, "output-http-elasticsearch", "", "Send request and response stats to ElasticSearch:\n\tgor --input-raw :8080 --output-http staging.com --output-http-elasticsearch 'es_host:api_port/index_name'") flag.Var(&Settings.outputHTTPUrlRewrite, "output-http-rewrite-url", "Rewrite the requst url based on a mapping:\n\tgor --input-raw :8080 --output-http staging.com --output-http-rewrite-url /xml_test/interface.php:/api/service.do") + flag.IntVar(&Settings.outputHTTPRedirects, "output-http-redirects", 0, "Enable how often redirects should be followed.") } func Debug(args ...interface{}) {