diff --git a/README.md b/README.md index 7272bed..dd7b819 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,4 @@ +[![Stories in Ready](https://badge.waffle.io/buger/gor.png?label=ready)](https://waffle.io/buger/gor) [![Build Status](https://travis-ci.org/buger/gor.png?branch=master)](https://travis-ci.org/buger/gor) ## About @@ -91,6 +92,16 @@ gor --input-raw :80 --output-http "http://staging.server" \ --output-http-header "Enable-Feature-X: true" ``` +## Filtering HTTP methods + +Requests not matching a specified whitelist can be filtered out. For example to strip non-nullipotent requests: + +``` +gor --input-raw :80 --output-http "http://staging.server" \ + --output-http-method GET \ + --output-http-method OPTIONS +``` + ### Basic Auth If your development or staging environment is protected by Basic Authentication then those credentials can be injected in during the replay: @@ -118,6 +129,47 @@ Feel free to ask question directly by email or by creating github issue. https://github.com/buger/gor/releases +## Command line reference +`gor -h` output: +``` + -cpuprofile="": write cpu profile to file + -memprofile="": write memory profile to this file + + -input-dummy=[]: Used for testing outputs. Emits 'Get /' request every 1s + + -input-file=[]: Read requests from file: + gor --input-file ./requests.gor --output-http staging.com + + -input-raw=[]: Capture traffic from given port (use RAW sockets and require *sudo* access): + # Capture traffic from 8080 port + gor --input-raw :8080 --output-http staging.com + + -input-tcp=[]: Used for internal communication between Gor instances. Example: + # Receive requests from other Gor instances on 28020 port, and redirect output to staging + gor --input-tcp :28020 --output-http staging.com + + -output-dummy=[]: Used for testing inputs. Just prints data coming from inputs. + + -output-file=[]: Write incoming requests to file: + gor --input-raw :80 --output-file ./requests.gor + + -output-http=[]: Forwards incoming requests to given http address. + # Redirect all incoming requests to staging.com address + gor --input-raw :80 --output-http http://staging.com + + -output-http-elasticsearch="": Send request and response stats to ElasticSearch: + gor --input-raw :8080 --output-http staging.com --output-http-elasticsearch 'es_host:api_port/index_name' + + -output-http-header=[]: Inject additional headers to http reqest: + gor --input-raw :8080 --output-http staging.com --output-http-header 'User-Agent: Gor' + + -output-tcp=[]: Used for internal communication between Gor instances. Example: + # Listen for requests on 80 port and forward them to other Gor instance on 28020 port + gor --input-raw :80 --output-tcp replay.local:28020 + + -split-output=false: By default each output gets same traffic. If set to `true` it splits traffic equally among all outputs. +``` + ## Building from source 1. Setup standard Go environment http://golang.org/doc/code.html and ensure that $GOPATH environment variable properly set. 2. `go get github.com/buger/gor`. @@ -140,6 +192,27 @@ Typical linux shell has a small open files soft limit at 1024. You can easily ra More about ulimit: http://blog.thecodingmachine.com/content/solving-too-many-open-files-exception-red5-or-any-other-application +## Tuning + +To achieve the top most performance you should tune the source server system limits: + + net.ipv4.tcp_max_tw_buckets = 65536 + net.ipv4.tcp_tw_recycle = 1 + net.ipv4.tcp_tw_reuse = 0 + net.ipv4.tcp_max_syn_backlog = 131072 + net.ipv4.tcp_syn_retries = 3 + net.ipv4.tcp_synack_retries = 3 + net.ipv4.tcp_retries1 = 3 + net.ipv4.tcp_retries2 = 8 + net.ipv4.tcp_rmem = 16384 174760 349520 + net.ipv4.tcp_wmem = 16384 131072 262144 + net.ipv4.tcp_mem = 262144 524288 1048576 + net.ipv4.tcp_max_orphans = 65536 + net.ipv4.tcp_fin_timeout = 10 + net.ipv4.tcp_low_latency = 1 + net.ipv4.tcp_syncookies = 0 + + ## Contributing 1. Fork it diff --git a/elasticsearch/elasticsearch.go b/elasticsearch/elasticsearch.go index d31735c..0eb9e32 100644 --- a/elasticsearch/elasticsearch.go +++ b/elasticsearch/elasticsearch.go @@ -21,7 +21,7 @@ type ESPlugin struct { ApiPort string Host string Index string - indexor *core.BulkIndexor + indexor *core.BulkIndexer done chan bool } @@ -80,7 +80,7 @@ func (p *ESPlugin) Init(URI string) { api.Domain = p.Host api.Port = p.ApiPort - p.indexor = core.NewBulkIndexorErrors(50, 60) + p.indexor = core.NewBulkIndexerErrors(50, 60) p.done = make(chan bool) p.indexor.Run(p.done) diff --git a/output_http.go b/output_http.go index a1dd3b9..fdc9f9a 100644 --- a/output_http.go +++ b/output_http.go @@ -43,12 +43,12 @@ type HTTPOutput struct { buf chan []byte headers HTTPHeaders + methods HTTPMethods elasticSearch *es.ESPlugin } -func NewHTTPOutput(options string, headers HTTPHeaders, elasticSearchAddr string) io.Writer { - +func NewHTTPOutput(options string, headers HTTPHeaders, methods HTTPMethods, elasticSearchAddr string) io.Writer { o := new(HTTPOutput) optionsArr := strings.Split(options, "|") @@ -60,6 +60,7 @@ func NewHTTPOutput(options string, headers HTTPHeaders, elasticSearchAddr string o.address = address o.headers = headers + o.methods = methods o.buf = make(chan []byte, 100) @@ -95,7 +96,10 @@ func (o *HTTPOutput) worker(n int) { } func (o *HTTPOutput) Write(data []byte) (n int, err error) { - o.buf <- data + buf := make([]byte, len(data)) + copy(buf, data) + + go o.sendRequest(buf) return len(data), nil } @@ -108,6 +112,14 @@ func (o *HTTPOutput) sendRequest(client *http.Client, data []byte) { return } + if len(o.methods) > 0 && !o.methods.Contains(request.Method) { + return + } + + client := &http.Client{ + CheckRedirect: customCheckRedirect, + } + // Change HOST of original request URL := o.address + request.URL.Path + "?" + request.URL.RawQuery diff --git a/output_http_test.go b/output_http_test.go index 5461df9..5733e68 100644 --- a/output_http_test.go +++ b/output_http_test.go @@ -29,11 +29,18 @@ func TestHTTPOutput(t *testing.T) { headers := HTTPHeaders{HTTPHeader{"User-Agent", "Gor"}} + methods := HTTPMethods{"GET", "PUT", "POST"} + output := NewHTTPOutput("127.0.0.1:50003", headers, methods, "") + listener := startHTTP(func(req *http.Request) { if req.Header.Get("User-Agent") != "Gor" { t.Error("Wrong header") } + if req.Method == "OPTIONS" { + t.Error("Wrong method") + } + wg.Done() }) @@ -46,8 +53,9 @@ func TestHTTPOutput(t *testing.T) { for i := 0; i < 100; i++ { wg.Add(2) - input.EmitGET() input.EmitPOST() + input.EmitOPTIONS() + input.EmitGET() } wg.Wait() diff --git a/plugins.go b/plugins.go index 3869771..e96e187 100644 --- a/plugins.go +++ b/plugins.go @@ -32,11 +32,15 @@ func InitPlugins() { Plugins.Outputs = append(Plugins.Outputs, NewTCPOutput(options)) } + for _, options := range Settings.inputFile { + Plugins.Inputs = append(Plugins.Inputs, NewFileInput(options)) + } + for _, options := range Settings.outputFile { Plugins.Outputs = append(Plugins.Outputs, NewFileOutput(options)) } for _, options := range Settings.outputHTTP { - Plugins.Outputs = append(Plugins.Outputs, NewHTTPOutput(options, Settings.outputHTTPHeaders, Settings.outputHTTPElasticSearch)) + Plugins.Outputs = append(Plugins.Outputs, NewHTTPOutput(options, Settings.outputHTTPHeaders, Settings.outputHTTPMethods, Settings.outputHTTPElasticSearch)) } } diff --git a/raw_socket_listener/listener.go b/raw_socket_listener/listener.go index 66a7f42..e73b09c 100644 --- a/raw_socket_listener/listener.go +++ b/raw_socket_listener/listener.go @@ -100,12 +100,12 @@ func (t *Listener) isIncomingDataPacket(buf []byte) bool { // Because RAW_SOCKET can't be bound to port, we have to control it by ourself if int(dest_port) == t.port { - // Check TCPPacket code for more description - flags := binary.BigEndian.Uint16(buf[12:14]) & 0x1FF + // Get the 'data offset' (size of the TCP header in 32-bit words) + dataOffset := (buf[12] & 0xF0) >> 4 // We need only packets with data inside - // TCP PSH flag indicate that packet have data inside - if (flags & TCP_PSH) != 0 { + // Check that the buffer is larger than the size of the TCP header + if len(buf) > int(dataOffset*4) { // We should create new buffer because go slices is pointers. So buffer data shoud be immutable. return true } diff --git a/settings.go b/settings.go index bd86c58..1e89e82 100644 --- a/settings.go +++ b/settings.go @@ -29,6 +29,7 @@ type AppSettings struct { outputHTTP MultiOption outputHTTPHeaders HTTPHeaders + outputHTTPMethods HTTPMethods outputHTTPElasticSearch string } @@ -60,6 +61,7 @@ func init() { flag.Var(&Settings.outputHTTP, "output-http", "Forwards incoming requests to given http address.\n\t# Redirect all incoming requests to staging.com address \n\tgor --input-raw :80 --output-http http://staging.com") flag.Var(&Settings.outputHTTPHeaders, "output-http-header", "Inject additional headers to http reqest:\n\tgor --input-raw :8080 --output-http staging.com --output-http-header 'User-Agent: Gor'") + flag.Var(&Settings.outputHTTPMethods, "output-http-method", "Whitelist of HTTP methods to replay. Anything else will be dropped:\n\tgor --input-raw :8080 --output-http staging.com --output-http-method GET --output-http-method OPTIONS") flag.StringVar(&Settings.outputHTTPElasticSearch, "output-http-elasticsearch", "", "Send request and response stats to ElasticSearch:\n\tgor --input-raw :8080 --output-http staging.com --output-http-elasticsearch 'es_host:api_port/index_name'") } diff --git a/settings_methods.go b/settings_methods.go new file mode 100644 index 0000000..82bb82c --- /dev/null +++ b/settings_methods.go @@ -0,0 +1,26 @@ +package gor + +import ( + "fmt" + "strings" +) + +type HTTPMethods []string + +func (h *HTTPMethods) String() string { + return fmt.Sprint(*h) +} + +func (h *HTTPMethods) Set(value string) error { + *h = append(*h, strings.ToUpper(value)) + return nil +} + +func (h *HTTPMethods) Contains(value string) bool { + for _, method := range *h { + if value == method { + return true + } + } + return false +} diff --git a/settings_methods_test.go b/settings_methods_test.go new file mode 100644 index 0000000..13563af --- /dev/null +++ b/settings_methods_test.go @@ -0,0 +1,24 @@ +package gor + +import ( + "testing" +) + +func TestHTTPMethods(t *testing.T) { + methods := HTTPMethods{} + + methods.Set("lower") + methods.Set("UPPER") + + if !methods.Contains("LOWER") { + t.Error("Does not contain LOWER") + } + + if !methods.Contains("UPPER") { + t.Error("Does not contain UPPER") + } + + if methods.Contains("ABSENT") { + t.Error("Does contain ABSENT") + } +} diff --git a/test_input.go b/test_input.go index 3c7cc2b..0ad6333 100644 --- a/test_input.go +++ b/test_input.go @@ -26,6 +26,10 @@ func (i *TestInput) EmitPOST() { i.data <- []byte("POST /pub/WWW/ HTTP/1.1\nHost: www.w3.org\r\n\r\na=1&b=2\r\n\r\n") } +func (i *TestInput) EmitOPTIONS() { + i.data <- []byte("OPTIONS / HTTP/1.1\nHost: www.w3.org\r\n\r\n") +} + func (i *TestInput) String() string { return "Test Input" }