diff --git a/http_client.go b/http_client.go index 50237ec..4e44f77 100644 --- a/http_client.go +++ b/http_client.go @@ -40,7 +40,6 @@ func NewHTTPClient(baseURL string, config *HTTPClientConfig) *HTTPClient { u.Host += ":" + defaultPorts[u.Scheme] } - client := new(HTTPClient) client.baseURL = u.String() client.host = u.Host diff --git a/http_modifier.go b/http_modifier.go index 3aa0476..5023008 100644 --- a/http_modifier.go +++ b/http_modifier.go @@ -11,7 +11,8 @@ type HTTPModifier struct { func NewHTTPModifier(config *HTTPModifierConfig) *HTTPModifier { // Optimization to skip modifier completely if we do not need it - if config.urlRegexp.regexp == nil && + if len(config.urlRegexp) == 0 && + len(config.urlNegativeRegexp) == 0 && len(config.urlRewrite) == 0 && len(config.headerFilters) == 0 && len(config.headerHashFilters) == 0 && @@ -29,21 +30,29 @@ func (m *HTTPModifier) Rewrite(payload []byte) (response []byte) { return } - if m.config.urlRegexp.regexp != nil { - host, _, _, _ := proto.Header(payload, []byte("Host")) - fullPath := append(host, proto.Path(payload)...) + if len(m.config.urlRegexp) > 0 { + path := proto.Path(payload) - if !m.config.urlRegexp.regexp.Match(fullPath) { + matched := false + + for _, f := range m.config.urlRegexp { + if f.regexp.Match(path) { + matched = true + } + } + + if !matched { return } } - if m.config.urlNegativeRegexp.regexp != nil { - host, _, _, _ := proto.Header(payload, []byte("Host")) - fullPath := append(host, proto.Path(payload)...) + if len(m.config.urlNegativeRegexp) > 0 { + path := proto.Path(payload) - if m.config.urlNegativeRegexp.regexp.Match(fullPath) { - return + for _, f := range m.config.urlNegativeRegexp { + if f.regexp.Match(path) { + return + } } } diff --git a/http_modifier_settings.go b/http_modifier_settings.go index a3a9148..cfbc82e 100644 --- a/http_modifier_settings.go +++ b/http_modifier_settings.go @@ -1,125 +1,124 @@ package main import ( - "errors" - "fmt" - "regexp" - "strings" - "strconv" - "bytes" + "bytes" + "errors" + "fmt" + "regexp" + "strconv" + "strings" ) type HTTPModifierConfig struct { - urlNegativeRegexp HTTPUrlRegexp - urlRegexp HTTPUrlRegexp - urlRewrite UrlRewriteMap - headerFilters HTTPHeaderFilters - headerHashFilters HTTPHashFilters - paramHashFilters HTTPHashFilters + urlNegativeRegexp HTTPUrlRegexp + urlRegexp HTTPUrlRegexp + urlRewrite UrlRewriteMap + headerFilters HTTPHeaderFilters + headerHashFilters HTTPHashFilters + paramHashFilters HTTPHashFilters - headers HTTPHeaders - methods HTTPMethods + headers HTTPHeaders + methods HTTPMethods } // // Handling of --http-allow-header options // type headerFilter struct { - name []byte - regexp *regexp.Regexp + name []byte + regexp *regexp.Regexp } type HTTPHeaderFilters []headerFilter func (h *HTTPHeaderFilters) String() string { - return fmt.Sprint(*h) + return fmt.Sprint(*h) } func (h *HTTPHeaderFilters) Set(value string) error { - valArr := strings.SplitN(value, ":", 2) - if len(valArr) < 2 { - return errors.New("need both header and value, colon-delimited (ex. user_id:^169$).") - } - r, err := regexp.Compile(valArr[1]) - if err != nil { - return err - } + valArr := strings.SplitN(value, ":", 2) + if len(valArr) < 2 { + return errors.New("need both header and value, colon-delimited (ex. user_id:^169$).") + } + r, err := regexp.Compile(valArr[1]) + if err != nil { + return err + } - *h = append(*h, headerFilter{name: []byte(valArr[0]), regexp: r}) + *h = append(*h, headerFilter{name: []byte(valArr[0]), regexp: r}) - return nil + return nil } // // Handling of --http-allow-header-hash and --http-allow-param-hash options // type hashFilter struct { - name []byte - percent uint32 + name []byte + percent uint32 } type HTTPHashFilters []hashFilter func (h *HTTPHashFilters) String() string { - return fmt.Sprint(*h) + return fmt.Sprint(*h) } func (h *HTTPHashFilters) Set(value string) error { - valArr := strings.SplitN(value, ":", 2) - if len(valArr) < 2 { - return errors.New("need both header and value, colon-delimited (ex. user_id:50%).") - } + valArr := strings.SplitN(value, ":", 2) + if len(valArr) < 2 { + return errors.New("need both header and value, colon-delimited (ex. user_id:50%).") + } - f := hashFilter{ name: []byte(valArr[0]) } + f := hashFilter{name: []byte(valArr[0])} - if strings.Contains(valArr[1], "%") { - p, _ := strconv.ParseInt(valArr[1][:len(valArr[1])-1], 0, 0) - f.percent = uint32(p) - } else if strings.Contains(valArr[1], "/") { - // DEPRECATED format - var num, den uint64 + if strings.Contains(valArr[1], "%") { + p, _ := strconv.ParseInt(valArr[1][:len(valArr[1])-1], 0, 0) + f.percent = uint32(p) + } else if strings.Contains(valArr[1], "/") { + // DEPRECATED format + var num, den uint64 - fracArr := strings.Split(valArr[1], "/") - num, _ = strconv.ParseUint(fracArr[0], 10, 64) - den, _ = strconv.ParseUint(fracArr[1], 10, 64) + fracArr := strings.Split(valArr[1], "/") + num, _ = strconv.ParseUint(fracArr[0], 10, 64) + den, _ = strconv.ParseUint(fracArr[1], 10, 64) - f.percent = uint32((float64(num) / float64(den)) * 100) - } else { - return errors.New("Value should be percent and contain '%'") - } + f.percent = uint32((float64(num) / float64(den)) * 100) + } else { + return errors.New("Value should be percent and contain '%'") + } - *h = append(*h, f) + *h = append(*h, f) - return nil + return nil } - // // Handling of --http-set-header option // type HTTPHeaders []HTTPHeader type HTTPHeader struct { - Name string - Value string + Name string + Value string } func (h *HTTPHeaders) String() string { - return fmt.Sprint(*h) + return fmt.Sprint(*h) } func (h *HTTPHeaders) Set(value string) error { - v := strings.SplitN(value, ":", 2) - if len(v) != 2 { - return errors.New("Expected `Key: Value`") - } + v := strings.SplitN(value, ":", 2) + if len(v) != 2 { + return errors.New("Expected `Key: Value`") + } - header := HTTPHeader{ - strings.TrimSpace(v[0]), - strings.TrimSpace(v[1]), - } + header := HTTPHeader{ + strings.TrimSpace(v[0]), + strings.TrimSpace(v[1]), + } - *h = append(*h, header) - return nil + *h = append(*h, header) + return nil } // @@ -128,66 +127,67 @@ func (h *HTTPHeaders) Set(value string) error { type HTTPMethods [][]byte func (h *HTTPMethods) String() string { - return fmt.Sprint(*h) + return fmt.Sprint(*h) } func (h *HTTPMethods) Set(value string) error { - *h = append(*h, []byte(value)) - return nil + *h = append(*h, []byte(value)) + return nil } func (h *HTTPMethods) Contains(value []byte) bool { - for _, method := range *h { - if bytes.Equal(value, method) { - return true - } - } - return false + for _, method := range *h { + if bytes.Equal(value, method) { + return true + } + } + return false } // // Handling of --http-rewrite-url option // type urlRewrite struct { - src *regexp.Regexp - target []byte + src *regexp.Regexp + target []byte } type UrlRewriteMap []urlRewrite func (r *UrlRewriteMap) String() string { - return fmt.Sprint(*r) + return fmt.Sprint(*r) } func (r *UrlRewriteMap) Set(value string) error { - valArr := strings.SplitN(value, ":", 2) - if len(valArr) < 2 { - return errors.New("need both src and target, colon-delimited (ex. /a:/b).") - } - regexp, err := regexp.Compile(valArr[0]) - if err != nil { - return err - } - *r = append(*r, urlRewrite{src: regexp, target: []byte(valArr[1])}) - return nil + valArr := strings.SplitN(value, ":", 2) + if len(valArr) < 2 { + return errors.New("need both src and target, colon-delimited (ex. /a:/b).") + } + regexp, err := regexp.Compile(valArr[0]) + if err != nil { + return err + } + *r = append(*r, urlRewrite{src: regexp, target: []byte(valArr[1])}) + return nil } // // Handling of --http-allow-url option // -type HTTPUrlRegexp struct { - regexp *regexp.Regexp +type urlRegexp struct { + regexp *regexp.Regexp } +type HTTPUrlRegexp []urlRegexp + func (r *HTTPUrlRegexp) String() string { - if r.regexp == nil { - return "" - } - return r.regexp.String() + return fmt.Sprint(*r) } func (r *HTTPUrlRegexp) Set(value string) error { - regexp, err := regexp.Compile(value) - r.regexp = regexp - return err + regexp, err := regexp.Compile(value) + + *r = append(*r, urlRegexp{regexp: regexp}) + + return err } diff --git a/http_modifier_settings_test.go b/http_modifier_settings_test.go index 1298115..5e135b6 100644 --- a/http_modifier_settings_test.go +++ b/http_modifier_settings_test.go @@ -1,81 +1,80 @@ package main import ( - "testing" + "testing" ) func TestHTTPHeaderFilters(t *testing.T) { - filters := HTTPHeaderFilters{} + filters := HTTPHeaderFilters{} - err := filters.Set("Header1:^$") - if err != nil { - t.Error("Should not error on Header1:^$") - } + err := filters.Set("Header1:^$") + if err != nil { + t.Error("Should not error on Header1:^$") + } - err = filters.Set("Header2:^:$") - if err != nil { - t.Error("Should not error on Header2:^:$") - } + err = filters.Set("Header2:^:$") + if err != nil { + t.Error("Should not error on Header2:^:$") + } - // Missing colon - err = filters.Set("Header3-^$") - if err == nil { - t.Error("Should error on Header2:^:$") - } + // Missing colon + err = filters.Set("Header3-^$") + if err == nil { + t.Error("Should error on Header2:^:$") + } } func TestHTTPHashFilters(t *testing.T) { - filters := HTTPHashFilters{} + filters := HTTPHashFilters{} - err := filters.Set("Header1:1/2") - if err != nil { - t.Error("Should support old syntax") - } + err := filters.Set("Header1:1/2") + if err != nil { + t.Error("Should support old syntax") + } - if filters[0].percent != 50 { - t.Error("Wrong percentage", filters[0].percent) - } + if filters[0].percent != 50 { + t.Error("Wrong percentage", filters[0].percent) + } - err = filters.Set("Header2:1") - if err == nil { - t.Error("Should error on Header2 because no % symbol") - } + err = filters.Set("Header2:1") + if err == nil { + t.Error("Should error on Header2 because no % symbol") + } - err = filters.Set("Header2:10%") - if err != nil { - t.Error("Should pass") - } + err = filters.Set("Header2:10%") + if err != nil { + t.Error("Should pass") + } - if filters[1].percent != 10 { - t.Error("Wrong percentage", filters[1].percent) - } + if filters[1].percent != 10 { + t.Error("Wrong percentage", filters[1].percent) + } } func TestHTTPMethods(t *testing.T) { - methods := HTTPMethods{} + methods := HTTPMethods{} - methods.Set("GET") - methods.Set("POST") + methods.Set("GET") + methods.Set("POST") - if !methods.Contains([]byte("GET")) { - t.Error("Does not contain GET") - } + if !methods.Contains([]byte("GET")) { + t.Error("Does not contain GET") + } - if !methods.Contains([]byte("POST")) { - t.Error("Does not contain POST") - } + if !methods.Contains([]byte("POST")) { + t.Error("Does not contain POST") + } } func TestUrlRewriteMap(t *testing.T) { - var err error - rewrites := UrlRewriteMap{} + var err error + rewrites := UrlRewriteMap{} - if err = rewrites.Set("/v1/user/([^\\/]+)/ping:/v2/user/$1/ping"); err != nil { - t.Error("Should set mapping", err) - } + if err = rewrites.Set("/v1/user/([^\\/]+)/ping:/v2/user/$1/ping"); err != nil { + t.Error("Should set mapping", err) + } - if err = rewrites.Set("/v1/user/([^\\/]+)/ping"); err == nil { - t.Error("Should not set mapping without :") - } + if err = rewrites.Set("/v1/user/([^\\/]+)/ping"); err == nil { + t.Error("Should not set mapping without :") + } } - diff --git a/http_modifier_test.go b/http_modifier_test.go index 80a3bb3..55d9a19 100644 --- a/http_modifier_test.go +++ b/http_modifier_test.go @@ -93,7 +93,6 @@ func TestHTTPModifierHeaderHashFilters(t *testing.T) { } } - func TestHTTPModifierParamHashFilters(t *testing.T) { filters := HTTPHashFilters{} filters.Set("user_id:1/2") @@ -135,3 +134,55 @@ func TestHTTPModifierHeaders(t *testing.T) { t.Error("Should update request headers", string(payload)) } } + +func TestHTTPModifierURLRegexp(t *testing.T) { + filters := HTTPUrlRegexp{} + filters.Set("/v1/app") + filters.Set("/v1/api") + + modifier := NewHTTPModifier(&HTTPModifierConfig{ + urlRegexp: filters, + }) + + payload := func(url string) []byte { + return []byte("POST " + url + " HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") + } + + if len(modifier.Rewrite(payload("/v1/app/test"))) == 0 { + t.Error("Should pass url") + } + + if len(modifier.Rewrite(payload("/v1/api/test"))) == 0 { + t.Error("Should pass url") + } + + if len(modifier.Rewrite(payload("/other"))) > 0 { + t.Error("Should not pass url") + } +} + +func TestHTTPModifierURLNegativeRegexp(t *testing.T) { + filters := HTTPUrlRegexp{} + filters.Set("/restricted1") + filters.Set("/some/restricted2") + + modifier := NewHTTPModifier(&HTTPModifierConfig{ + urlNegativeRegexp: filters, + }) + + payload := func(url string) []byte { + return []byte("POST " + url + " HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") + } + + if len(modifier.Rewrite(payload("/v1/app/test"))) == 0 { + t.Error("Should pass url") + } + + if len(modifier.Rewrite(payload("/restricted1"))) > 0 { + t.Error("Should not pass url") + } + + if len(modifier.Rewrite(payload("/some/restricted2"))) > 0 { + t.Error("Should not pass url") + } +} diff --git a/input_tcp.go b/input_tcp.go index 2e26c62..27b7de6 100644 --- a/input_tcp.go +++ b/input_tcp.go @@ -2,10 +2,10 @@ package main import ( "bufio" - "log" - "net" "encoding/hex" "fmt" + "log" + "net" "os" ) diff --git a/input_tcp_test.go b/input_tcp_test.go index 0f76e4c..27c00fc 100644 --- a/input_tcp_test.go +++ b/input_tcp_test.go @@ -1,12 +1,12 @@ package main import ( + "encoding/hex" "io" "log" "net" "sync" "testing" - "encoding/hex" ) func TestTCPInput(t *testing.T) { @@ -40,7 +40,7 @@ func TestTCPInput(t *testing.T) { for i := 0; i < 100; i++ { wg.Add(1) - encoded := make([]byte, len(msg)*2 + 1) + encoded := make([]byte, len(msg)*2+1) hex.Encode(encoded, msg) conn.Write(append(encoded, '\n')) } diff --git a/output_tcp.go b/output_tcp.go index fb06025..f6f0e9d 100644 --- a/output_tcp.go +++ b/output_tcp.go @@ -1,12 +1,12 @@ package main import ( + "encoding/hex" "fmt" "io" "log" "net" "time" - "encoding/hex" ) type TCPOutput struct { @@ -53,7 +53,7 @@ func (o *TCPOutput) worker() { func (o *TCPOutput) Write(data []byte) (n int, err error) { // Hex encoding always 2x number of bytes - encoded := make([]byte, len(data)*2 + 1) + encoded := make([]byte, len(data)*2+1) hex.Encode(encoded, data) o.buf <- append(encoded, '\n') diff --git a/output_tcp_test.go b/output_tcp_test.go index e42f14a..84c66b8 100644 --- a/output_tcp_test.go +++ b/output_tcp_test.go @@ -2,12 +2,12 @@ package main import ( "bufio" + "encoding/hex" "io" "log" "net" "sync" "testing" - "encoding/hex" ) func TestTCPOutput(t *testing.T) { diff --git a/settings.go b/settings.go index c39a2a1..8fd2127 100644 --- a/settings.go +++ b/settings.go @@ -23,7 +23,6 @@ func (h *MultiOption) Set(value string) error { return nil } - type AppSettings struct { verbose bool stats bool @@ -90,24 +89,20 @@ func init() { flag.Var(&Settings.modifierConfig.headers, "http-set-header", "Inject additional headers to http reqest:\n\tgor --input-raw :8080 --output-http staging.com --http-set-header 'User-Agent: Gor'") flag.Var(&Settings.modifierConfig.headers, "output-http-header", "WARNING: `--output-http-header` DEPRECATED, use `--http-set-header` instead") - flag.Var(&Settings.modifierConfig.methods, "http-allow-method", "Whitelist of HTTP methods to replay. Anything else will be dropped:\n\tgor --input-raw :8080 --output-http staging.com --http-allow-method GET --http-allow-method OPTIONS") flag.Var(&Settings.modifierConfig.methods, "output-http-method", "WARNING: `--output-http-method` DEPRECATED, use `--http-allow-method` instead") - flag.Var(&Settings.modifierConfig.urlRegexp, "http-allow-url", "A regexp to match requests against. Filter get matched agains full url with domain. Anything else will be dropped:\n\t gor --input-raw :8080 --output-http staging.com --http-allow-url ^www.") flag.Var(&Settings.modifierConfig.urlRegexp, "output-http-url-regexp", "WARNING: `--output-http-url-regexp` DEPRECATED, use `--http-allow-url` instead") flag.Var(&Settings.modifierConfig.urlNegativeRegexp, "http-diallow-url", "A regexp to match requests against. Filter get matched agains full url with domain. Anything else will be dropped:\n\t gor --input-raw :8080 --output-http staging.com --http-disallow-url ^www.") - flag.Var(&Settings.modifierConfig.urlRewrite, "http-rewrite-url", "Rewrite the requst url based on a mapping:\n\tgor --input-raw :8080 --output-http staging.com --http-rewrite-url /v1/user/([^\\/]+)/ping:/v2/user/$1/ping") flag.Var(&Settings.modifierConfig.urlRewrite, "output-http-rewrite-url", "WARNING: `--output-http-rewrite-url` DEPRECATED, use `--http-rewrite-url` instead") flag.Var(&Settings.modifierConfig.headerFilters, "http-allow-header", "A regexp to match a specific header against. Requests with non-matching headers will be dropped:\n\t gor --input-raw :8080 --output-http staging.com --http-allow-header api-version:^v1") flag.Var(&Settings.modifierConfig.headerFilters, "output-http-header-filter", "WARNING: `--output-http-header-filter` DEPRECATED, use `--http-allow-header` instead") - flag.Var(&Settings.modifierConfig.headerHashFilters, "http-allow-header-hash", "Takes a fraction of requests, consistently taking or rejecting a request based on the FNV32-1A hash of a specific header:\n\t gor --input-raw :8080 --output-http staging.com --http-allow-header-hash user-id:25%") flag.Var(&Settings.modifierConfig.headerHashFilters, "output-http-header-hash-filter", "WARNING: `output-http-header-hash-filter` DEPRECATED, use `--http-allow-header-hash` instead")