diff --git a/elasticsearch.go b/elasticsearch.go index 1819c9d..801e75c 100644 --- a/elasticsearch.go +++ b/elasticsearch.go @@ -26,27 +26,27 @@ type ESPlugin struct { } type ESRequestResponse struct { - ReqUrl []byte `json:"Req_URL"` - ReqMethod []byte `json:"Req_Method"` - ReqUserAgent []byte `json:"Req_User-Agent"` - ReqAcceptLanguage []byte `json:"Req_Accept-Language,omitempty"` - ReqAccept []byte `json:"Req_Accept,omitempty"` - ReqAcceptEncoding []byte `json:"Req_Accept-Encoding,omitempty"` - ReqIfModifiedSince []byte `json:"Req_If-Modified-Since,omitempty"` - ReqConnection []byte `json:"Req_Connection,omitempty"` - ReqCookies []byte `json:"Req_Cookies,omitempty"` - RespStatus []byte `json:"Resp_Status"` - RespStatusCode []byte `json:"Resp_Status-Code"` - RespProto []byte `json:"Resp_Proto,omitempty"` - RespContentLength []byte `json:"Resp_Content-Length,omitempty"` - RespContentType []byte `json:"Resp_Content-Type,omitempty"` - RespTransferEncoding []byte `json:"Resp_Transfer-Encoding,omitempty"` - RespContentEncoding []byte `json:"Resp_Content-Encoding,omitempty"` - RespExpires []byte `json:"Resp_Expires,omitempty"` - RespCacheControl []byte `json:"Resp_Cache-Control,omitempty"` - RespVary []byte `json:"Resp_Vary,omitempty"` - RespSetCookie []byte `json:"Resp_Set-Cookie,omitempty"` - Rtt int64 `json:"RTT"` + ReqUrl []byte `json:"Req_URL"` + ReqMethod []byte `json:"Req_Method"` + ReqUserAgent []byte `json:"Req_User-Agent"` + ReqAcceptLanguage []byte `json:"Req_Accept-Language,omitempty"` + ReqAccept []byte `json:"Req_Accept,omitempty"` + ReqAcceptEncoding []byte `json:"Req_Accept-Encoding,omitempty"` + ReqIfModifiedSince []byte `json:"Req_If-Modified-Since,omitempty"` + ReqConnection []byte `json:"Req_Connection,omitempty"` + ReqCookies []byte `json:"Req_Cookies,omitempty"` + RespStatus []byte `json:"Resp_Status"` + RespStatusCode []byte `json:"Resp_Status-Code"` + RespProto []byte `json:"Resp_Proto,omitempty"` + RespContentLength []byte `json:"Resp_Content-Length,omitempty"` + RespContentType []byte `json:"Resp_Content-Type,omitempty"` + RespTransferEncoding []byte `json:"Resp_Transfer-Encoding,omitempty"` + RespContentEncoding []byte `json:"Resp_Content-Encoding,omitempty"` + RespExpires []byte `json:"Resp_Expires,omitempty"` + RespCacheControl []byte `json:"Resp_Cache-Control,omitempty"` + RespVary []byte `json:"Resp_Vary,omitempty"` + RespSetCookie []byte `json:"Resp_Set-Cookie,omitempty"` + Rtt int64 `json:"RTT"` Timestamp time.Time } diff --git a/http_client.go b/http_client.go index e5f859d..3d8d5bf 100644 --- a/http_client.go +++ b/http_client.go @@ -2,12 +2,12 @@ package main import ( "crypto/tls" + "github.com/buger/gor/proto" "io" "net" "net/url" "strings" "time" - "github.com/buger/gor/proto" ) var defaultPorts = map[string]string{ @@ -17,14 +17,14 @@ var defaultPorts = map[string]string{ type HTTPClientConfig struct { FollowRedirects int - Debug bool + Debug bool } type HTTPClient struct { - baseURL *url.URL - conn net.Conn - respBuf []byte - config *HTTPClientConfig + baseURL *url.URL + conn net.Conn + respBuf []byte + config *HTTPClientConfig redirectsCount int } diff --git a/http_client_test.go b/http_client_test.go index 45a3ba9..b22eabd 100644 --- a/http_client_test.go +++ b/http_client_test.go @@ -244,4 +244,4 @@ func TestHTTPClientRedirectLimit(t *testing.T) { client.Send(GET_payload) wg.Wait() -} \ No newline at end of file +} diff --git a/http_modifier.go b/http_modifier.go index 2f1dc3d..5d2b3e3 100644 --- a/http_modifier.go +++ b/http_modifier.go @@ -1,99 +1,97 @@ package main import ( - "github.com/buger/gor/proto" - "hash/fnv" + "github.com/buger/gor/proto" + "hash/fnv" ) - type HTTPModifierConfig struct { - urlRegexp HTTPUrlRegexp - urlRewrite UrlRewriteMap - headerFilters HTTPHeaderFilters - headerHashFilters HTTPHeaderHashFilters + urlRegexp HTTPUrlRegexp + urlRewrite UrlRewriteMap + headerFilters HTTPHeaderFilters + headerHashFilters HTTPHeaderHashFilters - headers HTTPHeaders - methods HTTPMethods + headers HTTPHeaders + methods HTTPMethods } type HTTPModifier struct { - config *HTTPModifierConfig + config *HTTPModifierConfig } func NewHTTPModifier(config *HTTPModifierConfig) *HTTPModifier { - // Optimization to skip modifier completely if we do not need it - if config.urlRegexp.regexp == nil && - len(config.urlRewrite) == 0 && - len(config.headerFilters) == 0 && - len(config.headerHashFilters) == 0 && - len(config.headers) == 0 && - len(config.methods) == 0 { - return nil - } + // Optimization to skip modifier completely if we do not need it + if config.urlRegexp.regexp == nil && + len(config.urlRewrite) == 0 && + len(config.headerFilters) == 0 && + len(config.headerHashFilters) == 0 && + len(config.headers) == 0 && + len(config.methods) == 0 { + return nil + } - return &HTTPModifier{config: config} + return &HTTPModifier{config: config} } func (m *HTTPModifier) Rewrite(payload []byte) (response []byte) { - if len(m.config.methods) > 0 && !m.config.methods.Contains(proto.Method(payload)) { - return - } + if len(m.config.methods) > 0 && !m.config.methods.Contains(proto.Method(payload)) { + return + } - if m.config.urlRegexp.regexp != nil { - host, _, _, _ := proto.Header(payload, []byte("Host")) - fullPath := append(host, proto.Path(payload)...) + if m.config.urlRegexp.regexp != nil { + host, _, _, _ := proto.Header(payload, []byte("Host")) + fullPath := append(host, proto.Path(payload)...) - if !m.config.urlRegexp.regexp.Match(fullPath) { - return - } - } + if !m.config.urlRegexp.regexp.Match(fullPath) { + return + } + } - if len(m.config.headerFilters) > 0 { - for _, f := range m.config.headerFilters { - value, s, _, _ := proto.Header(payload, f.name) + if len(m.config.headerFilters) > 0 { + for _, f := range m.config.headerFilters { + value, s, _, _ := proto.Header(payload, f.name) - if s != -1 && !f.regexp.Match(value) { - return - } - } - } + if s != -1 && !f.regexp.Match(value) { + return + } + } + } - if len(m.config.headerHashFilters) > 0 { - for _, f := range m.config.headerHashFilters { - value, s, _, _ := proto.Header(payload, f.name) + if len(m.config.headerHashFilters) > 0 { + for _, f := range m.config.headerHashFilters { + value, s, _, _ := proto.Header(payload, f.name) - if s == -1 { - return - } + if s == -1 { + return + } - hasher := fnv.New32a() - hasher.Write(value) + hasher := fnv.New32a() + hasher.Write(value) - if (hasher.Sum32() % 100) >= f.percent { - return - } - } - } + if (hasher.Sum32() % 100) >= f.percent { + return + } + } + } - if len(m.config.urlRewrite) > 0 { - path := proto.Path(payload) + if len(m.config.urlRewrite) > 0 { + path := proto.Path(payload) - for _, f := range m.config.urlRewrite { - if f.src.Match(path) { - path = f.src.ReplaceAll(path, f.target) - payload = proto.SetPath(payload, path) + for _, f := range m.config.urlRewrite { + if f.src.Match(path) { + path = f.src.ReplaceAll(path, f.target) + payload = proto.SetPath(payload, path) - break - } - } - } + break + } + } + } - if len(m.config.headers) > 0 { - for _, header := range m.config.headers { - payload = proto.SetHeader(payload, []byte(header.Name), []byte(header.Value)) - } - } + if len(m.config.headers) > 0 { + for _, header := range m.config.headers { + payload = proto.SetHeader(payload, []byte(header.Name), []byte(header.Value)) + } + } - - return payload -} \ No newline at end of file + return payload +} diff --git a/http_modifier_test.go b/http_modifier_test.go index ad06719..4689aca 100644 --- a/http_modifier_test.go +++ b/http_modifier_test.go @@ -1,112 +1,111 @@ package main import ( - "testing" - "github.com/buger/gor/proto" - "bytes" + "bytes" + "github.com/buger/gor/proto" + "testing" ) func TestHTTPModifierWithoutConfig(t *testing.T) { - if NewHTTPModifier(&HTTPModifierConfig{}) != nil { - t.Error("If no config specified should not be initialized") - } + if NewHTTPModifier(&HTTPModifierConfig{}) != nil { + t.Error("If no config specified should not be initialized") + } } func TestHTTPModifierHeaderFilters(t *testing.T) { - filters := HTTPHeaderFilters{} - filters.Set("Host:^www.w3.org$") + filters := HTTPHeaderFilters{} + filters.Set("Host:^www.w3.org$") - modifier := NewHTTPModifier(&HTTPModifierConfig{ - headerFilters: filters, - }) + modifier := NewHTTPModifier(&HTTPModifierConfig{ + headerFilters: filters, + }) - payload := []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") + payload := []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") - if len(modifier.Rewrite(payload)) == 0 { - t.Error("Request should pass filters") - } + if len(modifier.Rewrite(payload)) == 0 { + t.Error("Request should pass filters") + } - filters = HTTPHeaderFilters{} - // Setting filter that not match our header - filters.Set("Host:^www.w4.org$") + filters = HTTPHeaderFilters{} + // Setting filter that not match our header + filters.Set("Host:^www.w4.org$") - modifier = NewHTTPModifier(&HTTPModifierConfig{ - headerFilters: filters, - }) + modifier = NewHTTPModifier(&HTTPModifierConfig{ + headerFilters: filters, + }) - if len(modifier.Rewrite(payload)) != 0 { - t.Error("Request should not pass filters") - } + if len(modifier.Rewrite(payload)) != 0 { + t.Error("Request should not pass filters") + } } - func TestHTTPModifierURLRewrite(t *testing.T) { - var url, new_url []byte + var url, new_url []byte - rewrites := UrlRewriteMap{} + rewrites := UrlRewriteMap{} - payload := func(url []byte) []byte { - return []byte("POST " + string(url) + " HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") - } + payload := func(url []byte) []byte { + return []byte("POST " + string(url) + " HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") + } - err := rewrites.Set("/v1/user/([^\\/]+)/ping:/v2/user/$1/ping") - if err != nil { - t.Error("Should not error on /v1/user/([^\\/]+)/ping:/v2/user/$1/ping") - } + err := rewrites.Set("/v1/user/([^\\/]+)/ping:/v2/user/$1/ping") + if err != nil { + t.Error("Should not error on /v1/user/([^\\/]+)/ping:/v2/user/$1/ping") + } - modifier := NewHTTPModifier(&HTTPModifierConfig{ - urlRewrite: rewrites, - }) + modifier := NewHTTPModifier(&HTTPModifierConfig{ + urlRewrite: rewrites, + }) - url = []byte("/v1/user/joe/ping") - if new_url = proto.Path(modifier.Rewrite(payload(url))); bytes.Equal(new_url, url) { - t.Error("Request url should have been rewritten, wasn't", string(new_url)) - } + url = []byte("/v1/user/joe/ping") + if new_url = proto.Path(modifier.Rewrite(payload(url))); bytes.Equal(new_url, url) { + t.Error("Request url should have been rewritten, wasn't", string(new_url)) + } - url = []byte("/v1/user/ping") - if new_url = proto.Path(modifier.Rewrite(payload(url))); !bytes.Equal(new_url, url) { - t.Error("Request url should have been rewritten, wasn't", string(new_url)) - } + url = []byte("/v1/user/ping") + if new_url = proto.Path(modifier.Rewrite(payload(url))); !bytes.Equal(new_url, url) { + t.Error("Request url should have been rewritten, wasn't", string(new_url)) + } } func TestHTTPModifierHeaderHashFilters(t *testing.T) { - filters := HTTPHeaderHashFilters{} - filters.Set("Header2:1/2") + filters := HTTPHeaderHashFilters{} + filters.Set("Header2:1/2") - modifier := NewHTTPModifier(&HTTPModifierConfig{ - headerHashFilters: filters, - }) + modifier := NewHTTPModifier(&HTTPModifierConfig{ + headerHashFilters: filters, + }) - payload := func(header []byte) []byte { - return []byte("POST / HTTP/1.1\r\n" + string(header) + "Content-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") - } + payload := func(header []byte) []byte { + return []byte("POST / HTTP/1.1\r\n" + string(header) + "Content-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") + } - if p := modifier.Rewrite(payload([]byte(""))); len(p) > 0 { - t.Error("Request should not pass filters, Header2 does not exist") - } + if p := modifier.Rewrite(payload([]byte(""))); len(p) > 0 { + t.Error("Request should not pass filters, Header2 does not exist") + } - if p := modifier.Rewrite(payload([]byte("Header2: 3\r\n"))); len(p) > 0 { - t.Error("Request should not pass filters, Header2 hash too high") - } + if p := modifier.Rewrite(payload([]byte("Header2: 3\r\n"))); len(p) > 0 { + t.Error("Request should not pass filters, Header2 hash too high") + } - if p := modifier.Rewrite(payload([]byte("Header2: 1\r\n"))); len(p) == 0 { - t.Error("Request should pass filters") - } + if p := modifier.Rewrite(payload([]byte("Header2: 1\r\n"))); len(p) == 0 { + t.Error("Request should pass filters") + } } func TestHTTPModifierHeaders(t *testing.T) { - headers := HTTPHeaders{} - headers.Set("Header1:1") - headers.Set("Host:localhost") + headers := HTTPHeaders{} + headers.Set("Header1:1") + headers.Set("Host:localhost") - modifier := NewHTTPModifier(&HTTPModifierConfig{ - headers: headers, - }) + modifier := NewHTTPModifier(&HTTPModifierConfig{ + headers: headers, + }) - payload := []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") - new_payload := []byte("POST /post HTTP/1.1\r\nHeader1: 1\r\nContent-Length: 7\r\nHost: localhost\r\n\r\na=1&b=2") + payload := []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") + new_payload := []byte("POST /post HTTP/1.1\r\nHeader1: 1\r\nContent-Length: 7\r\nHost: localhost\r\n\r\na=1&b=2") - if payload = modifier.Rewrite(payload); !bytes.Equal(payload, new_payload) { - t.Error("Should update request headers", string(payload)) - } + if payload = modifier.Rewrite(payload); !bytes.Equal(payload, new_payload) { + t.Error("Should update request headers", string(payload)) + } } diff --git a/output_http.go b/output_http.go index 9b77e37..e2a6c96 100644 --- a/output_http.go +++ b/output_http.go @@ -12,7 +12,7 @@ const InitialDynamicWorkers = 10 type HTTPOutputConfig struct { redirectLimit int - stats bool + stats bool workers int elasticSearch string @@ -87,7 +87,7 @@ func (o *HTTPOutput) WorkerMaster() { func (o *HTTPOutput) Worker() { client := NewHTTPClient(o.address, &HTTPClientConfig{ FollowRedirects: o.config.redirectLimit, - Debug: o.config.Debug, + Debug: o.config.Debug, }) death_count := 0 @@ -151,7 +151,7 @@ func (o *HTTPOutput) sendRequest(client *HTTPClient, request []byte) { } if o.elasticSearch != nil { - o.elasticSearch.ResponseAnalyze(request, resp, start, stop) + o.elasticSearch.ResponseAnalyze(request, resp, start, stop) } } diff --git a/settings.go b/settings.go index 411ca62..3b6a7d0 100644 --- a/settings.go +++ b/settings.go @@ -29,11 +29,11 @@ type AppSettings struct { inputRAW MultiOption - inputHTTP MultiOption - outputHTTP MultiOption + inputHTTP MultiOption + outputHTTP MultiOption outputHTTPConfig HTTPOutputConfig - modifierConfig HTTPModifierConfig + modifierConfig HTTPModifierConfig } var Settings AppSettings = AppSettings{} diff --git a/settings_header_filters.go b/settings_header_filters.go index 7a017a6..bf10aaa 100644 --- a/settings_header_filters.go +++ b/settings_header_filters.go @@ -31,4 +31,4 @@ func (h *HTTPHeaderFilters) Set(value string) error { *h = append(*h, headerFilter{name: []byte(valArr[0]), regexp: r}) return nil -} \ No newline at end of file +} diff --git a/settings_header_filters_test.go b/settings_header_filters_test.go index 499f910..bbffecb 100644 --- a/settings_header_filters_test.go +++ b/settings_header_filters_test.go @@ -1,25 +1,25 @@ package main import ( - "testing" + "testing" ) func TestHTTPHeaderFilters(t *testing.T) { - filters := HTTPHeaderFilters{} + filters := HTTPHeaderFilters{} - err := filters.Set("Header1:^$") - if err != nil { - t.Error("Should not error on Header1:^$") - } + err := filters.Set("Header1:^$") + if err != nil { + t.Error("Should not error on Header1:^$") + } - err = filters.Set("Header2:^:$") - if err != nil { - t.Error("Should not error on Header2:^:$") - } + err = filters.Set("Header2:^:$") + if err != nil { + t.Error("Should not error on Header2:^:$") + } - // Missing colon - err = filters.Set("Header3-^$") - if err == nil { - t.Error("Should error on Header2:^:$") - } -} \ No newline at end of file + // Missing colon + err = filters.Set("Header3-^$") + if err == nil { + t.Error("Should error on Header2:^:$") + } +} diff --git a/settings_header_hash_filters.go b/settings_header_hash_filters.go index 5c848fe..e8a2072 100644 --- a/settings_header_hash_filters.go +++ b/settings_header_hash_filters.go @@ -39,4 +39,4 @@ func (h *HTTPHeaderHashFilters) Set(value string) error { *h = append(*h, f) return nil -} \ No newline at end of file +} diff --git a/settings_methods.go b/settings_methods.go index cd5a977..8d84521 100644 --- a/settings_methods.go +++ b/settings_methods.go @@ -1,9 +1,9 @@ package main import ( + "bytes" "fmt" "strings" - "bytes" ) type HTTPMethods [][]byte diff --git a/settings_url_map.go b/settings_url_map.go index d07a313..d90ac6f 100644 --- a/settings_url_map.go +++ b/settings_url_map.go @@ -27,6 +27,6 @@ func (r *UrlRewriteMap) Set(value string) error { if err != nil { return err } - *r = append(*r, urlRewrite{src: regexp, target: []byte(valArr[1]) }) + *r = append(*r, urlRewrite{src: regexp, target: []byte(valArr[1])}) return nil -} \ No newline at end of file +} diff --git a/settings_url_map_test.go b/settings_url_map_test.go index de7ad7d..b822387 100644 --- a/settings_url_map_test.go +++ b/settings_url_map_test.go @@ -1,18 +1,18 @@ package main import ( - "testing" + "testing" ) func TestUrlRewriteMap(t *testing.T) { - var err error - rewrites := UrlRewriteMap{} + var err error + rewrites := UrlRewriteMap{} - if err = rewrites.Set("/v1/user/([^\\/]+)/ping:/v2/user/$1/ping"); err != nil { - t.Error("Should set mapping", err) - } + if err = rewrites.Set("/v1/user/([^\\/]+)/ping:/v2/user/$1/ping"); err != nil { + t.Error("Should set mapping", err) + } - if err = rewrites.Set("/v1/user/([^\\/]+)/ping"); err == nil { - t.Error("Should not set mapping without :") - } -} \ No newline at end of file + if err = rewrites.Set("/v1/user/([^\\/]+)/ping"); err == nil { + t.Error("Should not set mapping without :") + } +} diff --git a/settings_url_regexp.go b/settings_url_regexp.go index bea2003..e676860 100644 --- a/settings_url_regexp.go +++ b/settings_url_regexp.go @@ -19,4 +19,4 @@ func (r *HTTPUrlRegexp) Set(value string) error { regexp, err := regexp.Compile(value) r.regexp = regexp return err -} \ No newline at end of file +}