github/goreplay
4
0
Fork 0
mirror of https://github.com/buger/goreplay.git synced 2024-04-21 12:32:02 +00:00
Code Issues Releases Wiki Activity

Allow multiple url regexp and apply fmt

Browse Source
This commit is contained in:
Leonid Bugaev
2015-07-09 16:05:00 +05:00
parent 1ce10249be
commit 29035d4518
10 changed files with 224 additions and 171 deletions
Download Patch File Download Diff File Expand all files Collapse all files
http_client.go
-1
View File
@@ -40,7 +40,6 @@ func NewHTTPClient(baseURL string, config *HTTPClientConfig) *HTTPClient {
u.Host += ":" + defaultPorts[u.Scheme]
}
client := new(HTTPClient)
client.baseURL = u.String()
client.host = u.Host
http_modifier.go
+19 -10
View File
@@ -11,7 +11,8 @@ type HTTPModifier struct {
func NewHTTPModifier(config *HTTPModifierConfig) *HTTPModifier {
// Optimization to skip modifier completely if we do not need it
if config.urlRegexp.regexp == nil &&
if len(config.urlRegexp) == 0 &&
len(config.urlNegativeRegexp) == 0 &&
len(config.urlRewrite) == 0 &&
len(config.headerFilters) == 0 &&
len(config.headerHashFilters) == 0 &&
@@ -29,21 +30,29 @@ func (m *HTTPModifier) Rewrite(payload []byte) (response []byte) {
return
}
if m.config.urlRegexp.regexp != nil {
host, _, _, _ := proto.Header(payload, []byte("Host"))
fullPath := append(host, proto.Path(payload)...)
if len(m.config.urlRegexp) > 0 {
path := proto.Path(payload)
if !m.config.urlRegexp.regexp.Match(fullPath) {
matched := false
for _, f := range m.config.urlRegexp {
if f.regexp.Match(path) {
matched = true
}
}
if !matched {
return
}
}
if m.config.urlNegativeRegexp.regexp != nil {
host, _, _, _ := proto.Header(payload, []byte("Host"))
fullPath := append(host, proto.Path(payload)...)
if len(m.config.urlNegativeRegexp) > 0 {
path := proto.Path(payload)
if m.config.urlNegativeRegexp.regexp.Match(fullPath) {
return
for _, f := range m.config.urlNegativeRegexp {
if f.regexp.Match(path) {
return
}
}
}
http_modifier_settings.go
+95 -95
View File
@@ -1,125 +1,124 @@
package main
import (
"errors"
"fmt"
"regexp"
"strings"
"strconv"
"bytes"
"bytes"
"errors"
"fmt"
"regexp"
"strconv"
"strings"
)
type HTTPModifierConfig struct {
urlNegativeRegexp HTTPUrlRegexp
urlRegexp HTTPUrlRegexp
urlRewrite UrlRewriteMap
headerFilters HTTPHeaderFilters
headerHashFilters HTTPHashFilters
paramHashFilters HTTPHashFilters
urlNegativeRegexp HTTPUrlRegexp
urlRegexp HTTPUrlRegexp
urlRewrite UrlRewriteMap
headerFilters HTTPHeaderFilters
headerHashFilters HTTPHashFilters
paramHashFilters HTTPHashFilters
headers HTTPHeaders
methods HTTPMethods
headers HTTPHeaders
methods HTTPMethods
}
//
// Handling of --http-allow-header options
//
type headerFilter struct {
name []byte
regexp *regexp.Regexp
name []byte
regexp *regexp.Regexp
}
type HTTPHeaderFilters []headerFilter
func (h *HTTPHeaderFilters) String() string {
return fmt.Sprint(*h)
return fmt.Sprint(*h)
}
func (h *HTTPHeaderFilters) Set(value string) error {
valArr := strings.SplitN(value, ":", 2)
if len(valArr) < 2 {
return errors.New("need both header and value, colon-delimited (ex. user_id:^169$).")
}
r, err := regexp.Compile(valArr[1])
if err != nil {
return err
}
valArr := strings.SplitN(value, ":", 2)
if len(valArr) < 2 {
return errors.New("need both header and value, colon-delimited (ex. user_id:^169$).")
}
r, err := regexp.Compile(valArr[1])
if err != nil {
return err
}
*h = append(*h, headerFilter{name: []byte(valArr[0]), regexp: r})
*h = append(*h, headerFilter{name: []byte(valArr[0]), regexp: r})
return nil
return nil
}
//
// Handling of --http-allow-header-hash and --http-allow-param-hash options
//
type hashFilter struct {
name []byte
percent uint32
name []byte
percent uint32
}
type HTTPHashFilters []hashFilter
func (h *HTTPHashFilters) String() string {
return fmt.Sprint(*h)
return fmt.Sprint(*h)
}
func (h *HTTPHashFilters) Set(value string) error {
valArr := strings.SplitN(value, ":", 2)
if len(valArr) < 2 {
return errors.New("need both header and value, colon-delimited (ex. user_id:50%).")
}
valArr := strings.SplitN(value, ":", 2)
if len(valArr) < 2 {
return errors.New("need both header and value, colon-delimited (ex. user_id:50%).")
}
f := hashFilter{ name: []byte(valArr[0]) }
f := hashFilter{name: []byte(valArr[0])}
if strings.Contains(valArr[1], "%") {
p, _ := strconv.ParseInt(valArr[1][:len(valArr[1])-1], 0, 0)
f.percent = uint32(p)
} else if strings.Contains(valArr[1], "/") {
// DEPRECATED format
var num, den uint64
if strings.Contains(valArr[1], "%") {
p, _ := strconv.ParseInt(valArr[1][:len(valArr[1])-1], 0, 0)
f.percent = uint32(p)
} else if strings.Contains(valArr[1], "/") {
// DEPRECATED format
var num, den uint64
fracArr := strings.Split(valArr[1], "/")
num, _ = strconv.ParseUint(fracArr[0], 10, 64)
den, _ = strconv.ParseUint(fracArr[1], 10, 64)
fracArr := strings.Split(valArr[1], "/")
num, _ = strconv.ParseUint(fracArr[0], 10, 64)
den, _ = strconv.ParseUint(fracArr[1], 10, 64)
f.percent = uint32((float64(num) / float64(den)) * 100)
} else {
return errors.New("Value should be percent and contain '%'")
}
f.percent = uint32((float64(num) / float64(den)) * 100)
} else {
return errors.New("Value should be percent and contain '%'")
}
*h = append(*h, f)
*h = append(*h, f)
return nil
return nil
}
//
// Handling of --http-set-header option
//
type HTTPHeaders []HTTPHeader
type HTTPHeader struct {
Name string
Value string
Name string
Value string
}
func (h *HTTPHeaders) String() string {
return fmt.Sprint(*h)
return fmt.Sprint(*h)
}
func (h *HTTPHeaders) Set(value string) error {
v := strings.SplitN(value, ":", 2)
if len(v) != 2 {
return errors.New("Expected `Key: Value`")
}
v := strings.SplitN(value, ":", 2)
if len(v) != 2 {
return errors.New("Expected `Key: Value`")
}
header := HTTPHeader{
strings.TrimSpace(v[0]),
strings.TrimSpace(v[1]),
}
header := HTTPHeader{
strings.TrimSpace(v[0]),
strings.TrimSpace(v[1]),
}
*h = append(*h, header)
return nil
*h = append(*h, header)
return nil
}
//
@@ -128,66 +127,67 @@ func (h *HTTPHeaders) Set(value string) error {
type HTTPMethods [][]byte
func (h *HTTPMethods) String() string {
return fmt.Sprint(*h)
return fmt.Sprint(*h)
}
func (h *HTTPMethods) Set(value string) error {
*h = append(*h, []byte(value))
return nil
*h = append(*h, []byte(value))
return nil
}
func (h *HTTPMethods) Contains(value []byte) bool {
for _, method := range *h {
if bytes.Equal(value, method) {
return true
}
}
return false
for _, method := range *h {
if bytes.Equal(value, method) {
return true
}
}
return false
}
//
// Handling of --http-rewrite-url option
//
type urlRewrite struct {
src *regexp.Regexp
target []byte
src *regexp.Regexp
target []byte
}
type UrlRewriteMap []urlRewrite
func (r *UrlRewriteMap) String() string {
return fmt.Sprint(*r)
return fmt.Sprint(*r)
}
func (r *UrlRewriteMap) Set(value string) error {
valArr := strings.SplitN(value, ":", 2)
if len(valArr) < 2 {
return errors.New("need both src and target, colon-delimited (ex. /a:/b).")
}
regexp, err := regexp.Compile(valArr[0])
if err != nil {
return err
}
*r = append(*r, urlRewrite{src: regexp, target: []byte(valArr[1])})
return nil
valArr := strings.SplitN(value, ":", 2)
if len(valArr) < 2 {
return errors.New("need both src and target, colon-delimited (ex. /a:/b).")
}
regexp, err := regexp.Compile(valArr[0])
if err != nil {
return err
}
*r = append(*r, urlRewrite{src: regexp, target: []byte(valArr[1])})
return nil
}
//
// Handling of --http-allow-url option
//
type HTTPUrlRegexp struct {
regexp *regexp.Regexp
type urlRegexp struct {
regexp *regexp.Regexp
}
type HTTPUrlRegexp []urlRegexp
func (r *HTTPUrlRegexp) String() string {
if r.regexp == nil {
return ""
}
return r.regexp.String()
return fmt.Sprint(*r)
}
func (r *HTTPUrlRegexp) Set(value string) error {
regexp, err := regexp.Compile(value)
r.regexp = regexp
return err
regexp, err := regexp.Compile(value)
*r = append(*r, urlRegexp{regexp: regexp})
return err
}
http_modifier_settings_test.go
+51 -52
View File
@@ -1,81 +1,80 @@
package main
import (
"testing"
"testing"
)
func TestHTTPHeaderFilters(t *testing.T) {
filters := HTTPHeaderFilters{}
filters := HTTPHeaderFilters{}
err := filters.Set("Header1:^$")
if err != nil {
t.Error("Should not error on Header1:^$")
}
err := filters.Set("Header1:^$")
if err != nil {
t.Error("Should not error on Header1:^$")
}
err = filters.Set("Header2:^:$")
if err != nil {
t.Error("Should not error on Header2:^:$")
}
err = filters.Set("Header2:^:$")
if err != nil {
t.Error("Should not error on Header2:^:$")
}
// Missing colon
err = filters.Set("Header3-^$")
if err == nil {
t.Error("Should error on Header2:^:$")
}
// Missing colon
err = filters.Set("Header3-^$")
if err == nil {
t.Error("Should error on Header2:^:$")
}
}
func TestHTTPHashFilters(t *testing.T) {
filters := HTTPHashFilters{}
filters := HTTPHashFilters{}
err := filters.Set("Header1:1/2")
if err != nil {
t.Error("Should support old syntax")
}
err := filters.Set("Header1:1/2")
if err != nil {
t.Error("Should support old syntax")
}
if filters[0].percent != 50 {
t.Error("Wrong percentage", filters[0].percent)
}
if filters[0].percent != 50 {
t.Error("Wrong percentage", filters[0].percent)
}
err = filters.Set("Header2:1")
if err == nil {
t.Error("Should error on Header2 because no % symbol")
}
err = filters.Set("Header2:1")
if err == nil {
t.Error("Should error on Header2 because no % symbol")
}
err = filters.Set("Header2:10%")
if err != nil {
t.Error("Should pass")
}
err = filters.Set("Header2:10%")
if err != nil {
t.Error("Should pass")
}
if filters[1].percent != 10 {
t.Error("Wrong percentage", filters[1].percent)
}
if filters[1].percent != 10 {
t.Error("Wrong percentage", filters[1].percent)
}
}
func TestHTTPMethods(t *testing.T) {
methods := HTTPMethods{}
methods := HTTPMethods{}
methods.Set("GET")
methods.Set("POST")
methods.Set("GET")
methods.Set("POST")
if !methods.Contains([]byte("GET")) {
t.Error("Does not contain GET")
}
if !methods.Contains([]byte("GET")) {
t.Error("Does not contain GET")
}
if !methods.Contains([]byte("POST")) {
t.Error("Does not contain POST")
}
if !methods.Contains([]byte("POST")) {
t.Error("Does not contain POST")
}
}
func TestUrlRewriteMap(t *testing.T) {
var err error
rewrites := UrlRewriteMap{}
var err error
rewrites := UrlRewriteMap{}
if err = rewrites.Set("/v1/user/([^\\/]+)/ping:/v2/user/$1/ping"); err != nil {
t.Error("Should set mapping", err)
}
if err = rewrites.Set("/v1/user/([^\\/]+)/ping:/v2/user/$1/ping"); err != nil {
t.Error("Should set mapping", err)
}
if err = rewrites.Set("/v1/user/([^\\/]+)/ping"); err == nil {
t.Error("Should not set mapping without :")
}
if err = rewrites.Set("/v1/user/([^\\/]+)/ping"); err == nil {
t.Error("Should not set mapping without :")
}
}
http_modifier_test.go
+52 -1
View File
@@ -93,7 +93,6 @@ func TestHTTPModifierHeaderHashFilters(t *testing.T) {
}
}
func TestHTTPModifierParamHashFilters(t *testing.T) {
filters := HTTPHashFilters{}
filters.Set("user_id:1/2")
@@ -135,3 +134,55 @@ func TestHTTPModifierHeaders(t *testing.T) {
t.Error("Should update request headers", string(payload))
}
}
func TestHTTPModifierURLRegexp(t *testing.T) {
filters := HTTPUrlRegexp{}
filters.Set("/v1/app")
filters.Set("/v1/api")
modifier := NewHTTPModifier(&HTTPModifierConfig{
urlRegexp: filters,
})
payload := func(url string) []byte {
return []byte("POST " + url + " HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
}
if len(modifier.Rewrite(payload("/v1/app/test"))) == 0 {
t.Error("Should pass url")
}
if len(modifier.Rewrite(payload("/v1/api/test"))) == 0 {
t.Error("Should pass url")
}
if len(modifier.Rewrite(payload("/other"))) > 0 {
t.Error("Should not pass url")
}
}
func TestHTTPModifierURLNegativeRegexp(t *testing.T) {
filters := HTTPUrlRegexp{}
filters.Set("/restricted1")
filters.Set("/some/restricted2")
modifier := NewHTTPModifier(&HTTPModifierConfig{
urlNegativeRegexp: filters,
})
payload := func(url string) []byte {
return []byte("POST " + url + " HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
}
if len(modifier.Rewrite(payload("/v1/app/test"))) == 0 {
t.Error("Should pass url")
}
if len(modifier.Rewrite(payload("/restricted1"))) > 0 {
t.Error("Should not pass url")
}
if len(modifier.Rewrite(payload("/some/restricted2"))) > 0 {
t.Error("Should not pass url")
}
}
input_tcp.go
+2 -2
View File
@@ -2,10 +2,10 @@ package main
import (
"bufio"
"log"
"net"
"encoding/hex"
"fmt"
"log"
"net"
"os"
)
input_tcp_test.go
+2 -2
View File
@@ -1,12 +1,12 @@
package main
import (
"encoding/hex"
"io"
"log"
"net"
"sync"
"testing"
"encoding/hex"
)
func TestTCPInput(t *testing.T) {
@@ -40,7 +40,7 @@ func TestTCPInput(t *testing.T) {
for i := 0; i < 100; i++ {
wg.Add(1)
encoded := make([]byte, len(msg)*2 + 1)
encoded := make([]byte, len(msg)*2+1)
hex.Encode(encoded, msg)
conn.Write(append(encoded, '\n'))
}
output_tcp.go
+2 -2
View File
@@ -1,12 +1,12 @@
package main
import (
"encoding/hex"
"fmt"
"io"
"log"
"net"
"time"
"encoding/hex"
)
type TCPOutput struct {
@@ -53,7 +53,7 @@ func (o *TCPOutput) worker() {
func (o *TCPOutput) Write(data []byte) (n int, err error) {
// Hex encoding always 2x number of bytes
encoded := make([]byte, len(data)*2 + 1)
encoded := make([]byte, len(data)*2+1)
hex.Encode(encoded, data)
o.buf <- append(encoded, '\n')
output_tcp_test.go
+1 -1
View File
@@ -2,12 +2,12 @@ package main
import (
"bufio"
"encoding/hex"
"io"
"log"
"net"
"sync"
"testing"
"encoding/hex"
)
func TestTCPOutput(t *testing.T) {
settings.go
-5
View File
@@ -23,7 +23,6 @@ func (h *MultiOption) Set(value string) error {
return nil
}
type AppSettings struct {
verbose bool
stats bool
@@ -90,24 +89,20 @@ func init() {
flag.Var(&Settings.modifierConfig.headers, "http-set-header", "Inject additional headers to http reqest:\n\tgor --input-raw :8080 --output-http staging.com --http-set-header 'User-Agent: Gor'")
flag.Var(&Settings.modifierConfig.headers, "output-http-header", "WARNING: `--output-http-header` DEPRECATED, use `--http-set-header` instead")
flag.Var(&Settings.modifierConfig.methods, "http-allow-method", "Whitelist of HTTP methods to replay. Anything else will be dropped:\n\tgor --input-raw :8080 --output-http staging.com --http-allow-method GET --http-allow-method OPTIONS")
flag.Var(&Settings.modifierConfig.methods, "output-http-method", "WARNING: `--output-http-method` DEPRECATED, use `--http-allow-method` instead")
flag.Var(&Settings.modifierConfig.urlRegexp, "http-allow-url", "A regexp to match requests against. Filter get matched agains full url with domain. Anything else will be dropped:\n\t gor --input-raw :8080 --output-http staging.com --http-allow-url ^www.")
flag.Var(&Settings.modifierConfig.urlRegexp, "output-http-url-regexp", "WARNING: `--output-http-url-regexp` DEPRECATED, use `--http-allow-url` instead")
flag.Var(&Settings.modifierConfig.urlNegativeRegexp, "http-diallow-url", "A regexp to match requests against. Filter get matched agains full url with domain. Anything else will be dropped:\n\t gor --input-raw :8080 --output-http staging.com --http-disallow-url ^www.")
flag.Var(&Settings.modifierConfig.urlRewrite, "http-rewrite-url", "Rewrite the requst url based on a mapping:\n\tgor --input-raw :8080 --output-http staging.com --http-rewrite-url /v1/user/([^\\/]+)/ping:/v2/user/$1/ping")
flag.Var(&Settings.modifierConfig.urlRewrite, "output-http-rewrite-url", "WARNING: `--output-http-rewrite-url` DEPRECATED, use `--http-rewrite-url` instead")
flag.Var(&Settings.modifierConfig.headerFilters, "http-allow-header", "A regexp to match a specific header against. Requests with non-matching headers will be dropped:\n\t gor --input-raw :8080 --output-http staging.com --http-allow-header api-version:^v1")
flag.Var(&Settings.modifierConfig.headerFilters, "output-http-header-filter", "WARNING: `--output-http-header-filter` DEPRECATED, use `--http-allow-header` instead")
flag.Var(&Settings.modifierConfig.headerHashFilters, "http-allow-header-hash", "Takes a fraction of requests, consistently taking or rejecting a request based on the FNV32-1A hash of a specific header:\n\t gor --input-raw :8080 --output-http staging.com --http-allow-header-hash user-id:25%")
flag.Var(&Settings.modifierConfig.headerHashFilters, "output-http-header-hash-filter", "WARNING: `output-http-header-hash-filter` DEPRECATED, use `--http-allow-header-hash` instead")