github/goreplay
4
0
Fork 0
mirror of https://github.com/buger/goreplay.git synced 2024-04-21 12:32:02 +00:00
Code Issues Releases Wiki Activity

Merge branch 'master' into input-modifier

Browse Source 
Conflicts:
	Dockerfile
	emitter.go
	http_client.go
	plugins.go
	proto/proto.go
This commit is contained in:
Leonid Bugaev
2015-07-21 09:49:37 +05:00
parent 5e6f7e03a7 0a30582131
commit 5cc4228dcd
39 changed files with 966 additions and 751 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.travis.yml
+3 -3
View File
@@ -1,3 +1,3 @@
language: go
go: 1.3.3
script: sudo -E bash -c "source /etc/profile && eval '$(gimme 1.4)' && export GOPATH=$HOME/gopath:$GOPATH && go get && GORACE='halt_on_error=1' go test -race -v"
language: go
go: 1.4.2
script: sudo -E bash -c "source /etc/profile && eval '$(gimme 1.4.2)' && export GOPATH=$HOME/gopath:$GOPATH && go get && GORACE='halt_on_error=1' go test ./... -v -timeout 15s"
CHANGELOG.md
-25
View File
@@ -1,25 +0,0 @@
v0.7.0 - 31 Oct 2013
* New modular architecture. Listener and Replay functionality merged.
* Added option to equally split traffic between multiple outputs: --split-output true
* Saving requests to file and replaying from it
* Injecting custom headers to http requests
* Advanced stats using ElasticSearch
v0.3.5 - 15 Sep 2013
* Significantly improved test coverage
* Fixed bug with redirect replay https://github.com/buger/gor/pull/15
* Added limit on listener side
* Improved stability (catch and log panic, instead of exiting)
* Added License file
v0.3.3 - 22 Jun 2013
* Using TCP instead of UDP for communication between Listener and Replay
* Significantly improved performance
* Fixed bugs causing locking and message dropping (concurrency issues)
* Rewrote concurrency model to use more channels
v0.3 - 10 Jun 2013
* Use RAW_SOCKETS instead of tcpdump
* Own TCP stack
* All HTTP request types support
* Simplified request parsing
Dockerfile
+3 -2
View File
@@ -1,4 +1,4 @@
FROM google/golang
FROM google/golang:1.4
RUN cd /goroot/src/ && GOOS=linux GOARCH=386 ./make.bash --no-clean
@@ -8,4 +8,5 @@ WORKDIR /gopath/src/github.com/buger/gor/
ADD . /gopath/src/github.com/buger/gor/
RUN go get
RUN go get -u github.com/golang/lint/golint
RUN go get
Makefile
+8 -2
View File
@@ -13,15 +13,21 @@ release-x86:
dbuild:
docker build -t gor .
dlint:
docker run -v `pwd`:$(SOURCE_PATH) -t -i --env GORACE="halt_on_error=1" gor golint $(PKG)
drace:
docker run -v `pwd`:$(SOURCE_PATH) -t -i --env GORACE="halt_on_error=1" gor go test ./... $(ARGS) -v -race -timeout 15s
dtest:
docker run -v `pwd`:$(SOURCE_PATH) -t -i --env GORACE="halt_on_error=1" gor go test ./... $(ARGS) -race -v -timeout 15s
docker run -v `pwd`:$(SOURCE_PATH) -t -i gor go test ./... $(ARGS) -v -timeout 15s
dcover:
docker run -v `pwd`:$(SOURCE_PATH) -t -i --env GORACE="halt_on_error=1" gor go test $(ARGS) -race -v -timeout 15s -coverprofile=coverage.out
go tool cover -html=coverage.out
dfmt:
docker run -v `pwd`:$(SOURCE_PATH) -t -i gor go fmt
docker run -v `pwd`:$(SOURCE_PATH) -t -i gor go fmt ./...
dvet:
docker run -v `pwd`:$(SOURCE_PATH) -t -i gor go vet
README.md
+10 -1
View File
@@ -128,6 +128,9 @@ gor --input-raw :8080 --output-http staging.com --http-disallow-url /api
```
# only forward requests with an api version of 1.0x
gor --input-raw :8080 --output-http staging.com --http-allow-header api-version:^1\.0\d
# only forward requests NOT containing User-Agent header value "Replayed by Gor"
gor --input-raw :8080 --output-http staging.com --http-disallow-header "User-Agent: Replayed by Gor"
```
#### Filter based on http method
@@ -260,6 +263,8 @@ https://github.com/buger/gor/releases
```
-http-allow-header=[]: A regexp to match a specific header against. Requests with non-matching headers will be dropped:
gor --input-raw :8080 --output-http staging.com --http-allow-header api-version:^v1
-http-disallow-header=[]: A regexp to match a specific header against. Requests with matching headers will be dropped:
gor --input-raw :8080 --output-http staging.com --http-disallow-header "User-Agent: Replayed by Gor"
-http-allow-method=[]: Whitelist of HTTP methods to replay. Anything else will be dropped:
gor --input-raw :8080 --output-http staging.com --http-allow-method GET --http-allow-method OPTIONS
-http-allow-url=[]: A regexp to match requests against. Filter get matched agains full url with domain. Anything else will be dropped:
@@ -349,6 +354,10 @@ Basic idea is that SSL was made to protect itself from traffic interception. The
More can be find here: https://github.com/buger/gor/issues/85
### Is there a limit for size of HTTP request when using output-http?
Due to the fact that Gor can't guarantee interception of all packets, for large payloads > 200kb there is chance of missing some packets and corrupting body. Treat it as a feature and chance to test broken bodies handling :)
The only way to guarantee delivery is using `--input-http`, but you will miss some features.
### I'm getting 'too many open files' error
Typical linux shell has a small open files soft limit at 1024. You can easily raise that when you do this before starting your gor replay process:
@@ -397,4 +406,4 @@ To achieve the top most performance you should tune the source server system lim
* [3SCALE](http://www.3scale.net/)
* [Optionlab](http://www.opinionlab.com)
* [TubeMogul] (http://tubemogul.com)
* To add your company drop me a line to github.com/buger or leonsbox@gmail.com
* To add your company drop me a line to github.com/buger or leonsbox@gmail.com
byteutils/byteutils.go
+27 -24
View File
@@ -1,37 +1,40 @@
// Package byteutils probvides helpers for working with byte slices
package byteutils
// Cut elements from slice for a given range
func Cut(a []byte, from, to int) []byte {
copy(a[from:], a[to:])
a = a[:len(a)-to+from]
copy(a[from:], a[to:])
a = a[:len(a)-to+from]
return a
return a
}
// Insert new slice at specified position
func Insert(a []byte, i int, b []byte) []byte {
a = append(a, make([]byte, len(b))...)
copy(a[i+len(b):], a[i:])
copy(a[i:i+len(b)], b)
a = append(a, make([]byte, len(b))...)
copy(a[i+len(b):], a[i:])
copy(a[i:i+len(b)], b)
return a
return a
}
// Unlike bytes.Replace it allows you to specify range
// Replace function unlike bytes.Replace allows you to specify range
func Replace(a []byte, from, to int, new []byte) []byte {
lenDiff := len(new) - (to - from)
lenDiff := len(new) - (to - from)
if lenDiff > 0 {
// Extend if new segment bigger
a = append(a, make([]byte, lenDiff)...)
copy(a[to+lenDiff:], a[to:])
copy(a[from:from+len(new)], new)
if lenDiff > 0 {
// Extend if new segment bigger
a = append(a, make([]byte, lenDiff)...)
copy(a[to+lenDiff:], a[to:])
copy(a[from:from+len(new)], new)
return a
} else if lenDiff < 0 {
copy(a[from:], new)
copy(a[from+len(new):],a[to:])
return a[:len(a) + lenDiff]
} else { // same size
copy(a[from:], new)
return a
}
}
return a
} else if lenDiff < 0 {
copy(a[from:], new)
copy(a[from+len(new):], a[to:])
return a[:len(a)+lenDiff]
} else { // same size
copy(a[from:], new)
return a
}
}
byteutils/byteutils_test.go
+18 -18
View File
@@ -1,32 +1,32 @@
package byteutils
import (
"testing"
"bytes"
"bytes"
"testing"
)
func TestCut(t *testing.T) {
if !bytes.Equal(Cut([]byte("123456"), 2, 4), []byte("1256")) {
t.Error("Should properly cut")
}
if !bytes.Equal(Cut([]byte("123456"), 2, 4), []byte("1256")) {
t.Error("Should properly cut")
}
}
func TestInsert(t *testing.T) {
if !bytes.Equal(Insert([]byte("123456"), 2, []byte("abcd")), []byte("12abcd3456")) {
t.Error("Should insert into middle of slice")
}
if !bytes.Equal(Insert([]byte("123456"), 2, []byte("abcd")), []byte("12abcd3456")) {
t.Error("Should insert into middle of slice")
}
}
func TestReplace(t *testing.T) {
if !bytes.Equal(Replace([]byte("123456"), 2, 4, []byte("ab")), []byte("12ab56")) {
t.Error("Should replace when same length")
}
if !bytes.Equal(Replace([]byte("123456"), 2, 4, []byte("ab")), []byte("12ab56")) {
t.Error("Should replace when same length")
}
if !bytes.Equal(Replace([]byte("123456"), 2, 4, []byte("abcd")), []byte("12abcd56")) {
t.Error("Should replace when replacement length bigger")
}
if !bytes.Equal(Replace([]byte("123456"), 2, 4, []byte("abcd")), []byte("12abcd56")) {
t.Error("Should replace when replacement length bigger")
}
if !bytes.Equal(Replace([]byte("123456"), 2, 5, []byte("ab")), []byte("12ab6")) {
t.Error("Should replace when replacement length bigger")
}
}
if !bytes.Equal(Replace([]byte("123456"), 2, 5, []byte("ab")), []byte("12ab6")) {
t.Error("Should replace when replacement length bigger")
}
}
elasticsearch.go
+15 -15
View File
@@ -122,24 +122,24 @@ func (p *ESPlugin) ResponseAnalyze(req, resp []byte, start, stop time.Time) {
esResp := ESRequestResponse{
ReqUrl: proto.Path(req),
ReqMethod: proto.Method(req),
ReqUserAgent: proto.GetHeader(req, "User-Agent"),
ReqAcceptLanguage: proto.GetHeader(req, "Accept-Language"),
ReqAccept: proto.GetHeader(req, "Accept"),
ReqAcceptEncoding: proto.GetHeader(req, "Accept-Encoding"),
ReqIfModifiedSince: proto.GetHeader(req, "If-Modified-Since"),
ReqConnection: proto.GetHeader(req, "Connection"),
ReqCookies: proto.GetHeader(req, "Cookie"),
ReqUserAgent: proto.Header(req, []byte("User-Agent")),
ReqAcceptLanguage: proto.Header(req, []byte("Accept-Language")),
ReqAccept: proto.Header(req, []byte("Accept")),
ReqAcceptEncoding: proto.Header(req, []byte("Accept-Encoding")),
ReqIfModifiedSince: proto.Header(req, []byte("If-Modified-Since")),
ReqConnection: proto.Header(req, []byte("Connection")),
ReqCookies: proto.Header(req, []byte("Cookie")),
RespStatus: proto.Status(resp),
RespStatusCode: proto.Status(resp),
RespProto: proto.Method(resp),
RespContentLength: proto.GetHeader(resp, "Content-Length"),
RespContentType: proto.GetHeader(resp, "Content-Type"),
RespTransferEncoding: proto.GetHeader(resp, "Transfer-Encoding"),
RespContentEncoding: proto.GetHeader(resp, "Content-Encoding"),
RespExpires: proto.GetHeader(resp, "Expires"),
RespCacheControl: proto.GetHeader(resp, "Cache-Control"),
RespVary: proto.GetHeader(resp, "Vary"),
RespSetCookie: proto.GetHeader(resp, "Set-Cookie"),
RespContentLength: proto.Header(resp, []byte("Content-Length")),
RespContentType: proto.Header(resp, []byte("Content-Type")),
RespTransferEncoding: proto.Header(resp, []byte("Transfer-Encoding")),
RespContentEncoding: proto.Header(resp, []byte("Content-Encoding")),
RespExpires: proto.Header(resp, []byte("Expires")),
RespCacheControl: proto.Header(resp, []byte("Cache-Control")),
RespVary: proto.Header(resp, []byte("Vary")),
RespSetCookie: proto.Header(resp, []byte("Set-Cookie")),
Rtt: rtt,
Timestamp: t,
}
emitter.go
+10 -6
View File
@@ -1,18 +1,18 @@
package main
import (
"crypto/rand"
"io"
"time"
"crypto/rand"
)
func uuid() []byte {
b := make([]byte, 16)
rand.Read(b)
return b
b := make([]byte, 16)
rand.Read(b)
return b
}
// Start initialize loop for sending data from inputs to outputs
func Start(stop chan int) {
if Settings.middleware != "" {
middleware := NewMiddleware(Settings.middleware)
@@ -44,7 +44,7 @@ func Start(stop chan int) {
}
}
// Copy from 1 reader to multiple writers
// CopyMulty copies from 1 reader to multiple writers
func CopyMulty(src io.Reader, writers ...io.Writer) (err error) {
buf := make([]byte, 5*1024*1024)
wIndex := 0
@@ -67,6 +67,10 @@ func CopyMulty(src io.Reader, writers ...io.Writer) (err error) {
}
}
if Settings.debug {
Debug("[EMITTER] Sending payload, size:", len(payload), "First 500 bytes:", string(payload[0:500]))
}
if Settings.splitOutput {
// Simple round robin
writers[wIndex].Write(payload)
gor.go
+6
View File
@@ -7,6 +7,7 @@ import (
"fmt"
"log"
"os"
"runtime"
"runtime/debug"
"runtime/pprof"
"time"
@@ -19,6 +20,11 @@ var (
)
func main() {
// If not set via env cariable
if len(os.Getenv("GOMAXPROCS")) == 0 {
runtime.GOMAXPROCS(runtime.NumCPU() * 2)
}
// Don't exit on panic
defer func() {
if r := recover(); r != nil {
http_client.go
+27 -10
View File
@@ -4,8 +4,10 @@ import (
"crypto/tls"
"github.com/buger/gor/proto"
"io"
"log"
"net"
"net/url"
"runtime/debug"
"strings"
"time"
)
@@ -89,9 +91,24 @@ func (c *HTTPClient) isAlive() bool {
}
func (c *HTTPClient) Send(data []byte) (response []byte, err error) {
// Don't exit on panic
defer func() {
if r := recover(); r != nil {
Debug("[HTTPClient]", r, string(data))
if _, ok := r.(error); !ok {
log.Println("[HTTPClient] Failed to send request: ", string(data))
log.Println("PANIC: pkg:", r, debug.Stack())
}
}
}()
if c.conn == nil || !c.isAlive() {
Debug("[HTTP] Connecting:", c.baseURL)
c.Connect()
Debug("[HTTPClient] Connecting:", c.baseURL)
if err = c.Connect(); err != nil {
log.Println("[HTTPClient] Connection error:", err)
return
}
}
timeout := time.Now().Add(5 * time.Second)
@@ -101,11 +118,11 @@ func (c *HTTPClient) Send(data []byte) (response []byte, err error) {
data = proto.SetHost(data, []byte(c.baseURL), []byte(c.host))
if c.config.Debug {
Debug("[HTTP] Sending:", string(data))
Debug("[HTTPClient] Sending:", string(data))
}
if _, err = c.conn.Write(data); err != nil {
Debug("[HTTP] Write error:", err, c.baseURL)
Debug("[HTTPClient] Write error:", err, c.baseURL)
return
}
@@ -113,14 +130,14 @@ func (c *HTTPClient) Send(data []byte) (response []byte, err error) {
n, err := c.conn.Read(c.respBuf)
if err != nil {
Debug("[HTTP] READ ERRORR!", err, c.conn)
Debug("[HTTPClient] Response read error", err, c.conn)
return
}
payload := c.respBuf[:n]
if c.config.Debug {
Debug("[HTTP] Received:", string(payload))
Debug("[HTTPClient] Received:", string(payload))
}
if c.config.FollowRedirects > 0 && c.redirectsCount < c.config.FollowRedirects {
@@ -128,13 +145,13 @@ func (c *HTTPClient) Send(data []byte) (response []byte, err error) {
// 3xx requests
if status[0] == '3' {
c.redirectsCount += 1
c.redirectsCount++
location, _, _, _ := proto.Header(payload, []byte("Location"))
location := proto.Header(payload, []byte("Location"))
redirectPayload := []byte("GET " + string(location) + " HTTP/1.1\r\n\r\n")
if c.config.Debug {
Debug("[HTTP] Redirecting to: " + string(location))
Debug("[HTTPClient] Redirecting to: " + string(location))
}
return c.Send(redirectPayload)
@@ -150,4 +167,4 @@ func (c *HTTPClient) Get(path string) (response []byte, err error) {
payload := "GET " + path + " HTTP/1.1\r\n\r\n"
return c.Send([]byte(payload))
}
}
http_client_test.go
+12 -12
View File
@@ -132,7 +132,7 @@ func TestHTTPClientHTTPSSend(t *testing.T) {
func TestHTTPClientServerInstantDisconnect(t *testing.T) {
wg := new(sync.WaitGroup)
GET_payload := []byte("GET / HTTP/1.1\r\n\r\n")
GETPayload := []byte("GET / HTTP/1.1\r\n\r\n")
ln, _ := net.Listen("tcp", ":0")
@@ -148,8 +148,8 @@ func TestHTTPClientServerInstantDisconnect(t *testing.T) {
client := NewHTTPClient(ln.Addr().String(), &HTTPClientConfig{})
wg.Add(2)
client.Send(GET_payload)
client.Send(GET_payload)
client.Send(GETPayload)
client.Send(GETPayload)
wg.Wait()
}
@@ -157,7 +157,7 @@ func TestHTTPClientServerInstantDisconnect(t *testing.T) {
func TestHTTPClientServerNoKeepAlive(t *testing.T) {
wg := new(sync.WaitGroup)
GET_payload := []byte("GET / HTTP/1.1\r\n\r\n")
GETPayload := []byte("GET / HTTP/1.1\r\n\r\n")
ln, _ := net.Listen("tcp", ":0")
@@ -186,8 +186,8 @@ func TestHTTPClientServerNoKeepAlive(t *testing.T) {
client := NewHTTPClient(ln.Addr().String(), &HTTPClientConfig{})
wg.Add(2)
client.Send(GET_payload)
client.Send(GET_payload)
client.Send(GETPayload)
client.Send(GETPayload)
wg.Wait()
}
@@ -195,7 +195,7 @@ func TestHTTPClientServerNoKeepAlive(t *testing.T) {
func TestHTTPClientRedirect(t *testing.T) {
wg := new(sync.WaitGroup)
GET_payload := []byte("GET / HTTP/1.1\r\n\r\n")
GETPayload := []byte("GET / HTTP/1.1\r\n\r\n")
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@@ -210,7 +210,7 @@ func TestHTTPClientRedirect(t *testing.T) {
// Should do 2 queries
wg.Add(2)
client.Send(GET_payload)
client.Send(GETPayload)
wg.Wait()
}
@@ -218,7 +218,7 @@ func TestHTTPClientRedirect(t *testing.T) {
func TestHTTPClientRedirectLimit(t *testing.T) {
wg := new(sync.WaitGroup)
GET_payload := []byte("GET / HTTP/1.1\r\n\r\n")
GETPayload := []byte("GET / HTTP/1.1\r\n\r\n")
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@@ -241,7 +241,7 @@ func TestHTTPClientRedirectLimit(t *testing.T) {
// Have 3 redirects + 1 GET, but should do only 2 redirects + GET
wg.Add(3)
client.Send(GET_payload)
client.Send(GETPayload)
wg.Wait()
}
@@ -249,7 +249,7 @@ func TestHTTPClientRedirectLimit(t *testing.T) {
func TestHTTPClientHandleHTTP10(t *testing.T) {
wg := new(sync.WaitGroup)
GET_payload := []byte("GET http://foobar.com/path HTTP/1.0\r\n\r\n")
GETPayload := []byte("GET http://foobar.com/path HTTP/1.0\r\n\r\n")
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@@ -263,7 +263,7 @@ func TestHTTPClientHandleHTTP10(t *testing.T) {
client := NewHTTPClient(server.URL, &HTTPClientConfig{Debug: true})
wg.Add(1)
client.Send(GET_payload)
client.Send(GETPayload)
wg.Wait()
}
http_modifier.go
+17 -5
View File
@@ -2,8 +2,9 @@ package main
import (
"bytes"
"github.com/buger/gor/proto"
"hash/fnv"
"github.com/buger/gor/proto"
)
type HTTPModifier struct {
@@ -16,6 +17,7 @@ func NewHTTPModifier(config *HTTPModifierConfig) *HTTPModifier {
len(config.urlNegativeRegexp) == 0 &&
len(config.urlRewrite) == 0 &&
len(config.headerFilters) == 0 &&
len(config.headerNegativeFilters) == 0 &&
len(config.headerHashFilters) == 0 &&
len(config.paramHashFilters) == 0 &&
len(config.params) == 0 &&
@@ -86,9 +88,19 @@ func (m *HTTPModifier) Rewrite(payload []byte) (response []byte) {
if len(m.config.headerFilters) > 0 {
for _, f := range m.config.headerFilters {
value, s, _, _ := proto.Header(payload, f.name)
value := proto.Header(payload, f.name)
if s != -1 && !f.regexp.Match(value) {
if len(value) > 0 && !f.regexp.Match(value) {
return
}
}
}
if len(m.config.headerNegativeFilters) > 0 {
for _, f := range m.config.headerNegativeFilters {
value := proto.Header(payload, f.name)
if len(value) > 0 && f.regexp.Match(value) {
return
}
}
@@ -96,9 +108,9 @@ func (m *HTTPModifier) Rewrite(payload []byte) (response []byte) {
if len(m.config.headerHashFilters) > 0 {
for _, f := range m.config.headerHashFilters {
value, s, _, _ := proto.Header(payload, f.name)
value := proto.Header(payload, f.name)
if s != -1 {
if len(value) > 0 {
hasher := fnv.New32a()
hasher.Write(value)
http_modifier_settings.go
+12 -9
View File
@@ -8,13 +8,15 @@ import (
"strings"
)
// HTTPModifierConfig holds configuration options for built-in traffic modifier
type HTTPModifierConfig struct {
urlNegativeRegexp HTTPUrlRegexp
urlRegexp HTTPUrlRegexp
urlRewrite UrlRewriteMap
headerFilters HTTPHeaderFilters
headerHashFilters HTTPHashFilters
paramHashFilters HTTPHashFilters
urlNegativeRegexp HTTPUrlRegexp
urlRegexp HTTPUrlRegexp
urlRewrite UrlRewriteMap
headerFilters HTTPHeaderFilters
headerNegativeFilters HTTPHeaderFilters
headerHashFilters HTTPHashFilters
paramHashFilters HTTPHashFilters
params HTTPParams
headers HTTPHeaders
@@ -22,13 +24,14 @@ type HTTPModifierConfig struct {
}
//
// Handling of --http-allow-header options
// Handling of --http-allow-header, --http-disallow-header options
//
type headerFilter struct {
name []byte
regexp *regexp.Regexp
}
// HTTPHeaderFilters holds list of headers and their regexps
type HTTPHeaderFilters []headerFilter
func (h *HTTPHeaderFilters) String() string {
@@ -67,7 +70,7 @@ func (h *HTTPHashFilters) String() string {
func (h *HTTPHashFilters) Set(value string) error {
valArr := strings.SplitN(value, ":", 2)
if len(valArr) < 2 {
return errors.New("need both header and value, colon-delimited (ex. user_id:50%).")
return errors.New("need both header and value, colon-delimited (ex. user_id:50%)")
}
f := hashFilter{name: []byte(valArr[0])}
@@ -180,7 +183,7 @@ func (r *UrlRewriteMap) String() string {
func (r *UrlRewriteMap) Set(value string) error {
valArr := strings.SplitN(value, ":", 2)
if len(valArr) < 2 {
return errors.New("need both src and target, colon-delimited (ex. /a:/b).")
return errors.New("need both src and target, colon-delimited (ex. /a:/b)")
}
regexp, err := regexp.Compile(valArr[0])
if err != nil {
http_modifier_test.go
+38 -11
View File
@@ -39,8 +39,35 @@ func TestHTTPModifierHeaderFilters(t *testing.T) {
}
}
func TestHTTPModifierHeaderNegativeFilters(t *testing.T) {
filters := HTTPHeaderFilters{}
filters.Set("Host:^www.w3.org$")
modifier := NewHTTPModifier(&HTTPModifierConfig{
headerNegativeFilters: filters,
})
payload := []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w4.org\r\n\r\na=1&b=2")
if len(modifier.Rewrite(payload)) == 0 {
t.Error("Request should pass filters")
}
filters = HTTPHeaderFilters{}
// Setting filter that not match our header
filters.Set("Host:^www.w4.org$")
modifier = NewHTTPModifier(&HTTPModifierConfig{
headerNegativeFilters: filters,
})
if len(modifier.Rewrite(payload)) != 0 {
t.Error("Request should not pass filters")
}
}
func TestHTTPModifierURLRewrite(t *testing.T) {
var url, new_url []byte
var url, newURL []byte
rewrites := UrlRewriteMap{}
@@ -58,13 +85,13 @@ func TestHTTPModifierURLRewrite(t *testing.T) {
})
url = []byte("/v1/user/joe/ping")
if new_url = proto.Path(modifier.Rewrite(payload(url))); bytes.Equal(new_url, url) {
t.Error("Request url should have been rewritten, wasn't", string(new_url))
if newURL = proto.Path(modifier.Rewrite(payload(url))); bytes.Equal(newURL, url) {
t.Error("Request url should have been rewritten, wasn't", string(newURL))
}
url = []byte("/v1/user/ping")
if new_url = proto.Path(modifier.Rewrite(payload(url))); !bytes.Equal(new_url, url) {
t.Error("Request url should have been rewritten, wasn't", string(new_url))
if newURL = proto.Path(modifier.Rewrite(payload(url))); !bytes.Equal(newURL, url) {
t.Error("Request url should have been rewritten, wasn't", string(newURL))
}
}
@@ -128,9 +155,9 @@ func TestHTTPModifierHeaders(t *testing.T) {
})
payload := []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
new_payload := []byte("POST /post HTTP/1.1\r\nHeader1: 1\r\nContent-Length: 7\r\nHost: localhost\r\n\r\na=1&b=2")
newPayload := []byte("POST /post HTTP/1.1\r\nHeader1: 1\r\nContent-Length: 7\r\nHost: localhost\r\n\r\na=1&b=2")
if payload = modifier.Rewrite(payload); !bytes.Equal(payload, new_payload) {
if payload = modifier.Rewrite(payload); !bytes.Equal(payload, newPayload) {
t.Error("Should update request headers", string(payload))
}
}
@@ -196,9 +223,9 @@ func TestHTTPModifierSetHeader(t *testing.T) {
})
payload := []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
payload_after := []byte("POST /post HTTP/1.1\r\nUser-Agent: Gor\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
payloadAfter := []byte("POST /post HTTP/1.1\r\nUser-Agent: Gor\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
if payload = modifier.Rewrite(payload); !bytes.Equal(payload_after, payload) {
if payload = modifier.Rewrite(payload); !bytes.Equal(payloadAfter, payload) {
t.Error("Should add new header", string(payload))
}
}
@@ -212,9 +239,9 @@ func TestHTTPModifierSetParam(t *testing.T) {
})
payload := []byte("POST /post?api_key=1234 HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
payload_after := []byte("POST /post?api_key=1 HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
payloadAfter := []byte("POST /post?api_key=1 HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
if payload = modifier.Rewrite(payload); !bytes.Equal(payload_after, payload) {
if payload = modifier.Rewrite(payload); !bytes.Equal(payloadAfter, payload) {
t.Error("Should override param", string(payload))
}
}
input_dummy.go
+2
View File
@@ -4,10 +4,12 @@ import (
"time"
)
// DummyInput used for debugging. It generate 1 "GET /"" request per second.
type DummyInput struct {
data chan []byte
}
// NewDummyInput constructor for DummyInput
func NewDummyInput(options string) (di *DummyInput) {
di = new(DummyInput)
di.data = make(chan []byte)
input_file.go
+4 -2
View File
@@ -7,6 +7,7 @@ import (
"time"
)
// FileInput can read requests generated by FileOutput
type FileInput struct {
data chan []byte
path string
@@ -14,19 +15,20 @@ type FileInput struct {
speedFactor float64
}
// NewFileInput constructor for FileInput. Accepts file path as argument.
func NewFileInput(path string) (i *FileInput) {
i = new(FileInput)
i.data = make(chan []byte)
i.path = path
i.speedFactor = 1
i.Init(path)
i.init(path)
go i.emit()
return
}
func (i *FileInput) Init(path string) {
func (i *FileInput) init(path string) {
file, err := os.Open(path)
if err != nil {
input_http.go
+2
View File
@@ -7,12 +7,14 @@ import (
"net/http/httputil"
)
// HTTPInput used for sending requests to Gor via http
type HTTPInput struct {
data chan []byte
address string
listener net.Listener
}
// NewHTTPInput constructor for HTTPInput. Accepts address with port which he will listen on.
func NewHTTPInput(address string) (i *HTTPInput) {
i = new(HTTPInput)
i.data = make(chan []byte)
input_raw.go
+2
View File
@@ -7,11 +7,13 @@ import (
"strings"
)
// RAWInput used for intercepting traffic for given address
type RAWInput struct {
data chan []byte
address string
}
// NewRAWInput constructor for RAWInput. Accepts address with port as argument.
func NewRAWInput(address string) (i *RAWInput) {
i = new(RAWInput)
i.data = make(chan []byte)
input_raw_test.go
+81 -19
View File
@@ -6,11 +6,13 @@ import (
"io/ioutil"
"log"
"net/http"
"net/http/httptest"
"net/http/httputil"
"os/exec"
"strings"
"sync"
"testing"
"time"
)
func TestRAWInput(t *testing.T) {
@@ -30,12 +32,15 @@ func TestRAWInput(t *testing.T) {
address := strings.Replace(listener.Addr().String(), "[::]", "127.0.0.1", -1)
client := NewHTTPClient(address, &HTTPClientConfig{})
time.Sleep(time.Millisecond)
go Start(quit)
for i := 0; i < 100; i++ {
wg.Add(1)
res, _ := http.Get("http://" + address)
res.Body.Close()
client.Get("/")
}
wg.Wait()
@@ -47,7 +52,7 @@ func TestInputRAW100Expect(t *testing.T) {
wg := new(sync.WaitGroup)
quit := make(chan int)
file_content, _ := ioutil.ReadFile("README.md")
fileContent, _ := ioutil.ReadFile("README.md")
// Origing and Replay server initialization
origin := startHTTP(func(w http.ResponseWriter, req *http.Request) {
@@ -57,12 +62,12 @@ func TestInputRAW100Expect(t *testing.T) {
wg.Done()
})
origin_address := strings.Replace(origin.Addr().String(), "[::]", "127.0.0.1", -1)
originAddr := strings.Replace(origin.Addr().String(), "[::]", "127.0.0.1", -1)
input := NewRAWInput(origin_address)
input := NewRAWInput(originAddr)
// We will use it to get content of raw HTTP request
test_output := NewTestOutput(func(data []byte) {
testOutput := NewTestOutput(func(data []byte) {
if strings.Contains(string(data), "Expect: 100-continue") {
t.Error("Should not contain 100-continue header")
}
@@ -73,24 +78,24 @@ func TestInputRAW100Expect(t *testing.T) {
defer req.Body.Close()
body, _ := ioutil.ReadAll(req.Body)
if !bytes.Equal(body, file_content) {
if !bytes.Equal(body, fileContent) {
buf, _ := httputil.DumpRequest(req, true)
t.Error("Wrong POST body:", string(buf))
}
wg.Done()
})
replay_address := listener.Addr().String()
replayAddr := listener.Addr().String()
http_output := NewHTTPOutput(replay_address, &HTTPOutputConfig{})
httpOutput := NewHTTPOutput(replayAddr, &HTTPOutputConfig{})
Plugins.Inputs = []io.Reader{input}
Plugins.Outputs = []io.Writer{test_output, http_output}
Plugins.Outputs = []io.Writer{testOutput, httpOutput}
go Start(quit)
wg.Add(3)
curl := exec.Command("curl", "http://"+origin_address, "--data-binary", "@README.md")
curl := exec.Command("curl", "http://"+originAddr, "--data-binary", "@README.md")
err := curl.Run()
if err != nil {
log.Fatal(err)
@@ -104,7 +109,7 @@ func TestInputRAWChunkedEncoding(t *testing.T) {
wg := new(sync.WaitGroup)
quit := make(chan int)
file_content, _ := ioutil.ReadFile("README.md")
fileContent, _ := ioutil.ReadFile("README.md")
// Origing and Replay server initialization
origin := startHTTP(func(w http.ResponseWriter, req *http.Request) {
@@ -114,33 +119,33 @@ func TestInputRAWChunkedEncoding(t *testing.T) {
wg.Done()
})
origin_address := strings.Replace(origin.Addr().String(), "[::]", "127.0.0.1", -1)
originAddr := strings.Replace(origin.Addr().String(), "[::]", "127.0.0.1", -1)
input := NewRAWInput(origin_address)
input := NewRAWInput(originAddr)
listener := startHTTP(func(w http.ResponseWriter, req *http.Request) {
defer req.Body.Close()
body, _ := ioutil.ReadAll(req.Body)
if !bytes.Equal(body, file_content) {
if !bytes.Equal(body, fileContent) {
buf, _ := httputil.DumpRequest(req, true)
t.Error("Wrong POST body:", string(buf))
}
wg.Done()
})
replay_address := listener.Addr().String()
replayAddr := listener.Addr().String()
http_output := NewHTTPOutput(replay_address, &HTTPOutputConfig{Debug: true})
httpOutput := NewHTTPOutput(replayAddr, &HTTPOutputConfig{Debug: true})
Plugins.Inputs = []io.Reader{input}
Plugins.Outputs = []io.Writer{http_output}
Plugins.Outputs = []io.Writer{httpOutput}
go Start(quit)
wg.Add(2)
curl := exec.Command("curl", "http://"+origin_address, "--header", "Transfer-Encoding: chunked", "--data-binary", "@README.md")
curl := exec.Command("curl", "http://"+originAddr, "--header", "Transfer-Encoding: chunked", "--data-binary", "@README.md")
err := curl.Run()
if err != nil {
log.Fatal(err)
@@ -150,3 +155,60 @@ func TestInputRAWChunkedEncoding(t *testing.T) {
close(quit)
}
func TestInputRAWLargePayload(t *testing.T) {
wg := new(sync.WaitGroup)
quit := make(chan int)
// Generate 200kb file
dd := exec.Command("dd", "if=/dev/urandom", "of=/tmp/large", "bs=1KB", "count=100")
err := dd.Run()
if err != nil {
log.Fatal("dd error:", err)
}
origin := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
defer req.Body.Close()
body, _ := ioutil.ReadAll(req.Body)
if len(body) != 100*1000 {
t.Error("File size should be 1mb:", len(body))
}
wg.Done()
}))
originAddr := strings.Replace(origin.Listener.Addr().String(), "[::]", "127.0.0.1", -1)
input := NewRAWInput(originAddr)
replay := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
req.Body = http.MaxBytesReader(w, req.Body, 1*1024*1024)
buf := make([]byte, 1*1024*1024)
n, _ := req.Body.Read(buf)
body := buf[0:n]
if len(body) != 100*1000 {
t.Error("File size should be 100000 bytes:", len(body))
}
wg.Done()
}))
defer replay.Close()
httpOutput := NewHTTPOutput(replay.URL, &HTTPOutputConfig{Debug: false})
Plugins.Inputs = []io.Reader{input}
Plugins.Outputs = []io.Writer{httpOutput}
go Start(quit)
wg.Add(2)
curl := exec.Command("curl", "http://"+originAddr, "--header", "Transfer-Encoding: chunked", "--data-binary", "@/tmp/large")
err = curl.Run()
if err != nil {
log.Fatal("curl error:", err)
}
wg.Wait()
close(quit)
}
input_tcp.go
+3 -3
View File
@@ -9,15 +9,15 @@ import (
"os"
)
// Can be tested using nc tool:
// echo "asdad" | nc 127.0.0.1 27017
//
// TCPInput used for internal communication
// It expected hex encoded data
type TCPInput struct {
data chan []byte
address string
listener net.Listener
}
// NewTCPInput constructor for TCPInput, accepts address with port
func NewTCPInput(address string) (i *TCPInput) {
i = new(TCPInput)
i.data = make(chan []byte)
input_tcp_test.go
+3 -3
View File
@@ -83,10 +83,10 @@ func BenchmarkTCPInput(b *testing.B) {
for {
data := <-dataChan
new_buf := make([]byte, len(data)+2)
buf := make([]byte, len(data)+2)
data = append(data, []byte("¶")...)
copy(new_buf, data)
conn.Write(new_buf)
copy(buf, data)
conn.Write(buf)
}
}(conn)
}
limiter.go
+3
View File
@@ -9,6 +9,7 @@ import (
"time"
)
// Limiter is a wrapper for input or output plugin which adds rate limiting
type Limiter struct {
plugin interface{}
limit int
@@ -30,6 +31,8 @@ func parseLimitOptions(options string) (limit int, isPercent bool) {
return
}
// NewLimiter constructor for Limiter, accepts plugin and options
// `options` allow to sprcify relatve or absolute limiting
func NewLimiter(plugin interface{}, options string) io.ReadWriter {
l := new(Limiter)
l.limit, l.isPercent = parseLimitOptions(options)
middleware.go
+2 -2
View File
@@ -8,7 +8,7 @@ import (
"log"
"os"
"os/exec"
"strings"
"strings"
)
type Middleware struct {
@@ -25,7 +25,7 @@ func NewMiddleware(command string) *Middleware {
m.command = command
m.data = make(chan []byte, 1000)
commands := strings.Split(command, " ")
commands := strings.Split(command, " ")
cmd := exec.Command(commands[0], commands[1:]...)
m.Stdout, _ = cmd.StdoutPipe()
middleware_test.go
+103 -103
View File
@@ -3,15 +3,15 @@ package main
import (
"bytes"
"crypto/rand"
"encoding/hex"
"github.com/buger/gor/proto"
"io"
"net/http"
"net/http/httptest"
"strings"
"sync"
"testing"
"strings"
"github.com/buger/gor/proto"
"net/http/httptest"
"encoding/hex"
"time"
"time"
)
type fakeServiceCb func(string, int, []byte)
@@ -20,8 +20,8 @@ type fakeServiceCb func(string, int, []byte)
func NewFakeSecureService(wg *sync.WaitGroup, cb fakeServiceCb) string {
active_tokens := make([]string, 0)
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
Debug("Received request: " + req.URL.String())
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
Debug("Received request: " + req.URL.String())
switch req.URL.Path {
case "/token":
@@ -29,63 +29,63 @@ func NewFakeSecureService(wg *sync.WaitGroup, cb fakeServiceCb) string {
token_length := 10
buf := make([]byte, token_length)
rand.Read(buf)
token := hex.EncodeToString(buf)
token := hex.EncodeToString(buf)
active_tokens = append(active_tokens, token)
w.Write([]byte(token))
w.Write([]byte(token))
cb(req.URL.Path, 200, []byte(token))
cb(req.URL.Path, 200, []byte(token))
case "/secure":
token := req.URL.Query().Get("token")
token_found := false
token_found := false
for _, t := range active_tokens {
if t == token {
token_found = true
break
token_found = true
break
}
}
if token_found {
w.WriteHeader(http.StatusAccepted)
cb(req.URL.Path, 202, []byte(nil))
} else {
w.WriteHeader(http.StatusForbidden)
cb(req.URL.Path, 403, []byte(nil))
}
if token_found {
w.WriteHeader(http.StatusAccepted)
cb(req.URL.Path, 202, []byte(nil))
} else {
w.WriteHeader(http.StatusForbidden)
cb(req.URL.Path, 403, []byte(nil))
}
}
wg.Done()
}))
address := strings.Replace(server.Listener.Addr().String(), "[::]", "127.0.0.1", -1)
address := strings.Replace(server.Listener.Addr().String(), "[::]", "127.0.0.1", -1)
return address
}
func TestFakeSecureService(t *testing.T) {
var resp, token []byte
var resp, token []byte
wg := new(sync.WaitGroup)
addr := NewFakeSecureService(wg, func(path string, status int, resp []byte){
addr := NewFakeSecureService(wg, func(path string, status int, resp []byte) {
})
})
wg.Add(3)
client := NewHTTPClient("http://" + addr, &HTTPClientConfig{Debug: true})
resp, _ = client.Get("/token")
token = proto.Body(resp)
client := NewHTTPClient("http://"+addr, &HTTPClientConfig{Debug: true})
resp, _ = client.Get("/token")
token = proto.Body(resp)
// Right token
resp, _ = client.Get("/secure?token=" + string(token))
if !bytes.Equal(proto.Status(resp), []byte("202")) {
t.Error("Valid token should return status 202:", string(proto.Status(resp)))
}
// Right token
resp, _ = client.Get("/secure?token=" + string(token))
if !bytes.Equal(proto.Status(resp), []byte("202")) {
t.Error("Valid token should return status 202:", string(proto.Status(resp)))
}
// Wrong tokens forbidden
resp, _ = client.Get("/secure?token=wrong")
if !bytes.Equal(proto.Status(resp), []byte("403")) {
// Wrong tokens forbidden
resp, _ = client.Get("/secure?token=wrong")
if !bytes.Equal(proto.Status(resp), []byte("403")) {
t.Error("Wrong token should returns status 403:", string(proto.Status(resp)))
}
@@ -93,98 +93,98 @@ func TestFakeSecureService(t *testing.T) {
}
func TestEchoMiddleware(t *testing.T) {
wg := new(sync.WaitGroup)
wg := new(sync.WaitGroup)
from := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
wg.Done()
}))
to := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
wg.Done()
}))
from := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
wg.Done()
}))
to := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
wg.Done()
}))
quit := make(chan int)
quit := make(chan int)
// Catch traffic from one service
input := NewRAWInput(from.Listener.Addr().String())
// Catch traffic from one service
input := NewRAWInput(from.Listener.Addr().String())
// And redirect to another
output := NewHTTPOutput(to.URL, &HTTPOutputConfig{})
// And redirect to another
output := NewHTTPOutput(to.URL, &HTTPOutputConfig{})
Plugins.Inputs = []io.Reader{input}
Plugins.Outputs = []io.Writer{output}
Settings.middleware = "./examples/echo_modifier.sh"
Plugins.Inputs = []io.Reader{input}
Plugins.Outputs = []io.Writer{output}
Settings.middleware = "./examples/echo_modifier.sh"
// Start Gor
go Start(quit)
// Start Gor
go Start(quit)
time.Sleep(time.Millisecond)
time.Sleep(time.Millisecond)
// Should receive 2 requests from original + 2 from replayed
wg.Add(4)
// Should receive 2 requests from original + 2 from replayed
wg.Add(4)
client := NewHTTPClient(from.URL, &HTTPClientConfig{Debug: true})
client := NewHTTPClient(from.URL, &HTTPClientConfig{Debug: true})
// Request should be echoed
client.Get("/")
client.Get("/")
// Request should be echoed
client.Get("/")
client.Get("/")
wg.Wait()
close(quit)
Settings.middleware = ""
wg.Wait()
close(quit)
Settings.middleware = ""
}
func TestTokenMiddleware(t *testing.T) {
var resp, token []byte
var resp, token []byte
wg := new(sync.WaitGroup)
wg := new(sync.WaitGroup)
from := NewFakeSecureService(wg, func(path string, status int, tok []byte){
})
to := NewFakeSecureService(wg, func(path string, status int, tok []byte){
switch path {
case "/token":
if bytes.Equal(token, tok) {
t.Error("Tokens should not match")
}
case "/secure":
if status != 202 {
// t.Error("Server should receive valid rewritten token")
}
}
})
from := NewFakeSecureService(wg, func(path string, status int, tok []byte) {
})
to := NewFakeSecureService(wg, func(path string, status int, tok []byte) {
switch path {
case "/token":
if bytes.Equal(token, tok) {
t.Error("Tokens should not match")
}
case "/secure":
if status != 202 {
// t.Error("Server should receive valid rewritten token")
}
}
})
quit := make(chan int)
quit := make(chan int)
// Catch traffic from one service
input := NewRAWInput(from)
// Catch traffic from one service
input := NewRAWInput(from)
// And redirect to another
output := NewHTTPOutput(to, &HTTPOutputConfig{})
// And redirect to another
output := NewHTTPOutput(to, &HTTPOutputConfig{})
Plugins.Inputs = []io.Reader{input}
Plugins.Outputs = []io.Writer{output}
// Settings.middleware = "./examples/echo_modifier.sh"
Plugins.Inputs = []io.Reader{input}
Plugins.Outputs = []io.Writer{output}
// Settings.middleware = "./examples/echo_modifier.sh"
// Start Gor
go Start(quit)
// Start Gor
go Start(quit)
time.Sleep(time.Millisecond)
time.Sleep(time.Millisecond)
// Should receive 2 requests from original + 2 from replayed
wg.Add(4)
// Should receive 2 requests from original + 2 from replayed
wg.Add(4)
client := NewHTTPClient("http://" + from, &HTTPClientConfig{Debug: true})
client := NewHTTPClient("http://"+from, &HTTPClientConfig{Debug: true})
// Sending traffic to original service
resp, _ = client.Get("/token")
token = proto.Body(resp)
// Sending traffic to original service
resp, _ = client.Get("/token")
token = proto.Body(resp)
resp, _ = client.Get("/secure?token=" + string(token))
if !bytes.Equal(proto.Status(resp), []byte("202")) {
t.Error("Valid token should return 202:", proto.Status(resp))
}
resp, _ = client.Get("/secure?token=" + string(token))
if !bytes.Equal(proto.Status(resp), []byte("202")) {
t.Error("Valid token should return 202:", proto.Status(resp))
}
wg.Wait()
close(quit)
Settings.middleware = ""
}
wg.Wait()
close(quit)
Settings.middleware = ""
}
output_dummy.go
+2
View File
@@ -4,9 +4,11 @@ import (
"fmt"
)
// DummyOutput used for debugging, prints all incoming requests
type DummyOutput struct {
}
// NewDummyOutput constructor for DummyOutput
func NewDummyOutput(options string) (di *DummyOutput) {
di = new(DummyOutput)
output_file.go
+5 -2
View File
@@ -8,26 +8,29 @@ import (
"time"
)
// RawRequest stores original start time and request payload
type RawRequest struct {
Timestamp int64
Request []byte
}
// FileOutput output plugin
type FileOutput struct {
path string
encoder *gob.Encoder
file *os.File
}
// NewFileOutput constructor for FileOutput, accepts path
func NewFileOutput(path string) io.Writer {
o := new(FileOutput)
o.path = path
o.Init(path)
o.init(path)
return o
}
func (o *FileOutput) Init(path string) {
func (o *FileOutput) init(path string) {
var err error
o.file, err = os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0660)
output_http.go
+23 -17
View File
@@ -7,8 +7,9 @@ import (
"time"
)
const InitialDynamicWorkers = 10
const initialDynamicWorkers = 10
// HTTPOutputConfig struct for holding http output configuration
type HTTPOutputConfig struct {
redirectLimit int
@@ -20,16 +21,19 @@ type HTTPOutputConfig struct {
Debug bool
}
// HTTPOutput plugin manage pool of workers which send request to replayed server
// By default workers pool is dynamic and starts with 10 workers
// You can specify fixed number of workers using `--output-http-workers`
type HTTPOutput struct {
// Keep this as first element of struct because it guarantees 64bit
// alignment. atomic.* functions crash on 32bit machines if operand is not
// aligned at 64bit. See https://github.com/golang/go/issues/599
activeWorkers int64
address string
limit int
queue chan []byte
responses chan []byte
address string
limit int
queue chan []byte
responses chan []byte
needWorker chan int
@@ -40,6 +44,8 @@ type HTTPOutput struct {
elasticSearch *ESPlugin
}
// NewHTTPOutput constructor for HTTPOutput
// Initialize workers
func NewHTTPOutput(address string, config *HTTPOutputConfig) io.Writer {
o := new(HTTPOutput)
@@ -55,7 +61,7 @@ func NewHTTPOutput(address string, config *HTTPOutputConfig) io.Writer {
// Initial workers count
if o.config.workers == 0 {
o.needWorker <- InitialDynamicWorkers
o.needWorker <- initialDynamicWorkers
} else {
o.needWorker <- o.config.workers
}
@@ -65,16 +71,16 @@ func NewHTTPOutput(address string, config *HTTPOutputConfig) io.Writer {
o.elasticSearch.Init(o.config.elasticSearch)
}
go o.WorkerMaster()
go o.workerMaster()
return o
}
func (o *HTTPOutput) WorkerMaster() {
func (o *HTTPOutput) workerMaster() {
for {
new_workers := <-o.needWorker
for i := 0; i < new_workers; i++ {
go o.Worker()
newWorkers := <-o.needWorker
for i := 0; i < newWorkers; i++ {
go o.startWorker()
}
// Disable dynamic scaling if workers poll fixed size
@@ -84,13 +90,13 @@ func (o *HTTPOutput) WorkerMaster() {
}
}
func (o *HTTPOutput) Worker() {
func (o *HTTPOutput) startWorker() {
client := NewHTTPClient(o.address, &HTTPClientConfig{
FollowRedirects: o.config.redirectLimit,
Debug: o.config.Debug,
})
death_count := 0
deathCount := 0
atomic.AddInt64(&o.activeWorkers, 1)
@@ -98,19 +104,19 @@ func (o *HTTPOutput) Worker() {
select {
case data := <-o.queue:
o.sendRequest(client, data)
death_count = 0
deathCount = 0
case <-time.After(time.Millisecond * 100):
// When dynamic scaling enabled workers die after 2s of inactivity
if o.config.workers == 0 {
death_count += 1
deathCount++
} else {
continue
}
if death_count > 20 {
if deathCount > 20 {
workersCount := atomic.LoadInt64(&o.activeWorkers)
// At least 1 worker should be alive
// At least 1 startWorker should be alive
if workersCount != 1 {
atomic.AddInt64(&o.activeWorkers, -1)
return
output_http_test.go
+2 -2
View File
@@ -86,10 +86,10 @@ func TestOutputHTTPSSL(t *testing.T) {
}))
input := NewTestInput()
http_output := NewHTTPOutput(server.URL, &HTTPOutputConfig{})
output := NewHTTPOutput(server.URL, &HTTPOutputConfig{})
Plugins.Inputs = []io.Reader{input}
Plugins.Outputs = []io.Writer{http_output}
Plugins.Outputs = []io.Writer{output}
go Start(quit)
output_tcp.go
+5
View File
@@ -9,6 +9,9 @@ import (
"time"
)
// TCPOutput used for sending raw tcp payloads
// Currently used for internal communication between listener and replay server
// Can be used for transfering binary payloads like protocol buffers
type TCPOutput struct {
address string
limit int
@@ -16,6 +19,8 @@ type TCPOutput struct {
bufStats *GorStat
}
// NewTCPOutput constructor for TCPOutput
// Initialize 10 workers which hold keep-alive connection
func NewTCPOutput(address string) io.Writer {
o := new(TCPOutput)
plugins.go
+12 -7
View File
@@ -6,21 +6,25 @@ import (
"strings"
)
// InOutPlugins struct for holding references to plugins
type InOutPlugins struct {
Inputs []io.Reader
Outputs []io.Writer
}
// Plugins holds all the plugin objects
var Plugins *InOutPlugins = new(InOutPlugins)
// extractLimitOptions detects if plugin get called with limiter support
// Returns address and limit
func extractLimitOptions(options string) (string, string) {
split := strings.Split(options, "|")
if len(split) > 1 {
return split[0], split[1]
} else {
return split[0], ""
}
return split[0], ""
}
// Automatically detects type of plugin and initialize it
@@ -43,23 +47,24 @@ func registerPlugin(constructor interface{}, options ...interface{}) {
// Calling our constructor with list of given options
plugin := vc.Call(vo)[0].Interface()
plugin_wrapper := plugin
pluginWrapper := plugin
if limit != "" {
plugin_wrapper = NewLimiter(plugin, limit)
pluginWrapper = NewLimiter(plugin, limit)
} else {
plugin_wrapper = plugin
pluginWrapper = plugin
}
if _, ok := plugin.(io.Reader); ok {
Plugins.Inputs = append(Plugins.Inputs, plugin_wrapper.(io.Reader))
Plugins.Inputs = append(Plugins.Inputs, pluginWrapper.(io.Reader))
}
if _, ok := plugin.(io.Writer); ok {
Plugins.Outputs = append(Plugins.Outputs, plugin_wrapper.(io.Writer))
Plugins.Outputs = append(Plugins.Outputs, pluginWrapper.(io.Writer))
}
}
// InitPlugins specify and initialize all available plugins
func InitPlugins() {
for _, options := range Settings.inputDummy {
registerPlugin(NewDummyInput, options)
proto/proto.go
+145 -106
View File
@@ -1,171 +1,210 @@
// Low-level interaction with HTTP request payload
/*
Package proto provides byte-level interaction with HTTP request payload.
Example of HTTP payload for future references, new line symbols escaped:
POST /upload HTTP/1.1\r\n
User-Agent: Gor\r\n
Content-Length: 11\r\n
\r\n
Hello world
GET /index.html HTTP/1.1\r\n
User-Agent: Gor\r\n
\r\n
\r\n
*/
package proto
import (
"bytes"
"github.com/buger/gor/byteutils"
_ "log"
"bytes"
"github.com/buger/gor/byteutils"
)
// In HTTP newline defined by 2 bytes (for both windows and *nix support)
var CLRF = []byte("\r\n")
var EMPTY_LINE = []byte("\r\n\r\n")
var HEADER_DELIM = []byte(": ")
// Headers should end with empty line
// New line acts as separator: end of Headers or Body (in some cases)
var EmptyLine = []byte("\r\n\r\n")
// Separator for Header line. Header looks like: `HeaderName: value`
var HeaderDelim = []byte(": ")
// MIMEHeadersEndPos finds end of the Headers section, which should end with empty line.
func MIMEHeadersEndPos(payload []byte) int {
return bytes.Index(payload, EMPTY_LINE)
return bytes.Index(payload, EmptyLine)
}
// MIMEHeadersStartPos finds start of Headers section
// It just finds position of second line (first contains location and method).
func MIMEHeadersStartPos(payload []byte) int {
return bytes.Index(payload, CLRF) + 2 // Find first line end
return bytes.Index(payload, CLRF) + 2 // Find first line end
}
// Find header value or return error
// Do not support multi-line headers
func Header(payload []byte, name []byte) (value []byte, headerStart, valueStart, headerEnd int) {
headerStart = bytes.Index(payload, name)
// header return value and positions of header/value start/end.
// If not found, value will be blank, and headerStart will be -1
// Do not support multi-line headers.
func header(payload []byte, name []byte) (value []byte, headerStart, valueStart, headerEnd int) {
headerStart = bytes.Index(payload, name)
if headerStart == -1 {
return
}
if headerStart == -1 {
return
}
valueStart = headerStart + len(name) + 1 // Skip ":" after header name
if payload[valueStart] == ' ' { // Ignore empty space after ':'
valueStart += 1
}
headerEnd = valueStart + bytes.IndexByte(payload[valueStart:], '\r')
value = payload[valueStart:headerEnd]
valueStart = headerStart + len(name) + 1 // Skip ":" after header name
if payload[valueStart] == ' ' { // Ignore empty space after ':'
valueStart++
}
headerEnd = valueStart + bytes.IndexByte(payload[valueStart:], '\r')
value = payload[valueStart:headerEnd]
return
return
}
func GetHeader(payload []byte, name string) []byte {
val, _, _, _ := Header(payload, []byte(name))
// Header returns header value, if header not found, value will be blank
func Header(payload, name []byte) []byte {
val, _, _, _ := header(payload, name)
return val
return val
}
// SetHeader sets header value. If header not found it creates new one.
// Returns modified request payload
func SetHeader(payload, name, value []byte) []byte {
_, hs, vs, he := Header(payload, name)
_, hs, vs, he := header(payload, name)
// If header found
if hs != -1 {
return byteutils.Replace(payload, vs, he, value)
} else {
return AddHeader(payload, name, value)
}
if hs != -1 {
// If header found we just repace its value
return byteutils.Replace(payload, vs, he, value)
}
return AddHeader(payload, name, value)
}
// AddHeader takes http payload and appends new header to the start of headers section
// Returns modified request payload
func AddHeader(payload, name, value []byte) []byte {
header := make([]byte, len(name) + 2 + len(value) + 2)
copy(header[0:], name)
copy(header[len(name):], HEADER_DELIM)
copy(header[len(name)+2:], value)
copy(header[len(header)-2:], CLRF)
header := make([]byte, len(name)+2+len(value)+2)
copy(header[0:], name)
copy(header[len(name):], HeaderDelim)
copy(header[len(name)+2:], value)
copy(header[len(header)-2:], CLRF)
mimeStart := MIMEHeadersStartPos(payload)
mimeStart := MIMEHeadersStartPos(payload)
return byteutils.Insert(payload, mimeStart, header)
return byteutils.Insert(payload, mimeStart, header)
}
// Body returns request/response body
func Body(payload []byte) []byte {
// 4 -> len(EMPTY_LINE)
return payload[MIMEHeadersEndPos(payload) + 4:]
// 4 -> len(EMPTY_LINE)
return payload[MIMEHeadersEndPos(payload)+4:]
}
// Path takes payload and retuns request path: Split(firstLine, ' ')[1]
func Path(payload []byte) []byte {
start := bytes.IndexByte(payload, ' ')
start += 1
start := bytes.IndexByte(payload, ' ') + 1
end := bytes.IndexByte(payload[start:], ' ')
end := bytes.IndexByte(payload[start:], ' ')
return payload[start:start+end]
return payload[start : start+end]
}
// SetPath takes payload, sets new path and returns modified payload
func SetPath(payload, path []byte) []byte {
start := bytes.IndexByte(payload, ' ')
start += 1
start := bytes.IndexByte(payload, ' ') + 1
end := bytes.IndexByte(payload[start:], ' ')
end := bytes.IndexByte(payload[start:], ' ')
return byteutils.Replace(payload, start, start+end, path)
return byteutils.Replace(payload, start, start+end, path)
}
// PathParam returns URL query attribute by given name, if no found: valueStart will be -1
func PathParam(payload, name []byte) (value []byte, valueStart, valueEnd int) {
path := Path(payload)
path := Path(payload)
if paramStart := bytes.Index(path, append(name, '=')); paramStart != -1 {
valueStart := paramStart + len(name) + 1
paramEnd := bytes.IndexByte(path[valueStart:], '&')
if paramEnd == -1 { // It is final param
paramEnd = len(path)
} else {
paramEnd += valueStart
}
if paramStart := bytes.Index(path, append(name, '=')); paramStart != -1 {
valueStart := paramStart + len(name) + 1
paramEnd := bytes.IndexByte(path[valueStart:], '&')
return path[valueStart:paramEnd], valueStart, paramEnd
} else {
return []byte(""), -1, -1
}
// Param can end with '&' (another param), or end of line
if paramEnd == -1 { // It is final param
paramEnd = len(path)
} else {
paramEnd += valueStart
}
return path[valueStart:paramEnd], valueStart, paramEnd
}
return []byte(""), -1, -1
}
// SetPathParam takes payload and updates path Query attribute
// If query param not found, it will append new
// Returns modified payload
func SetPathParam(payload, name, value []byte) []byte {
path := Path(payload)
_, vs, ve := PathParam(payload, name)
path := Path(payload)
_, vs, ve := PathParam(payload, name)
if vs != -1 {
newPath := make([]byte, len(path))
copy(newPath, path)
newPath = byteutils.Replace(newPath, vs, ve, value)
if vs != -1 { // If param found, replace its value and set new Path
newPath := make([]byte, len(path))
copy(newPath, path)
newPath = byteutils.Replace(newPath, vs, ve, value)
return SetPath(payload, newPath)
} else { // if param not found append to end of url
// Adding 2 because of '?' or '&' at start, and '=' in middle
newParam := make([]byte, len(name) + len(value) + 2)
return SetPath(payload, newPath)
}
if bytes.IndexByte(path, '?') == -1 {
newParam[0] = '?'
} else {
newParam[0] = '&'
}
// if param not found append to end of url
// Adding 2 because of '?' or '&' at start, and '=' in middle
newParam := make([]byte, len(name)+len(value)+2)
copy(newParam[1:], name)
newParam[1+len(name)] = '='
copy(newParam[2+len(name):], value)
if bytes.IndexByte(path, '?') == -1 {
newParam[0] = '?'
} else {
newParam[0] = '&'
}
newPath := make([]byte, len(path) + len(newParam))
copy(newPath, path)
copy(newPath[len(path):], newParam)
// Copy "param=value" into buffer, after it looks like "?param=value"
copy(newParam[1:], name)
newParam[1+len(name)] = '='
copy(newParam[2+len(name):], value)
return SetPath(payload, newPath)
}
// Append param to the end of path
newPath := make([]byte, len(path)+len(newParam))
copy(newPath, path)
copy(newPath[len(path):], newParam)
return SetPath(payload, newPath)
}
// SetHost updates Host header for HTTP/1.1 or updates host in path for HTTP/1.0 or Proxy requests
// Returns modified payload
func SetHost(payload, url, host []byte) []byte {
// If this is HTTP 1.0 traffic or proxy traffic it may include host right into path variable, so instead of setting Host header we rewrite Path
// Fix for https://github.com/buger/gor/issues/156
if path := Path(payload); bytes.HasPrefix(path, []byte("http")) {
hostStart := bytes.IndexByte(path, ':') // : position "https?:"
hostStart += 3 // Skip 1 ':' and 2 '\'
hostEnd := hostStart + bytes.IndexByte(path[hostStart:], '/')
// If this is HTTP 1.0 traffic or proxy traffic it may include host right into path variable, so instead of setting Host header we rewrite Path
// Fix for https://github.com/buger/gor/issues/156
if path := Path(payload); bytes.HasPrefix(path, []byte("http")) {
hostStart := bytes.IndexByte(path, ':') // : position "https?:"
hostStart += 3 // Skip 1 ':' and 2 '\'
hostEnd := hostStart + bytes.IndexByte(path[hostStart:], '/')
newPath := make([]byte, len(path))
copy(newPath, path)
newPath = byteutils.Replace(newPath, 0, hostEnd, url)
newPath := make([]byte, len(path))
copy(newPath, path)
newPath = byteutils.Replace(newPath, 0, hostEnd, url)
return SetPath(payload, newPath)
} else {
return SetHeader(payload, []byte("Host"), host)
}
return SetPath(payload, newPath)
}
return SetHeader(payload, []byte("Host"), host)
}
// Method returns HTTP method
func Method(payload []byte) []byte {
end := bytes.IndexByte(payload, ' ')
end := bytes.IndexByte(payload, ' ')
return payload[:end]
return payload[:end]
}
// Status in response have same position as Path in request
// Status returns response status.
// It happend to be in same position as request payload path
func Status(payload []byte) []byte {
return Path(payload)
}
return Path(payload)
}
proto/proto_test.go
+86 -92
View File
@@ -1,152 +1,146 @@
package proto
import (
"testing"
"bytes"
"bytes"
"testing"
)
func TestHeader(t *testing.T) {
var payload, val []byte
var headerStart int
var payload, val []byte
var headerStart int
payload = []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
payload = []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
if val, _, _, _ = Header(payload, []byte("Content-Length")); !bytes.Equal(val, []byte("7")) {
t.Error("Should find header value")
}
if val = Header(payload, []byte("Content-Length")); !bytes.Equal(val, []byte("7")) {
t.Error("Should find header value")
}
payload = []byte("POST /post HTTP/1.1\r\nContent-Length:7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
payload = []byte("POST /post HTTP/1.1\r\nContent-Length:7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
if val, _, _, _ = Header(payload, []byte("Content-Length")); !bytes.Equal(val, []byte("7")) {
t.Error("Should find header value without space after :")
}
if val = Header(payload, []byte("Content-Length")); !bytes.Equal(val, []byte("7")) {
t.Error("Should find header value without space after :")
}
if _, headerStart, _, _ = Header(payload, []byte("Not-Found")); headerStart != -1 {
t.Error("Should not found header")
}
if _, headerStart, _, _ = header(payload, []byte("Not-Found")); headerStart != -1 {
t.Error("Should not found header")
}
}
func TestMIMEHeadersEndPos(t *testing.T) {
head := []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org")
payload := []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
head := []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org")
payload := []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
end := MIMEHeadersEndPos(payload)
end := MIMEHeadersEndPos(payload)
if !bytes.Equal(payload[:end], head) {
t.Error("Wrong headers end position:", end)
}
if !bytes.Equal(payload[:end], head) {
t.Error("Wrong headers end position:", end)
}
}
func TestMIMEHeadersStartPos(t *testing.T) {
headers := []byte("Content-Length: 7\r\nHost: www.w3.org")
payload := []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
headers := []byte("Content-Length: 7\r\nHost: www.w3.org")
payload := []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
start := MIMEHeadersStartPos(payload)
end := MIMEHeadersEndPos(payload)
start := MIMEHeadersStartPos(payload)
end := MIMEHeadersEndPos(payload)
if !bytes.Equal(payload[start:end], headers) {
t.Error("Wrong headers end position:", start, end)
}
if !bytes.Equal(payload[start:end], headers) {
t.Error("Wrong headers end position:", start, end)
}
}
func TestSetHeader(t *testing.T) {
var payload, payload_after []byte
var payload, payloadAfter []byte
payload = []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
payload_after = []byte("POST /post HTTP/1.1\r\nContent-Length: 14\r\nHost: www.w3.org\r\n\r\na=1&b=2")
payload = []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
payloadAfter = []byte("POST /post HTTP/1.1\r\nContent-Length: 14\r\nHost: www.w3.org\r\n\r\na=1&b=2")
if payload = SetHeader(payload, []byte("Content-Length"), []byte("14")); !bytes.Equal(payload, payload_after) {
t.Error("Should update header if it exists", string(payload))
}
if payload = SetHeader(payload, []byte("Content-Length"), []byte("14")); !bytes.Equal(payload, payloadAfter) {
t.Error("Should update header if it exists", string(payload))
}
payload = []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
payloadAfter = []byte("POST /post HTTP/1.1\r\nUser-Agent: Gor\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
payload = []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
payload_after = []byte("POST /post HTTP/1.1\r\nUser-Agent: Gor\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
if payload = SetHeader(payload, []byte("User-Agent"), []byte("Gor")); !bytes.Equal(payload, payload_after) {
t.Error("Should add header if not found", string(payload))
}
if payload = SetHeader(payload, []byte("User-Agent"), []byte("Gor")); !bytes.Equal(payload, payloadAfter) {
t.Error("Should add header if not found", string(payload))
}
}
func TestPath(t *testing.T) {
var path, payload []byte
var path, payload []byte
payload = []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
payload = []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
if path = Path(payload); !bytes.Equal(path, []byte("/post")) {
t.Error("Should find path", string(path))
}
if path = Path(payload); !bytes.Equal(path, []byte("/post")) {
t.Error("Should find path", string(path))
}
}
func TestSetPath(t *testing.T) {
var payload, payload_after []byte
var payload, payloadAfter []byte
payload = []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
payload_after = []byte("POST /new_path HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
payload = []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
payloadAfter = []byte("POST /new_path HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
if payload = SetPath(payload, []byte("/new_path")); !bytes.Equal(payload, payload_after) {
t.Error("Should replace path", string(payload))
}
if payload = SetPath(payload, []byte("/new_path")); !bytes.Equal(payload, payloadAfter) {
t.Error("Should replace path", string(payload))
}
}
func TestPathParam(t *testing.T) {
var payload []byte
var payload []byte
payload = []byte("POST /post?param=test&user_id=1 HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
payload = []byte("POST /post?param=test&user_id=1 HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
if val, _, _ := PathParam(payload, []byte("param")); !bytes.Equal(val, []byte("test")) {
t.Error("Should detect attribute", string(val))
}
if val, _, _ := PathParam(payload, []byte("param")); !bytes.Equal(val, []byte("test")) {
t.Error("Should detect attribute", string(val))
}
if val, _, _ := PathParam(payload, []byte("user_id")); !bytes.Equal(val, []byte("1")) {
t.Error("Should detect attribute", string(val))
}
if val, _, _ := PathParam(payload, []byte("user_id")); !bytes.Equal(val, []byte("1")) {
t.Error("Should detect attribute", string(val))
}
}
func TestSetPathParam(t *testing.T) {
var payload, payload_after []byte
var payload, payloadAfter []byte
payload = []byte("POST /post?param=test&user_id=1 HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
payload_after = []byte("POST /post?param=new&user_id=1 HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
payload = []byte("POST /post?param=test&user_id=1 HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
payloadAfter = []byte("POST /post?param=new&user_id=1 HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
if payload = SetPathParam(payload, []byte("param"), []byte("new")); !bytes.Equal(payload, payload_after) {
t.Error("Should replace existing value", string(payload))
}
if payload = SetPathParam(payload, []byte("param"), []byte("new")); !bytes.Equal(payload, payloadAfter) {
t.Error("Should replace existing value", string(payload))
}
payload = []byte("POST /post?param=test&user_id=1 HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
payload_after = []byte("POST /post?param=test&user_id=2 HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
payload = []byte("POST /post?param=test&user_id=1 HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
payloadAfter = []byte("POST /post?param=test&user_id=2 HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
if payload = SetPathParam(payload, []byte("user_id"), []byte("2")); !bytes.Equal(payload, payload_after) {
t.Error("Should replace existing value", string(payload))
}
if payload = SetPathParam(payload, []byte("user_id"), []byte("2")); !bytes.Equal(payload, payloadAfter) {
t.Error("Should replace existing value", string(payload))
}
payload = []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
payloadAfter = []byte("POST /post?param=test HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
payload = []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
payload_after = []byte("POST /post?param=test HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
if payload = SetPathParam(payload, []byte("param"), []byte("test")); !bytes.Equal(payload, payloadAfter) {
t.Error("Should set param if url have no params", string(payload))
}
if payload = SetPathParam(payload, []byte("param"), []byte("test")); !bytes.Equal(payload, payload_after) {
t.Error("Should set param if url have no params", string(payload))
}
payload = []byte("POST /post?param=test HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
payloadAfter = []byte("POST /post?param=test&user_id=1 HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
payload = []byte("POST /post?param=test HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
payload_after = []byte("POST /post?param=test&user_id=1 HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
if payload = SetPathParam(payload, []byte("user_id"), []byte("1")); !bytes.Equal(payload, payload_after) {
t.Error("Should set param at the end if url params", string(payload))
}
if payload = SetPathParam(payload, []byte("user_id"), []byte("1")); !bytes.Equal(payload, payloadAfter) {
t.Error("Should set param at the end if url params", string(payload))
}
}
func TestSetHostHTTP10(t *testing.T) {
var payload, payload_after []byte
var payload, payloadAfter []byte
payload = []byte("POST http://example.com/post HTTP/1.0\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
payload_after = []byte("POST http://new.com/post HTTP/1.0\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
payload = []byte("POST http://example.com/post HTTP/1.0\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
payloadAfter = []byte("POST http://new.com/post HTTP/1.0\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
if payload = SetHost(payload, []byte("http://new.com"), []byte("new.com")); !bytes.Equal(payload, payload_after) {
t.Error("Should replace host", string(payload))
}
}
if payload = SetHost(payload, []byte("http://new.com"), []byte("new.com")); !bytes.Equal(payload, payloadAfter) {
t.Error("Should replace host", string(payload))
}
}
raw_socket_listener/listener.go
+121 -116
View File
@@ -1,128 +1,135 @@
package raw_socket
/*
Package rawSocket provides traffic sniffier using RAW sockets.
Capture traffic from socket using RAW_SOCKET's
http://en.wikipedia.org/wiki/Raw_socket
RAW_SOCKET allow you listen for traffic on any port (e.g. sniffing) because they operate on IP level.
Ports is TCP feature, same as flow control, reliable transmission and etc.
This package implements own TCP layer: TCP packets is parsed using tcp_packet.go, and flow control is managed by tcp_message.go
*/
package rawSocket
import (
"encoding/binary"
"log"
"net"
"strconv"
"bytes"
"bytes"
"encoding/binary"
"log"
"net"
"strconv"
)
// Capture traffic from socket using RAW_SOCKET's
// http://en.wikipedia.org/wiki/Raw_socket
//
// RAW_SOCKET allow you listen for traffic on any port (e.g. sniffing) because they operate on IP level.
// Ports is TCP feature, same as flow control, reliable transmission and etc.
// Since we can't use default TCP libraries RAWTCPLitener implements own TCP layer
// TCP packets is parsed using tcp_packet.go, and flow control is managed by tcp_message.go
// Listener handle traffic capture
type Listener struct {
messages map[string]*TCPMessage // buffer of TCPMessages waiting to be send
// buffer of TCPMessages waiting to be send
messages map[string]*TCPMessage
// Expect: 100-continue request is send in 2 tcp messages
// We store ACK aliases to merge this packets together
ack_aliases map[uint32]uint32
seq_with_data map[uint32]uint32
// Expect: 100-continue request is send in 2 tcp messages
// We store ACK aliases to merge this packets together
ackAliases map[uint32]uint32
// To get ACK of second message we need to compute its Seq and wait for them message
seqWithData map[uint32]uint32
c_packets chan *TCPPacket
c_messages chan *TCPMessage // Messages ready to be send to client
// Messages ready to be send to client
packetsChan chan *TCPPacket
c_del_message chan *TCPMessage // Used for notifications about completed or expired messages
// Messages ready to be send to client
messagesChan chan *TCPMessage
addr string // IP to listen
port int // Port to listen
// Used for notifications about completed or expired messages
messageDelChan chan *TCPMessage
addr string // IP to listen
port int // Port to listen
}
// RAWTCPListen creates a listener to capture traffic from RAW_SOCKET
// NewListener creates and initializes new Listener object
func NewListener(addr string, port string) (rawListener *Listener) {
rawListener = &Listener{}
rawListener = &Listener{}
rawListener.c_packets = make(chan *TCPPacket, 10000)
rawListener.c_messages = make(chan *TCPMessage, 10000)
rawListener.c_del_message = make(chan *TCPMessage, 10000)
rawListener.packetsChan = make(chan *TCPPacket, 10000)
rawListener.messagesChan = make(chan *TCPMessage, 10000)
rawListener.messageDelChan = make(chan *TCPMessage, 10000)
rawListener.messages = make(map[string]*TCPMessage)
rawListener.ack_aliases = make(map[uint32]uint32)
rawListener.seq_with_data = make(map[uint32]uint32)
rawListener.messages = make(map[string]*TCPMessage)
rawListener.ackAliases = make(map[uint32]uint32)
rawListener.seqWithData = make(map[uint32]uint32)
rawListener.addr = addr
rawListener.port, _ = strconv.Atoi(port)
rawListener.addr = addr
rawListener.port, _ = strconv.Atoi(port)
go rawListener.listen()
go rawListener.readRAWSocket()
go rawListener.listen()
go rawListener.readRAWSocket()
return
return
}
func (t *Listener) listen() {
for {
select {
// If message ready for deletion it means that its also complete or expired by timeout
case message := <-t.c_del_message:
t.c_messages <- message
delete(t.ack_aliases, message.packets[0].Ack)
delete(t.messages, message.ID)
for {
select {
// If message ready for deletion it means that its also complete or expired by timeout
case message := <-t.messageDelChan:
t.messagesChan <- message
delete(t.ackAliases, message.Ack)
delete(t.messages, message.ID)
// We need to use channels to process each packet to avoid data races
case packet := <-t.c_packets:
t.processTCPPacket(packet)
}
}
// We need to use channels to process each packet to avoid data races
case packet := <-t.packetsChan:
t.processTCPPacket(packet)
}
}
}
func (t *Listener) readRAWSocket() {
conn, e := net.ListenPacket("ip4:tcp", t.addr)
conn, e := net.ListenPacket("ip4:tcp", t.addr)
if e != nil {
log.Fatal(e)
}
if e != nil {
log.Fatal(e)
}
defer conn.Close()
defer conn.Close()
buf := make([]byte, 4096*10)
for {
buf := make([]byte, 64*1024) // 64kb
// Note: ReadFrom receive messages without IP header
n, addr, err := conn.ReadFrom(buf)
for {
// Note: ReadFrom receive messages without IP header
n, addr, err := conn.ReadFrom(buf)
if err != nil {
log.Println("Error:", err)
continue
}
if err != nil {
log.Println("Error:", err)
continue
}
if n > 0 {
t.parsePacket(addr, buf[:n])
}
}
if n > 0 {
go t.parsePacket(addr, buf[:n])
}
}
}
func (t *Listener) parsePacket(addr net.Addr, buf []byte) {
if t.isIncomingDataPacket(buf) {
new_buf := make([]byte, len(buf))
copy(new_buf, buf)
t.c_packets <- ParseTCPPacket(addr, new_buf)
}
if t.isIncomingDataPacket(buf) {
t.packetsChan <- ParseTCPPacket(addr, buf)
}
}
func (t *Listener) isIncomingDataPacket(buf []byte) bool {
// To avoid full packet parsing every time, we manually parsing values needed for packet filtering
// http://en.wikipedia.org/wiki/Transmission_Control_Protocol
dest_port := binary.BigEndian.Uint16(buf[2:4])
// To avoid full packet parsing every time, we manually parsing values needed for packet filtering
// http://en.wikipedia.org/wiki/Transmission_Control_Protocol
destPort := binary.BigEndian.Uint16(buf[2:4])
// Because RAW_SOCKET can't be bound to port, we have to control it by ourself
if int(dest_port) == t.port {
// Get the 'data offset' (size of the TCP header in 32-bit words)
dataOffset := (buf[12] & 0xF0) >> 4
// Because RAW_SOCKET can't be bound to port, we have to control it by ourself
if int(destPort) == t.port {
// Get the 'data offset' (size of the TCP header in 32-bit words)
dataOffset := (buf[12] & 0xF0) >> 4
// We need only packets with data inside
// Check that the buffer is larger than the size of the TCP header
if len(buf) > int(dataOffset*4) {
// We should create new buffer because go slices is pointers. So buffer data shoud be immutable.
return true
}
}
// We need only packets with data inside
// Check that the buffer is larger than the size of the TCP header
if len(buf) > int(dataOffset*4) {
// We should create new buffer because go slices is pointers. So buffer data shoud be immutable.
return true
}
}
return false
return false
}
var bExpect100ContinueCheck = []byte("Expect: 100-continue")
@@ -132,44 +139,42 @@ var bPOST = []byte("POST")
//
// For TCP message unique id is Acknowledgment number (see tcp_packet.go)
func (t *Listener) processTCPPacket(packet *TCPPacket) {
defer func() { recover() }()
defer func() { recover() }()
var message *TCPMessage
var message *TCPMessage
parent_message_ack, parent_ok := t.seq_with_data[packet.Seq]
if parent_ok {
t.ack_aliases[packet.Ack] = parent_message_ack
delete(t.seq_with_data, packet.Seq)
}
if parentAck, ok := t.seqWithData[packet.Seq]; ok {
t.ackAliases[packet.Ack] = parentAck
delete(t.seqWithData, packet.Seq)
}
ack_alias, alias_ok := t.ack_aliases[packet.Ack]
if alias_ok {
packet.Ack = ack_alias
}
if alias, ok := t.ackAliases[packet.Ack]; ok {
packet.Ack = alias
}
m_id := packet.Addr.String() + strconv.Itoa(int(packet.Ack))
message, ok := t.messages[m_id]
mID := packet.Addr.String() + strconv.Itoa(int(packet.SrcPort)) + strconv.Itoa(int(packet.Ack))
message, ok := t.messages[mID]
if !ok {
// We sending c_del_message channel, so message object can communicate with Listener and notify it if message completed
message = NewTCPMessage(m_id, t.c_del_message)
t.messages[m_id] = message
}
if !ok {
// We sending messageDelChan channel, so message object can communicate with Listener and notify it if message completed
message = NewTCPMessage(mID, t.messageDelChan, packet.Ack)
t.messages[mID] = message
}
if bytes.Equal(packet.Data[0:4], bPOST) {
if bytes.Equal(packet.Data[len(packet.Data)-24:len(packet.Data)-4], bExpect100ContinueCheck) {
t.seq_with_data[packet.Seq + uint32(len(packet.Data))] = packet.Ack
if bytes.Equal(packet.Data[0:4], bPOST) {
if bytes.Equal(packet.Data[len(packet.Data)-24:len(packet.Data)-4], bExpect100ContinueCheck) {
t.seqWithData[packet.Seq+uint32(len(packet.Data))] = packet.Ack
// Removing `Expect: 100-continue` header
packet.Data = append(packet.Data[:len(packet.Data)-24], packet.Data[len(packet.Data)-2:]...)
}
}
// Removing `Expect: 100-continue` header
packet.Data = append(packet.Data[:len(packet.Data)-24], packet.Data[len(packet.Data)-2:]...)
}
}
// Adding packet to message
message.c_packets <- packet
// Adding packet to message
message.packetsChan <- packet
}
// Receive TCP messages from the listener channel
func (t *Listener) Receive() *TCPMessage {
return <-t.c_messages
return <-t.messagesChan
}
raw_socket_listener/tcp_message.go
+63 -61
View File
@@ -1,12 +1,13 @@
package raw_socket
package rawSocket
import (
"log"
"sort"
"time"
"log"
"sort"
"time"
)
const MSG_EXPIRE = 2000 * time.Millisecond
// MsgExpire specify period that message should wait before it considered as finished
const MsgExpire = 2000 * time.Millisecond
// TCPMessage ensure that all TCP packets for given request is received, and processed in right sequence
// Its needed because all TCP message can be fragmented or re-transmitted
@@ -15,91 +16,92 @@ const MSG_EXPIRE = 2000 * time.Millisecond
// Message can be compiled from unique packets with same message_id which sorted by sequence
// Message is received if we didn't receive any packets for 2000ms
type TCPMessage struct {
ID string // Message ID
packets []*TCPPacket
ID string // Message ID
Ack uint32
packets []*TCPPacket
timer *time.Timer // Used for expire check
timer *time.Timer // Used for expire check
c_packets chan *TCPPacket
packetsChan chan *TCPPacket
c_del_message chan *TCPMessage
delChan chan *TCPMessage
}
// NewTCPMessage pointer created from a Acknowledgment number and a channel of messages readuy to be deleted
func NewTCPMessage(ID string, c_del chan *TCPMessage) (msg *TCPMessage) {
msg = &TCPMessage{ID: ID}
func NewTCPMessage(ID string, delChan chan *TCPMessage, Ack uint32) (msg *TCPMessage) {
msg = &TCPMessage{ID: ID, Ack: Ack}
msg.c_packets = make(chan *TCPPacket)
msg.c_del_message = c_del // used for notifying that message completed or expired
msg.packetsChan = make(chan *TCPPacket)
msg.delChan = delChan // used for notifying that message completed or expired
// Every time we receive packet we reset this timer
msg.timer = time.AfterFunc(MSG_EXPIRE, msg.Timeout)
// Every time we receive packet we reset this timer
msg.timer = time.AfterFunc(MsgExpire, msg.Timeout)
go msg.listen()
go msg.listen()
return
return
}
func (t *TCPMessage) listen() {
for {
select {
case packet, more := <-t.c_packets:
if more {
t.AddPacket(packet)
} else {
// Stop loop if channel closed
return
}
}
}
for {
select {
case packet, more := <-t.packetsChan:
if more {
t.AddPacket(packet)
} else {
// Stop loop if channel closed
return
}
}
}
}
// Timeout notifies message to stop listening, close channel and message ready to be sent
func (t *TCPMessage) Timeout() {
select {
// In some cases Timeout can be called multiple times (do not know how yet)
// Ensure that we did not close channel 2 times
case packet, ok := <- t.c_packets:
if ok {
t.AddPacket(packet)
} else {
return
}
default:
close(t.c_packets)
t.c_del_message <- t // Notify RAWListener that message is ready to be send to replay server
}
select {
// In some cases Timeout can be called multiple times (do not know how yet)
// Ensure that we did not close channel 2 times
case packet, ok := <-t.packetsChan:
if ok {
t.AddPacket(packet)
} else {
return
}
default:
close(t.packetsChan)
t.delChan <- t // Notify RAWListener that message is ready to be send to replay server
}
}
// Bytes sorts packets in right orders and return message content
func (t *TCPMessage) Bytes() (output []byte) {
sort.Sort(BySeq(t.packets))
sort.Sort(sortBySeq(t.packets))
for _, v := range t.packets {
output = append(output, v.Data...)
}
for _, v := range t.packets {
output = append(output, v.Data...)
}
return output
return output
}
// AddPacket to the message and ensure packet uniqueness
// TCP allows that packet can be re-send multiple times
func (t *TCPMessage) AddPacket(packet *TCPPacket) {
packetFound := false
packetFound := false
for _, pkt := range t.packets {
if packet.Seq == pkt.Seq {
packetFound = true
break
}
}
for _, pkt := range t.packets {
if packet.Seq == pkt.Seq {
packetFound = true
break
}
}
if packetFound {
log.Println("Received packet with same sequence")
} else {
t.packets = append(t.packets, packet)
}
if packetFound {
log.Println("Received packet with same sequence")
} else {
t.packets = append(t.packets, packet)
}
// Reset message timeout timer
t.timer.Reset(MSG_EXPIRE)
// Reset message timeout timer
t.timer.Reset(MsgExpire)
}
raw_socket_listener/tcp_packet.go
+66 -66
View File
@@ -1,101 +1,101 @@
package raw_socket
package rawSocket
import (
"encoding/binary"
"net"
"strconv"
"strings"
"encoding/binary"
"net"
"strconv"
"strings"
)
// TCP Flags
const (
TCP_FIN = 1 << iota
TCP_SYN
TCP_RST
TCP_PSH
TCP_ACK
TCP_URG
TCP_ECE
TCP_CWR
TCP_NS
fFIN = 1 << iota
fSYN
fRST
fPSH
fACK
fURG
fECE
fCWR
fNS
)
// Simple TCP packet parser
//
// TCPPacket provides tcp packet parser
// Packet structure: http://en.wikipedia.org/wiki/Transmission_Control_Protocol
type TCPPacket struct {
SrcPort uint16
DestPort uint16
Seq uint32
Ack uint32
DataOffset uint8
Flags uint16
Window uint16
Checksum uint16
Urgent uint16
SrcPort uint16
DestPort uint16
Seq uint32
Ack uint32
DataOffset uint8
Flags uint16
Window uint16
Checksum uint16
Urgent uint16
Data []byte
Data []byte
Addr net.Addr
Addr net.Addr
}
// ParseTCPPacket takes address and tcp payload and returns parsed TCPPacket
func ParseTCPPacket(addr net.Addr, b []byte) (p *TCPPacket) {
p = &TCPPacket{Data: b}
p.ParseBasic()
p.Addr = addr
p = &TCPPacket{Data: b}
p.ParseBasic()
p.Addr = addr
return p
return p
}
// Parse TCP Packet, inspired by: https://github.com/miekg/pcap/blob/master/packet.go
func (t *TCPPacket) Parse() {
t.ParseBasic()
t.SrcPort = binary.BigEndian.Uint16(t.Data[0:2])
t.DestPort = binary.BigEndian.Uint16(t.Data[2:4])
t.Flags = binary.BigEndian.Uint16(t.Data[12:14]) & 0x1FF
t.Window = binary.BigEndian.Uint16(t.Data[14:16])
t.Checksum = binary.BigEndian.Uint16(t.Data[16:18])
t.Urgent = binary.BigEndian.Uint16(t.Data[18:20])
t.ParseBasic()
t.DestPort = binary.BigEndian.Uint16(t.Data[2:4])
t.Flags = binary.BigEndian.Uint16(t.Data[12:14]) & 0x1FF
t.Window = binary.BigEndian.Uint16(t.Data[14:16])
t.Checksum = binary.BigEndian.Uint16(t.Data[16:18])
t.Urgent = binary.BigEndian.Uint16(t.Data[18:20])
}
// ParseBasic set of fields
func (t *TCPPacket) ParseBasic() {
t.Seq = binary.BigEndian.Uint32(t.Data[4:8])
t.Ack = binary.BigEndian.Uint32(t.Data[8:12])
t.DataOffset = (t.Data[12] & 0xF0) >> 4
t.SrcPort = binary.BigEndian.Uint16(t.Data[0:2])
t.Seq = binary.BigEndian.Uint32(t.Data[4:8])
t.Ack = binary.BigEndian.Uint32(t.Data[8:12])
t.DataOffset = (t.Data[12] & 0xF0) >> 4
t.Data = t.Data[t.DataOffset*4:]
t.Data = t.Data[t.DataOffset*4:]
}
// String output for a TCP Packet
func (t *TCPPacket) String() string {
return strings.Join([]string{
"Source port: " + strconv.Itoa(int(t.SrcPort)),
"Dest port:" + strconv.Itoa(int(t.DestPort)),
"Sequence:" + strconv.Itoa(int(t.Seq)),
"Acknowledgment:" + strconv.Itoa(int(t.Ack)),
"Header len:" + strconv.Itoa(int(t.DataOffset)),
return strings.Join([]string{
"Source port: " + strconv.Itoa(int(t.SrcPort)),
"Dest port:" + strconv.Itoa(int(t.DestPort)),
"Sequence:" + strconv.Itoa(int(t.Seq)),
"Acknowledgment:" + strconv.Itoa(int(t.Ack)),
"Header len:" + strconv.Itoa(int(t.DataOffset)),
"Flag ns:" + strconv.FormatBool(t.Flags&TCP_NS != 0),
"Flag crw:" + strconv.FormatBool(t.Flags&TCP_CWR != 0),
"Flag ece:" + strconv.FormatBool(t.Flags&TCP_ECE != 0),
"Flag urg:" + strconv.FormatBool(t.Flags&TCP_URG != 0),
"Flag ack:" + strconv.FormatBool(t.Flags&TCP_ACK != 0),
"Flag psh:" + strconv.FormatBool(t.Flags&TCP_PSH != 0),
"Flag rst:" + strconv.FormatBool(t.Flags&TCP_RST != 0),
"Flag syn:" + strconv.FormatBool(t.Flags&TCP_SYN != 0),
"Flag fin:" + strconv.FormatBool(t.Flags&TCP_FIN != 0),
"Flag ns:" + strconv.FormatBool(t.Flags&fNS != 0),
"Flag crw:" + strconv.FormatBool(t.Flags&fCWR != 0),
"Flag ece:" + strconv.FormatBool(t.Flags&fECE != 0),
"Flag urg:" + strconv.FormatBool(t.Flags&fURG != 0),
"Flag ack:" + strconv.FormatBool(t.Flags&fACK != 0),
"Flag psh:" + strconv.FormatBool(t.Flags&fPSH != 0),
"Flag rst:" + strconv.FormatBool(t.Flags&fRST != 0),
"Flag syn:" + strconv.FormatBool(t.Flags&fSYN != 0),
"Flag fin:" + strconv.FormatBool(t.Flags&fFIN != 0),
"Window size:" + strconv.Itoa(int(t.Window)),
"Checksum:" + strconv.Itoa(int(t.Checksum)),
"Window size:" + strconv.Itoa(int(t.Window)),
"Checksum:" + strconv.Itoa(int(t.Checksum)),
"Data size:" + strconv.Itoa(len(t.Data)),
"Data:" + string(t.Data),
}, "\n")
"Data size:" + strconv.Itoa(len(t.Data)),
"Data:" + string(t.Data),
}, "\n")
}
type BySeq []*TCPPacket
type sortBySeq []*TCPPacket
func (a BySeq) Len() int { return len(a) }
func (a BySeq) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
func (a BySeq) Less(i, j int) bool { return a[i].Seq < a[j].Seq }
func (a sortBySeq) Len() int { return len(a) }
func (a sortBySeq) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
func (a sortBySeq) Less(i, j int) bool { return a[i].Seq < a[j].Seq }
settings.go
+13 -4
View File
@@ -8,23 +8,27 @@ import (
)
const (
VERSION = "0.9.6"
// VERSION specifies Gor current version
VERSION = "0.9.8"
)
// Allows to specify multiple flags with same name and collects all values to array
// MultiOption allows to specify multiple flags with same name and collects all values into array
type MultiOption []string
func (h *MultiOption) String() string {
return fmt.Sprint(*h)
}
// Set gets called multiple times for each flag with same name
func (h *MultiOption) Set(value string) error {
*h = append(*h, value)
return nil
}
// AppSettings is the struct of main configuration
type AppSettings struct {
verbose bool
debug bool
stats bool
splitOutput bool
@@ -50,7 +54,8 @@ type AppSettings struct {
modifierConfig HTTPModifierConfig
}
var Settings AppSettings = AppSettings{}
// Settings holds Gor configuration
var Settings AppSettings
func usage() {
fmt.Printf("Gor is a simple http traffic replication tool written in Go. Its main goal is to replay traffic from production servers to staging and dev environments.\nProject page: https://github.com/buger/gor\nAuthor: <Leonid Bugaev> leonsbox@gmail.com\nCurrent Version: %s\n\n", VERSION)
@@ -61,7 +66,8 @@ func usage() {
func init() {
flag.Usage = usage
flag.BoolVar(&Settings.verbose, "verbose", false, "Turn on verbose/debug output")
flag.BoolVar(&Settings.verbose, "verbose", false, "Turn on more verbose output")
flag.BoolVar(&Settings.debug, "debug", false, "Turn on debug output, shows all itercepted traffic. Works only when with `verbose` flag")
flag.BoolVar(&Settings.stats, "stats", false, "Turn on queue stats output")
flag.BoolVar(&Settings.splitOutput, "split-output", false, "By default each output gets same traffic. If set to `true` it splits traffic equally among all outputs.")
@@ -109,12 +115,15 @@ func init() {
flag.Var(&Settings.modifierConfig.headerFilters, "http-allow-header", "A regexp to match a specific header against. Requests with non-matching headers will be dropped:\n\t gor --input-raw :8080 --output-http staging.com --http-allow-header api-version:^v1")
flag.Var(&Settings.modifierConfig.headerFilters, "output-http-header-filter", "WARNING: `--output-http-header-filter` DEPRECATED, use `--http-allow-header` instead")
flag.Var(&Settings.modifierConfig.headerFilters, "http-disallow-header", "A regexp to match a specific header against. Requests with matching headers will be dropped:\n\t gor --input-raw :8080 --output-http staging.com --http-disallow-header \"User-Agent: Replayed by Gor\"")
flag.Var(&Settings.modifierConfig.headerHashFilters, "http-header-limiter", "Takes a fraction of requests, consistently taking or rejecting a request based on the FNV32-1A hash of a specific header:\n\t gor --input-raw :8080 --output-http staging.com --http-header-imiter user-id:25%")
flag.Var(&Settings.modifierConfig.headerHashFilters, "output-http-header-hash-filter", "WARNING: `output-http-header-hash-filter` DEPRECATED, use `--http-header-hash-limiter` instead")
flag.Var(&Settings.modifierConfig.paramHashFilters, "http-param-limiter", "Takes a fraction of requests, consistently taking or rejecting a request based on the FNV32-1A hash of a specific GET param:\n\t gor --input-raw :8080 --output-http staging.com --http-param-limiter user_id:25%")
}
// Debug gets called only if --verbose flag specified
func Debug(args ...interface{}) {
if Settings.verbose {
fmt.Print("[DEBUG] ")
test_input.go
+10 -3
View File
@@ -5,10 +5,12 @@ import (
"encoding/base64"
)
// TestInput used for testing purpose, it allows emitting requests on demand
type TestInput struct {
data chan []byte
}
// NewTestInput constructor for TestInput
func NewTestInput() (i *TestInput) {
i = new(TestInput)
i.data = make(chan []byte, 100)
@@ -23,28 +25,33 @@ func (i *TestInput) Read(data []byte) (int, error) {
return len(buf), nil
}
// EmitGET emits GET request without headers
func (i *TestInput) EmitGET() {
i.data <- []byte("GET / HTTP/1.1\r\n\r\n")
}
// EmitPOST emits POST request with Content-Length
func (i *TestInput) EmitPOST() {
i.data <- []byte("POST /pub/WWW/ HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
}
// EmitChunkedPOST emits POST request with `Transfer-Encoding: chunked` and chunked body
func (i *TestInput) EmitChunkedPOST() {
i.data <- []byte("POST /pub/WWW/ HTTP/1.1\r\nHost: www.w3.org\r\nTransfer-Encoding: chunked\r\n\r\n4\r\nWiki\r\n5\r\npedia\r\ne\r\n in\r\n\r\nchunks.\r\n0\r\n\r\n")
}
func (i *TestInput) EmitFile() {
size := 1024 * 1024 // 1 MB
// EmitLargePOST emits POST request with large payload (5mb)
func (i *TestInput) EmitLargePOST() {
size := 5 * 1024 * 1024 // 5 MB
rb := make([]byte, size)
rand.Read(rb)
rs := base64.URLEncoding.EncodeToString(rb)
i.data <- []byte("POST /pub/WWW/ HTTP/1.1\nHost: www.w3.org\r\n\r\n------WebKitFormBoundaryePkpFF7tjBAqx29L\nContent-Disposition: form-data; name=\"uploadedfile\"; filename=\"hello.o\"\nContent-Type: application/x-object\n" + rs + "\r\n\r\n")
i.data <- []byte("POST / HTTP/1.1\nHost: www.w3.org\nContent-Length:5242880\r\n\r\n" + rs)
}
// EmitOPTIONS emits OPTIONS request, similar to GET
func (i *TestInput) EmitOPTIONS() {
i.data <- []byte("OPTIONS / HTTP/1.1\nHost: www.w3.org\r\n\r\n")
}
test_output.go
+2
View File
@@ -2,10 +2,12 @@ package main
type writeCallback func(data []byte)
// TestOutput used in testing to intercept any output into callback
type TestOutput struct {
cb writeCallback
}
// NewTestOutput constructor for TestOutput, accepts callback which get called on each incoming Write
func NewTestOutput(cb writeCallback) (i *TestOutput) {
i = new(TestOutput)
i.cb = cb