mirror of
https://github.com/buger/goreplay.git
synced 2024-04-21 12:32:02 +00:00
Merge pull request #170 from buger/input-raw-performance
Improve performance of input-raw and improve handing of large payloads
This commit is contained in:
Leonid Bugaev
@@ -21,7 +21,7 @@ dcover:
|
||||
go tool cover -html=coverage.out
|
||||
|
||||
dfmt:
|
||||
docker run -v `pwd`:$(SOURCE_PATH) -t -i gor go fmt
|
||||
docker run -v `pwd`:$(SOURCE_PATH) -t -i gor go fmt ./...
|
||||
|
||||
dvet:
|
||||
docker run -v `pwd`:$(SOURCE_PATH) -t -i gor go vet
|
||||
|
||||
+23
-23
@@ -1,37 +1,37 @@
|
||||
package byteutils
|
||||
|
||||
func Cut(a []byte, from, to int) []byte {
|
||||
copy(a[from:], a[to:])
|
||||
a = a[:len(a)-to+from]
|
||||
copy(a[from:], a[to:])
|
||||
a = a[:len(a)-to+from]
|
||||
|
||||
return a
|
||||
return a
|
||||
}
|
||||
|
||||
func Insert(a []byte, i int, b []byte) []byte {
|
||||
a = append(a, make([]byte, len(b))...)
|
||||
copy(a[i+len(b):], a[i:])
|
||||
copy(a[i:i+len(b)], b)
|
||||
a = append(a, make([]byte, len(b))...)
|
||||
copy(a[i+len(b):], a[i:])
|
||||
copy(a[i:i+len(b)], b)
|
||||
|
||||
return a
|
||||
return a
|
||||
}
|
||||
|
||||
// Unlike bytes.Replace it allows you to specify range
|
||||
func Replace(a []byte, from, to int, new []byte) []byte {
|
||||
lenDiff := len(new) - (to - from)
|
||||
lenDiff := len(new) - (to - from)
|
||||
|
||||
if lenDiff > 0 {
|
||||
// Extend if new segment bigger
|
||||
a = append(a, make([]byte, lenDiff)...)
|
||||
copy(a[to+lenDiff:], a[to:])
|
||||
copy(a[from:from+len(new)], new)
|
||||
if lenDiff > 0 {
|
||||
// Extend if new segment bigger
|
||||
a = append(a, make([]byte, lenDiff)...)
|
||||
copy(a[to+lenDiff:], a[to:])
|
||||
copy(a[from:from+len(new)], new)
|
||||
|
||||
return a
|
||||
} else if lenDiff < 0 {
|
||||
copy(a[from:], new)
|
||||
copy(a[from+len(new):],a[to:])
|
||||
return a[:len(a) + lenDiff]
|
||||
} else { // same size
|
||||
copy(a[from:], new)
|
||||
return a
|
||||
}
|
||||
}
|
||||
return a
|
||||
} else if lenDiff < 0 {
|
||||
copy(a[from:], new)
|
||||
copy(a[from+len(new):], a[to:])
|
||||
return a[:len(a)+lenDiff]
|
||||
} else { // same size
|
||||
copy(a[from:], new)
|
||||
return a
|
||||
}
|
||||
}
|
||||
|
||||
+18
-18
@@ -1,32 +1,32 @@
|
||||
package byteutils
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"bytes"
|
||||
"bytes"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestCut(t *testing.T) {
|
||||
if !bytes.Equal(Cut([]byte("123456"), 2, 4), []byte("1256")) {
|
||||
t.Error("Should properly cut")
|
||||
}
|
||||
if !bytes.Equal(Cut([]byte("123456"), 2, 4), []byte("1256")) {
|
||||
t.Error("Should properly cut")
|
||||
}
|
||||
}
|
||||
|
||||
func TestInsert(t *testing.T) {
|
||||
if !bytes.Equal(Insert([]byte("123456"), 2, []byte("abcd")), []byte("12abcd3456")) {
|
||||
t.Error("Should insert into middle of slice")
|
||||
}
|
||||
if !bytes.Equal(Insert([]byte("123456"), 2, []byte("abcd")), []byte("12abcd3456")) {
|
||||
t.Error("Should insert into middle of slice")
|
||||
}
|
||||
}
|
||||
|
||||
func TestReplace(t *testing.T) {
|
||||
if !bytes.Equal(Replace([]byte("123456"), 2, 4, []byte("ab")), []byte("12ab56")) {
|
||||
t.Error("Should replace when same length")
|
||||
}
|
||||
if !bytes.Equal(Replace([]byte("123456"), 2, 4, []byte("ab")), []byte("12ab56")) {
|
||||
t.Error("Should replace when same length")
|
||||
}
|
||||
|
||||
if !bytes.Equal(Replace([]byte("123456"), 2, 4, []byte("abcd")), []byte("12abcd56")) {
|
||||
t.Error("Should replace when replacement length bigger")
|
||||
}
|
||||
if !bytes.Equal(Replace([]byte("123456"), 2, 4, []byte("abcd")), []byte("12abcd56")) {
|
||||
t.Error("Should replace when replacement length bigger")
|
||||
}
|
||||
|
||||
if !bytes.Equal(Replace([]byte("123456"), 2, 5, []byte("ab")), []byte("12ab6")) {
|
||||
t.Error("Should replace when replacement length bigger")
|
||||
}
|
||||
}
|
||||
if !bytes.Equal(Replace([]byte("123456"), 2, 5, []byte("ab")), []byte("12ab6")) {
|
||||
t.Error("Should replace when replacement length bigger")
|
||||
}
|
||||
}
|
||||
|
||||
+1
-1
@@ -40,7 +40,7 @@ func CopyMulty(src io.Reader, writers ...io.Writer) (err error) {
|
||||
}
|
||||
|
||||
if Settings.debug {
|
||||
Debug("[EMITTER] Sending paylod:", string(payload))
|
||||
Debug("[EMITTER] Sending payload, size:", len(payload), "First 500 bytes:", string(payload[0:500]))
|
||||
}
|
||||
|
||||
if Settings.splitOutput {
|
||||
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
"os"
|
||||
"runtime/debug"
|
||||
"runtime/pprof"
|
||||
"runtime"
|
||||
"time"
|
||||
)
|
||||
|
||||
@@ -19,6 +20,11 @@ var (
|
||||
)
|
||||
|
||||
func main() {
|
||||
// If not set via env cariable
|
||||
if len(os.Getenv("GOMAXPROCS")) == 0 {
|
||||
runtime.GOMAXPROCS(runtime.NumCPU() * 2)
|
||||
}
|
||||
|
||||
// Don't exit on panic
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
|
||||
+8
-3
@@ -4,12 +4,12 @@ import (
|
||||
"crypto/tls"
|
||||
"github.com/buger/gor/proto"
|
||||
"io"
|
||||
"log"
|
||||
"net"
|
||||
"net/url"
|
||||
"runtime/debug"
|
||||
"strings"
|
||||
"time"
|
||||
"runtime/debug"
|
||||
"log"
|
||||
)
|
||||
|
||||
var defaultPorts = map[string]string{
|
||||
@@ -103,7 +103,6 @@ func (c *HTTPClient) Send(data []byte) (response []byte, err error) {
|
||||
}
|
||||
}()
|
||||
|
||||
|
||||
if c.conn == nil || !c.isAlive() {
|
||||
Debug("[HTTPClient] Connecting:", c.baseURL)
|
||||
if err = c.Connect(); err != nil {
|
||||
@@ -163,3 +162,9 @@ func (c *HTTPClient) Send(data []byte) (response []byte, err error) {
|
||||
|
||||
return payload, err
|
||||
}
|
||||
|
||||
func (c *HTTPClient) Get(path string) (response []byte, err error) {
|
||||
payload := "GET " + path + " HTTP/1.1\r\n\r\n"
|
||||
|
||||
return c.Send([]byte(payload))
|
||||
}
|
||||
+62
-2
@@ -6,6 +6,7 @@ import (
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/http/httputil"
|
||||
"os/exec"
|
||||
"strings"
|
||||
@@ -30,12 +31,13 @@ func TestRAWInput(t *testing.T) {
|
||||
|
||||
address := strings.Replace(listener.Addr().String(), "[::]", "127.0.0.1", -1)
|
||||
|
||||
client := NewHTTPClient(address, &HTTPClientConfig{})
|
||||
|
||||
go Start(quit)
|
||||
|
||||
for i := 0; i < 100; i++ {
|
||||
wg.Add(1)
|
||||
res, _ := http.Get("http://" + address)
|
||||
res.Body.Close()
|
||||
client.Get("/")
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
@@ -150,3 +152,61 @@ func TestInputRAWChunkedEncoding(t *testing.T) {
|
||||
|
||||
close(quit)
|
||||
}
|
||||
|
||||
func TestInputRAWLargePayload(t *testing.T) {
|
||||
wg := new(sync.WaitGroup)
|
||||
quit := make(chan int)
|
||||
|
||||
// Generate 200kb file
|
||||
dd := exec.Command("dd", "if=/dev/urandom", "of=/tmp/large", "bs=1KB", "count=100")
|
||||
err := dd.Run()
|
||||
if err != nil {
|
||||
log.Fatal("dd error:", err)
|
||||
}
|
||||
|
||||
// Origing and Replay server initialization
|
||||
origin := startHTTP(func(req *http.Request) {
|
||||
defer req.Body.Close()
|
||||
body, _ := ioutil.ReadAll(req.Body)
|
||||
|
||||
if len(body) != 100*1000 {
|
||||
t.Error("File size should be 1mb:", len(body))
|
||||
}
|
||||
|
||||
wg.Done()
|
||||
})
|
||||
origin_address := strings.Replace(origin.Addr().String(), "[::]", "127.0.0.1", -1)
|
||||
|
||||
input := NewRAWInput(origin_address)
|
||||
|
||||
replay := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
|
||||
req.Body = http.MaxBytesReader(w, req.Body, 1*1024*1024)
|
||||
buf := make([]byte, 1*1024*1024)
|
||||
n, _ := req.Body.Read(buf)
|
||||
body := buf[0:n]
|
||||
|
||||
if len(body) != 100*1000 {
|
||||
t.Error("File size should be 100000 bytes:", len(body))
|
||||
}
|
||||
|
||||
wg.Done()
|
||||
}))
|
||||
defer replay.Close()
|
||||
|
||||
http_output := NewHTTPOutput(replay.URL, &HTTPOutputConfig{Debug: false})
|
||||
|
||||
Plugins.Inputs = []io.Reader{input}
|
||||
Plugins.Outputs = []io.Writer{http_output}
|
||||
|
||||
go Start(quit)
|
||||
|
||||
wg.Add(2)
|
||||
curl := exec.Command("curl", "http://"+origin_address, "--header", "Transfer-Encoding: chunked", "--data-binary", "@/tmp/large")
|
||||
err = curl.Run()
|
||||
if err != nil {
|
||||
log.Fatal("curl error:", err)
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
close(quit)
|
||||
}
|
||||
|
||||
+93
-93
@@ -2,9 +2,9 @@
|
||||
package proto
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"github.com/buger/gor/byteutils"
|
||||
_ "log"
|
||||
"bytes"
|
||||
"github.com/buger/gor/byteutils"
|
||||
_ "log"
|
||||
)
|
||||
|
||||
var CLRF = []byte("\r\n")
|
||||
@@ -13,154 +13,154 @@ var HEADER_DELIM = []byte(": ")
|
||||
|
||||
// Headers should end with empty line
|
||||
func MIMEHeadersEndPos(payload []byte) int {
|
||||
return bytes.Index(payload, EMPTY_LINE)
|
||||
return bytes.Index(payload, EMPTY_LINE)
|
||||
}
|
||||
|
||||
func MIMEHeadersStartPos(payload []byte) int {
|
||||
return bytes.Index(payload, CLRF) + 2 // Find first line end
|
||||
return bytes.Index(payload, CLRF) + 2 // Find first line end
|
||||
}
|
||||
|
||||
// Find header value or return error
|
||||
// Do not support multi-line headers
|
||||
func Header(payload []byte, name []byte) (value []byte, headerStart, valueStart, headerEnd int) {
|
||||
headerStart = bytes.Index(payload, name)
|
||||
headerStart = bytes.Index(payload, name)
|
||||
|
||||
if headerStart == -1 {
|
||||
return
|
||||
}
|
||||
if headerStart == -1 {
|
||||
return
|
||||
}
|
||||
|
||||
valueStart = headerStart + len(name) + 1 // Skip ":" after header name
|
||||
if payload[valueStart] == ' ' { // Ignore empty space after ':'
|
||||
valueStart += 1
|
||||
}
|
||||
headerEnd = valueStart + bytes.IndexByte(payload[valueStart:], '\r')
|
||||
value = payload[valueStart:headerEnd]
|
||||
valueStart = headerStart + len(name) + 1 // Skip ":" after header name
|
||||
if payload[valueStart] == ' ' { // Ignore empty space after ':'
|
||||
valueStart += 1
|
||||
}
|
||||
headerEnd = valueStart + bytes.IndexByte(payload[valueStart:], '\r')
|
||||
value = payload[valueStart:headerEnd]
|
||||
|
||||
return
|
||||
return
|
||||
}
|
||||
|
||||
func GetHeader(payload []byte, name string) []byte {
|
||||
val, _, _, _ := Header(payload, []byte(name))
|
||||
val, _, _, _ := Header(payload, []byte(name))
|
||||
|
||||
return val
|
||||
return val
|
||||
}
|
||||
|
||||
func SetHeader(payload, name, value []byte) []byte {
|
||||
_, hs, vs, he := Header(payload, name)
|
||||
_, hs, vs, he := Header(payload, name)
|
||||
|
||||
// If header found
|
||||
if hs != -1 {
|
||||
return byteutils.Replace(payload, vs, he, value)
|
||||
} else {
|
||||
return AddHeader(payload, name, value)
|
||||
}
|
||||
// If header found
|
||||
if hs != -1 {
|
||||
return byteutils.Replace(payload, vs, he, value)
|
||||
} else {
|
||||
return AddHeader(payload, name, value)
|
||||
}
|
||||
}
|
||||
|
||||
func AddHeader(payload, name, value []byte) []byte {
|
||||
header := make([]byte, len(name) + 2 + len(value) + 2)
|
||||
copy(header[0:], name)
|
||||
copy(header[len(name):], HEADER_DELIM)
|
||||
copy(header[len(name)+2:], value)
|
||||
copy(header[len(header)-2:], CLRF)
|
||||
header := make([]byte, len(name)+2+len(value)+2)
|
||||
copy(header[0:], name)
|
||||
copy(header[len(name):], HEADER_DELIM)
|
||||
copy(header[len(name)+2:], value)
|
||||
copy(header[len(header)-2:], CLRF)
|
||||
|
||||
mimeStart := MIMEHeadersStartPos(payload)
|
||||
mimeStart := MIMEHeadersStartPos(payload)
|
||||
|
||||
return byteutils.Insert(payload, mimeStart, header)
|
||||
return byteutils.Insert(payload, mimeStart, header)
|
||||
}
|
||||
|
||||
func Path(payload []byte) []byte {
|
||||
start := bytes.IndexByte(payload, ' ')
|
||||
start += 1
|
||||
start := bytes.IndexByte(payload, ' ')
|
||||
start += 1
|
||||
|
||||
end := bytes.IndexByte(payload[start:], ' ')
|
||||
end := bytes.IndexByte(payload[start:], ' ')
|
||||
|
||||
return payload[start:start+end]
|
||||
return payload[start : start+end]
|
||||
}
|
||||
|
||||
func SetPath(payload, path []byte) []byte {
|
||||
start := bytes.IndexByte(payload, ' ')
|
||||
start += 1
|
||||
start := bytes.IndexByte(payload, ' ')
|
||||
start += 1
|
||||
|
||||
end := bytes.IndexByte(payload[start:], ' ')
|
||||
end := bytes.IndexByte(payload[start:], ' ')
|
||||
|
||||
return byteutils.Replace(payload, start, start+end, path)
|
||||
return byteutils.Replace(payload, start, start+end, path)
|
||||
}
|
||||
|
||||
func PathParam(payload, name []byte) (value []byte, valueStart, valueEnd int) {
|
||||
path := Path(payload)
|
||||
path := Path(payload)
|
||||
|
||||
if paramStart := bytes.Index(path, append(name, '=')); paramStart != -1 {
|
||||
valueStart := paramStart + len(name) + 1
|
||||
paramEnd := bytes.IndexByte(path[valueStart:], '&')
|
||||
if paramEnd == -1 { // It is final param
|
||||
paramEnd = len(path)
|
||||
} else {
|
||||
paramEnd += valueStart
|
||||
}
|
||||
if paramStart := bytes.Index(path, append(name, '=')); paramStart != -1 {
|
||||
valueStart := paramStart + len(name) + 1
|
||||
paramEnd := bytes.IndexByte(path[valueStart:], '&')
|
||||
if paramEnd == -1 { // It is final param
|
||||
paramEnd = len(path)
|
||||
} else {
|
||||
paramEnd += valueStart
|
||||
}
|
||||
|
||||
return path[valueStart:paramEnd], valueStart, paramEnd
|
||||
} else {
|
||||
return []byte(""), -1, -1
|
||||
}
|
||||
return path[valueStart:paramEnd], valueStart, paramEnd
|
||||
} else {
|
||||
return []byte(""), -1, -1
|
||||
}
|
||||
}
|
||||
|
||||
func SetPathParam(payload, name, value []byte) []byte {
|
||||
path := Path(payload)
|
||||
_, vs, ve := PathParam(payload, name)
|
||||
path := Path(payload)
|
||||
_, vs, ve := PathParam(payload, name)
|
||||
|
||||
if vs != -1 {
|
||||
newPath := make([]byte, len(path))
|
||||
copy(newPath, path)
|
||||
newPath = byteutils.Replace(newPath, vs, ve, value)
|
||||
if vs != -1 {
|
||||
newPath := make([]byte, len(path))
|
||||
copy(newPath, path)
|
||||
newPath = byteutils.Replace(newPath, vs, ve, value)
|
||||
|
||||
return SetPath(payload, newPath)
|
||||
} else { // if param not found append to end of url
|
||||
// Adding 2 because of '?' or '&' at start, and '=' in middle
|
||||
newParam := make([]byte, len(name) + len(value) + 2)
|
||||
return SetPath(payload, newPath)
|
||||
} else { // if param not found append to end of url
|
||||
// Adding 2 because of '?' or '&' at start, and '=' in middle
|
||||
newParam := make([]byte, len(name)+len(value)+2)
|
||||
|
||||
if bytes.IndexByte(path, '?') == -1 {
|
||||
newParam[0] = '?'
|
||||
} else {
|
||||
newParam[0] = '&'
|
||||
}
|
||||
if bytes.IndexByte(path, '?') == -1 {
|
||||
newParam[0] = '?'
|
||||
} else {
|
||||
newParam[0] = '&'
|
||||
}
|
||||
|
||||
copy(newParam[1:], name)
|
||||
newParam[1+len(name)] = '='
|
||||
copy(newParam[2+len(name):], value)
|
||||
copy(newParam[1:], name)
|
||||
newParam[1+len(name)] = '='
|
||||
copy(newParam[2+len(name):], value)
|
||||
|
||||
newPath := make([]byte, len(path) + len(newParam))
|
||||
copy(newPath, path)
|
||||
copy(newPath[len(path):], newParam)
|
||||
newPath := make([]byte, len(path)+len(newParam))
|
||||
copy(newPath, path)
|
||||
copy(newPath[len(path):], newParam)
|
||||
|
||||
return SetPath(payload, newPath)
|
||||
}
|
||||
return SetPath(payload, newPath)
|
||||
}
|
||||
}
|
||||
|
||||
func SetHost(payload, url, host []byte) []byte {
|
||||
// If this is HTTP 1.0 traffic or proxy traffic it may include host right into path variable, so instead of setting Host header we rewrite Path
|
||||
// Fix for https://github.com/buger/gor/issues/156
|
||||
if path := Path(payload); bytes.HasPrefix(path, []byte("http")) {
|
||||
hostStart := bytes.IndexByte(path, ':') // : position "https?:"
|
||||
hostStart += 3 // Skip 1 ':' and 2 '\'
|
||||
hostEnd := hostStart + bytes.IndexByte(path[hostStart:], '/')
|
||||
// If this is HTTP 1.0 traffic or proxy traffic it may include host right into path variable, so instead of setting Host header we rewrite Path
|
||||
// Fix for https://github.com/buger/gor/issues/156
|
||||
if path := Path(payload); bytes.HasPrefix(path, []byte("http")) {
|
||||
hostStart := bytes.IndexByte(path, ':') // : position "https?:"
|
||||
hostStart += 3 // Skip 1 ':' and 2 '\'
|
||||
hostEnd := hostStart + bytes.IndexByte(path[hostStart:], '/')
|
||||
|
||||
newPath := make([]byte, len(path))
|
||||
copy(newPath, path)
|
||||
newPath = byteutils.Replace(newPath, 0, hostEnd, url)
|
||||
newPath := make([]byte, len(path))
|
||||
copy(newPath, path)
|
||||
newPath = byteutils.Replace(newPath, 0, hostEnd, url)
|
||||
|
||||
return SetPath(payload, newPath)
|
||||
} else {
|
||||
return SetHeader(payload, []byte("Host"), host)
|
||||
}
|
||||
return SetPath(payload, newPath)
|
||||
} else {
|
||||
return SetHeader(payload, []byte("Host"), host)
|
||||
}
|
||||
}
|
||||
|
||||
func Method(payload []byte) []byte {
|
||||
end := bytes.IndexByte(payload, ' ')
|
||||
end := bytes.IndexByte(payload, ' ')
|
||||
|
||||
return payload[:end]
|
||||
return payload[:end]
|
||||
}
|
||||
|
||||
// Status in response have same position as Path in request
|
||||
func Status(payload []byte) []byte {
|
||||
return Path(payload)
|
||||
}
|
||||
return Path(payload)
|
||||
}
|
||||
|
||||
+86
-92
@@ -1,152 +1,146 @@
|
||||
package proto
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"bytes"
|
||||
"bytes"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestHeader(t *testing.T) {
|
||||
var payload, val []byte
|
||||
var headerStart int
|
||||
var payload, val []byte
|
||||
var headerStart int
|
||||
|
||||
payload = []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
|
||||
payload = []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
|
||||
|
||||
if val, _, _, _ = Header(payload, []byte("Content-Length")); !bytes.Equal(val, []byte("7")) {
|
||||
t.Error("Should find header value")
|
||||
}
|
||||
if val, _, _, _ = Header(payload, []byte("Content-Length")); !bytes.Equal(val, []byte("7")) {
|
||||
t.Error("Should find header value")
|
||||
}
|
||||
|
||||
payload = []byte("POST /post HTTP/1.1\r\nContent-Length:7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
|
||||
payload = []byte("POST /post HTTP/1.1\r\nContent-Length:7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
|
||||
|
||||
if val, _, _, _ = Header(payload, []byte("Content-Length")); !bytes.Equal(val, []byte("7")) {
|
||||
t.Error("Should find header value without space after :")
|
||||
}
|
||||
if val, _, _, _ = Header(payload, []byte("Content-Length")); !bytes.Equal(val, []byte("7")) {
|
||||
t.Error("Should find header value without space after :")
|
||||
}
|
||||
|
||||
if _, headerStart, _, _ = Header(payload, []byte("Not-Found")); headerStart != -1 {
|
||||
t.Error("Should not found header")
|
||||
}
|
||||
if _, headerStart, _, _ = Header(payload, []byte("Not-Found")); headerStart != -1 {
|
||||
t.Error("Should not found header")
|
||||
}
|
||||
}
|
||||
|
||||
func TestMIMEHeadersEndPos(t *testing.T) {
|
||||
head := []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org")
|
||||
payload := []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
|
||||
head := []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org")
|
||||
payload := []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
|
||||
|
||||
end := MIMEHeadersEndPos(payload)
|
||||
end := MIMEHeadersEndPos(payload)
|
||||
|
||||
if !bytes.Equal(payload[:end], head) {
|
||||
t.Error("Wrong headers end position:", end)
|
||||
}
|
||||
if !bytes.Equal(payload[:end], head) {
|
||||
t.Error("Wrong headers end position:", end)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMIMEHeadersStartPos(t *testing.T) {
|
||||
headers := []byte("Content-Length: 7\r\nHost: www.w3.org")
|
||||
payload := []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
|
||||
headers := []byte("Content-Length: 7\r\nHost: www.w3.org")
|
||||
payload := []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
|
||||
|
||||
start := MIMEHeadersStartPos(payload)
|
||||
end := MIMEHeadersEndPos(payload)
|
||||
start := MIMEHeadersStartPos(payload)
|
||||
end := MIMEHeadersEndPos(payload)
|
||||
|
||||
if !bytes.Equal(payload[start:end], headers) {
|
||||
t.Error("Wrong headers end position:", start, end)
|
||||
}
|
||||
if !bytes.Equal(payload[start:end], headers) {
|
||||
t.Error("Wrong headers end position:", start, end)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSetHeader(t *testing.T) {
|
||||
var payload, payload_after []byte
|
||||
var payload, payload_after []byte
|
||||
|
||||
payload = []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
|
||||
payload_after = []byte("POST /post HTTP/1.1\r\nContent-Length: 14\r\nHost: www.w3.org\r\n\r\na=1&b=2")
|
||||
payload = []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
|
||||
payload_after = []byte("POST /post HTTP/1.1\r\nContent-Length: 14\r\nHost: www.w3.org\r\n\r\na=1&b=2")
|
||||
|
||||
if payload = SetHeader(payload, []byte("Content-Length"), []byte("14")); !bytes.Equal(payload, payload_after) {
|
||||
t.Error("Should update header if it exists", string(payload))
|
||||
}
|
||||
if payload = SetHeader(payload, []byte("Content-Length"), []byte("14")); !bytes.Equal(payload, payload_after) {
|
||||
t.Error("Should update header if it exists", string(payload))
|
||||
}
|
||||
|
||||
payload = []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
|
||||
payload_after = []byte("POST /post HTTP/1.1\r\nUser-Agent: Gor\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
|
||||
|
||||
payload = []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
|
||||
payload_after = []byte("POST /post HTTP/1.1\r\nUser-Agent: Gor\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
|
||||
|
||||
if payload = SetHeader(payload, []byte("User-Agent"), []byte("Gor")); !bytes.Equal(payload, payload_after) {
|
||||
t.Error("Should add header if not found", string(payload))
|
||||
}
|
||||
if payload = SetHeader(payload, []byte("User-Agent"), []byte("Gor")); !bytes.Equal(payload, payload_after) {
|
||||
t.Error("Should add header if not found", string(payload))
|
||||
}
|
||||
}
|
||||
|
||||
func TestPath(t *testing.T) {
|
||||
var path, payload []byte
|
||||
var path, payload []byte
|
||||
|
||||
payload = []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
|
||||
payload = []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
|
||||
|
||||
if path = Path(payload); !bytes.Equal(path, []byte("/post")) {
|
||||
t.Error("Should find path", string(path))
|
||||
}
|
||||
if path = Path(payload); !bytes.Equal(path, []byte("/post")) {
|
||||
t.Error("Should find path", string(path))
|
||||
}
|
||||
}
|
||||
|
||||
func TestSetPath(t *testing.T) {
|
||||
var payload, payload_after []byte
|
||||
var payload, payload_after []byte
|
||||
|
||||
payload = []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
|
||||
payload_after = []byte("POST /new_path HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
|
||||
payload = []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
|
||||
payload_after = []byte("POST /new_path HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
|
||||
|
||||
if payload = SetPath(payload, []byte("/new_path")); !bytes.Equal(payload, payload_after) {
|
||||
t.Error("Should replace path", string(payload))
|
||||
}
|
||||
if payload = SetPath(payload, []byte("/new_path")); !bytes.Equal(payload, payload_after) {
|
||||
t.Error("Should replace path", string(payload))
|
||||
}
|
||||
}
|
||||
|
||||
func TestPathParam(t *testing.T) {
|
||||
var payload []byte
|
||||
var payload []byte
|
||||
|
||||
payload = []byte("POST /post?param=test&user_id=1 HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
|
||||
payload = []byte("POST /post?param=test&user_id=1 HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
|
||||
|
||||
if val, _, _ := PathParam(payload, []byte("param")); !bytes.Equal(val, []byte("test")) {
|
||||
t.Error("Should detect attribute", string(val))
|
||||
}
|
||||
if val, _, _ := PathParam(payload, []byte("param")); !bytes.Equal(val, []byte("test")) {
|
||||
t.Error("Should detect attribute", string(val))
|
||||
}
|
||||
|
||||
if val, _, _ := PathParam(payload, []byte("user_id")); !bytes.Equal(val, []byte("1")) {
|
||||
t.Error("Should detect attribute", string(val))
|
||||
}
|
||||
if val, _, _ := PathParam(payload, []byte("user_id")); !bytes.Equal(val, []byte("1")) {
|
||||
t.Error("Should detect attribute", string(val))
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
func TestSetPathParam(t *testing.T) {
|
||||
var payload, payload_after []byte
|
||||
var payload, payload_after []byte
|
||||
|
||||
payload = []byte("POST /post?param=test&user_id=1 HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
|
||||
payload_after = []byte("POST /post?param=new&user_id=1 HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
|
||||
payload = []byte("POST /post?param=test&user_id=1 HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
|
||||
payload_after = []byte("POST /post?param=new&user_id=1 HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
|
||||
|
||||
if payload = SetPathParam(payload, []byte("param"), []byte("new")); !bytes.Equal(payload, payload_after) {
|
||||
t.Error("Should replace existing value", string(payload))
|
||||
}
|
||||
if payload = SetPathParam(payload, []byte("param"), []byte("new")); !bytes.Equal(payload, payload_after) {
|
||||
t.Error("Should replace existing value", string(payload))
|
||||
}
|
||||
|
||||
payload = []byte("POST /post?param=test&user_id=1 HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
|
||||
payload_after = []byte("POST /post?param=test&user_id=2 HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
|
||||
payload = []byte("POST /post?param=test&user_id=1 HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
|
||||
payload_after = []byte("POST /post?param=test&user_id=2 HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
|
||||
|
||||
if payload = SetPathParam(payload, []byte("user_id"), []byte("2")); !bytes.Equal(payload, payload_after) {
|
||||
t.Error("Should replace existing value", string(payload))
|
||||
}
|
||||
if payload = SetPathParam(payload, []byte("user_id"), []byte("2")); !bytes.Equal(payload, payload_after) {
|
||||
t.Error("Should replace existing value", string(payload))
|
||||
}
|
||||
|
||||
payload = []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
|
||||
payload_after = []byte("POST /post?param=test HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
|
||||
|
||||
payload = []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
|
||||
payload_after = []byte("POST /post?param=test HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
|
||||
if payload = SetPathParam(payload, []byte("param"), []byte("test")); !bytes.Equal(payload, payload_after) {
|
||||
t.Error("Should set param if url have no params", string(payload))
|
||||
}
|
||||
|
||||
if payload = SetPathParam(payload, []byte("param"), []byte("test")); !bytes.Equal(payload, payload_after) {
|
||||
t.Error("Should set param if url have no params", string(payload))
|
||||
}
|
||||
payload = []byte("POST /post?param=test HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
|
||||
payload_after = []byte("POST /post?param=test&user_id=1 HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
|
||||
|
||||
|
||||
payload = []byte("POST /post?param=test HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
|
||||
payload_after = []byte("POST /post?param=test&user_id=1 HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
|
||||
|
||||
if payload = SetPathParam(payload, []byte("user_id"), []byte("1")); !bytes.Equal(payload, payload_after) {
|
||||
t.Error("Should set param at the end if url params", string(payload))
|
||||
}
|
||||
if payload = SetPathParam(payload, []byte("user_id"), []byte("1")); !bytes.Equal(payload, payload_after) {
|
||||
t.Error("Should set param at the end if url params", string(payload))
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
func TestSetHostHTTP10(t *testing.T) {
|
||||
var payload, payload_after []byte
|
||||
var payload, payload_after []byte
|
||||
|
||||
payload = []byte("POST http://example.com/post HTTP/1.0\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
|
||||
payload_after = []byte("POST http://new.com/post HTTP/1.0\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
|
||||
payload = []byte("POST http://example.com/post HTTP/1.0\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
|
||||
payload_after = []byte("POST http://new.com/post HTTP/1.0\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2")
|
||||
|
||||
if payload = SetHost(payload, []byte("http://new.com"), []byte("new.com")); !bytes.Equal(payload, payload_after) {
|
||||
t.Error("Should replace host", string(payload))
|
||||
}
|
||||
}
|
||||
if payload = SetHost(payload, []byte("http://new.com"), []byte("new.com")); !bytes.Equal(payload, payload_after) {
|
||||
t.Error("Should replace host", string(payload))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,11 +1,11 @@
|
||||
package raw_socket
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"log"
|
||||
"net"
|
||||
"strconv"
|
||||
"bytes"
|
||||
"bytes"
|
||||
"encoding/binary"
|
||||
"log"
|
||||
"net"
|
||||
"strconv"
|
||||
)
|
||||
|
||||
// Capture traffic from socket using RAW_SOCKET's
|
||||
@@ -16,59 +16,58 @@ import (
|
||||
// Since we can't use default TCP libraries RAWTCPLitener implements own TCP layer
|
||||
// TCP packets is parsed using tcp_packet.go, and flow control is managed by tcp_message.go
|
||||
type Listener struct {
|
||||
messages map[string]*TCPMessage // buffer of TCPMessages waiting to be send
|
||||
messages map[string]*TCPMessage // buffer of TCPMessages waiting to be send
|
||||
|
||||
// Expect: 100-continue request is send in 2 tcp messages
|
||||
// We store ACK aliases to merge this packets together
|
||||
ack_aliases map[uint32]uint32
|
||||
seq_with_data map[uint32]uint32
|
||||
// Expect: 100-continue request is send in 2 tcp messages
|
||||
// We store ACK aliases to merge this packets together
|
||||
ack_aliases map[uint32]uint32
|
||||
seq_with_data map[uint32]uint32
|
||||
|
||||
c_packets chan *TCPPacket
|
||||
c_messages chan *TCPMessage // Messages ready to be send to client
|
||||
c_packets chan *TCPPacket
|
||||
c_messages chan *TCPMessage // Messages ready to be send to client
|
||||
|
||||
c_del_message chan *TCPMessage // Used for notifications about completed or expired messages
|
||||
c_del_message chan *TCPMessage // Used for notifications about completed or expired messages
|
||||
|
||||
addr string // IP to listen
|
||||
port int // Port to listen
|
||||
addr string // IP to listen
|
||||
port int // Port to listen
|
||||
}
|
||||
|
||||
// RAWTCPListen creates a listener to capture traffic from RAW_SOCKET
|
||||
func NewListener(addr string, port string) (rawListener *Listener) {
|
||||
rawListener = &Listener{}
|
||||
rawListener = &Listener{}
|
||||
|
||||
rawListener.c_packets = make(chan *TCPPacket, 10000)
|
||||
rawListener.c_messages = make(chan *TCPMessage, 10000)
|
||||
rawListener.c_del_message = make(chan *TCPMessage, 10000)
|
||||
rawListener.c_packets = make(chan *TCPPacket, 10000)
|
||||
rawListener.c_messages = make(chan *TCPMessage, 10000)
|
||||
rawListener.c_del_message = make(chan *TCPMessage, 10000)
|
||||
|
||||
rawListener.messages = make(map[string]*TCPMessage)
|
||||
rawListener.ack_aliases = make(map[uint32]uint32)
|
||||
rawListener.seq_with_data = make(map[uint32]uint32)
|
||||
rawListener.messages = make(map[string]*TCPMessage)
|
||||
rawListener.ack_aliases = make(map[uint32]uint32)
|
||||
rawListener.seq_with_data = make(map[uint32]uint32)
|
||||
|
||||
rawListener.addr = addr
|
||||
rawListener.port, _ = strconv.Atoi(port)
|
||||
rawListener.addr = addr
|
||||
rawListener.port, _ = strconv.Atoi(port)
|
||||
|
||||
go rawListener.listen()
|
||||
go rawListener.readRAWSocket()
|
||||
go rawListener.listen()
|
||||
go rawListener.readRAWSocket()
|
||||
|
||||
return
|
||||
return
|
||||
}
|
||||
|
||||
func (t *Listener) listen() {
|
||||
for {
|
||||
select {
|
||||
// If message ready for deletion it means that its also complete or expired by timeout
|
||||
case message := <-t.c_del_message:
|
||||
t.c_messages <- message
|
||||
delete(t.ack_aliases, message.packets[0].Ack)
|
||||
delete(t.messages, message.ID)
|
||||
for {
|
||||
select {
|
||||
// If message ready for deletion it means that its also complete or expired by timeout
|
||||
case message := <-t.c_del_message:
|
||||
t.c_messages <- message
|
||||
delete(t.ack_aliases, message.Ack)
|
||||
delete(t.messages, message.ID)
|
||||
|
||||
// We need to use channels to process each packet to avoid data races
|
||||
case packet := <-t.c_packets:
|
||||
t.processTCPPacket(packet)
|
||||
}
|
||||
}
|
||||
// We need to use channels to process each packet to avoid data races
|
||||
case packet := <-t.c_packets:
|
||||
t.processTCPPacket(packet)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (t *Listener) readRAWSocket() {
|
||||
conn, e := net.ListenPacket("ip4:tcp", t.addr)
|
||||
|
||||
@@ -78,9 +77,8 @@ func (t *Listener) readRAWSocket() {
|
||||
|
||||
defer conn.Close()
|
||||
|
||||
buf := make([]byte, 4096*10)
|
||||
|
||||
for {
|
||||
buf := make([]byte, 64*1024) // 64kb
|
||||
// Note: ReadFrom receive messages without IP header
|
||||
n, addr, err := conn.ReadFrom(buf)
|
||||
|
||||
@@ -90,39 +88,36 @@ func (t *Listener) readRAWSocket() {
|
||||
}
|
||||
|
||||
if n > 0 {
|
||||
t.parsePacket(addr, buf[:n])
|
||||
go t.parsePacket(addr, buf[:n])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (t *Listener) parsePacket(addr net.Addr, buf []byte) {
|
||||
if t.isIncomingDataPacket(buf) {
|
||||
new_buf := make([]byte, len(buf))
|
||||
copy(new_buf, buf)
|
||||
|
||||
t.c_packets <- ParseTCPPacket(addr, new_buf)
|
||||
t.c_packets <- ParseTCPPacket(addr, buf)
|
||||
}
|
||||
}
|
||||
|
||||
func (t *Listener) isIncomingDataPacket(buf []byte) bool {
|
||||
// To avoid full packet parsing every time, we manually parsing values needed for packet filtering
|
||||
// http://en.wikipedia.org/wiki/Transmission_Control_Protocol
|
||||
dest_port := binary.BigEndian.Uint16(buf[2:4])
|
||||
// To avoid full packet parsing every time, we manually parsing values needed for packet filtering
|
||||
// http://en.wikipedia.org/wiki/Transmission_Control_Protocol
|
||||
dest_port := binary.BigEndian.Uint16(buf[2:4])
|
||||
|
||||
// Because RAW_SOCKET can't be bound to port, we have to control it by ourself
|
||||
if int(dest_port) == t.port {
|
||||
// Get the 'data offset' (size of the TCP header in 32-bit words)
|
||||
dataOffset := (buf[12] & 0xF0) >> 4
|
||||
// Because RAW_SOCKET can't be bound to port, we have to control it by ourself
|
||||
if int(dest_port) == t.port {
|
||||
// Get the 'data offset' (size of the TCP header in 32-bit words)
|
||||
dataOffset := (buf[12] & 0xF0) >> 4
|
||||
|
||||
// We need only packets with data inside
|
||||
// Check that the buffer is larger than the size of the TCP header
|
||||
if len(buf) > int(dataOffset*4) {
|
||||
// We should create new buffer because go slices is pointers. So buffer data shoud be immutable.
|
||||
return true
|
||||
}
|
||||
}
|
||||
// We need only packets with data inside
|
||||
// Check that the buffer is larger than the size of the TCP header
|
||||
if len(buf) > int(dataOffset*4) {
|
||||
// We should create new buffer because go slices is pointers. So buffer data shoud be immutable.
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
return false
|
||||
}
|
||||
|
||||
var bExpect100ContinueCheck = []byte("Expect: 100-continue")
|
||||
@@ -132,44 +127,44 @@ var bPOST = []byte("POST")
|
||||
//
|
||||
// For TCP message unique id is Acknowledgment number (see tcp_packet.go)
|
||||
func (t *Listener) processTCPPacket(packet *TCPPacket) {
|
||||
defer func() { recover() }()
|
||||
defer func() { recover() }()
|
||||
|
||||
var message *TCPMessage
|
||||
var message *TCPMessage
|
||||
|
||||
parent_message_ack, parent_ok := t.seq_with_data[packet.Seq]
|
||||
if parent_ok {
|
||||
t.ack_aliases[packet.Ack] = parent_message_ack
|
||||
delete(t.seq_with_data, packet.Seq)
|
||||
}
|
||||
parent_message_ack, parent_ok := t.seq_with_data[packet.Seq]
|
||||
if parent_ok {
|
||||
t.ack_aliases[packet.Ack] = parent_message_ack
|
||||
delete(t.seq_with_data, packet.Seq)
|
||||
}
|
||||
|
||||
ack_alias, alias_ok := t.ack_aliases[packet.Ack]
|
||||
if alias_ok {
|
||||
packet.Ack = ack_alias
|
||||
}
|
||||
ack_alias, alias_ok := t.ack_aliases[packet.Ack]
|
||||
if alias_ok {
|
||||
packet.Ack = ack_alias
|
||||
}
|
||||
|
||||
m_id := packet.Addr.String() + strconv.Itoa(int(packet.Ack))
|
||||
message, ok := t.messages[m_id]
|
||||
m_id := packet.Addr.String() + strconv.Itoa(int(packet.SrcPort)) + strconv.Itoa(int(packet.Ack))
|
||||
message, ok := t.messages[m_id]
|
||||
|
||||
if !ok {
|
||||
// We sending c_del_message channel, so message object can communicate with Listener and notify it if message completed
|
||||
message = NewTCPMessage(m_id, t.c_del_message)
|
||||
t.messages[m_id] = message
|
||||
}
|
||||
if !ok {
|
||||
// We sending c_del_message channel, so message object can communicate with Listener and notify it if message completed
|
||||
message = NewTCPMessage(m_id, t.c_del_message, packet.Ack)
|
||||
t.messages[m_id] = message
|
||||
}
|
||||
|
||||
if bytes.Equal(packet.Data[0:4], bPOST) {
|
||||
if bytes.Equal(packet.Data[len(packet.Data)-24:len(packet.Data)-4], bExpect100ContinueCheck) {
|
||||
t.seq_with_data[packet.Seq + uint32(len(packet.Data))] = packet.Ack
|
||||
if bytes.Equal(packet.Data[0:4], bPOST) {
|
||||
if bytes.Equal(packet.Data[len(packet.Data)-24:len(packet.Data)-4], bExpect100ContinueCheck) {
|
||||
t.seq_with_data[packet.Seq+uint32(len(packet.Data))] = packet.Ack
|
||||
|
||||
// Removing `Expect: 100-continue` header
|
||||
packet.Data = append(packet.Data[:len(packet.Data)-24], packet.Data[len(packet.Data)-2:]...)
|
||||
}
|
||||
}
|
||||
// Removing `Expect: 100-continue` header
|
||||
packet.Data = append(packet.Data[:len(packet.Data)-24], packet.Data[len(packet.Data)-2:]...)
|
||||
}
|
||||
}
|
||||
|
||||
// Adding packet to message
|
||||
message.c_packets <- packet
|
||||
// Adding packet to message
|
||||
message.c_packets <- packet
|
||||
}
|
||||
|
||||
// Receive TCP messages from the listener channel
|
||||
func (t *Listener) Receive() *TCPMessage {
|
||||
return <-t.c_messages
|
||||
return <-t.c_messages
|
||||
}
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
package raw_socket
|
||||
|
||||
import (
|
||||
"log"
|
||||
"sort"
|
||||
"time"
|
||||
"log"
|
||||
"sort"
|
||||
"time"
|
||||
)
|
||||
|
||||
const MSG_EXPIRE = 2000 * time.Millisecond
|
||||
@@ -15,91 +15,92 @@ const MSG_EXPIRE = 2000 * time.Millisecond
|
||||
// Message can be compiled from unique packets with same message_id which sorted by sequence
|
||||
// Message is received if we didn't receive any packets for 2000ms
|
||||
type TCPMessage struct {
|
||||
ID string // Message ID
|
||||
packets []*TCPPacket
|
||||
ID string // Message ID
|
||||
Ack uint32
|
||||
packets []*TCPPacket
|
||||
|
||||
timer *time.Timer // Used for expire check
|
||||
timer *time.Timer // Used for expire check
|
||||
|
||||
c_packets chan *TCPPacket
|
||||
c_packets chan *TCPPacket
|
||||
|
||||
c_del_message chan *TCPMessage
|
||||
c_del_message chan *TCPMessage
|
||||
}
|
||||
|
||||
// NewTCPMessage pointer created from a Acknowledgment number and a channel of messages readuy to be deleted
|
||||
func NewTCPMessage(ID string, c_del chan *TCPMessage) (msg *TCPMessage) {
|
||||
msg = &TCPMessage{ID: ID}
|
||||
func NewTCPMessage(ID string, c_del chan *TCPMessage, Ack uint32) (msg *TCPMessage) {
|
||||
msg = &TCPMessage{ID: ID, Ack: Ack}
|
||||
|
||||
msg.c_packets = make(chan *TCPPacket)
|
||||
msg.c_del_message = c_del // used for notifying that message completed or expired
|
||||
msg.c_packets = make(chan *TCPPacket)
|
||||
msg.c_del_message = c_del // used for notifying that message completed or expired
|
||||
|
||||
// Every time we receive packet we reset this timer
|
||||
msg.timer = time.AfterFunc(MSG_EXPIRE, msg.Timeout)
|
||||
// Every time we receive packet we reset this timer
|
||||
msg.timer = time.AfterFunc(MSG_EXPIRE, msg.Timeout)
|
||||
|
||||
go msg.listen()
|
||||
go msg.listen()
|
||||
|
||||
return
|
||||
return
|
||||
}
|
||||
|
||||
func (t *TCPMessage) listen() {
|
||||
for {
|
||||
select {
|
||||
case packet, more := <-t.c_packets:
|
||||
if more {
|
||||
t.AddPacket(packet)
|
||||
} else {
|
||||
// Stop loop if channel closed
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
for {
|
||||
select {
|
||||
case packet, more := <-t.c_packets:
|
||||
if more {
|
||||
t.AddPacket(packet)
|
||||
} else {
|
||||
// Stop loop if channel closed
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Timeout notifies message to stop listening, close channel and message ready to be sent
|
||||
func (t *TCPMessage) Timeout() {
|
||||
select {
|
||||
// In some cases Timeout can be called multiple times (do not know how yet)
|
||||
// Ensure that we did not close channel 2 times
|
||||
case packet, ok := <- t.c_packets:
|
||||
if ok {
|
||||
t.AddPacket(packet)
|
||||
} else {
|
||||
return
|
||||
}
|
||||
default:
|
||||
close(t.c_packets)
|
||||
t.c_del_message <- t // Notify RAWListener that message is ready to be send to replay server
|
||||
}
|
||||
select {
|
||||
// In some cases Timeout can be called multiple times (do not know how yet)
|
||||
// Ensure that we did not close channel 2 times
|
||||
case packet, ok := <-t.c_packets:
|
||||
if ok {
|
||||
t.AddPacket(packet)
|
||||
} else {
|
||||
return
|
||||
}
|
||||
default:
|
||||
close(t.c_packets)
|
||||
t.c_del_message <- t // Notify RAWListener that message is ready to be send to replay server
|
||||
}
|
||||
}
|
||||
|
||||
// Bytes sorts packets in right orders and return message content
|
||||
func (t *TCPMessage) Bytes() (output []byte) {
|
||||
sort.Sort(BySeq(t.packets))
|
||||
sort.Sort(BySeq(t.packets))
|
||||
|
||||
for _, v := range t.packets {
|
||||
output = append(output, v.Data...)
|
||||
}
|
||||
for _, v := range t.packets {
|
||||
output = append(output, v.Data...)
|
||||
}
|
||||
|
||||
return output
|
||||
return output
|
||||
}
|
||||
|
||||
// AddPacket to the message and ensure packet uniqueness
|
||||
// TCP allows that packet can be re-send multiple times
|
||||
func (t *TCPMessage) AddPacket(packet *TCPPacket) {
|
||||
packetFound := false
|
||||
packetFound := false
|
||||
|
||||
for _, pkt := range t.packets {
|
||||
if packet.Seq == pkt.Seq {
|
||||
packetFound = true
|
||||
break
|
||||
}
|
||||
}
|
||||
for _, pkt := range t.packets {
|
||||
if packet.Seq == pkt.Seq {
|
||||
packetFound = true
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if packetFound {
|
||||
log.Println("Received packet with same sequence")
|
||||
} else {
|
||||
t.packets = append(t.packets, packet)
|
||||
}
|
||||
if packetFound {
|
||||
log.Println("Received packet with same sequence")
|
||||
} else {
|
||||
t.packets = append(t.packets, packet)
|
||||
}
|
||||
|
||||
// Reset message timeout timer
|
||||
t.timer.Reset(MSG_EXPIRE)
|
||||
// Reset message timeout timer
|
||||
t.timer.Reset(MSG_EXPIRE)
|
||||
}
|
||||
|
||||
@@ -1,101 +1,101 @@
|
||||
package raw_socket
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"net"
|
||||
"strconv"
|
||||
"strings"
|
||||
"encoding/binary"
|
||||
"net"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// TCP Flags
|
||||
const (
|
||||
TCP_FIN = 1 << iota
|
||||
TCP_SYN
|
||||
TCP_RST
|
||||
TCP_PSH
|
||||
TCP_ACK
|
||||
TCP_URG
|
||||
TCP_ECE
|
||||
TCP_CWR
|
||||
TCP_NS
|
||||
TCP_FIN = 1 << iota
|
||||
TCP_SYN
|
||||
TCP_RST
|
||||
TCP_PSH
|
||||
TCP_ACK
|
||||
TCP_URG
|
||||
TCP_ECE
|
||||
TCP_CWR
|
||||
TCP_NS
|
||||
)
|
||||
|
||||
// Simple TCP packet parser
|
||||
//
|
||||
// Packet structure: http://en.wikipedia.org/wiki/Transmission_Control_Protocol
|
||||
type TCPPacket struct {
|
||||
SrcPort uint16
|
||||
DestPort uint16
|
||||
Seq uint32
|
||||
Ack uint32
|
||||
DataOffset uint8
|
||||
Flags uint16
|
||||
Window uint16
|
||||
Checksum uint16
|
||||
Urgent uint16
|
||||
SrcPort uint16
|
||||
DestPort uint16
|
||||
Seq uint32
|
||||
Ack uint32
|
||||
DataOffset uint8
|
||||
Flags uint16
|
||||
Window uint16
|
||||
Checksum uint16
|
||||
Urgent uint16
|
||||
|
||||
Data []byte
|
||||
Data []byte
|
||||
|
||||
Addr net.Addr
|
||||
Addr net.Addr
|
||||
}
|
||||
|
||||
func ParseTCPPacket(addr net.Addr, b []byte) (p *TCPPacket) {
|
||||
p = &TCPPacket{Data: b}
|
||||
p.ParseBasic()
|
||||
p.Addr = addr
|
||||
p = &TCPPacket{Data: b}
|
||||
p.ParseBasic()
|
||||
p.Addr = addr
|
||||
|
||||
return p
|
||||
return p
|
||||
}
|
||||
|
||||
// Parse TCP Packet, inspired by: https://github.com/miekg/pcap/blob/master/packet.go
|
||||
func (t *TCPPacket) Parse() {
|
||||
t.ParseBasic()
|
||||
t.SrcPort = binary.BigEndian.Uint16(t.Data[0:2])
|
||||
t.DestPort = binary.BigEndian.Uint16(t.Data[2:4])
|
||||
t.Flags = binary.BigEndian.Uint16(t.Data[12:14]) & 0x1FF
|
||||
t.Window = binary.BigEndian.Uint16(t.Data[14:16])
|
||||
t.Checksum = binary.BigEndian.Uint16(t.Data[16:18])
|
||||
t.Urgent = binary.BigEndian.Uint16(t.Data[18:20])
|
||||
t.ParseBasic()
|
||||
t.DestPort = binary.BigEndian.Uint16(t.Data[2:4])
|
||||
t.Flags = binary.BigEndian.Uint16(t.Data[12:14]) & 0x1FF
|
||||
t.Window = binary.BigEndian.Uint16(t.Data[14:16])
|
||||
t.Checksum = binary.BigEndian.Uint16(t.Data[16:18])
|
||||
t.Urgent = binary.BigEndian.Uint16(t.Data[18:20])
|
||||
}
|
||||
|
||||
// ParseBasic set of fields
|
||||
func (t *TCPPacket) ParseBasic() {
|
||||
t.Seq = binary.BigEndian.Uint32(t.Data[4:8])
|
||||
t.Ack = binary.BigEndian.Uint32(t.Data[8:12])
|
||||
t.DataOffset = (t.Data[12] & 0xF0) >> 4
|
||||
t.SrcPort = binary.BigEndian.Uint16(t.Data[0:2])
|
||||
t.Seq = binary.BigEndian.Uint32(t.Data[4:8])
|
||||
t.Ack = binary.BigEndian.Uint32(t.Data[8:12])
|
||||
t.DataOffset = (t.Data[12] & 0xF0) >> 4
|
||||
|
||||
t.Data = t.Data[t.DataOffset*4:]
|
||||
t.Data = t.Data[t.DataOffset*4:]
|
||||
}
|
||||
|
||||
// String output for a TCP Packet
|
||||
func (t *TCPPacket) String() string {
|
||||
return strings.Join([]string{
|
||||
"Source port: " + strconv.Itoa(int(t.SrcPort)),
|
||||
"Dest port:" + strconv.Itoa(int(t.DestPort)),
|
||||
"Sequence:" + strconv.Itoa(int(t.Seq)),
|
||||
"Acknowledgment:" + strconv.Itoa(int(t.Ack)),
|
||||
"Header len:" + strconv.Itoa(int(t.DataOffset)),
|
||||
return strings.Join([]string{
|
||||
"Source port: " + strconv.Itoa(int(t.SrcPort)),
|
||||
"Dest port:" + strconv.Itoa(int(t.DestPort)),
|
||||
"Sequence:" + strconv.Itoa(int(t.Seq)),
|
||||
"Acknowledgment:" + strconv.Itoa(int(t.Ack)),
|
||||
"Header len:" + strconv.Itoa(int(t.DataOffset)),
|
||||
|
||||
"Flag ns:" + strconv.FormatBool(t.Flags&TCP_NS != 0),
|
||||
"Flag crw:" + strconv.FormatBool(t.Flags&TCP_CWR != 0),
|
||||
"Flag ece:" + strconv.FormatBool(t.Flags&TCP_ECE != 0),
|
||||
"Flag urg:" + strconv.FormatBool(t.Flags&TCP_URG != 0),
|
||||
"Flag ack:" + strconv.FormatBool(t.Flags&TCP_ACK != 0),
|
||||
"Flag psh:" + strconv.FormatBool(t.Flags&TCP_PSH != 0),
|
||||
"Flag rst:" + strconv.FormatBool(t.Flags&TCP_RST != 0),
|
||||
"Flag syn:" + strconv.FormatBool(t.Flags&TCP_SYN != 0),
|
||||
"Flag fin:" + strconv.FormatBool(t.Flags&TCP_FIN != 0),
|
||||
"Flag ns:" + strconv.FormatBool(t.Flags&TCP_NS != 0),
|
||||
"Flag crw:" + strconv.FormatBool(t.Flags&TCP_CWR != 0),
|
||||
"Flag ece:" + strconv.FormatBool(t.Flags&TCP_ECE != 0),
|
||||
"Flag urg:" + strconv.FormatBool(t.Flags&TCP_URG != 0),
|
||||
"Flag ack:" + strconv.FormatBool(t.Flags&TCP_ACK != 0),
|
||||
"Flag psh:" + strconv.FormatBool(t.Flags&TCP_PSH != 0),
|
||||
"Flag rst:" + strconv.FormatBool(t.Flags&TCP_RST != 0),
|
||||
"Flag syn:" + strconv.FormatBool(t.Flags&TCP_SYN != 0),
|
||||
"Flag fin:" + strconv.FormatBool(t.Flags&TCP_FIN != 0),
|
||||
|
||||
"Window size:" + strconv.Itoa(int(t.Window)),
|
||||
"Checksum:" + strconv.Itoa(int(t.Checksum)),
|
||||
"Window size:" + strconv.Itoa(int(t.Window)),
|
||||
"Checksum:" + strconv.Itoa(int(t.Checksum)),
|
||||
|
||||
"Data size:" + strconv.Itoa(len(t.Data)),
|
||||
"Data:" + string(t.Data),
|
||||
}, "\n")
|
||||
"Data size:" + strconv.Itoa(len(t.Data)),
|
||||
"Data:" + string(t.Data),
|
||||
}, "\n")
|
||||
}
|
||||
|
||||
type BySeq []*TCPPacket
|
||||
|
||||
func (a BySeq) Len() int { return len(a) }
|
||||
func (a BySeq) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
|
||||
func (a BySeq) Less(i, j int) bool { return a[i].Seq < a[j].Seq }
|
||||
func (a BySeq) Len() int { return len(a) }
|
||||
func (a BySeq) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
|
||||
func (a BySeq) Less(i, j int) bool { return a[i].Seq < a[j].Seq }
|
||||
|
||||
+1
-1
@@ -25,7 +25,7 @@ func (h *MultiOption) Set(value string) error {
|
||||
|
||||
type AppSettings struct {
|
||||
verbose bool
|
||||
debug bool
|
||||
debug bool
|
||||
stats bool
|
||||
|
||||
splitOutput bool
|
||||
|
||||
+3
-3
@@ -35,14 +35,14 @@ func (i *TestInput) EmitChunkedPOST() {
|
||||
i.data <- []byte("POST /pub/WWW/ HTTP/1.1\r\nHost: www.w3.org\r\nTransfer-Encoding: chunked\r\n\r\n4\r\nWiki\r\n5\r\npedia\r\ne\r\n in\r\n\r\nchunks.\r\n0\r\n\r\n")
|
||||
}
|
||||
|
||||
func (i *TestInput) EmitFile() {
|
||||
size := 1024 * 1024 // 1 MB
|
||||
func (i *TestInput) EmitLargePOST() {
|
||||
size := 5 * 1024 * 1024 // 5 MB
|
||||
rb := make([]byte, size)
|
||||
rand.Read(rb)
|
||||
|
||||
rs := base64.URLEncoding.EncodeToString(rb)
|
||||
|
||||
i.data <- []byte("POST /pub/WWW/ HTTP/1.1\nHost: www.w3.org\r\n\r\n------WebKitFormBoundaryePkpFF7tjBAqx29L\nContent-Disposition: form-data; name=\"uploadedfile\"; filename=\"hello.o\"\nContent-Type: application/x-object\n" + rs + "\r\n\r\n")
|
||||
i.data <- []byte("POST / HTTP/1.1\nHost: www.w3.org\nContent-Length:5242880\r\n\r\n" + rs)
|
||||
}
|
||||
|
||||
func (i *TestInput) EmitOPTIONS() {
|
||||
|
||||
Reference in New Issue
Block a user