Benchmarking, packaging, and fix issues, tests and perfomance (#797)

### performance
- handling of the very big packet(any size that can be buffered)
- speeding up TCP sessions by using message hints: Added **proto.HasFullPayload** that helps to validate the entire HTTP request, it supports `Chunked` encoding too! Added **proto.HasRequestTitle** and **proto.HasResponseTitle** for validating the beginning of HTTP request. Those methods are used `input_raw.go` with `TCP`.
- supports Keep-Alive: the above functions helps to support keep-alive

### Packaging
- **capture:** engines(capture/doc.go)
- **tcp:** tcp message parser (tcp/doc.go)

### benchmarking
- **capture.BenchmarkPcapDump:** the benchmarks regarding dumping packets in a pcap file
- **capture.BenchmarkPcapFile:** the benchmarks of reading packets from a pcap file
- **capture.BenchmarkPcap:** the benchmarks of parsing packets from the loopback interface with pcap handles
- **proto.BenchmarkHasFullPayload:**: benchmarking this function which validates the HTTP payload
- **tcp.BenchmarkPacketParseAndSort:** benchmarks of parsing and sorting packets
- **tcp.BenchmarkMessageParserWithoutHint:** benchmarks of message reasembling by using `SYN` and `FIN` flag
- **tcp.BenchmarkMessageParserWithHint:** benchmarks of message reasembling by using `proto.HasRequestTitle` and `proto.HasFullPayload` flag

### issues
see linked issues

###  tests
- fixed input raw and engine tests

**Most of the changed of the files, was about using functionalities of** `tcp` **and** `capture` **in existing functionalities**
This commit is contained in:
Urban Ishimwe
2020-08-11 12:44:53 +03:00
committed by GitHub
parent 2f81fba370
commit fdc8b094f0
60 changed files with 3106 additions and 3825 deletions
+25 -19
View File
@@ -1,13 +1,13 @@
package main
import (
"bytes"
"io"
"log"
"net/http"
"os/exec"
"strings"
"sync"
"testing"
"time"
"github.com/buger/goreplay/proto"
)
@@ -17,6 +17,7 @@ func TestHTTPInput(t *testing.T) {
quit := make(chan int)
input := NewHTTPInput("127.0.0.1:0")
time.Sleep(time.Millisecond)
output := NewTestOutput(func(data []byte) {
wg.Done()
})
@@ -30,7 +31,7 @@ func TestHTTPInput(t *testing.T) {
emitter := NewEmitter(quit)
go emitter.Start(plugins, Settings.Middleware)
address := strings.Replace(input.listener.Addr().String(), "[::]", "127.0.0.1", -1)
address := strings.Replace(input.address, "[::]", "127.0.0.1", -1)
for i := 0; i < 100; i++ {
wg.Add(1)
@@ -43,18 +44,17 @@ func TestHTTPInput(t *testing.T) {
func TestInputHTTPLargePayload(t *testing.T) {
wg := new(sync.WaitGroup)
quit := make(chan int)
dd := exec.Command("dd", "if=/dev/urandom", "of=/tmp/large", "bs=1", "count=4000000")
err := dd.Run()
if err != nil {
log.Fatal("dd error:", err)
}
quit := make(chan int, 1)
const n = 10 << 20 // 10MB
var large [n]byte
large[n-1] = '0'
input := NewHTTPInput("127.0.0.1:0")
time.Sleep(time.Millisecond)
output := NewTestOutput(func(data []byte) {
if len(proto.Body(payloadBody(data))) != 4000000 {
t.Error("Should receive full file")
_len := len(proto.Body(payloadBody(data)))
if _len >= n { // considering http body CRLF
t.Errorf("expected body to be >= %d", n)
}
wg.Done()
})
@@ -65,16 +65,22 @@ func TestInputHTTPLargePayload(t *testing.T) {
plugins.All = append(plugins.All, input, output)
emitter := NewEmitter(quit)
defer emitter.Close()
go emitter.Start(plugins, Settings.Middleware)
wg.Add(1)
address := strings.Replace(input.listener.Addr().String(), "[::]", "127.0.0.1", -1)
curl := exec.Command("curl", "http://"+address, "--data-binary", "@/tmp/large")
err = curl.Run()
address := strings.Replace(input.address, "[::]", "127.0.0.1", -1)
var req *http.Request
var err error
req, err = http.NewRequest("POST", "http://"+address, bytes.NewBuffer(large[:]))
if err != nil {
log.Fatal("curl error:", err)
t.Error(err)
return
}
wg.Add(1)
_, err = http.DefaultClient.Do(req)
if err != nil {
t.Error(err)
return
}
wg.Wait()
emitter.Close()
}