mirror of
https://github.com/buger/goreplay.git
synced 2024-04-21 12:32:02 +00:00
Benchmarking, packaging, and fix issues, tests and perfomance (#797)
### performance - handling of the very big packet(any size that can be buffered) - speeding up TCP sessions by using message hints: Added **proto.HasFullPayload** that helps to validate the entire HTTP request, it supports `Chunked` encoding too! Added **proto.HasRequestTitle** and **proto.HasResponseTitle** for validating the beginning of HTTP request. Those methods are used `input_raw.go` with `TCP`. - supports Keep-Alive: the above functions helps to support keep-alive ### Packaging - **capture:** engines(capture/doc.go) - **tcp:** tcp message parser (tcp/doc.go) ### benchmarking - **capture.BenchmarkPcapDump:** the benchmarks regarding dumping packets in a pcap file - **capture.BenchmarkPcapFile:** the benchmarks of reading packets from a pcap file - **capture.BenchmarkPcap:** the benchmarks of parsing packets from the loopback interface with pcap handles - **proto.BenchmarkHasFullPayload:**: benchmarking this function which validates the HTTP payload - **tcp.BenchmarkPacketParseAndSort:** benchmarks of parsing and sorting packets - **tcp.BenchmarkMessageParserWithoutHint:** benchmarks of message reasembling by using `SYN` and `FIN` flag - **tcp.BenchmarkMessageParserWithHint:** benchmarks of message reasembling by using `proto.HasRequestTitle` and `proto.HasFullPayload` flag ### issues see linked issues ### tests - fixed input raw and engine tests **Most of the changed of the files, was about using functionalities of** `tcp` **and** `capture` **in existing functionalities**
This commit is contained in:
Urban Ishimwe
+159
-76
@@ -1,125 +1,208 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log"
|
||||
"net"
|
||||
"strconv"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
raw "github.com/buger/goreplay/capture"
|
||||
"github.com/buger/goreplay/capture"
|
||||
"github.com/buger/goreplay/proto"
|
||||
"github.com/buger/goreplay/size"
|
||||
"github.com/buger/goreplay/tcp"
|
||||
)
|
||||
|
||||
// TCPProtocol is a number to indicate type of protocol
|
||||
type TCPProtocol uint8
|
||||
|
||||
const (
|
||||
// ProtocolHTTP ...
|
||||
ProtocolHTTP TCPProtocol = iota
|
||||
// ProtocolBinary ...
|
||||
ProtocolBinary
|
||||
)
|
||||
|
||||
// Set is here so that TCPProtocol can implement flag.Var
|
||||
func (protocol *TCPProtocol) Set(v string) error {
|
||||
switch v {
|
||||
case "", "http":
|
||||
*protocol = ProtocolHTTP
|
||||
case "binary":
|
||||
*protocol = ProtocolBinary
|
||||
default:
|
||||
return fmt.Errorf("unsupported protocol %s", v)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (protocol *TCPProtocol) String() string {
|
||||
switch *protocol {
|
||||
case ProtocolBinary:
|
||||
return "binary"
|
||||
case ProtocolHTTP:
|
||||
return "http"
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
// RAWInputConfig represents configuration that can be applied on raw input
|
||||
type RAWInputConfig struct {
|
||||
capture.PcapOptions
|
||||
Expire time.Duration `json:"input-raw-expire"`
|
||||
CopyBufferSize size.Size `json:"copy-buffer-size"`
|
||||
Engine capture.EngineType `json:"input-raw-engine"`
|
||||
TrackResponse bool `json:"input-raw-track-response"`
|
||||
Protocol TCPProtocol `json:"input-raw-protocol"`
|
||||
RealIPHeader string `json:"input-raw-realip-header"`
|
||||
Stats bool `json:"input-raw-stats"`
|
||||
quit chan bool // Channel used only to indicate goroutine should shutdown
|
||||
host string
|
||||
port uint16
|
||||
}
|
||||
|
||||
// RAWInput used for intercepting traffic for given address
|
||||
type RAWInput struct {
|
||||
data chan *raw.TCPMessage
|
||||
address string
|
||||
expire time.Duration
|
||||
quit chan bool // Channel used only to indicate goroutine should shutdown
|
||||
engine int
|
||||
realIPHeader []byte
|
||||
trackResponse bool
|
||||
listener *raw.Listener
|
||||
protocol raw.TCPProtocol
|
||||
bpfFilter string
|
||||
timestampType string
|
||||
bufferSize int64
|
||||
sync.Mutex
|
||||
RAWInputConfig
|
||||
messageStats []tcp.Stats
|
||||
listener *capture.Listener
|
||||
message chan *tcp.Message
|
||||
cancelListener context.CancelFunc
|
||||
}
|
||||
|
||||
// Available engines for intercepting traffic
|
||||
const (
|
||||
EngineRawSocket = 1 << iota
|
||||
EnginePcap
|
||||
EnginePcapFile
|
||||
)
|
||||
|
||||
// NewRAWInput constructor for RAWInput. Accepts address with port as argument.
|
||||
func NewRAWInput(address string, engine int, trackResponse bool, expire time.Duration, realIPHeader string, protocol string, bpfFilter string, timestampType string, bufferSize int64) (i *RAWInput) {
|
||||
// NewRAWInput constructor for RAWInput. Accepts raw input config as arguments.
|
||||
func NewRAWInput(address string, config RAWInputConfig) (i *RAWInput) {
|
||||
i = new(RAWInput)
|
||||
i.data = make(chan *raw.TCPMessage)
|
||||
i.address = address
|
||||
i.expire = expire
|
||||
i.engine = engine
|
||||
i.bpfFilter = bpfFilter
|
||||
i.realIPHeader = []byte(realIPHeader)
|
||||
i.RAWInputConfig = config
|
||||
i.message = make(chan *tcp.Message, 1000)
|
||||
i.quit = make(chan bool)
|
||||
i.trackResponse = trackResponse
|
||||
i.timestampType = timestampType
|
||||
i.bufferSize = bufferSize
|
||||
|
||||
switch protocol {
|
||||
case "http":
|
||||
i.protocol = raw.ProtocolHTTP
|
||||
case "binary":
|
||||
i.protocol = raw.ProtocolBinary
|
||||
if !PRO {
|
||||
log.Fatal("Binary protocols can be used only with PRO license")
|
||||
}
|
||||
default:
|
||||
log.Fatal("Unsupported protocol:", protocol)
|
||||
var host, _port string
|
||||
var err error
|
||||
var port int
|
||||
host, _port, err = net.SplitHostPort(address)
|
||||
if err != nil {
|
||||
log.Fatalf("input-raw: error while parsing address: %s", err)
|
||||
}
|
||||
if _port != "" {
|
||||
port, err = strconv.Atoi(_port)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
log.Fatalf("parsing port error: %v", err)
|
||||
}
|
||||
i.host = host
|
||||
i.port = uint16(port)
|
||||
|
||||
i.listen(address)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
func (i *RAWInput) Read(data []byte) (int, error) {
|
||||
var msg *raw.TCPMessage
|
||||
func (i *RAWInput) Read(data []byte) (n int, err error) {
|
||||
var msg *tcp.Message
|
||||
var buf []byte
|
||||
select {
|
||||
case <-i.quit:
|
||||
return 0, ErrorStopped
|
||||
case msg = <-i.data:
|
||||
case msg = <-i.message:
|
||||
buf = msg.Data()
|
||||
}
|
||||
|
||||
buf := msg.Bytes()
|
||||
|
||||
var header []byte
|
||||
|
||||
var msgType byte = ResponsePayload
|
||||
if msg.IsIncoming {
|
||||
header = payloadHeader(RequestPayload, msg.UUID(), msg.Start.UnixNano(), -1)
|
||||
if len(i.realIPHeader) > 0 {
|
||||
buf = proto.SetHeader(buf, i.realIPHeader, []byte(msg.IP().String()))
|
||||
msgType = RequestPayload
|
||||
if i.RealIPHeader != "" {
|
||||
buf = proto.SetHeader(buf, []byte(i.RealIPHeader), []byte(msg.SrcAddr))
|
||||
}
|
||||
} else {
|
||||
header = payloadHeader(ResponsePayload, msg.UUID(), msg.Start.UnixNano(), msg.End.UnixNano()-msg.AssocMessage.End.UnixNano())
|
||||
}
|
||||
header = payloadHeader(msgType, msg.UUID(), msg.Start.UnixNano(), msg.End.UnixNano()-msg.Start.UnixNano())
|
||||
|
||||
copy(data[0:len(header)], header)
|
||||
copy(data[len(header):], buf)
|
||||
|
||||
return len(buf) + len(header), nil
|
||||
n = copy(data, header)
|
||||
if len(data) > len(header) {
|
||||
n += copy(data[len(header):], buf)
|
||||
}
|
||||
dis := len(header) + len(buf) - n
|
||||
if dis > 0 {
|
||||
go Debug(2, "[INPUT-RAW] discarded", dis, "bytes increase copy buffer size")
|
||||
}
|
||||
if msg.Truncated {
|
||||
go Debug(2, "[INPUT-RAW] message truncated, copy-buffer-size")
|
||||
}
|
||||
go i.addStats(msg.Stats)
|
||||
return n, nil
|
||||
}
|
||||
|
||||
func (i *RAWInput) listen(address string) {
|
||||
Debug("Listening for traffic on: " + address)
|
||||
|
||||
host, port, err := net.SplitHostPort(address)
|
||||
var err error
|
||||
i.listener, err = capture.NewListener(i.host, i.port, "", i.Engine, i.TrackResponse)
|
||||
if err != nil {
|
||||
log.Fatalf("input-raw: error while parsing address: %s", err)
|
||||
log.Fatal(err)
|
||||
}
|
||||
i.listener.SetPcapOptions(i.PcapOptions)
|
||||
err = i.listener.Activate()
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
pool := tcp.NewMessagePool(i.CopyBufferSize, i.Expire, Debug, i.handler)
|
||||
pool.End = endHint
|
||||
pool.Start = startHint
|
||||
var ctx context.Context
|
||||
ctx, i.cancelListener = context.WithCancel(context.Background())
|
||||
errCh := i.listener.ListenBackground(ctx, pool.Handler)
|
||||
select {
|
||||
case err := <-errCh:
|
||||
log.Fatal(err)
|
||||
case <-i.listener.Reading:
|
||||
Debug(1, i)
|
||||
}
|
||||
}
|
||||
|
||||
i.listener = raw.NewListener(host, port, i.engine, i.trackResponse, i.expire, i.protocol, i.bpfFilter, i.timestampType, i.bufferSize, Settings.InputRAWConfig.OverrideSnapLen, Settings.InputRAWConfig.ImmediateMode)
|
||||
|
||||
ch := i.listener.Receiver()
|
||||
|
||||
go func() {
|
||||
for {
|
||||
select {
|
||||
case <-i.quit:
|
||||
return
|
||||
case i.data <- <-ch: // Receiving TCPMessage object
|
||||
}
|
||||
}
|
||||
}()
|
||||
func (i *RAWInput) handler(m *tcp.Message) {
|
||||
i.message <- m
|
||||
}
|
||||
|
||||
func (i *RAWInput) String() string {
|
||||
return "Intercepting traffic from: " + i.address
|
||||
return fmt.Sprintf("Intercepting traffic from: %s:%d", i.host, i.port)
|
||||
}
|
||||
|
||||
// GetStats returns the stats so far and reset the stats
|
||||
func (i *RAWInput) GetStats() []tcp.Stats {
|
||||
i.Lock()
|
||||
defer func() {
|
||||
i.messageStats = []tcp.Stats{}
|
||||
i.Unlock()
|
||||
}()
|
||||
return i.messageStats
|
||||
}
|
||||
|
||||
// Close closes the input raw listener
|
||||
func (i *RAWInput) Close() error {
|
||||
i.listener.Close()
|
||||
i.cancelListener()
|
||||
close(i.quit)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (i *RAWInput) addStats(mStats tcp.Stats) {
|
||||
if i.Stats {
|
||||
i.Lock()
|
||||
if len(i.messageStats) >= 10000 {
|
||||
i.messageStats = []tcp.Stats{}
|
||||
}
|
||||
i.messageStats = append(i.messageStats, mStats)
|
||||
|
||||
i.Unlock()
|
||||
}
|
||||
}
|
||||
|
||||
func startHint(pckt *tcp.Packet) (isIncoming, isOutgoing bool) {
|
||||
return proto.HasRequestTitle(pckt.Payload), proto.HasResponseTitle(pckt.Payload)
|
||||
}
|
||||
|
||||
func endHint(m *tcp.Message) bool {
|
||||
return proto.HasFullPayload(m.Data())
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user