Benchmarking, packaging, and fix issues, tests and perfomance (#797)

### performance
- handling of the very big packet(any size that can be buffered)
- speeding up TCP sessions by using message hints: Added **proto.HasFullPayload** that helps to validate the entire HTTP request, it supports `Chunked` encoding too! Added **proto.HasRequestTitle** and **proto.HasResponseTitle** for validating the beginning of HTTP request. Those methods are used `input_raw.go` with `TCP`.
- supports Keep-Alive: the above functions helps to support keep-alive

### Packaging
- **capture:** engines(capture/doc.go)
- **tcp:** tcp message parser (tcp/doc.go)

### benchmarking
- **capture.BenchmarkPcapDump:** the benchmarks regarding dumping packets in a pcap file
- **capture.BenchmarkPcapFile:** the benchmarks of reading packets from a pcap file
- **capture.BenchmarkPcap:** the benchmarks of parsing packets from the loopback interface with pcap handles
- **proto.BenchmarkHasFullPayload:**: benchmarking this function which validates the HTTP payload
- **tcp.BenchmarkPacketParseAndSort:** benchmarks of parsing and sorting packets
- **tcp.BenchmarkMessageParserWithoutHint:** benchmarks of message reasembling by using `SYN` and `FIN` flag
- **tcp.BenchmarkMessageParserWithHint:** benchmarks of message reasembling by using `proto.HasRequestTitle` and `proto.HasFullPayload` flag

### issues
see linked issues

###  tests
- fixed input raw and engine tests

**Most of the changed of the files, was about using functionalities of** `tcp` **and** `capture` **in existing functionalities**
This commit is contained in:
Urban Ishimwe
2020-08-11 12:44:53 +03:00
committed by GitHub
parent 2f81fba370
commit fdc8b094f0
60 changed files with 3106 additions and 3825 deletions
+12 -38
View File
@@ -4,7 +4,7 @@ import (
"bytes"
"crypto/rand"
"encoding/hex"
"strconv"
"fmt"
)
// These constants help to indicate the type of payload
@@ -48,37 +48,9 @@ func payloadScanner(data []byte, atEOF bool) (advance int, token []byte, err err
// Timing is request start or round-trip time, depending on payloadType
func payloadHeader(payloadType byte, uuid []byte, timing int64, latency int64) (header []byte) {
var sTime, sLatency string
sTime = strconv.FormatInt(timing, 10)
if latency != -1 {
sLatency = strconv.FormatInt(latency, 10)
}
//Example:
// 3 f45590522cd1838b4a0d5c5aab80b77929dea3b3 1231\n
// `+ 1` indicates space characters or end of line
headerLen := 1 + 1 + len(uuid) + 1 + len(sTime) + 1
if latency != -1 {
headerLen += len(sLatency) + 1
}
header = make([]byte, headerLen)
header[0] = payloadType
header[1] = ' '
header[2+len(uuid)] = ' '
header[len(header)-1] = '\n'
copy(header[2:], uuid)
copy(header[3+len(uuid):], sTime)
if latency != -1 {
header[3+len(uuid)+len(sTime)] = ' '
copy(header[4+len(uuid)+len(sTime):], sLatency)
}
return header
// 3 f45590522cd1838b4a0d5c5aab80b77929dea3b3 13923489726487326 1231\n
return []byte(fmt.Sprintf("%c %s %d %d\n", payloadType, uuid, timing, latency))
}
func payloadBody(payload []byte) []byte {
@@ -89,19 +61,21 @@ func payloadBody(payload []byte) []byte {
func payloadMeta(payload []byte) [][]byte {
headerSize := bytes.IndexByte(payload, '\n')
if headerSize < 0 {
headerSize = 0
return nil
}
return bytes.Split(payload[:headerSize], []byte{' '})
}
func payloadID(payload []byte) []byte {
idx := bytes.IndexByte(payload[2:], ' ')
func payloadID(payload []byte) (id []byte) {
meta := payloadMeta(payload)
if idx == -1 {
return []byte{}
if len(meta) < 2 {
return
}
return payload[2 : 2+idx]
// id is encoded in hex, we need to revert to how it was
id = make([]byte, 20)
hex.Decode(id, meta[1])
return
}
func isOriginPayload(payload []byte) bool {