diff --git a/.gitignore b/.gitignore index 4c1635a..6bc8d66 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,7 @@ *.bin *.gz +*.zip *.class @@ -17,3 +18,5 @@ gor *.mprof *.pcap + +.DS_Store diff --git a/Dockerfile b/Dockerfile index 91435a0..6f1e3cb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,5 +18,7 @@ RUN go get -u github.com/golang/lint/golint WORKDIR /go/src/github.com/buger/gor-pro/ ADD . /go/src/github.com/buger/gor-pro/ -RUN javac -cp /tmp/commons-io-2.4/commons-io-2.4.jar ./examples/middleware/echo.java +RUN wget http://archive.apache.org/dist/commons/io/binaries/commons-io-2.4-bin.tar.gz && tar xzf commons-io-2.4-bin.tar.gz && cd commons-io-2.4 && mv commons-io-2.4.jar /tmp/ +RUN wget http://archive.apache.org/dist/commons/codec/binaries/commons-codec-1.9-bin.tar.gz && tar xzf commons-codec-1.9-bin.tar.gz +RUN javac -cp commons-io-2.4/commons-io-2.4.jar -cp commons-codec-1.9/commons-codec-1.9.jar ./examples/middleware/echo.java RUN go get \ No newline at end of file diff --git a/elasticsearch.go b/elasticsearch.go index a626853..2635b89 100644 --- a/elasticsearch.go +++ b/elasticsearch.go @@ -84,9 +84,11 @@ func (p *ESPlugin) Init(URI string) { p.done = make(chan bool) p.indexor.Start() - // Only start the ErrorHandler goroutine when in verbose mode - // no need to burn ressources otherwise - go p.ErrorHandler() + if Settings.verbose { + // Only start the ErrorHandler goroutine when in verbose mode + // no need to burn ressources otherwise + go p.ErrorHandler() + } log.Println("Initialized Elasticsearch Plugin") return diff --git a/emitter.go b/emitter.go index 1cedd35..49db03a 100644 --- a/emitter.go +++ b/emitter.go @@ -16,7 +16,7 @@ func Start(stop chan int) { middleware.ReadFrom(in) } - // We going only to read responses, so using same ReadFrom method + // We are going only to read responses, so using same ReadFrom method for _, out := range Plugins.Outputs { if r, ok := out.(io.Reader); ok { middleware.ReadFrom(r) diff --git a/examples/middleware/echo.java b/examples/middleware/echo.java index ffa885c..9fba333 100644 --- a/examples/middleware/echo.java +++ b/examples/middleware/echo.java @@ -6,19 +6,19 @@ import org.apache.commons.codec.DecoderException; import org.apache.commons.codec.binary.Hex; -public class Echo { - public static String decodeHexString(String s) throws DecoderException { - return new String(Hex.decodeHex(s.toCharArray())); - } +class Echo { + public static String decodeHexString(String s) throws DecoderException { + return new String(Hex.decodeHex(s.toCharArray())); + } - public static String encodeHexString(String s) { - return new String(Hex.encodeHex(s.getBytes())); - } + public static String encodeHexString(String s) { + return new String(Hex.encodeHex(s.getBytes())); + } - public static String transformHTTPMessage(String req) { - // do actual transformations here - return req; - } + public static String transformHTTPMessage(String req) { + // do actual transformations here + return req; + } public static void main(String[] args) throws DecoderException { if(args != null){ @@ -29,7 +29,7 @@ public class Echo { } BufferedReader stdin = new BufferedReader(new InputStreamReader( - System.in)); + System.in)); String line = null; try { diff --git a/http_client.go b/http_client.go index c4ec8f2..aeb322f 100644 --- a/http_client.go +++ b/http_client.go @@ -58,15 +58,10 @@ func NewHTTPClient(baseURL string, config *HTTPClientConfig) *HTTPClient { } u, _ := url.Parse(baseURL) - if !strings.Contains(u.Host, ":") { - if u.Scheme != "http" { - u.Host += ":" + defaultPorts[u.Scheme] - } - } - if config.Timeout.Nanoseconds() == 0 { - config.Timeout = 5 * time.Second - } + if config.Timeout == 0 { + config.Timeout = time.Second + } config.ConnectionTimeout = config.Timeout @@ -92,13 +87,13 @@ func (c *HTTPClient) Connect() (err error) { c.Disconnect() if !strings.Contains(c.host, ":") { - c.conn, err = net.DialTimeout("tcp", c.host+":80", c.config.ConnectionTimeout) + c.conn, err = net.DialTimeout("tcp", c.host + ":" + defaultPorts[c.scheme], c.config.ConnectionTimeout) } else { c.conn, err = net.DialTimeout("tcp", c.host, c.config.ConnectionTimeout) } if c.scheme == "https" { - tlsConn := tls.Client(c.conn, &tls.Config{InsecureSkipVerify: true}) + tlsConn := tls.Client(c.conn, &tls.Config{InsecureSkipVerify: true, ServerName: c.host}) if err = tlsConn.Handshake(); err != nil { return @@ -216,9 +211,14 @@ func (c *HTTPClient) Send(data []byte) (response []byte, err error) { if bytes.Equal(proto.Header(c.respBuf, []byte("Transfer-Encoding")), []byte("chunked")) { chunked = true } else { - l := proto.Header(c.respBuf, []byte("Content-Length")) - if len(l) > 0 { - contentLength, _ = strconv.Atoi(string(l)) + status, _ := strconv.Atoi(string(proto.Status(c.respBuf))) + if (status >= 100 && status < 200) || status == 204 || status == 304 { + contentLength = 0 + } else { + l := proto.Header(c.respBuf, []byte("Content-Length")) + if len(l) > 0 { + contentLength, _ = strconv.Atoi(string(l)) + } } } @@ -322,6 +322,11 @@ func (c *HTTPClient) Send(data []byte) (response []byte, err error) { } } + if bytes.Equal(proto.Status(payload), []byte("400")) { + c.Disconnect() + Debug("[HTTPClient] Closed connection on 400 response") + } + c.redirectsCount = 0 return payload, err diff --git a/output_kafka.go b/output_kafka.go new file mode 100644 index 0000000..4ebfc4c --- /dev/null +++ b/output_kafka.go @@ -0,0 +1,96 @@ +package main + +import ( + "encoding/json" + "github.com/Shopify/sarama" + "github.com/buger/gor/proto" + "io" + "log" + "strings" + "time" +) + +// KafkaConfig should contains required information to +// build producers. +type KafkaConfig struct { + host string + topic string +} + +// KafkaOutput should make producer client. +type KafkaOutput struct { + config *KafkaConfig + producer sarama.AsyncProducer +} + +// KafkaMessage should contains catched request information that should be +// passed as Json to Apache Kafka. +type KafkaMessage struct { + ReqURL string `json:"Req_URL"` + ReqMethod string `json:"Req_Method"` + ReqBody string `json:"Req_Body,omitempty"` + ReqHeaders map[string]string `json:"Req_Headers,omitempty"` +} + +// KafkaOutputFrequency in milliseconds +const KafkaOutputFrequency = 500 + +// NewKafkaOutput creates instance of kafka producer client. +func NewKafkaOutput(address string, config *KafkaConfig) io.Writer { + c := sarama.NewConfig() + c.Producer.RequiredAcks = sarama.WaitForLocal + c.Producer.Compression = sarama.CompressionSnappy + c.Producer.Flush.Frequency = KafkaOutputFrequency * time.Millisecond + + brokerList := strings.Split(config.host, ",") + + producer, err := sarama.NewAsyncProducer(brokerList, c) + if err != nil { + log.Fatalln("Failed to start Sarama(Kafka) producer:", err) + } + + o := &KafkaOutput{ + config: config, + producer: producer, + } + + if Settings.verbose { + // Start infinite loop for tracking errors for kafka producer. + go o.ErrorHandler() + } + + return o +} + +// ErrorHandler should receive errors +func (o *KafkaOutput) ErrorHandler() { + for err := range o.producer.Errors() { + log.Println("Failed to write access log entry:", err) + } +} + +func (o *KafkaOutput) Write(data []byte) (n int, err error) { + headers := make(map[string]string) + proto.ParseHeaders([][]byte{data}, func(header []byte, value []byte) bool { + headers[string(header)] = string(value) + return true + }) + + req := payloadBody(data) + + kafkaMessage := KafkaMessage{ + ReqURL: string(proto.Path(req)), + ReqMethod: string(proto.Method(req)), + ReqBody: string(proto.Body(req)), + ReqHeaders: headers, + } + jsonMessage, _ := json.Marshal(&kafkaMessage) + message := sarama.StringEncoder(jsonMessage) + + o.producer.Input() <- &sarama.ProducerMessage{ + Topic: o.config.topic, + Value: message, + } + + return len(message), nil +} diff --git a/plugins.go b/plugins.go index 42e5699..5a215b0 100644 --- a/plugins.go +++ b/plugins.go @@ -146,4 +146,8 @@ func InitPlugins() { for _, options := range Settings.outputBinary { registerPlugin(NewBinaryOutput, options, &Settings.outputBinaryConfig) } + + if Settings.outputKafkaConfig.host != "" && Settings.outputKafkaConfig.topic != "" { + registerPlugin(NewKafkaOutput, "", &Settings.outputKafkaConfig) + } } diff --git a/proto/proto.go b/proto/proto.go index 0bdb819..e63ac9f 100644 --- a/proto/proto.go +++ b/proto/proto.go @@ -190,6 +190,7 @@ func ParseHeaders(payloads [][]byte, cb func(header []byte, value []byte) bool) i := 0 pIdx := 0 lineBreaks := 0 + newLineBreak := true for { if len(payloads)-1 < pIdx { @@ -206,6 +207,7 @@ func ParseHeaders(payloads [][]byte, cb func(header []byte, value []byte) bool) switch p[i] { case '\r', '\n': + newLineBreak = true lineBreaks++ // End of headers @@ -254,7 +256,10 @@ func ParseHeaders(payloads [][]byte, cb func(header []byte, value []byte) bool) hS = [2]int{-1, -1} hE = [2]int{-1, -1} case ':': - hE = [2]int{pIdx, i} + if newLineBreak { + hE = [2]int{pIdx, i} + newLineBreak = false + } default: lineBreaks = 0 @@ -443,7 +448,7 @@ func Status(payload []byte) []byte { } var httpMethods []string = []string{ - "GET ", "OPTI", "HEAD", "POST", "PUT ", "DELE", "TRAC", "CONN" /* custom methods */, "BAN", "PURG", + "GET ", "OPTI", "HEAD", "POST", "PUT ", "DELE", "TRAC", "CONN", "PATC" /* custom methods */, "BAN", "PURG", } func IsHTTPPayload(payload []byte) bool { diff --git a/proto/proto_test.go b/proto/proto_test.go index 32da8c9..800799b 100644 --- a/proto/proto_test.go +++ b/proto/proto_test.go @@ -139,11 +139,64 @@ func TestParseHeaders(t *testing.T) { "Host": "www.w3.org", "User-Agent": "Chrome", } + if !reflect.DeepEqual(headers, expected) { t.Error("Headers do not properly parsed", headers) } } +func TestParseHeadersWithComplexUserAgent(t *testing.T) { + // User-Agent could contain inside ':' + // Parser should wait for \r\n + payload := [][]byte{[]byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.or"), []byte("g\r\nUser-Ag"), []byte("ent:Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko\r\n\r\n"), []byte("Fake-Header: asda")} + + headers := make(map[string]string) + + ParseHeaders(payload, func(header []byte, value []byte) bool { + headers[string(header)] = string(value) + return true + }) + + expected := map[string]string{ + "User-Agent": "Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko", + } + + if expected["User-Agent"] != headers["User-Agent"] { + t.Errorf("Header 'User-Agent' expected '%s' and parsed: '%s'", expected["User-Agent"], headers["User-Agent"]) + } +} + +func TestParseHeadersWithOrigin(t *testing.T) { + // User-Agent could contain inside ':' + // Parser should wait for \r\n + payload := [][]byte{[]byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.or"), []byte("g\r\nReferrer: http://127.0.0.1:3000\r\nOrigi"), []byte("n: https://www.example.com\r\nUser-Ag"), []byte("ent:Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko\r\n\r\n"), []byte("in:https://www.example.com\r\n\r\n"), []byte("Fake-Header: asda")} + + headers := make(map[string]string) + + ParseHeaders(payload, func(header []byte, value []byte) bool { + headers[string(header)] = string(value) + return true + }) + + expected := map[string]string{ + "Origin": "https://www.example.com", + "User-Agent": "Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko", + "Referrer": "http://127.0.0.1:3000", + } + + if expected["Referrer"] != headers["Referrer"] { + t.Errorf("Header 'Referrer' expected '%s' and parsed: '%s'", expected["Referrer"], headers["Referrer"]) + } + + if expected["Origin"] != headers["Origin"] { + t.Errorf("Header 'Origin' expected '%s' and parsed: '%s'", expected["Origin"], headers["Origin"]) + } + + if expected["User-Agent"] != headers["User-Agent"] { + t.Errorf("Header 'User-Agent' expected '%s' and parsed: '%s'", expected["User-Agent"], headers["User-Agent"]) + } +} + func TestHeaderEquals(t *testing.T) { tests := []struct { h1 string diff --git a/raw_socket_listener/listener.go b/raw_socket_listener/listener.go index b9d87b7..cf7e38a 100644 --- a/raw_socket_listener/listener.go +++ b/raw_socket_listener/listener.go @@ -33,6 +33,12 @@ import ( var _ = fmt.Println +type packet struct { + srcIP []byte + data []byte + timestamp time.Time +} + // Listener handle traffic capture type Listener struct { mu sync.Mutex @@ -53,7 +59,7 @@ type Listener struct { respWithoutReq map[uint32]tcpID // Messages ready to be send to client - packetsChan chan []byte + packetsChan chan *packet // Messages ready to be send to client messagesChan chan *TCPMessage @@ -90,7 +96,7 @@ const ( func NewListener(addr string, port string, engine int, trackResponse bool, expire time.Duration, protocol TCPProtocol) (l *Listener) { l = &Listener{} - l.packetsChan = make(chan []byte, 10000) + l.packetsChan = make(chan *packet, 10000) l.messagesChan = make(chan *TCPMessage, 10000) l.quit = make(chan bool) l.readyCh = make(chan bool, 1) @@ -140,9 +146,9 @@ func (t *Listener) listen() { t.conn.Close() } return - case data := <-t.packetsChan: - packet := ParseTCPPacket(data[:16], data[16:]) - t.processTCPPacket(packet) + case packet := <-t.packetsChan: + tcpPacket := ParseTCPPacket(packet.srcIP, packet.data, packet.timestamp) + t.processTCPPacket(tcpPacket) case <-gcTicker: now := time.Now() @@ -468,7 +474,7 @@ func (t *Listener) readPcap() { } // Truncated TCP info - if len(data) < 13 { + if len(data) <= 13 { continue } @@ -525,11 +531,7 @@ func (t *Listener) readPcap() { } } - newBuf := make([]byte, len(data)+16) - copy(newBuf[:16], srcIP) - copy(newBuf[16:], data) - - t.packetsChan <- newBuf + t.packetsChan <- t.buildPacket(srcIP, data, packet.Metadata().Timestamp) } } }(d) @@ -592,11 +594,7 @@ func (t *Listener) readPcapFile() { continue } - newBuf := make([]byte, len(data)+16) - copy(newBuf[:16], addr) - copy(newBuf[16:], data) - - t.packetsChan <- newBuf + t.packetsChan <- t.buildPacket(addr, data, packet.Metadata().Timestamp) } } } @@ -629,16 +627,26 @@ func (t *Listener) readRAWSocket() { if n > 0 { if t.isValidPacket(buf[:n]) { - newBuf := make([]byte, n+16) - copy(newBuf[16:], buf[:n]) - copy(newBuf[:16], []byte(addr.(*net.IPAddr).IP)) - - t.packetsChan <- newBuf + t.packetsChan <- t.buildPacket([]byte(addr.(*net.IPAddr).IP), buf[:n], time.Now()) } } } } +func (t *Listener) buildPacket(packetSrcIP []byte, packetData []byte, timestamp time.Time) *packet { + copyPacketSrcIP := make([]byte, 16) + copyPacketData := make([]byte, len(packetData)) + + copy(copyPacketSrcIP, packetSrcIP) + copy(copyPacketData, packetSrcIP) + + return &packet{ + srcIP: packetSrcIP, + data: packetData, + timestamp:timestamp, + } +} + func (t *Listener) isValidPacket(buf []byte) bool { // To avoid full packet parsing every time, we manually parsing values needed for packet filtering // http://en.wikipedia.org/wiki/Transmission_Control_Protocol @@ -723,7 +731,7 @@ func (t *Listener) processTCPPacket(packet *TCPPacket) { message, ok := t.messages[packet.ID] if !ok { - message = NewTCPMessage(packet.Seq, packet.Ack, isIncoming, t.protocol) + message = NewTCPMessage(packet.Seq, packet.Ack, isIncoming, t.protocol, packet.timestamp)) t.messages[packet.ID] = message if !isIncoming { diff --git a/raw_socket_listener/listener_test.go b/raw_socket_listener/listener_test.go index 9f81174..ef11333 100644 --- a/raw_socket_listener/listener_test.go +++ b/raw_socket_listener/listener_test.go @@ -15,13 +15,13 @@ func TestRawListenerInput(t *testing.T) { listener := NewListener("", "0", EnginePcap, true, 10*time.Millisecond, ProtocolHTTP) defer listener.Close() - reqPacket := buildPacket(true, 1, 1, []byte("GET / HTTP/1.1\r\n\r\n")) + reqPacket := buildPacket(true, 1, 1, []byte("GET / HTTP/1.1\r\n\r\n"), time.Now()) respAck := reqPacket.Seq + uint32(len(reqPacket.Data)) - respPacket := buildPacket(false, respAck, reqPacket.Seq+1, []byte("HTTP/1.1 200 OK\r\n\r\n")) + respPacket := buildPacket(false, respAck, reqPacket.Seq+1, []byte("HTTP/1.1 200 OK\r\n\r\n"), time.Now()) - listener.packetsChan <- reqPacket.Dump() - listener.packetsChan <- respPacket.Dump() + listener.packetsChan <- reqPacket.dump() + listener.packetsChan <- respPacket.dump() select { case req = <-listener.messagesChan: @@ -52,16 +52,16 @@ func TestRawListenerInputResponseByClose(t *testing.T) { listener := NewListener("", "0", EnginePcap, true, 10*time.Millisecond) defer listener.Close() - reqPacket := buildPacket(true, 1, 1, []byte("GET / HTTP/1.1\r\n\r\n")) + reqPacket := buildPacket(true, 1, 1, []byte("GET / HTTP/1.1\r\n\r\n"), time.Now()) respAck := reqPacket.Seq + uint32(len(reqPacket.Data)) - respPacket := buildPacket(false, respAck, reqPacket.Seq+1, []byte("HTTP/1.1 200 OK\r\nConnection: close\r\n\r\nasd")) - finPacket := buildPacket(false, respAck, reqPacket.Seq+2, []byte("")) + respPacket := buildPacket(false, respAck, reqPacket.Seq+1, []byte("HTTP/1.1 200 OK\r\nConnection: close\r\n\r\nasd"), time.Now()) + finPacket := buildPacket(false, respAck, reqPacket.Seq+2, []byte(""), time.Now()) finPacket.IsFIN = true - listener.packetsChan <- reqPacket.Dump() - listener.packetsChan <- respPacket.Dump() - listener.packetsChan <- finPacket.Dump() + listener.packetsChan <- reqPacket.dump() + listener.packetsChan <- respPacket.dump() + listener.packetsChan <- finPacket.dump() select { case req = <-listener.messagesChan: @@ -92,9 +92,9 @@ func TestRawListenerInputWithoutResponse(t *testing.T) { listener := NewListener("", "0", EnginePcap, false, 10*time.Millisecond, ProtocolHTTP) defer listener.Close() - reqPacket := buildPacket(true, 1, 1, []byte("GET / HTTP/1.1\r\n\r\n")) + reqPacket := buildPacket(true, 1, 1, []byte("GET / HTTP/1.1\r\n\r\n"), time.Now()) - listener.packetsChan <- reqPacket.Dump() + listener.packetsChan <- reqPacket.dump() select { case req = <-listener.messagesChan: @@ -114,12 +114,12 @@ func TestRawListenerResponse(t *testing.T) { listener := NewListener("", "0", EnginePcap, true, 10*time.Millisecond, ProtocolHTTP) defer listener.Close() - reqPacket := buildPacket(true, 1, 1, []byte("GET / HTTP/1.1\r\n\r\n")) - respPacket := buildPacket(false, 1+uint32(len(reqPacket.Data)), 2, []byte("HTTP/1.1 200 OK\r\n\r\n")) + reqPacket := buildPacket(true, 1, 1, []byte("GET / HTTP/1.1\r\n\r\n"), time.Now()) + respPacket := buildPacket(false, 1+uint32(len(reqPacket.Data)), 2, []byte("HTTP/1.1 200 OK\r\n\r\n"), time.Now()) // If response packet comes before request - listener.packetsChan <- respPacket.Dump() - listener.packetsChan <- reqPacket.Dump() + listener.packetsChan <- respPacket.dump() + listener.packetsChan <- reqPacket.dump() select { case req = <-listener.messagesChan: @@ -152,15 +152,15 @@ func TestShort100Continue(t *testing.T) { listener := NewListener("", "0", EnginePcap, true, 10*time.Millisecond, ProtocolHTTP) defer listener.Close() - reqPacket1 := buildPacket(true, 1, 1, []byte("POST / HTTP/1.1\r\nContent-Length: 2\r\nExpect: 100-continue\r\n\r\n")) + reqPacket1 := buildPacket(true, 1, 1, []byte("POST / HTTP/1.1\r\nContent-Length: 2\r\nExpect: 100-continue\r\n\r\n"), time.Now()) // Packet with data have different Seq - reqPacket2 := buildPacket(true, 2, reqPacket1.Seq+uint32(len(reqPacket1.Data)), []byte("a")) - reqPacket3 := buildPacket(true, 2, reqPacket2.Seq+1, []byte("b")) + reqPacket2 := buildPacket(true, 2, reqPacket1.Seq+uint32(len(reqPacket1.Data)), []byte("a"), time.Now()) + reqPacket3 := buildPacket(true, 2, reqPacket2.Seq+1, []byte("b"), time.Now()) - respPacket1 := buildPacket(false, 10, 3, []byte("HTTP/1.1 100 Continue\r\n\r\n")) + respPacket1 := buildPacket(false, 10, 3, []byte("HTTP/1.1 100 Continue\r\n\r\n"), time.Now()) // panic(int(uint32(len(reqPacket1.Data)) + uint32(len(reqPacket2.Data)) + uint32(len(reqPacket3.Data)))) - respPacket2 := buildPacket(false, reqPacket3.Seq+1 /* len of data */, 2, []byte("HTTP/1.1 200 OK\r\n\r\n")) + respPacket2 := buildPacket(false, reqPacket3.Seq+1 /* len of data */, 2, []byte("HTTP/1.1 200 OK\r\n\r\n"), time.Now()) result := []byte("POST / HTTP/1.1\r\nContent-Length: 2\r\n\r\nab") @@ -172,15 +172,15 @@ func Test100ContinueWrongOrder(t *testing.T) { listener := NewListener("", "0", EnginePcap, true, 10*time.Millisecond, ProtocolHTTP) defer listener.Close() - reqPacket1 := buildPacket(true, 1, 1, []byte("POST / HTTP/1.1\r\nContent-Length: 2\r\nExpect: 100-continue\r\n\r\n")) + reqPacket1 := buildPacket(true, 1, 1, []byte("POST / HTTP/1.1\r\nContent-Length: 2\r\nExpect: 100-continue\r\n\r\n"), time.Now()) // Packet with data have different Seq - reqPacket2 := buildPacket(true, 2, reqPacket1.Seq+uint32(len(reqPacket1.Data)), []byte("a")) - reqPacket3 := buildPacket(true, 2, reqPacket2.Seq+1, []byte("b")) + reqPacket2 := buildPacket(true, 2, reqPacket1.Seq+uint32(len(reqPacket1.Data)), []byte("a"), time.Now()) + reqPacket3 := buildPacket(true, 2, reqPacket2.Seq+1, []byte("b"), time.Now()) - respPacket1 := buildPacket(false, 10, 3, []byte("HTTP/1.1 100 Continue\r\n")) + respPacket1 := buildPacket(false, 10, 3, []byte("HTTP/1.1 100 Continue\r\n"), time.Now()) // panic(int(uint32(len(reqPacket1.Data)) + uint32(len(reqPacket2.Data)) + uint32(len(reqPacket3.Data)))) - respPacket2 := buildPacket(false, reqPacket3.Seq+1 /* len of data */, 2, []byte("HTTP/1.1 200 OK\r\n\r\n")) + respPacket2 := buildPacket(false, reqPacket3.Seq+1 /* len of data */, 2, []byte("HTTP/1.1 200 OK\r\n\r\n"), time.Now()) result := []byte("POST / HTTP/1.1\r\nContent-Length: 2\r\n\r\nab") @@ -191,15 +191,15 @@ func TestAlt100ContinueHeaderOrder(t *testing.T) { listener := NewListener("", "0", EnginePcap, true, 10*time.Millisecond, ProtocolHTTP) defer listener.Close() - reqPacket1 := buildPacket(true, 1, 1, []byte("POST / HTTP/1.1\r\nExpect: 100-continue\r\nContent-Length: 2\r\n\r\n")) + reqPacket1 := buildPacket(true, 1, 1, []byte("POST / HTTP/1.1\r\nExpect: 100-continue\r\nContent-Length: 2\r\n\r\n"), time.Now()) // Packet with data have different Seq - reqPacket2 := buildPacket(true, 2, reqPacket1.Seq+uint32(len(reqPacket1.Data)), []byte("a")) - reqPacket3 := buildPacket(true, 2, reqPacket2.Seq+1, []byte("b")) + reqPacket2 := buildPacket(true, 2, reqPacket1.Seq+uint32(len(reqPacket1.Data)), []byte("a"), time.Now()) + reqPacket3 := buildPacket(true, 2, reqPacket2.Seq+1, []byte("b"), time.Now()) - respPacket1 := buildPacket(false, 10, 3, []byte("HTTP/1.1 100 Continue\r\n")) + respPacket1 := buildPacket(false, 10, 3, []byte("HTTP/1.1 100 Continue\r\n"), time.Now()) // panic(int(uint32(len(reqPacket1.Data)) + uint32(len(reqPacket2.Data)) + uint32(len(reqPacket3.Data)))) - respPacket2 := buildPacket(false, reqPacket3.Seq+1 /* len of data */, 2, []byte("HTTP/1.1 200 OK\r\n\r\n")) + respPacket2 := buildPacket(false, reqPacket3.Seq+1 /* len of data */, 2, []byte("HTTP/1.1 200 OK\r\n\r\n"), time.Now()) result := []byte("POST / HTTP/1.1\r\nContent-Length: 2\r\n\r\nab") @@ -209,7 +209,7 @@ func TestAlt100ContinueHeaderOrder(t *testing.T) { func testRawListener100Continue(t *testing.T, listener *Listener, result []byte, packets ...*TCPPacket) { var req, resp *TCPMessage for _, p := range packets { - listener.packetsChan <- p.Dump() + listener.packetsChan <- p.dump() } select { @@ -249,7 +249,7 @@ func testChunkedSequence(t *testing.T, listener *Listener, packets ...*TCPPacket var r, req, resp *TCPMessage for _, p := range packets { - listener.packetsChan <- p.Dump() + listener.packetsChan <- p.dump() } select { @@ -352,16 +352,16 @@ func TestRawListenerChunkedWrongOrder(t *testing.T) { listener := NewListener("", "0", EnginePcap, true, 10*time.Millisecond, ProtocolHTTP) defer listener.Close() - reqPacket1 := buildPacket(true, 1, 1, []byte("POST / HTTP/1.1\r\nTransfer-Encoding: chunked\r\nExpect: 100-continue\r\n\r\n")) + reqPacket1 := buildPacket(true, 1, 1, []byte("POST / HTTP/1.1\r\nTransfer-Encoding: chunked\r\nExpect: 100-continue\r\n\r\n"), time.Now()) // Packet with data have different Seq - reqPacket2 := buildPacket(true, 2, reqPacket1.Seq+uint32(len(reqPacket1.Data)), []byte("1\r\na\r\n")) - reqPacket3 := buildPacket(true, 2, reqPacket2.Seq+uint32(len(reqPacket2.Data)), []byte("1\r\nb\r\n")) - reqPacket4 := buildPacket(true, 2, reqPacket3.Seq+uint32(len(reqPacket3.Data)), []byte("0\r\n\r\n")) + reqPacket2 := buildPacket(true, 2, reqPacket1.Seq+uint32(len(reqPacket1.Data)), []byte("1\r\na\r\n"), time.Now()) + reqPacket3 := buildPacket(true, 2, reqPacket2.Seq+uint32(len(reqPacket2.Data)), []byte("1\r\nb\r\n"), time.Now()) + reqPacket4 := buildPacket(true, 2, reqPacket3.Seq+uint32(len(reqPacket3.Data)), []byte("0\r\n\r\n"), time.Now()) - respPacket1 := buildPacket(false, 10, 3, []byte("HTTP/1.1 100 Continue\r\n\r\n")) + respPacket1 := buildPacket(false, 10, 3, []byte("HTTP/1.1 100 Continue\r\n\r\n"), time.Now()) // panic(int(uint32(len(reqPacket1.Data)) + uint32(len(reqPacket2.Data)) + uint32(len(reqPacket3.Data)))) - respPacket2 := buildPacket(false, reqPacket4.Seq+5 /* len of data */, 2, []byte("HTTP/1.1 200 OK\r\n\r\n")) + respPacket2 := buildPacket(false, reqPacket4.Seq+5 /* len of data */, 2, []byte("HTTP/1.1 200 OK\r\n\r\n"), time.Now()) // Should re-construct message from all possible combinations for i := 0; i < 6*5*4*3*2*1; i++ { @@ -381,13 +381,13 @@ func chunkedPostMessage() []*TCPPacket { ack := uint32(rand.Int63()) seq := uint32(rand.Int63()) - reqPacket1 := buildPacket(true, ack, seq, []byte("POST / HTTP/1.1\r\nTransfer-Encoding: chunked\r\n\r\n")) + reqPacket1 := buildPacket(true, ack, seq, []byte("POST / HTTP/1.1\r\nTransfer-Encoding: chunked\r\n\r\n"), time.Now()) // Packet with data have different Seq - reqPacket2 := buildPacket(true, ack, seq+47, []byte("1\r\na\r\n")) - reqPacket3 := buildPacket(true, ack, reqPacket2.Seq+5, []byte("1\r\nb\r\n")) - reqPacket4 := buildPacket(true, ack, reqPacket3.Seq+5, []byte("0\r\n\r\n")) + reqPacket2 := buildPacket(true, ack, seq+47, []byte("1\r\na\r\n"), time.Now()) + reqPacket3 := buildPacket(true, ack, reqPacket2.Seq+5, []byte("1\r\nb\r\n"), time.Now()) + reqPacket4 := buildPacket(true, ack, reqPacket3.Seq+5, []byte("0\r\n\r\n"), time.Now()) - respPacket := buildPacket(false, reqPacket4.Seq+5 /* len of data */, ack, []byte("HTTP/1.1 200 OK\r\n\r\n")) + respPacket := buildPacket(false, reqPacket4.Seq+5 /* len of data */, ack, []byte("HTTP/1.1 200 OK\r\n\r\n"), time.Now()) return []*TCPPacket{ reqPacket1, reqPacket2, reqPacket3, reqPacket4, respPacket, @@ -409,8 +409,8 @@ func postMessage() []*TCPPacket { } return []*TCPPacket{ - buildPacket(true, ack, seq, data), - buildPacket(false, seq+uint32(len(data)), seq2, []byte("HTTP/1.1 200 OK\r\n\r\n")), + buildPacket(true, ack, seq, data, time.Now()), + buildPacket(false, seq+uint32(len(data)), seq2, []byte("HTTP/1.1 200 OK\r\n\r\n"), time.Now()), } } @@ -420,8 +420,8 @@ func getMessage() []*TCPPacket { seq := uint32(rand.Int63()) return []*TCPPacket{ - buildPacket(true, ack, seq, []byte("GET / HTTP/1.1\r\n\r\n")), - buildPacket(false, seq+18, seq2, []byte("HTTP/1.1 200 OK\r\n\r\n")), + buildPacket(true, ack, seq, []byte("GET / HTTP/1.1\r\n\r\n"), time.Now()), + buildPacket(false, seq+18, seq2, []byte("HTTP/1.1 200 OK\r\n\r\n"), time.Now()), } } @@ -452,7 +452,7 @@ func TestRawListenerBench(t *testing.T) { } } - l.packetsChan <- p.Dump() + l.packetsChan <- p.dump() time.Sleep(time.Millisecond) } diff --git a/raw_socket_listener/tcp_message.go b/raw_socket_listener/tcp_message.go index c6152c5..f1bbc1b 100644 --- a/raw_socket_listener/tcp_message.go +++ b/raw_socket_listener/tcp_message.go @@ -57,8 +57,8 @@ type TCPMessage struct { } // NewTCPMessage pointer created from a Acknowledgment number and a channel of messages readuy to be deleted -func NewTCPMessage(Seq, Ack uint32, IsIncoming bool, protocol TCPProtocol) (msg *TCPMessage) { - msg = &TCPMessage{Seq: Seq, Ack: Ack, IsIncoming: IsIncoming, protocol: protocol} +func NewTCPMessage(Seq, Ack uint32, IsIncoming bool, protocol TCPProtocol, timestamp time.Time) (msg *TCPMessage) { + msg = &TCPMessage{Seq: Seq, Ack: Ack, IsIncoming: IsIncoming, protocol: protocol, Start: timestamp} msg.Start = time.Now() return @@ -144,6 +144,10 @@ func (t *TCPMessage) AddPacket(packet *TCPPacket) { t.DataAck = packet.OrigAck } + if packet.timestamp.Before(t.Start) { + t.Start = packet.timestamp + } + t.checkSeqIntegrity() if t.protocol == ProtocolHTTP { diff --git a/raw_socket_listener/tcp_message_test.go b/raw_socket_listener/tcp_message_test.go index 6847bc0..14c4e6e 100644 --- a/raw_socket_listener/tcp_message_test.go +++ b/raw_socket_listener/tcp_message_test.go @@ -5,9 +5,10 @@ import ( "encoding/binary" _ "log" "testing" + "time" ) -func buildPacket(isIncoming bool, Ack, Seq uint32, Data []byte) (packet *TCPPacket) { +func buildPacket(isIncoming bool, Ack, Seq uint32, Data []byte, timestamp time.Time) (packet *TCPPacket) { var srcPort, destPort uint16 // For tests `listening` port is 0 @@ -25,7 +26,7 @@ func buildPacket(isIncoming bool, Ack, Seq uint32, Data []byte) (packet *TCPPack buf[12] = 64 buf = append(buf, Data...) - packet = ParseTCPPacket([]byte("123"), buf) + packet = ParseTCPPacket([]byte("123"), buf, timestamp) return packet } @@ -43,24 +44,24 @@ func buildMessage(p *TCPPacket) *TCPMessage { } func TestTCPMessagePacketsOrder(t *testing.T) { - msg := buildMessage(buildPacket(true, 1, 1, []byte("a"))) - msg.AddPacket(buildPacket(true, 1, 2, []byte("b"))) + msg := buildMessage(buildPacket(true, 1, 1, []byte("a"), time.Now())) + msg.AddPacket(buildPacket(true, 1, 2, []byte("b"), time.Now())) if !bytes.Equal(msg.Bytes(), []byte("ab")) { t.Error("Should contatenate packets in right order") } // When first packet have wrong order (Seq) - msg = buildMessage(buildPacket(true, 1, 2, []byte("b"))) - msg.AddPacket(buildPacket(true, 1, 1, []byte("a"))) + msg = buildMessage(buildPacket(true, 1, 2, []byte("b"), time.Now())) + msg.AddPacket(buildPacket(true, 1, 1, []byte("a"), time.Now())) if !bytes.Equal(msg.Bytes(), []byte("ab")) { t.Error("Should contatenate packets in right order") } // Should ignore packets with same sequence - msg = buildMessage(buildPacket(true, 1, 1, []byte("a"))) - msg.AddPacket(buildPacket(true, 1, 1, []byte("a"))) + msg = buildMessage(buildPacket(true, 1, 1, []byte("a"), time.Now())) + msg.AddPacket(buildPacket(true, 1, 1, []byte("a"), time.Now())) if !bytes.Equal(msg.Bytes(), []byte("a")) { t.Error("Should ignore packet with same Seq") @@ -68,8 +69,8 @@ func TestTCPMessagePacketsOrder(t *testing.T) { } func TestTCPMessageSize(t *testing.T) { - msg := buildMessage(buildPacket(true, 1, 1, []byte("POST / HTTP/1.1\r\nContent-Length: 2\r\n\r\na"))) - msg.AddPacket(buildPacket(true, 1, 2, []byte("b"))) + msg := buildMessage(buildPacket(true, 1, 1, []byte("POST / HTTP/1.1\r\nContent-Length: 2\r\n\r\na"), time.Now())) + msg.AddPacket(buildPacket(true, 1, 2, []byte("b"), time.Now())) if msg.BodySize() != 2 { t.Error("Should count only body", msg.BodySize()) @@ -110,7 +111,7 @@ func TestTCPMessageIsComplete(t *testing.T) { } for _, tc := range testCases { - msg := buildMessage(buildPacket(tc.direction, 1, 1, []byte(tc.payload))) + msg := buildMessage(buildPacket(tc.direction, 1, 1, []byte(tc.payload), time.Now())) if tc.assocMessage { msg.AssocMessage = &TCPMessage{} } @@ -123,9 +124,9 @@ func TestTCPMessageIsComplete(t *testing.T) { } func TestTCPMessageIsSeqMissing(t *testing.T) { - p1 := buildPacket(false, 1, 1, []byte("HTTP/1.1 200 OK\r\n")) - p2 := buildPacket(false, 1, p1.Seq+uint32(len(p1.Data)), []byte("Content-Length: 10\r\n\r\n")) - p3 := buildPacket(false, 1, p2.Seq+uint32(len(p2.Data)), []byte("a")) + p1 := buildPacket(false, 1, 1, []byte("HTTP/1.1 200 OK\r\n"), time.Now()) + p2 := buildPacket(false, 1, p1.Seq+uint32(len(p1.Data)), []byte("Content-Length: 10\r\n\r\n"), time.Now()) + p3 := buildPacket(false, 1, p2.Seq+uint32(len(p2.Data)), []byte("a"), time.Now()) msg := buildMessage(p1) if msg.seqMissing { @@ -144,8 +145,8 @@ func TestTCPMessageIsSeqMissing(t *testing.T) { } func TestTCPMessageIsHeadersReceived(t *testing.T) { - p1 := buildPacket(false, 1, 1, []byte("HTTP/1.1 200 OK\r\n\r\n")) - p2 := buildPacket(false, 1, p1.Seq+uint32(len(p1.Data)), []byte("Content-Length: 10\r\n\r\n")) + p1 := buildPacket(false, 1, 1, []byte("HTTP/1.1 200 OK\r\n\r\n"), time.Now()) + p2 := buildPacket(false, 1, p1.Seq+uint32(len(p1.Data)), []byte("Content-Length: 10\r\n\r\n"), time.Now()) msg := buildMessage(p1) if msg.headerPacket == -1 { @@ -157,7 +158,7 @@ func TestTCPMessageIsHeadersReceived(t *testing.T) { t.Error("Should found double new line: headers received") } - msg = buildMessage(buildPacket(true, 1, 1, []byte("GET / HTTP/1.1\r\nContent-Length: 1\r\n"))) + msg = buildMessage(buildPacket(true, 1, 1, []byte("GET / HTTP/1.1\r\nContent-Length: 1\r\n"), time.Now())) if msg.headerPacket != -1 { t.Error("Should not find headers end") } @@ -183,7 +184,7 @@ func TestTCPMessageMethodType(t *testing.T) { } for _, tc := range testCases { - msg := buildMessage(buildPacket(tc.direction, 1, 1, []byte(tc.payload))) + msg := buildMessage(buildPacket(tc.direction, 1, 1, []byte(tc.payload), time.Now())) if msg.methodType != tc.expectedMethodType { t.Errorf("Expected %d, got %d", tc.expectedMethodType, msg.methodType) @@ -208,7 +209,7 @@ func TestTCPMessageBodyType(t *testing.T) { } for _, tc := range testCases { - msg := buildMessage(buildPacket(tc.direction, 1, 1, []byte(tc.payload))) + msg := buildMessage(buildPacket(tc.direction, 1, 1, []byte(tc.payload), time.Now())) if msg.bodyType != tc.expectedBodyType { t.Errorf("Expected %d, got %d", tc.expectedBodyType, msg.bodyType) @@ -229,12 +230,12 @@ func TestTCPMessageBodySize(t *testing.T) { } for _, tc := range testCases { - msg := buildMessage(buildPacket(tc.direction, 1, 1, []byte(tc.payloads[0]))) + msg := buildMessage(buildPacket(tc.direction, 1, 1, []byte(tc.payloads[0]), time.Now())) if len(tc.payloads) > 1 { for _, p := range tc.payloads[1:] { seq := uint32(1 + msg.Size()) - msg.AddPacket(buildPacket(tc.direction, 1, seq, []byte(p))) + msg.AddPacket(buildPacket(tc.direction, 1, seq, []byte(p), time.Now())) } } @@ -243,3 +244,15 @@ func TestTCPMessageBodySize(t *testing.T) { } } } + +func TestTcpMessageStart(t *testing.T) { + start := time.Now().Add(-1 * time.Second) + + msg := buildMessage(buildPacket(true, 1, 2, []byte("b"), time.Now())) + msg.AddPacket(buildPacket(true, 1, 1, []byte("POST / HTTP/1.1\r\nContent-Length: 2\r\n\r\na"), start)) + + if msg.Start != start { + t.Error("Message timestamp should be equal to the lowest related packet timestamp", start, msg.Start) + } +} + diff --git a/raw_socket_listener/tcp_packet.go b/raw_socket_listener/tcp_packet.go index 0e3e832..f2ba82c 100644 --- a/raw_socket_listener/tcp_packet.go +++ b/raw_socket_listener/tcp_packet.go @@ -5,6 +5,7 @@ import ( "log" "strconv" "strings" + "time" ) var _ = log.Println @@ -38,14 +39,16 @@ type TCPPacket struct { Raw []byte Data []byte Addr []byte + timestamp time.Time ID tcpID } // ParseTCPPacket takes address and tcp payload and returns parsed TCPPacket -func ParseTCPPacket(addr []byte, data []byte) (p *TCPPacket) { +func ParseTCPPacket(addr []byte, data []byte, timestamp time.Time) (p *TCPPacket) { p = &TCPPacket{Raw: data} p.ParseBasic() p.Addr = addr + p.timestamp = timestamp p.GenID() return @@ -79,27 +82,33 @@ func (t *TCPPacket) ParseBasic() { t.Data = t.Raw[t.DataOffset*4:] } -func (t *TCPPacket) Dump() []byte { - buf := make([]byte, len(t.Data)+16+16) - copy(buf[:16], t.Addr) +func (t *TCPPacket) dump() *packet { - tcpBuf := buf[16:] + packetSrcIP := make([]byte, 16) + packetData := make([]byte, len(t.Data) + 16) - binary.BigEndian.PutUint16(tcpBuf[2:4], t.DestPort) - binary.BigEndian.PutUint16(tcpBuf[0:2], t.SrcPort) + copy(packetSrcIP, t.Addr) - binary.BigEndian.PutUint32(tcpBuf[4:8], t.Seq) - binary.BigEndian.PutUint32(tcpBuf[8:12], t.Ack) + binary.BigEndian.PutUint16(packetData[0:2], t.SrcPort) + binary.BigEndian.PutUint16(packetData[2:4], t.DestPort) - tcpBuf[12] = 64 + binary.BigEndian.PutUint32(packetData[4:8], t.Seq) + binary.BigEndian.PutUint32(packetData[8:12], t.Ack) + + packetData[12] = 64 if t.IsFIN { - tcpBuf[13] = tcpBuf[13] | 0x01 + packetData[13] = packetData[13] | 0x01 } - copy(tcpBuf[16:], t.Data) + copy(packetData[16:], t.Data) + + return &packet{ + srcIP: packetSrcIP, + data:packetData, + timestamp:t.timestamp, + } - return buf } // String output for a TCP Packet diff --git a/settings.go b/settings.go index 5227c34..3068924 100644 --- a/settings.go +++ b/settings.go @@ -64,6 +64,8 @@ type AppSettings struct { outputBinaryConfig BinaryOutputConfig modifierConfig HTTPModifierConfig + + outputKafkaConfig KafkaConfig } // Settings holds Gor configuration @@ -130,7 +132,8 @@ func init() { flag.IntVar(&Settings.outputHTTPConfig.BufferSize, "output-http-response-buffer", 0, "HTTP response buffer size, all data after this size will be discarded.") flag.IntVar(&Settings.outputHTTPConfig.workers, "output-http-workers", 0, "Gor uses dynamic worker scaling by default. Enter a number to run a set number of workers.") flag.IntVar(&Settings.outputHTTPConfig.redirectLimit, "output-http-redirects", 0, "Enable how often redirects should be followed.") - flag.DurationVar(&Settings.outputHTTPConfig.Timeout, "output-http-timeout", 0, "Specify HTTP request/response timeout. By default 5s. Example: --output-http-timeout 30s") + flag.DurationVar(&Settings.outputHTTPConfig.Timeout, "output-http-timeout", 5*time.Second, "Specify HTTP request/response timeout. By default 5s. Example: --output-http-timeout 30s") + flag.BoolVar(&Settings.outputHTTPConfig.stats, "output-http-stats", false, "Report http output queue stats to console every 5 seconds.") flag.BoolVar(&Settings.outputHTTPConfig.OriginalHost, "http-original-host", false, "Normally gor replaces the Host http header with the host supplied with --output-http. This option disables that behavior, preserving the original Host header.") flag.BoolVar(&Settings.outputHTTPConfig.Debug, "output-http-debug", false, "Enables http debug output.") @@ -147,6 +150,9 @@ func init() { flag.BoolVar(&Settings.outputBinaryConfig.Debug, "output-binary-debug", false, "Enables binary debug output.") /* outputBinaryConfig */ + flag.StringVar(&Settings.outputKafkaConfig.host, "output-kafka-host", "", "Send request and response stats to Kafka:\n\tgor --input-raw :8080 --output-kafka-host '192.168.0.1:9092,192.168.0.2:9092'") + flag.StringVar(&Settings.outputKafkaConfig.topic, "output-kafka-topic", "", "Send request and response stats to Kafka:\n\tgor --input-raw :8080 --output-kafka-topic 'kafka-log'") + flag.Var(&Settings.modifierConfig.headers, "http-set-header", "Inject additional headers to http reqest:\n\tgor --input-raw :8080 --output-http staging.com --http-set-header 'User-Agent: Gor'") flag.Var(&Settings.modifierConfig.headers, "output-http-header", "WARNING: `--output-http-header` DEPRECATED, use `--http-set-header` instead")