diff --git a/Makefile b/Makefile index 6f43cab..11ce2c6 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -SOURCE = emitter.go gor.go gor_stat.go input_dummy.go input_file.go input_raw.go input_tcp.go limiter.go output_dummy.go output_file.go input_http.go output_http.go output_tcp.go plugins.go settings.go test_input.go elasticsearch.go http_modifier.go http_modifier_settings.go http_client.go middleware.go payload_format.go +SOURCE = emitter.go gor.go gor_stat.go input_dummy.go input_file.go input_raw.go input_tcp.go limiter.go output_dummy.go output_file.go input_http.go output_http.go output_tcp.go plugins.go settings.go test_input.go elasticsearch.go http_modifier.go http_modifier_settings.go http_client.go middleware.go protocol.go SOURCE_PATH = /gopath/src/github.com/buger/gor/ diff --git a/README.md b/README.md index 8e5dbf0..e341d98 100644 --- a/README.md +++ b/README.md @@ -213,16 +213,30 @@ gor --input-raw :80 --middleware "/opt/middleware_executable" --output-http "htt #### Communication protocol All messages should be hex encoded, new line character specifieds the end of the message, eg. new message per line. -Decoded payload consist of 2 parts: header and HTTP payload, separated by new line character. Example: +Decoded payload consist of 2 parts: header and HTTP payload, separated by new line character. + +Example request payload: + ``` -1 932079936fa4306fc308d67588178d17d823647c +1 932079936fa4306fc308d67588178d17d823647c 1439818823587396305 GET /a HTTP/1.1 Host: 127.0.0.1 ``` -First header byte `1` represent payload type, possible values: `1` - request, `2` - original response, `3` - replayed response -After empty spaces, goes request id `932079936fa4306fc308d67588178d17d823647c`. Request id unique among all requests (sha1 of time and Ack), but remain same for original and replayed response, so you can create associations between request and responses. +Example response payload: + +``` +2 8e091765ae902fef8a2b7d9dd960e9d52222bd8c 2782013 +HTTP/1.1 200 OK +Date: Mon, 17 Aug 2015 13:40:23 GMT +Content-Length: 0 +Content-Type: text/plain; charset=utf-8 + +``` + +Header contains request meta information separated by spaces. First value is payload type, possible values: `1` - request, `2` - original response, `3` - replayed response. +Next goes request id: unique among all requests (sha1 of time and Ack), but remain same for original and replayed response, so you can create associations between request and responses. Third argument varies depending on payload type: for request - start time, for responses - round-trip time. HTTP payload is unmodified HTTP requests/responses intercepted from network. You can read more about request format [here](http://www.jmarshall.com/easy/http/), [here](https://en.wikipedia.org/wiki/Hypertext_Transfer_Protocol) and [here](http://www.w3.org/Protocols/rfc2616/rfc2616.html). You can operate with payload as you want, add headers, change path, and etc. Basically you just editing a string, just ensure that it is RCF compliant. diff --git a/examples/middleware/token_modifier.go b/examples/middleware/token_modifier.go index 3b871e2..9da3fc8 100644 --- a/examples/middleware/token_modifier.go +++ b/examples/middleware/token_modifier.go @@ -55,10 +55,13 @@ func process(buf []byte) { // 2 - Response // 3 - ReplayedResponse payloadType := buf[0] - headerSize := 42 - header := buf[:headerSize] + headerSize := bytes.IndexByte(buf, '\n') + 1 + header := buf[:headerSize-1] + + // Header contains space separated values of: request type, request id, and request start time (or round-trip time for responses) + meta := bytes.Split(header, []byte(" ")) // For each request you should receive 3 payloads (request, response, replayed response) with same request id - reqID := string(header[2:headerSize]) + reqID := string(meta[1]) payload := buf[headerSize:] Debug("Received payload:", string(buf)) @@ -83,7 +86,7 @@ func process(buf []byte) { } // Re-compute length in case if payload was modified - bufLen := len(header) + len(payload) + bufLen := headerSize + len(payload) // Encoding request and sending it back dst := make([]byte, bufLen*2+1) hex.Encode(dst, buf[:bufLen]) diff --git a/input_raw.go b/input_raw.go index cfdfc8b..75a37ec 100644 --- a/input_raw.go +++ b/input_raw.go @@ -34,12 +34,13 @@ func (i *RAWInput) Read(data []byte) (int, error) { buf := msg.Bytes() if i.captureResponse { - payloadType := RequestPayload - if !msg.IsIncoming { - payloadType = ResponsePayload - } + var header []byte - header := payloadHeader(payloadType, msg.UUID()) + if msg.IsIncoming { + header = payloadHeader(RequestPayload, msg.UUID(), msg.Start) + } else { + header = payloadHeader(ResponsePayload, msg.UUID(), msg.End-msg.RequestStart) + } copy(data[0:len(header)], header) copy(data[len(header):], buf) diff --git a/middleware.go b/middleware.go index cb4ea12..e88be4d 100644 --- a/middleware.go +++ b/middleware.go @@ -108,7 +108,6 @@ func (m *Middleware) read(from io.Reader) { } func (m *Middleware) Read(data []byte) (int, error) { - Debug("Trying to read channel!") buf := <-m.data copy(data, buf) diff --git a/output_http.go b/output_http.go index c3e0450..a44c7fd 100644 --- a/output_http.go +++ b/output_http.go @@ -1,6 +1,7 @@ package main import ( + "bytes" "io" "log" "sync/atomic" @@ -10,8 +11,9 @@ import ( const initialDynamicWorkers = 10 type response struct { - payload []byte - uuid []byte + payload []byte + uuid []byte + roundTripTime int64 } // HTTPOutputConfig struct for holding http output configuration @@ -163,7 +165,7 @@ func (o *HTTPOutput) Read(data []byte) (int, error) { Debug("[OUTPUT-HTTP] Received response", string(resp.payload)) - header := payloadHeader(ReplayedResponsePayload, resp.uuid) + header := payloadHeader(ReplayedResponsePayload, resp.uuid, resp.roundTripTime) copy(data[0:len(header)], header) copy(data[len(header):], resp.payload) @@ -174,8 +176,10 @@ func (o *HTTPOutput) sendRequest(client *HTTPClient, request []byte) { var uuid []byte if len(Settings.middleware) > 0 { - uuid = request[2:42] - request = request[43:] + headerSize := bytes.IndexByte(request, '\n') + meta := bytes.Split(request[:headerSize], []byte{' '}) + uuid = meta[1] + request = request[headerSize+1:] } start := time.Now() @@ -187,7 +191,7 @@ func (o *HTTPOutput) sendRequest(client *HTTPClient, request []byte) { } if len(Settings.middleware) > 0 { - o.responses <- response{resp, uuid} + o.responses <- response{resp, uuid, stop.UnixNano() - start.UnixNano()} } if o.elasticSearch != nil { diff --git a/payload_format.go b/payload_format.go deleted file mode 100644 index f26b103..0000000 --- a/payload_format.go +++ /dev/null @@ -1,26 +0,0 @@ -package main - -const ( - RequestPayload = 1 << iota - ResponsePayload - ReplayedResponsePayload -) - -func payloadHeader(payloadType int, uuid []byte) (header []byte) { - header = make([]byte, 43) - header[1] = ' ' - header[len(header)-1] = '\n' - - switch payloadType { - case RequestPayload: - header[0] = '1' - case ResponsePayload: - header[0] = '2' - case ReplayedResponsePayload: - header[0] = '3' - } - - copy(header[2:], uuid) - - return header -} diff --git a/protocol.go b/protocol.go new file mode 100644 index 0000000..a92ac80 --- /dev/null +++ b/protocol.go @@ -0,0 +1,38 @@ +package main + +import ( + "strconv" +) + +const ( + RequestPayload = 1 << iota + ResponsePayload + ReplayedResponsePayload +) + +// Timing is request start or round-trip time, depending on payloadType +func payloadHeader(payloadType int, uuid []byte, timing int64) (header []byte) { + sTime := strconv.FormatInt(timing, 10) + + //Example: + // 3 f45590522cd1838b4a0d5c5aab80b77929dea3b3 1231\n + // `+ 1` indicates space characters or end of line + header = make([]byte, 1+1+len(uuid)+1+len(sTime)+1) + header[1] = ' ' + header[2+len(uuid)] = ' ' + header[len(header)-1] = '\n' + + switch payloadType { + case RequestPayload: + header[0] = '1' + case ResponsePayload: + header[0] = '2' + case ReplayedResponsePayload: + header[0] = '3' + } + + copy(header[2:], uuid) + copy(header[3+len(uuid):], sTime) + + return header +} diff --git a/raw_socket_listener/tcp_message.go b/raw_socket_listener/tcp_message.go index 359e62e..956f4ec 100644 --- a/raw_socket_listener/tcp_message.go +++ b/raw_socket_listener/tcp_message.go @@ -23,6 +23,7 @@ type TCPMessage struct { RequestStart int64 RequestAck uint32 Start int64 + End int64 IsIncoming bool packets []*TCPPacket @@ -119,6 +120,7 @@ func (t *TCPMessage) AddPacket(packet *TCPPacket) { log.Println("Received packet with same sequence") } else { t.packets = append(t.packets, packet) + t.End = time.Now().UnixNano() } if !t.isMultipart() {